Current section
Files
Jump to
Current section
Files
examples/agent_retry_live.exs
# examples/agent_retry_live.exs
#
# The agent spike's retry path against REAL claude calls -- this script COSTS
# MONEY (one completed haiku turn; the failed attempt times out before claude
# can answer, so it bills at or near zero).
#
# The worker opts into retries (max_attempts: 3) and delegates its callbacks
# to ObanClaude.Agent.Job, whose routing is terminal-aware. The only
# manipulation is chaos injection at the :query_fun seam: the FIRST call
# forces a 1500ms subprocess timeout onto the real ClaudeWrapper.query/2, so
# attempt 1 dies with a genuine %Error{kind: :timeout} (a retryable {:error,
# :timeout} verdict); later calls run untouched. What that exercises, all
# real:
#
# * Oban schedules and re-runs the retry (attempt bookkeeping in SQLite)
# * the agent stays :running across the failed attempt -- job_retrying/2
# records {:retrying, ...} and re-arms the watchdog
# * the final attempt succeeds and lands as ONE finished turn
#
# mix run examples/agent_retry_live.exs
# ---------------------------------------------------------------------------
# 1. Throwaway SQLite repo + Oban, as in the other examples.
# ---------------------------------------------------------------------------
db_path = Path.join(System.tmp_dir!(), "oban_claude_agent_retry_live.db")
for suffix <- ["", "-shm", "-wal"], do: File.rm(db_path <> suffix)
defmodule RetryLiveRepo do
use Ecto.Repo, otp_app: :oban_claude, adapter: Ecto.Adapters.SQLite3
end
Application.put_env(:oban_claude, RetryLiveRepo,
database: db_path,
pool_size: 1,
busy_timeout: 5_000,
log: false
)
defmodule RetryLiveMigration do
use Ecto.Migration
def up, do: Oban.Migrations.up()
def down, do: Oban.Migrations.down()
end
# ---------------------------------------------------------------------------
# 2. A retrying agent worker: real claude, chaos on the first call only.
# ---------------------------------------------------------------------------
:persistent_term.put(:retry_live_calls, :counters.new(1, []))
defmodule RetryLiveWorker do
use ObanClaude.Worker,
queue: :agents,
max_attempts: 3,
query_fun: &__MODULE__.chaos_query/2
# Snappy demo: retry ~2s after a failure instead of Oban's default ~16s+.
@impl Oban.Worker
def backoff(%Oban.Job{}), do: 2
@impl ObanClaude.Worker
def handle_result(result, job), do: ObanClaude.Agent.Job.handle_result(result, job)
@impl ObanClaude.Worker
def handle_error(verdict, payload, job),
do: ObanClaude.Agent.Job.handle_error(verdict, payload, job)
# Call 1: the real claude subprocess, crippled by a 1500ms timeout it cannot
# possibly meet. Call 2+: the same real call, untouched.
def chaos_query(prompt, opts) do
counter = :persistent_term.get(:retry_live_calls)
:counters.add(counter, 1, 1)
call = :counters.get(counter, 1)
opts = if call == 1, do: Keyword.put(opts, :timeout, 1_500), else: opts
IO.puts(" [claude] call #{call}, timeout=#{inspect(opts[:timeout])}")
ClaudeWrapper.query(prompt, opts)
end
end
# ---------------------------------------------------------------------------
# 3. Boot + printers for every seam event.
# ---------------------------------------------------------------------------
{:ok, _} = RetryLiveRepo.start_link()
Ecto.Migrator.up(RetryLiveRepo, 1, RetryLiveMigration, log: false)
{:ok, _} =
Oban.start_link(
repo: RetryLiveRepo,
engine: Oban.Engines.Lite,
notifier: Oban.Notifiers.PG,
peer: Oban.Peers.Isolated,
plugins: [],
queues: [agents: 1]
)
{:ok, _} = ObanClaude.Agent.Supervisor.start_link()
:telemetry.attach(
"retry-live-transitions",
[:oban_claude, :agent, :transition],
fn _e, _m, meta, _c -> IO.puts(" [transition] #{meta.from} -> #{meta.to}") end,
nil
)
:telemetry.attach(
"retry-live-stop",
[:oban_claude, :run, :stop],
fn _e, meas, meta, _c ->
IO.puts(" [run ok] $#{Float.round(meas.cost_usd, 4)}: #{inspect(meta.result.result)}")
end,
nil
)
:telemetry.attach(
"retry-live-exception",
[:oban_claude, :run, :exception],
fn _e, _m, meta, _c ->
IO.puts(" [run failed] kind=#{inspect(meta.error.kind)} (attempt #{meta.job.attempt})")
end,
nil
)
# ---------------------------------------------------------------------------
# 4. One cast prompt; the retry happens underneath while the agent holds
# :running the whole way.
# ---------------------------------------------------------------------------
{:ok, _} =
ObanClaude.Agent.start_agent("retry-demo",
worker: RetryLiveWorker,
args: %{"model" => "haiku"},
job_timeout: 120_000
)
IO.puts("\ncasting prompt (fire-and-forget)...")
:ok = ObanClaude.Agent.cast_prompt("retry-demo", "Reply with exactly: recovered")
{:ok, :running} = ObanClaude.Agent.await("retry-demo", :running, 10_000)
{:ok, :idle} = ObanClaude.Agent.await("retry-demo", :idle, 180_000)
# ---------------------------------------------------------------------------
# 5. Proof: the agent's ledger and Oban's own attempt bookkeeping.
# ---------------------------------------------------------------------------
{:ok, info} = ObanClaude.Agent.info("retry-demo")
{:ok, history} = ObanClaude.Agent.history("retry-demo")
IO.puts("\nAgent info: turns=#{info.turns} cost=$#{Float.round(info.cost_usd, 4)}")
IO.puts("Agent history (oldest first):")
for entry <- history, do: IO.puts(" #{inspect(entry, printable_limit: 120)}")
import Ecto.Query, only: [from: 2]
[{state, attempt, max_attempts}] =
RetryLiveRepo.all(from(j in "oban_jobs", select: {j.state, j.attempt, j.max_attempts}))
IO.puts("\nOban's row: state=#{state} attempt=#{attempt}/#{max_attempts}")