Current section
Files
Jump to
Current section
Files
lib/asm/run/server.ex
defmodule ASM.Run.Server do
@moduledoc """
Per-run worker that owns backend lifecycle and event fanout.
"""
use GenServer, restart: :temporary
alias ASM.{Error, Event, Metadata, Provider, ProviderRegistry, ProviderRuntimeProfile, Run}
alias ASM.Execution.{Config, PolicyPlug}
alias ASM.ProviderBackend.Event, as: BackendEvent
alias ASM.ProviderBackend.Info, as: BackendInfo
alias CliSubprocessCore.Payload
@boot_timeout_ms 15_000
@spec start_link(keyword()) :: GenServer.on_start()
def start_link(opts) when is_list(opts) do
caller = self()
reply_ref = make_ref()
with_trap_exit(fn ->
case GenServer.start_link(__MODULE__, {caller, reply_ref, opts}) do
{:ok, pid} ->
await_bootstrap(pid, reply_ref, Keyword.get(opts, :boot_timeout_ms, @boot_timeout_ms))
{:error, reason} ->
{:error, reason}
end
end)
end
@spec interrupt(GenServer.server()) :: :ok
def interrupt(server) do
GenServer.cast(server, :interrupt)
:ok
end
@spec ingest_event(GenServer.server(), Event.t()) :: :ok
def ingest_event(server, %Event{} = event) do
GenServer.cast(server, {:ingest_event, event})
:ok
end
@spec get_state(GenServer.server()) :: Run.State.t()
def get_state(server), do: GenServer.call(server, :get_state)
@spec resolve_approval(GenServer.server(), String.t(), :allow | :deny) :: :ok
def resolve_approval(server, approval_id, decision) do
GenServer.cast(server, {:resolve_approval, approval_id, decision})
:ok
end
@impl true
def init({caller, reply_ref, opts}) when is_list(opts) do
trap_exits()
Process.put({__MODULE__, :boot_waiter}, {caller, reply_ref})
{:ok, Run.State.new(opts), {:continue, :bootstrap}}
end
def init(opts) when is_list(opts) do
trap_exits()
{:ok, Run.State.new(opts), {:continue, :bootstrap}}
end
# The run process owns the backend handle, and `terminate/2` is what closes
# it. Without trapping exits that callback never runs when the run
# supervisor shuts the run down — a session stop would leave the provider
# process group orphaned.
defp trap_exits do
_ = Process.flag(:trap_exit, true)
:ok
end
@impl true
def handle_continue(:bootstrap, state) do
case start_backend(state) do
{:ok, next_state} ->
_ = ASM.Telemetry.run_started(state.session_id, state.run_id, state.provider)
notify_bootstrap_waiter(:ok)
{:noreply, arm_run_deadline(next_state)}
{:error, %Error{} = error, next_state} ->
notify_bootstrap_waiter({:error, error})
_ = maybe_close_backend(next_state)
{:stop, :normal, %{next_state | error: error}}
end
end
@impl true
def handle_call(:get_state, _from, state) do
{:reply, Run.State.materialize(state), state}
end
@impl true
def handle_cast({:ingest_event, %Event{} = event}, state) do
consume_event(state, event)
end
def handle_cast(:interrupt, state) do
_ = maybe_interrupt_backend(state)
event =
Event.new(
:error,
Payload.Error.new(message: "Run interrupted", code: "user_cancelled", severity: :warning),
run_id: state.run_id,
session_id: state.session_id,
provider: state.provider,
timestamp: DateTime.utc_now()
)
next_state =
state
|> process_events([event])
|> Map.put(:status, :interrupted)
|> Map.put(:finished_at, DateTime.utc_now())
|> Map.put(:error, Error.new(:user_cancelled, :runtime, "Run interrupted"))
finish_run(next_state)
end
def handle_cast({:resolve_approval, approval_id, decision}, state) do
if Map.has_key?(state.pending_approvals, approval_id) do
state = clear_approval_timer(state, approval_id)
notify_session(state, {:clear_approval, approval_id})
event =
Event.new(
:approval_resolved,
Payload.ApprovalResolved.new(approval_id: approval_id, decision: decision),
run_id: state.run_id,
session_id: state.session_id,
provider: state.provider,
timestamp: DateTime.utc_now()
)
next_state = process_events(state, [event])
{:noreply, next_state}
else
{:noreply, state}
end
end
@impl true
def handle_info(
%BackendEvent{subscription_ref: ref, core_event: %CliSubprocessCore.Event{} = core_event},
%{backend_subscription_ref: ref} = state
) do
event =
Event.wrap_core(
%{
run_id: state.run_id,
session_id: state.session_id,
provider: state.provider,
metadata: state.metadata
},
core_event
)
consume_event(state, event)
end
def handle_info(
%BackendEvent{subscription_ref: ref, asm_event: %Event{} = event},
%{backend_subscription_ref: ref} = state
) do
consume_event(state, event)
end
def handle_info({:approval_timeout, approval_id}, state) do
if Map.has_key?(state.pending_approvals, approval_id) do
state = clear_approval_timer(state, approval_id)
notify_session(state, {:clear_approval, approval_id})
event =
Event.new(
:approval_resolved,
Payload.ApprovalResolved.new(
approval_id: approval_id,
decision: :deny,
reason: "timeout"
),
run_id: state.run_id,
session_id: state.session_id,
provider: state.provider,
timestamp: DateTime.utc_now()
)
{:noreply, process_events(state, [event])}
else
{:noreply, state}
end
end
# The backend is monitored, not linked (`detach_backend_link/1`), and the
# parent exit is handled by `:gen_server` itself, so a stray link exit is
# not a run-terminating event.
def handle_info({:EXIT, _pid, _reason}, state), do: {:noreply, state}
def handle_info(:run_deadline_reached, state) do
state = %{state | deadline_timer_ref: nil}
if Run.EventReducer.final?(state) do
{:noreply, state}
else
message = run_deadline_message(state)
_ = maybe_interrupt_backend(state)
event =
Event.new(
:error,
Payload.Error.new(
message: message,
code: "timeout",
severity: :fatal,
metadata: %{asm_error_domain: :runtime}
),
run_id: state.run_id,
session_id: state.session_id,
provider: state.provider,
timestamp: DateTime.utc_now()
)
next_state =
state
|> process_events([event])
|> Map.put(:status, :failed)
|> Map.put(:finished_at, DateTime.utc_now())
|> Map.put(:error, Error.new(:timeout, :runtime, message))
finish_run(next_state)
end
end
def handle_info({:DOWN, ref, :process, _pid, reason}, %{backend_ref: ref} = state) do
next_state = %{state | backend_pid: nil, backend_ref: nil}
cond do
Run.EventReducer.final?(next_state) ->
{:noreply, next_state}
reason in [:normal, :shutdown] ->
event =
Event.new(
:run_completed,
%{status: :completed},
run_id: next_state.run_id,
session_id: next_state.session_id,
provider: next_state.provider,
timestamp: DateTime.utc_now()
)
finish_run(process_events(next_state, [event]))
true ->
event =
Event.new(
:error,
Payload.Error.new(
message: "backend crashed: #{inspect(reason)}",
code: "transport_error"
),
run_id: next_state.run_id,
session_id: next_state.session_id,
provider: next_state.provider,
timestamp: DateTime.utc_now()
)
finish_run(process_events(next_state, [event]))
end
end
@impl true
def terminate(_reason, state) do
cleanup_approval_timers(state)
cancel_run_deadline(state)
_ = maybe_close_backend(state)
:ok
end
# The total-run deadline is armed once, at backend start, and never re-armed:
# a backend that keeps emitting events re-arms `:stream_timeout_ms` forever,
# so only a wall-clock budget over the whole run can end it.
defp arm_run_deadline(%Run.State{run_deadline_ms: :infinity} = state), do: state
defp arm_run_deadline(%Run.State{run_deadline_ms: deadline_ms} = state)
when is_integer(deadline_ms) and deadline_ms > 0 do
%{state | deadline_timer_ref: Process.send_after(self(), :run_deadline_reached, deadline_ms)}
end
defp cancel_run_deadline(%Run.State{deadline_timer_ref: ref}) when is_reference(ref) do
_ = Process.cancel_timer(ref, async: true, info: false)
:ok
end
defp cancel_run_deadline(_state), do: :ok
defp run_deadline_message(%Run.State{run_deadline_ms: deadline_ms, run_id: run_id}) do
"run #{run_id} exceeded its total run deadline of #{deadline_ms}ms"
end
defp start_backend(%Run.State{} = state) do
with {:ok, provider} <- Provider.resolve(state.provider),
{:ok, resolution} <-
resolve_backend(provider, state),
start_config <- backend_start_config(provider, state, resolution),
{:ok, pid, info} <- resolution.backend.start_run(start_config) do
detach_backend_link(pid)
bootstrap_backend_session(state, resolution, pid, info, start_config)
else
{:error, %Error{} = error} ->
{:error, error, state}
{:error, reason} ->
{:error,
Error.new(:runtime, :runtime, "backend start failed: #{inspect(reason)}", cause: reason),
state}
end
end
defp bootstrap_backend_session(state, resolution, pid, info, start_config) do
with :ok <- maybe_subscribe_backend(resolution.backend, pid, start_config.subscription_ref),
next_state <- put_backend_state(state, resolution, pid, info, start_config),
:ok <- deliver_prompt(next_state) do
{:ok, next_state}
else
{:error, %Error{} = error} ->
_ = safe_close_backend(resolution.backend, pid)
{:error, error, state}
end
end
defp maybe_subscribe_backend(ASM.ProviderBackend.Core, _pid, _ref), do: :ok
defp maybe_subscribe_backend(backend, pid, ref), do: subscribe_backend(backend, pid, ref)
defp resolve_backend(provider, %Run.State{backend: backend} = state)
when is_atom(backend) and not is_nil(backend) do
with {:ok, runtime_profile} <- ProviderRuntimeProfile.resolve(provider.name),
:ok <- validate_backend_override(provider, backend, runtime_profile) do
lane = backend_override_lane(state.lane)
capabilities = backend_override_capabilities(provider, lane)
execution_mode = execution_mode(state)
{:ok,
%{
provider: provider,
backend: backend,
requested_lane: state.lane || lane,
preferred_lane: lane,
lane: lane,
capabilities: capabilities,
provider_runtime_profile: runtime_profile,
observability:
%{
provider: provider.name,
provider_display_name: provider.display_name,
requested_lane: state.lane || lane,
preferred_lane: lane,
lane: lane,
lane_reason: :backend_override,
lane_fallback_reason: nil,
execution_mode: execution_mode,
backend: backend,
capabilities: capabilities
}
|> Map.merge(ProviderRuntimeProfile.observability(runtime_profile))
}}
end
end
defp resolve_backend(provider, %Run.State{} = state) do
ProviderRegistry.resolve(provider.name,
lane: state.lane || :auto,
execution_mode: execution_mode(state)
)
end
defp backend_start_config(provider, state, resolution) do
subscription_ref = make_ref()
%{
provider: provider,
lane: resolution.lane,
backend: resolution.backend,
prompt: state.prompt,
continuation: state.continuation,
provider_opts: state.provider_opts,
backend_opts: state.backend_opts,
codex_materialized_runtime: state.codex_materialized_runtime,
tools: state.tools,
execution_config: state.execution_config,
subscriber_pid: self(),
subscription_ref: subscription_ref,
metadata: Map.merge(state.metadata, %{run_id: state.run_id, session_id: state.session_id})
}
end
defp detach_backend_link(pid) when is_pid(pid) do
Process.unlink(pid)
:ok
rescue
_ -> :ok
end
defp subscribe_backend(backend, pid, ref) when is_atom(backend) and is_pid(pid) do
case backend.subscribe(pid, self(), ref) do
:ok ->
:ok
{:error, %Error{} = error} ->
_ = safe_close_backend(backend, pid)
{:error, error}
{:error, reason} ->
_ = safe_close_backend(backend, pid)
{:error,
Error.new(
:runtime,
:runtime,
"backend subscribe failed: #{inspect(reason)}",
cause: reason
)}
end
rescue
error ->
_ = safe_close_backend(backend, pid)
{:error, Error.new(:runtime, :runtime, Exception.message(error), cause: error)}
end
defp put_backend_state(state, resolution, pid, info, start_config) do
ref = Process.monitor(pid)
backend_info = resolved_backend_info(info, resolution, pid)
%{
state
| backend: resolution.backend,
backend_pid: pid,
backend_ref: ref,
backend_subscription_ref: start_config.subscription_ref,
backend_info: backend_info,
lane: resolution.lane,
metadata: Map.merge(state.metadata, backend_info.observability)
}
end
defp deliver_prompt(%Run.State{backend: ASM.ProviderBackend.SDK, provider: :claude} = state) do
case state.backend.send_input(state.backend_pid, state.prompt, []) do
:ok ->
:ok
{:error, reason} ->
{:error,
Error.new(:runtime, :runtime, "prompt delivery failed: #{inspect(reason)}",
cause: reason
)}
end
end
defp deliver_prompt(_state), do: :ok
defp consume_event(state, %Event{} = event) do
case apply_pipeline(state, event) do
{:ok, events, next_state} ->
next_state = process_events(next_state, events)
if Run.EventReducer.final?(next_state) do
finish_run(next_state)
else
{:noreply, next_state}
end
{:error, %Error{} = error, next_state} ->
error_event =
Event.new(
:error,
error_payload(error),
run_id: next_state.run_id,
session_id: next_state.session_id,
provider: next_state.provider,
timestamp: DateTime.utc_now()
)
finish_run(process_events(next_state, [error_event]))
end
end
defp apply_pipeline(state, event) do
case ASM.Pipeline.run(
event,
execution_policy_pipeline(state) ++ state.pipeline,
state.pipeline_ctx
) do
{:ok, events, pipeline_ctx} ->
{:ok, events, %{state | pipeline_ctx: pipeline_ctx}}
{:error, %Error{} = error, pipeline_ctx} ->
{:error, error, %{state | pipeline_ctx: pipeline_ctx}}
end
rescue
error ->
{:error, Error.new(:runtime, :runtime, Exception.message(error), cause: error), state}
end
defp execution_policy_pipeline(%Run.State{execution_config: %Config{} = config}) do
case Config.to_execution_environment(config).allowed_tools do
[] -> []
allowed_tools -> [{PolicyPlug, allowed_tools: allowed_tools}]
end
end
defp execution_policy_pipeline(_state), do: []
defp process_events(state, events) when is_list(events) do
Enum.reduce(events, state, fn event, acc ->
event = merge_event_metadata(event, acc.metadata)
next_state = Run.EventReducer.apply_event!(acc, event)
maybe_capture_checkpoint(next_state, event)
fanout(next_state, event)
maybe_track_approval(next_state, event)
end)
end
defp maybe_track_approval(state, %Event{kind: :approval_requested} = event) do
payload = Event.legacy_payload(event)
notify_session(state, {:register_approval, self(), payload})
timer_ref =
Process.send_after(
self(),
{:approval_timeout, payload.approval_id},
state.approval_timeout_ms
)
put_in(state.approval_timers[payload.approval_id], timer_ref)
end
defp maybe_track_approval(state, _event), do: state
defp fanout(%Run.State{} = state, %Event{kind: :cost_update} = event) do
notify_session(state, {:cost_update, Event.legacy_payload(event)})
fanout_to_subscriber(state, event)
end
defp fanout(%Run.State{} = state, event) do
fanout_to_subscriber(state, event)
end
defp fanout_to_subscriber(%Run.State{subscriber: subscriber}, event) when is_pid(subscriber) do
send(subscriber, {:asm_run_event, event.run_id, event})
end
defp fanout_to_subscriber(_state, _event), do: :ok
defp finish_run(state) do
_ = maybe_close_backend(state)
notify_done(state)
_ = ASM.Telemetry.run_completed(state.session_id, state.run_id, state.provider, state.status)
{:stop, :normal, state}
end
defp notify_done(%Run.State{subscriber: subscriber, run_id: run_id}) when is_pid(subscriber) do
send(subscriber, {:asm_run_done, run_id})
end
defp notify_done(_state), do: :ok
defp notify_session(%Run.State{session_pid: session_pid}, message) when is_pid(session_pid) do
send(session_pid, message)
end
defp notify_session(_state, _message), do: :ok
defp maybe_capture_checkpoint(
%Run.State{} = state,
%Event{provider_session_id: provider_session_id} = event
)
when is_binary(provider_session_id) and provider_session_id != "" do
notify_session(
state,
{:capture_checkpoint, event.run_id, provider_session_id,
normalize_checkpoint_metadata(event.metadata)}
)
end
defp maybe_capture_checkpoint(_state, _event), do: :ok
defp clear_approval_timer(state, approval_id) do
case Map.pop(state.approval_timers, approval_id) do
{nil, timers} ->
%{state | approval_timers: timers}
{timer_ref, timers} ->
_ = Process.cancel_timer(timer_ref, async: true, info: false)
%{state | approval_timers: timers}
end
end
defp cleanup_approval_timers(state) do
Enum.each(state.approval_timers, fn {_approval_id, timer_ref} ->
_ = Process.cancel_timer(timer_ref, async: true, info: false)
end)
end
defp maybe_interrupt_backend(%Run.State{backend: backend, backend_pid: pid})
when is_atom(backend) and is_pid(pid) do
backend.interrupt(pid)
rescue
_ -> :ok
end
defp maybe_interrupt_backend(_state), do: :ok
defp maybe_close_backend(%Run.State{backend: backend, backend_pid: pid})
when is_atom(backend) and is_pid(pid) do
safe_close_backend(backend, pid)
rescue
_ -> :ok
end
defp maybe_close_backend(_state), do: :ok
defp safe_close_backend(backend, pid) when is_atom(backend) and is_pid(pid) do
backend.close(pid)
rescue
_ -> :ok
end
defp await_bootstrap(pid, reply_ref, timeout_ms) when is_pid(pid) and is_reference(reply_ref) do
receive do
{:asm_run_bootstrapped, ^reply_ref} ->
{:ok, pid}
{:asm_run_boot_failed, ^reply_ref, %Error{} = error} ->
safe_stop_run(pid)
{:error, error}
{:EXIT, ^pid, %Error{} = error} ->
{:error, error}
{:EXIT, ^pid, reason} ->
{:error, normalize_bootstrap_exit(reason)}
after
timeout_ms ->
safe_stop_run(pid)
{:error, Error.new(:timeout, :runtime, "run bootstrap timed out")}
end
end
defp safe_stop_run(pid) when is_pid(pid) do
GenServer.stop(pid, :normal)
catch
:exit, _reason -> :ok
end
defp normalize_bootstrap_exit(%Error{} = error), do: error
defp normalize_bootstrap_exit({%Error{} = error, _stacktrace}), do: error
defp normalize_bootstrap_exit(reason) do
Error.new(:runtime, :runtime, "run bootstrap failed: #{inspect(reason)}", cause: reason)
end
defp notify_bootstrap_waiter(message) do
case Process.delete({__MODULE__, :boot_waiter}) do
{caller, reply_ref} when is_pid(caller) and is_reference(reply_ref) ->
payload =
case message do
:ok -> {:asm_run_bootstrapped, reply_ref}
{:error, %Error{} = error} -> {:asm_run_boot_failed, reply_ref, error}
end
send(caller, payload)
:ok
_ ->
:ok
end
end
defp with_trap_exit(fun) when is_function(fun, 0) do
previous_trap_exit? = Process.flag(:trap_exit, true)
try do
fun.()
after
Process.flag(:trap_exit, previous_trap_exit?)
end
end
defp error_payload(%Error{} = error) do
Payload.Error.new(
message: error.message,
code: to_string(error.kind),
metadata: %{asm_error_domain: error.domain}
)
end
defp merge_event_metadata(%Event{} = event, metadata) when is_map(metadata) do
%{event | metadata: Metadata.merge_run_metadata(metadata, event.metadata)}
end
defp normalize_checkpoint_metadata(metadata), do: metadata
defp resolved_backend_info(info, resolution, pid) do
fallback =
normalize_backend_info(info, resolution, pid)
|> normalize_backend_observability(resolution)
case fetch_backend_info(resolution.backend, pid, resolution) do
%BackendInfo{} = refreshed ->
normalize_backend_observability(refreshed, resolution)
_other ->
fallback
end
end
defp normalize_backend_info(info, resolution, pid) do
case info do
%BackendInfo{} = normalized ->
BackendInfo.normalize(normalized,
provider: resolution.provider.name,
lane: resolution.lane,
backend: resolution.backend,
session_pid: pid
)
other ->
BackendInfo.normalize(other,
provider: resolution.provider.name,
lane: resolution.lane,
backend: resolution.backend,
runtime: resolution.backend,
capabilities: resolution.capabilities,
session_pid: pid
)
end
end
defp fetch_backend_info(backend, pid, resolution) when is_atom(backend) and is_pid(pid) do
backend.info(pid)
|> normalize_backend_info(resolution, pid)
rescue
_error ->
nil
catch
:exit, _reason ->
nil
end
defp normalize_backend_observability(%BackendInfo{} = info, resolution) do
%{info | observability: backend_observability(info, resolution)}
end
defp backend_observability(%BackendInfo{} = info, resolution) do
capabilities =
case info.capabilities do
[] -> resolution.capabilities
values -> values
end
resolution.observability
|> Map.merge(info.observability)
|> Map.put(:provider, info.provider || resolution.provider.name)
|> Map.put(:lane, info.lane || resolution.lane)
|> Map.put(:backend, info.backend || resolution.backend)
|> Map.put(:runtime, info.runtime || resolution.backend)
|> Map.put(:capabilities, capabilities)
end
defp execution_mode(%Run.State{execution_config: %ASM.Execution.Config{execution_mode: mode}})
when mode in [:local, :remote_node],
do: mode
defp execution_mode(_state), do: :local
defp backend_override_lane(lane) when lane in [:core, :sdk], do: lane
defp backend_override_lane(_lane), do: :core
defp backend_override_capabilities(%Provider{} = provider, :core) do
module_capabilities(provider.core_profile)
end
defp backend_override_capabilities(%Provider{} = provider, :sdk) do
module_capabilities(provider.sdk_runtime)
end
defp validate_backend_override(_provider, _backend, nil), do: :ok
defp validate_backend_override(_provider, ASM.ProviderBackend.Core, %{}), do: :ok
defp validate_backend_override(%Provider{} = provider, backend, %{ref: ref}) do
{:error,
Error.new(
:config_invalid,
:config,
"backend override #{inspect(backend)} is unavailable while provider runtime profile #{inspect(ref)} is active for #{inspect(provider.name)}",
provider: provider.name,
cause: %{provider: provider.name, backend: backend, provider_runtime_profile_ref: ref}
)}
end
defp module_capabilities(module) when is_atom(module) do
if function_exported?(module, :capabilities, 0) do
module.capabilities()
else
[]
end
end
end