Current section

Files

Jump to
codex_sdk lib codex realtime.ex
Raw

lib/codex/realtime.ex

defmodule Codex.Realtime do
@moduledoc """
Realtime audio streaming with OpenAI's Realtime API.
This module provides a high-level interface for building voice-enabled
AI applications using WebSocket-based real-time communication.
## Quick Start
# Define an agent
agent = Codex.Realtime.agent(
name: "VoiceAssistant",
instructions: "You are a helpful voice assistant.",
tools: [weather_tool]
)
# Create and run a session
{:ok, session} = Codex.Realtime.run(agent)
# Send audio and receive events
Codex.Realtime.send_audio(session, audio_bytes)
Codex.Realtime.subscribe(session, self())
receive do
{:session_event, event} -> handle_event(event)
end
## Features
- Real-time audio streaming (PCM16, G.711)
- Voice activity detection (semantic VAD, server VAD)
- Tool execution during conversations
- Agent handoffs
- Output guardrails
- Dynamic instructions (string or function)
## Architecture
The realtime feature consists of:
- `Codex.Realtime.Agent` - Agent definition with tools and handoffs
- `Codex.Realtime.Session` - Session management (GenServer)
- `Codex.Realtime.Runner` - Session orchestration
- `Codex.Realtime.Config` - Configuration types
- `Codex.Realtime.Events` - Event types for subscribers
- `Codex.Realtime.Items` - Conversation history items
- `Codex.Realtime.Audio` - Audio format utilities
## Configuration
Sessions can be configured with various options:
{:ok, session} = Codex.Realtime.run(agent,
config: %Codex.Realtime.Config.RunConfig{
model_settings: %Codex.Realtime.Config.SessionModelSettings{
voice: "nova",
turn_detection: %Codex.Realtime.Config.TurnDetectionConfig{
type: :semantic_vad,
eagerness: :medium
}
}
}
)
## Event Handling
Subscribers receive events as `{:session_event, event}` messages:
Codex.Realtime.subscribe(session, self())
receive do
{:session_event, %Codex.Realtime.Events.AgentStartEvent{}} ->
IO.puts("Agent started")
{:session_event, %Codex.Realtime.Events.AudioEvent{audio: audio}} ->
play_audio(audio.data)
{:session_event, %Codex.Realtime.Events.ToolStartEvent{tool: tool}} ->
IO.puts("Calling tool: \#{tool.name}")
{:session_event, %Codex.Realtime.Events.AgentEndEvent{}} ->
IO.puts("Turn completed")
end
"""
alias Codex.Realtime.Agent
alias Codex.Realtime.Config
alias Codex.Realtime.Config.ModelConfig
alias Codex.Realtime.Runner
alias Codex.Realtime.Session
@doc """
Create and start a realtime session with an agent.
This is a convenience function that creates a runner and starts a session
in one step. For more control, use `Codex.Realtime.Runner` directly.
## Options
* `:config` - Run configuration (`%Codex.Realtime.Config.RunConfig{}`)
* `:model_config` - Model connection config (`%Codex.Realtime.Config.ModelConfig{}`)
* `:context` - Context map passed to the session
## Returns
* `{:ok, pid}` - Session started successfully
* `{:error, reason}` - Failed to start session
## Example
{:ok, session} = Codex.Realtime.run(agent,
config: %Codex.Realtime.Config.RunConfig{
model_settings: %{voice: "nova"}
}
)
# Use the session
Codex.Realtime.send_message(session, "Hello!")
Codex.Realtime.subscribe(session, self())
"""
@spec run(Agent.t(), keyword()) :: {:ok, pid()} | {:error, term()}
def run(agent, opts \\ []) do
config = Keyword.get(opts, :config)
model_config = Keyword.get(opts, :model_config, %ModelConfig{})
context = Keyword.get(opts, :context, %{})
# Pass through testing options
websocket_pid = Keyword.get(opts, :websocket_pid)
websocket_module = Keyword.get(opts, :websocket_module)
runner = Runner.new(agent, config: config)
runner_opts =
[context: context, model_config: model_config]
|> maybe_add_opt(:websocket_pid, websocket_pid)
|> maybe_add_opt(:websocket_module, websocket_module)
Runner.run(runner, runner_opts)
end
@doc """
Create a realtime agent.
This is a convenience function for creating an agent struct from keyword
options.
## Options
* `:name` - Agent name (default: "Agent")
* `:instructions` - System instructions (string or function)
* `:model` - Model name (default: "gpt-4o-realtime-preview")
* `:tools` - List of tools available to the agent
* `:handoffs` - List of agents or handoffs for transfers
* `:output_guardrails` - Output guardrails to apply
* `:hooks` - Event hooks
## Example
agent = Codex.Realtime.agent(
name: "Assistant",
instructions: "Be helpful and concise.",
tools: [my_tool],
handoffs: [support_agent]
)
# With dynamic instructions
agent = Codex.Realtime.agent(
name: "Greeter",
instructions: fn ctx -> "Hello \#{ctx.user_name}!" end
)
"""
@spec agent(keyword()) :: Agent.t()
def agent(opts) do
Agent.new(opts)
end
@doc """
Create a runner for more control over session creation.
Use this when you need to configure the runner separately from running it,
or when you want to reuse the same runner for multiple sessions.
## Example
runner = Codex.Realtime.runner(agent,
config: %RunConfig{tracing_disabled: true}
)
{:ok, session1} = Codex.Realtime.Runner.run(runner)
{:ok, session2} = Codex.Realtime.Runner.run(runner, context: %{user: "Alice"})
"""
@spec runner(Agent.t(), keyword()) :: Runner.t()
def runner(agent, opts \\ []) do
Runner.new(agent, opts)
end
# Delegate session operations
@doc """
Send audio data to the model.
## Options
* `:commit` - Whether to commit the audio buffer (default: false)
## Example
Codex.Realtime.send_audio(session, audio_bytes)
Codex.Realtime.send_audio(session, audio_bytes, commit: true)
"""
@spec send_audio(GenServer.server(), binary(), keyword()) :: :ok
defdelegate send_audio(session, audio, opts \\ []), to: Session
@doc """
Send a text message to the model.
Can be a simple string or a structured message map.
## Example
Codex.Realtime.send_message(session, "Hello!")
Codex.Realtime.send_message(session, %{
"type" => "message",
"role" => "user",
"content" => [%{"type" => "input_text", "text" => "Hello!"}]
})
"""
@spec send_message(GenServer.server(), String.t() | map()) :: :ok
defdelegate send_message(session, message), to: Session
@doc """
Interrupt the current response.
Sends a cancel signal to stop the model from generating more output.
"""
@spec interrupt(GenServer.server()) :: :ok
defdelegate interrupt(session), to: Session
@doc """
Subscribe to session events.
The subscriber process will receive `{:session_event, event}` messages
for all session events.
## Example
Codex.Realtime.subscribe(session, self())
receive do
{:session_event, %Codex.Realtime.Events.AudioEvent{} = event} ->
play_audio(event.audio.data)
end
"""
@spec subscribe(GenServer.server(), pid()) :: :ok
defdelegate subscribe(session, pid), to: Session
@doc """
Get the conversation history.
Returns all items in the conversation history.
"""
@spec history(GenServer.server()) :: [Codex.Realtime.Items.item()]
defdelegate history(session), to: Session
@doc """
Close the session.
Closes the WebSocket connection and stops the session process.
"""
@spec close(GenServer.server()) :: :ok
defdelegate close(session), to: Session
@doc """
Unsubscribe from session events.
"""
@spec unsubscribe(GenServer.server(), pid()) :: :ok
defdelegate unsubscribe(session, pid), to: Session
@doc """
Send a raw event to the model.
Use this for advanced scenarios where you need to send custom events.
"""
@spec send_raw_event(GenServer.server(), map()) :: :ok
defdelegate send_raw_event(session, event), to: Session
@doc """
Update session settings.
Use this to change model settings mid-session, such as voice or modalities.
## Example
settings = %Codex.Realtime.Config.SessionModelSettings{voice: "nova"}
Codex.Realtime.update_session(session, settings)
"""
@spec update_session(GenServer.server(), Config.SessionModelSettings.t()) :: :ok
defdelegate update_session(session, settings), to: Session
@doc """
Get the current agent.
"""
@spec current_agent(GenServer.server()) :: Agent.t()
defdelegate current_agent(session), to: Session
# Helpers
defp maybe_add_opt(opts, _key, nil), do: opts
defp maybe_add_opt(opts, key, value), do: Keyword.put(opts, key, value)
end