Current section
Files
Jump to
Current section
Files
lib/codex/voice/models/openai_stt.ex
defmodule Codex.Voice.Models.OpenAISTT do
@moduledoc """
OpenAI speech-to-text model implementation.
This module implements the `Codex.Voice.Model.STTModel` behaviour using
OpenAI's audio transcription API. It supports both single-shot transcription
and streaming transcription sessions via WebSocket.
## Default Model
The default model is `gpt-4o-transcribe`, which provides high-quality
transcriptions with support for multiple languages.
## Example
model = OpenAISTT.new()
audio = AudioInput.new(wav_data)
settings = STTSettings.new(language: "en")
{:ok, text} = OpenAISTT.transcribe(model, audio, settings, true, false)
"""
@behaviour Codex.Voice.Model.STTModel
alias Codex.Auth
alias Codex.Config.Defaults
alias Codex.Voice.Config.STTSettings
alias Codex.Voice.Input.AudioInput
alias Codex.Voice.Input.StreamedAudioInput
alias Codex.Voice.Models.OpenAISTTSession
defstruct [:model, :client, :api_key, :base_url]
@type t :: %__MODULE__{
model: String.t(),
client: term(),
api_key: String.t() | nil,
base_url: String.t()
}
@default_model Defaults.stt_model()
@default_base_url Defaults.openai_api_base_url()
@doc """
Create a new OpenAI STT model.
## Options
- `:client` - Optional HTTP client (for testing)
- `:api_key` - API key (defaults to OPENAI_API_KEY env var)
- `:base_url` - API base URL (defaults to OpenAI)
## Examples
iex> model = Codex.Voice.Models.OpenAISTT.new()
iex> model.model
"gpt-4o-transcribe"
iex> model = Codex.Voice.Models.OpenAISTT.new("whisper-1")
iex> model.model
"whisper-1"
"""
@spec new(String.t() | nil, keyword()) :: t()
def new(model \\ nil, opts \\ []) do
%__MODULE__{
model: model || @default_model,
client: Keyword.get(opts, :client),
api_key: Keyword.get(opts, :api_key),
base_url: Keyword.get(opts, :base_url, @default_base_url)
}
end
@impl true
def model_name, do: @default_model
@doc """
Transcribe audio input to text.
Makes a POST request to OpenAI's audio transcriptions endpoint with
the audio data in WAV format.
## Parameters
- `model` - The OpenAISTT model struct
- `input` - AudioInput with the audio data
- `settings` - STTSettings with transcription options
- `_trace_include_sensitive_data` - Whether to include text in traces (unused)
- `_trace_include_sensitive_audio_data` - Whether to include audio in traces (unused)
## Returns
- `{:ok, text}` - The transcribed text
- `{:error, reason}` - If the request fails
"""
@spec transcribe(
t(),
AudioInput.t(),
STTSettings.t(),
boolean(),
boolean()
) :: {:ok, String.t()} | {:error, term()}
def transcribe(
%__MODULE__{} = model,
%AudioInput{} = input,
%STTSettings{} = settings,
_trace_include_sensitive_data,
_trace_include_sensitive_audio_data
) do
api_key = model.api_key || Auth.direct_api_key()
{filename, wav_data, content_type} = AudioInput.to_audio_file(input)
# Build multipart form for Req library
# Req expects: {name, {value, options}} where options is a keyword list
# Options can include :filename, :content_type, :size
multipart =
[
{:file, {wav_data, filename: filename, content_type: content_type}},
{:model, model.model}
]
|> maybe_add_param(:prompt, settings.prompt)
|> maybe_add_param(:language, settings.language)
|> maybe_add_param(:temperature, format_temperature(settings.temperature))
case Req.post("#{model.base_url}/audio/transcriptions",
headers: [{"Authorization", "Bearer #{api_key}"}],
form_multipart: multipart
) do
{:ok, %{status: 200, body: %{"text" => text}}} ->
{:ok, text}
{:ok, %{status: status, body: body}} ->
{:error, {:api_error, status, body}}
{:error, reason} ->
{:error, reason}
end
end
@impl true
def create_session(
%StreamedAudioInput{} = input,
%STTSettings{} = settings,
trace_include_sensitive_data,
trace_include_sensitive_audio_data
) do
OpenAISTTSession.start_link(
input: input,
settings: settings,
model: @default_model,
trace_include_sensitive_data: trace_include_sensitive_data,
trace_include_sensitive_audio_data: trace_include_sensitive_audio_data
)
end
defp maybe_add_param(list, _key, nil), do: list
defp maybe_add_param(list, key, value) when is_atom(key), do: list ++ [{key, to_string(value)}]
@spec format_temperature(float() | nil) :: String.t() | nil
defp format_temperature(nil), do: nil
defp format_temperature(temp), do: Float.to_string(temp)
end
defmodule Codex.Voice.Models.OpenAISTTSession do
@moduledoc """
Streaming transcription session using WebSocket.
This GenServer manages a WebSocket connection to OpenAI's realtime
transcription API. It receives audio input from a `StreamedAudioInput`
and produces text transcriptions for each detected turn.
## Turn Detection
The session uses semantic VAD (Voice Activity Detection) by default
to detect turn boundaries in the audio stream.
"""
use GenServer
alias Codex.Auth
alias Codex.Config.Defaults
alias Codex.Voice.Config.STTSettings
alias Codex.Voice.Input.StreamedAudioInput
@behaviour Codex.Voice.Model.StreamedTranscriptionSession
defstruct [
:input,
:settings,
:model,
:api_key,
:trace_include_sensitive_data,
:trace_include_sensitive_audio_data,
:websocket,
:listener_task,
:stream_task,
transcripts: [],
waiters: []
]
@type t :: %__MODULE__{
input: StreamedAudioInput.t(),
settings: STTSettings.t(),
model: String.t(),
api_key: String.t() | nil,
trace_include_sensitive_data: boolean(),
trace_include_sensitive_audio_data: boolean(),
websocket: pid() | nil,
listener_task: Task.t() | nil,
stream_task: Task.t() | nil,
transcripts: [String.t()],
waiters: [{GenServer.from(), reference()}]
}
@default_turn_detection Defaults.stt_default_turn_detection()
@doc """
Start a new streaming transcription session.
## Options
- `:input` - StreamedAudioInput to read audio from (required)
- `:settings` - STTSettings for transcription options (required)
- `:model` - Model name to use
- `:api_key` - API key (defaults to OPENAI_API_KEY env var)
- `:trace_include_sensitive_data` - Whether to include text in traces
- `:trace_include_sensitive_audio_data` - Whether to include audio in traces
"""
@spec start_link(keyword()) :: GenServer.on_start()
def start_link(opts) do
GenServer.start_link(__MODULE__, opts)
end
@impl Codex.Voice.Model.StreamedTranscriptionSession
def transcribe_turns(session) do
Stream.resource(
fn -> session end,
fn session ->
case GenServer.call(session, :get_transcript, :infinity) do
{:ok, text} -> {[text], session}
:done -> {:halt, session}
{:error, reason} -> raise "Transcription error: #{inspect(reason)}"
end
end,
fn _ -> :ok end
)
end
@impl Codex.Voice.Model.StreamedTranscriptionSession
def close(session) do
GenServer.stop(session, :normal)
:ok
end
@impl GenServer
def init(opts) do
input = Keyword.fetch!(opts, :input)
settings = Keyword.fetch!(opts, :settings)
state = %__MODULE__{
input: input,
settings: settings,
model: Keyword.get(opts, :model, "gpt-4o-transcribe"),
api_key: Keyword.get(opts, :api_key, Auth.direct_api_key()),
trace_include_sensitive_data: Keyword.get(opts, :trace_include_sensitive_data, true),
trace_include_sensitive_audio_data:
Keyword.get(opts, :trace_include_sensitive_audio_data, false)
}
# Connection will be established when transcribe_turns is first called
{:ok, state}
end
@impl GenServer
def handle_call(:get_transcript, from, state) do
case state.transcripts do
[text | rest] ->
{:reply, {:ok, text}, %{state | transcripts: rest}}
[] ->
# No transcripts available, add to waiters
monitor_ref = monitor_waiter(from)
{:noreply, %{state | waiters: state.waiters ++ [{from, monitor_ref}]}}
end
end
@impl GenServer
def handle_info({:transcript, text}, state) do
case state.waiters do
[{waiter, monitor_ref} | rest] ->
Process.demonitor(monitor_ref, [:flush])
GenServer.reply(waiter, {:ok, text})
{:noreply, %{state | waiters: rest}}
[] ->
{:noreply, %{state | transcripts: state.transcripts ++ [text]}}
end
end
@impl GenServer
def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
waiters =
Enum.reject(state.waiters, fn {_waiter, monitor_ref} ->
monitor_ref == ref
end)
{:noreply, %{state | waiters: waiters}}
end
@impl GenServer
def handle_info(:session_complete, state) do
# Notify all waiters that we're done
for {waiter, monitor_ref} <- state.waiters do
Process.demonitor(monitor_ref, [:flush])
GenServer.reply(waiter, :done)
end
{:noreply, %{state | waiters: []}}
end
@impl GenServer
def handle_info({:error, reason}, state) do
# Notify all waiters of the error
for {waiter, monitor_ref} <- state.waiters do
Process.demonitor(monitor_ref, [:flush])
GenServer.reply(waiter, {:error, reason})
end
{:noreply, %{state | waiters: []}}
end
@impl GenServer
def terminate(_reason, state) do
Enum.each(state.waiters, fn {waiter, monitor_ref} ->
Process.demonitor(monitor_ref, [:flush])
GenServer.reply(waiter, {:error, :closed})
end)
close_websocket(state.websocket)
shutdown_task(state.listener_task)
shutdown_task(state.stream_task)
:ok
end
@doc false
def default_turn_detection, do: @default_turn_detection
defp close_websocket(pid) when is_pid(pid) do
if Process.alive?(pid) do
Process.exit(pid, :shutdown)
end
rescue
_ -> :ok
catch
:exit, _ -> :ok
end
defp close_websocket(_), do: :ok
defp shutdown_task(%Task{} = task) do
case task.pid do
pid when is_pid(pid) ->
if Process.alive?(pid) do
Process.exit(pid, :kill)
end
:ok
_ ->
:ok
end
rescue
_ -> :ok
catch
:exit, _ -> :ok
end
defp shutdown_task(_), do: :ok
defp monitor_waiter({pid, _tag}) when is_pid(pid) do
Process.monitor(pid)
end
end