Current section

Files

Jump to
codex_sdk lib codex voice models openai_stt.ex
Raw

lib/codex/voice/models/openai_stt.ex

defmodule Codex.Voice.Models.OpenAISTT do
@moduledoc """
OpenAI speech-to-text model implementation.
This module implements the `Codex.Voice.Model.STTModel` behaviour using
OpenAI's audio transcription API. It supports both single-shot transcription
and streamed-input sessions that transcribe buffered audio once the input
closes.
## Default Model
The default model is `gpt-4o-transcribe`, which provides high-quality
transcriptions with support for multiple languages.
## Example
model = OpenAISTT.new()
audio = AudioInput.new(wav_data)
settings = STTSettings.new(language: "en")
{:ok, text} = OpenAISTT.transcribe(model, audio, settings, true, false)
"""
@behaviour Codex.Voice.Model.STTModel
alias Codex.Auth
alias Codex.Config.Defaults
alias Codex.Net.CA
alias Codex.Voice.Config.STTSettings
alias Codex.Voice.Input.AudioInput
alias Codex.Voice.Input.StreamedAudioInput
alias Codex.Voice.Models.OpenAISTTSession
@derive {Inspect, except: [:api_key]}
defstruct [:model, :client, :api_key, :base_url]
@type t :: %__MODULE__{
model: String.t(),
client: term(),
api_key: String.t() | nil,
base_url: String.t()
}
@default_model Defaults.stt_model()
@default_base_url Defaults.openai_api_base_url()
@doc """
Create a new OpenAI STT model.
## Options
- `:client` - Optional HTTP client (for testing)
- `:api_key` - API key (defaults to OPENAI_API_KEY env var)
- `:base_url` - API base URL (defaults to OpenAI)
## Examples
iex> model = Codex.Voice.Models.OpenAISTT.new()
iex> model.model
"gpt-4o-transcribe"
iex> model = Codex.Voice.Models.OpenAISTT.new("whisper-1")
iex> model.model
"whisper-1"
"""
@spec new(String.t() | nil, keyword()) :: t()
def new(model \\ nil, opts \\ []) do
%__MODULE__{
model: model || @default_model,
client: Keyword.get(opts, :client),
api_key: Keyword.get(opts, :api_key),
base_url: Keyword.get(opts, :base_url, @default_base_url)
}
end
@impl true
def model_name, do: @default_model
@doc """
Transcribe audio input to text.
Makes a POST request to OpenAI's audio transcriptions endpoint with
the audio data in WAV format.
## Parameters
- `model` - The OpenAISTT model struct
- `input` - AudioInput with the audio data
- `settings` - STTSettings with transcription options
- `_trace_include_sensitive_data` - Whether to include text in traces (unused)
- `_trace_include_sensitive_audio_data` - Whether to include audio in traces (unused)
## Returns
- `{:ok, text}` - The transcribed text
- `{:error, reason}` - If the request fails
"""
@spec transcribe(
t(),
AudioInput.t(),
STTSettings.t(),
boolean(),
boolean()
) :: {:ok, String.t()} | {:error, term()}
def transcribe(
%__MODULE__{} = model,
%AudioInput{} = input,
%STTSettings{} = settings,
_trace_include_sensitive_data,
_trace_include_sensitive_audio_data
) do
api_key = model.api_key || Auth.direct_api_key()
{filename, wav_data, content_type} = AudioInput.to_audio_file(input)
# Build multipart form for Req library
# Req expects: {name, {value, options}} where options is a keyword list
# Options can include :filename, :content_type, :size
multipart =
[
{:file, {wav_data, filename: filename, content_type: content_type}},
{:model, model.model}
]
|> maybe_add_param(:prompt, settings.prompt)
|> maybe_add_param(:language, settings.language)
|> maybe_add_param(:temperature, format_temperature(settings.temperature))
request_client = resolve_request_client(model.client)
req_opts =
[headers: [{"Authorization", "Bearer #{api_key}"}], form_multipart: multipart]
|> CA.merge_req_options()
case request_client.(
"#{model.base_url}/audio/transcriptions",
req_opts
) do
{:ok, %{status: 200, body: %{"text" => text}}} ->
{:ok, text}
{:ok, %{status: status, body: body}} ->
{:error, {:api_error, status, body}}
{:error, reason} ->
{:error, reason}
end
end
@impl true
def create_session(
%__MODULE__{} = model,
%StreamedAudioInput{} = input,
%STTSettings{} = settings,
trace_include_sensitive_data,
trace_include_sensitive_audio_data
) do
OpenAISTTSession.start_link(
input: input,
settings: settings,
stt_model: model,
trace_include_sensitive_data: trace_include_sensitive_data,
trace_include_sensitive_audio_data: trace_include_sensitive_audio_data
)
end
@doc false
def create_session(
%StreamedAudioInput{} = input,
%STTSettings{} = settings,
trace_include_sensitive_data,
trace_include_sensitive_audio_data
) do
create_session(
new(),
input,
settings,
trace_include_sensitive_data,
trace_include_sensitive_audio_data
)
end
defp maybe_add_param(list, _key, nil), do: list
defp maybe_add_param(list, key, value) when is_atom(key), do: list ++ [{key, to_string(value)}]
defp resolve_request_client(nil), do: &Req.post/2
defp resolve_request_client(client) when is_function(client, 2), do: client
defp resolve_request_client(client) when is_atom(client), do: &client.post/2
@spec format_temperature(float() | nil) :: String.t() | nil
defp format_temperature(nil), do: nil
defp format_temperature(temp), do: Float.to_string(temp)
end
defmodule Codex.Voice.Models.OpenAISTTSession do
@moduledoc """
Buffered streamed transcription session.
This GenServer consumes a `StreamedAudioInput` until it closes, then submits
the aggregated audio to the configured STT model and yields completed
transcript turns to callers.
"""
use GenServer
alias Codex.Auth
alias Codex.Config.Defaults
alias Codex.TaskSupport
alias Codex.Voice.Config.STTSettings
alias Codex.Voice.Input.AudioInput
alias Codex.Voice.Input.StreamedAudioInput
alias Codex.Voice.Models.OpenAISTT
@behaviour Codex.Voice.Model.StreamedTranscriptionSession
@derive {Inspect, except: [:api_key]}
defstruct [
:input,
:settings,
:model,
:api_key,
:trace_include_sensitive_data,
:trace_include_sensitive_audio_data,
:stt_model,
:websocket,
:listener_task,
:stream_task,
:completion,
transcripts: [],
waiters: []
]
@type t :: %__MODULE__{
input: StreamedAudioInput.t(),
settings: STTSettings.t(),
model: String.t(),
api_key: String.t() | nil,
trace_include_sensitive_data: boolean(),
trace_include_sensitive_audio_data: boolean(),
stt_model: OpenAISTT.t(),
websocket: pid() | nil,
listener_task: Task.t() | nil,
stream_task: Task.t() | nil,
completion: :done | {:error, term()} | nil,
transcripts: [String.t()],
waiters: [{GenServer.from(), reference()}]
}
@default_turn_detection Defaults.stt_default_turn_detection()
@doc """
Start a new streaming transcription session.
## Options
- `:input` - StreamedAudioInput to read audio from (required)
- `:settings` - STTSettings for transcription options (required)
- `:stt_model` - Optional `OpenAISTT` model struct to use as-is
- `:model` - Model name to use when `:stt_model` is not provided
- `:api_key` - API key (defaults to OPENAI_API_KEY env var)
- `:base_url` - API base URL when `:stt_model` is not provided
- `:client` - Optional HTTP client when `:stt_model` is not provided
- `:trace_include_sensitive_data` - Whether to include text in traces
- `:trace_include_sensitive_audio_data` - Whether to include audio in traces
"""
@spec start_link(keyword()) :: GenServer.on_start()
def start_link(opts) do
GenServer.start_link(__MODULE__, opts)
end
@impl Codex.Voice.Model.StreamedTranscriptionSession
def transcribe_turns(session) do
Stream.resource(
fn -> session end,
fn session ->
case GenServer.call(session, :get_transcript, :infinity) do
{:ok, text} -> {[text], session}
:done -> {:halt, session}
{:error, reason} -> raise "Transcription error: #{inspect(reason)}"
end
end,
fn _ -> :ok end
)
end
@impl Codex.Voice.Model.StreamedTranscriptionSession
def close(session) do
GenServer.stop(session, :normal)
:ok
end
@impl GenServer
def init(opts) do
input = Keyword.fetch!(opts, :input)
settings = Keyword.fetch!(opts, :settings)
stt_model = resolve_stt_model(opts)
api_key = stt_model.api_key || Auth.direct_api_key()
state = %__MODULE__{
input: input,
settings: settings,
model: stt_model.model,
api_key: api_key,
trace_include_sensitive_data: Keyword.get(opts, :trace_include_sensitive_data, true),
trace_include_sensitive_audio_data:
Keyword.get(opts, :trace_include_sensitive_audio_data, false),
stt_model: %{stt_model | api_key: api_key},
completion: nil
}
case start_transcription_task(state) do
{:ok, task} ->
{:ok, %{state | stream_task: task}}
{:error, reason} ->
{:stop, reason}
end
end
@impl GenServer
def handle_call(:get_transcript, from, state) do
case state.transcripts do
[text | rest] ->
{:reply, {:ok, text}, %{state | transcripts: rest}}
[] ->
case state.completion do
:done ->
{:reply, :done, state}
{:error, reason} ->
{:reply, {:error, reason}, state}
nil ->
monitor_ref = monitor_waiter(from)
{:noreply, %{state | waiters: state.waiters ++ [{from, monitor_ref}]}}
end
end
end
@impl GenServer
def handle_info({:transcript, text}, state) do
case state.waiters do
[{waiter, monitor_ref} | rest] ->
Process.demonitor(monitor_ref, [:flush])
GenServer.reply(waiter, {:ok, text})
{:noreply, %{state | waiters: rest}}
[] ->
{:noreply, %{state | transcripts: state.transcripts ++ [text]}}
end
end
def handle_info({ref, _result}, %{stream_task: %Task{ref: ref}} = state) do
{:noreply, state}
end
@impl GenServer
def handle_info(
{:DOWN, ref, :process, pid, reason},
%{stream_task: %Task{ref: ref, pid: pid}} = state
) do
state = %{state | stream_task: nil}
case {reason, state.completion} do
{:normal, _} ->
{:noreply, state}
{:shutdown, _} ->
{:noreply, state}
{{:shutdown, _}, _} ->
{:noreply, state}
{_reason, nil} ->
finish_with_error(state, {:task_exit, reason})
_ ->
{:noreply, state}
end
end
def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
waiters =
Enum.reject(state.waiters, fn {_waiter, monitor_ref} ->
monitor_ref == ref
end)
{:noreply, %{state | waiters: waiters}}
end
@impl GenServer
def handle_info(:session_complete, state) do
finish_with_completion(state)
end
@impl GenServer
def handle_info({:session_error, reason}, state) do
finish_with_error(state, reason)
end
@impl GenServer
def terminate(_reason, state) do
Enum.each(state.waiters, fn {waiter, monitor_ref} ->
Process.demonitor(monitor_ref, [:flush])
GenServer.reply(waiter, {:error, :closed})
end)
close_websocket(state.websocket)
shutdown_task(state.listener_task)
shutdown_task(state.stream_task)
:ok
end
@doc false
def default_turn_detection, do: @default_turn_detection
defp resolve_stt_model(opts) do
case Keyword.get(opts, :stt_model) do
%OpenAISTT{} = model ->
model
nil ->
OpenAISTT.new(
Keyword.get(opts, :model, OpenAISTT.model_name()),
api_key: Keyword.get(opts, :api_key),
base_url: Keyword.get(opts, :base_url, Defaults.openai_api_base_url()),
client: Keyword.get(opts, :client)
)
end
end
defp start_transcription_task(state) do
session = self()
runner = fn ->
run_transcription(session, state)
end
TaskSupport.async_nolink(runner)
end
defp run_transcription(session, state) do
case collect_audio(state.input) do
<<>> ->
send(session, :session_complete)
audio_data ->
audio_input = AudioInput.new(audio_data)
case OpenAISTT.transcribe(
state.stt_model,
audio_input,
state.settings,
state.trace_include_sensitive_data,
state.trace_include_sensitive_audio_data
) do
{:ok, text} ->
maybe_send_transcript(session, text)
send(session, :session_complete)
{:error, reason} ->
send(session, {:session_error, reason})
end
end
rescue
error ->
send(session, {:session_error, error})
catch
kind, reason ->
send(session, {:session_error, {kind, reason}})
end
defp collect_audio(%StreamedAudioInput{} = input) do
input
|> StreamedAudioInput.stream()
|> Enum.reduce([], fn chunk, acc -> [chunk | acc] end)
|> Enum.reverse()
|> IO.iodata_to_binary()
end
defp maybe_send_transcript(session, text) do
if String.trim(text) != "" do
send(session, {:transcript, text})
end
:ok
end
defp finish_with_completion(state) do
Enum.each(state.waiters, fn {waiter, monitor_ref} ->
Process.demonitor(monitor_ref, [:flush])
GenServer.reply(waiter, :done)
end)
{:noreply, %{state | waiters: [], completion: :done}}
end
defp finish_with_error(state, reason) do
Enum.each(state.waiters, fn {waiter, monitor_ref} ->
Process.demonitor(monitor_ref, [:flush])
GenServer.reply(waiter, {:error, reason})
end)
{:noreply, %{state | waiters: [], completion: {:error, reason}}}
end
defp close_websocket(pid) when is_pid(pid) do
if Process.alive?(pid) do
Process.exit(pid, :shutdown)
end
rescue
_ -> :ok
catch
:exit, _ -> :ok
end
defp close_websocket(_), do: :ok
defp shutdown_task(%Task{} = task) do
case task.pid do
pid when is_pid(pid) ->
if Process.alive?(pid) do
Process.exit(pid, :kill)
end
:ok
_ ->
:ok
end
rescue
_ -> :ok
catch
:exit, _ -> :ok
end
defp shutdown_task(_), do: :ok
defp monitor_waiter({pid, _tag}) when is_pid(pid) do
Process.monitor(pid)
end
end