Current section

Files

Jump to
gemini_ex lib gemini client websocket.ex
Raw

lib/gemini/client/websocket.ex

defmodule Gemini.Client.WebSocket do
@moduledoc """
WebSocket client for Gemini Live API using :gun.
This module provides low-level WebSocket connectivity with:
- TLS/HTTP2 connection management
- Automatic reconnection handling with configurable retry logic
- Message framing and parsing
- Auth strategy integration (Gemini API / Vertex AI)
- Comprehensive telemetry integration
## Usage
Typically used through `Gemini.Live.Session` rather than directly.
{:ok, conn} = WebSocket.connect(:gemini, model: "gemini-2.5-flash")
:ok = WebSocket.send(conn, %{setup: setup_config})
{:ok, message} = WebSocket.receive(conn)
:ok = WebSocket.close(conn)
## Connection Options
- `:model` - Required. Model name for the Live API
- `:project_id` - Required for Vertex AI
- `:location` - Vertex AI location (default: "us-central1")
- `:api_version` - API version (default: "v1beta")
- `:timeout` - Connection timeout in ms (default: 30000)
- `:retry_attempts` - Number of retry attempts for transient failures (default: 3)
- `:retry_delay` - Initial delay between retries in ms (default: 1000)
- `:retry_backoff` - Backoff multiplier for retries (default: 2.0)
## Connection State
The connection struct tracks:
- Gun connection PID
- Stream reference
- Authentication strategy
- Connection status
## Endpoints
- **Gemini API**: `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=API_KEY`
- **Vertex AI**: `wss://{location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent?project=...&location=...`
## Telemetry Events
This module emits the following telemetry events:
- `[:gemini, :live, :websocket, :connect, :start]` - Connection attempt started
- `[:gemini, :live, :websocket, :connect, :stop]` - Connection established
- `[:gemini, :live, :websocket, :connect, :exception]` - Connection failed
- `[:gemini, :live, :websocket, :send]` - Message sent
- `[:gemini, :live, :websocket, :receive]` - Message received
- `[:gemini, :live, :websocket, :close]` - Connection closed
- `[:gemini, :live, :websocket, :retry]` - Retry attempt
"""
require Logger
alias Gemini.Auth.MultiAuthCoordinator
alias Gemini.Config
alias Gemini.Telemetry
@type auth_strategy :: :gemini | :vertex_ai
@type connection_status :: :connecting | :connected | :closing | :closed
@type t :: %__MODULE__{
gun_pid: pid() | nil,
stream_ref: reference() | nil,
auth_strategy: auth_strategy() | nil,
status: connection_status(),
model: String.t() | nil,
project_id: String.t() | nil,
location: String.t() | nil,
api_version: String.t(),
retry_config: retry_config()
}
@type retry_config :: %{
attempts: non_neg_integer(),
delay: non_neg_integer(),
backoff: float()
}
@type connection_error ::
:project_id_required_for_vertex_ai
| :no_api_key
| {:open_failed, term()}
| {:connection_failed, term()}
| {:upgrade_failed, integer(), list()}
| {:upgrade_error, term()}
| :upgrade_timeout
| {:max_retries_exceeded, term()}
@enforce_keys []
defstruct [
:gun_pid,
:stream_ref,
:auth_strategy,
:model,
:project_id,
:location,
status: :connecting,
api_version: "v1beta",
retry_config: %{attempts: 3, delay: 1000, backoff: 2.0}
]
# Gemini API endpoint - v1beta is the default Live API version.
# Use v1alpha for native audio extras (affective dialog, proactivity, thinking).
@gemini_host "generativelanguage.googleapis.com"
# Vertex AI Live endpoint (v1)
@vertex_path "/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent"
# Connection timeouts
@connect_timeout 30_000
@upgrade_timeout 10_000
# Default retry configuration
@default_retry_attempts 3
@default_retry_delay 1_000
@default_retry_backoff 2.0
# Retryable error types
@retryable_errors [:timeout, :closed, :econnrefused, :econnreset, :etimedout]
@redact_query_params ~w(key access_token token)
# Build gun options at runtime to avoid compile-time function capture issue
# WebSocket connections require HTTP/1.1 for the upgrade handshake
@spec gun_opts() :: map()
defp gun_opts do
%{
protocols: [:http],
transport: :tls,
tls_opts: [
verify: :verify_peer,
cacerts: :public_key.cacerts_get(),
depth: 3,
customize_hostname_check: [
match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
]
]
}
end
@doc """
Establishes a WebSocket connection to the Live API.
## Parameters
- `auth_strategy` - `:gemini` or `:vertex_ai`
- `opts` - Connection options:
- `:model` - Required. Model name
- `:project_id` - Required for Vertex AI
- `:location` - Vertex AI location (default: "us-central1")
- `:api_version` - Gemini API version for `:gemini` connections (default: "v1beta")
- `:timeout` - Connection timeout in ms (default: 30000)
- `:retry_attempts` - Number of retry attempts (default: 3)
- `:retry_delay` - Initial retry delay in ms (default: 1000)
- `:retry_backoff` - Backoff multiplier (default: 2.0)
## Returns
- `{:ok, connection}` - Successfully connected
- `{:error, reason}` - Connection failed
## Examples
{:ok, conn} = WebSocket.connect(:gemini, model: "gemini-2.5-flash-native-audio-preview-12-2025")
{:ok, conn} = WebSocket.connect(:vertex_ai,
model: "gemini-2.5-flash-native-audio-preview-12-2025",
project_id: "my-project",
location: "us-central1"
)
# With custom retry configuration
{:ok, conn} = WebSocket.connect(:gemini,
model: "gemini-2.5-flash",
retry_attempts: 5,
retry_delay: 2000,
retry_backoff: 1.5
)
"""
@spec connect(auth_strategy(), keyword()) :: {:ok, t()} | {:error, connection_error()}
def connect(auth_strategy, opts \\ []) do
start_time = System.monotonic_time()
model = Keyword.fetch!(opts, :model)
project_id = Keyword.get(opts, :project_id)
location = Keyword.get(opts, :location, "us-central1")
api_version = Keyword.get(opts, :api_version, "v1beta")
timeout = Keyword.get(opts, :timeout, @connect_timeout)
retry_config = %{
attempts: Keyword.get(opts, :retry_attempts, @default_retry_attempts),
delay: Keyword.get(opts, :retry_delay, @default_retry_delay),
backoff: Keyword.get(opts, :retry_backoff, @default_retry_backoff)
}
conn = %__MODULE__{
auth_strategy: auth_strategy,
model: model,
project_id: project_id,
location: location,
api_version: api_version,
retry_config: retry_config
}
# Emit telemetry start event
emit_connect_start(conn)
result =
with {:ok, conn} <- validate_config(conn) do
connect_with_retry(conn, timeout, retry_config.attempts, retry_config.delay)
end
# Emit telemetry stop/exception event
case result do
{:ok, conn} ->
emit_connect_stop(conn, start_time)
{:ok, conn}
{:error, reason} ->
emit_connect_exception(conn, reason, start_time)
{:error, reason}
end
end
@doc """
Sends a message over the WebSocket connection.
The message should be a map that will be JSON-encoded.
## Parameters
- `conn` - The WebSocket connection struct
- `message` - A map to be JSON-encoded and sent
## Returns
- `:ok` - Message sent successfully
- `{:error, reason}` - Send failed
## Example
:ok = WebSocket.send(conn, %{
"clientContent" => %{
"turns" => [%{"role" => "user", "parts" => [%{"text" => "Hello"}]}],
"turnComplete" => true
}
})
"""
@spec send(t(), map()) :: :ok | {:error, term()}
def send(%__MODULE__{status: :connected, gun_pid: pid, stream_ref: ref} = conn, message)
when is_map(message) do
json = Jason.encode!(message)
:gun.ws_send(pid, ref, {:text, json})
emit_send(conn, message, byte_size(json))
:ok
catch
:error, reason -> {:error, reason}
end
def send(%__MODULE__{status: status}, _message) do
{:error, {:not_connected, status}}
end
@doc """
Receives the next message from the WebSocket connection.
This is a blocking call that waits for the next message.
## Parameters
- `conn` - The connection struct
- `timeout` - Timeout in milliseconds (default: 60000)
## Returns
- `{:ok, message}` - Received and parsed JSON message
- `{:error, :timeout}` - No message received within timeout
- `{:error, :closed}` - Connection closed
- `{:error, reason}` - Other error
"""
@spec receive(t(), timeout()) :: {:ok, map()} | {:error, term()}
def receive(%__MODULE__{gun_pid: pid, stream_ref: ref} = conn, timeout \\ 60_000) do
start_time = System.monotonic_time()
result =
receive do
{:gun_ws, ^pid, ^ref, {:text, data}} ->
case Jason.decode(data) do
{:ok, message} -> {:ok, message}
{:error, _} = error -> error
end
# Live API sends binary frames containing JSON
{:gun_ws, ^pid, ^ref, {:binary, data}} ->
case Jason.decode(data) do
{:ok, message} -> {:ok, message}
{:error, _} = error -> error
end
{:gun_ws, ^pid, ^ref, {:close, code, reason}} ->
Logger.debug("WebSocket closed: code=#{code}, reason=#{reason}")
{:error, {:closed, code, reason}}
{:gun_down, ^pid, :http2, reason, _} ->
Logger.warning("Gun connection down: #{inspect(reason)}")
{:error, {:connection_down, reason}}
{:gun_error, ^pid, ^ref, reason} ->
Logger.error("WebSocket error: #{inspect(reason)}")
{:error, reason}
{:gun_error, ^pid, reason} ->
Logger.error("Gun error: #{inspect(reason)}")
{:error, reason}
after
timeout ->
{:error, :timeout}
end
# Emit telemetry for received message
case result do
{:ok, message} ->
emit_receive(conn, message, start_time)
_ ->
:ok
end
result
catch
:exit, reason ->
{:error, {:exit, reason}}
end
@doc """
Receives all available messages without blocking.
Returns a list of messages that are immediately available.
"""
@spec receive_all(t()) :: [map()]
def receive_all(%__MODULE__{} = conn) do
receive_all(conn, [])
end
defp receive_all(conn, acc) do
case __MODULE__.receive(conn, 0) do
{:ok, message} -> receive_all(conn, [message | acc])
{:error, :timeout} -> Enum.reverse(acc)
{:error, _} -> Enum.reverse(acc)
end
end
@doc """
Closes the WebSocket connection gracefully.
## Parameters
- `conn` - The WebSocket connection struct
## Returns
- `:ok` - Always returns :ok
"""
@spec close(t()) :: :ok
def close(%__MODULE__{gun_pid: nil} = conn) do
emit_close(conn, :already_closed)
:ok
end
def close(%__MODULE__{gun_pid: pid, stream_ref: ref, status: :connected} = conn) do
:gun.ws_send(pid, ref, :close)
:gun.close(pid)
emit_close(conn, :graceful)
:ok
end
def close(%__MODULE__{gun_pid: pid} = conn) do
:gun.close(pid)
emit_close(conn, :forced)
:ok
end
@doc """
Returns the current connection status.
"""
@spec status(t()) :: connection_status()
def status(%__MODULE__{status: status}), do: status
@doc """
Checks if the connection is active.
"""
@spec connected?(t()) :: boolean()
def connected?(%__MODULE__{status: :connected}), do: true
def connected?(_), do: false
@doc """
Checks if an error is retryable.
Returns true if the error is a transient failure that might succeed on retry.
## Parameters
- `error` - The error to check
## Returns
- `true` if the error is retryable
- `false` otherwise
"""
@spec retryable_error?(term()) :: boolean()
def retryable_error?(:timeout), do: true
def retryable_error?(:closed), do: true
def retryable_error?(:econnrefused), do: true
def retryable_error?(:econnreset), do: true
def retryable_error?(:etimedout), do: true
def retryable_error?({:connection_failed, _}), do: true
def retryable_error?({:open_failed, reason}) when reason in @retryable_errors, do: true
def retryable_error?({:upgrade_error, {:stream_error, _, _}}), do: true
def retryable_error?(:upgrade_timeout), do: true
def retryable_error?(_), do: false
# Private Functions
@spec validate_config(t()) :: {:ok, t()} | {:error, atom()}
defp validate_config(%__MODULE__{auth_strategy: :vertex_ai, project_id: nil}) do
{:error, :project_id_required_for_vertex_ai}
end
defp validate_config(conn), do: {:ok, conn}
@spec connect_with_retry(t(), timeout(), non_neg_integer(), non_neg_integer()) ::
{:ok, t()} | {:error, connection_error()}
defp connect_with_retry(conn, timeout, attempts_remaining, current_delay) do
case do_connect(conn, timeout) do
{:ok, connected_conn} ->
{:ok, connected_conn}
{:error, reason} when attempts_remaining > 0 ->
if retryable_error?(reason) do
Logger.warning(
"WebSocket connection failed (#{inspect(reason)}), " <>
"retrying in #{current_delay}ms (#{attempts_remaining} attempts remaining)"
)
emit_retry(conn, reason, attempts_remaining, current_delay)
Process.sleep(current_delay)
next_delay = round(current_delay * conn.retry_config.backoff)
connect_with_retry(conn, timeout, attempts_remaining - 1, next_delay)
else
# Non-retryable error
Logger.error("WebSocket connection failed with non-retryable error: #{inspect(reason)}")
{:error, reason}
end
{:error, reason} ->
# No more retries
Logger.error("WebSocket connection failed after all retries: #{inspect(reason)}")
{:error, {:max_retries_exceeded, reason}}
end
end
@spec do_connect(t(), timeout()) :: {:ok, t()} | {:error, term()}
defp do_connect(conn, timeout) do
with {:ok, conn} <- open_connection(conn, timeout),
{:ok, conn} <- upgrade_to_websocket(conn) do
{:ok, %{conn | status: :connected}}
end
end
@spec open_connection(t(), timeout()) :: {:ok, t()} | {:error, term()}
defp open_connection(%__MODULE__{auth_strategy: :gemini} = conn, timeout) do
Logger.debug("Opening connection to Gemini API Live endpoint")
do_open_connection(conn, @gemini_host, 443, timeout)
end
defp open_connection(%__MODULE__{auth_strategy: :vertex_ai, location: location} = conn, timeout) do
host = "#{location}-aiplatform.googleapis.com"
Logger.debug("Opening connection to Vertex AI Live endpoint: #{host}")
do_open_connection(conn, host, 443, timeout)
end
@spec do_open_connection(t(), String.t(), pos_integer(), timeout()) ::
{:ok, t()} | {:error, term()}
defp do_open_connection(conn, host, port, timeout) do
case :gun.open(String.to_charlist(host), port, gun_opts()) do
{:ok, pid} ->
case :gun.await_up(pid, timeout) do
{:ok, protocol} when protocol in [:http, :http2] ->
Logger.debug("Gun connection established with #{protocol}")
{:ok, %{conn | gun_pid: pid}}
{:error, reason} ->
:gun.close(pid)
{:error, {:connection_failed, reason}}
end
{:error, reason} ->
{:error, {:open_failed, reason}}
end
end
@spec upgrade_to_websocket(t()) :: {:ok, t()} | {:error, term()}
defp upgrade_to_websocket(%__MODULE__{} = conn) do
path = build_websocket_path(conn)
headers = build_upgrade_headers(conn)
Logger.debug("Upgrading to WebSocket: #{redact_websocket_path(path)}")
stream_ref = :gun.ws_upgrade(conn.gun_pid, path, headers, %{})
receive do
{:gun_upgrade, _pid, ^stream_ref, ["websocket"], _headers} ->
Logger.debug("WebSocket upgrade successful")
{:ok, %{conn | stream_ref: stream_ref}}
{:gun_response, _pid, ^stream_ref, :fin, status, resp_headers} ->
Logger.error("WebSocket upgrade failed: status=#{status}")
{:error, {:upgrade_failed, status, resp_headers}}
{:gun_error, _pid, ^stream_ref, reason} ->
Logger.error("WebSocket upgrade error: #{inspect(reason)}")
{:error, {:upgrade_error, reason}}
after
@upgrade_timeout ->
{:error, :upgrade_timeout}
end
end
@spec build_websocket_path(t()) :: String.t()
defp build_websocket_path(%__MODULE__{auth_strategy: :gemini} = conn) do
case get_auth_params(conn) do
{:ok, %{api_key: api_key}} ->
"#{gemini_path(conn.api_version)}?key=#{api_key}"
{:error, _} ->
# Fallback - will fail at server
gemini_path(conn.api_version)
end
end
defp build_websocket_path(%__MODULE__{auth_strategy: :vertex_ai}) do
@vertex_path
end
@doc false
@spec redact_websocket_path(String.t()) :: String.t()
def redact_websocket_path(path) when is_binary(path) do
Enum.reduce(@redact_query_params, path, &redact_query_param/2)
end
@doc false
@spec redacted_websocket_path(t()) :: String.t()
def redacted_websocket_path(%__MODULE__{} = conn) do
conn
|> build_websocket_path()
|> redact_websocket_path()
end
defp redact_query_param(param, path) do
regex = Regex.compile!("([?&]#{Regex.escape(param)}=)[^&]+", "i")
Regex.replace(regex, path, "\\1[REDACTED]")
end
defp gemini_path(api_version) when is_binary(api_version) do
"/ws/google.ai.generativelanguage.#{api_version}.GenerativeService.BidiGenerateContent"
end
@spec build_upgrade_headers(t()) :: [{String.t(), String.t()}]
defp build_upgrade_headers(%__MODULE__{auth_strategy: :gemini}) do
[
{"content-type", "application/json"}
]
end
defp build_upgrade_headers(%__MODULE__{auth_strategy: :vertex_ai} = conn) do
case get_vertex_token(conn) do
{:ok, token} ->
[
{"authorization", "Bearer #{token}"},
{"content-type", "application/json"}
]
{:error, _} ->
[{"content-type", "application/json"}]
end
end
@spec get_auth_params(t()) :: {:ok, map()} | {:error, term()}
defp get_auth_params(%__MODULE__{auth_strategy: :gemini}) do
case Config.api_key() do
nil -> {:error, :no_api_key}
key -> {:ok, %{api_key: key}}
end
end
@spec get_vertex_token(t()) :: {:ok, String.t()} | {:error, term()}
defp get_vertex_token(conn) do
case get_vertex_credentials(conn) do
{:ok, %{access_token: token}} when is_binary(token) ->
{:ok, token}
{:ok, creds} ->
extract_token_from_auth(creds)
error ->
error
end
end
@spec extract_token_from_auth(map()) :: {:ok, String.t()} | {:error, term()}
defp extract_token_from_auth(creds) do
case Gemini.Auth.build_headers(:vertex_ai, creds) do
{:ok, headers} ->
extract_bearer_token(headers)
error ->
error
end
end
@spec extract_bearer_token([{String.t(), String.t()}]) :: {:ok, String.t()} | {:error, term()}
defp extract_bearer_token(headers) do
case List.keyfind(headers, "Authorization", 0) do
{_, "Bearer " <> token} -> {:ok, token}
_ -> {:error, :no_token_in_headers}
end
end
@spec get_vertex_credentials(t()) :: {:ok, map()} | {:error, term()}
defp get_vertex_credentials(conn) do
opts = [
project_id: conn.project_id,
location: conn.location
]
MultiAuthCoordinator.get_credentials(:vertex_ai, opts)
end
# Telemetry helpers
@spec emit_connect_start(t()) :: :ok
defp emit_connect_start(conn) do
Telemetry.execute(
[:gemini, :live, :websocket, :connect, :start],
%{system_time: System.system_time()},
%{
auth_strategy: conn.auth_strategy,
model: conn.model,
location: conn.location
}
)
end
@spec emit_connect_stop(t(), integer()) :: :ok
defp emit_connect_stop(conn, start_time) do
duration = Telemetry.calculate_duration(start_time)
Telemetry.execute(
[:gemini, :live, :websocket, :connect, :stop],
%{duration: duration},
%{
auth_strategy: conn.auth_strategy,
model: conn.model,
location: conn.location,
status: :connected
}
)
end
@spec emit_connect_exception(t(), term(), integer()) :: :ok
defp emit_connect_exception(conn, reason, start_time) do
duration = Telemetry.calculate_duration(start_time)
Telemetry.execute(
[:gemini, :live, :websocket, :connect, :exception],
%{duration: duration},
%{
auth_strategy: conn.auth_strategy,
model: conn.model,
location: conn.location,
error: reason
}
)
end
@spec emit_send(t(), map(), non_neg_integer()) :: :ok
defp emit_send(conn, message, size) do
Telemetry.execute(
[:gemini, :live, :websocket, :send],
%{size: size},
%{
auth_strategy: conn.auth_strategy,
model: conn.model,
message_type: detect_message_type(message)
}
)
end
@spec emit_receive(t(), map(), integer()) :: :ok
defp emit_receive(conn, message, start_time) do
duration = Telemetry.calculate_duration(start_time)
Telemetry.execute(
[:gemini, :live, :websocket, :receive],
%{duration: duration},
%{
auth_strategy: conn.auth_strategy,
model: conn.model,
message_type: detect_message_type(message)
}
)
end
@spec emit_close(t(), atom()) :: :ok
defp emit_close(conn, reason) do
Telemetry.execute(
[:gemini, :live, :websocket, :close],
%{},
%{
auth_strategy: conn.auth_strategy,
model: conn.model,
close_reason: reason
}
)
end
@spec emit_retry(t(), term(), non_neg_integer(), non_neg_integer()) :: :ok
defp emit_retry(conn, error, attempts_remaining, delay) do
Telemetry.execute(
[:gemini, :live, :websocket, :retry],
%{delay: delay, attempts_remaining: attempts_remaining},
%{
auth_strategy: conn.auth_strategy,
model: conn.model,
error: error
}
)
end
# Key mappings for message type detection: {string_key, atom_key, type}
@message_type_keys [
{"setup", :setup, :setup},
{"setupComplete", :setup_complete, :setup_complete},
{"clientContent", :client_content, :client_content},
{"serverContent", :server_content, :server_content},
{"realtimeInput", :realtime_input, :realtime_input},
{"toolCall", :tool_call, :tool_call},
{"toolResponse", :tool_response, :tool_response},
{"goAway", :go_away, :go_away}
]
@spec detect_message_type(map()) :: atom()
defp detect_message_type(message) when is_map(message) do
Enum.find_value(@message_type_keys, :unknown, fn {string_key, atom_key, type} ->
if Map.has_key?(message, string_key) or Map.has_key?(message, atom_key) do
type
end
end)
end
end