Current section
Files
Jump to
Current section
Files
lib/gemini/live/session.ex
defmodule Gemini.Live.Session do
@moduledoc """
GenServer for managing WebSocket connections to the Gemini Live API.
The Live API provides bidirectional streaming communication with Gemini models,
supporting real-time text, audio, and video interactions.
## Features
- WebSocket-based real-time communication
- Automatic reconnection with exponential backoff
- Message queuing during connection setup
- Callback-based event handling
- Graceful connection lifecycle management
## Usage
# Start a session
{:ok, session} = LiveSession.start_link(
model: "gemini-2.5-flash",
generation_config: %{temperature: 0.8},
on_message: fn message -> IO.inspect(message, label: "Received") end
)
# Connect to the Live API
:ok = LiveSession.connect(session)
# Send a text message
:ok = LiveSession.send(session, "Hello, how are you?")
# Send client content with tools
:ok = LiveSession.send_client_content(session, [
%{role: "user", parts: [%{text: "What's the weather?"}]}
])
# Send tool response
:ok = LiveSession.send_tool_response(session, [
%{name: "get_weather", response: %{temperature: 72, condition: "sunny"}}
])
# Close the session
:ok = LiveSession.close(session)
## Callbacks
The session supports several callbacks for handling events:
- `:on_message` - Called when a message is received from the server
- `:on_connect` - Called when successfully connected
- `:on_disconnect` - Called when disconnected
- `:on_error` - Called when an error occurs
## Example with Callbacks
LiveSession.start_link(
model: "gemini-2.5-flash",
on_message: fn message ->
case message do
%{server_content: content} ->
IO.puts("Model: \#{inspect(content)}")
%{tool_call: calls} ->
IO.puts("Function calls: \#{inspect(calls)}")
_ ->
IO.puts("Other: \#{inspect(message)}")
end
end,
on_connect: fn -> IO.puts("Connected!") end,
on_error: fn error -> IO.puts("Error: \#{inspect(error)}") end
)
"""
use GenServer
require Logger
alias Gemini.Auth.MultiAuthCoordinator
alias Gemini.Config
alias Gemini.Live.Message
alias Gemini.Live.Message.{ClientContent, ClientMessage, LiveClientSetup, ToolResponse}
@type session :: pid() | atom()
@type message :: String.t() | map()
@type callback :: (term() -> any())
@default_reconnect_delay 1000
@max_reconnect_delay 30_000
@ping_interval 30_000
defstruct [
:conn_pid,
:stream_ref,
:model,
:config,
:auth_strategy,
:websocket_url,
:auth_headers,
:on_message,
:on_connect,
:on_disconnect,
:on_error,
:message_queue,
:reconnect_delay,
:reconnect_timer,
:ping_timer,
:setup_sent,
:setup_complete,
status: :disconnected
]
@type t :: %__MODULE__{
conn_pid: pid() | nil,
stream_ref: reference() | nil,
model: String.t(),
config: map(),
auth_strategy: :gemini | :vertex_ai,
websocket_url: String.t() | nil,
auth_headers: [{String.t(), String.t()}] | nil,
on_message: callback() | nil,
on_connect: callback() | nil,
on_disconnect: callback() | nil,
on_error: callback() | nil,
message_queue: [map()],
reconnect_delay: non_neg_integer(),
reconnect_timer: reference() | nil,
ping_timer: reference() | nil,
setup_sent: boolean(),
setup_complete: boolean(),
status: :disconnected | :connecting | :connected | :error
}
# Client API
@doc """
Start a Live API session.
## Options
- `:model` - Model to use (required)
- `:auth` - Authentication strategy (`:gemini` or `:vertex_ai`, auto-detected if not provided)
- `:generation_config` - Generation configuration
- `:system_instruction` - System instruction
- `:tools` - Available tools/functions
- `:tool_config` - Tool configuration
- `:on_message` - Callback when message received
- `:on_connect` - Callback when connected
- `:on_disconnect` - Callback when disconnected
- `:on_error` - Callback when error occurs
- `:name` - Optional name for the GenServer
## Examples
{:ok, session} = LiveSession.start_link(
model: "gemini-2.5-flash",
on_message: &handle_message/1
)
"""
@spec start_link(keyword()) :: GenServer.on_start()
def start_link(opts) do
{name, opts} = Keyword.pop(opts, :name)
if name do
GenServer.start_link(__MODULE__, opts, name: name)
else
GenServer.start_link(__MODULE__, opts)
end
end
@doc """
Connect to the Live API WebSocket endpoint.
This must be called after start_link to establish the connection.
## Examples
{:ok, session} = LiveSession.start_link(model: "gemini-2.5-flash")
:ok = LiveSession.connect(session)
"""
@spec connect(session()) :: :ok | {:error, term()}
def connect(session) do
GenServer.call(session, :connect)
end
@doc """
Send a text message to the model.
## Examples
:ok = LiveSession.send(session, "What is the weather today?")
"""
@spec send(session(), String.t()) :: :ok | {:error, term()}
def send(session, text) when is_binary(text) do
content = %ClientContent{
turns: [%{role: "user", parts: [%{text: text}]}],
turn_complete: true
}
send_client_content(session, content)
end
@doc """
Send client content (turns) to the model.
## Parameters
- `session` - Session PID or name
- `turns` - List of content turns or ClientContent struct
- `turn_complete` - Whether the turn is complete (default: true)
## Examples
# Send simple content
:ok = LiveSession.send_client_content(session, [
%{role: "user", parts: [%{text: "Hello"}]}
])
# Send ClientContent struct
content = %ClientContent{
turns: [...],
turn_complete: true
}
:ok = LiveSession.send_client_content(session, content)
"""
@spec send_client_content(session(), [map()] | ClientContent.t(), boolean()) ::
:ok | {:error, term()}
def send_client_content(session, turns, turn_complete \\ true)
def send_client_content(session, %ClientContent{} = content, _turn_complete) do
message = %ClientMessage{client_content: content}
GenServer.call(session, {:send_message, message})
end
def send_client_content(session, turns, turn_complete) when is_list(turns) do
content = %ClientContent{
turns: turns,
turn_complete: turn_complete
}
send_client_content(session, content)
end
@doc """
Send real-time input (audio/video) to the model.
## Parameters
- `session` - Session PID or name
- `media_chunks` - List of media chunks
- `opts` - Additional options
## Examples
# Send audio chunk
:ok = LiveSession.send_realtime_input(session, [
%{data: audio_data, mime_type: "audio/pcm"}
])
"""
@spec send_realtime_input(session(), [map()], keyword()) :: :ok | {:error, term()}
def send_realtime_input(session, media_chunks, _opts \\ []) when is_list(media_chunks) do
message = %ClientMessage{
realtime_input: %Message.RealtimeInput{media_chunks: media_chunks}
}
GenServer.call(session, {:send_message, message})
end
@doc """
Send tool/function response to the model.
## Parameters
- `session` - Session PID or name
- `function_responses` - List of function responses
- `opts` - Additional options
## Examples
:ok = LiveSession.send_tool_response(session, [
%{
name: "get_weather",
response: %{temperature: 72, condition: "sunny"}
}
])
"""
@spec send_tool_response(session(), [map()], keyword()) :: :ok | {:error, term()}
def send_tool_response(session, function_responses, _opts \\ [])
when is_list(function_responses) do
message = %ClientMessage{
tool_response: %ToolResponse{function_responses: function_responses}
}
GenServer.call(session, {:send_message, message})
end
@doc """
Close the Live API session.
## Examples
:ok = LiveSession.close(session)
"""
@spec close(session()) :: :ok
def close(session) do
GenServer.stop(session, :normal)
end
@doc """
Get the current status of the session.
Returns one of: `:disconnected`, `:connecting`, `:connected`, `:error`
## Examples
status = LiveSession.status(session)
# => :connected
"""
@spec status(session()) :: atom()
def status(session) do
GenServer.call(session, :status)
end
# GenServer Callbacks
@impl true
def init(opts) do
model = Keyword.fetch!(opts, :model)
auth_strategy = Keyword.get(opts, :auth, Config.current_api_type())
state = %__MODULE__{
model: model,
config: opts,
auth_strategy: auth_strategy,
on_message: Keyword.get(opts, :on_message),
on_connect: Keyword.get(opts, :on_connect),
on_disconnect: Keyword.get(opts, :on_disconnect),
on_error: Keyword.get(opts, :on_error),
message_queue: [],
reconnect_delay: @default_reconnect_delay,
setup_sent: false,
setup_complete: false,
status: :disconnected
}
{:ok, state}
end
@impl true
def handle_call(:connect, _from, state) do
case do_connect(state) do
{:ok, new_state} ->
{:reply, :ok, new_state}
{:error, _reason} = error ->
{:reply, error, %{state | status: :error}}
end
end
@impl true
def handle_call({:send_message, message}, _from, state) do
case state.status do
:connected when state.setup_complete ->
case send_websocket_message(state, message) do
:ok ->
{:reply, :ok, state}
{:error, reason} = error ->
Logger.error("Failed to send message: #{inspect(reason)}")
{:reply, error, state}
end
status when status in [:connecting, :connected] ->
# Queue message until setup is complete
new_state = %{state | message_queue: state.message_queue ++ [message]}
{:reply, :ok, new_state}
_ ->
{:reply, {:error, :not_connected}, state}
end
end
@impl true
def handle_call(:status, _from, state) do
{:reply, state.status, state}
end
@impl true
def handle_info({:gun_up, conn_pid, _protocol}, %{conn_pid: conn_pid} = state) do
Logger.debug("Gun connection established")
{:noreply, state}
end
@impl true
def handle_info(
{:gun_upgrade, conn_pid, stream_ref, ["websocket"], _headers},
%{conn_pid: conn_pid, stream_ref: stream_ref} = state
) do
Logger.info("WebSocket upgrade successful")
# Send setup message
setup_message = build_setup_message(state)
case send_websocket_message(state, setup_message) do
:ok ->
# Start ping timer
ping_timer = Process.send_after(self(), :send_ping, @ping_interval)
new_state = %{
state
| status: :connected,
setup_sent: true,
ping_timer: ping_timer
}
invoke_callback(state.on_connect, nil)
{:noreply, new_state}
{:error, reason} ->
Logger.error("Failed to send setup message: #{inspect(reason)}")
invoke_callback(state.on_error, reason)
{:noreply, schedule_reconnect(state)}
end
end
@impl true
def handle_info(
{:gun_ws, conn_pid, stream_ref, {:text, text}},
%{
conn_pid: conn_pid,
stream_ref: stream_ref
} = state
) do
case Message.from_json(text) do
{:ok, message} ->
handle_server_message(message, state)
{:error, reason} ->
Logger.error("Failed to parse WebSocket message: #{inspect(reason)}")
{:noreply, state}
end
end
@impl true
def handle_info(
{:gun_ws, conn_pid, stream_ref, {:close, code, reason}},
%{
conn_pid: conn_pid,
stream_ref: stream_ref
} = state
) do
Logger.warning("WebSocket closed: code=#{code}, reason=#{inspect(reason)}")
invoke_callback(state.on_disconnect, {code, reason})
{:noreply, schedule_reconnect(state)}
end
@impl true
def handle_info({:gun_down, conn_pid, _protocol, reason, _}, %{conn_pid: conn_pid} = state) do
Logger.warning("Gun connection down: #{inspect(reason)}")
invoke_callback(state.on_disconnect, reason)
{:noreply, schedule_reconnect(state)}
end
@impl true
def handle_info({:gun_error, conn_pid, reason}, %{conn_pid: conn_pid} = state) do
Logger.error("Gun error: #{inspect(reason)}")
invoke_callback(state.on_error, reason)
{:noreply, schedule_reconnect(state)}
end
@impl true
def handle_info(
{:gun_error, conn_pid, stream_ref, reason},
%{
conn_pid: conn_pid,
stream_ref: stream_ref
} = state
) do
Logger.error("Gun stream error: #{inspect(reason)}")
invoke_callback(state.on_error, reason)
{:noreply, state}
end
@impl true
def handle_info(:send_ping, state) do
if state.status == :connected do
:gun.ws_send(state.conn_pid, state.stream_ref, :ping)
end
ping_timer = Process.send_after(self(), :send_ping, @ping_interval)
{:noreply, %{state | ping_timer: ping_timer}}
end
@impl true
def handle_info(:reconnect, state) do
Logger.info("Attempting to reconnect...")
case do_connect(state) do
{:ok, new_state} ->
{:noreply, new_state}
{:error, _reason} ->
{:noreply, schedule_reconnect(state)}
end
end
@impl true
def terminate(_reason, state) do
# Clean up timers
if state.reconnect_timer, do: Process.cancel_timer(state.reconnect_timer)
if state.ping_timer, do: Process.cancel_timer(state.ping_timer)
# Close connection
if state.conn_pid do
:gun.close(state.conn_pid)
end
:ok
end
# Private Functions
defp do_connect(state) do
with {:ok, url, headers} <- build_websocket_url_and_headers(state),
{:ok, conn_pid, stream_ref} <- connect_websocket(url, headers) do
new_state = %{
state
| conn_pid: conn_pid,
stream_ref: stream_ref,
websocket_url: url,
auth_headers: headers,
status: :connecting,
reconnect_delay: @default_reconnect_delay
}
{:ok, new_state}
else
{:error, reason} = error ->
Logger.error("Failed to connect: #{inspect(reason)}")
invoke_callback(state.on_error, reason)
error
end
end
defp build_websocket_url_and_headers(state) do
case MultiAuthCoordinator.coordinate_auth(state.auth_strategy, Enum.into(state.config, %{})) do
{:ok, auth_strategy, headers} ->
url = build_live_api_url(state.model, auth_strategy, state.config)
{:ok, url, headers}
{:error, reason} ->
{:error, "Authentication failed: #{inspect(reason)}"}
end
end
defp build_live_api_url(_model, :gemini, config) do
api_key = Config.api_key() || Keyword.get(config, :api_key)
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key=#{api_key}"
end
defp build_live_api_url(_model, :vertex_ai, config) do
vertex_config = Config.get_auth_config(:vertex_ai)
project_id = Keyword.get(config, :project_id) || Map.get(vertex_config, :project_id)
location =
Keyword.get(config, :location, "us-central1") ||
Map.get(vertex_config, :location, "us-central1")
"wss://#{location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent?project=#{project_id}&location=#{location}"
end
defp connect_websocket(url, headers) do
uri = URI.parse(url)
port = uri.port || 443
host = String.to_charlist(uri.host)
# Build path with query string
path =
if uri.query do
"#{uri.path}?#{uri.query}"
else
uri.path
end
# Convert headers to gun format
gun_headers =
Enum.map(headers, fn {k, v} ->
{String.to_charlist(k), String.to_charlist(v)}
end)
with {:ok, conn_pid} <- :gun.open(host, port, %{protocols: [:http], transport: :tls}),
{:ok, _protocol} <- :gun.await_up(conn_pid),
stream_ref <- :gun.ws_upgrade(conn_pid, path, gun_headers) do
{:ok, conn_pid, stream_ref}
else
{:error, reason} ->
{:error, reason}
end
end
defp build_setup_message(state) do
setup = %LiveClientSetup{
model: state.model,
generation_config: Keyword.get(state.config, :generation_config),
system_instruction: Keyword.get(state.config, :system_instruction),
tools: Keyword.get(state.config, :tools),
tool_config: Keyword.get(state.config, :tool_config)
}
%ClientMessage{setup: setup}
end
defp send_websocket_message(state, %ClientMessage{} = message) do
case Message.to_json(message) do
{:ok, json} ->
:gun.ws_send(state.conn_pid, state.stream_ref, {:text, json})
:ok
{:error, reason} ->
{:error, reason}
end
end
defp handle_server_message(message, state) do
# Invoke callback
invoke_callback(state.on_message, message)
# Handle specific message types
new_state =
cond do
message.setup_complete ->
Logger.info("Setup complete")
# Flush queued messages
flush_message_queue(%{state | setup_complete: true})
message.server_content ->
state
message.tool_call ->
state
message.tool_call_cancellation ->
state
true ->
state
end
{:noreply, new_state}
end
defp flush_message_queue(state) do
Enum.each(state.message_queue, fn message ->
send_websocket_message(state, message)
end)
%{state | message_queue: []}
end
defp schedule_reconnect(state) do
# Cancel existing timer if any
if state.reconnect_timer do
Process.cancel_timer(state.reconnect_timer)
end
# Cancel ping timer
if state.ping_timer do
Process.cancel_timer(state.ping_timer)
end
# Close existing connection
if state.conn_pid do
:gun.close(state.conn_pid)
end
# Calculate next delay with exponential backoff
next_delay = min(state.reconnect_delay * 2, @max_reconnect_delay)
# Schedule reconnect
timer = Process.send_after(self(), :reconnect, state.reconnect_delay)
%{
state
| status: :disconnected,
conn_pid: nil,
stream_ref: nil,
setup_sent: false,
setup_complete: false,
reconnect_timer: timer,
reconnect_delay: next_delay,
ping_timer: nil
}
end
defp invoke_callback(nil, _arg), do: :ok
defp invoke_callback(callback, _arg) when is_function(callback, 0) do
callback.()
rescue
e -> Logger.error("Callback error: #{inspect(e)}")
end
defp invoke_callback(callback, arg) when is_function(callback, 1) do
callback.(arg)
rescue
e -> Logger.error("Callback error: #{inspect(e)}")
end
end