Packages

A Gleam WebSocket consumer for AT Protocol Jetstream events

Current section

Files

Jump to
goose src goose.gleam
Raw

src/goose.gleam

import gleam/dynamic.{type Dynamic}
import gleam/dynamic/decode
import gleam/erlang/atom
import gleam/erlang/process.{type Pid}
import gleam/int
import gleam/io
import gleam/json
import gleam/list
import gleam/option.{type Option}
import gleam/string
/// Jetstream event types
pub type JetstreamEvent {
CommitEvent(did: String, time_us: Int, commit: CommitData)
IdentityEvent(did: String, time_us: Int, identity: IdentityData)
AccountEvent(did: String, time_us: Int, account: AccountData)
UnknownEvent(raw: String)
}
pub type CommitData {
CommitData(
rev: String,
operation: String,
collection: String,
rkey: String,
record: Option(Dynamic),
cid: Option(String),
)
}
pub type IdentityData {
IdentityData(did: String, handle: String, seq: Int, time: String)
}
pub type AccountData {
AccountData(active: Bool, did: String, seq: Int, time: String)
}
/// Configuration for Jetstream consumer
pub type JetstreamConfig {
JetstreamConfig(
endpoint: String,
wanted_collections: List(String),
wanted_dids: List(String),
cursor: Option(Int),
max_message_size_bytes: Option(Int),
compress: Bool,
require_hello: Bool,
/// Maximum backoff time in seconds for retry logic (default: 60)
max_backoff_seconds: Int,
/// Whether to log connection events (connected, disconnected) (default: True)
log_connection_events: Bool,
/// Whether to log retry attempts and errors (default: True)
log_retry_attempts: Bool,
)
}
/// Create a default configuration for US East endpoint
/// Includes automatic retry with exponential backoff (1s, 2s, 4s, 8s, 16s, 32s, capped at 60s)
pub fn default_config() -> JetstreamConfig {
JetstreamConfig(
endpoint: "wss://jetstream2.us-east.bsky.network/subscribe",
wanted_collections: [],
wanted_dids: [],
cursor: option.None,
max_message_size_bytes: option.None,
compress: False,
require_hello: False,
max_backoff_seconds: 60,
log_connection_events: True,
log_retry_attempts: True,
)
}
/// Build the WebSocket URL with query parameters
pub fn build_url(config: JetstreamConfig) -> String {
let base = config.endpoint
let mut_params = []
// Add wanted collections (each as a separate query parameter)
let mut_params = case config.wanted_collections {
[] -> mut_params
collections -> {
let collection_params =
list.map(collections, fn(col) { "wantedCollections=" <> col })
list.append(collection_params, mut_params)
}
}
// Add wanted DIDs (each as a separate query parameter)
let mut_params = case config.wanted_dids {
[] -> mut_params
dids -> {
let did_params = list.map(dids, fn(did) { "wantedDids=" <> did })
list.append(did_params, mut_params)
}
}
// Add cursor if specified
let mut_params = case config.cursor {
option.None -> mut_params
option.Some(cursor_val) ->
list.append(["cursor=" <> string.inspect(cursor_val)], mut_params)
}
// Add maxMessageSizeBytes if specified
let mut_params = case config.max_message_size_bytes {
option.None -> mut_params
option.Some(size_val) ->
list.append(
["maxMessageSizeBytes=" <> string.inspect(size_val)],
mut_params,
)
}
// Add compress parameter (always include it)
let mut_params = case config.compress {
False -> list.append(["compress=false"], mut_params)
True -> list.append(["compress=true"], mut_params)
}
// Add requireHello parameter (always include it)
let mut_params = case config.require_hello {
False -> list.append(["requireHello=false"], mut_params)
True -> list.append(["requireHello=true"], mut_params)
}
case mut_params {
[] -> base
params -> base <> "?" <> string.join(list.reverse(params), "&")
}
}
/// Connect to Jetstream WebSocket using Erlang gun library
@external(erlang, "goose_ws_ffi", "connect")
pub fn connect(
url: String,
handler_pid: Pid,
compress: Bool,
) -> Result(Pid, Dynamic)
/// Start consuming the Jetstream feed with automatic retry logic
///
/// Handles connection failures gracefully with exponential backoff and automatic reconnection.
/// The retry behavior is configured through the JetstreamConfig fields:
/// - max_backoff_seconds: Maximum wait time between retries
/// - log_connection_events: Log connects/disconnects
/// - log_retry_attempts: Log retry attempts and errors
///
/// Example:
/// ```gleam
/// let config = goose.default_config()
///
/// goose.start_consumer(config, fn(event_json) {
/// // Handle event
/// io.println(event_json)
/// })
/// ```
pub fn start_consumer(
config: JetstreamConfig,
on_event: fn(String) -> Nil,
) -> Nil {
start_with_retry_internal(config, on_event, 0)
}
/// Internal function to handle connection with retry
fn start_with_retry_internal(
config: JetstreamConfig,
on_event: fn(String) -> Nil,
retry_count: Int,
) -> Nil {
let url = build_url(config)
let self = process.self()
let result = connect(url, self, config.compress)
case result {
Ok(_conn_pid) -> {
case config.log_connection_events {
True -> io.println("Connected to Jetstream successfully")
False -> Nil
}
// Start receiving with retry support
receive_with_retry(config, on_event)
}
Error(err) -> {
// Connection failed, calculate backoff and retry
let backoff_seconds = calculate_backoff(retry_count, config)
case config.log_retry_attempts {
True -> {
io.println(
"Failed to connect to Jetstream (attempt "
<> int.to_string(retry_count + 1)
<> "): "
<> string.inspect(err),
)
io.println(
"Retrying in " <> int.to_string(backoff_seconds) <> " seconds...",
)
}
False -> Nil
}
// Sleep for backoff period
process.sleep(backoff_seconds * 1000)
// Retry connection
start_with_retry_internal(config, on_event, retry_count + 1)
}
}
}
/// Calculate exponential backoff with configurable maximum
fn calculate_backoff(retry_count: Int, config: JetstreamConfig) -> Int {
let backoff = case retry_count {
0 -> 1
1 -> 2
2 -> 4
3 -> 8
4 -> 16
5 -> 32
_ -> config.max_backoff_seconds
}
// Cap at max_backoff_seconds
case backoff > config.max_backoff_seconds {
True -> config.max_backoff_seconds
False -> backoff
}
}
/// Receive messages with retry logic
fn receive_with_retry(
config: JetstreamConfig,
on_event: fn(String) -> Nil,
) -> Nil {
case receive_ws_message() {
Ok(text) -> {
on_event(text)
receive_with_retry(config, on_event)
}
Error(error_dynamic) -> {
// Decode error type
let atm = atom.cast_from_dynamic(error_dynamic)
let error_type = atom.to_string(atm)
case error_type {
"timeout" -> {
// No messages in 60s, connection is alive - continue
receive_with_retry(config, on_event)
}
"closed" -> {
// Connection closed - log if configured
case config.log_connection_events {
True -> io.println("Jetstream connection closed, reconnecting...")
False -> Nil
}
start_with_retry_internal(config, on_event, 0)
}
"connection_error" -> {
// Connection error - log if configured
case config.log_connection_events {
True -> io.println("Jetstream connection error, reconnecting...")
False -> Nil
}
start_with_retry_internal(config, on_event, 0)
}
_ -> {
// Unknown error - log if retry logging is enabled
case config.log_retry_attempts {
True -> {
io.println("Unknown Jetstream error: " <> error_type)
io.println("Reconnecting...")
}
False -> Nil
}
start_with_retry_internal(config, on_event, 0)
}
}
}
}
}
/// Receive a WebSocket message from the message queue
/// Returns Ok(text) for messages, or Error with one of: timeout, closed, connection_error
@external(erlang, "goose_ffi", "receive_ws_message")
fn receive_ws_message() -> Result(String, Dynamic)
/// Parse a JSON event string into a JetstreamEvent
pub fn parse_event(json_string: String) -> JetstreamEvent {
// Try to parse as commit event first
case json.parse(json_string, commit_event_decoder()) {
Ok(event) -> event
Error(_) -> {
// Try identity event
case json.parse(json_string, identity_event_decoder()) {
Ok(event) -> event
Error(_) -> {
// Try account event
case json.parse(json_string, account_event_decoder()) {
Ok(event) -> event
Error(_) -> UnknownEvent(json_string)
}
}
}
}
}
}
/// Decoder for commit events
fn commit_event_decoder() {
use did <- decode.field("did", decode.string)
use time_us <- decode.field("time_us", decode.int)
use commit <- decode.field("commit", commit_data_decoder())
decode.success(CommitEvent(did: did, time_us: time_us, commit: commit))
}
/// Decoder for commit data - handles both create/update (with record) and delete (without)
fn commit_data_decoder() {
// Try decoder with record and cid fields first (for create/update)
// If that fails, try without (for delete)
decode.one_of(commit_with_record_decoder(), or: [
commit_without_record_decoder(),
])
}
/// Decoder for commit with record (create/update operations)
fn commit_with_record_decoder() {
use rev <- decode.field("rev", decode.string)
use operation <- decode.field("operation", decode.string)
use collection <- decode.field("collection", decode.string)
use rkey <- decode.field("rkey", decode.string)
use record <- decode.field("record", decode.dynamic)
use cid <- decode.field("cid", decode.string)
decode.success(CommitData(
rev: rev,
operation: operation,
collection: collection,
rkey: rkey,
record: option.Some(record),
cid: option.Some(cid),
))
}
/// Decoder for commit without record (delete operations)
fn commit_without_record_decoder() {
use rev <- decode.field("rev", decode.string)
use operation <- decode.field("operation", decode.string)
use collection <- decode.field("collection", decode.string)
use rkey <- decode.field("rkey", decode.string)
decode.success(CommitData(
rev: rev,
operation: operation,
collection: collection,
rkey: rkey,
record: option.None,
cid: option.None,
))
}
/// Decoder for identity events
fn identity_event_decoder() {
use did <- decode.field("did", decode.string)
use time_us <- decode.field("time_us", decode.int)
use identity <- decode.field("identity", identity_data_decoder())
decode.success(IdentityEvent(did: did, time_us: time_us, identity: identity))
}
/// Decoder for identity data
fn identity_data_decoder() {
use did <- decode.field("did", decode.string)
use handle <- decode.field("handle", decode.string)
use seq <- decode.field("seq", decode.int)
use time <- decode.field("time", decode.string)
decode.success(IdentityData(did: did, handle: handle, seq: seq, time: time))
}
/// Decoder for account events
fn account_event_decoder() {
use did <- decode.field("did", decode.string)
use time_us <- decode.field("time_us", decode.int)
use account <- decode.field("account", account_data_decoder())
decode.success(AccountEvent(did: did, time_us: time_us, account: account))
}
/// Decoder for account data
fn account_data_decoder() {
use active <- decode.field("active", decode.bool)
use did <- decode.field("did", decode.string)
use seq <- decode.field("seq", decode.int)
use time <- decode.field("time", decode.string)
decode.success(AccountData(active: active, did: did, seq: seq, time: time))
}