Current section
Files
Jump to
Current section
Files
lib/gepa/data_loader.ex
defmodule GEPA.DataLoader do
@moduledoc """
Protocol for data access with flexible ID types.
DataLoader provides an abstraction over data storage, allowing GEPA
to work with in-memory lists, databases, lazy streams, etc.
## Required Callbacks
- `all_ids/1`: Return all available data IDs
- `fetch/2`: Fetch data instances by ID
- `size/1`: Return total count of data instances
## Example Implementation
defmodule MyDatabaseLoader do
@behaviour GEPA.DataLoader
defstruct [:db_conn]
@impl true
def all_ids(%__MODULE__{db_conn: conn}) do
DB.query!(conn, "SELECT id FROM examples")
end
@impl true
def fetch(%__MODULE__{db_conn: conn}, ids) do
DB.query!(conn, "SELECT * FROM examples WHERE id IN (?)", [ids])
end
@impl true
def size(%__MODULE__{db_conn: conn}) do
DB.query_one!(conn, "SELECT COUNT(*) FROM examples")
end
end
"""
@type data_id :: term()
@type data_inst :: term()
@type t :: term()
@doc """
Return ordered list of all available data IDs.
The order should be stable across calls.
"""
@callback all_ids(t()) :: [data_id()]
@doc """
Fetch data instances for given IDs.
Must preserve order: `fetch(loader, [id1, id2])` returns data in same order.
## Contract
- `length(fetch(loader, ids)) == length(ids)`
- Order must match input IDs
- Missing IDs may raise or return nil - behavior is implementation-defined
"""
@callback fetch(t(), [data_id()]) :: [data_inst()]
@doc """
Return total number of data instances.
"""
@callback size(t()) :: non_neg_integer()
@doc """
Delegates to the implementation's all_ids/1.
"""
@spec all_ids(t()) :: [data_id()]
def all_ids(%module{} = loader) do
module.all_ids(loader)
end
@doc """
Delegates to the implementation's fetch/2.
"""
@spec fetch(t(), [data_id()]) :: [data_inst()]
def fetch(%module{} = loader, ids) do
module.fetch(loader, ids)
end
@doc """
Delegates to the implementation's size/1.
"""
@spec size(t()) :: non_neg_integer()
def size(%module{} = loader) do
module.size(loader)
end
end
defmodule GEPA.DataLoader.List do
@moduledoc """
Simple in-memory list-based data loader.
Uses integer indices as data IDs.
## Example
loader = GEPA.DataLoader.List.new([
%{input: "Q1", answer: "A1"},
%{input: "Q2", answer: "A2"}
])
GEPA.DataLoader.all_ids(loader) # => [0, 1]
GEPA.DataLoader.fetch(loader, [1, 0]) # => [%{input: "Q2", ...}, %{input: "Q1", ...}]
"""
@behaviour GEPA.DataLoader
defstruct [:items]
@type t :: %__MODULE__{items: [term()]}
@doc """
Create a list-based data loader.
"""
@spec new([term()]) :: t()
def new(items) when is_list(items) do
%__MODULE__{items: items}
end
@impl true
def all_ids(%__MODULE__{items: items}) do
Enum.to_list(0..(length(items) - 1))
end
@impl true
def fetch(%__MODULE__{items: items}, ids) do
Enum.map(ids, &Enum.at(items, &1))
end
@impl true
def size(%__MODULE__{items: items}) do
length(items)
end
end