Packages
llama_cpp_ex
0.6.3
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.28
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.10
0.8.9
0.8.8
0.8.7
0.8.6
0.8.5
0.8.4
0.8.3
0.8.2
0.8.1
0.8.0
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.0
0.6.14
0.6.13
0.6.12
0.6.11
0.6.10
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.0
0.4.4
0.4.3
0.4.2
0.4.1
0.3.0
0.2.0
Elixir bindings for llama.cpp — run LLMs locally with Metal, CUDA, Vulkan, or CPU acceleration.
Current section
Files
Jump to
Current section
Files
lib/llama_cpp_ex/embedding.ex
defmodule LlamaCppEx.Embedding do
@moduledoc "Generate embeddings from text using an embedding model."
alias LlamaCppEx.{Model, Context, Tokenizer}
@type t :: [float()]
@doc """
Computes an embedding for a single text.
## Options
* `:n_ctx` - Context size. Defaults to `2048`.
* `:pooling_type` - Pooling type. Defaults to `:unspecified` (model's default).
Values: `:unspecified`, `:none`, `:mean`, `:cls`, `:last`.
* `:normalize` - Normalization mode. `2` = L2 (default), `0` = max-abs, `-1` = none.
"""
@spec embed(Model.t(), String.t(), keyword()) :: {:ok, t()} | {:error, String.t()}
def embed(%Model{} = model, text, opts \\ []) when is_binary(text) do
n_ctx = Keyword.get(opts, :n_ctx, 2048)
pooling_type = Keyword.get(opts, :pooling_type, :unspecified)
normalize = Keyword.get(opts, :normalize, 2)
{:ok, tokens} = Tokenizer.encode(model, text)
ctx_size = max(n_ctx, length(tokens) + 8)
with {:ok, ctx} <-
Context.create(model,
n_ctx: ctx_size,
embeddings: true,
pooling_type: pooling_type
),
:ok <- embed_decode(ctx, tokens, 0) do
get_embeddings(ctx, 0, normalize)
end
end
@doc """
Computes embeddings for multiple texts.
Uses a fresh context for each text. Accepts the same options as `embed/3`.
"""
@spec embed_batch(Model.t(), [String.t()], keyword()) :: {:ok, [t()]} | {:error, String.t()}
def embed_batch(%Model{} = model, texts, opts \\ []) when is_list(texts) do
results =
Enum.reduce_while(texts, {:ok, []}, fn text, {:ok, acc} ->
case embed(model, text, opts) do
{:ok, emb} -> {:cont, {:ok, [emb | acc]}}
{:error, _} = err -> {:halt, err}
end
end)
case results do
{:ok, embeddings} -> {:ok, Enum.reverse(embeddings)}
{:error, _} = err -> err
end
end
defp embed_decode(%Context{ref: ref}, tokens, seq_id) do
case LlamaCppEx.NIF.embed_decode(ref, tokens, seq_id) do
:ok -> :ok
{:error, _} = err -> err
end
end
defp get_embeddings(%Context{ref: ref}, seq_id, normalize) do
case LlamaCppEx.NIF.get_embeddings(ref, seq_id, normalize) do
{:ok, _} = result -> result
{:error, _} = err -> err
end
end
end