Packages
llama_cpp_ex
0.6.7
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.28
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.10
0.8.9
0.8.8
0.8.7
0.8.6
0.8.5
0.8.4
0.8.3
0.8.2
0.8.1
0.8.0
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.0
0.6.14
0.6.13
0.6.12
0.6.11
0.6.10
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.0
0.4.4
0.4.3
0.4.2
0.4.1
0.3.0
0.2.0
Elixir bindings for llama.cpp — run LLMs locally with Metal, CUDA, Vulkan, or CPU acceleration.
Current section
Files
Jump to
Current section
Files
lib/llama_cpp_ex/tokenizer.ex
defmodule LlamaCppEx.Tokenizer do
@moduledoc """
Text tokenization and detokenization.
"""
@doc """
Encodes text into a list of token IDs.
## Options
* `:add_special` - Add special tokens (BOS/EOS). Defaults to `true`.
* `:parse_special` - Parse special token text (e.g., `<|im_start|>`). Defaults to `true`.
"""
@spec encode(LlamaCppEx.Model.t(), String.t(), keyword()) ::
{:ok, [integer()]} | {:error, String.t()}
def encode(%LlamaCppEx.Model{ref: ref}, text, opts \\ []) do
add_special = Keyword.get(opts, :add_special, true)
parse_special = Keyword.get(opts, :parse_special, true)
{:ok, LlamaCppEx.NIF.tokenize(ref, text, add_special, parse_special)}
rescue
e in ErlangError -> {:error, "tokenize failed: #{inspect(e.original)}"}
end
@doc """
Decodes a list of token IDs back into text.
"""
@spec decode(LlamaCppEx.Model.t(), [integer()]) :: {:ok, String.t()} | {:error, String.t()}
def decode(%LlamaCppEx.Model{ref: ref}, tokens) when is_list(tokens) do
{:ok, LlamaCppEx.NIF.detokenize(ref, tokens)}
rescue
e in ErlangError -> {:error, "detokenize failed: #{inspect(e.original)}"}
end
@doc """
Converts a single token ID to its text representation.
"""
@spec token_to_piece(LlamaCppEx.Model.t(), integer()) :: String.t()
def token_to_piece(%LlamaCppEx.Model{ref: ref}, token) do
LlamaCppEx.NIF.token_to_piece(ref, token)
end
@doc "Returns the vocabulary size."
@spec vocab_size(LlamaCppEx.Model.t()) :: integer()
def vocab_size(%LlamaCppEx.Model{ref: ref}), do: LlamaCppEx.NIF.vocab_n_tokens(ref)
@doc "Returns the BOS (beginning of sentence) token ID."
@spec bos_token(LlamaCppEx.Model.t()) :: integer()
def bos_token(%LlamaCppEx.Model{ref: ref}), do: LlamaCppEx.NIF.vocab_bos(ref)
@doc "Returns the EOS (end of sentence) token ID."
@spec eos_token(LlamaCppEx.Model.t()) :: integer()
def eos_token(%LlamaCppEx.Model{ref: ref}), do: LlamaCppEx.NIF.vocab_eos(ref)
@doc "Returns whether a token is an end-of-generation token."
@spec eog?(LlamaCppEx.Model.t(), integer()) :: boolean()
def eog?(%LlamaCppEx.Model{ref: ref}, token), do: LlamaCppEx.NIF.vocab_is_eog(ref, token)
end