Packages
fnord
0.7.12
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/ai/tokenizer/default.ex
defmodule AI.Tokenizer.Default do
@moduledoc """
Implements the default tokenizer behaviour for AI.Tokenizer.
"""
@tokenizer %{
"o3-mini" => AI.Tokenizer.Tokens_o200k_base,
"gpt-4o" => AI.Tokenizer.Tokens_o200k_base,
"gpt-4o-mini" => AI.Tokenizer.Tokens_o200k_base,
"text-embedding-3-large" => AI.Tokenizer.Tokens_cl100k_base,
"default" => AI.Tokenizer.Tokens_cl100k_base
}
# A special fallback token ID for unknown tokens
@fallback_token 200_020
# -----------------------------------------------------------------------------
# Behaviour implementation
# -----------------------------------------------------------------------------
@behaviour AI.Tokenizer
@impl AI.Tokenizer
def decode(token_ids, model) do
tokenizer = get_tokenizer!(model)
reverse_vocab = tokenizer.get_reverse_vocab()
token_ids
|> Enum.map(fn
{@fallback_token, raw_token} ->
# Return the original token text for fallback entries
raw_token
token_id when is_integer(token_id) ->
# Lookup in reverse_vocab, or empty string if not found
Map.get(reverse_vocab, token_id, "")
# In case something unexpected slips through
other ->
IO.warn("Unexpected token in decode: #{inspect(other)}")
""
end)
|> Enum.join("")
end
@impl AI.Tokenizer
def encode(text, model) do
tokenizer = get_tokenizer!(model)
pattern = tokenizer.get_pattern()
vocab = tokenizer.get_vocab()
special_tokens = tokenizer.get_special_tokens()
try do
# Step 1: Split text using the tokenizer-specific pattern
pattern
|> Regex.scan(text)
|> List.flatten()
# Step 2: Apply BPE merging for each token
|> Enum.map(&apply_bpe(&1, model))
# Step 3: Convert to token IDs, falling back to special fallback tokens for unknowns
|> Enum.map(fn token ->
case Map.get(vocab, token) do
nil ->
# Check if it's a special token
case special_tokens[token] do
nil ->
# **FALLBACK**: if not in vocab or special_tokens, return tuple
{@fallback_token, token}
special_id ->
# Known special token
special_id
end
id ->
# Known vocab token
id
end
end)
rescue
e in ArgumentError ->
IO.inspect(text, label: "Input text (raw)", limit: :infinity)
{:error, e}
end
end
# -----------------------------------------------------------------------------
# Private functions
# -----------------------------------------------------------------------------
defp get_tokenizer!(model) do
@tokenizer[model] || @tokenizer["default"]
end
defp apply_bpe(token, model) do
token
|> String.graphemes()
|> loop_merge(model)
end
# Merge adjacent pairs based on BPE rules
defp loop_merge(tokens, model) do
tokenizer = get_tokenizer!(model)
tokens
# Create pairs of adjacent tokens
|> Enum.zip(Enum.drop(tokens, 1))
# Find the first pair that exists in the merge rules
|> Enum.find(&MapSet.member?(tokenizer.get_merges(), &1))
|> case do
# No more pairs to merge, return the tokens as a single string
nil ->
Enum.join(tokens, "")
# Merge the pair and continue
pair ->
tokens
|> Enum.reduce([], fn token, acc ->
case acc do
# Merge the pair into one token
[last | rest] when {last, token} == pair ->
[Enum.join(Tuple.to_list(pair), "") | rest]
# Add the token to the accumulator as-is
_ ->
[token | acc]
end
end)
|> Enum.reverse()
# Recursive call to handle further merges
|> loop_merge(model)
end
end
end