Packages
fnord
0.4.38
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/ai/tokenizer/default.ex
defmodule AI.Tokenizer.Default do
@moduledoc """
Implements the default tokenizer behaviour for AI.Tokenizer.
"""
@tokenizer %{
"gpt-4o" => AI.Tokenizer.Tokens_o200k_base,
"gpt-4o-mini" => AI.Tokenizer.Tokens_o200k_base,
"text-embedding-3-large" => AI.Tokenizer.Tokens_cl100k_base
}
# -----------------------------------------------------------------------------
# Behaviour implementation
# -----------------------------------------------------------------------------
@behaviour AI.Tokenizer
@impl AI.Tokenizer
def decode(token_ids, model) do
tokenizer = get_tokenizer!(model)
reverse_vocab = tokenizer.get_reverse_vocab()
token_ids
|> Enum.map(&Map.get(reverse_vocab, &1, ""))
|> Enum.join("")
end
@impl AI.Tokenizer
def encode(text, model) do
tokenizer = get_tokenizer!(model)
pattern = tokenizer.get_pattern()
vocab = tokenizer.get_vocab()
special_tokens = tokenizer.get_special_tokens()
# Step 1: Split text with the tokenizer-specific pattern
pattern
|> Regex.scan(text)
|> List.flatten()
# Step 2: Apply byte pair encoding merging
|> Enum.map(&apply_bpe(&1, model))
# Step 3: Map tokens to vocabulary IDs
|> Enum.map(&Map.get(vocab, &1, special_tokens[&1] || nil))
end
# -----------------------------------------------------------------------------
# Private functions
# -----------------------------------------------------------------------------
defp get_tokenizer!(model) do
tokenizer = @tokenizer[model]
if is_nil(tokenizer) do
raise ArgumentError, "Unknown model: #{model}"
end
tokenizer
end
defp apply_bpe(token, model) do
token
|> String.graphemes()
|> loop_merge(model)
end
# Merge adjacent pairs based on BPE rules
defp loop_merge(tokens, model) do
tokenizer = get_tokenizer!(model)
tokens
# Create pairs of adjacent tokens
|> Enum.zip(Enum.drop(tokens, 1))
# Find the first pair that exists in the merge rules
|> Enum.find(&MapSet.member?(tokenizer.get_merges(), &1))
|> case do
# No more pairs to merge, return the tokens as a single string
nil ->
Enum.join(tokens, "")
# Merge the pair and continue
pair ->
tokens
|> Enum.reduce([], fn token, acc ->
case acc do
# Merge the pair into one token
[last | rest] when {last, token} == pair ->
[Enum.join(Tuple.to_list(pair), "") | rest]
# Add the token to the accumulator as-is
_ ->
[token | acc]
end
end)
|> Enum.reverse()
# Recursive call to merge further
|> loop_merge(model)
end
end
end