Packages
fnord
0.4.34
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/ai/tokenizer.ex
defmodule AI.Tokenizer do
@moduledoc """
> Oh yeah? I'm gonna make my own tokenizer, with blackjack and hookers!
-- ~Bender~ ChatGPT
The only tokenizer modules available when this was written are either older
and don't correctly count for OpenAI's newer models (Gpt3Tokenizer) or can't
be used in an escript because they require priv access or OTP support beyond
escript's abilities (Tokenizers).
This module tokenizes text using the `o200k_base` vocabulary and merges files
from OpenAI's tiktoken repo.
"""
import AI.Tokens_o200k_base
# -----------------------------------------------------------------------------
# Behaviour definition
# -----------------------------------------------------------------------------
@callback decode(list()) :: String.t()
@callback encode(String.t()) :: list()
def get_impl() do
Application.get_env(:fnord, :tokenizer_module) || __MODULE__
end
# -----------------------------------------------------------------------------
# Behaviour implementation
# -----------------------------------------------------------------------------
@behaviour AI.Tokenizer
@impl AI.Tokenizer
def decode(token_ids) do
Enum.map(token_ids, fn id ->
# Return `nil` for unknown IDs
Map.get(get_reverse_vocab(), id, nil)
end)
|> Enum.join("")
end
@impl AI.Tokenizer
def encode(text) do
# Step 1: Split text into initial tokens using the regex pattern
tokens = Regex.scan(get_pattern(), text) |> List.flatten()
# Step 2: Apply BPE merging
bpe_tokens = Enum.map(tokens, &apply_bpe(&1))
# Step 3: Map tokens to vocabulary IDs
Enum.map(bpe_tokens, fn token ->
Map.get(get_vocab(), token, get_special_tokens()[token] || nil)
end)
end
# -----------------------------------------------------------------------------
# Public functions
# -----------------------------------------------------------------------------
def chunk(input, max_tokens) do
tokenizer = AI.Tokenizer.get_impl()
input
|> tokenizer.encode()
|> Enum.chunk_every(max_tokens)
|> Enum.map(&tokenizer.decode(&1))
end
# -----------------------------------------------------------------------------
# Private functions
# -----------------------------------------------------------------------------
defp apply_bpe(token) do
# Split token into characters for BPE merging
chars = String.graphemes(token)
# Merge adjacent pairs based on BPE rules
loop_merge(chars)
end
defp loop_merge(tokens) do
tokens
# Create pairs of adjacent tokens
|> Enum.zip(Enum.drop(tokens, 1))
# Find the first pair that exists in the merge rules
|> Enum.find(&MapSet.member?(get_merges(), &1))
|> case do
# No more pairs to merge, return the tokens as a single string
nil ->
Enum.join(tokens, "")
# Merge the pair and continue
pair ->
tokens
|> Enum.reduce([], fn token, acc ->
case acc do
# Merge the pair into one token
[last | rest] when {last, token} == pair ->
[Enum.join(Tuple.to_list(pair), "") | rest]
# Add the token to the accumulator as-is
_ ->
[token | acc]
end
end)
|> Enum.reverse()
# Recursive call to merge further
|> loop_merge()
end
end
end