Packages
fnord
0.8.21
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/ai/splitter.ex
defmodule AI.Splitter do
@moduledoc """
This module is used to split a string into chunks by the number of tokens,
while accounting for *other* data that might be going with it to the API
endpoint with the limited token count.
For example, the search entry agent may be processing a large file, one that
must be split into 3 slices just to fit it into the payload of an API call.
In order to retain context between chunks, the agent essentially _reduces_
over the file, keeping track of information in the previous chunks to
generate a final summary. Doing that means that we need to not only split the
file by the number of tokens in each slice, but also keep some space for the
bespoke data that will be added to the payload as the agent's "accumulator".
"""
defstruct [
:model,
:input,
:done
]
@type t :: %__MODULE__{
model: AI.Model.t(),
input: binary,
done: boolean
}
def new(input, model) do
%AI.Splitter{
model: model,
input: input,
done: false
}
end
def next_chunk(%AI.Splitter{done: true} = tok, _bespoke_input) do
{:done, tok}
end
def next_chunk(tok, bespoke_input) do
next_chunk(tok, bespoke_input, nil)
end
@doc """
Returns the next chunk and updated splitter state, accounting for the bespoke input tokens.
Optionally, a `max_chunk_tokens` can be provided to limit the chunk size explicitly.
"""
def next_chunk(tok, bespoke_input, max_chunk_tokens) do
bespoke_tokens = AI.PretendTokenizer.guesstimate_tokens(bespoke_input)
max_tokens = max_chunk_tokens || max_tokens(tok.model)
remaining_tokens = max_tokens - bespoke_tokens
if remaining_tokens <= 0 do
{"", %{tok | done: true, input: ""}}
else
remaining_chars = remaining_tokens * 4
{slice, remaining} = String.split_at(tok.input, remaining_chars)
is_done? = remaining == ""
{slice, %{tok | done: is_done?, input: remaining}}
end
end
defp max_tokens(model) do
# Leave some space since we just guestimate token counts
round(model.context * 0.9)
end
end