Packages
fnord
0.9.3
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/ai/accumulator.ex
defmodule AI.Accumulator do
@backoff_threshold 0.6
@backoff_step 0.2
@moduledoc """
When file or other input to too large for the model's context window, this
module may be used to process the file in chunks. It automatically modifies
the supplied agent prompt to include instructions for accumulating a response
across multiple chunks based on the `context` (max context window tokens)
parameter supplied by the `model` parameter.
Note that while this makes use of the `AI.Completion` module, it does NOT
have the same interface and cannot be used for long-running conversations, as
it does not accept a list of messages as its input.
"""
defstruct [
:splitter,
:buffer,
:model,
:toolbox,
:prompt,
:question,
:completion_args,
:line_numbers,
:compact?
]
@type t :: %__MODULE__{
splitter: AI.Splitter.t(),
buffer: binary,
model: AI.Model.t(),
toolbox: AI.Tools.toolbox() | nil,
prompt: binary,
question: binary,
completion_args: Keyword.t(),
compact?: boolean
}
@line_numbers_prompt """
The input you are processing includes line numbers.
Each line is prefixed by a line number, separated by a pipe character (|).
"""
@accumulator_prompt """
You are processing input chunks in sequence.
Each chunk is paired with an "accumulator" string to update, in the following format:
```
# Question / Goal
[user question or goal]
# Accumulated Response
[your accumulated response]
-----
[current input chunk]
```
Guidelines for updating the accumulator:
1. Add Relevant Content: Update the accumulator with your response to the current chunk
2. Continuity: Build on the existing response, preserving its structure
3. Handle Incompletes: If a chunk is incomplete, mark it (e.g., `<partial>`) to complete later
4. Consistent Format: Append new content under "Accumulated Response"
Respond ONLY with the `Accumulated Response` section, including your updates from the current chunk, using the guidelines below.
-----
"""
@final_prompt """
You have processed the user's input in chunks and collected your accumulated notes.
Please review your notes and ensure that they are coherent and consistent.
Your input will be in the format:
```
# Question / Goal
[user question or goal, if any]
# Accumulated Response
[your accumulated response]
```
Respond ONLY with your cleaned up `Accumulated Response` section, without the header, `# Accumulated Response`.
-----
"""
@type success :: {:ok, AI.Completion.t()}
@type error :: {:error, binary}
@type response :: success | error
@spec get_response(Keyword.t()) :: response
def get_response(opts \\ []) do
with {:ok, model} <- Keyword.fetch(opts, :model),
{:ok, prompt} <- Keyword.fetch(opts, :prompt),
{:ok, input} <- Keyword.fetch(opts, :input) do
question = Keyword.get(opts, :question, nil)
line_numbers = Keyword.get(opts, :line_numbers, false)
compact? = Keyword.get(opts, :compact?, true)
input =
if line_numbers do
Util.numbered_lines(input)
else
input
end
toolbox =
opts
|> Keyword.get(:toolbox, nil)
|> AI.Tools.build_toolbox()
%__MODULE__{
splitter: AI.Splitter.new(input, model),
buffer: "",
model: model,
toolbox: toolbox,
prompt: prompt,
question: question,
completion_args: Keyword.get(opts, :completion_args, []),
line_numbers: line_numbers,
compact?: compact?
}
|> reduce()
else
:error -> {:error, :missing_required_option}
end
end
defp reduce(%{splitter: %{done: true}} = acc) do
system_prompt = """
#{@final_prompt}
#{if acc.line_numbers, do: @line_numbers_prompt, else: ""}
#{acc.prompt}
"""
user_prompt =
if is_nil(acc.question) or acc.question == "" do
"""
# Accumulated Response
#{acc.buffer}
"""
else
"""
# Question / Goal
#{acc.question}
# Accumulated Response
#{acc.buffer}
"""
end
args =
acc.completion_args
|> Keyword.put(:model, acc.model)
|> Keyword.put(:toolbox, acc.toolbox)
|> Keyword.put(:messages, [
AI.Util.system_msg(system_prompt),
AI.Util.user_msg(user_prompt)
])
AI.Completion.get(args)
end
defp reduce(%{splitter: %{done: false}} = acc) do
with {:ok, acc} <- process_chunk(acc) do
reduce(acc)
end
end
def process_chunk(acc), do: process_chunk(acc, 1.0)
# ----------------------------------------------------------------------------
# Processes a chunk of input by generating an updated accumulator response.
#
# The `frac` parameter represents the fraction of the model context to use
# for this chunk. On context length exceeded errors, it backs off by subtracting
# @backoff_step from frac, and retries. If frac drops below @backoff_threshold,
# returns an error indicating unable to back off further.
# ----------------------------------------------------------------------------
defp process_chunk(_acc, frac) when frac < @backoff_threshold do
{:error, "context window length exceeded: unable to back off further to fit the input"}
end
defp process_chunk(acc, frac) do
max_chunk_tokens = round(acc.model.context * frac)
# Build the "user message" prompt, which contains the accumulated response.
user_prompt = """
# Question / Goal
#{acc.question}
# Accumulated Response
#{acc.buffer}
"""
# Get the next chunk from the splitter and update the splitter state.
# The next chunk size is limited by the remaining context tokens.
{chunk, splitter} = AI.Splitter.next_chunk(acc.splitter, user_prompt, max_chunk_tokens)
acc = %{acc | splitter: splitter}
user_prompt = """
#{user_prompt}
-----
#{chunk}
"""
# Build the system prompt with the accumulator instructions and any relevant line number info.
system_prompt = """
#{@accumulator_prompt}
#{if acc.line_numbers, do: @line_numbers_prompt, else: ""}
#{acc.prompt}
"""
args =
acc.completion_args
# Auto-compaction could bork the accumulator. The accumulator is also
# used for compaction, so we want to avoid recursion.
|> Keyword.put(:compact?, acc.compact?)
|> Keyword.put(:model, acc.model)
|> Keyword.put(:toolbox, acc.toolbox)
|> Keyword.put(:messages, [
AI.Util.system_msg(system_prompt),
AI.Util.user_msg(user_prompt)
])
# Call AI.Completion.get and handle responses centrally
args
|> AI.Completion.get()
|> case do
{:ok, %{response: response}} ->
{:ok, %{acc | splitter: splitter, buffer: response}}
{:error, :context_length_exceeded, usage} ->
# Context length exceeded error handling:
# Back off and retry with smaller fraction only if frac >= @backoff_threshold
# If frac < @backoff_threshold, do not retry further and return error.
if frac >= @backoff_threshold do
UI.warn(
"Context length exceeded (#{usage} tokens used), backing off to fraction #{frac - @backoff_step}"
)
process_chunk(acc, frac - @backoff_step)
else
UI.error(
"Context length exceeded (#{usage} tokens used), unable to back off further. THIS. IS. SPARTA!"
)
{:error, "context window length exceeded: unable to back off further to fit the input"}
end
{:error, %AI.Completion{response: resp}} when is_binary(resp) ->
{:error, resp}
{:error, reason} ->
{:error, reason}
end
end
end