Packages
fnord
0.6.5
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/ai/agent/fact_checker.ex
defmodule AI.Agent.FactChecker do
@model AI.Model.fast()
@prompt """
#{AI.Util.note_format_prompt()}
You are the Fact Checker Agent, responsible for confirming, refuting, and correcting facts learned from prior research.
Your role is to:
- Examine the notes provided to determine their accuracy.
- Provide a response that confirms, refutes, or corrects the facts.
- Use the file's content to support your response.
- Ensure that the response is accurate and relevant to the query.
Note that facts may have been learned by examining multiple files.
Use your tool calls to perform a complete investigation and determine the accuracy of the information.
If a note mentions a file that does not exist, you should assume that the fact is invalid.
**IT IS ESSENTIAL THAT NO FACTUAL INFORMATION IS LOST.**
# Response
Your response template varies based on your findings:
- CORRECT: respond with `CONFIRMED:<original note>`
- CORRECTABLE WITH THE INFORMATION YOU FOUND: respond with `CONFIRMED:<corrected note>`
- INCORRECT: respond with `REFUTED:<original note>`
"""
@invalid_format_prompt """
The fact-checked note was not in the expected format.
Please correct the format. Respond ONLY with the expected note format.
The correct format is ALWAYS one of:
- `CONFIRMED:<original note>`
- `CONFIRMED:<corrected note>`
- `REFUTED:<original note>`
"""
@non_git_tools [
AI.Tools.tool_spec!("file_contents_tool"),
AI.Tools.tool_spec!("file_info_tool"),
AI.Tools.tool_spec!("file_list_tool"),
AI.Tools.tool_spec!("file_search_tool"),
AI.Tools.tool_spec!("file_spelunker_tool")
]
@git_tools [
AI.Tools.tool_spec!("git_diff_branch_tool"),
AI.Tools.tool_spec!("git_list_branches_tool"),
AI.Tools.tool_spec!("git_grep_tool"),
AI.Tools.tool_spec!("git_log_tool"),
AI.Tools.tool_spec!("git_pickaxe_tool"),
AI.Tools.tool_spec!("git_show_tool")
]
@tools @non_git_tools ++ @git_tools
# -----------------------------------------------------------------------------
# Behaviour implementation
# -----------------------------------------------------------------------------
@behaviour AI.Agent
@impl AI.Agent
def get_response(ai, opts) do
with {:ok, note} <- Map.fetch(opts, :note),
{confirmed, refuted} <- fact_check_note(note, ai) do
{:ok, {confirmed, refuted}}
else
:error ->
{:error, "missing required argument: note"}
:invalid_format ->
{:error, "the LLM responded with the incorrect format after 3 attempts"}
end
end
# -----------------------------------------------------------------------------
# Private functions
# -----------------------------------------------------------------------------
defp fact_check_note(note, ai) do
1..3
|> Enum.reduce_while(nil, fn _attempt, acc ->
note
|> get_completion(ai, acc)
|> process_response()
|> validate()
|> case do
{:ok, {confirmed, refuted}} -> {:halt, {confirmed, refuted}}
{:error, :invalid_format} -> {:cont, :invalid_format}
end
end)
end
defp get_completion(note, ai, prior_failure) do
messages = [
AI.Util.system_msg(@prompt),
AI.Util.user_msg(note)
]
messages =
case prior_failure do
nil -> messages
:invalid_format -> messages ++ [AI.Util.system_msg(@invalid_format_prompt)]
end
AI.Completion.get(ai,
log_msgs: false,
log_tool_calls: false,
model: @model,
tools: available_tools(),
messages: messages
)
end
defp process_response({:ok, %{response: response}}) do
response
|> String.split("\n")
|> Enum.reduce_while({[], []}, fn line, {confirmed, refuted} ->
line
|> parse_line
|> case do
{:confirmed, note} -> {:cont, {[note | confirmed], refuted}}
{:refuted, note} -> {:cont, {confirmed, [note | refuted]}}
{:error, :invalid_format} -> {:halt, {:error, :invalid_format}}
end
end)
|> case do
{:error, :invalid_format} -> {:error, :invalid_format}
{confirmed, refuted} -> {:ok, {join(confirmed), join(refuted)}}
end
end
defp parse_line(line) do
line
|> String.trim()
|> String.trim_leading("-")
|> String.trim()
|> String.split(":", parts: 2)
|> case do
["CONFIRMED", note] -> {:confirmed, note}
["REFUTED", note] -> {:refuted, note}
_ -> {:error, :invalid_format}
end
end
defp validate({:error, error}), do: {:error, error}
defp validate({:ok, {confirmed, refuted}}) do
case AI.Util.validate_notes_string(confirmed) do
{:ok, _} -> {:ok, {confirmed, refuted}}
{:error, :invalid_format} -> {:error, :invalid_format}
end
end
defp join({a, b}), do: {join(a), join(b)}
defp join(notes), do: notes |> Enum.join("\n")
defp available_tools() do
if Git.is_git_repo?() do
@tools
else
@non_git_tools
end
end
end