Packages
fnord
0.6.7
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/cmd/defrag.ex
defmodule Cmd.Defrag do
@behaviour Cmd
# ----------------------------------------------------------------------------
# Behaviour implementation
# ----------------------------------------------------------------------------
@impl Cmd
def spec do
[
defrag: [
name: "defrag",
about: "Consolidate and fact-check saved notes",
options: [
project: [
value_name: "PROJECT",
long: "--project",
short: "-p",
help: "Project name",
required: true
],
workers: [
value_name: "WORKERS",
long: "--workers",
short: "-w",
help: "Limits the number of concurrent OpenAI requests",
parser: :integer,
default: Cmd.default_workers()
]
],
flags: [
quiet: [
long: "--quiet",
short: "-Q",
help: "Suppress interactive output; automatically enabled when executed in a pipe",
required: false,
default: false
]
]
]
]
end
@impl Cmd
def run(opts, _unknown) do
project_name = Map.get(opts, :project)
Application.put_env(:fnord, :project, project_name)
project_name
|> Store.get_project()
|> defrag()
end
# -----------------------------------------------------------------------------
# Notes clean up
# -----------------------------------------------------------------------------
defp defrag(project) do
notes = get_notes(project)
count = Enum.count(notes)
UI.info("Project", project.name)
UI.info("Workers", Application.get_env(:fnord, :workers) |> to_string())
UI.info(" Topics", to_string(count))
with {:ok, confirmed, refuted} <- fact_check_saved_notes(notes),
{:ok, consolidated} <- consolidate_saved_notes(confirmed),
:ok <- confirm_updated_notes(consolidated, refuted, count) do
save_updated_notes(project, consolidated)
else
{:error, :invalid_format} ->
UI.error("Error defragmenting notes", "Invalid note format")
:no_change ->
UI.info("Defragmenting notes", "No changes needed")
:aborted ->
UI.warn("Defragmenting notes", "Aborted at user request")
end
end
# ----------------------------------------------------------------------------
# Consolidating saved notes
# ----------------------------------------------------------------------------
@type note :: String.t()
@spec consolidate_saved_notes([note]) :: {:ok, [note]} | {:error, :invalid_format}
defp consolidate_saved_notes([]) do
UI.info("Consolidating prior research", "Nothing to do")
{:ok, []}
end
defp consolidate_saved_notes(original_notes) do
ai = AI.new()
count = Enum.count(original_notes)
UI.progress_bar_start(:consolidation, "Consolidating #{count} notes", 1)
with {:ok, note} <- AI.Agent.NotesConsolidator.get_response(ai, %{notes: original_notes}),
{:ok, result} <- AI.Util.validate_notes_string(note) do
new_count = Enum.count(result)
UI.progress_bar_update(:consolidation)
UI.report_step("Consolidated", "from #{count} to #{new_count}")
{:ok, result}
end
end
# ----------------------------------------------------------------------------
# Fact-checking
# ----------------------------------------------------------------------------
@spec fact_check_saved_notes([note]) :: {:ok, [note], [note]}
defp fact_check_saved_notes([]) do
UI.info("Fact-checking prior research", "Nothing to do")
{:ok, [], []}
end
defp fact_check_saved_notes(notes) do
notes
|> UI.async_stream(&fact_check_saved_note/1, "Fact-checking notes")
|> Enum.reduce({[], []}, fn
{:ok, {:ok, confirmed, refuted}}, {acc_confirmed, acc_refuted} ->
{
confirmed ++ acc_confirmed,
[refuted | acc_refuted]
}
_, acc ->
acc
end)
|> then(fn {confirmed, refuted} ->
{:ok, confirmed, refuted}
end)
end
@spec fact_check_saved_note(note) ::
{:ok, note, note}
| {:error, String.t()}
| {:error, :invalid_format}
defp fact_check_saved_note(note) do
ai = AI.new()
args = %{note: note}
with {:ok, {confirmed, refuted}} <- AI.Agent.FactChecker.get_response(ai, args),
{:ok, validated} <- AI.Util.validate_notes_string(confirmed) do
if refuted != "" do
UI.report_step("Refuted prior research", refuted)
end
{:ok, validated, refuted}
end
end
# -----------------------------------------------------------------------------
# Confirmation and validation
# -----------------------------------------------------------------------------
@spec confirm_updated_notes([note], [note], non_neg_integer) ::
:ok
| :aborted
| :no_change
defp confirm_updated_notes([], [], 0), do: :no_change
defp confirm_updated_notes(confirmed, refuted, original_count) do
new_count = Enum.count(confirmed)
change = abs(original_count - new_count)
difference =
case change do
0 ->
0
_ ->
(change / original_count * 100)
|> Float.round(0)
|> trunc()
end
IO.puts("""
/-----------------------------------------------------------
| Refuted Notes
\\-----------------------------------------------------------
""")
refuted
|> Enum.each(fn note ->
note = String.trim(note)
if note != "" do
IO.puts(note)
end
end)
IO.puts("""
/-----------------------------------------------------------
| Confirmed Notes
\\-----------------------------------------------------------
""")
confirmed
|> Enum.each(fn note ->
note
|> note_to_string()
|> IO.puts()
IO.puts("")
end)
IO.puts("""
/-----------------------------------------------------------
| There is a #{difference}% change in note count:
| * Original: #{original_count}
| * Updated: #{new_count}
\\-----------------------------------------------------------
""")
if UI.confirm("Replace your saved notes with the updated list?") do
:ok
else
:aborted
end
end
# -----------------------------------------------------------------------------
# Retrieval and saving
# -----------------------------------------------------------------------------
@spec get_notes(Store.Project.t()) :: [note]
defp get_notes(project) do
project
|> Store.Project.notes()
|> Enum.reduce([], fn note, acc ->
with {:ok, text} <- Store.Project.Note.read_note(note) do
[text | acc]
else
_ -> acc
end
end)
|> Enum.reverse()
end
@spec save_updated_notes(Store.Project.t(), [note]) :: :ok
defp save_updated_notes(project, notes) do
count = Enum.count(notes)
UI.report_step("Saving updated notes")
UI.report_step(" - Clearing old notes")
Store.Project.reset_notes(project)
UI.report_step(" - Saving consolidated notes")
notes
|> UI.async_stream(
fn text ->
project
|> Store.Project.Note.new()
|> Store.Project.Note.write(text)
end,
"Saving notes"
)
|> Enum.to_list()
UI.info(" - #{count} notes saved")
:ok
end
@spec note_to_string(note) :: String.t()
defp note_to_string(note) do
with {:ok, {topic, facts}} <- Store.Project.Note.parse_string(note) do
facts
|> Enum.map(&"- #{&1}")
|> Enum.join("\n")
|> then(fn str -> "## #{topic}\n#{str}" end)
else
{:error, :invalid_format, type} ->
"""
WARNING - Invalid note format - Improperly formatted #{to_string(type)}:
#{note}
"""
end
|> String.trim()
end
end