Packages
fnord
0.4.42
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/cmd/indexer.ex
defmodule Cmd.Indexer do
defstruct [
:opts,
:indexer_module,
:indexer,
:project
]
def new(opts, indexer \\ Indexer) do
project =
Store.get_project()
|> Store.Project.save_settings(
Map.get(opts, :directory),
Map.get(opts, :exclude)
)
if is_nil(project.source_root) do
raise """
Error: the project root was not found in the settings file.
This can happen under the following circumstances:
- the first index of a project
- the first index reindexing after moving the project directory
- the first index after the upgrade that made --dir optional
"""
end
%__MODULE__{
opts: opts,
indexer_module: indexer,
indexer: indexer.new(),
project: project
}
end
def run(idx) do
UI.info("Project", idx.project.name)
UI.info("Root", idx.project.source_root)
UI.info(
"Exclude",
idx.project.exclude
|> Enum.join(" | ")
|> case do
"" -> "None"
x -> x
end
)
if reindex?(idx) do
Store.Project.delete(idx.project)
UI.report_step("Burned all of the old data to the ground to force a full reindex!")
else
Store.Project.delete_missing_files(idx.project)
UI.report_step("Deleted missing and newly excluded files")
end
{:ok, queue} = Queue.start_link(&process_entry(idx, &1))
all_files = Store.Project.source_files(idx.project)
stale_files = Store.Project.stale_source_files(idx.project)
total = Enum.count(all_files)
count = Enum.count(stale_files)
if count == 0 do
UI.warn("No files to index in #{idx.project.name}")
else
spin("Indexing #{count} / #{total} files", fn ->
# files * 3 for each step in indexing a file (summary, outline, embeddings)
progress_bar_start(:indexing, "Tasks", count * 3)
# queue files
Enum.each(stale_files, &Queue.queue(queue, &1))
# start a monitor that displays in-progress files
monitor = start_in_progress_jobs_monitor(queue)
# wait on queue to complete
Queue.shutdown(queue)
Queue.join(queue)
# wait on monitor to terminate
Task.await(monitor)
{"All tasks complete", :ok}
end)
end
end
# ----------------------------------------------------------------------------
# Options
# ----------------------------------------------------------------------------
defp quiet?(), do: Application.get_env(:fnord, :quiet)
defp reindex?(idx), do: Map.get(idx.opts, :reindex, false)
# ----------------------------------------------------------------------------
# Indexing process
# ----------------------------------------------------------------------------
defp process_entry(idx, entry) do
with {:ok, contents} <- Store.Entry.read_source_file(entry),
{:ok, summary, outline} <- get_derivatives(idx, entry.file, contents),
{:ok, embeddings} <- get_embeddings(idx, entry.file, summary, outline, contents),
:ok <- Store.Entry.save(entry, summary, outline, embeddings) do
:ok
else
{:error, reason} -> UI.warn("Error processing #{entry.file}", inspect(reason))
end
end
defp get_derivatives(idx, file, file_contents) do
summary_task = Task.async(fn -> get_summary(idx, file, file_contents) end)
outline_task = Task.async(fn -> get_outline(idx, file, file_contents) end)
with {:ok, summary} <- Task.await(summary_task, :infinity),
{:ok, outline} <- Task.await(outline_task, :infinity) do
{:ok, summary, outline}
end
end
defp get_outline(idx, file, file_contents) do
res = idx.indexer_module.get_outline(idx.indexer, file, file_contents)
progress_bar_update(:indexing)
res
end
defp get_summary(idx, file, file_contents) do
res = idx.indexer_module.get_summary(idx.indexer, file, file_contents)
progress_bar_update(:indexing)
res
end
defp get_embeddings(idx, file, summary, outline, file_contents) do
to_embed = """
# File
`#{file}`
## Summary
#{summary}
## Outline
#{outline}
## Contents
```
#{file_contents}
```
"""
result = idx.indexer_module.get_embeddings(idx.indexer, to_embed)
progress_bar_update(:indexing)
case result do
{:error, reason} -> IO.inspect(reason)
_ -> nil
end
result
end
# ----------------------------------------------------------------------------
# UI interaction
# ----------------------------------------------------------------------------
defp spin(processing, func) do
if quiet?() do
{_msg, result} = func.()
result
else
Spinner.run(func, processing)
end
end
defp progress_bar_start(name, label, total) do
if !quiet?() do
Owl.ProgressBar.start(
id: name,
label: label,
total: total,
timer: true,
absolute_values: true
)
end
end
defp progress_bar_update(name) do
if !quiet?() do
Owl.ProgressBar.inc(id: name)
Owl.LiveScreen.await_render()
end
end
defp start_in_progress_jobs_monitor(queue) do
if quiet?() do
Task.async(fn -> :ok end)
else
Owl.LiveScreen.add_block(:in_progress, state: "")
Task.async(fn ->
in_progress_jobs(queue)
Owl.LiveScreen.update(:in_progress, "Indexing complete")
Owl.LiveScreen.await_render()
end)
end
end
defp in_progress_jobs(queue) do
unless Queue.is_idle(queue) do
jobs =
queue
|> Queue.in_progress_jobs()
|> Enum.map(&"- #{&1.rel_path}")
|> Enum.join("\n")
box =
Owl.Box.new(jobs,
title: "[ In Progress ]",
border_style: :solid_rounded,
horizontal_aling: :left,
padding_x: 1
)
Owl.LiveScreen.update(:in_progress, box)
Process.sleep(250)
in_progress_jobs(queue)
end
end
end