Packages
fnord
0.5.2
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/cmd/index/embeddings.ex
defmodule Cmd.Index.Embeddings do
# ----------------------------------------------------------------------------
# Options
# ----------------------------------------------------------------------------
defp reindex?(idx), do: Map.get(idx.opts, :reindex, false)
# ----------------------------------------------------------------------------
# Indexing process
# ----------------------------------------------------------------------------
def index_project(idx) do
if reindex?(idx) do
Store.Project.delete(idx.project)
UI.report_step("Burned all of the old data to the ground to force a full reindex!")
else
UI.info("Scanning project...")
Store.Project.delete_missing_files(idx.project)
UI.report_step("Deleted missing and newly excluded files")
end
all_files = Store.Project.source_files(idx.project)
stale_files = Store.Project.stale_source_files(idx.project)
total = Enum.count(all_files)
count = Enum.count(stale_files)
if count == 0 do
UI.warn("No files to index in #{idx.project.name}")
else
{:ok, queue} = Queue.start_link(&process_entry(idx, &1))
Cmd.Index.UI.spin("Indexing #{count} / #{total} files", fn ->
# files * 3 for each step in indexing a file (summary, outline, embeddings)
Cmd.Index.UI.progress_bar_start(:indexing, "Tasks", count * 3)
# queue files
Enum.each(stale_files, &Queue.queue(queue, &1))
# start a monitor that displays in-progress files
monitor = Cmd.Index.UI.start_in_progress_jobs_monitor(queue)
# wait on queue to complete
Queue.shutdown(queue)
Queue.join(queue)
# wait on monitor to terminate
Task.await(monitor)
{"All file indexing tasks complete", :ok}
end)
end
end
defp process_entry(idx, entry) do
with {:ok, contents} <- Store.Project.Entry.read_source_file(entry),
{:ok, summary, outline} <- get_derivatives(idx, entry.file, contents),
{:ok, embeddings} <- get_embeddings(idx, entry.file, summary, outline, contents),
:ok <- Store.Project.Entry.save(entry, summary, outline, embeddings) do
UI.debug("✓ #{entry.file}")
:ok
else
{:error, reason} -> UI.warn("Error processing #{entry.file}", inspect(reason))
end
end
defp get_derivatives(idx, file, file_contents) do
summary_task = Task.async(fn -> get_summary(idx, file, file_contents) end)
outline_task = Task.async(fn -> get_outline(idx, file, file_contents) end)
with {:ok, summary} <- Task.await(summary_task, :infinity),
{:ok, outline} <- Task.await(outline_task, :infinity) do
{:ok, summary, outline}
end
end
defp get_outline(idx, file, file_contents) do
res = idx.indexer_module.get_outline(idx.indexer, file, file_contents)
Cmd.Index.UI.progress_bar_update(:indexing)
res
end
defp get_summary(idx, file, file_contents) do
res = idx.indexer_module.get_summary(idx.indexer, file, file_contents)
Cmd.Index.UI.progress_bar_update(:indexing)
res
end
defp get_embeddings(idx, file, summary, outline, file_contents) do
to_embed = """
# File
`#{file}`
## Summary
#{summary}
## Outline
#{outline}
## Contents
```
#{file_contents}
```
"""
result = idx.indexer_module.get_embeddings(idx.indexer, to_embed)
Cmd.Index.UI.progress_bar_update(:indexing)
case result do
{:error, reason} -> IO.inspect(reason)
_ -> nil
end
result
end
end