Packages
fnord
0.4.27
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/cmd/indexer.ex
defmodule Cmd.Indexer do
@moduledoc """
This module provides the functionality for the `index` sub-command.
"""
require Logger
defstruct [
:opts,
:project,
:root,
:store,
:concurrency,
:reindex,
:ai_module,
:ai,
:quiet
]
@doc """
Create a new `Indexer` struct.
"""
def new(opts, ai_module \\ AI) do
concurrency = Map.get(opts, :concurrency, 1)
reindex = Map.get(opts, :reindex, false)
quiet = Map.get(opts, :quiet, false)
project = opts.project
root =
case Map.get(opts, :directory, nil) do
root when is_binary(root) ->
Path.absname(root)
nil ->
settings = Settings.new()
case get_root(settings, project) do
{:ok, root} ->
root
{:error, :not_found} ->
raise """
Error: the project root was not found in the settings file.
This can happen under the following circumstances:
- the first index of a project
- the first index reindexing after moving the project directory
- the first index after the upgrade that made --dir optional
Re-run with --dir to specify the project root directory and update the settings file.
"""
end
end
Settings.new()
|> Settings.set_project(project, %{"root" => root})
idx =
%__MODULE__{
opts: opts,
project: opts.project,
root: root,
store: Store.new(project),
concurrency: concurrency,
reindex: reindex,
ai_module: ai_module,
ai: ai_module.new(),
quiet: quiet
}
idx
end
# Retrieves the project root from settings. If the project is not yet found
# in settings or it _is_ but the value of `"root"` is `nil`, it will return
# an error.
defp get_root(settings, project) do
with {:ok, info} <- Settings.get_project(settings, project),
{:ok, root} <- Map.fetch(info, "root") do
case root do
nil -> {:error, :not_found}
_ -> {:ok, Path.absname(root)}
end
end
end
@doc """
Run the indexing process using the given `Indexer` struct. If `force_reindex`
is `true`, the project will be deleted and reindexed from scratch.
"""
def run(idx) do
if idx.reindex do
# Delete entire project directory when --force-reindex is passed
msg = "Deleting all embeddings to force full reindexing of #{idx.project}"
spin(idx, msg, fn ->
{"Burned it all to the ground", Store.delete_project(idx.store)}
end)
else
# Otherwise, just delete any files that no longer exist.
msg = "Deleting missing files from #{idx.project}"
spin(idx, msg, fn ->
{"Deleted missing files", Store.delete_missing_files(idx.store, idx.root)}
end)
end
{:ok, queue} =
Queue.start_link(idx.concurrency, fn file ->
process_file(idx, file)
end)
scanner =
Scanner.new(idx.root, fn file ->
Queue.queue(queue, file)
end)
count =
spin(idx, "Scanning project files", fn ->
num_files = count_files_to_index(idx)
total_files = Scanner.count_files(scanner)
msg = "Indexing #{num_files} of #{total_files} total files"
{msg, num_files}
end)
spin(idx, "Indexing #{idx.project}", fn ->
# count * 3 for each step in indexing a file (summary, outline, embeddings)
progress_bar_start(idx, :indexing, "tasks", count * 3)
Scanner.scan(scanner)
Queue.shutdown(queue)
Queue.join(queue)
progress_bar_end(idx)
{"All tasks complete", :ok}
end)
end
# -----------------------------------------------------------------------------
# Indexing process
# -----------------------------------------------------------------------------
defp process_file(idx, file) do
with {:ok, file_hash} <- get_file_hash(idx, file) do
file_contents = File.read!(file)
with {:ok, summary} <- get_summary(idx, file, file_contents),
{:ok, outline} <- get_outline(idx, file, file_contents),
{:ok, embeddings} <- get_embeddings(idx, file, summary, outline, file_contents) do
Store.put(idx.store, file, file_hash, summary, outline, embeddings)
else
{:error, reason} -> UI.warn("Error processing file #{file}", reason)
end
end
end
defp count_files_to_index(idx) do
idx.root
|> Scanner.new(fn _ -> nil end)
|> Scanner.reduce(0, nil, fn file, acc ->
case get_file_hash(idx, file) do
{:ok, _} -> acc + 1
{:error, :unchanged} -> acc
end
end)
end
defp get_file_hash(idx, file) do
# Files that are missing or have missing indexes must be reindexed.
cond do
!Store.has_summary?(idx.store, file) ->
{:ok, nil}
!Store.has_outline?(idx.store, file) ->
{:ok, nil}
!Store.has_embeddings?(idx.store, file) ->
{:ok, nil}
true ->
existing_hash = Store.get_hash(idx.store, file)
file_hash = sha256(file)
if is_nil(existing_hash) or existing_hash != file_hash do
{:ok, file_hash}
else
{:error, :unchanged}
end
end
end
defp get_outline(idx, file, file_contents) do
res = idx.ai_module.get_outline(idx.ai, file, file_contents)
progress_bar_update(idx, :indexing)
res
end
defp get_summary(idx, file, file_contents) do
res = idx.ai_module.get_summary(idx.ai, file, file_contents)
progress_bar_update(idx, :indexing)
res
end
defp get_embeddings(idx, file, summary, outline, file_contents, attempt \\ 0) do
to_embed = """
# File
`#{file}`
## Summary
#{summary}
## Outline
#{outline}
## Contents
```
#{file_contents}
```
"""
idx.ai_module.get_embeddings(idx.ai, to_embed)
|> case do
{:ok, embeddings} ->
progress_bar_update(idx, :indexing)
{:ok, embeddings}
{:error, %OpenaiEx.Error{message: "Request timed out."}} ->
if attempt < 3 do
UI.warn("request to index file timed out, retrying", "attempt #{attempt + 1}/3")
get_embeddings(idx, file, summary, outline, file_contents, attempt + 1)
else
{:error, "request to index file timed out after 3 attempts"}
end
{:error, reason} ->
{:error, reason}
end
end
defp sha256(file_path) do
case File.read(file_path) do
{:ok, content} -> :crypto.hash(:sha256, content) |> Base.encode16(case: :lower)
{:error, reason} -> {:error, reason}
end
end
# -----------------------------------------------------------------------------
# UI interaction
# -----------------------------------------------------------------------------
defp spin(idx, processing, func) do
if idx.quiet do
{_msg, result} = func.()
result
else
Spinner.run(func, processing)
end
end
defp progress_bar_start(idx, name, label, total) do
if !idx.quiet do
Owl.ProgressBar.start(
id: name,
label: label,
total: total,
timer: true,
absolute_values: true
)
end
end
defp progress_bar_end(idx) do
if !idx.quiet do
Owl.LiveScreen.await_render()
end
end
defp progress_bar_update(idx, name) do
if !idx.quiet do
Owl.ProgressBar.inc(id: name)
end
end
end