Packages
fnord
0.4.23
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/cmd/indexer.ex
defmodule Cmd.Indexer do
@moduledoc """
This module provides the functionality for the `index` sub-command.
"""
defstruct [
:project,
:root,
:store,
:concurrency,
:reindex,
:ai_module,
:ai,
:quiet,
:tui
]
@doc """
Create a new `Indexer` struct.
"""
def new(opts, ai_module \\ AI) do
concurrency = Map.get(opts, :concurrency, 1)
reindex = Map.get(opts, :reindex, false)
quiet = Map.get(opts, :quiet, false)
root = Path.absname(opts.directory)
{:ok, tui} = Tui.start_link(opts)
idx = %__MODULE__{
project: opts.project,
root: root,
store: Store.new(opts.project),
concurrency: concurrency,
reindex: reindex,
ai_module: ai_module,
ai: ai_module.new(),
quiet: quiet,
tui: tui
}
idx
end
@doc """
Run the indexing process using the given `Indexer` struct. If `force_reindex`
is `true`, the project will be deleted and reindexed from scratch.
"""
def run(idx) do
if idx.reindex do
# Delete entire project directory when --force-reindex is passed
status_id = Tui.add_step("Deleting all data to force full reindexing", idx.project)
Store.delete_project(idx.store)
Tui.finish_step(status_id, :ok)
else
# Otherwise, just delete any files that no longer exist.
status_id = Tui.add_step("Deleting missing files", idx.project)
Store.delete_missing_files(idx.store, idx.root)
Tui.finish_step(status_id, :ok)
end
scan_status_id = Tui.add_step("Scanning files", idx.root)
{:ok, queue} =
Queue.start_link(idx.concurrency, fn file ->
process_file(idx, file)
end)
scanner =
Scanner.new(idx.root, fn file ->
Queue.queue(queue, file)
end)
total_files = Scanner.count_files(scanner)
num_files = count_files_to_index(idx)
Tui.finish_step(scan_status_id, :ok)
index_status_id =
Tui.add_step("Indexing", "#{num_files} / #{total_files} files in #{idx.root}")
bs_status_id = Tui.add_step()
Scanner.scan(scanner)
Queue.shutdown(queue)
Queue.join(queue)
Tui.finish_step(index_status_id, :ok)
Tui.finish_step(bs_status_id, :ok)
Tui.stop(idx.tui)
end
defp process_file(idx, file) do
with {:ok, file_hash} <- get_file_hash(idx, file) do
file_contents = File.read!(file)
with {:ok, summary} <- get_summary(idx, file, file_contents),
{:ok, outline} <- get_outline(idx, file, file_contents),
{:ok, embeddings} <- get_embeddings(idx, file, summary, outline, file_contents) do
Store.put(idx.store, file, file_hash, summary, outline, embeddings)
else
{:error, reason} -> IO.puts("Error processing file: #{file} - #{inspect(reason)}")
end
end
end
defp count_files_to_index(idx) do
idx.root
|> Scanner.new(fn _ -> nil end)
|> Scanner.reduce(0, nil, fn file, acc ->
case get_file_hash(idx, file) do
{:ok, _} -> acc + 1
{:error, :unchanged} -> acc
end
end)
end
defp get_file_hash(idx, file) do
# Files that are missing or have missing indexes must be reindexed.
cond do
!Store.has_summary?(idx.store, file) ->
{:ok, nil}
!Store.has_outline?(idx.store, file) ->
{:ok, nil}
!Store.has_embeddings?(idx.store, file) ->
{:ok, nil}
true ->
existing_hash = Store.get_hash(idx.store, file)
file_hash = sha256(file)
if is_nil(existing_hash) or existing_hash != file_hash do
{:ok, file_hash}
else
{:error, :unchanged}
end
end
end
defp get_outline(idx, file, file_contents) do
idx.ai_module.get_outline(idx.ai, file, file_contents)
end
defp get_summary(idx, file, file_contents) do
idx.ai_module.get_summary(idx.ai, file, file_contents)
end
defp get_embeddings(idx, file, summary, outline, file_contents, attempt \\ 0) do
to_embed = """
# File
`#{file}`
## Summary
#{summary}
## Outline
#{outline}
## Contents
```
#{file_contents}
```
"""
idx.ai_module.get_embeddings(idx.ai, to_embed)
|> case do
{:ok, embeddings} ->
{:ok, embeddings}
{:error, %OpenaiEx.Error{message: "Request timed out."}} ->
if attempt < 3 do
IO.puts("request to index file timed out, retrying (attempt #{attempt + 1}/3)")
get_embeddings(idx, file, summary, outline, file_contents, attempt + 1)
else
{:error, "request to index file timed out after 3 attempts"}
end
{:error, reason} ->
{:error, reason}
end
end
defp sha256(file_path) do
case File.read(file_path) do
{:ok, content} -> :crypto.hash(:sha256, content) |> Base.encode16(case: :lower)
{:error, reason} -> {:error, reason}
end
end
end