Packages
fnord
0.4.12
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/indexer.ex
defmodule Indexer do
@moduledoc """
This module provides the functionality for the `index` and `delete`
sub-commands.
"""
defstruct [
:project,
:root,
:store,
:concurrency,
:reindex,
:ai_module,
:ai,
:quiet
]
@doc """
Create a new `Indexer` struct.
"""
def new(opts, ai_module \\ AI) do
concurrency = Map.get(opts, :concurrency, 1)
reindex = Map.get(opts, :reindex, false)
quiet = Map.get(opts, :quiet, false)
idx = %Indexer{
project: opts.project,
root: opts.directory,
store: Store.new(opts.project),
concurrency: concurrency,
reindex: reindex,
ai_module: ai_module,
quiet: quiet
}
idx
end
defp spin(idx, processing, func) do
if idx.quiet do
func.()
else
Spinner.run(func, processing)
end
end
defp progress_bar_start(idx, name, label, total) do
if !idx.quiet do
Owl.ProgressBar.start(id: name, label: label, total: total)
end
end
defp progress_bar_end(idx) do
if !idx.quiet do
Owl.LiveScreen.await_render()
end
end
defp progress_bar_update(idx, name) do
if !idx.quiet do
Owl.ProgressBar.inc(id: name)
end
end
defp info(idx, msg) do
if !idx.quiet do
IO.puts(msg)
end
end
@doc """
Run the indexing process using the given `Indexer` struct. If `force_reindex`
is `true`, the project will be deleted and reindexed from scratch.
"""
def run(idx) do
if idx.reindex do
# When --force-reindex is passed, delete the project completely and start
# from scratch.
spin(
idx,
"Deleting all embeddings to force full reindexing of #{idx.project}",
fn -> Store.delete_project(idx.store) end
)
else
# Otherwise, just delete any files that no longer exist.
spin(
idx,
"Deleting missing files from #{idx.project}",
fn -> Store.delete_missing_files(idx.store, idx.root) end
)
end
spin(
idx,
"Indexing files in #{idx.root}",
fn ->
{:ok, queue} =
Queue.start_link(idx.concurrency, fn file ->
process_file(idx, file)
progress_bar_update(idx, :indexing)
end)
scanner = Scanner.new(idx.root, fn file -> Queue.queue(queue, file) end)
num_files = Scanner.count_files(scanner)
progress_bar_start(idx, :indexing, "Indexing", num_files + 1)
Scanner.scan(scanner)
Queue.shutdown(queue)
Queue.join(queue)
progress_bar_end(idx)
info(idx, "All tasks complete")
end
)
end
@doc """
Permanently deletes the project from the store.
"""
def delete_project(idx) do
Store.delete_project(idx.store)
end
defp process_file(idx, file) do
existing_hash = Store.get_hash(idx.store, file)
file_hash = sha256(file)
if is_nil(existing_hash) or existing_hash != file_hash do
file_contents = File.read!(file)
with {:ok, summary} <- get_summary(idx, file, file_contents),
{:ok, embeddings} <- get_embeddings(idx, file, summary, file_contents) do
Store.put(idx.store, file, file_hash, summary, embeddings)
else
{:error, reason} -> IO.puts("Error processing file: #{file} - #{inspect(reason)}")
end
end
end
defp get_summary(idx, file, file_contents) do
idx.ai_module.get_summary(file, file_contents)
|> case do
{:ok, summary} -> {:ok, summary}
{:error, reason} -> {:error, reason}
end
end
defp get_embeddings(idx, file, summary, file_contents) do
to_embed = """
# File
`#{file}`
## Summary
#{summary}
## Contents
```
#{file_contents}
```
"""
idx.ai_module.get_embeddings(to_embed)
|> case do
{:ok, embeddings} -> {:ok, embeddings}
{:error, reason} -> {:error, reason}
end
end
defp sha256(file_path) do
case File.read(file_path) do
{:ok, content} -> :crypto.hash(:sha256, content) |> Base.encode16(case: :lower)
{:error, reason} -> {:error, reason}
end
end
end