Packages
fnord
0.4.0
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/index.ex
defmodule Index do
defstruct [
:project,
:root,
:ai_module,
:ai,
:store
]
def new(project, root, ai_module \\ AI) do
idx = %Index{
project: project,
root: root,
ai_module: ai_module,
ai: ai_module.new(),
store: Store.new(project)
}
idx
end
def run(idx, force_reindex \\ false) do
if force_reindex do
# When --force-reindex is passed, delete the project completely and start
# from scratch.
IO.puts("Deleting all embeddings to force full reindexing of #{idx.project}")
Store.delete_project(idx.store)
else
# Otherwise, just delete any files that no longer exist.
IO.puts("Deleting missing files from #{idx.project}")
Store.delete_missing_files(idx.store, idx.root)
end
{:ok, queue} =
Queue.start_link(4, fn file ->
IO.write(".")
process_file(idx, file)
IO.write(".")
end)
scanner = Scanner.new(idx.root, fn file -> Queue.queue(queue, file) end)
IO.puts("Indexing files in #{idx.root}")
Scanner.scan(scanner)
Queue.shutdown(queue)
Queue.join(queue)
IO.puts("done!")
end
def delete_project(project) do
store = Store.new(project)
Store.delete_project(store)
end
defp process_file(idx, file) do
existing_hash = Store.get_hash(idx.store, file)
file_hash = sha256(file)
if is_nil(existing_hash) or existing_hash != file_hash do
file_contents = File.read!(file)
with {:ok, summary} <- get_summary(idx, file, file_contents),
{:ok, embeddings} <- get_embeddings(idx, file, summary, file_contents) do
Store.put(idx.store, file, file_hash, summary, embeddings)
else
{:error, reason} -> IO.puts("Error processing file: #{file} - #{reason}")
end
end
end
defp get_summary(idx, file, file_contents, attempt \\ 0) do
idx.ai_module.get_summary(idx.ai, file, file_contents)
|> case do
{:ok, summary} ->
{:ok, summary}
{:error, %OpenaiEx.Error{message: "Request timed out."}} ->
if attempt < 3 do
IO.puts("request to summarize file timed out, retrying (attempt #{attempt + 1}/3)")
get_summary(idx, file, file_contents, attempt + 1)
else
{:error, "request to summarize file timed out after 3 attempts"}
end
{:error, reason} ->
{:error, reason}
end
end
defp get_embeddings(idx, file, summary, file_contents, attempt \\ 0) do
to_embed = """
# File
`#{file}`
## Summary
#{summary}
## Contents
```
#{file_contents}
```
"""
idx.ai_module.get_embeddings(idx.ai, to_embed)
|> case do
{:ok, embeddings} ->
{:ok, embeddings}
{:error, %OpenaiEx.Error{message: "Request timed out."}} ->
if attempt < 3 do
IO.puts("request to index file timed out, retrying (attempt #{attempt + 1}/3)")
get_embeddings(idx, file, summary, file_contents, attempt + 1)
else
{:error, "request to index file timed out after 3 attempts"}
end
{:error, reason} ->
{:error, reason}
end
end
defp sha256(file_path) do
case File.read(file_path) do
{:ok, content} -> :crypto.hash(:sha256, content) |> Base.encode16(case: :lower)
{:error, reason} -> {:error, reason}
end
end
end