Packages
fnord
0.4.21
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/search.ex
defmodule Search do
@moduledoc """
This module provides the functionality for the `search` sub-command.
"""
defstruct [
:project,
:query,
:limit,
:detail,
:store,
:concurrency,
:ai_module
]
@doc """
Creates a new `Search` struct with the given options.
"""
def new(opts, ai_module \\ AI) do
%Search{
project: opts[:project],
query: opts[:query],
limit: opts[:limit],
detail: opts[:detail],
store: Store.new(opts[:project]),
concurrency: opts[:concurrency],
ai_module: ai_module
}
end
@doc """
Searches the given project for previously indexed files (see `Indexing`) that
match the given query. The search results are printed to the console.
Note that the query input is first sent to OpenAI's API to generate an
embedding to match against the vector store.
"""
def run(opts, ai_module \\ AI) do
search = new(opts, ai_module)
search
|> get_results()
|> Enum.each(fn {file, score, data} ->
output_file(search, file, score, data)
end)
end
def get_results(search) do
needle = get_query_embeddings(search.query, search.ai_module)
{:ok, queue} =
Queue.start_link(search.concurrency, fn file ->
with {:ok, data} <- get_file_data(search, file) do
get_score(needle, data)
|> case do
{:ok, score} -> {file, score, data}
{:error, :no_embeddings} -> nil
end
else
_ -> nil
end
end)
results =
search
|> list_files()
|> Queue.map(queue)
|> Enum.reject(&is_nil/1)
|> Enum.sort(fn {_, score1, _}, {_, score2, _} -> score1 >= score2 end)
|> Enum.take(search.limit)
Queue.shutdown(queue)
Queue.join(queue)
results
end
defp output_file(search, file, score, data) do
if search.detail do
summary = Map.get(data, "summary")
IO.puts("""
-----
# File: #{file} | Score: #{score}
#{summary}
""")
else
IO.puts(file)
end
end
defp get_score(needle, data) do
data
|> Map.get("embeddings", [])
|> Enum.map(fn emb -> cosine_similarity(needle, emb) end)
|> case do
[] -> {:error, :no_embeddings}
scores -> {:ok, Enum.max(scores)}
end
end
defp get_query_embeddings(query, ai_module) do
{:ok, [needle]} = ai_module.get_embeddings(ai_module.new(), query)
needle
end
defp list_files(search) do
Store.list_files(search.store)
end
defp get_file_data(search, file) do
if search.detail do
Store.get(search.store, file)
else
with {:ok, embeddings} <- Store.get_embeddings(search.store, file) do
{:ok, %{"embeddings" => embeddings}}
else
{:error, _} -> {:error, :file}
end
end
end
# Computes the cosine similarity between two vectors
def cosine_similarity(vec1, vec2) do
dot_product = Enum.zip(vec1, vec2) |> Enum.reduce(0.0, fn {a, b}, acc -> acc + a * b end)
magnitude1 = :math.sqrt(Enum.reduce(vec1, 0.0, fn x, acc -> acc + x * x end))
magnitude2 = :math.sqrt(Enum.reduce(vec2, 0.0, fn x, acc -> acc + x * x end))
if magnitude1 == 0.0 or magnitude2 == 0.0 do
0.0
else
dot_product / (magnitude1 * magnitude2)
end
end
end