Packages
fnord
0.4.27
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/search.ex
defmodule Search do
defstruct [
:project,
:query,
:limit,
:detail,
:store,
:concurrency,
:ai_module
]
@doc """
Creates a new `Search` struct with the given options.
"""
def new(opts, ai_module \\ AI) do
%__MODULE__{
project: opts[:project],
query: opts[:query],
limit: opts[:limit],
detail: opts[:detail],
store: Store.new(opts[:project]),
concurrency: opts[:concurrency],
ai_module: ai_module
}
end
def get_results(search) do
needle = get_query_embeddings(search.query, search.ai_module)
{:ok, queue} =
Queue.start_link(search.concurrency, fn file ->
with {:ok, data} <- get_file_data(search, file) do
get_score(needle, data)
|> case do
{:ok, score} -> {file, score, data}
{:error, :no_embeddings} -> nil
end
else
_ -> nil
end
end)
results =
search
|> list_files()
|> Queue.map(queue)
|> Enum.reject(&is_nil/1)
|> Enum.sort(fn {_, score1, _}, {_, score2, _} -> score1 >= score2 end)
|> Enum.take(search.limit)
Queue.shutdown(queue)
Queue.join(queue)
results
end
defp get_score(needle, data) do
data
|> Map.get("embeddings", [])
|> Enum.map(fn emb -> cosine_similarity(needle, emb) end)
|> case do
[] -> {:error, :no_embeddings}
scores -> {:ok, Enum.max(scores)}
end
end
defp get_query_embeddings(query, ai_module) do
{:ok, [needle]} = ai_module.get_embeddings(ai_module.new(), query)
needle
end
defp list_files(search) do
Store.list_files(search.store)
end
defp get_file_data(search, file) do
if search.detail do
Store.get(search.store, file)
else
with {:ok, embeddings} <- Store.get_embeddings(search.store, file) do
{:ok, %{"embeddings" => embeddings}}
else
{:error, _} -> {:error, :file}
end
end
end
# Computes the cosine similarity between two vectors
def cosine_similarity(vec1, vec2) do
dot_product = Enum.zip(vec1, vec2) |> Enum.reduce(0.0, fn {a, b}, acc -> acc + a * b end)
magnitude1 = :math.sqrt(Enum.reduce(vec1, 0.0, fn x, acc -> acc + x * x end))
magnitude2 = :math.sqrt(Enum.reduce(vec2, 0.0, fn x, acc -> acc + x * x end))
if magnitude1 == 0.0 or magnitude2 == 0.0 do
0.0
else
dot_product / (magnitude1 * magnitude2)
end
end
end