Packages
fnord
0.4.5
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/store.ex
defmodule Store do
@moduledoc """
This module provides the functionality for storing and retrieving embeddings
and metadata for files.
"""
defstruct [:project, :path]
@doc """
Get the path to the store root directory.
"""
def home() do
"#{System.get_env("HOME")}/.fnord"
end
@doc """
Create a new `Store` struct.
"""
def new(project) do
File.mkdir_p!(home())
path = Path.join(home(), project)
File.mkdir_p!(path)
%Store{
project: project,
path: path
}
end
@doc """
Permanently deletes the specified project's index directory and all its
contents.
"""
def delete_project(store) do
File.rm_rf!(store.path)
end
@doc """
Permanently deletes the specified file from the store. Note that this is
_only_ exposed for the sake of testing.
"""
def delete_file(store, file) do
path = get_entry_path(store, file)
File.rm_rf!(path)
end
@doc """
Permanently delete any files that are indexed but no longer exist on disk.
"""
def delete_missing_files(store, root, callback \\ fn -> nil end) do
store
|> list_files()
|> Enum.each(fn file ->
cond do
!File.exists?(file) ->
delete_file(store, file)
callback.()
# There was a bug allowing git-ignored files to be indexed
Git.is_ignored?(file, root) ->
delete_file(store, file)
callback.()
true ->
:ok
end
end)
end
defp get_key(file_path) do
full_path = Path.expand(file_path)
:crypto.hash(:sha256, full_path) |> Base.encode16(case: :lower)
end
defp get_entry_path(store, file_path) do
Path.join(store.path, get_key(file_path))
end
@doc """
Get the metadata for the specified file. Returns `{:ok, data}` if the file
exists, or `{:error, :not_found}` if it does not. The structure of the
metadata is as follows:
```
%{
file: "path/to/file.ext",
hash: "DEADBEEF",
timestamp: "2021-01-01T00:00:00Z",
fnord_path: "path/to/store/FEEBDAED"
}
```
"""
def info(store, file) do
with path = get_entry_path(store, file),
file = Path.join(path, "metadata.json"),
{:ok, data} <- File.read(file),
{:ok, meta} <- Jason.decode(data) do
{:ok, Map.put(meta, "fnord_path", path)}
else
_ -> {:error, :not_found}
end
end
@doc """
Get the hash for the specified file. Returns `{:ok, hash}` if the file
exists, or `{:error, :not_found}` if it does not.
"""
def get_hash(store, file) do
Store.info(store, file)
|> case do
{:ok, data} -> Map.get(data, "hash")
{:error, :not_found} -> {:error, :not_found}
end
end
@doc """
Get the summary for the specified file. Returns `{:ok, summary}` if the file
exists, or `{:error, :not_found}` if it does not.
"""
def get_summary(store, file) do
with path = get_entry_path(store, file),
file = Path.join(path, "summary"),
{:ok, summary} <- File.read(file) do
{:ok, summary}
else
{:error, :not_found} -> {:error, :not_found}
end
end
@doc """
Get the embeddings for the specified file. Returns `{:ok, embeddings}` if the
file exists, or `{:error, :not_found}` if it does not. Note that if the input
file was greater than 8192 tokens, the embeddings will be split into multiple
chunks.
"""
def get_embeddings(store, file) do
with {:ok, meta} <- Store.info(store, file) do
path = Map.get(meta, "fnord_path")
embeddings =
Path.join(path, "embedding_*.json")
|> Path.wildcard()
|> Enum.map(&File.read!(&1))
|> Enum.map(&Jason.decode!(&1))
{:ok, embeddings}
else
{:error, :not_found} -> {:error, :not_found}
end
end
@doc """
Get the metadata, summary, and embeddings for the specified file. Returns
`{:ok, info}` if the file exists, or `{:error, :not_found}` if it does not.
The structure of the returned `info` is as follows:
```
%{
file: "path/to/file.ext",
hash: "DEADBEEF",
timestamp: "2021-01-01T00:00:00Z",
summary: "AI-generated summary of the file",
embeddings: [
[0.1, 0.2, 0.3, ...],
[0.4, 0.5, 0.6, ...],
...
]
}
```
"""
def get(store, file) do
with {:ok, meta} <- Store.info(store, file),
{:ok, summary} <- get_summary(store, file),
{:ok, embeddings} <- get_embeddings(store, file) do
info =
meta
|> Map.put("summary", summary)
|> Map.put("embeddings", embeddings)
{:ok, info}
else
{:error, :not_found} -> {:error, :not_found}
end
end
@doc """
Store the metadata, summary, and embeddings for the specified file. If the
file already exists in the store, it will be overwritten.
"""
def put(store, file, hash, summary, embeddings) do
path = get_entry_path(store, file)
# Delete the existing directory if it exists
File.rm_rf!(path)
# Recreate the subject file dir under the project dir
File.mkdir_p!(path)
# Write the metadata to a separate file
metadata =
Jason.encode!(%{
file: Path.expand(file),
hash: hash,
timestamp: DateTime.utc_now()
})
Path.join(path, "metadata.json") |> File.write!(metadata)
# Write the summary to a separate file
Path.join(path, "summary") |> File.write!(summary)
# Write each chunk of embeddings to separate files
embeddings
|> Enum.with_index()
|> Enum.each(fn {embedding, chunk_no} ->
entry = Jason.encode!(embedding)
Path.join(path, "embedding_#{chunk_no}.json") |> File.write!(entry)
end)
end
@doc """
List all projects in the store. Returns a list of project names.
"""
def list_projects() do
Path.wildcard(Path.join(home(), "*"))
|> Enum.map(fn path -> Path.basename(path) end)
end
@doc """
List all indexed files in the project. Returns a list of absolute, expanded,
file paths.
"""
def list_files(store) do
Path.wildcard(Path.join(store.path, "*"))
|> Enum.map(fn path -> Path.join(path, "metadata.json") end)
|> Enum.filter(&File.exists?/1)
|> Enum.map(fn path ->
path
|> File.read!()
|> Jason.decode!()
|> Map.get("file")
end)
end
# Computes the cosine similarity between two vectors
def cosine_similarity(vec1, vec2) do
dot_product = Enum.zip(vec1, vec2) |> Enum.reduce(0.0, fn {a, b}, acc -> acc + a * b end)
magnitude1 = :math.sqrt(Enum.reduce(vec1, 0.0, fn x, acc -> acc + x * x end))
magnitude2 = :math.sqrt(Enum.reduce(vec2, 0.0, fn x, acc -> acc + x * x end))
if magnitude1 == 0.0 or magnitude2 == 0.0 do
0.0
else
dot_product / (magnitude1 * magnitude2)
end
end
end