Packages
fnord
0.6.1
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/store/project.ex
defmodule Store.Project do
defstruct [
:name,
:store_path,
:source_root,
:exclude,
:conversation_dir,
:notes_dir
]
@type t :: %__MODULE__{}
@conversation_dir "conversations"
@notes_dir "notes"
def new(project_name, store_path) do
settings = Settings.new() |> Settings.get(project_name, %{})
exclude = Map.get(settings, "exclude", [])
root = Map.get(settings, "root")
%__MODULE__{
name: project_name,
store_path: store_path,
source_root: root,
exclude: exclude,
conversation_dir: Path.join(store_path, @conversation_dir),
notes_dir: Path.join(store_path, @notes_dir)
}
end
def save_settings(project, source_root \\ nil, exclude \\ nil) do
settings = %{
"exclude" => project.exclude,
"root" =>
case project.source_root do
nil -> nil
_ -> Path.expand(project.source_root)
end
}
settings =
if is_nil(source_root) do
settings
else
Map.put(settings, "root", Path.expand(source_root))
end
settings =
case exclude do
nil -> settings
[] -> settings
_ -> Map.put(settings, "exclude", exclude)
end
Settings.new()
|> Settings.set(project.name, settings)
new(project.name, project.store_path)
end
def create(project) do
project.store_path |> File.mkdir_p!()
project.store_path |> Path.join("conversations") |> File.mkdir_p!()
end
def delete(project) do
# Delete indexed files
project.store_path
|> Path.join("*/metadata.json")
|> Path.wildcard()
|> Enum.map(&Path.dirname/1)
|> Enum.each(fn path -> File.rm_rf!(path) end)
end
def torch(project) do
# Delete entire directory
File.rm_rf!(project.store_path)
# Remove from settings
Settings.new() |> Settings.delete(project.name)
end
# ----------------------------------------------------------------------------
# Entries
# ----------------------------------------------------------------------------
@spec find_entry(Store.Project.t(), String.t()) ::
{:ok, Store.Project.Entry.t()}
| {:error, atom()}
def find_entry(project, path) do
with {:ok, resolved} <- find_file(project, path) do
{:ok, Store.Project.Entry.new_from_file_path(project, resolved)}
end
end
@spec find_file(Store.Project.t(), String.t()) ::
{:ok, String.t()}
| {:error, atom()}
def find_file(project, path) do
[
&find_abs_file_root/2,
&find_abs_file_project/2,
&find_file_project/2
]
|> Enum.find_value(fn f ->
with {:ok, path} <- f.(project, path) do
{:ok, path}
else
_ -> false
end
end)
|> case do
{:ok, path} -> {:ok, path}
_ -> {:error, :not_found}
end
end
defp find_abs_file_root(_project, path) do
if String.starts_with?(path, "/") && File.exists?(path) do
{:ok, path}
else
{:error, :not_found}
end
end
defp find_abs_file_project(project, path) do
if String.starts_with?(path, "/") do
path =
Path.join(project.source_root, path)
|> Path.expand(project.source_root)
if File.exists?(path) do
{:ok, path}
else
{:error, :not_found}
end
else
{:error, :not_found}
end
end
defp find_file_project(project, path) do
project
|> stored_files()
|> Enum.find(&String.ends_with?(&1.file, path))
|> case do
nil -> {:error, :not_found}
entry -> {:ok, entry.file}
end
end
def expand_path(path, project) do
Path.expand(path, project.source_root)
end
def relative_path(path, project) do
path
|> expand_path(project)
|> Path.relative_to(project.source_root)
end
def find_path_in_source_root(project, path) do
path = expand_path(project, path)
cond do
File.dir?(path) -> {:ok, :dir, path}
File.regular?(path) -> {:ok, :file, path}
true -> {:ok, :not_found, path}
end
end
def exists_in_store?(project) do
path = project.store_path
files = Path.wildcard(Path.join(path, "*"))
cond do
!File.dir?(path) -> false
# There was a bug at one point where store files were deleted but not the
# directory itself. This check is to ensure that the directory is empty.
Enum.empty?(files) -> false
true -> true
end
end
def stored_files(project) do
# Start with the path to the project in the store
project.store_path
# Each entry is a dir that contains metadata.json; this step ignores things
# like the conversations directory.
|> Path.join("*/metadata.json")
# Expand the glob
|> Path.wildcard()
# Strip metadata.json from the each listing, leaving the directory path for
# the individual entry.
|> Enum.map(&Path.dirname/1)
# Create an Entry for each directory.
|> Enum.map(&Store.Project.Entry.new_from_entry_path(project, &1))
end
def source_files(project) do
excluded = excluded_paths(project)
DirStream.new(project.source_root, &want_dir?(&1, excluded, project.source_root))
|> Stream.filter(&want_file?(&1, excluded, project.source_root))
|> Stream.map(&Store.Project.Entry.new_from_file_path(project, &1))
end
def stale_source_files(project) do
project
|> source_files()
|> Stream.filter(&Store.Project.Entry.is_stale?/1)
end
def delete_missing_files(project) do
excluded_files = excluded_paths(project)
project
|> stored_files()
|> Enum.each(fn entry ->
cond do
!Store.Project.Entry.source_file_exists?(entry) -> Store.Project.Entry.delete(entry)
Store.Project.Entry.is_git_ignored?(entry) -> Store.Project.Entry.delete(entry)
MapSet.member?(excluded_files, entry.file) -> Store.Project.Entry.delete(entry)
true -> is_text?(entry.file)
end
end)
end
# -----------------------------------------------------------------------------
# Conversations
# -----------------------------------------------------------------------------
def conversations(project) do
conversations =
project.store_path
|> Path.join(["conversations/*.json"])
|> Path.wildcard()
|> Enum.map(&Path.basename(&1, ".json"))
|> Enum.map(&Store.Project.Conversation.new(&1, project))
timestamps =
conversations
|> Enum.reduce(%{}, fn conversation, acc ->
timestamp = Store.Project.Conversation.timestamp(conversation)
Map.put(acc, conversation.id, timestamp)
end)
conversations
|> Enum.sort(fn a, b ->
timestamps[a.id] > timestamps[b.id]
end)
end
# -----------------------------------------------------------------------------
# Notes
# -----------------------------------------------------------------------------
def notes(project) do
project.notes_dir
|> File.ls()
|> case do
{:ok, dirs} ->
dirs
|> Enum.sort()
|> Enum.map(&Store.Project.Note.new(project, &1))
_ ->
[]
end
end
def search_notes(project, query, max_results \\ 10) do
needle = AI.get_embeddings!(AI.new(), query)
notes = notes(project)
workers = Enum.count(notes)
notes
# Retrieve embeddings for each note
|> Util.async_stream(
fn note ->
with {:ok, embeddings} <- Store.Project.Note.read_embeddings(note) do
{:ok, {note, embeddings}}
end
end,
max_concurrency: workers
)
# Calculate the similarity between the query and each note
|> Util.async_stream(
fn
{:ok, {:ok, {note, embeddings}}} ->
score = AI.Util.cosine_similarity(needle, embeddings)
{score, note}
_ ->
nil
end,
max_concurrency: workers
)
# Collect the results
|> Enum.reduce([], fn
{:ok, {score, note}}, acc -> [{score, note} | acc]
_, acc -> acc
end)
# Sort by similarity
|> Enum.sort(fn {a, _}, {b, _} -> a >= b end)
# Take the top N results
|> Enum.take(max_results)
end
def reset_notes(project) do
File.rm_rf(project.notes_dir)
end
# -----------------------------------------------------------------------------
# Private functions
# -----------------------------------------------------------------------------
defp want_dir?(path, excluded, source_root) do
cond do
# hidden
is_hidden?(path) -> false
# explicitly excluded
MapSet.member?(excluded, path) -> false
# git-ignored
Git.is_ignored?(path, source_root) -> false
# keeper
true -> true
end
end
defp want_file?(path, excluded, source_root) do
cond do
# hidden
is_hidden?(path) -> false
# explicitly excluded
MapSet.member?(excluded, path) -> false
# git-ignored
Git.is_ignored?(path, source_root) -> false
# text only
true -> is_text?(path)
end
end
defp is_hidden?(path) do
base = Path.basename(path)
cond do
base == ".github" -> false
String.starts_with?(base, ".") -> true
true -> false
end
end
defp is_text?(file_path) do
try do
file_path
|> File.stream!(1024)
|> Enum.reduce_while(true, fn chunk, _acc ->
if String.valid?(chunk) do
{:cont, true}
else
{:halt, false}
end
end)
rescue
_ -> false
end
end
defp excluded_paths(project) do
if is_nil(project.exclude) do
MapSet.new()
else
project.exclude
|> Enum.flat_map(fn exclude ->
exclude
|> find_path_in_source_root(project)
|> case do
# If it's a directory, exclude all files in that directory
{:ok, :dir, path} -> Path.wildcard(Path.join(path, "**/*"), match_dot: true)
# If it's a single file, exclude just that file
{:ok, :file, path} -> [path]
# Otherwise, treat it as a glob
_ -> Path.wildcard(exclude, match_dot: true)
end
end)
# Filter out directories and non-existent files
|> Enum.filter(&File.regular?/1)
# Convert everything to absolute paths
|> Enum.map(&Path.absname/1)
# Convert to a MapSet for faster lookups
|> MapSet.new()
end
end
end