Packages
fnord
0.9.26
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/store/project/entry.ex
defmodule Store.Project.Entry do
defstruct [
:project,
:file,
:rel_path,
:key,
:store_path,
:metadata,
:summary,
:embeddings
]
@type t :: %__MODULE__{}
@id_prefix_reversible "r1-"
@id_prefix_hash "h1-"
@max_id_len 240
@spec new_from_file_path(Store.Project.t(), String.t()) :: t()
def new_from_file_path(project, file) do
abs_path = Store.Project.expand_path(file, project)
rel_path = Store.Project.relative_path(abs_path, project)
key = id_for_rel_path(rel_path)
store_base = Store.Project.files_root(project)
store_path = Path.join(store_base, key)
metadata = Store.Project.Entry.Metadata.new(store_path, abs_path)
summary = Store.Project.Entry.Summary.new(store_path, abs_path)
embeddings = Store.Project.Entry.Embeddings.new(store_path, abs_path)
%__MODULE__{
project: project,
file: abs_path,
rel_path: rel_path,
key: key,
store_path: store_path,
metadata: metadata,
summary: summary,
embeddings: embeddings
}
end
@spec new_from_entry_path(Store.Project.t(), String.t()) :: t()
def new_from_entry_path(project, entry_path) do
with {:ok, content} <- File.read(Path.join(entry_path, "metadata.json")),
{:ok, metadata} <- SafeJson.decode(content),
{:ok, file_field} <- Map.fetch(metadata, "file"),
true <- is_binary(file_field) do
resolved =
cond do
# Legacy absolute path
String.starts_with?(file_field, "/") ->
file_field
# New relative path
true ->
Path.expand(file_field, project.source_root)
end
new_from_file_path(project, resolved)
else
_ ->
raise """
Invalid or corrupted metadata in #{entry_path}.
This may indicate a corrupted project index. Try running:
fnord index --reindex
"""
end
end
@spec exists_in_store?(t()) :: boolean()
def exists_in_store?(entry) do
File.dir?(entry.store_path)
end
@spec create(t()) :: :ok
def create(entry), do: File.mkdir_p!(entry.store_path)
@spec delete(t()) :: [binary()]
def delete(entry), do: File.rm_rf!(entry.store_path)
@spec is_incomplete?(t()) :: boolean()
def is_incomplete?(entry) do
cond do
!has_metadata?(entry) -> true
!has_summary?(entry) -> true
!has_embeddings?(entry) -> true
true -> false
end
end
@spec is_stale?(t()) :: boolean()
def is_stale?(entry) do
cond do
!exists_in_store?(entry) ->
true
!has_summary?(entry) ->
true
!has_embeddings?(entry) ->
true
true ->
# Read metadata.json once and thread it through the remaining
# checks. Each per-file scan pays a single disk+decode cost
# instead of one read for the hash and another (indirectly, via
# the embeddings file) for the dim.
case read_metadata(entry) do
{:ok, metadata} ->
not hash_is_current?(entry, metadata) or
not embedding_dim_is_current?(entry, metadata)
_ ->
true
end
end
end
# An entry whose stored embedding vector was produced by a different
# model (e.g. pre-migration OpenAI 3072-dim data that the cross-format
# hash upgrade marked as "fresh") is still stale from the embedding
# layer's perspective: cosine_similarity would crash against the new
# query vectors. The dim is recorded in metadata at save time so this
# check is O(1) disk; older stores that predate that field fall back
# to reading the embeddings file once and backfilling so the next
# scan takes the fast path.
@spec embedding_dim_is_current?(t(), map()) :: boolean()
defp embedding_dim_is_current?(entry, metadata) do
current = AI.Embeddings.dimensions()
case Map.get(metadata, "embedding_dim") do
dim when is_integer(dim) ->
dim == current
_ ->
case Store.Project.Entry.Embeddings.read(entry.embeddings) do
{:ok, list} when is_list(list) ->
actual = length(list)
if actual == current do
backfill_embedding_dim(entry, metadata, actual)
true
else
false
end
_ ->
false
end
end
end
# Re-stamps metadata with the discovered dim so subsequent scans
# answer embedding_dim_is_current? from metadata alone. No-op when
# metadata is missing the hash (we don't invent one) - the next
# successful save/3 will record both fields together.
defp backfill_embedding_dim(entry, metadata, dim) do
case Map.get(metadata, "hash") do
hash when is_binary(hash) -> save_metadata(entry, hash, embedding_dim: dim)
_ -> :ok
end
end
@spec read(t()) :: {:ok, map()} | {:error, any()}
def read(entry) do
with {:ok, metadata} <- read_metadata(entry),
{:ok, summary} <- read_summary(entry),
{:ok, embeddings} <- read_embeddings(entry) do
info =
metadata
|> Map.put("file", entry.file)
|> Map.put("summary", summary)
|> Map.put("embeddings", embeddings)
{:ok, info}
end
end
@spec save(t(), String.t(), [float]) :: :ok | {:error, any()}
def save(entry, summary, embeddings) do
delete(entry)
create(entry)
with {:ok, hash} <- Store.Project.Source.hash(entry.project, entry.rel_path),
:ok <- save_metadata(entry, hash, embedding_dim: embedding_length(embeddings)),
:ok <- save_summary(entry, summary),
:ok <- save_embeddings(entry, embeddings) do
:ok
end
end
# Width of a single embedding vector. Indexer.get_embeddings returns a
# flat list of floats today, but accept the legacy nested shape too so
# this helper stays robust if a caller ever hands us the pre-collapse
# per-chunk layout.
defp embedding_length([]), do: 0
defp embedding_length([first | _]) when is_list(first), do: length(first)
defp embedding_length(list) when is_list(list), do: length(list)
defp embedding_length(_), do: 0
@spec hash_is_current?(t()) :: boolean()
def hash_is_current?(entry) do
case read_metadata(entry) do
{:ok, metadata} -> hash_is_current?(entry, metadata)
_ -> false
end
end
@spec hash_is_current?(t(), map()) :: boolean()
def hash_is_current?(entry, metadata) do
with {:ok, stored} <- Map.fetch(metadata, "hash"),
{:ok, current} <- Store.Project.Source.hash(entry.project, entry.rel_path) do
cond do
stored == current ->
true
# Cross-format match: stored hash was produced by an older fnord
# (sha256 of working-tree content) while the current source mode
# produces a different hash (git blob SHA). If the actual content
# is unchanged, we re-stamp the metadata to the current format in
# place rather than blowing a full LLM summarize + embed pass on
# a file that hasn't changed at all.
content_unchanged?(entry, stored) ->
restamp_metadata_hash(entry, metadata, current)
true
true ->
false
end
else
_ -> false
end
end
# The cross-format hash upgrade rewrites metadata with the
# current-format hash. Preserves embedding_dim if the older metadata
# had it - otherwise the next scan would have to reopen the
# embeddings file to rediscover the dim we just threw away.
defp restamp_metadata_hash(entry, metadata, current) do
opts =
case Map.get(metadata, "embedding_dim") do
dim when is_integer(dim) -> [embedding_dim: dim]
_ -> []
end
save_metadata(entry, current, opts)
end
# Stored is sha256 (64 hex) only if it came from the pre-Source/fs-mode
# era. Compare the current source content's sha256 to the stored value;
# equal means "same content, different hash format".
defp content_unchanged?(entry, stored) when is_binary(stored) do
case byte_size(stored) do
64 ->
case Store.Project.Source.read(entry.project, entry.rel_path) do
{:ok, content} -> sha256(content) == stored
_ -> false
end
_ ->
false
end
end
defp content_unchanged?(_entry, _stored), do: false
@spec read_source_file(t()) :: {:ok, binary} | {:error, any()}
def read_source_file(entry) do
Store.Project.Source.read(entry.project, entry.rel_path)
end
# -----------------------------------------------------------------------------
# metadata.json
# -----------------------------------------------------------------------------
def metadata_file_path(entry), do: Store.Project.Entry.Metadata.store_path(entry.metadata)
def has_metadata?(entry), do: Store.Project.Entry.Metadata.exists?(entry.metadata)
def read_metadata(entry), do: Store.Project.Entry.Metadata.read(entry.metadata)
@spec save_metadata(t(), binary, keyword) :: :ok | {:error, any()}
def save_metadata(entry, hash, opts \\ []) when is_binary(hash) do
base = %{rel_path: entry.rel_path, hash: hash}
attrs =
case Keyword.get(opts, :embedding_dim) do
dim when is_integer(dim) -> Map.put(base, :embedding_dim, dim)
_ -> base
end
Store.Project.Entry.Metadata.write(entry.metadata, attrs)
end
# Back-compat for callers that still hash the working tree directly.
# Used only by tests / tooling that reach into the entry directly
# rather than going through the Source-aware `save/3` pipeline.
def save_metadata(entry) do
case Store.Project.Source.hash(entry.project, entry.rel_path) do
{:ok, hash} -> save_metadata(entry, hash)
{:error, reason} -> {:error, reason}
end
end
# -----------------------------------------------------------------------------
# summary
# -----------------------------------------------------------------------------
def summary_file_path(entry), do: Store.Project.Entry.Summary.store_path(entry.summary)
def has_summary?(entry), do: Store.Project.Entry.Summary.exists?(entry.summary)
def read_summary(entry), do: Store.Project.Entry.Summary.read(entry.summary)
def save_summary(entry, data), do: Store.Project.Entry.Summary.write(entry.summary, data)
# -----------------------------------------------------------------------------
# embeddings
# -----------------------------------------------------------------------------
def embeddings_file_paths(entry) do
Store.Project.Entry.Embeddings.store_path(entry.embeddings)
end
def has_embeddings?(entry), do: Store.Project.Entry.Embeddings.exists?(entry.embeddings)
def read_embeddings(entry), do: Store.Project.Entry.Embeddings.read(entry.embeddings)
def save_embeddings(entry, embeddings) do
Store.Project.Entry.Embeddings.write(entry.embeddings, embeddings)
end
# -----------------------------------------------------------------------------
# Private functions
# -----------------------------------------------------------------------------
defp sha256(content) do
:crypto.hash(:sha256, content)
|> Base.encode16(case: :lower)
end
@spec id_for_rel_path(String.t()) :: String.t()
def id_for_rel_path(rel_path) do
# Try reversible ID first
reversible_id = @id_prefix_reversible <> Base.url_encode64(rel_path, padding: false)
if byte_size(reversible_id) <= @max_id_len do
reversible_id
else
# Fall back to hash for long paths
@id_prefix_hash <> sha256(rel_path)
end
end
@spec rel_path_from_id(String.t()) :: {:ok, String.t()} | {:error, :not_reversible}
def rel_path_from_id(id) do
cond do
String.starts_with?(id, @id_prefix_reversible) ->
encoded = String.slice(id, String.length(@id_prefix_reversible)..-1//1)
case Base.url_decode64(encoded, padding: false) do
{:ok, rel_path} -> {:ok, rel_path}
:error -> {:error, :not_reversible}
end
String.starts_with?(id, @id_prefix_hash) ->
{:error, :not_reversible}
true ->
# Legacy absolute path hash
{:error, :not_reversible}
end
end
end