Packages

fnord

0.9.37

AI code archaeology

Current section

Files

Jump to
fnord lib store project entry.ex
Raw

lib/store/project/entry.ex

defmodule Store.Project.Entry do
defstruct [
:project,
:file,
:rel_path,
:key,
:store_path,
:metadata,
:summary,
:embeddings
]
@type t :: %__MODULE__{}
@id_prefix_reversible "r1-"
@id_prefix_hash "h1-"
@max_id_len 240
@spec new_from_file_path(Store.Project.t(), String.t()) :: t()
def new_from_file_path(project, file) do
abs_path = Store.Project.expand_path(file, project)
rel_path = Store.Project.relative_path(abs_path, project)
key = id_for_rel_path(rel_path)
store_base = Store.Project.files_root(project)
store_path = Path.join(store_base, key)
metadata = Store.Project.Entry.Metadata.new(store_path, abs_path)
summary = Store.Project.Entry.Summary.new(store_path, abs_path)
embeddings = Store.Project.Entry.Embeddings.new(store_path, abs_path)
%__MODULE__{
project: project,
file: abs_path,
rel_path: rel_path,
key: key,
store_path: store_path,
metadata: metadata,
summary: summary,
embeddings: embeddings
}
end
@spec new_from_entry_path(Store.Project.t(), String.t()) :: t()
def new_from_entry_path(project, entry_path) do
with {:ok, content} <- File.read(Path.join(entry_path, "metadata.json")),
{:ok, metadata} <- SafeJson.decode(content),
{:ok, file_field} <- Map.fetch(metadata, "file"),
true <- is_binary(file_field) do
resolved =
cond do
# Legacy absolute path
String.starts_with?(file_field, "/") ->
file_field
# New relative path
true ->
Path.expand(file_field, project.source_root)
end
new_from_file_path(project, resolved)
else
_ ->
raise """
Invalid or corrupted metadata in #{entry_path}.
This may indicate a corrupted project index. Try running:
fnord index --reindex
"""
end
end
@spec exists_in_store?(t()) :: boolean()
def exists_in_store?(entry) do
File.dir?(entry.store_path)
end
@spec create(t()) :: :ok
def create(entry), do: File.mkdir_p!(entry.store_path)
@spec delete(t()) :: [binary()]
def delete(entry), do: File.rm_rf!(entry.store_path)
@spec is_incomplete?(t()) :: boolean()
def is_incomplete?(entry) do
cond do
!has_metadata?(entry) -> true
!has_summary?(entry) -> true
!has_embeddings?(entry) -> true
true -> false
end
end
@spec is_stale?(t()) :: boolean()
def is_stale?(entry) do
cond do
!exists_in_store?(entry) ->
true
!has_summary?(entry) ->
true
!has_embeddings?(entry) ->
true
true ->
# Read metadata.json once and thread it through the remaining
# checks. Each per-file scan pays a single disk+decode cost
# instead of one read for the hash and another (indirectly, via
# the embeddings file) for the dim.
case read_metadata(entry) do
{:ok, metadata} ->
not hash_is_current?(entry, metadata) or
not embedding_dim_is_current?(entry, metadata)
_ ->
true
end
end
end
# An entry whose stored embedding vector was produced by a different
# model (e.g. pre-migration OpenAI 3072-dim data that the cross-format
# hash upgrade marked as "fresh") is still stale from the embedding
# layer's perspective: cosine_similarity would crash against the new
# query vectors. The dim is recorded in metadata at save time so this
# check is O(1) disk; older stores that predate that field fall back
# to reading the embeddings file once and backfilling so the next
# scan takes the fast path.
@spec embedding_dim_is_current?(t(), map()) :: boolean()
defp embedding_dim_is_current?(entry, metadata) do
current = AI.Embeddings.dimensions()
case Map.get(metadata, "embedding_dim") do
dim when is_integer(dim) ->
dim == current
_ ->
case Store.Project.Entry.Embeddings.read(entry.embeddings) do
{:ok, list} when is_list(list) ->
actual = length(list)
if actual == current do
backfill_embedding_dim(entry, metadata, actual)
true
else
false
end
_ ->
false
end
end
end
# Re-stamps metadata with the discovered dim so subsequent scans
# answer embedding_dim_is_current? from metadata alone. No-op when
# metadata is missing the hash (we don't invent one) - the next
# successful save/3 will record both fields together.
defp backfill_embedding_dim(entry, metadata, dim) do
case Map.get(metadata, "hash") do
hash when is_binary(hash) -> save_metadata(entry, hash, embedding_dim: dim)
_ -> :ok
end
end
@spec read(t()) :: {:ok, map()} | {:error, any()}
def read(entry) do
with {:ok, metadata} <- read_metadata(entry),
{:ok, summary} <- read_summary(entry),
{:ok, embeddings} <- read_embeddings(entry) do
info =
metadata
|> Map.put("file", entry.file)
|> Map.put("summary", summary)
|> Map.put("embeddings", embeddings)
{:ok, info}
end
end
@spec save(t(), String.t(), [float]) :: :ok | {:error, any()}
def save(entry, summary, embeddings) do
delete(entry)
create(entry)
with {:ok, hash} <- Store.Project.Source.hash(entry.project, entry.rel_path),
:ok <- save_metadata(entry, hash, embedding_dim: embedding_length(embeddings)),
:ok <- save_summary(entry, summary),
:ok <- save_embeddings(entry, embeddings) do
:ok
end
end
# Width of a single embedding vector. Indexer.get_embeddings returns a
# flat list of floats today, but accept the legacy nested shape too so
# this helper stays robust if a caller ever hands us the pre-collapse
# per-chunk layout.
defp embedding_length([]), do: 0
defp embedding_length([first | _]) when is_list(first), do: length(first)
defp embedding_length(list) when is_list(list), do: length(list)
defp embedding_length(_), do: 0
@spec hash_is_current?(t()) :: boolean()
def hash_is_current?(entry) do
case read_metadata(entry) do
{:ok, metadata} -> hash_is_current?(entry, metadata)
_ -> false
end
end
@spec hash_is_current?(t(), map()) :: boolean()
def hash_is_current?(entry, metadata) do
with {:ok, stored} <- Map.fetch(metadata, "hash"),
{:ok, current} <- Store.Project.Source.hash(entry.project, entry.rel_path) do
cond do
stored == current ->
true
# Cross-format match: stored hash was produced by an older fnord
# (sha256 of working-tree content) while the current source mode
# produces a different hash (git blob SHA). If the actual content
# is unchanged, we re-stamp the metadata to the current format in
# place rather than blowing a full LLM summarize + embed pass on
# a file that hasn't changed at all.
content_unchanged?(entry, stored) ->
restamp_metadata_hash(entry, metadata, current)
true
true ->
false
end
else
_ -> false
end
end
# The cross-format hash upgrade rewrites metadata with the
# current-format hash. Preserves embedding_dim if the older metadata
# had it - otherwise the next scan would have to reopen the
# embeddings file to rediscover the dim we just threw away.
defp restamp_metadata_hash(entry, metadata, current) do
opts =
case Map.get(metadata, "embedding_dim") do
dim when is_integer(dim) -> [embedding_dim: dim]
_ -> []
end
save_metadata(entry, current, opts)
end
# Stored is sha256 (64 hex) only if it came from the pre-Source/fs-mode
# era. Compare the current source content's sha256 to the stored value;
# equal means "same content, different hash format".
defp content_unchanged?(entry, stored) when is_binary(stored) do
case byte_size(stored) do
64 ->
case Store.Project.Source.read(entry.project, entry.rel_path) do
{:ok, content} -> sha256(content) == stored
_ -> false
end
_ ->
false
end
end
defp content_unchanged?(_entry, _stored), do: false
@spec read_source_file(t()) :: {:ok, binary} | {:error, any()}
def read_source_file(entry) do
Store.Project.Source.read(entry.project, entry.rel_path)
end
# -----------------------------------------------------------------------------
# metadata.json
# -----------------------------------------------------------------------------
def metadata_file_path(entry), do: Store.Project.Entry.Metadata.store_path(entry.metadata)
def has_metadata?(entry), do: Store.Project.Entry.Metadata.exists?(entry.metadata)
def read_metadata(entry), do: Store.Project.Entry.Metadata.read(entry.metadata)
@spec save_metadata(t(), binary, keyword) :: :ok | {:error, any()}
def save_metadata(entry, hash, opts \\ []) when is_binary(hash) do
base = %{rel_path: entry.rel_path, hash: hash}
attrs =
case Keyword.get(opts, :embedding_dim) do
dim when is_integer(dim) -> Map.put(base, :embedding_dim, dim)
_ -> base
end
Store.Project.Entry.Metadata.write(entry.metadata, attrs)
end
# Back-compat for callers that still hash the working tree directly.
# Used only by tests / tooling that reach into the entry directly
# rather than going through the Source-aware `save/3` pipeline.
def save_metadata(entry) do
case Store.Project.Source.hash(entry.project, entry.rel_path) do
{:ok, hash} -> save_metadata(entry, hash)
{:error, reason} -> {:error, reason}
end
end
# -----------------------------------------------------------------------------
# summary
# -----------------------------------------------------------------------------
def summary_file_path(entry), do: Store.Project.Entry.Summary.store_path(entry.summary)
def has_summary?(entry), do: Store.Project.Entry.Summary.exists?(entry.summary)
def read_summary(entry), do: Store.Project.Entry.Summary.read(entry.summary)
def save_summary(entry, data), do: Store.Project.Entry.Summary.write(entry.summary, data)
# -----------------------------------------------------------------------------
# embeddings
# -----------------------------------------------------------------------------
def embeddings_file_paths(entry) do
Store.Project.Entry.Embeddings.store_path(entry.embeddings)
end
def has_embeddings?(entry), do: Store.Project.Entry.Embeddings.exists?(entry.embeddings)
def read_embeddings(entry), do: Store.Project.Entry.Embeddings.read(entry.embeddings)
def save_embeddings(entry, embeddings) do
Store.Project.Entry.Embeddings.write(entry.embeddings, embeddings)
end
# -----------------------------------------------------------------------------
# Private functions
# -----------------------------------------------------------------------------
defp sha256(content) do
:crypto.hash(:sha256, content)
|> Base.encode16(case: :lower)
end
@spec id_for_rel_path(String.t()) :: String.t()
def id_for_rel_path(rel_path) do
# Try reversible ID first
reversible_id = @id_prefix_reversible <> Base.url_encode64(rel_path, padding: false)
if byte_size(reversible_id) <= @max_id_len do
reversible_id
else
# Fall back to hash for long paths
@id_prefix_hash <> sha256(rel_path)
end
end
@spec rel_path_from_id(String.t()) :: {:ok, String.t()} | {:error, :not_reversible}
def rel_path_from_id(id) do
cond do
String.starts_with?(id, @id_prefix_reversible) ->
encoded = String.slice(id, String.length(@id_prefix_reversible)..-1//1)
case Base.url_decode64(encoded, padding: false) do
{:ok, rel_path} -> {:ok, rel_path}
:error -> {:error, :not_reversible}
end
String.starts_with?(id, @id_prefix_hash) ->
{:error, :not_reversible}
true ->
# Legacy absolute path hash
{:error, :not_reversible}
end
end
end