Packages
fnord
0.9.13
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/store/project/conversation_index.ex
defmodule Store.Project.ConversationIndex do
@moduledoc """
Manages semantic index data for conversations within a project.
Index data is stored under the project's store path in a parallel
directory tree to conversations themselves:
<store_path>/conversations/index/<conversation-id>/
embeddings.json
metadata.json
This module is responsible for tracking which conversations are indexed,
determining which ones are new or stale, and reading/writing embeddings
and associated metadata.
"""
alias Store.Project
alias Store.Project.Conversation
@type metadata :: %{
optional(String.t()) => any
}
@type status :: %{
new: [Conversation.t()],
stale: [Conversation.t()],
deleted: [String.t()]
}
@index_dir "conversations/index"
@embeddings_filename "embeddings.json"
@metadata_filename "metadata.json"
@spec root(Project.t()) :: String.t()
def root(%Project{store_path: store_path}) do
Path.join(store_path, @index_dir)
end
@spec path_for(Project.t(), String.t()) :: String.t()
def path_for(project, conversation_id) do
project
|> root()
|> Path.join(conversation_id)
end
@doc """
Returns the status of the conversation index for the given project.
It classifies conversations into:
* `:deleted` - indexed conversations whose source conversation no longer exists
* `:stale` - conversations whose index metadata is stale compared to the
on-disk conversation timestamp or embedding model
* `:new` - conversations that exist but have no index entry
"""
@spec index_status(Project.t()) :: status
def index_status(%Project{} = project) do
source_conversations = Conversation.list(project.store_path)
indexed_conversations =
project
|> root()
|> Path.join("*/#{@metadata_filename}")
|> Path.wildcard()
|> Enum.map(&Path.dirname/1)
|> Enum.map(&Path.basename/1)
source_by_id = Map.new(source_conversations, &{&1.id, &1})
indexed_ids = MapSet.new(indexed_conversations)
source_ids = MapSet.new(Map.keys(source_by_id))
new_ids = MapSet.difference(source_ids, indexed_ids)
deleted_ids = MapSet.difference(indexed_ids, source_ids)
last_updated =
source_conversations
|> Enum.map(fn c ->
ts = Conversation.timestamp(c)
ts_int = if is_integer(ts), do: ts, else: DateTime.to_unix(ts)
{c.id, ts_int}
end)
|> Map.new()
last_indexed =
source_conversations
|> Enum.map(fn convo ->
case read_metadata(project, convo.id) do
{:ok, %{"last_indexed_ts" => ts}} -> {convo.id, ts}
_ -> {convo.id, -1}
end
end)
|> Map.new()
stale_convs =
source_conversations
|> Enum.filter(fn convo ->
ts_updated = Map.get(last_updated, convo.id, 0)
ts_indexed = Map.get(last_indexed, convo.id, 0)
ts_indexed < ts_updated
end)
|> Enum.reject(fn convo -> not MapSet.member?(indexed_ids, convo.id) end)
|> Enum.sort_by(&Map.get(last_indexed, &1.id, 0))
%{
new:
new_ids
|> Enum.map(&Map.fetch!(source_by_id, &1)),
stale: stale_convs,
deleted: MapSet.to_list(deleted_ids)
}
end
@doc """
Writes embeddings and metadata for a conversation.
The embeddings are stored in `embeddings.json` and the metadata in
`metadata.json` under the conversation's index directory.
"""
@spec write_embeddings(Project.t(), String.t(), any, metadata) :: :ok | {:error, term}
def write_embeddings(%Project{} = project, conversation_id, embeddings, metadata) do
dir = path_for(project, conversation_id)
with :ok <- File.mkdir_p(dir),
:ok <- write_json(Path.join(dir, @embeddings_filename), embeddings),
:ok <- write_json(Path.join(dir, @metadata_filename), metadata) do
:ok
end
end
@doc """
Reads embeddings and metadata for a conversation.
Returns `{:ok, %{embeddings: embeddings, metadata: metadata}}` on success
or an error tuple if either file cannot be read/decoded.
"""
@spec read_embeddings(Project.t(), String.t()) ::
{:ok, %{embeddings: any, metadata: metadata}} | {:error, term}
def read_embeddings(%Project{} = project, conversation_id) do
dir = path_for(project, conversation_id)
embeddings_path = Path.join(dir, @embeddings_filename)
metadata_path = Path.join(dir, @metadata_filename)
with {:ok, embeddings} <- read_json(embeddings_path),
{:ok, metadata} <- read_json(metadata_path) do
{:ok, %{embeddings: embeddings, metadata: metadata}}
end
end
@doc """
Reads only the metadata for a conversation index entry.
"""
@spec read_metadata(Project.t(), String.t()) :: {:ok, metadata} | {:error, term}
def read_metadata(%Project{} = project, conversation_id) do
dir = path_for(project, conversation_id)
metadata_path = Path.join(dir, @metadata_filename)
read_json(metadata_path)
end
@doc """
Enumerates all indexed conversations, yielding `{id, embedding_vector, metadata}`.
"""
@spec all_embeddings(Project.t()) :: Enumerable.t()
def all_embeddings(%Project{} = project) do
project
|> root()
|> Path.join("*/#{@embeddings_filename}")
|> Path.wildcard()
|> Stream.map(fn path ->
id =
path
|> Path.dirname()
|> Path.basename()
embeddings_result = read_json(path)
metadata_result = read_metadata(project, id)
case {embeddings_result, metadata_result} do
{{:ok, embeddings}, {:ok, metadata}} -> {id, embeddings, metadata}
_ -> nil
end
end)
|> Stream.reject(&is_nil/1)
end
@doc """
Deletes the index entry for the given conversation id.
"""
@spec delete(Project.t(), String.t()) :: :ok
def delete(%Project{} = project, conversation_id) do
project
|> path_for(conversation_id)
|> File.rm_rf!()
:ok
end
# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------
@spec write_json(String.t(), any) :: :ok | {:error, term}
defp write_json(path, data) do
case SafeJson.encode(data) do
{:ok, json} -> File.write(path, json)
error -> error
end
end
@spec read_json(String.t()) :: {:ok, any} | {:error, term}
defp read_json(path) do
case File.read(path) do
{:ok, contents} -> SafeJson.decode(contents)
error -> error
end
end
end