Packages
fnord
0.4.43
0.9.40
0.9.39
0.9.38
0.9.37
0.9.36
0.9.35
0.9.34
0.9.33
0.9.32
0.9.31
0.9.30
0.9.29
0.9.28
0.9.27
0.9.26
0.9.25
0.9.24
0.9.23
0.9.22
0.9.21
0.9.20
0.9.19
0.9.18
0.9.17
0.9.16
0.9.15
0.9.14
0.9.13
0.9.12
0.9.11
0.9.10
0.9.9
0.9.8
0.9.7
0.9.6
0.9.5
0.9.4
0.9.3
0.9.2
0.9.1
0.9.0
0.8.99
0.8.98
0.8.97
0.8.96
0.8.95
0.8.94
0.8.93
0.8.92
0.8.91
0.8.90
0.8.89
0.8.88
0.8.87
0.8.86
0.8.85
0.8.84
0.8.83
0.8.82
0.8.81
0.8.80
0.8.79
0.8.78
0.8.77
0.8.76
0.8.75
0.8.74
0.8.73
0.8.72
0.8.71
0.8.70
0.8.69
0.8.68
0.8.67
0.8.66
0.8.65
0.8.64
0.8.63
0.8.62
0.8.61
0.8.60
0.8.59
0.8.58
0.8.57
0.8.56
0.8.55
0.8.54
0.8.53
0.8.52
0.8.51
0.8.50
0.8.49
0.8.48
0.8.47
0.8.46
0.8.45
0.8.44
0.8.43
0.8.42
0.8.41
0.8.40
0.8.39
0.8.38
0.8.37
0.8.36
0.8.35
0.8.34
0.8.33
0.8.32
0.8.31
0.8.30
0.8.29
0.8.27
0.8.26
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.1
0.8.0
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.3
0.7.2
0.7.1
0.7.0
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.1
0.6.0
0.5.9
0.5.8
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.44
0.4.43
0.4.42
0.4.41
0.4.40
0.4.39
0.4.38
0.4.37
0.4.36
0.4.35
0.4.34
0.4.33
0.4.32
0.4.30
0.4.29
0.4.28
0.4.27
0.4.26
0.4.25
0.4.24
0.4.23
0.4.22
0.4.21
0.4.20
0.4.19
0.4.18
0.4.17
0.4.16
0.4.15
0.4.14
0.4.13
0.4.12
0.4.11
0.4.10
0.4.9
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.0
0.2.0
0.1.0
AI code archaeology
Current section
Files
Jump to
Current section
Files
lib/store/embeddings.ex
defmodule Store.Embeddings do
defstruct [:store_path, :source_file]
@filename "embeddings.json"
@behaviour Store.EntryFile
@impl Store.EntryFile
def new(entry_path, source_file) do
entry = %__MODULE__{
store_path: Path.join(entry_path, @filename),
source_file: source_file
}
if has_old_style_embeddings?(entry) do
entry
|> upgrade_old_style_embeddings()
|> case do
{:error, reason} ->
UI.warn("Error upgrading embeddings format", "#{entry.source_file}: #{inspect(reason)}")
_ ->
nil
end
end
entry
end
@impl Store.EntryFile
def store_path(entry), do: entry.store_path
@impl Store.EntryFile
def exists?(entry), do: File.exists?(entry.store_path)
@impl Store.EntryFile
def read(entry) do
entry
|> store_path()
|> File.read()
|> case do
{:ok, contents} -> Jason.decode(contents)
error -> error
end
end
@impl Store.EntryFile
def write(entry, embeddings) do
# Embeddings used to be stored separately for each chunk of the file (when
# the file was too large for the model to process in one go). This is no
# longer the case, so we need to remove the old embedding files, which were
# named embeddings_N.json.
entry.store_path
|> Path.dirname()
|> Path.join("embeddings_*.json")
|> Path.wildcard()
|> Enum.each(&File.rm_rf!/1)
embeddings
# For each dimension, find the maximum value across all embeddings. This
# isn't necessarily the _most_ accurate, but it selects the highest rating
# for each dimension found in the file, which should be reasonable for
# semantic searching.
|> Enum.zip_with(&Enum.max/1)
|> Jason.encode()
|> case do
{:ok, json} -> File.write(entry.store_path, json)
error -> error
end
end
defp get_old_style_embeddings(entry) do
entry.store_path
|> Path.dirname()
|> Path.join("embedding_*.json")
|> Path.wildcard()
end
defp has_old_style_embeddings?(entry) do
count =
entry
|> get_old_style_embeddings()
|> Enum.count()
count > 0
end
defp upgrade_old_style_embeddings(entry) do
UI.debug("Upgrading embeddings format", entry.source_file)
# First, find all of the old-style embeddings files.
embeddings_files = get_old_style_embeddings(entry)
# Next, read the contents of each file and combine them, selecting the max
# value for each position.
combined_embeddings =
embeddings_files
|> Enum.map(&File.read!/1)
|> Enum.map(&Jason.decode!/1)
|> Enum.zip_with(&Enum.max/1)
# Write the combined embeddings to the new file.
with {:ok, json} <- Jason.encode(combined_embeddings),
:ok <- File.write(entry.store_path, json) do
# Lastly, delete the old embeddings files.
embeddings_files
|> Enum.each(fn path ->
UI.debug("Deleting old-style embeddings file", path)
File.rm_rf!(path)
end)
:ok
else
error ->
UI.warn("Error upgrading embeddings format", inspect(error))
{:error, error}
end
end
end