Current section

Files

Jump to
arcana lib mix tasks arcana.gen.embedding_migration.ex
Raw

lib/mix/tasks/arcana.gen.embedding_migration.ex

defmodule Mix.Tasks.Arcana.Gen.EmbeddingMigration do
@moduledoc """
Generates a migration to update vector column dimensions.
Use this when switching to an embedding model with different dimensions.
$ mix arcana.gen.embedding_migration
The task will:
1. Detect the current embedding configuration dimensions
2. Show the detected dimensions
3. Generate a migration to update the vector column
## Options
* `--dimensions` - Override auto-detected dimensions
"""
use Mix.Task
@shortdoc "Generates a migration for embedding dimension changes"
@impl Mix.Task
def run(args) do
{opts, _, _} =
OptionParser.parse(args,
strict: [dimensions: :integer]
)
# Start the app to access config
Mix.Task.run("app.config")
dimensions =
case Keyword.get(opts, :dimensions) do
nil ->
case detect_dimensions() do
{:ok, dims} ->
Mix.shell().info("Detected embedding dimensions: #{dims}")
dims
{:error, reason} ->
Mix.raise("Could not detect dimensions: #{inspect(reason)}")
end
dims ->
Mix.shell().info("Using specified dimensions: #{dims}")
dims
end
generate_migration(dimensions)
end
defp detect_dimensions do
embedder = Arcana.embedder()
{:ok, Arcana.Embedder.dimensions(embedder)}
rescue
e -> {:error, e}
end
defp generate_migration(dimensions) do
timestamp = Calendar.strftime(DateTime.utc_now(), "%Y%m%d%H%M%S")
filename = "#{timestamp}_update_embedding_dimensions.exs"
# Find the priv/repo/migrations directory
repo_config = Application.get_env(:arcana, Application.get_env(:arcana, :repo))
priv_dir = Keyword.get(repo_config, :priv, "priv/repo")
migrations_dir = Path.join(priv_dir, "migrations")
# Ensure the directory exists
File.mkdir_p!(migrations_dir)
path = Path.join(migrations_dir, filename)
content = migration_content(dimensions)
File.write!(path, content)
Mix.shell().info("""
Generated migration: #{path}
This migration will:
1. Drop the HNSW index on the embedding column
2. Alter the embedding column to #{dimensions} dimensions
3. Recreate the HNSW index
Run the migration with:
mix ecto.migrate
After migrating, re-embed all documents with:
mix arcana.reembed
""")
end
defp migration_content(dimensions) do
"""
defmodule Arcana.Repo.Migrations.UpdateEmbeddingDimensions do
use Ecto.Migration
def up do
# Drop the existing HNSW index
drop_if_exists index(:arcana_chunks, [:embedding])
# Alter the embedding column to new dimensions
alter table(:arcana_chunks) do
modify :embedding, :vector, size: #{dimensions}
end
# Recreate the HNSW index with the new dimensions
create index(:arcana_chunks, [:embedding],
using: :hnsw,
options: "vector_cosine_ops"
)
end
def down do
# Note: Down migration requires knowing the previous dimensions
# This is a best-effort reversal - you may need to adjust the size
drop_if_exists index(:arcana_chunks, [:embedding])
alter table(:arcana_chunks) do
modify :embedding, :vector, size: 384
end
create index(:arcana_chunks, [:embedding],
using: :hnsw,
options: "vector_cosine_ops"
)
end
end
"""
end
end