Current section
Files
Jump to
Current section
Files
lib/mix/tasks/arcana.reembed.ex
defmodule Mix.Tasks.Arcana.Reembed do
@moduledoc """
Re-embeds all documents with the current embedding configuration.
Use this after switching embedding models or updating to a new version.
$ mix arcana.reembed
## Options
* `--batch-size` - Number of chunks to process at once (default: 50)
* `--quiet` - Suppress progress output
## Examples
# Default usage
mix arcana.reembed
# With larger batch size
mix arcana.reembed --batch-size 100
# Quiet mode (no progress bar)
mix arcana.reembed --quiet
"""
use Mix.Task
@shortdoc "Re-embeds all documents with current embedding config"
@impl Mix.Task
def run(args) do
{opts, _, _} =
OptionParser.parse(args,
strict: [batch_size: :integer, quiet: :boolean]
)
batch_size = Keyword.get(opts, :batch_size, 50)
quiet = Keyword.get(opts, :quiet, false)
# Start the host application (which will start the repo)
Mix.Task.run("app.start")
repo = Application.get_env(:arcana, :repo)
unless repo do
Mix.raise("No repo configured. Set config :arcana, repo: YourApp.Repo")
end
# Show current embedding info
info = Arcana.Maintenance.embedding_info()
Mix.shell().info("Embedding config: #{format_info(info)}")
# Build progress callback
progress_fn =
if quiet do
fn _, _ -> :ok end
else
build_progress_fn()
end
Mix.shell().info("Re-embedding chunks...")
{:ok, %{rechunked_documents: docs, total_chunks: total}} =
Arcana.Maintenance.reembed(repo, batch_size: batch_size, progress: progress_fn)
unless quiet, do: IO.puts("")
if docs > 0 do
Mix.shell().info("Rechunked #{docs} documents.")
end
Mix.shell().info("Done! #{total} chunks total.")
end
defp format_info(%{type: :local, model: model, dimensions: dims}) do
"local (#{model}, #{dims} dims)"
end
defp format_info(%{type: :openai, model: model, dimensions: dims}) do
"openai (#{model}, #{dims} dims)"
end
defp format_info(%{type: :custom, dimensions: dims}) do
"custom (#{dims} dims)"
end
defp format_info(%{type: type, dimensions: dims}) do
"#{type} (#{dims} dims)"
end
defp build_progress_fn do
# Simple progress indicator
fn current, total ->
percent = round(current / total * 100)
bar_width = 40
filled = round(percent / 100 * bar_width)
empty = bar_width - filled
bar = String.duplicate("=", filled) <> String.duplicate(" ", empty)
IO.write("\r[#{bar}] #{percent}% (#{current}/#{total})")
end
end
end