Current section
Files
Jump to
Current section
Files
lib/mix/tasks/arcana.reembed_chunks.ex
defmodule Mix.Tasks.Arcana.ReembedChunks do
@moduledoc """
Re-embeds all chunks with the current embedding configuration.
Use this after switching embedding models or updating to a new version.
$ mix arcana.reembed_chunks
## Options
* `--collection` - Only re-embed chunks in the specified collection
* `--batch-size` - Number of chunks to process at once (default: 50)
* `--concurrency N` - Number of parallel embedding requests (default: 5)
* `--skip N` - Skip first N chunks (for resuming interrupted runs)
* `--quiet` - Suppress progress output
## Examples
# Default usage (all collections)
mix arcana.reembed_chunks
# Re-embed only a specific collection
mix arcana.reembed_chunks --collection my-docs
# With larger batch size and higher concurrency
mix arcana.reembed_chunks --batch-size 100 --concurrency 10
# Resume from chunk 500
mix arcana.reembed_chunks --skip 500
# Quiet mode (no progress bar)
mix arcana.reembed_chunks --quiet
"""
use Mix.Task
@shortdoc "Re-embeds all chunks with current embedding config"
@impl Mix.Task
def run(args) do
{opts, _, _} =
OptionParser.parse(args,
strict: [
batch_size: :integer,
quiet: :boolean,
collection: :string,
concurrency: :integer,
skip: :integer
]
)
batch_size = Keyword.get(opts, :batch_size, 50)
quiet = Keyword.get(opts, :quiet, false)
collection = Keyword.get(opts, :collection)
concurrency = Keyword.get(opts, :concurrency, 5)
skip = Keyword.get(opts, :skip, 0)
# Start the host application (which will start the repo)
Mix.Task.run("app.start")
repo = Application.get_env(:arcana, :repo)
unless repo do
Mix.raise("No repo configured. Set config :arcana, repo: YourApp.Repo")
end
# Show current embedding info
info = Arcana.Maintenance.embedding_info()
Mix.shell().info("Embedding config: #{format_info(info)}")
# Build progress callback
progress_fn =
if quiet do
fn _, _ -> :ok end
else
build_progress_fn()
end
scope = if collection, do: "collection '#{collection}'", else: "all collections"
skip_msg = if skip > 0, do: " (skipping first #{skip})", else: ""
Mix.shell().info("Re-embedding chunks in #{scope}#{skip_msg}...")
reembed_opts = [
batch_size: batch_size,
progress: progress_fn,
concurrency: concurrency,
skip: skip
]
reembed_opts =
if collection, do: Keyword.put(reembed_opts, :collection, collection), else: reembed_opts
{:ok, %{rechunked_documents: docs, total_chunks: total, skipped: skipped}} =
Arcana.Maintenance.reembed(repo, reembed_opts)
unless quiet, do: IO.puts("")
if docs > 0 do
Mix.shell().info("Rechunked #{docs} documents.")
end
skip_info = if skipped > 0, do: " (#{skipped} skipped)", else: ""
Mix.shell().info("Done! #{total} chunks total#{skip_info}.")
end
defp format_info(%{type: :local, model: model, dimensions: dims}) do
"local (#{model}, #{dims} dims)"
end
defp format_info(%{type: :openai, model: model, dimensions: dims}) do
"openai (#{model}, #{dims} dims)"
end
defp format_info(%{type: :custom, dimensions: dims}) do
"custom (#{dims} dims)"
end
defp format_info(%{type: type, dimensions: dims}) do
"#{type} (#{dims} dims)"
end
defp build_progress_fn do
# Simple progress indicator
fn current, total ->
percent = round(current / total * 100)
bar_width = 40
filled = round(percent / 100 * bar_width)
empty = bar_width - filled
bar = String.duplicate("=", filled) <> String.duplicate(" ", empty)
IO.write("\r[#{bar}] #{percent}% (#{current}/#{total})")
end
end
end