xberg

# This file is auto-generated by alef — DO NOT EDIT.
# alef:hash:0ba7524a2759609dc6f17c04fa9ae89940b7a540ae2e9a3708ac3fca410f8701
# To regenerate: alef generate
# To verify freshness: alef verify --exit-code
defmodule Xberg.ChunkSizing do
  @moduledoc """
  How chunk size is measured.

  Defaults to `Characters` (Unicode character count). When using token-based sizing,
  chunks are sized by token count according to the specified tokenizer.

  Token-based sizing uses HuggingFace tokenizers loaded at runtime, or a tokenizer
  backend you register yourself. Any tokenizer available on HuggingFace Hub can be
  used, including OpenAI-compatible tokenizers (e.g., `Xenova/gpt-4o`,
  `Xenova/cl100k_base`). To size chunks with your own tokenizer instead (llama.cpp/GGUF
  vocabularies, SentencePiece models, custom vocabs), register a `TokenizerBackend`
  with `register_tokenizer_backend` and set `model` to the registered name.
  """

  @typedoc "How chunk size is measured."
  @type t :: term()

  @typedoc "Size measured in Unicode characters (default)."
  @type characters :: :characters
  @typedoc "Size measured in tokens from a HuggingFace tokenizer or a registered tokenizer backend."
  @type tokenizer :: %{type: :tokenizer, model: String.t(), cache_dir: String.t()}

  def tokenizer(model, cache_dir), do: {:tokenizer, %{model: model, cache_dir: cache_dir}}
end