dspex

# Generated by SnakeBridge v0.14.0 - DO NOT EDIT MANUALLY
# Regenerate with: mix compile
# Library: dspy 3.1.2
# Python module: dspy.dsp.utils
# Python class: Tokenizer

defmodule Dspy.Dsp.Utils.Tokenizer do
  @moduledoc """
  Base tokenizer class.

  Tokenizers implement tokenize, which should return a Tokens class.
  """
  def __snakebridge_python_name__, do: "dspy.dsp.utils"
  def __snakebridge_python_class__, do: "Tokenizer"
  def __snakebridge_library__, do: "dspy"
  @opaque t :: SnakeBridge.Ref.t()

  @doc """
  Initialize self.  See help(type(self)) for accurate signature.

  ## Parameters

  - `args` (term())
  - `kwargs` (term())
  """
  @spec new(list(term()), keyword()) :: {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}
  def new(args, opts \\ []) do
    {args, opts} = SnakeBridge.Runtime.normalize_args_opts(args, opts)
    SnakeBridge.Runtime.call_class(__MODULE__, :__init__, [] ++ List.wrap(args), opts)
  end

  @doc """
  Python method `Tokenizer.shutdown`.

  ## Returns

  - `term()`
  """
  @spec shutdown(SnakeBridge.Ref.t(), keyword()) :: {:ok, term()} | {:error, Snakepit.Error.t()}
  def shutdown(ref, opts \\ []) do
    SnakeBridge.Runtime.call_method(ref, :shutdown, [], opts)
  end

  @doc """
  Python method `Tokenizer.tokenize`.

  ## Parameters

  - `text` (term())

  ## Returns

  - `term()`
  """
  @spec tokenize(SnakeBridge.Ref.t(), term(), keyword()) ::
          {:ok, term()} | {:error, Snakepit.Error.t()}
  def tokenize(ref, text, opts \\ []) do
    SnakeBridge.Runtime.call_method(ref, :tokenize, [text], opts)
  end
end