dspex

# Generated by SnakeBridge v0.14.0 - DO NOT EDIT MANUALLY
# Regenerate with: mix compile
# Library: dspy 3.1.2
# Python module: dspy.dsp.utils
# Python class: SimpleTokenizer

defmodule Dspy.Dsp.Utils.SimpleTokenizer do
  @moduledoc """
  Base tokenizer class.

  Tokenizers implement tokenize, which should return a Tokens class.
  """
  def __snakebridge_python_name__, do: "dspy.dsp.utils"
  def __snakebridge_python_class__, do: "SimpleTokenizer"
  def __snakebridge_library__, do: "dspy"
  @opaque t :: SnakeBridge.Ref.t()

  @doc """
  Args:

  annotators: None or empty set (only tokenizes).

  ## Parameters

  - `kwargs` (term())
  """
  @spec new(keyword()) :: {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}
  def new(opts \\ []) do
    SnakeBridge.Runtime.call_class(__MODULE__, :__init__, [], opts)
  end

  @doc """
  Python method `SimpleTokenizer.shutdown`.

  ## Returns

  - `term()`
  """
  @spec shutdown(SnakeBridge.Ref.t(), keyword()) :: {:ok, term()} | {:error, Snakepit.Error.t()}
  def shutdown(ref, opts \\ []) do
    SnakeBridge.Runtime.call_method(ref, :shutdown, [], opts)
  end

  @doc """
  Python method `SimpleTokenizer.tokenize`.

  ## Parameters

  - `text` (term())

  ## Returns

  - `term()`
  """
  @spec tokenize(SnakeBridge.Ref.t(), term(), keyword()) ::
          {:ok, term()} | {:error, Snakepit.Error.t()}
  def tokenize(ref, text, opts \\ []) do
    SnakeBridge.Runtime.call_method(ref, :tokenize, [text], opts)
  end

  @spec alpha_num(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}
  def alpha_num(ref) do
    SnakeBridge.Runtime.get_attr(ref, :ALPHA_NUM)
  end

  @spec non_ws(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}
  def non_ws(ref) do
    SnakeBridge.Runtime.get_attr(ref, :NON_WS)
  end
end