Packages

fastest_tiktoken

0.1.1
0.1.2 0.1.1

Fast Elixir bindings for the pure-Rust tiktoken crate

Current section

Files

Jump to
Readme
2 Versions
2 Dependencies
Dependants
Files
Activity
fastest_tiktoken lib fastest_tiktoken native.ex
Raw

lib/fastest_tiktoken/native.ex 

defmodule FastestTiktoken.Native do

  @moduledoc false



  version = Mix.Project.config()[:version]

  checksum_file = Path.expand("../../checksum-Elixir.FastestTiktoken.Native.exs", __DIR__)



  use RustlerPrecompiled,

    otp_app: :fastest_tiktoken,

    crate: "fastest_tiktoken",

    base_url: "https://github.com/caudena/fastest_tiktoken/releases/download/v#{version}",

    force_build:

      System.get_env("FASTEST_TIKTOKEN_BUILD") in ["1", "true"] or not File.exists?(checksum_file),

    targets: ~w(

      aarch64-apple-darwin

      aarch64-unknown-linux-gnu

      x86_64-apple-darwin

      x86_64-pc-windows-msvc

      x86_64-unknown-linux-gnu

      x86_64-unknown-linux-musl

    ),

    nif_versions: ["2.15"],

    version: version



  def list_encodings, do: :erlang.nif_error(:nif_not_loaded)

  def encoding_for_model(_model), do: :erlang.nif_error(:nif_not_loaded)

  def encoding_exists(_encoding), do: :erlang.nif_error(:nif_not_loaded)



  def encode(_encoding, _text, _special_mode, _allowed_special),

    do: :erlang.nif_error(:nif_not_loaded)



  def decode(_encoding, _ids), do: :erlang.nif_error(:nif_not_loaded)



  def count_tokens(_encoding, _text, _special_mode, _allowed_special),

    do: :erlang.nif_error(:nif_not_loaded)



  def split_tokens(_encoding, _text, _special_mode, _allowed_special),

    do: :erlang.nif_error(:nif_not_loaded)

end