Packages

tokenizers

0.3.2
0.5.1 0.5.0 0.4.0 0.3.2 0.3.1 0.3.0 0.2.0 0.1.2 0.1.1 0.1.0

Bindings to Hugging Face Tokenizers for Elixir

HexDocs

Current section

Files

Jump to
Readme
10 Versions
3 Dependencies
Dependants
Files
Activity
tokenizers native ex_tokenizers Cargo.toml
Raw

native/ex_tokenizers/Cargo.toml 

[package]

name = "ex_tokenizers"

version = "0.3.1"

authors = []

edition = "2021"



[lib]

name = "ex_tokenizers"

path = "src/lib.rs"

crate-type = ["cdylib"]



[dependencies]

anyhow = "1"

rustler = "0.27.0"

thiserror = "1"

regex = "1.5.5"

tokenizers = { version = "0.13.3", default-features = false, features = ["onig", "esaxx_fast"]}