Packages

tokenizers

0.5.1
0.5.1 0.5.0 0.4.0 0.3.2 0.3.1 0.3.0 0.2.0 0.1.2 0.1.1 0.1.0

Bindings to Hugging Face Tokenizers for Elixir

HexDocs

Current section

Files

Jump to
Readme
10 Versions
3 Dependencies
11 Dependants
Files
Activity
tokenizers native ex_tokenizers Cargo.toml
Raw

native/ex_tokenizers/Cargo.toml 

[package]

name = "ex_tokenizers"

version = "0.1.0"

authors = []

edition = "2021"



[lib]

name = "ex_tokenizers"

path = "src/lib.rs"

crate-type = ["cdylib"]



[dependencies]

anyhow = "1"

rustler = "0.34.0"

thiserror = "1"

tokenizers = { version = "0.20.0", default-features = false, features = ["onig", "esaxx_fast"]}

serde = { version = "1.0", features = [ "rc", "derive" ] }