Packages

tokenizers

0.5.1
0.5.1 0.5.0 0.4.0 0.3.2 0.3.1 0.3.0 0.2.0 0.1.2 0.1.1 0.1.0

Bindings to Hugging Face Tokenizers for Elixir

HexDocs

Current section

Files

Jump to
Readme
10 Versions
3 Dependencies
Dependants
Files
Activity
tokenizers native ex_tokenizers src lib.rs
Raw

native/ex_tokenizers/src/lib.rs 

mod added_token;

mod decoders;

mod encoding;

mod error;

mod models;

mod normalizers;

mod post_processors;

mod pre_tokenizers;

mod tokenizer;

mod trainers;

mod util;



use rustler::{Env, Term};



pub use error::ExTokenizersError;



fn on_load(_env: Env, _info: Term) -> bool {

    true

}



rustler::init!("Elixir.Tokenizers.Native", load = on_load);