Current section

Files

Jump to
viva_tensor include viva_tensor@text@unigram_SentencePieceTokenizer.hrl
Raw

include/viva_tensor@text@unigram_SentencePieceTokenizer.hrl

-record(sentence_piece_tokenizer, {
mode :: viva_tensor@text@unigram:sentence_piece_mode(),
unigram :: gleam@option:option(viva_tensor@text@unigram:unigram_tokenizer()),
bpe :: gleam@option:option(viva_tensor@text@tokenizer:bpe_tokenizer())
}).