Current section

Files

Jump to
viva_tensor include viva_tensor@text@tokenizer_WhitespaceTokenizer.hrl
Raw

include/viva_tensor@text@tokenizer_WhitespaceTokenizer.hrl

-record(whitespace_tokenizer, {
vocab :: gleam@dict:dict(binary(), integer()),
inverse_vocab :: gleam@dict:dict(integer(), binary()),
unk_id :: integer(),
pad_id :: integer()
}).