Packages

iree_tokenizers

0.8.4
0.8.6 0.8.5 0.8.4 0.8.3 0.8.2 0.8.1 0.8.0 0.7.0 0.6.0 0.5.0 0.4.0 0.3.0 0.2.0 0.1.0 0.1.0-dev

Fast Hugging Face tokenizer.json, OpenAI tiktoken, and SentencePiece model bindings for Elixir via the IREE tokenizer runtime

HexDocs

Current section

Files

Jump to
Readme
15 Versions
4 Dependencies
Dependants
Files
Activity
iree_tokenizers test fixtures minimal_wordpiece_batch_longest_padded.json
Raw

test/fixtures/minimal_wordpiece_batch_longest_padded.json 

{

  "version": "1.0",

  "padding": {

    "strategy": "BatchLongest",

    "direction": "Left",

    "pad_to_multiple_of": null,

    "pad_id": 0,

    "pad_type_id": 0,

    "pad_token": "[PAD]"

  },

  "model": {

    "type": "WordPiece",

    "unk_token": "[UNK]",

    "continuing_subword_prefix": "##",

    "max_input_chars_per_word": 100,

    "vocab": {

      "[PAD]": 0,

      "[UNK]": 1,

      "hello": 2,

      "world": 3,

      "token": 4,

      "##s": 5,

      "more": 6,

      "text": 7

    }

  },

  "pre_tokenizer": {

    "type": "Whitespace"

  },

  "decoder": {

    "type": "WordPiece",

    "prefix": "##",

    "cleanup": false

  }

}