Packages

iree_tokenizers

0.8.3
0.8.6 0.8.5 0.8.4 0.8.3 0.8.2 0.8.1 0.8.0 0.7.0 0.6.0 0.5.0 0.4.0 0.3.0 0.2.0 0.1.0 0.1.0-dev

Fast Hugging Face tokenizer.json, OpenAI tiktoken, and SentencePiece model bindings for Elixir via the IREE tokenizer runtime

HexDocs

Current section

Files

Jump to
Readme
15 Versions
4 Dependencies
Dependants
Files
Activity
iree_tokenizers test fixtures bytelevel_utf8_split.json
Raw

test/fixtures/bytelevel_utf8_split.json 

{

  "model": {

    "type": "BPE",

    "vocab": {

      "ð": 0,

      "Ł": 1,

      "ļ": 2,

      "Ģ": 3,

      "ðŁ": 4,

      "ļĢ": 5

    },

    "merges": [

      "ð Ł",

      "ļ Ģ"

    ]

  },

  "pre_tokenizer": {

    "type": "ByteLevel",

    "add_prefix_space": false,

    "trim_offsets": false,

    "use_regex": true

  },

  "decoder": {

    "type": "ByteLevel",

    "add_prefix_space": false,

    "trim_offsets": false

  }

}