iree_tokenizers

{
  "model": {
    "type": "BPE",
    "vocab": {
      "h": 0,
      "e": 1,
      "l": 2,
      "o": 3,
      "Ġ": 4,
      "1": 5,
      "2": 6,
      "3": 7,
      "4": 8,
      "5": 9,
      "6": 10,
      "7": 11,
      "he": 12,
      "hel": 13,
      "hell": 14,
      "hello": 15,
      "Ġ1": 16,
      "23": 17,
      "45": 18,
      "456": 19,
      "123": 20,
      "<BOS_TOKEN>": 21
    },
    "merges": [
      "h e",
      "he l",
      "hel l",
      "hell o",
      "Ġ 1",
      "2 3",
      "4 5",
      "45 6",
      "1 23"
    ]
  },
  "pre_tokenizer": {
    "type": "Sequence",
    "pretokenizers": [
      {
        "type": "Split",
        "pattern": {
          "Regex": "\\d{1,3}(?=(?:\\d{3})*\\b)"
        },
        "behavior": "Isolated",
        "invert": false
      },
      {
        "type": "Split",
        "pattern": {
          "Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
        },
        "behavior": "Isolated",
        "invert": false
      },
      {
        "type": "ByteLevel",
        "add_prefix_space": false,
        "trim_offsets": true,
        "use_regex": false
      }
    ]
  },
  "post_processor": {
    "add_prefix_space": true,
    "trim_offsets": false,
    "use_regex": true,
    "type": "TemplateProcessing",
    "single": [
      {
        "SpecialToken": {
          "id": "<BOS_TOKEN>",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      }
    ],
    "pair": [
      {
        "SpecialToken": {
          "id": "<BOS_TOKEN>",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "B",
          "type_id": 1
        }
      }
    ],
    "special_tokens": {
      "<BOS_TOKEN>": {
        "id": "<BOS_TOKEN>",
        "ids": [
          21
        ],
        "tokens": [
          "<BOS_TOKEN>"
        ]
      }
    }
  }
}