{
  "transformers": "5.18.0",
  "tokenizers": "0.23.2",
  "add_special_tokens": false,
  "records": [
    {
      "repo": "Qwen/Qwen3-30B-A3B-Instruct-2507",
      "revision": "0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe",
      "loader": "AutoTokenizer.from_pretrained local hash-verified pinned files, trust_remote_code=False",
      "class": "Qwen2Tokenizer",
      "is_fast": true,
      "examples": [
        {
          "text": "zzz",
          "utf8_hex": "7a7a7a",
          "n_tokens": 2,
          "ids": [
            10400,
            89
          ],
          "pieces": [
            "zz",
            "z"
          ]
        },
        {
          "text": "______",
          "utf8_hex": "5f5f5f5f5f5f",
          "n_tokens": 2,
          "ids": [
            2130,
            563
          ],
          "pieces": [
            "____",
            "__"
          ]
        },
        {
          "text": "________",
          "utf8_hex": "5f5f5f5f5f5f5f5f",
          "n_tokens": 1,
          "ids": [
            3979
          ],
          "pieces": [
            "________"
          ]
        },
        {
          "text": "aaaaaaaa",
          "utf8_hex": "6161616161616161",
          "n_tokens": 1,
          "ids": [
            69440
          ],
          "pieces": [
            "aaaaaaaa"
          ]
        },
        {
          "text": "aaaaaaaaaaaaaaaa",
          "utf8_hex": "61616161616161616161616161616161",
          "n_tokens": 2,
          "ids": [
            69440,
            69440
          ],
          "pieces": [
            "aaaaaaaa",
            "aaaaaaaa"
          ]
        }
      ]
    },
    {
      "repo": "Qwen/Qwen3-Next-80B-A3B-Instruct",
      "revision": "9c7f2fbe84465e40164a94cc16cd30b6999b0cc7",
      "loader": "AutoTokenizer.from_pretrained local hash-verified pinned files, trust_remote_code=False",
      "class": "Qwen2Tokenizer",
      "is_fast": true,
      "examples": [
        {
          "text": "zzz",
          "utf8_hex": "7a7a7a",
          "n_tokens": 2,
          "ids": [
            10400,
            89
          ],
          "pieces": [
            "zz",
            "z"
          ]
        },
        {
          "text": "______",
          "utf8_hex": "5f5f5f5f5f5f",
          "n_tokens": 2,
          "ids": [
            2130,
            563
          ],
          "pieces": [
            "____",
            "__"
          ]
        },
        {
          "text": "________",
          "utf8_hex": "5f5f5f5f5f5f5f5f",
          "n_tokens": 1,
          "ids": [
            3979
          ],
          "pieces": [
            "________"
          ]
        },
        {
          "text": "aaaaaaaa",
          "utf8_hex": "6161616161616161",
          "n_tokens": 1,
          "ids": [
            69440
          ],
          "pieces": [
            "aaaaaaaa"
          ]
        },
        {
          "text": "aaaaaaaaaaaaaaaa",
          "utf8_hex": "61616161616161616161616161616161",
          "n_tokens": 2,
          "ids": [
            69440,
            69440
          ],
          "pieces": [
            "aaaaaaaa",
            "aaaaaaaa"
          ]
        }
      ]
    },
    {
      "repo": "Qwen/Qwen3-0.6B",
      "revision": "c1899de289a04d12100db370d81485cdf75e47ca",
      "loader": "AutoTokenizer public current default Space model, revision pinned at retrieval, no remote code",
      "class": "Qwen2Tokenizer",
      "is_fast": true,
      "examples": [
        {
          "text": "zzz",
          "utf8_hex": "7a7a7a",
          "n_tokens": 2,
          "ids": [
            10400,
            89
          ],
          "pieces": [
            "zz",
            "z"
          ]
        },
        {
          "text": "______",
          "utf8_hex": "5f5f5f5f5f5f",
          "n_tokens": 2,
          "ids": [
            2130,
            563
          ],
          "pieces": [
            "____",
            "__"
          ]
        },
        {
          "text": "________",
          "utf8_hex": "5f5f5f5f5f5f5f5f",
          "n_tokens": 1,
          "ids": [
            3979
          ],
          "pieces": [
            "________"
          ]
        },
        {
          "text": "aaaaaaaa",
          "utf8_hex": "6161616161616161",
          "n_tokens": 1,
          "ids": [
            69440
          ],
          "pieces": [
            "aaaaaaaa"
          ]
        },
        {
          "text": "aaaaaaaaaaaaaaaa",
          "utf8_hex": "61616161616161616161616161616161",
          "n_tokens": 2,
          "ids": [
            69440,
            69440
          ],
          "pieces": [
            "aaaaaaaa",
            "aaaaaaaa"
          ]
        }
      ]
    }
  ]
}