| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455 |
- {
- "added_tokens_decoder": {
- "0": {
- "content": "<s>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "1": {
- "content": "<pad>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "2": {
- "content": "</s>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "3": {
- "content": "<unk>",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- },
- "250001": {
- "content": "<mask>",
- "lstrip": true,
- "normalized": false,
- "rstrip": false,
- "single_word": false,
- "special": true
- }
- },
- "bos_token": "<s>",
- "clean_up_tokenization_spaces": true,
- "cls_token": "<s>",
- "eos_token": "</s>",
- "mask_token": "<mask>",
- "model_max_length": 8192,
- "pad_token": "<pad>",
- "sep_token": "</s>",
- "sp_model_kwargs": {},
- "tokenizer_class": "XLMRobertaTokenizer",
- "unk_token": "<unk>"
- }
|