| 1234567891011121314151617181920 |
- {
- "bos_token": "<s>",
- "clean_up_tokenization_spaces": true,
- "cls_token": "<s>",
- "eos_token": "</s>",
- "mask_token": {
- "__type": "AddedToken",
- "content": "<mask>",
- "lstrip": true,
- "normalized": true,
- "rstrip": false,
- "single_word": false
- },
- "model_max_length": 512,
- "pad_token": "<pad>",
- "sep_token": "</s>",
- "sp_model_kwargs": {},
- "tokenizer_class": "XLMRobertaTokenizer",
- "unk_token": "<unk>"
- }
|
