| { |
| "add_prefix_space": false, |
| "backend": "tokenizers", |
| "bos_token": "<|endoftext|>", |
| "clean_up_tokenization_spaces": true, |
| "eos_token": "<|endoftext|>", |
| "errors": "replace", |
| "extra_special_tokens": [ |
| "<|endoftext|>", |
| "<fim_prefix>", |
| "<fim_middle>", |
| "<fim_suffix>", |
| "<fim_pad>", |
| "<repo_name>", |
| "<file_sep>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<jupyter_script>", |
| "<empty_output>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<NAME>", |
| "<EMAIL>", |
| "<KEY>", |
| "<PASSWORD>" |
| ], |
| "is_local": false, |
| "model_max_length": 1000000000000000019884624838656, |
| "pad_token": "<|endoftext|>", |
| "tokenizer_class": "GPT2Tokenizer", |
| "unk_token": "<|endoftext|>", |
| "vocab_size": 49152 |
| } |
|
|
