voidful commited on
Commit
f7821c8
·
verified ·
1 Parent(s): d3b84ba

Training completed: lora mode

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +4 -0
tokenizer_config.json CHANGED
@@ -164071,9 +164071,13 @@
164071
  "eos_token": "<|im_end|>",
164072
  "errors": "replace",
164073
  "extra_special_tokens": {},
 
164074
  "model_max_length": 1010000,
164075
  "pad_token": "<|endoftext|>",
164076
  "split_special_tokens": false,
 
164077
  "tokenizer_class": "Qwen2Tokenizer",
 
 
164078
  "unk_token": null
164079
  }
 
164071
  "eos_token": "<|im_end|>",
164072
  "errors": "replace",
164073
  "extra_special_tokens": {},
164074
+ "max_length": 4096,
164075
  "model_max_length": 1010000,
164076
  "pad_token": "<|endoftext|>",
164077
  "split_special_tokens": false,
164078
+ "stride": 0,
164079
  "tokenizer_class": "Qwen2Tokenizer",
164080
+ "truncation_side": "right",
164081
+ "truncation_strategy": "longest_first",
164082
  "unk_token": null
164083
  }