add trained BBPE Tokenizer model

Tokenizer_code/README.md

@@ -1,2 +1,3 @@
 # USAGE
-使用sentencepiece和huggingface transformers分别进行tokenizer的训练
+使用sentencepiece和huggingface transformers分别进行tokenizer的训练
+support BPE and BBPE

Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py

@@ -9,7 +9,7 @@ tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
 # tokenizer.pre_tokenizer = Whitespace()
 tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True)
 trainer = BpeTrainer(
-    vocab_size=30000,           
+    vocab_size=12800,           
     special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],  
     min_frequency=2,            
     show_progress=True