add trained BBPE Tokenizer model

Tokenizer_code/README.md

@@ -1,2 +1,3 @@
 # USAGE
-使用sentencepiece和huggingface transformers分别进行tokenizer的训练
+使用sentencepiece和huggingface transformers分别进行tokenizer的训练
+support BPE and BBPE

Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py

@@ -9,7 +9,7 @@ tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
 # tokenizer.pre_tokenizer = Whitespace()
 tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True)
 trainer = BpeTrainer(
-    vocab_size=30000,           
+    vocab_size=12800,           
     special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],  
     min_frequency=2,            
     show_progress=True         

tokenizer_file/hf_tokenizer_BBPE/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_file/hf_tokenizer_BBPE/tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizer",
+  "unk_token": "[UNK]"
+}