tokenizer support BBPE

Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py

@@ -20,4 +20,4 @@ tokenizer = old_tokenizer.train_new_from_iterator(
 )
 
 
-tokenizer.save_pretrained("new_tokenizer2")
+tokenizer.save_pretrained("tokenizer_file/new_tokenizer2")

Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py

@@ -1,11 +1,13 @@
 from tokenizers import Tokenizer
 from tokenizers.models import BPE  
 from tokenizers.trainers import BpeTrainer
+from tokenizers.pre_tokenizers import ByteLevel
 from tokenizers.pre_tokenizers import Whitespace
 from transformers import PreTrainedTokenizerFast
 
 tokenizer = Tokenizer(BPE(unk_token="[UNK]"))  
-tokenizer.pre_tokenizer = Whitespace()
+# tokenizer.pre_tokenizer = Whitespace()
+tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True)
 trainer = BpeTrainer(
     vocab_size=30000,           
     special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],  
@@ -23,4 +25,4 @@ fast_tokenizer = PreTrainedTokenizerFast(
     sep_token="[SEP]",
     mask_token="[MASK]",
 )
-fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer")
+fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer_BBPE")