tokenizer support BBPE

README.md

@@ -3,10 +3,13 @@ Dolly is a custom LLM built from scratch
 
 ## 🗓️ 更新日志
 
-### 2025-04-29
+#### 2025-04-30
+- 📝 添加Tokenizer的BBPE方式训练
+
+#### 2025-04-29
 - 📝 添加tokenizer构建代码，支持sentencepiece和transfomers方式，支持从文本构建和从已有的tokenzier构建。
 
-### 2025-04-24
+#### 2025-04-24
 - ✅ 测试从transformers构建自定义的LLM模型结构。
 
 ## 致谢

Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py

@@ -20,4 +20,4 @@ tokenizer = old_tokenizer.train_new_from_iterator(
 )
 
 
-tokenizer.save_pretrained("new_tokenizer2")
+tokenizer.save_pretrained("tokenizer_file/new_tokenizer2")

Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py

@@ -1,11 +1,13 @@
 from tokenizers import Tokenizer
 from tokenizers.models import BPE  
 from tokenizers.trainers import BpeTrainer
+from tokenizers.pre_tokenizers import ByteLevel
 from tokenizers.pre_tokenizers import Whitespace
 from transformers import PreTrainedTokenizerFast
 
 tokenizer = Tokenizer(BPE(unk_token="[UNK]"))  
-tokenizer.pre_tokenizer = Whitespace()
+# tokenizer.pre_tokenizer = Whitespace()
+tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True)
 trainer = BpeTrainer(
     vocab_size=30000,           
     special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],  
@@ -23,4 +25,4 @@ fast_tokenizer = PreTrainedTokenizerFast(
     sep_token="[SEP]",
     mask_token="[MASK]",
 )
-fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer")
+fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer_BBPE")