From c5c5a560dfc2eebd602bdccc48fd2c973b8207fa Mon Sep 17 00:00:00 2001 From: coffee Date: Wed, 30 Apr 2025 16:38:18 +0800 Subject: [PATCH] tokenizer support BBPE --- README.md | 7 +++++-- .../step1.1_hf_train_from_old_largefile.py | 2 +- ...cratch.py => step2_hf_train_from_scratch_BPEandBBPE.py} | 6 ++++-- 3 files changed, 10 insertions(+), 5 deletions(-) rename Tokenizer_code/hf_tokenizer_code/{step2_hf_train_from_scratch.py => step2_hf_train_from_scratch_BPEandBBPE.py} (78%) diff --git a/README.md b/README.md index 688ede3..db9f013 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,13 @@ Dolly is a custom LLM built from scratch ## 🗓️ 更新日志 -### 2025-04-29 +#### 2025-04-30 +- 📝 添加Tokenizer的BBPE方式训练 + +#### 2025-04-29 - 📝 添加tokenizer构建代码，支持sentencepiece和transfomers方式，支持从文本构建和从已有的tokenzier构建。 -### 2025-04-24 +#### 2025-04-24 - ✅ 测试从transformers构建自定义的LLM模型结构。 ## 致谢 diff --git a/Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py b/Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py index 28a11fb..bb5257c 100644 --- a/Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py +++ b/Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py @@ -20,4 +20,4 @@ tokenizer = old_tokenizer.train_new_from_iterator( ) -tokenizer.save_pretrained("new_tokenizer2") \ No newline at end of file +tokenizer.save_pretrained("tokenizer_file/new_tokenizer2") \ No newline at end of file diff --git a/Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py b/Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py similarity index 78% rename from Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py rename to Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py index e9bef44..e9d5fec 100644 --- a/Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py +++ b/Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py @@ -1,11 +1,13 @@ from tokenizers import Tokenizer from tokenizers.models import BPE from tokenizers.trainers import BpeTrainer +from tokenizers.pre_tokenizers import ByteLevel from tokenizers.pre_tokenizers import Whitespace from transformers import PreTrainedTokenizerFast tokenizer = Tokenizer(BPE(unk_token="[UNK]")) -tokenizer.pre_tokenizer = Whitespace() +# tokenizer.pre_tokenizer = Whitespace() +tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True) trainer = BpeTrainer( vocab_size=30000, special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"], @@ -23,4 +25,4 @@ fast_tokenizer = PreTrainedTokenizerFast( sep_token="[SEP]", mask_token="[MASK]", ) -fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer") \ No newline at end of file +fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer_BBPE") \ No newline at end of file