admin/LLM-from-scratch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tokenizer support BBPE

Browse Source
This commit is contained in:
coffee
2025-04-30 16:38:18 +08:00
parent 96c99c8e1a
commit c5c5a560df
3 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py
+1 -1
View File
@@ -20,4 +20,4 @@ tokenizer = old_tokenizer.train_new_from_iterator(
)
tokenizer.save_pretrained("new_tokenizer2")
tokenizer.save_pretrained("tokenizer_file/new_tokenizer2")
Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py → Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py
+4 -2
View File
@@ -1,11 +1,13 @@
from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.pre_tokenizers import Whitespace
from transformers import PreTrainedTokenizerFast
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
tokenizer.pre_tokenizer = Whitespace()
# tokenizer.pre_tokenizer = Whitespace()
tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True)
trainer = BpeTrainer(
vocab_size=30000,
special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],
@@ -23,4 +25,4 @@ fast_tokenizer = PreTrainedTokenizerFast(
sep_token="[SEP]",
mask_token="[MASK]",
)
fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer")
fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer_BBPE")