add trained BBPE Tokenizer model

2025-04-30 17:25:53 +08:00
parent a68b346429
commit 9e604a0339
6 changed files with 126549 additions and 2 deletions
@@ -9,7 +9,7 @@ tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
 # tokenizer.pre_tokenizer = Whitespace()
 tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True)
 trainer = BpeTrainer(
-    vocab_size=30000,           
+    vocab_size=12800,           
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],  
    min_frequency=2,            
    show_progress=True