[2025.06.19] update train code, pretoken corpus

src/step0_train_tokenizer_parquet.py

@@ -1,5 +1,5 @@
 import glob
-from tokenizers import Tokenizer, normalizers, pre_tokenizers,  processors, decoders
+from tokenizers import Tokenizer, normalizers, pre_tokenizers,  processors, decoders,Regex
 from tokenizers.normalizers import NFC
 from tokenizers.models import BPE
 from tokenizers.trainers import BpeTrainer
@@ -68,7 +68,7 @@ tokenizer.normalizer = normalizers.Sequence([NFC()])
 regex_pattern = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+"""
 tokenizer.pre_tokenizer = pre_tokenizers.Sequence([
     pre_tokenizers.Split(
-        pattern=pre_tokenizers.Regex(regex_pattern),
+        pattern=Regex(regex_pattern),
         behavior="isolated",
         invert=False
     ),
@@ -92,7 +92,7 @@ tokenizer.decoder = decoders.ByteLevel(
 )
 
 trainer = BpeTrainer(
-    vocab_size=32000,  
+    vocab_size=151936,  
     min_frequency=1,    
     special_tokens=SPECIAL_TOKENS,
     show_progress=True,