From 9c1739baf3737bfe992d13fe9fbf3e57c77f8a2c Mon Sep 17 00:00:00 2001 From: coffee Date: Tue, 10 Jun 2025 17:56:00 +0800 Subject: [PATCH] [2025.06.10] train EN tokenizer use parquet files --- src/step0_train_tokenizer_parquet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/step0_train_tokenizer_parquet.py b/src/step0_train_tokenizer_parquet.py index 4d2c777..85c2bc7 100755 --- a/src/step0_train_tokenizer_parquet.py +++ b/src/step0_train_tokenizer_parquet.py @@ -62,13 +62,13 @@ tokenizer = Tokenizer(BPE(unk_token=None)) tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False) trainer = BpeTrainer( - vocab_size=151643, + vocab_size=32000, min_frequency=2, special_tokens=SPECIAL_TOKENS, show_progress=True ) -corpus_files = 'corpus/train-*-of-00399.parquet' +corpus_files = glob.glob('corpus/parquet/train-0000*-of-00399.parquet' ) tokenizer.train_from_iterator( iterator=read_parquet_shards(corpus_files), trainer=trainer,