admin/LLM-from-scratch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[2025.06.10] train EN tokenizer use parquet files

Browse Source
This commit is contained in:
coffee
2025-06-10 17:56:00 +08:00
parent eb1c6c0c17
commit 9c1739baf3
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/step0_train_tokenizer_parquet.py
+2 -2
View File
@@ -62,13 +62,13 @@ tokenizer = Tokenizer(BPE(unk_token=None))
tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False) tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)
trainer = BpeTrainer( trainer = BpeTrainer(
vocab_size=151643, vocab_size=32000,
min_frequency=2, min_frequency=2,
special_tokens=SPECIAL_TOKENS, special_tokens=SPECIAL_TOKENS,
show_progress=True show_progress=True
) )
corpus_files = 'corpus/train-*-of-00399.parquet' corpus_files = glob.glob('corpus/parquet/train-0000*-of-00399.parquet' )
tokenizer.train_from_iterator( tokenizer.train_from_iterator(
iterator=read_parquet_shards(corpus_files), iterator=read_parquet_shards(corpus_files),
trainer=trainer, trainer=trainer,