From 6a3030984833f2ed29eb02dca188709fa21ff050 Mon Sep 17 00:00:00 2001 From: coffee Date: Thu, 5 Jun 2025 18:09:18 +0800 Subject: [PATCH] [2025.06.05] update tokenizer train code, use the style of Qwen --- .gitignore | 3 +- .../step2_hf_train_from_scratch_BPEandBBPE.py | 68 ++++----- .../step1_spm_train_tokenizer.py | 130 +++++++++++++++--- 3 files changed, 149 insertions(+), 52 deletions(-) diff --git a/.gitignore b/.gitignore index 616afb3..96f64c1 100755 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,4 @@ dolly_env_local llm_dolly.egg-info src_llm/__pycache__ weights_MoE -weights_llm -weights_tokenizer/* \ No newline at end of file +weights_llm \ No newline at end of file diff --git a/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py b/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py index 7775b6a..63d4e67 100755 --- a/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py +++ b/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py @@ -20,81 +20,82 @@ def get_training_corpus(): if "text" in data: yield data["text"] -# 使用Path对象处理路径 -corpus_dir = Path("./corpus") + +corpus_dir = Path("./corpus/skypile_2020_head") jsonl_files = list(corpus_dir.glob("**/*.jsonl")) if not jsonl_files: raise ValueError("No training files found") -# 使用Qwen2官方特殊标记 + special_tokens = [ "<|endoftext|>", "<|im_start|>", "<|im_end|>", ] -# 创建BPE tokenizer + tokenizer = Tokenizer(models.BPE()) -# 使用Qwen2的normalizer配置 + tokenizer.normalizer = normalizers.Sequence([ normalizers.NFKC(), normalizers.StripAccents(), ]) -# 简化预分词器：使用ByteLevel代替多个Split规则 -tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel( - add_prefix_space=False, # 对齐Qwen2配置 - use_regex=True -) -# 移除后处理器（Qwen2不需要特殊后处理） -tokenizer.post_processor = None +tokenizer.pre_tokenizer = pre_tokenizers.Sequence( + [ + pre_tokenizers.Split(r"[\r\n]", "isolated"), + pre_tokenizers.Split(r"\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+","isolated"), + pre_tokenizers.Split(r"\s?[!-/:-~！-／：-～‘-‟　-。]+", "isolated"), + pre_tokenizers.Split(r"\s+$", "isolated"), + pre_tokenizers.Split("[一-龥ࠀ-一가-퟿]+","isolated"), + pre_tokenizers.Digits(individual_digits=True), + pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False), + ]) + +# tokenizer.post_processor = None +tokenizer.post_processor = processors.ByteLevel(use_regex=True, trim_offsets=False, add_prefix_space=True) -# 配置解码器 tokenizer.decoder = decoders.ByteLevel( use_regex=True, trim_offsets=True, - add_prefix_space=False # 对齐Qwen2的add_prefix_space配置 + add_prefix_space=False ) -# 配置训练器 + trainer = trainers.BpeTrainer( vocab_size=12800, - special_tokens=special_tokens, - min_frequency=2, # 添加最小频率阈值 - show_progress=True # 显示进度条 + special_tokens=special_tokens, + show_progress=True ) print("Training tokenizer...") tokenizer.train_from_iterator( get_training_corpus(), trainer=trainer, - length=sum(1 for _ in get_training_corpus()) # 添加总长度用于进度显示 + length=sum(1 for _ in get_training_corpus()) ) -# 创建PreTrainedTokenizerFast时添加Qwen2关键配置 wrapped_tokenizer = PreTrainedTokenizerFast( tokenizer_object=tokenizer, - bos_token=None, # 对齐Qwen2配置 + bos_token=None, eos_token="<|endoftext|>", pad_token="<|endoftext|>", - unk_token=None, # 对齐Qwen2配置 + unk_token=None, additional_special_tokens=["<|im_start|>", "<|im_end|>"], - model_max_length=32768, # 添加model_max_length - clean_up_tokenization_spaces=False, # 对齐Qwen2配置 - errors="replace", # 对齐Qwen2配置 - split_special_tokens=False, # 对齐Qwen2配置 + model_max_length=32768, + clean_up_tokenization_spaces=False, + errors="replace", + split_special_tokens=False, ) output_dir = Path("./llm_tokenizer") output_dir.mkdir(exist_ok=True, parents=True) -# 保存tokenizer wrapped_tokenizer.save_pretrained(output_dir) print(f"Tokenizer saved to {output_dir}") -# 使用Qwen2官方chat_template QWEN2_CHAT_TEMPLATE = ( "{% for message in messages %}" "{% if loop.first and messages[0]['role'] != 'system' %}" @@ -107,13 +108,10 @@ QWEN2_CHAT_TEMPLATE = ( "{% endif %}" ) -# 直接设置chat_template避免重新加载 wrapped_tokenizer.chat_template = QWEN2_CHAT_TEMPLATE -# 再次保存包含chat_template的配置 wrapped_tokenizer.save_pretrained(output_dir) -# 测试对话模板 messages = [ {"role": "user", "content": "你好！"}, {"role": "assistant", "content": "你好，有什么可以帮您？"}, @@ -128,7 +126,11 @@ formatted = wrapped_tokenizer.apply_chat_template( print("\n格式化后的对话:") print(formatted) -# 测试tokenization test_text = "深度学习是人工智能的一个重要分支。Deep learning is a subset of machine learning." print("\nTest tokenization:") -print(wrapped_tokenizer.tokenize(test_text)) \ No newline at end of file +encoded = wrapped_tokenizer.tokenize(test_text) +print(encoded) +print(encoded) +decoded_text = wrapped_tokenizer.decode(encoded) +print(decoded_text) +print(decoded_text) \ No newline at end of file diff --git a/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py b/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py index 34ef9f0..12c1be8 100755 --- a/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py +++ b/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py @@ -1,22 +1,118 @@ +import os +import json +from pathlib import Path import sentencepiece as spm +import logging +import tempfile +import unicodedata +from transformers import PreTrainedTokenizerFast +from tqdm import tqdm +logging.basicConfig(level=logging.INFO, format='%(message)s') -###推荐使用 utils/spm_train_tokenizer.sh，使用多核加速 +def normalize_text(text): + text = text.replace('\x00', '') + text = unicodedata.normalize('NFKC', text) + text = ''.join(c for c in unicodedata.normalize('NFD', text) + if unicodedata.category(c) != 'Mn') + return text -#英文分词器 -# corpus_path = 'a.txt,b.txt' -corpus_path_en= 'corpus/pg16457.txt' -vocab_size_en = 1000 -model_en= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_en' +corpus_dir = Path("./corpus/skypile_2020_head") +jsonl_files = list(corpus_dir.glob("**/*.jsonl")) +if not jsonl_files: + raise ValueError("No training files found") -# model_type 使用 bpe -model_type = 'bpe' -# byte_fallback=True，避免 out of vocabulary 问题 -spm.SentencePieceTrainer.train(input=corpus_path_en, model_prefix=model_en, - vocab_size=vocab_size_en, model_type=model_type, byte_fallback=True,character_coverage=1) +special_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"] +logging.info("Staring processing corpus") +with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_corpus: + temp_corpus_path = temp_corpus.name + file_progress = tqdm(jsonl_files, desc="处理文件中", unit="file") + for file_path in file_progress: + file_progress.set_description(f"处理: {file_path.name[:40]}...") + with open(file_path, "r", encoding="utf-8") as f: + for line in f: + try: + data = json.loads(line) + if "text" in data: + normalized_text = normalize_text(data["text"]) + temp_corpus.write(normalized_text + "\n") + except json.JSONDecodeError as e: + logging.warning(f"JSON 解析错误在 {file_path}: {e}") + except UnicodeDecodeError as e: + logging.warning(f"编码错误在 {file_path}: {e}") +logging.info("corpus processing completed") +logging.info("Starting SentencePiece training") -#中文分词器 -corpus_path_zh= 'corpus/wikipedia.txt' -vocab_size_zh = 32000 -model_zh= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_zh' -spm.SentencePieceTrainer.train(input=corpus_path_zh, model_prefix=model_zh, - vocab_size=vocab_size_zh, model_type=model_type, byte_fallback=True,character_coverage=1) +vocab_size = 32000 + +# character_coverage = 0.9995 + +spm.SentencePieceTrainer.train( + input=temp_corpus_path, + model_prefix='tokenizer_model', + vocab_size=vocab_size, + model_type='bpe', + character_coverage=1.0, + pad_id=-1, + bos_id=-1, + eos_id=-1, + unk_id=0, + user_defined_symbols=special_tokens, + split_digits=True, + byte_fallback=True, + remove_extra_whitespaces=False, + add_dummy_prefix=False, + normalization_rule_name='identity', + num_threads=os.cpu_count(), + max_sentencepiece_length=32, + shuffle_input_sentence=True, +) + +os.unlink(temp_corpus_path) +logging.info("SentencePiece training completed") + +tokenizer = PreTrainedTokenizerFast( + tokenizer_file='tokenizer_model.model', + bos_token=None, + eos_token="<|endoftext|>", + pad_token="<|endoftext|>", + unk_token=None, + additional_special_tokens=["<|im_start|>", "<|im_end|>"], + model_max_length=32768, + clean_up_tokenization_spaces=False, + split_special_tokens=False, + errors="replace" +) + +output_dir = Path("./llm_tokenizer") +output_dir.mkdir(exist_ok=True, parents=True) +tokenizer.save_pretrained(output_dir) +logging.info(f"Tokenizer saved to {output_dir}") + +QWEN2_CHAT_TEMPLATE = ( + "{% for message in messages %}" + "{% if loop.first and messages[0]['role'] != 'system' %}" + "{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}" + "{% endif %}" + "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '<|im_start|>assistant\n' }}" + "{% endif %}" +) + +tokenizer.chat_template = QWEN2_CHAT_TEMPLATE +tokenizer.save_pretrained(output_dir) + +messages = [ + {"role": "user", "content": "你好！"}, + {"role": "assistant", "content": "你好，有什么可以帮您？"}, + {"role": "user", "content": "Python怎么用？"} +] + +formatted = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True +) +print("\n格式化后的对话:") +print(formatted) \ No newline at end of file