[2025.06.05] update tokenizer train code, use the style of Qwen

.gitignore

@@ -6,4 +6,3 @@ llm_dolly.egg-info
 src_llm/__pycache__
 weights_MoE
 weights_llm
-weights_tokenizer/*

src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py

@@ -20,81 +20,82 @@ def get_training_corpus():
                 if "text" in data:
                     yield data["text"]
 
-# 使用Path对象处理路径
+
-corpus_dir = Path("./corpus")
+corpus_dir = Path("./corpus/skypile_2020_head")
 jsonl_files = list(corpus_dir.glob("**/*.jsonl"))
 if not jsonl_files:
     raise ValueError("No training files found")
 
-# 使用Qwen2官方特殊标记
+
 special_tokens = [
     "<|endoftext|>",
     "<|im_start|>",
     "<|im_end|>",
 ]
 
-# 创建BPE tokenizer
+
 tokenizer = Tokenizer(models.BPE())
 
-# 使用Qwen2的normalizer配置
+
 tokenizer.normalizer = normalizers.Sequence([
     normalizers.NFKC(),
     normalizers.StripAccents(),
 ])
 
-# 简化预分词器：使用ByteLevel代替多个Split规则
-tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
-    add_prefix_space=False,  # 对齐Qwen2配置
-    use_regex=True
-)
 
-# 移除后处理器（Qwen2不需要特殊后处理）
+tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
-tokenizer.post_processor = None
+    [
+        pre_tokenizers.Split(r"[\r\n]", "isolated"),   
+        pre_tokenizers.Split(r"\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+","isolated"), 
+        pre_tokenizers.Split(r"\s?[!-/:-~！-／：-～‘-‟　-。]+", "isolated"),   
+        pre_tokenizers.Split(r"\s+$", "isolated"), 
+        pre_tokenizers.Split("[一-龥ࠀ-一가-퟿]+","isolated"),
+        pre_tokenizers.Digits(individual_digits=True),
+        pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
+    ])
+
+# tokenizer.post_processor = None
+tokenizer.post_processor = processors.ByteLevel(use_regex=True, trim_offsets=False, add_prefix_space=True)
 
-# 配置解码器
 tokenizer.decoder = decoders.ByteLevel(
     use_regex=True,
     trim_offsets=True,
-    add_prefix_space=False  # 对齐Qwen2的add_prefix_space配置
+    add_prefix_space=False  
 )
 
-# 配置训练器
+
 trainer = trainers.BpeTrainer(
     vocab_size=12800,
     special_tokens=special_tokens, 
-    min_frequency=2,  # 添加最小频率阈值
+    show_progress=True  
-    show_progress=True  # 显示进度条
 )
 
 print("Training tokenizer...")
 tokenizer.train_from_iterator(
     get_training_corpus(),
     trainer=trainer,
-    length=sum(1 for _ in get_training_corpus())  # 添加总长度用于进度显示
+    length=sum(1 for _ in get_training_corpus())  
 )
 
-# 创建PreTrainedTokenizerFast时添加Qwen2关键配置
 wrapped_tokenizer = PreTrainedTokenizerFast(
     tokenizer_object=tokenizer,
-    bos_token=None,  # 对齐Qwen2配置
+    bos_token=None,  
     eos_token="<|endoftext|>",
     pad_token="<|endoftext|>",
-    unk_token=None,  # 对齐Qwen2配置
+    unk_token=None,  
     additional_special_tokens=["<|im_start|>", "<|im_end|>"],
-    model_max_length=32768,  # 添加model_max_length
+    model_max_length=32768,  
-    clean_up_tokenization_spaces=False,  # 对齐Qwen2配置
+    clean_up_tokenization_spaces=False,  
-    errors="replace",  # 对齐Qwen2配置
+    errors="replace", 
-    split_special_tokens=False,  # 对齐Qwen2配置
+    split_special_tokens=False,  
 )
 
 output_dir = Path("./llm_tokenizer")
 output_dir.mkdir(exist_ok=True, parents=True)
 
-# 保存tokenizer
 wrapped_tokenizer.save_pretrained(output_dir)
 print(f"Tokenizer saved to {output_dir}")
 
-# 使用Qwen2官方chat_template
 QWEN2_CHAT_TEMPLATE = (
     "{% for message in messages %}"
     "{% if loop.first and messages[0]['role'] != 'system' %}"
@@ -107,13 +108,10 @@ QWEN2_CHAT_TEMPLATE = (
     "{% endif %}"
 )
 
-# 直接设置chat_template避免重新加载
 wrapped_tokenizer.chat_template = QWEN2_CHAT_TEMPLATE
 
-# 再次保存包含chat_template的配置
 wrapped_tokenizer.save_pretrained(output_dir)
 
-# 测试对话模板
 messages = [
     {"role": "user", "content": "你好！"},
     {"role": "assistant", "content": "你好，有什么可以帮您？"},
@@ -128,7 +126,11 @@ formatted = wrapped_tokenizer.apply_chat_template(
 print("\n格式化后的对话:")
 print(formatted)
 
-# 测试tokenization
 test_text = "深度学习是人工智能的一个重要分支。Deep learning is a subset of machine learning."
 print("\nTest tokenization:")
-print(wrapped_tokenizer.tokenize(test_text))
+encoded = wrapped_tokenizer.tokenize(test_text)
+print(encoded)
+print(encoded)
+decoded_text = wrapped_tokenizer.decode(encoded) 
+print(decoded_text)
+print(decoded_text)

src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py

@@ -1,22 +1,118 @@
+import os
+import json
+from pathlib import Path
 import sentencepiece as spm
+import logging
+import tempfile
+import unicodedata
+from transformers import PreTrainedTokenizerFast
+from tqdm import tqdm
+logging.basicConfig(level=logging.INFO, format='%(message)s')
 
-###推荐使用 utils/spm_train_tokenizer.sh，使用多核加速
+def normalize_text(text):
+    text = text.replace('\x00', '')
+    text = unicodedata.normalize('NFKC', text)
+    text = ''.join(c for c in unicodedata.normalize('NFD', text)
+                  if unicodedata.category(c) != 'Mn')
+    return text
 
-#英文分词器
+corpus_dir = Path("./corpus/skypile_2020_head")
-# corpus_path = 'a.txt,b.txt'
+jsonl_files = list(corpus_dir.glob("**/*.jsonl"))
-corpus_path_en= 'corpus/pg16457.txt'
+if not jsonl_files:
-vocab_size_en = 1000
+    raise ValueError("No training files found")
-model_en= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_en'
 
-# model_type 使用 bpe
+special_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
-model_type = 'bpe'
+logging.info("Staring processing corpus")
-# byte_fallback=True，避免 out of vocabulary 问题
+with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_corpus:
-spm.SentencePieceTrainer.train(input=corpus_path_en, model_prefix=model_en, 
+    temp_corpus_path = temp_corpus.name
-    vocab_size=vocab_size_en, model_type=model_type, byte_fallback=True,character_coverage=1)
+    file_progress = tqdm(jsonl_files, desc="处理文件中", unit="file")
+    for file_path in file_progress:
+        file_progress.set_description(f"处理: {file_path.name[:40]}...")
+        with open(file_path, "r", encoding="utf-8") as f:
+            for line in f:
+                try:
+                    data = json.loads(line)
+                    if "text" in data:
+                        normalized_text = normalize_text(data["text"])
+                        temp_corpus.write(normalized_text + "\n")
+                except json.JSONDecodeError as e:
+                    logging.warning(f"JSON 解析错误在 {file_path}: {e}")
+                except UnicodeDecodeError as e:
+                    logging.warning(f"编码错误在 {file_path}: {e}")
+logging.info("corpus processing completed")
+logging.info("Starting SentencePiece training")
 
-#中文分词器
+vocab_size = 32000  
-corpus_path_zh= 'corpus/wikipedia.txt'
+
-vocab_size_zh = 32000
+# character_coverage = 0.9995  
-model_zh= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_zh'
+
-spm.SentencePieceTrainer.train(input=corpus_path_zh, model_prefix=model_zh, 
+spm.SentencePieceTrainer.train(
-    vocab_size=vocab_size_zh, model_type=model_type, byte_fallback=True,character_coverage=1)
+    input=temp_corpus_path,
+    model_prefix='tokenizer_model',
+    vocab_size=vocab_size,  
+    model_type='bpe',
+    character_coverage=1.0,  
+    pad_id=-1,
+    bos_id=-1,
+    eos_id=-1,
+    unk_id=0,
+    user_defined_symbols=special_tokens,
+    split_digits=True,
+    byte_fallback=True,
+    remove_extra_whitespaces=False,
+    add_dummy_prefix=False,
+    normalization_rule_name='identity',
+    num_threads=os.cpu_count(),
+    max_sentencepiece_length=32,  
+    shuffle_input_sentence=True, 
+)
+
+os.unlink(temp_corpus_path)
+logging.info("SentencePiece training completed")
+
+tokenizer = PreTrainedTokenizerFast(
+    tokenizer_file='tokenizer_model.model',
+    bos_token=None,
+    eos_token="<|endoftext|>",
+    pad_token="<|endoftext|>",
+    unk_token=None,
+    additional_special_tokens=["<|im_start|>", "<|im_end|>"],
+    model_max_length=32768,
+    clean_up_tokenization_spaces=False,
+    split_special_tokens=False,
+    errors="replace"
+)
+
+output_dir = Path("./llm_tokenizer")
+output_dir.mkdir(exist_ok=True, parents=True)
+tokenizer.save_pretrained(output_dir)
+logging.info(f"Tokenizer saved to {output_dir}")
+
+QWEN2_CHAT_TEMPLATE = (
+    "{% for message in messages %}"
+    "{% if loop.first and messages[0]['role'] != 'system' %}"
+    "{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}"
+    "{% endif %}"
+    "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
+    "{% endfor %}"
+    "{% if add_generation_prompt %}"
+    "{{ '<|im_start|>assistant\n' }}"
+    "{% endif %}"
+)
+
+tokenizer.chat_template = QWEN2_CHAT_TEMPLATE
+tokenizer.save_pretrained(output_dir)
+
+messages = [
+    {"role": "user", "content": "你好！"},
+    {"role": "assistant", "content": "你好，有什么可以帮您？"},
+    {"role": "user", "content": "Python怎么用？"}
+]
+
+formatted = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True
+)
+print("\n格式化后的对话:")
+print(formatted)