[2025.06.10] adjust directory structure

src_dev/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py

@@ -0,0 +1,118 @@
+import os
+import json
+from pathlib import Path
+import sentencepiece as spm
+import logging
+import tempfile
+import unicodedata
+from transformers import PreTrainedTokenizerFast
+from tqdm import tqdm
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+def normalize_text(text):
+    text = text.replace('\x00', '')
+    text = unicodedata.normalize('NFKC', text)
+    text = ''.join(c for c in unicodedata.normalize('NFD', text)
+                  if unicodedata.category(c) != 'Mn')
+    return text
+
+corpus_dir = Path("./corpus/skypile_2020_head")
+jsonl_files = list(corpus_dir.glob("**/*.jsonl"))
+if not jsonl_files:
+    raise ValueError("No training files found")
+
+special_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
+logging.info("Staring processing corpus")
+with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_corpus:
+    temp_corpus_path = temp_corpus.name
+    file_progress = tqdm(jsonl_files, desc="处理文件中", unit="file")
+    for file_path in file_progress:
+        file_progress.set_description(f"处理: {file_path.name[:40]}...")
+        with open(file_path, "r", encoding="utf-8") as f:
+            for line in f:
+                try:
+                    data = json.loads(line)
+                    if "text" in data:
+                        normalized_text = normalize_text(data["text"])
+                        temp_corpus.write(normalized_text + "\n")
+                except json.JSONDecodeError as e:
+                    logging.warning(f"JSON 解析错误在 {file_path}: {e}")
+                except UnicodeDecodeError as e:
+                    logging.warning(f"编码错误在 {file_path}: {e}")
+logging.info("corpus processing completed")
+logging.info("Starting SentencePiece training")
+
+vocab_size = 32000  
+
+# character_coverage = 0.9995  
+
+spm.SentencePieceTrainer.train(
+    input=temp_corpus_path,
+    model_prefix='tokenizer_model',
+    vocab_size=vocab_size,  
+    model_type='bpe',
+    character_coverage=1.0,  
+    pad_id=-1,
+    bos_id=-1,
+    eos_id=-1,
+    unk_id=0,
+    user_defined_symbols=special_tokens,
+    split_digits=True,
+    byte_fallback=True,
+    remove_extra_whitespaces=False,
+    add_dummy_prefix=False,
+    normalization_rule_name='identity',
+    num_threads=os.cpu_count(),
+    max_sentencepiece_length=32,  
+    shuffle_input_sentence=True, 
+)
+
+os.unlink(temp_corpus_path)
+logging.info("SentencePiece training completed")
+
+tokenizer = PreTrainedTokenizerFast(
+    tokenizer_file='tokenizer_model.model',
+    bos_token=None,
+    eos_token="<|endoftext|>",
+    pad_token="<|endoftext|>",
+    unk_token=None,
+    additional_special_tokens=["<|im_start|>", "<|im_end|>"],
+    model_max_length=32768,
+    clean_up_tokenization_spaces=False,
+    split_special_tokens=False,
+    errors="replace"
+)
+
+output_dir = Path("./llm_tokenizer")
+output_dir.mkdir(exist_ok=True, parents=True)
+tokenizer.save_pretrained(output_dir)
+logging.info(f"Tokenizer saved to {output_dir}")
+
+QWEN2_CHAT_TEMPLATE = (
+    "{% for message in messages %}"
+    "{% if loop.first and messages[0]['role'] != 'system' %}"
+    "{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}"
+    "{% endif %}"
+    "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
+    "{% endfor %}"
+    "{% if add_generation_prompt %}"
+    "{{ '<|im_start|>assistant\n' }}"
+    "{% endif %}"
+)
+
+tokenizer.chat_template = QWEN2_CHAT_TEMPLATE
+tokenizer.save_pretrained(output_dir)
+
+messages = [
+    {"role": "user", "content": "你好！"},
+    {"role": "assistant", "content": "你好，有什么可以帮您？"},
+    {"role": "user", "content": "Python怎么用？"}
+]
+
+formatted = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True
+)
+print("\n格式化后的对话:")
+print(formatted)

src_dev/src_tokenizer/spm_tokenizer_code/step2_spm_merge_tokenizer.py

@@ -0,0 +1,33 @@
+import sentencepiece as spm
+from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
+
+model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_zh.model"
+en_model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_en.model"
+
+model_file_new = "tokenizer_part/dolly_tokenizer_spm/tokenizer.model"
+
+chinese_sp = spm.SentencePieceProcessor(model_file=model_file)
+en_sp = spm.SentencePieceProcessor(model_file=en_model_file)
+
+print(chinese_sp.vocab_size())#32000
+
+chinese_spm = sp_model_pb2.ModelProto()
+chinese_spm.ParseFromString(chinese_sp.serialized_model_proto())
+
+en_spm = sp_model_pb2.ModelProto()
+en_spm.ParseFromString(en_sp.serialized_model_proto())
+
+chinese_sp_set = set(p.piece for p in chinese_spm.pieces)
+
+for p in en_spm.pieces:
+    piece = p.piece
+    if piece not in chinese_sp_set:
+        chinese_sp_set.add(piece)
+        new_piece = sp_model_pb2.ModelProto().SentencePiece()
+        new_piece.piece = piece
+        new_piece.score = 0
+        chinese_spm.pieces.append(new_piece)
+
+print(f"after expand vocab={len(chinese_spm.pieces)}") # 32361,增加361个
+with open(model_file_new, 'wb') as file:
+    file.write(chinese_spm.SerializeToString())

src_dev/src_tokenizer/spm_tokenizer_code/step3_spm_to_hf.py

@@ -0,0 +1,18 @@
+from transformers import AutoTokenizer,XLMRobertaTokenizer
+#由spm转为hf_tokenizer
+tokenizer = XLMRobertaTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_spm/tokenizer.model")
+text = "A Frenchman from Paris"
+encoded_input = tokenizer.encode(text)
+print(encoded_input)
+decoded_output = tokenizer.decode(encoded_input)
+print(decoded_output)
+tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf')
+
+#转换为常见的tokenzier.json格式
+tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf")
+text = "A Frenchman from Paris"
+encoded_input = tokenizer.encode(text)
+print(encoded_input)
+decoded_output = tokenizer.decode(encoded_input)
+print(decoded_output)
+tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf2')

src_dev/src_tokenizer/spm_tokenizer_code/step4_spm_expand_vocab.py

@@ -0,0 +1,21 @@
+import sentencepiece as spm
+from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
+
+sp = spm.SentencePieceProcessor(model_file='tokenizer_file/dolly_tokenizer_spm/tokenizer.model')
+print(sp.vocab_size())  # 32461
+
+old_model = sp_model_pb2.ModelProto()
+old_model.ParseFromString(sp.serialized_model_proto())
+pieces = ['拉布拉多', '货拉拉', '拉不拉']
+existed_pieces = set(p.piece for p in old_model.pieces)
+for piece in pieces:
+    if piece not in existed_pieces:
+        existed_pieces.add(piece)
+        new_piece = sp_model_pb2.ModelProto().SentencePiece()
+        new_piece.piece = piece
+        new_piece.score = 0
+        old_model.pieces.append(new_piece)
+    
+print(len(old_model.pieces))  # 32461
+with open('tokenizer_file/dolly_tokenizer_spm/expand_tokenizer.model', 'wb') as f:
+    f.write(old_model.SerializeToString())

src_dev/src_tokenizer/spm_tokenizer_code/step5_test_spm_tokenizer.py

@@ -0,0 +1,15 @@
+import sentencepiece as spm
+sp = spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer.model')
+
+sp_en= spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer_en.model')
+sentence = 'A Frenchman from Paris'
+pieces = sp.encode_as_pieces(sentence)
+pieces_en = sp_en.encode_as_pieces(sentence)
+print(pieces)  # ['▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'ar', 'is']
+print(pieces_en) #'▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'aris'
+tokens = sp.encode(sentence)
+tokens_en = sp_en.encode(sentence)
+print(tokens)  # [338, 493, 587, 349, 807, 429, 373, 279, 282]
+print(tokens_en) #364, 490, 5981, 398, 2036, 32046, 386, 9020] 
+# 解码后的结果应等于编码前的输入
+# assert sp.decode(tokens) == sentence