[2025.06.05] update tokenizer train code, use the style of Qwen
This commit is contained in:
coffee
+1
-2
@@ -5,5 +5,4 @@ dolly_env_local
|
||||
llm_dolly.egg-info
|
||||
src_llm/__pycache__
|
||||
weights_MoE
|
||||
weights_llm
|
||||
weights_tokenizer/*
|
||||
weights_llm
|
||||
@@ -20,81 +20,82 @@ def get_training_corpus():
|
||||
if "text" in data:
|
||||
yield data["text"]
|
||||
|
||||
# 使用Path对象处理路径
|
||||
corpus_dir = Path("./corpus")
|
||||
|
||||
corpus_dir = Path("./corpus/skypile_2020_head")
|
||||
jsonl_files = list(corpus_dir.glob("**/*.jsonl"))
|
||||
if not jsonl_files:
|
||||
raise ValueError("No training files found")
|
||||
|
||||
# 使用Qwen2官方特殊标记
|
||||
|
||||
special_tokens = [
|
||||
"<|endoftext|>",
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
]
|
||||
|
||||
# 创建BPE tokenizer
|
||||
|
||||
tokenizer = Tokenizer(models.BPE())
|
||||
|
||||
# 使用Qwen2的normalizer配置
|
||||
|
||||
tokenizer.normalizer = normalizers.Sequence([
|
||||
normalizers.NFKC(),
|
||||
normalizers.StripAccents(),
|
||||
])
|
||||
|
||||
# 简化预分词器：使用ByteLevel代替多个Split规则
|
||||
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
|
||||
add_prefix_space=False, # 对齐Qwen2配置
|
||||
use_regex=True
|
||||
)
|
||||
|
||||
# 移除后处理器（Qwen2不需要特殊后处理）
|
||||
tokenizer.post_processor = None
|
||||
tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
|
||||
[
|
||||
pre_tokenizers.Split(r"[\r\n]", "isolated"),
|
||||
pre_tokenizers.Split(r"\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+","isolated"),
|
||||
pre_tokenizers.Split(r"\s?[!-/:-~！-／：-～‘-‟ -。]+", "isolated"),
|
||||
pre_tokenizers.Split(r"\s+$", "isolated"),
|
||||
pre_tokenizers.Split("[一-龥ࠀ-一가-]+","isolated"),
|
||||
pre_tokenizers.Digits(individual_digits=True),
|
||||
pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
|
||||
])
|
||||
|
||||
# tokenizer.post_processor = None
|
||||
tokenizer.post_processor = processors.ByteLevel(use_regex=True, trim_offsets=False, add_prefix_space=True)
|
||||
|
||||
# 配置解码器
|
||||
tokenizer.decoder = decoders.ByteLevel(
|
||||
use_regex=True,
|
||||
trim_offsets=True,
|
||||
add_prefix_space=False # 对齐Qwen2的add_prefix_space配置
|
||||
add_prefix_space=False
|
||||
)
|
||||
|
||||
# 配置训练器
|
||||
|
||||
trainer = trainers.BpeTrainer(
|
||||
vocab_size=12800,
|
||||
special_tokens=special_tokens,
|
||||
min_frequency=2, # 添加最小频率阈值
|
||||
show_progress=True # 显示进度条
|
||||
special_tokens=special_tokens,
|
||||
show_progress=True
|
||||
)
|
||||
|
||||
print("Training tokenizer...")
|
||||
tokenizer.train_from_iterator(
|
||||
get_training_corpus(),
|
||||
trainer=trainer,
|
||||
length=sum(1 for _ in get_training_corpus()) # 添加总长度用于进度显示
|
||||
length=sum(1 for _ in get_training_corpus())
|
||||
)
|
||||
|
||||
# 创建PreTrainedTokenizerFast时添加Qwen2关键配置
|
||||
wrapped_tokenizer = PreTrainedTokenizerFast(
|
||||
tokenizer_object=tokenizer,
|
||||
bos_token=None, # 对齐Qwen2配置
|
||||
bos_token=None,
|
||||
eos_token="<|endoftext|>",
|
||||
pad_token="<|endoftext|>",
|
||||
unk_token=None, # 对齐Qwen2配置
|
||||
unk_token=None,
|
||||
additional_special_tokens=["<|im_start|>", "<|im_end|>"],
|
||||
model_max_length=32768, # 添加model_max_length
|
||||
clean_up_tokenization_spaces=False, # 对齐Qwen2配置
|
||||
errors="replace", # 对齐Qwen2配置
|
||||
split_special_tokens=False, # 对齐Qwen2配置
|
||||
model_max_length=32768,
|
||||
clean_up_tokenization_spaces=False,
|
||||
errors="replace",
|
||||
split_special_tokens=False,
|
||||
)
|
||||
|
||||
output_dir = Path("./llm_tokenizer")
|
||||
output_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
# 保存tokenizer
|
||||
wrapped_tokenizer.save_pretrained(output_dir)
|
||||
print(f"Tokenizer saved to {output_dir}")
|
||||
|
||||
# 使用Qwen2官方chat_template
|
||||
QWEN2_CHAT_TEMPLATE = (
|
||||
"{% for message in messages %}"
|
||||
"{% if loop.first and messages[0]['role'] != 'system' %}"
|
||||
@@ -107,13 +108,10 @@ QWEN2_CHAT_TEMPLATE = (
|
||||
"{% endif %}"
|
||||
)
|
||||
|
||||
# 直接设置chat_template避免重新加载
|
||||
wrapped_tokenizer.chat_template = QWEN2_CHAT_TEMPLATE
|
||||
|
||||
# 再次保存包含chat_template的配置
|
||||
wrapped_tokenizer.save_pretrained(output_dir)
|
||||
|
||||
# 测试对话模板
|
||||
messages = [
|
||||
{"role": "user", "content": "你好！"},
|
||||
{"role": "assistant", "content": "你好，有什么可以帮您？"},
|
||||
@@ -128,7 +126,11 @@ formatted = wrapped_tokenizer.apply_chat_template(
|
||||
print("\n格式化后的对话:")
|
||||
print(formatted)
|
||||
|
||||
# 测试tokenization
|
||||
test_text = "深度学习是人工智能的一个重要分支。Deep learning is a subset of machine learning."
|
||||
print("\nTest tokenization:")
|
||||
print(wrapped_tokenizer.tokenize(test_text))
|
||||
encoded = wrapped_tokenizer.tokenize(test_text)
|
||||
print(encoded)
|
||||
print(encoded)
|
||||
decoded_text = wrapped_tokenizer.decode(encoded)
|
||||
print(decoded_text)
|
||||
print(decoded_text)
|
||||
@@ -1,22 +1,118 @@
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sentencepiece as spm
|
||||
import logging
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
from tqdm import tqdm
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
|
||||
###推荐使用 utils/spm_train_tokenizer.sh，使用多核加速
|
||||
def normalize_text(text):
|
||||
text = text.replace('\x00', '')
|
||||
text = unicodedata.normalize('NFKC', text)
|
||||
text = ''.join(c for c in unicodedata.normalize('NFD', text)
|
||||
if unicodedata.category(c) != 'Mn')
|
||||
return text
|
||||
|
||||
#英文分词器
|
||||
# corpus_path = 'a.txt,b.txt'
|
||||
corpus_path_en= 'corpus/pg16457.txt'
|
||||
vocab_size_en = 1000
|
||||
model_en= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_en'
|
||||
corpus_dir = Path("./corpus/skypile_2020_head")
|
||||
jsonl_files = list(corpus_dir.glob("**/*.jsonl"))
|
||||
if not jsonl_files:
|
||||
raise ValueError("No training files found")
|
||||
|
||||
# model_type 使用 bpe
|
||||
model_type = 'bpe'
|
||||
# byte_fallback=True，避免 out of vocabulary 问题
|
||||
spm.SentencePieceTrainer.train(input=corpus_path_en, model_prefix=model_en,
|
||||
vocab_size=vocab_size_en, model_type=model_type, byte_fallback=True,character_coverage=1)
|
||||
special_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
|
||||
logging.info("Staring processing corpus")
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_corpus:
|
||||
temp_corpus_path = temp_corpus.name
|
||||
file_progress = tqdm(jsonl_files, desc="处理文件中", unit="file")
|
||||
for file_path in file_progress:
|
||||
file_progress.set_description(f"处理: {file_path.name[:40]}...")
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
if "text" in data:
|
||||
normalized_text = normalize_text(data["text"])
|
||||
temp_corpus.write(normalized_text + "\n")
|
||||
except json.JSONDecodeError as e:
|
||||
logging.warning(f"JSON 解析错误在 {file_path}: {e}")
|
||||
except UnicodeDecodeError as e:
|
||||
logging.warning(f"编码错误在 {file_path}: {e}")
|
||||
logging.info("corpus processing completed")
|
||||
logging.info("Starting SentencePiece training")
|
||||
|
||||
#中文分词器
|
||||
corpus_path_zh= 'corpus/wikipedia.txt'
|
||||
vocab_size_zh = 32000
|
||||
model_zh= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_zh'
|
||||
spm.SentencePieceTrainer.train(input=corpus_path_zh, model_prefix=model_zh,
|
||||
vocab_size=vocab_size_zh, model_type=model_type, byte_fallback=True,character_coverage=1)
|
||||
vocab_size = 32000
|
||||
|
||||
# character_coverage = 0.9995
|
||||
|
||||
spm.SentencePieceTrainer.train(
|
||||
input=temp_corpus_path,
|
||||
model_prefix='tokenizer_model',
|
||||
vocab_size=vocab_size,
|
||||
model_type='bpe',
|
||||
character_coverage=1.0,
|
||||
pad_id=-1,
|
||||
bos_id=-1,
|
||||
eos_id=-1,
|
||||
unk_id=0,
|
||||
user_defined_symbols=special_tokens,
|
||||
split_digits=True,
|
||||
byte_fallback=True,
|
||||
remove_extra_whitespaces=False,
|
||||
add_dummy_prefix=False,
|
||||
normalization_rule_name='identity',
|
||||
num_threads=os.cpu_count(),
|
||||
max_sentencepiece_length=32,
|
||||
shuffle_input_sentence=True,
|
||||
)
|
||||
|
||||
os.unlink(temp_corpus_path)
|
||||
logging.info("SentencePiece training completed")
|
||||
|
||||
tokenizer = PreTrainedTokenizerFast(
|
||||
tokenizer_file='tokenizer_model.model',
|
||||
bos_token=None,
|
||||
eos_token="<|endoftext|>",
|
||||
pad_token="<|endoftext|>",
|
||||
unk_token=None,
|
||||
additional_special_tokens=["<|im_start|>", "<|im_end|>"],
|
||||
model_max_length=32768,
|
||||
clean_up_tokenization_spaces=False,
|
||||
split_special_tokens=False,
|
||||
errors="replace"
|
||||
)
|
||||
|
||||
output_dir = Path("./llm_tokenizer")
|
||||
output_dir.mkdir(exist_ok=True, parents=True)
|
||||
tokenizer.save_pretrained(output_dir)
|
||||
logging.info(f"Tokenizer saved to {output_dir}")
|
||||
|
||||
QWEN2_CHAT_TEMPLATE = (
|
||||
"{% for message in messages %}"
|
||||
"{% if loop.first and messages[0]['role'] != 'system' %}"
|
||||
"{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}"
|
||||
"{% endif %}"
|
||||
"{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
|
||||
"{% endfor %}"
|
||||
"{% if add_generation_prompt %}"
|
||||
"{{ '<|im_start|>assistant\n' }}"
|
||||
"{% endif %}"
|
||||
)
|
||||
|
||||
tokenizer.chat_template = QWEN2_CHAT_TEMPLATE
|
||||
tokenizer.save_pretrained(output_dir)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "你好！"},
|
||||
{"role": "assistant", "content": "你好，有什么可以帮您？"},
|
||||
{"role": "user", "content": "Python怎么用？"}
|
||||
]
|
||||
|
||||
formatted = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
print("\n格式化后的对话:")
|
||||
print(formatted)
|
||||
Reference in New Issue
Block a user