[2025.06.10] adjust directory structure
This commit is contained in:
coffee
Executable
+3
@@ -0,0 +1,3 @@
|
||||
# USAGE
|
||||
使用sentencepiece和huggingface transformers分别进行tokenizer的训练
|
||||
support BPE and BBPE
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
def get_training_corpus(file_path, batch_size=1000):
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
batch = []
|
||||
for line in f:
|
||||
batch.append(line.strip())
|
||||
if len(batch) >= batch_size:
|
||||
yield batch
|
||||
batch = []
|
||||
if batch:
|
||||
yield batch
|
||||
|
||||
old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")
|
||||
training_corpus = get_training_corpus("corpus/wikipedia.txt")
|
||||
|
||||
tokenizer = old_tokenizer.train_new_from_iterator(
|
||||
training_corpus,
|
||||
vocab_size=34000
|
||||
)
|
||||
|
||||
|
||||
tokenizer.save_pretrained("tokenizer_file/new_tokenizer2")
|
||||
@@ -0,0 +1,16 @@
|
||||
from datasets import load_dataset
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
dataset = load_dataset("text", data_files="corpus/wikipedia.txt", streaming=True)
|
||||
|
||||
def batch_iterator(batch_size=1000):
|
||||
for batch in dataset["train"].iter(batch_size):
|
||||
yield batch["text"]
|
||||
|
||||
old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")
|
||||
tokenizer = old_tokenizer.train_new_from_iterator(
|
||||
batch_iterator(),
|
||||
vocab_size=33000,
|
||||
)
|
||||
|
||||
tokenizer.save_pretrained("new_tokenizer")
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tokenizers import (
|
||||
decoders,
|
||||
models,
|
||||
normalizers,
|
||||
pre_tokenizers,
|
||||
processors,
|
||||
trainers,
|
||||
Tokenizer,
|
||||
)
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
import logging
|
||||
from tqdm import tqdm
|
||||
log_dir = "/mnt/hanluzhi/dolly/logs"
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(message)s',
|
||||
handlers=[
|
||||
logging.StreamHandler(),
|
||||
logging.FileHandler(os.path.join(log_dir, "training.log"))
|
||||
]
|
||||
)
|
||||
|
||||
def get_training_corpus():
|
||||
for file_path in tqdm(jsonl_files, desc="processing files", unit="file"):
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
data = json.loads(line)
|
||||
if "text" in data:
|
||||
yield data["text"]
|
||||
|
||||
|
||||
corpus_dir = Path("./corpus/skypile_2020_head")
|
||||
jsonl_files = list(corpus_dir.glob("**/*.jsonl"))
|
||||
if not jsonl_files:
|
||||
raise ValueError("No training files found")
|
||||
|
||||
|
||||
special_tokens = [
|
||||
"<|endoftext|>",
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
]
|
||||
|
||||
|
||||
tokenizer = Tokenizer(models.BPE())
|
||||
|
||||
|
||||
tokenizer.normalizer = normalizers.Sequence([
|
||||
normalizers.NFKC(),
|
||||
normalizers.StripAccents(),
|
||||
])
|
||||
|
||||
|
||||
tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
|
||||
[
|
||||
pre_tokenizers.Split(r"[\r\n]", "isolated"),
|
||||
pre_tokenizers.Split(r"\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+","isolated"),
|
||||
pre_tokenizers.Split(r"\s?[!-/:-~！-／：-～‘-‟ -。]+", "isolated"),
|
||||
pre_tokenizers.Split(r"\s+$", "isolated"),
|
||||
pre_tokenizers.Split("[一-龥ࠀ-一가-]+","isolated"),
|
||||
pre_tokenizers.Digits(individual_digits=True),
|
||||
pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
|
||||
])
|
||||
|
||||
# tokenizer.post_processor = None
|
||||
tokenizer.post_processor = processors.ByteLevel(use_regex=True, trim_offsets=False, add_prefix_space=True)
|
||||
|
||||
tokenizer.decoder = decoders.ByteLevel(
|
||||
use_regex=True,
|
||||
trim_offsets=True,
|
||||
add_prefix_space=False
|
||||
)
|
||||
|
||||
|
||||
trainer = trainers.BpeTrainer(
|
||||
vocab_size=12800,
|
||||
special_tokens=special_tokens,
|
||||
show_progress=True
|
||||
)
|
||||
|
||||
logging.info("Training tokenizer...")
|
||||
tokenizer.train_from_iterator(
|
||||
get_training_corpus(),
|
||||
trainer=trainer,
|
||||
length=sum(1 for _ in get_training_corpus())
|
||||
)
|
||||
|
||||
wrapped_tokenizer = PreTrainedTokenizerFast(
|
||||
tokenizer_object=tokenizer,
|
||||
bos_token=None,
|
||||
eos_token="<|endoftext|>",
|
||||
pad_token="<|endoftext|>",
|
||||
unk_token=None,
|
||||
additional_special_tokens=["<|im_start|>", "<|im_end|>"],
|
||||
model_max_length=32768,
|
||||
clean_up_tokenization_spaces=False,
|
||||
errors="replace",
|
||||
split_special_tokens=False,
|
||||
)
|
||||
logging.info("train done ")
|
||||
output_dir = Path("./llm_tokenizer")
|
||||
output_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
wrapped_tokenizer.save_pretrained(output_dir)
|
||||
logging.info(f"Tokenizer saved to {output_dir}")
|
||||
|
||||
QWEN2_CHAT_TEMPLATE = (
|
||||
"{% for message in messages %}"
|
||||
"{% if loop.first and messages[0]['role'] != 'system' %}"
|
||||
"{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}"
|
||||
"{% endif %}"
|
||||
"{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
|
||||
"{% endfor %}"
|
||||
"{% if add_generation_prompt %}"
|
||||
"{{ '<|im_start|>assistant\n' }}"
|
||||
"{% endif %}"
|
||||
)
|
||||
|
||||
wrapped_tokenizer.chat_template = QWEN2_CHAT_TEMPLATE
|
||||
|
||||
wrapped_tokenizer.save_pretrained(output_dir)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "你好！"},
|
||||
{"role": "assistant", "content": "你好，有什么可以帮您？"},
|
||||
{"role": "user", "content": "Python怎么用？"}
|
||||
]
|
||||
|
||||
formatted = wrapped_tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
logging.info("\n格式化后的对话:")
|
||||
logging.info(formatted)
|
||||
|
||||
test_text = "深度学习是人工智能的一个重要分支。Deep learning is a subset of machine learning."
|
||||
logging.info("\nTest tokenization:")
|
||||
encoded = wrapped_tokenizer.tokenize(test_text)
|
||||
logging.info(encoded)
|
||||
logging.info(encoded)
|
||||
decoded_text = wrapped_tokenizer.decode(encoded)
|
||||
logging.info(decoded_text)
|
||||
logging.info(decoded_text)
|
||||
@@ -0,0 +1,10 @@
|
||||
from transformers import AutoTokenizer, AutoModel
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")
|
||||
|
||||
new_tokens = ["<ACTION_1>", "<ACTION_2>"]
|
||||
tokenizer.add_tokens(new_tokens)
|
||||
|
||||
text = "<ACTION_1>"
|
||||
print(tokenizer.tokenize(text))
|
||||
tokenizer.save_pretrained("updated_tokenizer")
|
||||
@@ -0,0 +1,118 @@
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sentencepiece as spm
|
||||
import logging
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
from tqdm import tqdm
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
|
||||
def normalize_text(text):
|
||||
text = text.replace('\x00', '')
|
||||
text = unicodedata.normalize('NFKC', text)
|
||||
text = ''.join(c for c in unicodedata.normalize('NFD', text)
|
||||
if unicodedata.category(c) != 'Mn')
|
||||
return text
|
||||
|
||||
corpus_dir = Path("./corpus/skypile_2020_head")
|
||||
jsonl_files = list(corpus_dir.glob("**/*.jsonl"))
|
||||
if not jsonl_files:
|
||||
raise ValueError("No training files found")
|
||||
|
||||
special_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
|
||||
logging.info("Staring processing corpus")
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_corpus:
|
||||
temp_corpus_path = temp_corpus.name
|
||||
file_progress = tqdm(jsonl_files, desc="处理文件中", unit="file")
|
||||
for file_path in file_progress:
|
||||
file_progress.set_description(f"处理: {file_path.name[:40]}...")
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
if "text" in data:
|
||||
normalized_text = normalize_text(data["text"])
|
||||
temp_corpus.write(normalized_text + "\n")
|
||||
except json.JSONDecodeError as e:
|
||||
logging.warning(f"JSON 解析错误在 {file_path}: {e}")
|
||||
except UnicodeDecodeError as e:
|
||||
logging.warning(f"编码错误在 {file_path}: {e}")
|
||||
logging.info("corpus processing completed")
|
||||
logging.info("Starting SentencePiece training")
|
||||
|
||||
vocab_size = 32000
|
||||
|
||||
# character_coverage = 0.9995
|
||||
|
||||
spm.SentencePieceTrainer.train(
|
||||
input=temp_corpus_path,
|
||||
model_prefix='tokenizer_model',
|
||||
vocab_size=vocab_size,
|
||||
model_type='bpe',
|
||||
character_coverage=1.0,
|
||||
pad_id=-1,
|
||||
bos_id=-1,
|
||||
eos_id=-1,
|
||||
unk_id=0,
|
||||
user_defined_symbols=special_tokens,
|
||||
split_digits=True,
|
||||
byte_fallback=True,
|
||||
remove_extra_whitespaces=False,
|
||||
add_dummy_prefix=False,
|
||||
normalization_rule_name='identity',
|
||||
num_threads=os.cpu_count(),
|
||||
max_sentencepiece_length=32,
|
||||
shuffle_input_sentence=True,
|
||||
)
|
||||
|
||||
os.unlink(temp_corpus_path)
|
||||
logging.info("SentencePiece training completed")
|
||||
|
||||
tokenizer = PreTrainedTokenizerFast(
|
||||
tokenizer_file='tokenizer_model.model',
|
||||
bos_token=None,
|
||||
eos_token="<|endoftext|>",
|
||||
pad_token="<|endoftext|>",
|
||||
unk_token=None,
|
||||
additional_special_tokens=["<|im_start|>", "<|im_end|>"],
|
||||
model_max_length=32768,
|
||||
clean_up_tokenization_spaces=False,
|
||||
split_special_tokens=False,
|
||||
errors="replace"
|
||||
)
|
||||
|
||||
output_dir = Path("./llm_tokenizer")
|
||||
output_dir.mkdir(exist_ok=True, parents=True)
|
||||
tokenizer.save_pretrained(output_dir)
|
||||
logging.info(f"Tokenizer saved to {output_dir}")
|
||||
|
||||
QWEN2_CHAT_TEMPLATE = (
|
||||
"{% for message in messages %}"
|
||||
"{% if loop.first and messages[0]['role'] != 'system' %}"
|
||||
"{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}"
|
||||
"{% endif %}"
|
||||
"{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
|
||||
"{% endfor %}"
|
||||
"{% if add_generation_prompt %}"
|
||||
"{{ '<|im_start|>assistant\n' }}"
|
||||
"{% endif %}"
|
||||
)
|
||||
|
||||
tokenizer.chat_template = QWEN2_CHAT_TEMPLATE
|
||||
tokenizer.save_pretrained(output_dir)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "你好！"},
|
||||
{"role": "assistant", "content": "你好，有什么可以帮您？"},
|
||||
{"role": "user", "content": "Python怎么用？"}
|
||||
]
|
||||
|
||||
formatted = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
print("\n格式化后的对话:")
|
||||
print(formatted)
|
||||
@@ -0,0 +1,33 @@
|
||||
import sentencepiece as spm
|
||||
from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
|
||||
|
||||
model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_zh.model"
|
||||
en_model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_en.model"
|
||||
|
||||
model_file_new = "tokenizer_part/dolly_tokenizer_spm/tokenizer.model"
|
||||
|
||||
chinese_sp = spm.SentencePieceProcessor(model_file=model_file)
|
||||
en_sp = spm.SentencePieceProcessor(model_file=en_model_file)
|
||||
|
||||
print(chinese_sp.vocab_size())#32000
|
||||
|
||||
chinese_spm = sp_model_pb2.ModelProto()
|
||||
chinese_spm.ParseFromString(chinese_sp.serialized_model_proto())
|
||||
|
||||
en_spm = sp_model_pb2.ModelProto()
|
||||
en_spm.ParseFromString(en_sp.serialized_model_proto())
|
||||
|
||||
chinese_sp_set = set(p.piece for p in chinese_spm.pieces)
|
||||
|
||||
for p in en_spm.pieces:
|
||||
piece = p.piece
|
||||
if piece not in chinese_sp_set:
|
||||
chinese_sp_set.add(piece)
|
||||
new_piece = sp_model_pb2.ModelProto().SentencePiece()
|
||||
new_piece.piece = piece
|
||||
new_piece.score = 0
|
||||
chinese_spm.pieces.append(new_piece)
|
||||
|
||||
print(f"after expand vocab={len(chinese_spm.pieces)}") # 32361,增加361个
|
||||
with open(model_file_new, 'wb') as file:
|
||||
file.write(chinese_spm.SerializeToString())
|
||||
@@ -0,0 +1,18 @@
|
||||
from transformers import AutoTokenizer,XLMRobertaTokenizer
|
||||
#由spm转为hf_tokenizer
|
||||
tokenizer = XLMRobertaTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_spm/tokenizer.model")
|
||||
text = "A Frenchman from Paris"
|
||||
encoded_input = tokenizer.encode(text)
|
||||
print(encoded_input)
|
||||
decoded_output = tokenizer.decode(encoded_input)
|
||||
print(decoded_output)
|
||||
tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf')
|
||||
|
||||
#转换为常见的tokenzier.json格式
|
||||
tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf")
|
||||
text = "A Frenchman from Paris"
|
||||
encoded_input = tokenizer.encode(text)
|
||||
print(encoded_input)
|
||||
decoded_output = tokenizer.decode(encoded_input)
|
||||
print(decoded_output)
|
||||
tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf2')
|
||||
@@ -0,0 +1,21 @@
|
||||
import sentencepiece as spm
|
||||
from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
|
||||
|
||||
sp = spm.SentencePieceProcessor(model_file='tokenizer_file/dolly_tokenizer_spm/tokenizer.model')
|
||||
print(sp.vocab_size()) # 32461
|
||||
|
||||
old_model = sp_model_pb2.ModelProto()
|
||||
old_model.ParseFromString(sp.serialized_model_proto())
|
||||
pieces = ['拉布拉多', '货拉拉', '拉不拉']
|
||||
existed_pieces = set(p.piece for p in old_model.pieces)
|
||||
for piece in pieces:
|
||||
if piece not in existed_pieces:
|
||||
existed_pieces.add(piece)
|
||||
new_piece = sp_model_pb2.ModelProto().SentencePiece()
|
||||
new_piece.piece = piece
|
||||
new_piece.score = 0
|
||||
old_model.pieces.append(new_piece)
|
||||
|
||||
print(len(old_model.pieces)) # 32461
|
||||
with open('tokenizer_file/dolly_tokenizer_spm/expand_tokenizer.model', 'wb') as f:
|
||||
f.write(old_model.SerializeToString())
|
||||
@@ -0,0 +1,15 @@
|
||||
import sentencepiece as spm
|
||||
sp = spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer.model')
|
||||
|
||||
sp_en= spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer_en.model')
|
||||
sentence = 'A Frenchman from Paris'
|
||||
pieces = sp.encode_as_pieces(sentence)
|
||||
pieces_en = sp_en.encode_as_pieces(sentence)
|
||||
print(pieces) # ['▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'ar', 'is']
|
||||
print(pieces_en) #'▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'aris'
|
||||
tokens = sp.encode(sentence)
|
||||
tokens_en = sp_en.encode(sentence)
|
||||
print(tokens) # [338, 493, 587, 349, 807, 429, 373, 279, 282]
|
||||
print(tokens_en) #364, 490, 5981, 398, 2036, 32046, 386, 9020]
|
||||
# 解码后的结果应等于编码前的输入
|
||||
# assert sp.decode(tokens) == sentence
|
||||
Reference in New Issue
Block a user