From c5ab40c2dfe7bb2f39bf14fcddd59819abd4f4ed Mon Sep 17 00:00:00 2001 From: coffee Date: Wed, 7 May 2025 11:24:51 +0800 Subject: [PATCH] update tokenizer code --- src_tokenizer/README.md | 3 ++ .../step1.1_hf_train_from_old_largefile.py | 23 +++++++++++++ .../step1_hf_train_from_old.py | 16 +++++++++ .../step2_hf_train_from_scratch_BPEandBBPE.py | 28 ++++++++++++++++ .../step3_hf_add_new_token.py | 10 ++++++ .../step1_spm_train_tokenizer.py | 22 +++++++++++++ .../step2_spm_merge_tokenizer.py | 33 +++++++++++++++++++ .../spm_tokenizer_code/step3_spm_to_hf.py | 18 ++++++++++ .../step4_spm_expand_vocab.py | 21 ++++++++++++ .../step5_test_spm_tokenizer.py | 15 +++++++++ 10 files changed, 189 insertions(+) create mode 100755 src_tokenizer/README.md create mode 100755 src_tokenizer/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py create mode 100755 src_tokenizer/hf_tokenizer_code/step1_hf_train_from_old.py create mode 100755 src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py create mode 100755 src_tokenizer/hf_tokenizer_code/step3_hf_add_new_token.py create mode 100755 src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py create mode 100755 src_tokenizer/spm_tokenizer_code/step2_spm_merge_tokenizer.py create mode 100755 src_tokenizer/spm_tokenizer_code/step3_spm_to_hf.py create mode 100755 src_tokenizer/spm_tokenizer_code/step4_spm_expand_vocab.py create mode 100755 src_tokenizer/spm_tokenizer_code/step5_test_spm_tokenizer.py diff --git a/src_tokenizer/README.md b/src_tokenizer/README.md new file mode 100755 index 0000000..ba5b27a --- /dev/null +++ b/src_tokenizer/README.md @@ -0,0 +1,3 @@ +# USAGE +使用sentencepiece和huggingface transformers分别进行tokenizer的训练 +support BPE and BBPE \ No newline at end of file diff --git a/src_tokenizer/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py b/src_tokenizer/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py new file mode 100755 index 0000000..bb5257c --- /dev/null +++ b/src_tokenizer/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py @@ -0,0 +1,23 @@ +from transformers import AutoTokenizer + +def get_training_corpus(file_path, batch_size=1000): + with open(file_path, "r", encoding="utf-8") as f: + batch = [] + for line in f: + batch.append(line.strip()) + if len(batch) >= batch_size: + yield batch + batch = [] + if batch: + yield batch + +old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2") +training_corpus = get_training_corpus("corpus/wikipedia.txt") + +tokenizer = old_tokenizer.train_new_from_iterator( + training_corpus, + vocab_size=34000 +) + + +tokenizer.save_pretrained("tokenizer_file/new_tokenizer2") \ No newline at end of file diff --git a/src_tokenizer/hf_tokenizer_code/step1_hf_train_from_old.py b/src_tokenizer/hf_tokenizer_code/step1_hf_train_from_old.py new file mode 100755 index 0000000..49598cf --- /dev/null +++ b/src_tokenizer/hf_tokenizer_code/step1_hf_train_from_old.py @@ -0,0 +1,16 @@ +from datasets import load_dataset +from transformers import AutoTokenizer + +dataset = load_dataset("text", data_files="corpus/wikipedia.txt", streaming=True) + +def batch_iterator(batch_size=1000): + for batch in dataset["train"].iter(batch_size): + yield batch["text"] + +old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2") +tokenizer = old_tokenizer.train_new_from_iterator( + batch_iterator(), + vocab_size=33000, +) + +tokenizer.save_pretrained("new_tokenizer") \ No newline at end of file diff --git a/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py b/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py new file mode 100755 index 0000000..3af40a3 --- /dev/null +++ b/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py @@ -0,0 +1,28 @@ +from tokenizers import Tokenizer +from tokenizers.models import BPE +from tokenizers.trainers import BpeTrainer +from tokenizers.pre_tokenizers import ByteLevel +from tokenizers.pre_tokenizers import Whitespace +from transformers import PreTrainedTokenizerFast + +tokenizer = Tokenizer(BPE(unk_token="[UNK]")) +# tokenizer.pre_tokenizer = Whitespace() +tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True) +trainer = BpeTrainer( + vocab_size=12800, + special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"], + min_frequency=2, + show_progress=True +) + +tokenizer.train(files=["corpus/wikipedia.txt"], trainer=trainer) +tokenizer.save("tokenizer_file/tokenizer.json") +fast_tokenizer = PreTrainedTokenizerFast( + tokenizer_file="tokenizer_file/tokenizer.json", + unk_token="[UNK]", + pad_token="[PAD]", + cls_token="[CLS]", + sep_token="[SEP]", + mask_token="[MASK]", +) +fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer_BBPE") \ No newline at end of file diff --git a/src_tokenizer/hf_tokenizer_code/step3_hf_add_new_token.py b/src_tokenizer/hf_tokenizer_code/step3_hf_add_new_token.py new file mode 100755 index 0000000..369a948 --- /dev/null +++ b/src_tokenizer/hf_tokenizer_code/step3_hf_add_new_token.py @@ -0,0 +1,10 @@ +from transformers import AutoTokenizer, AutoModel + +tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer") + +new_tokens = ["", ""] +tokenizer.add_tokens(new_tokens) + +text = "" +print(tokenizer.tokenize(text)) +tokenizer.save_pretrained("updated_tokenizer") \ No newline at end of file diff --git a/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py b/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py new file mode 100755 index 0000000..34ef9f0 --- /dev/null +++ b/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py @@ -0,0 +1,22 @@ +import sentencepiece as spm + +###推荐使用 utils/spm_train_tokenizer.sh，使用多核加速 + +#英文分词器 +# corpus_path = 'a.txt,b.txt' +corpus_path_en= 'corpus/pg16457.txt' +vocab_size_en = 1000 +model_en= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_en' + +# model_type 使用 bpe +model_type = 'bpe' +# byte_fallback=True，避免 out of vocabulary 问题 +spm.SentencePieceTrainer.train(input=corpus_path_en, model_prefix=model_en, + vocab_size=vocab_size_en, model_type=model_type, byte_fallback=True,character_coverage=1) + +#中文分词器 +corpus_path_zh= 'corpus/wikipedia.txt' +vocab_size_zh = 32000 +model_zh= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_zh' +spm.SentencePieceTrainer.train(input=corpus_path_zh, model_prefix=model_zh, + vocab_size=vocab_size_zh, model_type=model_type, byte_fallback=True,character_coverage=1) diff --git a/src_tokenizer/spm_tokenizer_code/step2_spm_merge_tokenizer.py b/src_tokenizer/spm_tokenizer_code/step2_spm_merge_tokenizer.py new file mode 100755 index 0000000..ebd7050 --- /dev/null +++ b/src_tokenizer/spm_tokenizer_code/step2_spm_merge_tokenizer.py @@ -0,0 +1,33 @@ +import sentencepiece as spm +from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2 + +model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_zh.model" +en_model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_en.model" + +model_file_new = "tokenizer_part/dolly_tokenizer_spm/tokenizer.model" + +chinese_sp = spm.SentencePieceProcessor(model_file=model_file) +en_sp = spm.SentencePieceProcessor(model_file=en_model_file) + +print(chinese_sp.vocab_size())#32000 + +chinese_spm = sp_model_pb2.ModelProto() +chinese_spm.ParseFromString(chinese_sp.serialized_model_proto()) + +en_spm = sp_model_pb2.ModelProto() +en_spm.ParseFromString(en_sp.serialized_model_proto()) + +chinese_sp_set = set(p.piece for p in chinese_spm.pieces) + +for p in en_spm.pieces: + piece = p.piece + if piece not in chinese_sp_set: + chinese_sp_set.add(piece) + new_piece = sp_model_pb2.ModelProto().SentencePiece() + new_piece.piece = piece + new_piece.score = 0 + chinese_spm.pieces.append(new_piece) + +print(f"after expand vocab={len(chinese_spm.pieces)}") # 32361,增加361个 +with open(model_file_new, 'wb') as file: + file.write(chinese_spm.SerializeToString()) \ No newline at end of file diff --git a/src_tokenizer/spm_tokenizer_code/step3_spm_to_hf.py b/src_tokenizer/spm_tokenizer_code/step3_spm_to_hf.py new file mode 100755 index 0000000..54f8441 --- /dev/null +++ b/src_tokenizer/spm_tokenizer_code/step3_spm_to_hf.py @@ -0,0 +1,18 @@ +from transformers import AutoTokenizer,XLMRobertaTokenizer +#由spm转为hf_tokenizer +tokenizer = XLMRobertaTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_spm/tokenizer.model") +text = "A Frenchman from Paris" +encoded_input = tokenizer.encode(text) +print(encoded_input) +decoded_output = tokenizer.decode(encoded_input) +print(decoded_output) +tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf') + +#转换为常见的tokenzier.json格式 +tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf") +text = "A Frenchman from Paris" +encoded_input = tokenizer.encode(text) +print(encoded_input) +decoded_output = tokenizer.decode(encoded_input) +print(decoded_output) +tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf2') \ No newline at end of file diff --git a/src_tokenizer/spm_tokenizer_code/step4_spm_expand_vocab.py b/src_tokenizer/spm_tokenizer_code/step4_spm_expand_vocab.py new file mode 100755 index 0000000..5cca6ce --- /dev/null +++ b/src_tokenizer/spm_tokenizer_code/step4_spm_expand_vocab.py @@ -0,0 +1,21 @@ +import sentencepiece as spm +from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2 + +sp = spm.SentencePieceProcessor(model_file='tokenizer_file/dolly_tokenizer_spm/tokenizer.model') +print(sp.vocab_size()) # 32461 + +old_model = sp_model_pb2.ModelProto() +old_model.ParseFromString(sp.serialized_model_proto()) +pieces = ['拉布拉多', '货拉拉', '拉不拉'] +existed_pieces = set(p.piece for p in old_model.pieces) +for piece in pieces: + if piece not in existed_pieces: + existed_pieces.add(piece) + new_piece = sp_model_pb2.ModelProto().SentencePiece() + new_piece.piece = piece + new_piece.score = 0 + old_model.pieces.append(new_piece) + +print(len(old_model.pieces)) # 32461 +with open('tokenizer_file/dolly_tokenizer_spm/expand_tokenizer.model', 'wb') as f: + f.write(old_model.SerializeToString()) \ No newline at end of file diff --git a/src_tokenizer/spm_tokenizer_code/step5_test_spm_tokenizer.py b/src_tokenizer/spm_tokenizer_code/step5_test_spm_tokenizer.py new file mode 100755 index 0000000..39fb692 --- /dev/null +++ b/src_tokenizer/spm_tokenizer_code/step5_test_spm_tokenizer.py @@ -0,0 +1,15 @@ +import sentencepiece as spm +sp = spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer.model') + +sp_en= spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer_en.model') +sentence = 'A Frenchman from Paris' +pieces = sp.encode_as_pieces(sentence) +pieces_en = sp_en.encode_as_pieces(sentence) +print(pieces) # ['▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'ar', 'is'] +print(pieces_en) #'▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'aris' +tokens = sp.encode(sentence) +tokens_en = sp_en.encode(sentence) +print(tokens) # [338, 493, 587, 349, 807, 429, 373, 279, 282] +print(tokens_en) #364, 490, 5981, 398, 2036, 32046, 386, 9020] +# 解码后的结果应等于编码前的输入 +# assert sp.decode(tokens) == sentence \ No newline at end of file