add tokenizer code

tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py

@@ -0,0 +1,23 @@
+from transformers import AutoTokenizer
+
+def get_training_corpus(file_path, batch_size=1000):
+    with open(file_path, "r", encoding="utf-8") as f:
+        batch = []
+        for line in f:
+            batch.append(line.strip())  
+            if len(batch) >= batch_size:
+                yield batch
+                batch = []
+        if batch:  
+            yield batch
+
+old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")  
+training_corpus = get_training_corpus("corpus/wikipedia.txt")  
+
+tokenizer = old_tokenizer.train_new_from_iterator(
+    training_corpus,
+    vocab_size=34000
+)
+
+
+tokenizer.save_pretrained("new_tokenizer2")

tokenizer_code/hf_tokenizer_code/step1_hf_train_from_old.py

@@ -0,0 +1,16 @@
+from datasets import load_dataset
+from transformers import AutoTokenizer
+
+dataset = load_dataset("text", data_files="corpus/wikipedia.txt", streaming=True)
+
+def batch_iterator(batch_size=1000):
+    for batch in dataset["train"].iter(batch_size):
+        yield batch["text"]
+
+old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")
+tokenizer = old_tokenizer.train_new_from_iterator(
+    batch_iterator(),
+    vocab_size=33000,
+)
+
+tokenizer.save_pretrained("new_tokenizer")

tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py

@@ -0,0 +1,26 @@
+from tokenizers import Tokenizer
+from tokenizers.models import BPE  
+from tokenizers.trainers import BpeTrainer
+from tokenizers.pre_tokenizers import Whitespace
+from transformers import PreTrainedTokenizerFast
+
+tokenizer = Tokenizer(BPE(unk_token="[UNK]"))  
+tokenizer.pre_tokenizer = Whitespace()
+trainer = BpeTrainer(
+    vocab_size=30000,           
+    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],  
+    min_frequency=2,            
+    show_progress=True         
+)
+
+tokenizer.train(files=["corpus/wikipedia.txt"], trainer=trainer)
+tokenizer.save("tokenizer_file/tokenizer.json")
+fast_tokenizer = PreTrainedTokenizerFast(
+    tokenizer_file="tokenizer_file/tokenizer.json",
+    unk_token="[UNK]",
+    pad_token="[PAD]",
+    cls_token="[CLS]",
+    sep_token="[SEP]",
+    mask_token="[MASK]",
+)
+fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer")

tokenizer_code/hf_tokenizer_code/step3_hf_add_new_token.py

@@ -0,0 +1,10 @@
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")
+
+new_tokens = ["<ACTION_1>", "<ACTION_2>"]
+tokenizer.add_tokens(new_tokens)
+
+text = "<ACTION_1>"
+print(tokenizer.tokenize(text))  
+tokenizer.save_pretrained("updated_tokenizer")

tokenizer_code/spm_tokenizer_code/step1_spm_train_tokenizer.py

@@ -0,0 +1,22 @@
+import sentencepiece as spm
+
+###推荐使用 utils/spm_train_tokenizer.sh，使用多核加速
+
+#英文分词器
+# corpus_path = 'a.txt,b.txt'
+corpus_path_en= 'corpus/pg16457.txt'
+vocab_size_en = 1000
+model_en= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_en'
+
+# model_type 使用 bpe
+model_type = 'bpe'
+# byte_fallback=True，避免 out of vocabulary 问题
+spm.SentencePieceTrainer.train(input=corpus_path_en, model_prefix=model_en, 
+    vocab_size=vocab_size_en, model_type=model_type, byte_fallback=True,character_coverage=1)
+
+#中文分词器
+corpus_path_zh= 'corpus/wikipedia.txt'
+vocab_size_zh = 32000
+model_zh= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_zh'
+spm.SentencePieceTrainer.train(input=corpus_path_zh, model_prefix=model_zh, 
+    vocab_size=vocab_size_zh, model_type=model_type, byte_fallback=True,character_coverage=1)

tokenizer_code/spm_tokenizer_code/step2_spm_merge_tokenizer.py

@@ -0,0 +1,33 @@
+import sentencepiece as spm
+from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
+
+model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_zh.model"
+en_model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_en.model"
+
+model_file_new = "tokenizer_part/dolly_tokenizer_spm/tokenizer.model"
+
+chinese_sp = spm.SentencePieceProcessor(model_file=model_file)
+en_sp = spm.SentencePieceProcessor(model_file=en_model_file)
+
+print(chinese_sp.vocab_size())#32000
+
+chinese_spm = sp_model_pb2.ModelProto()
+chinese_spm.ParseFromString(chinese_sp.serialized_model_proto())
+
+en_spm = sp_model_pb2.ModelProto()
+en_spm.ParseFromString(en_sp.serialized_model_proto())
+
+chinese_sp_set = set(p.piece for p in chinese_spm.pieces)
+
+for p in en_spm.pieces:
+    piece = p.piece
+    if piece not in chinese_sp_set:
+        chinese_sp_set.add(piece)
+        new_piece = sp_model_pb2.ModelProto().SentencePiece()
+        new_piece.piece = piece
+        new_piece.score = 0
+        chinese_spm.pieces.append(new_piece)
+
+print(f"after expand vocab={len(chinese_spm.pieces)}") # 32361,增加361个
+with open(model_file_new, 'wb') as file:
+    file.write(chinese_spm.SerializeToString())

tokenizer_code/spm_tokenizer_code/step3_spm_to_hf.py

@@ -0,0 +1,18 @@
+from transformers import AutoTokenizer,XLMRobertaTokenizer
+#由spm转为hf_tokenizer
+tokenizer = XLMRobertaTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_spm/tokenizer.model")
+text = "A Frenchman from Paris"
+encoded_input = tokenizer.encode(text)
+print(encoded_input)
+decoded_output = tokenizer.decode(encoded_input)
+print(decoded_output)
+tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf')
+
+#转换为常见的tokenzier.json格式
+tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf")
+text = "A Frenchman from Paris"
+encoded_input = tokenizer.encode(text)
+print(encoded_input)
+decoded_output = tokenizer.decode(encoded_input)
+print(decoded_output)
+tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf2')

tokenizer_code/spm_tokenizer_code/step4_spm_expand_vocab.py

@@ -0,0 +1,21 @@
+import sentencepiece as spm
+from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
+
+sp = spm.SentencePieceProcessor(model_file='tokenizer_file/dolly_tokenizer_spm/tokenizer.model')
+print(sp.vocab_size())  # 32461
+
+old_model = sp_model_pb2.ModelProto()
+old_model.ParseFromString(sp.serialized_model_proto())
+pieces = ['拉布拉多', '货拉拉', '拉不拉']
+existed_pieces = set(p.piece for p in old_model.pieces)
+for piece in pieces:
+    if piece not in existed_pieces:
+        existed_pieces.add(piece)
+        new_piece = sp_model_pb2.ModelProto().SentencePiece()
+        new_piece.piece = piece
+        new_piece.score = 0
+        old_model.pieces.append(new_piece)
+    
+print(len(old_model.pieces))  # 32461
+with open('tokenizer_file/dolly_tokenizer_spm/expand_tokenizer.model', 'wb') as f:
+    f.write(old_model.SerializeToString())

tokenizer_code/spm_tokenizer_code/test_spm_tokenizer.py

@@ -0,0 +1,15 @@
+import sentencepiece as spm
+sp = spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer.model')
+
+sp_en= spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer_en.model')
+sentence = 'A Frenchman from Paris'
+pieces = sp.encode_as_pieces(sentence)
+pieces_en = sp_en.encode_as_pieces(sentence)
+print(pieces)  # ['▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'ar', 'is']
+print(pieces_en) #'▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'aris'
+tokens = sp.encode(sentence)
+tokens_en = sp_en.encode(sentence)
+print(tokens)  # [338, 493, 587, 349, 807, 429, 373, 279, 282]
+print(tokens_en) #364, 490, 5981, 398, 2036, 32046, 386, 9020] 
+# 解码后的结果应等于编码前的输入
+# assert sp.decode(tokens) == sentence