update llm code

LLM_code/test.py

@@ -1,3 +0,0 @@
-from transformers import AutoTokenizer
-tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")  
-print(tokenizer.vocab_size)

LLM_code/test_dolly.py

@@ -1,18 +0,0 @@
-from transformers import PreTrainedModel
-import torch
-from torch import nn
-from configuration_dolly import DollyConfig
-from modeling_dolly import DollyModel
-model = DollyModel(DollyConfig())
-model.save_pretrained("./my_model")
-model.config.save_pretrained("./my_model")
-
-
-# 加载配置和模型
-config = MyModelConfig.from_pretrained("./my_model")
-model = MyModel.from_pretrained("./my_model", config=config)
-
-# 使用模型进行推理
-input_ids = torch.tensor([[101, 1024, 102]]).long()  # 示例输入
-outputs = model(input_ids)
-print(outputs)

LLM_code/test_huggingface_transformer.py

@@ -1,17 +0,0 @@
-import transformers
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from transformers import LlamaForCausalLM
-# tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
-from transformers import LlamaConfig
-config = LlamaConfig(
-    vocab_size=32000,  
-    hidden_size=4096,  
-    num_attention_heads=32,  
-    num_hidden_layers=32, 
-    intermediate_size=11008,  
-)
-
-model = LlamaForCausalLM(config)
-# model = transformers.Qwen2ForCausalLM(config)
-model_size = sum(t.numel() for t in model.parameters())
-print(f"Model Size: {model_size/1000**2:.1f}M parameters")

Tokenizer_code/README.md

@@ -1,3 +0,0 @@
-# USAGE
-使用sentencepiece和huggingface transformers分别进行tokenizer的训练
-support BPE and BBPE

Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py

@@ -1,23 +0,0 @@
-from transformers import AutoTokenizer
-
-def get_training_corpus(file_path, batch_size=1000):
-    with open(file_path, "r", encoding="utf-8") as f:
-        batch = []
-        for line in f:
-            batch.append(line.strip())  
-            if len(batch) >= batch_size:
-                yield batch
-                batch = []
-        if batch:  
-            yield batch
-
-old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")  
-training_corpus = get_training_corpus("corpus/wikipedia.txt")  
-
-tokenizer = old_tokenizer.train_new_from_iterator(
-    training_corpus,
-    vocab_size=34000
-)
-
-
-tokenizer.save_pretrained("tokenizer_file/new_tokenizer2")

Tokenizer_code/hf_tokenizer_code/step1_hf_train_from_old.py

@@ -1,16 +0,0 @@
-from datasets import load_dataset
-from transformers import AutoTokenizer
-
-dataset = load_dataset("text", data_files="corpus/wikipedia.txt", streaming=True)
-
-def batch_iterator(batch_size=1000):
-    for batch in dataset["train"].iter(batch_size):
-        yield batch["text"]
-
-old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")
-tokenizer = old_tokenizer.train_new_from_iterator(
-    batch_iterator(),
-    vocab_size=33000,
-)
-
-tokenizer.save_pretrained("new_tokenizer")

Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py

@@ -1,28 +0,0 @@
-from tokenizers import Tokenizer
-from tokenizers.models import BPE  
-from tokenizers.trainers import BpeTrainer
-from tokenizers.pre_tokenizers import ByteLevel
-from tokenizers.pre_tokenizers import Whitespace
-from transformers import PreTrainedTokenizerFast
-
-tokenizer = Tokenizer(BPE(unk_token="[UNK]"))  
-# tokenizer.pre_tokenizer = Whitespace()
-tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True)
-trainer = BpeTrainer(
-    vocab_size=12800,           
-    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],  
-    min_frequency=2,            
-    show_progress=True         
-)
-
-tokenizer.train(files=["corpus/wikipedia.txt"], trainer=trainer)
-tokenizer.save("tokenizer_file/tokenizer.json")
-fast_tokenizer = PreTrainedTokenizerFast(
-    tokenizer_file="tokenizer_file/tokenizer.json",
-    unk_token="[UNK]",
-    pad_token="[PAD]",
-    cls_token="[CLS]",
-    sep_token="[SEP]",
-    mask_token="[MASK]",
-)
-fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer_BBPE")

Tokenizer_code/hf_tokenizer_code/step3_hf_add_new_token.py

@@ -1,10 +0,0 @@
-from transformers import AutoTokenizer, AutoModel
-
-tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")
-
-new_tokens = ["<ACTION_1>", "<ACTION_2>"]
-tokenizer.add_tokens(new_tokens)
-
-text = "<ACTION_1>"
-print(tokenizer.tokenize(text))  
-tokenizer.save_pretrained("updated_tokenizer")

Tokenizer_code/spm_tokenizer_code/step1_spm_train_tokenizer.py

@@ -1,22 +0,0 @@
-import sentencepiece as spm
-
-###推荐使用 utils/spm_train_tokenizer.sh，使用多核加速
-
-#英文分词器
-# corpus_path = 'a.txt,b.txt'
-corpus_path_en= 'corpus/pg16457.txt'
-vocab_size_en = 1000
-model_en= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_en'
-
-# model_type 使用 bpe
-model_type = 'bpe'
-# byte_fallback=True，避免 out of vocabulary 问题
-spm.SentencePieceTrainer.train(input=corpus_path_en, model_prefix=model_en, 
-    vocab_size=vocab_size_en, model_type=model_type, byte_fallback=True,character_coverage=1)
-
-#中文分词器
-corpus_path_zh= 'corpus/wikipedia.txt'
-vocab_size_zh = 32000
-model_zh= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_zh'
-spm.SentencePieceTrainer.train(input=corpus_path_zh, model_prefix=model_zh, 
-    vocab_size=vocab_size_zh, model_type=model_type, byte_fallback=True,character_coverage=1)

Tokenizer_code/spm_tokenizer_code/step2_spm_merge_tokenizer.py

@@ -1,33 +0,0 @@
-import sentencepiece as spm
-from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
-
-model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_zh.model"
-en_model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_en.model"
-
-model_file_new = "tokenizer_part/dolly_tokenizer_spm/tokenizer.model"
-
-chinese_sp = spm.SentencePieceProcessor(model_file=model_file)
-en_sp = spm.SentencePieceProcessor(model_file=en_model_file)
-
-print(chinese_sp.vocab_size())#32000
-
-chinese_spm = sp_model_pb2.ModelProto()
-chinese_spm.ParseFromString(chinese_sp.serialized_model_proto())
-
-en_spm = sp_model_pb2.ModelProto()
-en_spm.ParseFromString(en_sp.serialized_model_proto())
-
-chinese_sp_set = set(p.piece for p in chinese_spm.pieces)
-
-for p in en_spm.pieces:
-    piece = p.piece
-    if piece not in chinese_sp_set:
-        chinese_sp_set.add(piece)
-        new_piece = sp_model_pb2.ModelProto().SentencePiece()
-        new_piece.piece = piece
-        new_piece.score = 0
-        chinese_spm.pieces.append(new_piece)
-
-print(f"after expand vocab={len(chinese_spm.pieces)}") # 32361,增加361个
-with open(model_file_new, 'wb') as file:
-    file.write(chinese_spm.SerializeToString())

Tokenizer_code/spm_tokenizer_code/step3_spm_to_hf.py

@@ -1,18 +0,0 @@
-from transformers import AutoTokenizer,XLMRobertaTokenizer
-#由spm转为hf_tokenizer
-tokenizer = XLMRobertaTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_spm/tokenizer.model")
-text = "A Frenchman from Paris"
-encoded_input = tokenizer.encode(text)
-print(encoded_input)
-decoded_output = tokenizer.decode(encoded_input)
-print(decoded_output)
-tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf')
-
-#转换为常见的tokenzier.json格式
-tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf")
-text = "A Frenchman from Paris"
-encoded_input = tokenizer.encode(text)
-print(encoded_input)
-decoded_output = tokenizer.decode(encoded_input)
-print(decoded_output)
-tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf2')

Tokenizer_code/spm_tokenizer_code/step4_spm_expand_vocab.py

@@ -1,21 +0,0 @@
-import sentencepiece as spm
-from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
-
-sp = spm.SentencePieceProcessor(model_file='tokenizer_file/dolly_tokenizer_spm/tokenizer.model')
-print(sp.vocab_size())  # 32461
-
-old_model = sp_model_pb2.ModelProto()
-old_model.ParseFromString(sp.serialized_model_proto())
-pieces = ['拉布拉多', '货拉拉', '拉不拉']
-existed_pieces = set(p.piece for p in old_model.pieces)
-for piece in pieces:
-    if piece not in existed_pieces:
-        existed_pieces.add(piece)
-        new_piece = sp_model_pb2.ModelProto().SentencePiece()
-        new_piece.piece = piece
-        new_piece.score = 0
-        old_model.pieces.append(new_piece)
-    
-print(len(old_model.pieces))  # 32461
-with open('tokenizer_file/dolly_tokenizer_spm/expand_tokenizer.model', 'wb') as f:
-    f.write(old_model.SerializeToString())

Tokenizer_code/spm_tokenizer_code/step5_test_spm_tokenizer.py

@@ -1,15 +0,0 @@
-import sentencepiece as spm
-sp = spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer.model')
-
-sp_en= spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer_en.model')
-sentence = 'A Frenchman from Paris'
-pieces = sp.encode_as_pieces(sentence)
-pieces_en = sp_en.encode_as_pieces(sentence)
-print(pieces)  # ['▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'ar', 'is']
-print(pieces_en) #'▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'aris'
-tokens = sp.encode(sentence)
-tokens_en = sp_en.encode(sentence)
-print(tokens)  # [338, 493, 587, 349, 807, 429, 373, 279, 282]
-print(tokens_en) #364, 490, 5981, 398, 2036, 32046, 386, 9020] 
-# 解码后的结果应等于编码前的输入
-# assert sp.decode(tokens) == sentence

src_llm/configuration_dolly.py

@@ -1,5 +1,5 @@
 from transformers import PretrainedConfig
-import logging
+from transformers import logging
 
 logger = logging.getLogger(__name__)
 

src_llm/modeling_dolly.py

@@ -43,7 +43,7 @@ class DollyMLP(nn.Module):
         self.intermediate_size = config.intermediate_size
         self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
         self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
-        self.down_proj = nn.Linear(self.inDynamicCachetermediate_size, self.hidden_size, bias=False)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
         self.act_fn = ACT2FN[config.hidden_act]
 
     def forward(self, x):

tokenizer_file/hf_tokenizer/special_tokens_map.json

@@ -1,7 +0,0 @@
-{
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
-}

tokenizer_file/hf_tokenizer/tokenizer_config.json

@@ -1,53 +0,0 @@
-{
-  "added_tokens_decoder": {
-    "0": {
-      "content": "[UNK]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "[CLS]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "[SEP]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "[PAD]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": "[MASK]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "clean_up_tokenization_spaces": false,
-  "cls_token": "[CLS]",
-  "extra_special_tokens": {},
-  "mask_token": "[MASK]",
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "tokenizer_class": "PreTrainedTokenizer",
-  "unk_token": "[UNK]"
-}

tokenizer_file/hf_tokenizer_BBPE/special_tokens_map.json

@@ -1,7 +0,0 @@
-{
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
-}

tokenizer_file/hf_tokenizer_BBPE/tokenizer_config.json

@@ -1,53 +0,0 @@
-{
-  "added_tokens_decoder": {
-    "0": {
-      "content": "[UNK]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "[CLS]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "[SEP]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "[PAD]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": "[MASK]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "clean_up_tokenization_spaces": false,
-  "cls_token": "[CLS]",
-  "extra_special_tokens": {},
-  "mask_token": "[MASK]",
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "tokenizer_class": "PreTrainedTokenizer",
-  "unk_token": "[UNK]"
-}