add pretrain code
This commit is contained in:
coffee
@@ -4,8 +4,11 @@ LLM-Dolly is a custom LLM built from scratch
|
||||
|
||||
## 🗓️ 更新日志
|
||||
|
||||
#### 2025-05-16
|
||||
- 📝 添加dolly_llm的预训练代码，进行一次预训练测试：模型0.6B，语料500M，46GB*4显卡.
|
||||
|
||||
#### 2025-05-14
|
||||
- ✅ 优化[Attention]代码，同时将dolly_llm作为pip包，进行安装
|
||||
- ✅ 将dolly_llm作为pip包，进行安装
|
||||
|
||||
#### 2025-05-09
|
||||
- 📝 优化[RMSNorm](https://arxiv.org/pdf/1910.07467)、MLP、[RoPE](https://arxiv.org/pdf/2104.09864)代码。
|
||||
|
||||
@@ -1,12 +1,32 @@
|
||||
<<<<<<< HEAD
|
||||
certifi==2025.4.26
|
||||
charset-normalizer==3.4.2
|
||||
filelock==3.18.0
|
||||
fsspec==2025.3.2
|
||||
=======
|
||||
accelerate==1.7.0
|
||||
aiohappyeyeballs==2.6.1
|
||||
aiohttp==3.11.18
|
||||
aiosignal==1.3.2
|
||||
attrs==25.3.0
|
||||
certifi==2025.4.26
|
||||
charset-normalizer==3.4.2
|
||||
datasets==3.6.0
|
||||
dill==0.3.8
|
||||
filelock==3.18.0
|
||||
frozenlist==1.6.0
|
||||
fsspec==2025.3.0
|
||||
>>>>>>> fc9eefc (add pretrain code)
|
||||
huggingface-hub==0.30.2
|
||||
idna==3.10
|
||||
Jinja2==3.1.6
|
||||
MarkupSafe==3.0.2
|
||||
mpmath==1.3.0
|
||||
<<<<<<< HEAD
|
||||
=======
|
||||
multidict==6.4.3
|
||||
multiprocess==0.70.16
|
||||
>>>>>>> fc9eefc (add pretrain code)
|
||||
networkx==3.4.2
|
||||
numpy==2.2.5
|
||||
nvidia-cublas-cu12==12.1.3.1
|
||||
@@ -22,17 +42,41 @@ nvidia-nccl-cu12==2.20.5
|
||||
nvidia-nvjitlink-cu12==12.9.41
|
||||
nvidia-nvtx-cu12==12.1.105
|
||||
packaging==25.0
|
||||
<<<<<<< HEAD
|
||||
protobuf==3.20.0
|
||||
=======
|
||||
pandas==2.2.3
|
||||
propcache==0.3.1
|
||||
protobuf==3.20.0
|
||||
psutil==7.0.0
|
||||
pyarrow==20.0.0
|
||||
python-dateutil==2.9.0.post0
|
||||
pytz==2025.2
|
||||
>>>>>>> fc9eefc (add pretrain code)
|
||||
PyYAML==6.0.2
|
||||
regex==2024.11.6
|
||||
requests==2.32.3
|
||||
safetensors==0.5.3
|
||||
sentencepiece==0.2.0
|
||||
<<<<<<< HEAD
|
||||
=======
|
||||
setuptools==78.1.1
|
||||
six==1.17.0
|
||||
>>>>>>> fc9eefc (add pretrain code)
|
||||
sympy==1.14.0
|
||||
tokenizers==0.21.1
|
||||
torch==2.3.1
|
||||
tqdm==4.67.1
|
||||
transformers==4.51.3
|
||||
<<<<<<< HEAD
|
||||
triton==2.3.1
|
||||
typing_extensions==4.13.2
|
||||
urllib3==2.4.0
|
||||
=======
|
||||
typing_extensions==4.13.2
|
||||
tzdata==2025.2
|
||||
urllib3==2.4.0
|
||||
wheel==0.45.1
|
||||
xxhash==3.5.0
|
||||
yarl==1.20.0
|
||||
>>>>>>> fc9eefc (add pretrain code)
|
||||
|
||||
@@ -6,10 +6,18 @@ setup(
|
||||
packages=find_packages(include=["src_llm"]),
|
||||
install_requires=[
|
||||
"torch==2.3.1",
|
||||
<<<<<<< HEAD
|
||||
=======
|
||||
"datasets==3.6.0",
|
||||
>>>>>>> fc9eefc (add pretrain code)
|
||||
"sentencepiece==0.2.0",
|
||||
"protobuf==3.20.0",
|
||||
"transformers==4.51.3",
|
||||
"tokenizers==0.21.1",
|
||||
<<<<<<< HEAD
|
||||
=======
|
||||
"accelerate==1.7.0",
|
||||
>>>>>>> fc9eefc (add pretrain code)
|
||||
],
|
||||
author="IP127000",
|
||||
author_email="hanluzhi@outlook.com",
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
model_name = "Qwen/Qwen3-8B"
|
||||
|
||||
# load the tokenizer and the model
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name,
|
||||
torch_dtype="auto",
|
||||
device_map="auto"
|
||||
)
|
||||
|
||||
# prepare the model input
|
||||
prompt = "Give me a short introduction to large language model."
|
||||
messages = [
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=True # Switches between thinking and non-thinking modes. Default is True.
|
||||
)
|
||||
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
|
||||
|
||||
# conduct text completion
|
||||
generated_ids = model.generate(
|
||||
**model_inputs,
|
||||
max_new_tokens=32768
|
||||
)
|
||||
output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()
|
||||
|
||||
# parsing thinking content
|
||||
try:
|
||||
# rindex finding 151668 (</think>)
|
||||
index = len(output_ids) - output_ids[::-1].index(151668)
|
||||
except ValueError:
|
||||
index = 0
|
||||
|
||||
thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n")
|
||||
content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n")
|
||||
|
||||
print("thinking content:", thinking_content)
|
||||
print("content:", content)
|
||||
@@ -0,0 +1,99 @@
|
||||
import torch
|
||||
from src_llm.modeling_dolly import DollyForCausalLM
|
||||
from transformers import (
|
||||
Trainer,
|
||||
TrainingArguments,
|
||||
DataCollatorForLanguageModeling,
|
||||
AutoTokenizer,
|
||||
set_seed
|
||||
)
|
||||
from datasets import load_dataset
|
||||
from accelerate import Accelerator
|
||||
import os
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
accelerator = Accelerator()
|
||||
set_seed(42)
|
||||
|
||||
model_name = "/mnt/han.luzhi/weights_llm"
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
if tokenizer.pad_token is None:
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
|
||||
model = DollyForCausalLM.from_pretrained(model_name)
|
||||
model.gradient_checkpointing_enable()
|
||||
dataset = load_dataset('text', data_files={'train': '/mnt/han.luzhi/corpus/wikipedia.txt'})
|
||||
|
||||
def preprocess_function(examples):
|
||||
return tokenizer(
|
||||
examples['text'],
|
||||
truncation=True,
|
||||
max_length=512,
|
||||
padding='max_length',
|
||||
return_tensors='pt'
|
||||
)
|
||||
|
||||
tokenized_datasets = dataset.map(
|
||||
preprocess_function,
|
||||
batched=True,
|
||||
remove_columns=['text'],
|
||||
num_proc=4
|
||||
)
|
||||
|
||||
data_collator = DataCollatorForLanguageModeling(
|
||||
tokenizer=tokenizer,
|
||||
mlm=False
|
||||
)
|
||||
|
||||
training_args = TrainingArguments(
|
||||
output_dir="/mnt/han.luzhi/result",
|
||||
do_eval=False,
|
||||
save_strategy="steps",
|
||||
save_steps=500,
|
||||
learning_rate=5e-5,
|
||||
per_device_train_batch_size=26,
|
||||
num_train_epochs=5,
|
||||
logging_dir="/mnt/han.luzhi/logs",
|
||||
logging_steps=10,
|
||||
save_total_limit=2,
|
||||
fp16=True,
|
||||
dataloader_num_workers=4,
|
||||
gradient_accumulation_steps=2,
|
||||
report_to="tensorboard",
|
||||
optim="adamw_torch",
|
||||
lr_scheduler_type="cosine",
|
||||
warmup_steps=500,
|
||||
weight_decay=0.01,
|
||||
local_rank=int(os.environ.get("LOCAL_RANK", 0)),
|
||||
disable_tqdm=False,
|
||||
logging_first_step=True,
|
||||
log_level="info",
|
||||
log_level_replica="info",
|
||||
ddp_find_unused_parameters=False,
|
||||
)
|
||||
|
||||
trainer = Trainer(
|
||||
model=model,
|
||||
args=training_args,
|
||||
train_dataset=tokenized_datasets['train'],
|
||||
processing_class=tokenizer,
|
||||
data_collator=data_collator,
|
||||
)
|
||||
|
||||
logger.info("***** 开始训练 *****")
|
||||
logger.info(f" 进程数 = {accelerator.num_processes}")
|
||||
logger.info(f" 设备 = {accelerator.device}")
|
||||
logger.info(f" 总批大小 = {training_args.per_device_train_batch_size * accelerator.num_processes * training_args.gradient_accumulation_steps}")
|
||||
|
||||
trainer.train()
|
||||
|
||||
if accelerator.is_main_process:
|
||||
trainer.save_model("/mnt/han.luzhi/result/final")
|
||||
tokenizer.save_pretrained("/mnt/han.luzhi/result/final")
|
||||
logger.info("***** 训练完成，模型已保存 *****")
|
||||
|
||||
## python3 -m torch.distributed.launch --nproc_per_node=4 --master_port=12345 pretrain_transformers.py
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151645,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 1024,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 3072,
|
||||
"max_position_embeddings": 40960,
|
||||
"max_window_layers": 28,
|
||||
"model_type": "qwen3",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 28,
|
||||
"num_key_value_heads": 8,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 1000000,
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.51.0",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 151643,
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"temperature": 0.6,
|
||||
"top_k": 20,
|
||||
"top_p": 0.95,
|
||||
"transformers_version": "4.51.0"
|
||||
}
|
||||
+151388
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,239 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151646": {
|
||||
"content": "<|object_ref_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151647": {
|
||||
"content": "<|object_ref_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151648": {
|
||||
"content": "<|box_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151649": {
|
||||
"content": "<|box_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151650": {
|
||||
"content": "<|quad_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151651": {
|
||||
"content": "<|quad_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151652": {
|
||||
"content": "<|vision_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151653": {
|
||||
"content": "<|vision_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151654": {
|
||||
"content": "<|vision_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151655": {
|
||||
"content": "<|image_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151656": {
|
||||
"content": "<|video_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151657": {
|
||||
"content": "<tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151658": {
|
||||
"content": "</tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151659": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151660": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151661": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151662": {
|
||||
"content": "<|fim_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151663": {
|
||||
"content": "<|repo_name|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151664": {
|
||||
"content": "<|file_sep|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151665": {
|
||||
"content": "<tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151666": {
|
||||
"content": "</tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151667": {
|
||||
"content": "<think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151668": {
|
||||
"content": "</think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set content = message.content %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is defined and message.reasoning_content is not none %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in message.content %}\n {%- set content = message.content.split('</think>')[-1].lstrip('\\n') %}\n {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "PreTrainedTokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user