add pretrain code
This commit is contained in:
coffee
@@ -4,8 +4,11 @@ LLM-Dolly is a custom LLM built from scratch
|
|||||||
|
|
||||||
## 🗓️ 更新日志
|
## 🗓️ 更新日志
|
||||||
|
|
||||||
|
#### 2025-05-16
|
||||||
|
- 📝 添加dolly_llm的预训练代码，进行一次预训练测试：模型0.6B，语料500M，46GB*4显卡.
|
||||||
|
|
||||||
#### 2025-05-14
|
#### 2025-05-14
|
||||||
- ✅ 优化[Attention]代码，同时将dolly_llm作为pip包，进行安装
|
- ✅ 将dolly_llm作为pip包，进行安装
|
||||||
|
|
||||||
#### 2025-05-09
|
#### 2025-05-09
|
||||||
- 📝 优化[RMSNorm](https://arxiv.org/pdf/1910.07467)、MLP、[RoPE](https://arxiv.org/pdf/2104.09864)代码。
|
- 📝 优化[RMSNorm](https://arxiv.org/pdf/1910.07467)、MLP、[RoPE](https://arxiv.org/pdf/2104.09864)代码。
|
||||||
|
|||||||
@@ -1,12 +1,32 @@
|
|||||||
|
<<<<<<< HEAD
|
||||||
certifi==2025.4.26
|
certifi==2025.4.26
|
||||||
charset-normalizer==3.4.2
|
charset-normalizer==3.4.2
|
||||||
filelock==3.18.0
|
filelock==3.18.0
|
||||||
fsspec==2025.3.2
|
fsspec==2025.3.2
|
||||||
|
=======
|
||||||
|
accelerate==1.7.0
|
||||||
|
aiohappyeyeballs==2.6.1
|
||||||
|
aiohttp==3.11.18
|
||||||
|
aiosignal==1.3.2
|
||||||
|
attrs==25.3.0
|
||||||
|
certifi==2025.4.26
|
||||||
|
charset-normalizer==3.4.2
|
||||||
|
datasets==3.6.0
|
||||||
|
dill==0.3.8
|
||||||
|
filelock==3.18.0
|
||||||
|
frozenlist==1.6.0
|
||||||
|
fsspec==2025.3.0
|
||||||
|
>>>>>>> fc9eefc (add pretrain code)
|
||||||
huggingface-hub==0.30.2
|
huggingface-hub==0.30.2
|
||||||
idna==3.10
|
idna==3.10
|
||||||
Jinja2==3.1.6
|
Jinja2==3.1.6
|
||||||
MarkupSafe==3.0.2
|
MarkupSafe==3.0.2
|
||||||
mpmath==1.3.0
|
mpmath==1.3.0
|
||||||
|
<<<<<<< HEAD
|
||||||
|
=======
|
||||||
|
multidict==6.4.3
|
||||||
|
multiprocess==0.70.16
|
||||||
|
>>>>>>> fc9eefc (add pretrain code)
|
||||||
networkx==3.4.2
|
networkx==3.4.2
|
||||||
numpy==2.2.5
|
numpy==2.2.5
|
||||||
nvidia-cublas-cu12==12.1.3.1
|
nvidia-cublas-cu12==12.1.3.1
|
||||||
@@ -22,17 +42,41 @@ nvidia-nccl-cu12==2.20.5
|
|||||||
nvidia-nvjitlink-cu12==12.9.41
|
nvidia-nvjitlink-cu12==12.9.41
|
||||||
nvidia-nvtx-cu12==12.1.105
|
nvidia-nvtx-cu12==12.1.105
|
||||||
packaging==25.0
|
packaging==25.0
|
||||||
|
<<<<<<< HEAD
|
||||||
protobuf==3.20.0
|
protobuf==3.20.0
|
||||||
|
=======
|
||||||
|
pandas==2.2.3
|
||||||
|
propcache==0.3.1
|
||||||
|
protobuf==3.20.0
|
||||||
|
psutil==7.0.0
|
||||||
|
pyarrow==20.0.0
|
||||||
|
python-dateutil==2.9.0.post0
|
||||||
|
pytz==2025.2
|
||||||
|
>>>>>>> fc9eefc (add pretrain code)
|
||||||
PyYAML==6.0.2
|
PyYAML==6.0.2
|
||||||
regex==2024.11.6
|
regex==2024.11.6
|
||||||
requests==2.32.3
|
requests==2.32.3
|
||||||
safetensors==0.5.3
|
safetensors==0.5.3
|
||||||
sentencepiece==0.2.0
|
sentencepiece==0.2.0
|
||||||
|
<<<<<<< HEAD
|
||||||
|
=======
|
||||||
|
setuptools==78.1.1
|
||||||
|
six==1.17.0
|
||||||
|
>>>>>>> fc9eefc (add pretrain code)
|
||||||
sympy==1.14.0
|
sympy==1.14.0
|
||||||
tokenizers==0.21.1
|
tokenizers==0.21.1
|
||||||
torch==2.3.1
|
torch==2.3.1
|
||||||
tqdm==4.67.1
|
tqdm==4.67.1
|
||||||
transformers==4.51.3
|
transformers==4.51.3
|
||||||
|
<<<<<<< HEAD
|
||||||
triton==2.3.1
|
triton==2.3.1
|
||||||
typing_extensions==4.13.2
|
typing_extensions==4.13.2
|
||||||
urllib3==2.4.0
|
urllib3==2.4.0
|
||||||
|
=======
|
||||||
|
typing_extensions==4.13.2
|
||||||
|
tzdata==2025.2
|
||||||
|
urllib3==2.4.0
|
||||||
|
wheel==0.45.1
|
||||||
|
xxhash==3.5.0
|
||||||
|
yarl==1.20.0
|
||||||
|
>>>>>>> fc9eefc (add pretrain code)
|
||||||
|
|||||||
@@ -6,10 +6,18 @@ setup(
|
|||||||
packages=find_packages(include=["src_llm"]),
|
packages=find_packages(include=["src_llm"]),
|
||||||
install_requires=[
|
install_requires=[
|
||||||
"torch==2.3.1",
|
"torch==2.3.1",
|
||||||
|
<<<<<<< HEAD
|
||||||
|
=======
|
||||||
|
"datasets==3.6.0",
|
||||||
|
>>>>>>> fc9eefc (add pretrain code)
|
||||||
"sentencepiece==0.2.0",
|
"sentencepiece==0.2.0",
|
||||||
"protobuf==3.20.0",
|
"protobuf==3.20.0",
|
||||||
"transformers==4.51.3",
|
"transformers==4.51.3",
|
||||||
"tokenizers==0.21.1",
|
"tokenizers==0.21.1",
|
||||||
|
<<<<<<< HEAD
|
||||||
|
=======
|
||||||
|
"accelerate==1.7.0",
|
||||||
|
>>>>>>> fc9eefc (add pretrain code)
|
||||||
],
|
],
|
||||||
author="IP127000",
|
author="IP127000",
|
||||||
author_email="hanluzhi@outlook.com",
|
author_email="hanluzhi@outlook.com",
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||||
|
|
||||||
|
model_name = "Qwen/Qwen3-8B"
|
||||||
|
|
||||||
|
# load the tokenizer and the model
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||||
|
model = AutoModelForCausalLM.from_pretrained(
|
||||||
|
model_name,
|
||||||
|
torch_dtype="auto",
|
||||||
|
device_map="auto"
|
||||||
|
)
|
||||||
|
|
||||||
|
# prepare the model input
|
||||||
|
prompt = "Give me a short introduction to large language model."
|
||||||
|
messages = [
|
||||||
|
{"role": "user", "content": prompt}
|
||||||
|
]
|
||||||
|
text = tokenizer.apply_chat_template(
|
||||||
|
messages,
|
||||||
|
tokenize=False,
|
||||||
|
add_generation_prompt=True,
|
||||||
|
enable_thinking=True # Switches between thinking and non-thinking modes. Default is True.
|
||||||
|
)
|
||||||
|
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
|
||||||
|
|
||||||
|
# conduct text completion
|
||||||
|
generated_ids = model.generate(
|
||||||
|
**model_inputs,
|
||||||
|
max_new_tokens=32768
|
||||||
|
)
|
||||||
|
output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()
|
||||||
|
|
||||||
|
# parsing thinking content
|
||||||
|
try:
|
||||||
|
# rindex finding 151668 (</think>)
|
||||||
|
index = len(output_ids) - output_ids[::-1].index(151668)
|
||||||
|
except ValueError:
|
||||||
|
index = 0
|
||||||
|
|
||||||
|
thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n")
|
||||||
|
content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n")
|
||||||
|
|
||||||
|
print("thinking content:", thinking_content)
|
||||||
|
print("content:", content)
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import torch
|
||||||
|
from src_llm.modeling_dolly import DollyForCausalLM
|
||||||
|
from transformers import (
|
||||||
|
Trainer,
|
||||||
|
TrainingArguments,
|
||||||
|
DataCollatorForLanguageModeling,
|
||||||
|
AutoTokenizer,
|
||||||
|
set_seed
|
||||||
|
)
|
||||||
|
from datasets import load_dataset
|
||||||
|
from accelerate import Accelerator
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
accelerator = Accelerator()
|
||||||
|
set_seed(42)
|
||||||
|
|
||||||
|
model_name = "/mnt/han.luzhi/weights_llm"
|
||||||
|
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||||
|
if tokenizer.pad_token is None:
|
||||||
|
tokenizer.pad_token = tokenizer.eos_token
|
||||||
|
|
||||||
|
model = DollyForCausalLM.from_pretrained(model_name)
|
||||||
|
model.gradient_checkpointing_enable()
|
||||||
|
dataset = load_dataset('text', data_files={'train': '/mnt/han.luzhi/corpus/wikipedia.txt'})
|
||||||
|
|
||||||
|
def preprocess_function(examples):
|
||||||
|
return tokenizer(
|
||||||
|
examples['text'],
|
||||||
|
truncation=True,
|
||||||
|
max_length=512,
|
||||||
|
padding='max_length',
|
||||||
|
return_tensors='pt'
|
||||||
|
)
|
||||||
|
|
||||||
|
tokenized_datasets = dataset.map(
|
||||||
|
preprocess_function,
|
||||||
|
batched=True,
|
||||||
|
remove_columns=['text'],
|
||||||
|
num_proc=4
|
||||||
|
)
|
||||||
|
|
||||||
|
data_collator = DataCollatorForLanguageModeling(
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
mlm=False
|
||||||
|
)
|
||||||
|
|
||||||
|
training_args = TrainingArguments(
|
||||||
|
output_dir="/mnt/han.luzhi/result",
|
||||||
|
do_eval=False,
|
||||||
|
save_strategy="steps",
|
||||||
|
save_steps=500,
|
||||||
|
learning_rate=5e-5,
|
||||||
|
per_device_train_batch_size=26,
|
||||||
|
num_train_epochs=5,
|
||||||
|
logging_dir="/mnt/han.luzhi/logs",
|
||||||
|
logging_steps=10,
|
||||||
|
save_total_limit=2,
|
||||||
|
fp16=True,
|
||||||
|
dataloader_num_workers=4,
|
||||||
|
gradient_accumulation_steps=2,
|
||||||
|
report_to="tensorboard",
|
||||||
|
optim="adamw_torch",
|
||||||
|
lr_scheduler_type="cosine",
|
||||||
|
warmup_steps=500,
|
||||||
|
weight_decay=0.01,
|
||||||
|
local_rank=int(os.environ.get("LOCAL_RANK", 0)),
|
||||||
|
disable_tqdm=False,
|
||||||
|
logging_first_step=True,
|
||||||
|
log_level="info",
|
||||||
|
log_level_replica="info",
|
||||||
|
ddp_find_unused_parameters=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
trainer = Trainer(
|
||||||
|
model=model,
|
||||||
|
args=training_args,
|
||||||
|
train_dataset=tokenized_datasets['train'],
|
||||||
|
processing_class=tokenizer,
|
||||||
|
data_collator=data_collator,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("***** 开始训练 *****")
|
||||||
|
logger.info(f" 进程数 = {accelerator.num_processes}")
|
||||||
|
logger.info(f" 设备 = {accelerator.device}")
|
||||||
|
logger.info(f" 总批大小 = {training_args.per_device_train_batch_size * accelerator.num_processes * training_args.gradient_accumulation_steps}")
|
||||||
|
|
||||||
|
trainer.train()
|
||||||
|
|
||||||
|
if accelerator.is_main_process:
|
||||||
|
trainer.save_model("/mnt/han.luzhi/result/final")
|
||||||
|
tokenizer.save_pretrained("/mnt/han.luzhi/result/final")
|
||||||
|
logger.info("***** 训练完成，模型已保存 *****")
|
||||||
|
|
||||||
|
## python3 -m torch.distributed.launch --nproc_per_node=4 --master_port=12345 pretrain_transformers.py
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"Qwen3ForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 151643,
|
||||||
|
"eos_token_id": 151645,
|
||||||
|
"head_dim": 128,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 1024,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 3072,
|
||||||
|
"max_position_embeddings": 40960,
|
||||||
|
"max_window_layers": 28,
|
||||||
|
"model_type": "qwen3",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_hidden_layers": 28,
|
||||||
|
"num_key_value_heads": 8,
|
||||||
|
"rms_norm_eps": 1e-06,
|
||||||
|
"rope_scaling": null,
|
||||||
|
"rope_theta": 1000000,
|
||||||
|
"sliding_window": null,
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"torch_dtype": "bfloat16",
|
||||||
|
"transformers_version": "4.51.0",
|
||||||
|
"use_cache": true,
|
||||||
|
"use_sliding_window": false,
|
||||||
|
"vocab_size": 151936
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"bos_token_id": 151643,
|
||||||
|
"do_sample": true,
|
||||||
|
"eos_token_id": [
|
||||||
|
151645,
|
||||||
|
151643
|
||||||
|
],
|
||||||
|
"pad_token_id": 151643,
|
||||||
|
"temperature": 0.6,
|
||||||
|
"top_k": 20,
|
||||||
|
"top_p": 0.95,
|
||||||
|
"transformers_version": "4.51.0"
|
||||||
|
}
|
||||||
+151388
File diff suppressed because it is too large
Load Diff
+757480
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,239 @@
|
|||||||
|
{
|
||||||
|
"add_bos_token": false,
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"151643": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151644": {
|
||||||
|
"content": "<|im_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151645": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151646": {
|
||||||
|
"content": "<|object_ref_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151647": {
|
||||||
|
"content": "<|object_ref_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151648": {
|
||||||
|
"content": "<|box_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151649": {
|
||||||
|
"content": "<|box_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151650": {
|
||||||
|
"content": "<|quad_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151651": {
|
||||||
|
"content": "<|quad_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151652": {
|
||||||
|
"content": "<|vision_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151653": {
|
||||||
|
"content": "<|vision_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151654": {
|
||||||
|
"content": "<|vision_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151655": {
|
||||||
|
"content": "<|image_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151656": {
|
||||||
|
"content": "<|video_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151657": {
|
||||||
|
"content": "<tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151658": {
|
||||||
|
"content": "</tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151659": {
|
||||||
|
"content": "<|fim_prefix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151660": {
|
||||||
|
"content": "<|fim_middle|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151661": {
|
||||||
|
"content": "<|fim_suffix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151662": {
|
||||||
|
"content": "<|fim_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151663": {
|
||||||
|
"content": "<|repo_name|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151664": {
|
||||||
|
"content": "<|file_sep|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151665": {
|
||||||
|
"content": "<tool_response>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151666": {
|
||||||
|
"content": "</tool_response>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151667": {
|
||||||
|
"content": "<think>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151668": {
|
||||||
|
"content": "</think>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"bos_token": null,
|
||||||
|
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set content = message.content %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is defined and message.reasoning_content is not none %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in message.content %}\n {%- set content = message.content.split('</think>')[-1].lstrip('\\n') %}\n {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|im_end|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"model_max_length": 131072,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "PreTrainedTokenizer",
|
||||||
|
"unk_token": null
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user