[2025.06.17] update train code

src_dev/src_pretrain/pretrain_deepspeed.py

@@ -19,10 +19,10 @@ logger = logging.getLogger(__name__)
 accelerator = Accelerator()
 set_seed(42)
 
-model_name = "/mnt/han.luzhi/dolly_llm/weights_llm"  
+model_name = "../weights_llm"  
 resume_option = None                     
 # resume_option = True                    
-# resume_option = "/mnt/han.luzhi/dolly_llm/checkpoints_ds/checkpoint-500"  
+# resume_option = "../checkpoints_ds/checkpoint-500"  
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 if tokenizer.pad_token is None:
     tokenizer.pad_token = tokenizer.eos_token
@@ -36,11 +36,11 @@ else:
     model = DollyForCausalLM.from_pretrained(model_name)
 model.gradient_checkpointing_enable()
 
-data_files = glob.glob("/mnt/han.luzhi/dolly_llm/corpus/*.jsonl")
+data_files = glob.glob("../corpus/*.jsonl")
 logger.info(f"找到训练语料: {data_files}")
 
 dataset = load_dataset('json', data_files=data_files, split='train')
-# dataset = load_dataset('text', data_files={'train': '/mnt/han.luzhi/dolly_llm/corpus/wikipedia.txt'})
+# dataset = load_dataset('text', data_files={'train': '../corpus/wikipedia.txt'})
 
 def preprocess_function(examples):
     return tokenizer(
@@ -65,15 +65,15 @@ data_collator = DataCollatorForLanguageModeling(
 )
 
 training_args = TrainingArguments(
-    output_dir="/mnt/han.luzhi/dolly_llm/checkpoints_ds",
-    deepspeed="/mnt/han.luzhi/dolly_llm/deepspeed_config/ds_stage2.json",  
+    output_dir="../checkpoints_ds",
+    deepspeed="../deepspeed_config/ds_stage2.json",  
     do_eval=False,
     save_strategy="steps",
     save_steps=500,
     learning_rate=1e-4,
     per_device_train_batch_size=36,
     num_train_epochs=2,
-    logging_dir="/mnt/han.luzhi/dolly_llm/logs",
+    logging_dir="../logs",
     logging_steps=20,
     save_total_limit=2,
     fp16=True,  
@@ -112,8 +112,8 @@ else:
 
 
 if accelerator.is_main_process: 
-    trainer.save_model("/mnt/han.luzhi/dolly_llm/result/final")
-    tokenizer.save_pretrained("/mnt/han.luzhi/dolly_llm/result/final")
+    trainer.save_model("../result/final")
+    tokenizer.save_pretrained("../result/final")
     logger.info("***** 训练完成，模型已保存 *****")
 
 ##  deepspeed --num_gpus=4 --master_port=12345 pretrain_transformers.py

src_dev/src_pretrain/pretrain_deepspeed_MoE.py

@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 accelerator = Accelerator()
 set_seed(42)
 
-model_name = "/mnt/han.luzhi/dolly/weights_MoE"  
+model_name = "../dolly/weights_MoE"  
 resume_option = None                     
 # resume_option = True                    
 # resume_option = "/mnt/han.luzhi/dolly_llm/checkpoints_ds/checkpoint-500"  
@@ -36,7 +36,7 @@ else:
     model = DollyMoEForCausalLM.from_pretrained(model_name)
 model.gradient_checkpointing_enable()
 
-data_files = glob.glob("/mnt/han.luzhi/dolly/corpus/*.jsonl")
+data_files = glob.glob("../dolly/corpus/*.jsonl")
 logger.info(f"找到训练语料: {data_files}")
 
 dataset = load_dataset('json', data_files=data_files, split='train')
@@ -65,15 +65,15 @@ data_collator = DataCollatorForLanguageModeling(
 )
 
 training_args = TrainingArguments(
-    output_dir="/mnt/han.luzhi/dolly/checkpoints_MoE",
-    deepspeed="/mnt/han.luzhi/dolly/deepspeed_config/ds_stage2.json",  
+    output_dir="../checkpoints_MoE",
+    deepspeed="../deepspeed_config/ds_stage2.json",  
     do_eval=False,
     save_strategy="steps",
     save_steps=500,
     learning_rate=1e-4,
     per_device_train_batch_size=16,
     num_train_epochs=2,
-    logging_dir="/mnt/han.luzhi/dolly/logs",
+    logging_dir="../dolly/logs",
     logging_steps=20,
     save_total_limit=2,
     fp16=True,  
@@ -112,8 +112,8 @@ else:
 
 
 if accelerator.is_main_process: 
-    trainer.save_model("/mnt/han.luzhi/dolly/chechpoints_MoE/final")
-    tokenizer.save_pretrained("/mnt/han.luzhi/dolly/chechpoints_MoE/final")
+    trainer.save_model("../chechpoints_MoE/final")
+    tokenizer.save_pretrained("../chechpoints_MoE/final")
     logger.info("***** 训练完成，模型已保存 *****")
 
 ## deepspeed --num_gpus=4 --master_port=12345 pretrain_transformers.py

src_dev/src_pretrain/pretrain_transformers.py

@@ -17,11 +17,11 @@ logger = logging.getLogger(__name__)
 accelerator = Accelerator()
 set_seed(42)
 
-model_name = "/mnt/han.luzhi/dolly_llm/weights_llm"  
+model_name = "../weights_llm"  
 # resume_option = None                     
 # resume_option = True                    
-resume_option = "/mnt/han.luzhi/dolly_llm/result/checkpoint-59000"  
-# resume_option = "/mnt/han.luzhi/dolly_llm/result/final" 
+resume_option = "../result/checkpoint-59000"  
+# resume_option = "../result/final" 
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 if tokenizer.pad_token is None:
     tokenizer.pad_token = tokenizer.eos_token
@@ -36,7 +36,7 @@ else:
 
 model.gradient_checkpointing_enable()
 
-dataset = load_dataset('text', data_files={'train': '/mnt/han.luzhi/dolly_llm/corpus/wikipedia.txt'})
+dataset = load_dataset('text', data_files={'train': '../corpus/wikipedia.txt'})
 
 def preprocess_function(examples):
     return tokenizer(
@@ -60,14 +60,14 @@ data_collator = DataCollatorForLanguageModeling(
 )
 
 training_args = TrainingArguments(
-    output_dir="/mnt/han.luzhi/dolly_llm/result",
+    output_dir="../result",
     do_eval=False,  
     save_strategy="steps", 
     save_steps=500,
     learning_rate=5e-5,
     per_device_train_batch_size=26,
     num_train_epochs=5,
-    logging_dir="/mnt/han.luzhi/dolly_llm/logs",
+    logging_dir="../logs",
     logging_steps=10,
     save_total_limit=2,
     fp16=True, 
@@ -102,8 +102,8 @@ logger.info(f" 总批大小 = {training_args.per_device_train_batch_size * accel
 trainer.train(resume_from_checkpoint=resume_option)
 
 if accelerator.is_main_process: 
-    trainer.save_model("/mnt/han.luzhi/dolly_llm/result/final")
-    tokenizer.save_pretrained("/mnt/han.luzhi/dolly_llm/result/final")
+    trainer.save_model("../result/final")
+    tokenizer.save_pretrained("../result/final")
     logger.info("***** 训练完成，模型已保存 *****")
 
 ##  python3 -m torch.distributed.launch --nproc_per_node=4 --master_port=12345 pretrain_transformers.py

src_dev/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py

@@ -13,7 +13,7 @@ from tokenizers import (
 from transformers import PreTrainedTokenizerFast
 import logging
 from tqdm import tqdm
-log_dir = "/mnt/hanluzhi/dolly/logs"
+log_dir = "../logs"
 os.makedirs(log_dir, exist_ok=True)
 
 logging.basicConfig(