admin/LLM-from-scratch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[2025.06.17] update train code

Browse Source
This commit is contained in:
root
2025-06-17 17:17:49 +08:00
parent c4f3a25653
commit ae76142fdc
17 changed files with 1517 additions and 261413 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src_dev/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py
+1 -1
View File
@@ -13,7 +13,7 @@ from tokenizers import (
from transformers import PreTrainedTokenizerFast
import logging
from tqdm import tqdm
log_dir = "/mnt/hanluzhi/dolly/logs"
log_dir = "../logs"
os.makedirs(log_dir, exist_ok=True)
logging.basicConfig(