add pretrain code

2025-05-19 08:37:39 +08:00
parent b48ff8a049
commit 1ee644e8ae
10 changed files with 151870 additions and 1 deletions
@@ -6,10 +6,18 @@ setup(
    packages=find_packages(include=["src_llm"]),  
    install_requires=[
        "torch==2.3.1",
+<<<<<<< HEAD
+=======
+        "datasets==3.6.0",
+>>>>>>> fc9eefc (add pretrain code)
        "sentencepiece==0.2.0",
        "protobuf==3.20.0",
        "transformers==4.51.3",
        "tokenizers==0.21.1",
+<<<<<<< HEAD
+=======
+        "accelerate==1.7.0",
+>>>>>>> fc9eefc (add pretrain code)
        ],  
    author="IP127000",  
    author_email="hanluzhi@outlook.com",