From c6e171e8ed408facbae5668cc89b139cc04740eb Mon Sep 17 00:00:00 2001 From: coffee Date: Tue, 10 Jun 2025 14:04:57 +0800 Subject: [PATCH] [2025.06.10] adjust directory structure --- {src_inference => src_dev/src_inference}/dolly_inference.py | 0 {src_llm => src_dev/src_llm}/__init__.py | 0 {src_llm => src_dev/src_llm}/configuration_dolly.py | 0 {src_llm => src_dev/src_llm}/modeling_dolly.py | 0 {src_llm_moe => src_dev/src_llm_moe}/__init__.py | 0 {src_llm_moe => src_dev/src_llm_moe}/configuration_dollyMoE.py | 0 {src_llm_moe => src_dev/src_llm_moe}/modeling_dollyMoE.py | 0 {src_pretrain => src_dev/src_pretrain}/pretrain_deepspeed.py | 0 {src_pretrain => src_dev/src_pretrain}/pretrain_deepspeed_MoE.py | 0 {src_pretrain => src_dev/src_pretrain}/pretrain_transformers.py | 0 {src_tokenizer => src_dev/src_tokenizer}/README.md | 0 .../hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py | 0 .../src_tokenizer}/hf_tokenizer_code/step1_hf_train_from_old.py | 0 .../hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py | 0 .../src_tokenizer}/hf_tokenizer_code/step3_hf_add_new_token.py | 0 .../spm_tokenizer_code/step1_spm_train_tokenizer.py | 0 .../spm_tokenizer_code/step2_spm_merge_tokenizer.py | 0 .../src_tokenizer}/spm_tokenizer_code/step3_spm_to_hf.py | 0 .../src_tokenizer}/spm_tokenizer_code/step4_spm_expand_vocab.py | 0 .../src_tokenizer}/spm_tokenizer_code/step5_test_spm_tokenizer.py | 0 {weights_llm => weights/weights_llm}/config.json | 0 {weights_llm => weights/weights_llm}/generation_config.json | 0 {weights_llm => weights/weights_llm}/merges.txt | 0 {weights_llm => weights/weights_llm}/tokenizer.json | 0 {weights_llm => weights/weights_llm}/tokenizer_config.json | 0 {weights_llm => weights/weights_llm}/vocab.json | 0 .../weights_tokenizer}/hf_tokenizer_BBPE/special_tokens_map.json | 0 .../weights_tokenizer}/hf_tokenizer_BBPE/tokenizer.json | 0 .../weights_tokenizer}/hf_tokenizer_BBPE/tokenizer_config.json | 0 29 files changed, 0 insertions(+), 0 deletions(-) rename {src_inference => src_dev/src_inference}/dolly_inference.py (100%) rename {src_llm => src_dev/src_llm}/__init__.py (100%) rename {src_llm => src_dev/src_llm}/configuration_dolly.py (100%) rename {src_llm => src_dev/src_llm}/modeling_dolly.py (100%) rename {src_llm_moe => src_dev/src_llm_moe}/__init__.py (100%) rename {src_llm_moe => src_dev/src_llm_moe}/configuration_dollyMoE.py (100%) rename {src_llm_moe => src_dev/src_llm_moe}/modeling_dollyMoE.py (100%) rename {src_pretrain => src_dev/src_pretrain}/pretrain_deepspeed.py (100%) rename {src_pretrain => src_dev/src_pretrain}/pretrain_deepspeed_MoE.py (100%) rename {src_pretrain => src_dev/src_pretrain}/pretrain_transformers.py (100%) rename {src_tokenizer => src_dev/src_tokenizer}/README.md (100%) rename {src_tokenizer => src_dev/src_tokenizer}/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py (100%) rename {src_tokenizer => src_dev/src_tokenizer}/hf_tokenizer_code/step1_hf_train_from_old.py (100%) rename {src_tokenizer => src_dev/src_tokenizer}/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py (100%) rename {src_tokenizer => src_dev/src_tokenizer}/hf_tokenizer_code/step3_hf_add_new_token.py (100%) rename {src_tokenizer => src_dev/src_tokenizer}/spm_tokenizer_code/step1_spm_train_tokenizer.py (100%) rename {src_tokenizer => src_dev/src_tokenizer}/spm_tokenizer_code/step2_spm_merge_tokenizer.py (100%) rename {src_tokenizer => src_dev/src_tokenizer}/spm_tokenizer_code/step3_spm_to_hf.py (100%) rename {src_tokenizer => src_dev/src_tokenizer}/spm_tokenizer_code/step4_spm_expand_vocab.py (100%) rename {src_tokenizer => src_dev/src_tokenizer}/spm_tokenizer_code/step5_test_spm_tokenizer.py (100%) rename {weights_llm => weights/weights_llm}/config.json (100%) rename {weights_llm => weights/weights_llm}/generation_config.json (100%) rename {weights_llm => weights/weights_llm}/merges.txt (100%) rename {weights_llm => weights/weights_llm}/tokenizer.json (100%) rename {weights_llm => weights/weights_llm}/tokenizer_config.json (100%) rename {weights_llm => weights/weights_llm}/vocab.json (100%) rename {weights_tokenizer => weights/weights_tokenizer}/hf_tokenizer_BBPE/special_tokens_map.json (100%) rename {weights_tokenizer => weights/weights_tokenizer}/hf_tokenizer_BBPE/tokenizer.json (100%) rename {weights_tokenizer => weights/weights_tokenizer}/hf_tokenizer_BBPE/tokenizer_config.json (100%) diff --git a/src_inference/dolly_inference.py b/src_dev/src_inference/dolly_inference.py similarity index 100% rename from src_inference/dolly_inference.py rename to src_dev/src_inference/dolly_inference.py diff --git a/src_llm/__init__.py b/src_dev/src_llm/__init__.py similarity index 100% rename from src_llm/__init__.py rename to src_dev/src_llm/__init__.py diff --git a/src_llm/configuration_dolly.py b/src_dev/src_llm/configuration_dolly.py similarity index 100% rename from src_llm/configuration_dolly.py rename to src_dev/src_llm/configuration_dolly.py diff --git a/src_llm/modeling_dolly.py b/src_dev/src_llm/modeling_dolly.py similarity index 100% rename from src_llm/modeling_dolly.py rename to src_dev/src_llm/modeling_dolly.py diff --git a/src_llm_moe/__init__.py b/src_dev/src_llm_moe/__init__.py similarity index 100% rename from src_llm_moe/__init__.py rename to src_dev/src_llm_moe/__init__.py diff --git a/src_llm_moe/configuration_dollyMoE.py b/src_dev/src_llm_moe/configuration_dollyMoE.py similarity index 100% rename from src_llm_moe/configuration_dollyMoE.py rename to src_dev/src_llm_moe/configuration_dollyMoE.py diff --git a/src_llm_moe/modeling_dollyMoE.py b/src_dev/src_llm_moe/modeling_dollyMoE.py similarity index 100% rename from src_llm_moe/modeling_dollyMoE.py rename to src_dev/src_llm_moe/modeling_dollyMoE.py diff --git a/src_pretrain/pretrain_deepspeed.py b/src_dev/src_pretrain/pretrain_deepspeed.py similarity index 100% rename from src_pretrain/pretrain_deepspeed.py rename to src_dev/src_pretrain/pretrain_deepspeed.py diff --git a/src_pretrain/pretrain_deepspeed_MoE.py b/src_dev/src_pretrain/pretrain_deepspeed_MoE.py similarity index 100% rename from src_pretrain/pretrain_deepspeed_MoE.py rename to src_dev/src_pretrain/pretrain_deepspeed_MoE.py diff --git a/src_pretrain/pretrain_transformers.py b/src_dev/src_pretrain/pretrain_transformers.py similarity index 100% rename from src_pretrain/pretrain_transformers.py rename to src_dev/src_pretrain/pretrain_transformers.py diff --git a/src_tokenizer/README.md b/src_dev/src_tokenizer/README.md similarity index 100% rename from src_tokenizer/README.md rename to src_dev/src_tokenizer/README.md diff --git a/src_tokenizer/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py b/src_dev/src_tokenizer/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py similarity index 100% rename from src_tokenizer/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py rename to src_dev/src_tokenizer/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py diff --git a/src_tokenizer/hf_tokenizer_code/step1_hf_train_from_old.py b/src_dev/src_tokenizer/hf_tokenizer_code/step1_hf_train_from_old.py similarity index 100% rename from src_tokenizer/hf_tokenizer_code/step1_hf_train_from_old.py rename to src_dev/src_tokenizer/hf_tokenizer_code/step1_hf_train_from_old.py diff --git a/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py b/src_dev/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py similarity index 100% rename from src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py rename to src_dev/src_tokenizer/hf_tokenizer_code/step2_hf_train_from_scratch_BPEandBBPE.py diff --git a/src_tokenizer/hf_tokenizer_code/step3_hf_add_new_token.py b/src_dev/src_tokenizer/hf_tokenizer_code/step3_hf_add_new_token.py similarity index 100% rename from src_tokenizer/hf_tokenizer_code/step3_hf_add_new_token.py rename to src_dev/src_tokenizer/hf_tokenizer_code/step3_hf_add_new_token.py diff --git a/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py b/src_dev/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py similarity index 100% rename from src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py rename to src_dev/src_tokenizer/spm_tokenizer_code/step1_spm_train_tokenizer.py diff --git a/src_tokenizer/spm_tokenizer_code/step2_spm_merge_tokenizer.py b/src_dev/src_tokenizer/spm_tokenizer_code/step2_spm_merge_tokenizer.py similarity index 100% rename from src_tokenizer/spm_tokenizer_code/step2_spm_merge_tokenizer.py rename to src_dev/src_tokenizer/spm_tokenizer_code/step2_spm_merge_tokenizer.py diff --git a/src_tokenizer/spm_tokenizer_code/step3_spm_to_hf.py b/src_dev/src_tokenizer/spm_tokenizer_code/step3_spm_to_hf.py similarity index 100% rename from src_tokenizer/spm_tokenizer_code/step3_spm_to_hf.py rename to src_dev/src_tokenizer/spm_tokenizer_code/step3_spm_to_hf.py diff --git a/src_tokenizer/spm_tokenizer_code/step4_spm_expand_vocab.py b/src_dev/src_tokenizer/spm_tokenizer_code/step4_spm_expand_vocab.py similarity index 100% rename from src_tokenizer/spm_tokenizer_code/step4_spm_expand_vocab.py rename to src_dev/src_tokenizer/spm_tokenizer_code/step4_spm_expand_vocab.py diff --git a/src_tokenizer/spm_tokenizer_code/step5_test_spm_tokenizer.py b/src_dev/src_tokenizer/spm_tokenizer_code/step5_test_spm_tokenizer.py similarity index 100% rename from src_tokenizer/spm_tokenizer_code/step5_test_spm_tokenizer.py rename to src_dev/src_tokenizer/spm_tokenizer_code/step5_test_spm_tokenizer.py diff --git a/weights_llm/config.json b/weights/weights_llm/config.json similarity index 100% rename from weights_llm/config.json rename to weights/weights_llm/config.json diff --git a/weights_llm/generation_config.json b/weights/weights_llm/generation_config.json similarity index 100% rename from weights_llm/generation_config.json rename to weights/weights_llm/generation_config.json diff --git a/weights_llm/merges.txt b/weights/weights_llm/merges.txt similarity index 100% rename from weights_llm/merges.txt rename to weights/weights_llm/merges.txt diff --git a/weights_llm/tokenizer.json b/weights/weights_llm/tokenizer.json similarity index 100% rename from weights_llm/tokenizer.json rename to weights/weights_llm/tokenizer.json diff --git a/weights_llm/tokenizer_config.json b/weights/weights_llm/tokenizer_config.json similarity index 100% rename from weights_llm/tokenizer_config.json rename to weights/weights_llm/tokenizer_config.json diff --git a/weights_llm/vocab.json b/weights/weights_llm/vocab.json similarity index 100% rename from weights_llm/vocab.json rename to weights/weights_llm/vocab.json diff --git a/weights_tokenizer/hf_tokenizer_BBPE/special_tokens_map.json b/weights/weights_tokenizer/hf_tokenizer_BBPE/special_tokens_map.json similarity index 100% rename from weights_tokenizer/hf_tokenizer_BBPE/special_tokens_map.json rename to weights/weights_tokenizer/hf_tokenizer_BBPE/special_tokens_map.json diff --git a/weights_tokenizer/hf_tokenizer_BBPE/tokenizer.json b/weights/weights_tokenizer/hf_tokenizer_BBPE/tokenizer.json similarity index 100% rename from weights_tokenizer/hf_tokenizer_BBPE/tokenizer.json rename to weights/weights_tokenizer/hf_tokenizer_BBPE/tokenizer.json diff --git a/weights_tokenizer/hf_tokenizer_BBPE/tokenizer_config.json b/weights/weights_tokenizer/hf_tokenizer_BBPE/tokenizer_config.json similarity index 100% rename from weights_tokenizer/hf_tokenizer_BBPE/tokenizer_config.json rename to weights/weights_tokenizer/hf_tokenizer_BBPE/tokenizer_config.json