From 96c99c8e1a486f10b72dc36374ebf65354ee3db6 Mon Sep 17 00:00:00 2001 From: coffee Date: Tue, 29 Apr 2025 19:06:17 +0800 Subject: [PATCH] update Tokenizer code --- {tokenizer_code => Tokenizer_code}/README.md | 0 .../hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py | 0 .../hf_tokenizer_code/step1_hf_train_from_old.py | 0 .../hf_tokenizer_code/step2_hf_train_from_scratch.py | 0 .../hf_tokenizer_code/step3_hf_add_new_token.py | 0 .../spm_tokenizer_code/step1_spm_train_tokenizer.py | 0 .../spm_tokenizer_code/step2_spm_merge_tokenizer.py | 0 .../spm_tokenizer_code/step3_spm_to_hf.py | 0 .../spm_tokenizer_code/step4_spm_expand_vocab.py | 0 .../spm_tokenizer_code/step5_test_spm_tokenizer.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename {tokenizer_code => Tokenizer_code}/README.md (100%) rename {tokenizer_code => Tokenizer_code}/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py (100%) rename {tokenizer_code => Tokenizer_code}/hf_tokenizer_code/step1_hf_train_from_old.py (100%) rename {tokenizer_code => Tokenizer_code}/hf_tokenizer_code/step2_hf_train_from_scratch.py (100%) rename {tokenizer_code => Tokenizer_code}/hf_tokenizer_code/step3_hf_add_new_token.py (100%) rename {tokenizer_code => Tokenizer_code}/spm_tokenizer_code/step1_spm_train_tokenizer.py (100%) rename {tokenizer_code => Tokenizer_code}/spm_tokenizer_code/step2_spm_merge_tokenizer.py (100%) rename {tokenizer_code => Tokenizer_code}/spm_tokenizer_code/step3_spm_to_hf.py (100%) rename {tokenizer_code => Tokenizer_code}/spm_tokenizer_code/step4_spm_expand_vocab.py (100%) rename {tokenizer_code => Tokenizer_code}/spm_tokenizer_code/step5_test_spm_tokenizer.py (100%) diff --git a/tokenizer_code/README.md b/Tokenizer_code/README.md similarity index 100% rename from tokenizer_code/README.md rename to Tokenizer_code/README.md diff --git a/tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py b/Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py similarity index 100% rename from tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py rename to Tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py diff --git a/tokenizer_code/hf_tokenizer_code/step1_hf_train_from_old.py b/Tokenizer_code/hf_tokenizer_code/step1_hf_train_from_old.py similarity index 100% rename from tokenizer_code/hf_tokenizer_code/step1_hf_train_from_old.py rename to Tokenizer_code/hf_tokenizer_code/step1_hf_train_from_old.py diff --git a/tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py b/Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py similarity index 100% rename from tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py rename to Tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py diff --git a/tokenizer_code/hf_tokenizer_code/step3_hf_add_new_token.py b/Tokenizer_code/hf_tokenizer_code/step3_hf_add_new_token.py similarity index 100% rename from tokenizer_code/hf_tokenizer_code/step3_hf_add_new_token.py rename to Tokenizer_code/hf_tokenizer_code/step3_hf_add_new_token.py diff --git a/tokenizer_code/spm_tokenizer_code/step1_spm_train_tokenizer.py b/Tokenizer_code/spm_tokenizer_code/step1_spm_train_tokenizer.py similarity index 100% rename from tokenizer_code/spm_tokenizer_code/step1_spm_train_tokenizer.py rename to Tokenizer_code/spm_tokenizer_code/step1_spm_train_tokenizer.py diff --git a/tokenizer_code/spm_tokenizer_code/step2_spm_merge_tokenizer.py b/Tokenizer_code/spm_tokenizer_code/step2_spm_merge_tokenizer.py similarity index 100% rename from tokenizer_code/spm_tokenizer_code/step2_spm_merge_tokenizer.py rename to Tokenizer_code/spm_tokenizer_code/step2_spm_merge_tokenizer.py diff --git a/tokenizer_code/spm_tokenizer_code/step3_spm_to_hf.py b/Tokenizer_code/spm_tokenizer_code/step3_spm_to_hf.py similarity index 100% rename from tokenizer_code/spm_tokenizer_code/step3_spm_to_hf.py rename to Tokenizer_code/spm_tokenizer_code/step3_spm_to_hf.py diff --git a/tokenizer_code/spm_tokenizer_code/step4_spm_expand_vocab.py b/Tokenizer_code/spm_tokenizer_code/step4_spm_expand_vocab.py similarity index 100% rename from tokenizer_code/spm_tokenizer_code/step4_spm_expand_vocab.py rename to Tokenizer_code/spm_tokenizer_code/step4_spm_expand_vocab.py diff --git a/tokenizer_code/spm_tokenizer_code/step5_test_spm_tokenizer.py b/Tokenizer_code/spm_tokenizer_code/step5_test_spm_tokenizer.py similarity index 100% rename from tokenizer_code/spm_tokenizer_code/step5_test_spm_tokenizer.py rename to Tokenizer_code/spm_tokenizer_code/step5_test_spm_tokenizer.py