add tokenizer code

.gitignore

@@ -0,0 +1 @@
+corpus/

README.md

@@ -1,2 +1,3 @@
 # dolly
 Dolly is a custom LLM built from scratch
+# useage

new_tokenizer2/special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

new_tokenizer2/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizerFast",
+  "unk_token": "<unk>"
+}

requirements.txt

@@ -0,0 +1,19 @@
+sentencepiece==0.2.0
+protobuf==3.20.0 #important
+transformers==4.51.3
+certifi==2025.4.26
+charset-normalizer==3.4.1
+filelock==3.18.0
+fsspec==2025.3.2
+huggingface-hub==0.30.2
+idna==3.10
+numpy==2.2.5
+packaging==25.0
+PyYAML==6.0.2
+regex==2024.11.6
+requests==2.32.3
+safetensors==0.5.3
+tokenizers==0.21.1
+tqdm==4.67.1
+typing_extensions==4.13.2
+urllib3==2.4.0

test.py

@@ -0,0 +1,3 @@
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")  
+print(tokenizer.vocab_size)

tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py

@@ -0,0 +1,23 @@
+from transformers import AutoTokenizer
+
+def get_training_corpus(file_path, batch_size=1000):
+    with open(file_path, "r", encoding="utf-8") as f:
+        batch = []
+        for line in f:
+            batch.append(line.strip())  
+            if len(batch) >= batch_size:
+                yield batch
+                batch = []
+        if batch:  
+            yield batch
+
+old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")  
+training_corpus = get_training_corpus("corpus/wikipedia.txt")  
+
+tokenizer = old_tokenizer.train_new_from_iterator(
+    training_corpus,
+    vocab_size=34000
+)
+
+
+tokenizer.save_pretrained("new_tokenizer2")

tokenizer_code/hf_tokenizer_code/step1_hf_train_from_old.py

@@ -0,0 +1,16 @@
+from datasets import load_dataset
+from transformers import AutoTokenizer
+
+dataset = load_dataset("text", data_files="corpus/wikipedia.txt", streaming=True)
+
+def batch_iterator(batch_size=1000):
+    for batch in dataset["train"].iter(batch_size):
+        yield batch["text"]
+
+old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")
+tokenizer = old_tokenizer.train_new_from_iterator(
+    batch_iterator(),
+    vocab_size=33000,
+)
+
+tokenizer.save_pretrained("new_tokenizer")

tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py

@@ -0,0 +1,26 @@
+from tokenizers import Tokenizer
+from tokenizers.models import BPE  
+from tokenizers.trainers import BpeTrainer
+from tokenizers.pre_tokenizers import Whitespace
+from transformers import PreTrainedTokenizerFast
+
+tokenizer = Tokenizer(BPE(unk_token="[UNK]"))  
+tokenizer.pre_tokenizer = Whitespace()
+trainer = BpeTrainer(
+    vocab_size=30000,           
+    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],  
+    min_frequency=2,            
+    show_progress=True         
+)
+
+tokenizer.train(files=["corpus/wikipedia.txt"], trainer=trainer)
+tokenizer.save("tokenizer_file/tokenizer.json")
+fast_tokenizer = PreTrainedTokenizerFast(
+    tokenizer_file="tokenizer_file/tokenizer.json",
+    unk_token="[UNK]",
+    pad_token="[PAD]",
+    cls_token="[CLS]",
+    sep_token="[SEP]",
+    mask_token="[MASK]",
+)
+fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer")

tokenizer_code/hf_tokenizer_code/step3_hf_add_new_token.py

@@ -0,0 +1,10 @@
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")
+
+new_tokens = ["<ACTION_1>", "<ACTION_2>"]
+tokenizer.add_tokens(new_tokens)
+
+text = "<ACTION_1>"
+print(tokenizer.tokenize(text))  
+tokenizer.save_pretrained("updated_tokenizer")

tokenizer_code/spm_tokenizer_code/step1_spm_train_tokenizer.py

@@ -0,0 +1,22 @@
+import sentencepiece as spm
+
+###推荐使用 utils/spm_train_tokenizer.sh，使用多核加速
+
+#英文分词器
+# corpus_path = 'a.txt,b.txt'
+corpus_path_en= 'corpus/pg16457.txt'
+vocab_size_en = 1000
+model_en= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_en'
+
+# model_type 使用 bpe
+model_type = 'bpe'
+# byte_fallback=True，避免 out of vocabulary 问题
+spm.SentencePieceTrainer.train(input=corpus_path_en, model_prefix=model_en, 
+    vocab_size=vocab_size_en, model_type=model_type, byte_fallback=True,character_coverage=1)
+
+#中文分词器
+corpus_path_zh= 'corpus/wikipedia.txt'
+vocab_size_zh = 32000
+model_zh= 'tokenizer_file/dolly_tokenizer_spm/tokenizer_zh'
+spm.SentencePieceTrainer.train(input=corpus_path_zh, model_prefix=model_zh, 
+    vocab_size=vocab_size_zh, model_type=model_type, byte_fallback=True,character_coverage=1)

tokenizer_code/spm_tokenizer_code/step2_spm_merge_tokenizer.py

@@ -0,0 +1,33 @@
+import sentencepiece as spm
+from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
+
+model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_zh.model"
+en_model_file = "tokenizer_part/dolly_tokenizer_spm/tokenizer_en.model"
+
+model_file_new = "tokenizer_part/dolly_tokenizer_spm/tokenizer.model"
+
+chinese_sp = spm.SentencePieceProcessor(model_file=model_file)
+en_sp = spm.SentencePieceProcessor(model_file=en_model_file)
+
+print(chinese_sp.vocab_size())#32000
+
+chinese_spm = sp_model_pb2.ModelProto()
+chinese_spm.ParseFromString(chinese_sp.serialized_model_proto())
+
+en_spm = sp_model_pb2.ModelProto()
+en_spm.ParseFromString(en_sp.serialized_model_proto())
+
+chinese_sp_set = set(p.piece for p in chinese_spm.pieces)
+
+for p in en_spm.pieces:
+    piece = p.piece
+    if piece not in chinese_sp_set:
+        chinese_sp_set.add(piece)
+        new_piece = sp_model_pb2.ModelProto().SentencePiece()
+        new_piece.piece = piece
+        new_piece.score = 0
+        chinese_spm.pieces.append(new_piece)
+
+print(f"after expand vocab={len(chinese_spm.pieces)}") # 32361,增加361个
+with open(model_file_new, 'wb') as file:
+    file.write(chinese_spm.SerializeToString())

tokenizer_code/spm_tokenizer_code/step3_spm_to_hf.py

@@ -0,0 +1,18 @@
+from transformers import AutoTokenizer,XLMRobertaTokenizer
+#由spm转为hf_tokenizer
+tokenizer = XLMRobertaTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_spm/tokenizer.model")
+text = "A Frenchman from Paris"
+encoded_input = tokenizer.encode(text)
+print(encoded_input)
+decoded_output = tokenizer.decode(encoded_input)
+print(decoded_output)
+tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf')
+
+#转换为常见的tokenzier.json格式
+tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf")
+text = "A Frenchman from Paris"
+encoded_input = tokenizer.encode(text)
+print(encoded_input)
+decoded_output = tokenizer.decode(encoded_input)
+print(decoded_output)
+tokenizer.save_pretrained('tokenizer_file/dolly_tokenizer_hf2')

tokenizer_code/spm_tokenizer_code/step4_spm_expand_vocab.py

@@ -0,0 +1,21 @@
+import sentencepiece as spm
+from sentencepiece import sentencepiece_model_pb2 as sp_model_pb2
+
+sp = spm.SentencePieceProcessor(model_file='tokenizer_file/dolly_tokenizer_spm/tokenizer.model')
+print(sp.vocab_size())  # 32461
+
+old_model = sp_model_pb2.ModelProto()
+old_model.ParseFromString(sp.serialized_model_proto())
+pieces = ['拉布拉多', '货拉拉', '拉不拉']
+existed_pieces = set(p.piece for p in old_model.pieces)
+for piece in pieces:
+    if piece not in existed_pieces:
+        existed_pieces.add(piece)
+        new_piece = sp_model_pb2.ModelProto().SentencePiece()
+        new_piece.piece = piece
+        new_piece.score = 0
+        old_model.pieces.append(new_piece)
+    
+print(len(old_model.pieces))  # 32461
+with open('tokenizer_file/dolly_tokenizer_spm/expand_tokenizer.model', 'wb') as f:
+    f.write(old_model.SerializeToString())

tokenizer_code/spm_tokenizer_code/test_spm_tokenizer.py

@@ -0,0 +1,15 @@
+import sentencepiece as spm
+sp = spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer.model')
+
+sp_en= spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer_en.model')
+sentence = 'A Frenchman from Paris'
+pieces = sp.encode_as_pieces(sentence)
+pieces_en = sp_en.encode_as_pieces(sentence)
+print(pieces)  # ['▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'ar', 'is']
+print(pieces_en) #'▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'aris'
+tokens = sp.encode(sentence)
+tokens_en = sp_en.encode(sentence)
+print(tokens)  # [338, 493, 587, 349, 807, 429, 373, 279, 282]
+print(tokens_en) #364, 490, 5981, 398, 2036, 32046, 386, 9020] 
+# 解码后的结果应等于编码前的输入
+# assert sp.decode(tokens) == sentence

tokenizer_file/dolly_tokenizer_hf/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer_file/dolly_tokenizer_hf/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32362": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}

tokenizer_file/dolly_tokenizer_hf2/special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_file/dolly_tokenizer_hf2/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32362": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}

tokenizer_file/hf_tokenizer/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_file/hf_tokenizer/tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizer",
+  "unk_token": "[UNK]"
+}

tokenizer_file/new_tokenizer/special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_file/new_tokenizer/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizerFast",
+  "unk_token": "<unk>"
+}

tokenizer_part/train_tokenizer.py

@@ -1,21 +0,0 @@
-from datasets import load_dataset
-import os
-os.environ["HF_DATASETS_CACHE"] = "./data/"
-wikipedia = load_dataset("wikipedia", language="zh")
-bookcorpus = load_dataset("bookcorpus")
-openwebtext = load_dataset("openwebtext")
-cc100 = load_dataset("cc100", language="en")
-
-wikipedia_text = wikipedia['train']['text']
-bookcorpus_text= bookcorpus["train"]["text"]
-openwebtext_text = openwebtext['train']['text']
-cc100_text = cc100['train']['text']
-combined_text = wikipedia_text + openwebtext_text+ bookcorpus_text
-print("finished")
-from tokenizers import Tokenizer, models, pre_tokenizers, trainers
-tokenizer = Tokenizer(models.BPE())
-tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
-trainer = trainers.BpeTrainer(vocab_size=30000, min_frequency=2)
-tokenizer.train_from_iterator(combined_text, trainer=trainer)
-tokenizer.save("path_to_save_your_tokenizer.json")
-

updated_tokenizer/special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

updated_tokenizer/tokenizer_config.json

@@ -0,0 +1,69 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30000": {
+      "content": "<ACTION_1>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30001": {
+      "content": "<ACTION_2>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizer",
+  "unk_token": "[UNK]"
+}

utils/download_corpus.sh

@@ -0,0 +1,3 @@
+mkdir corpus
+git clone https://huggingface.co/datasets/pleisto/wikipedia-cn-20230720-filtered corpus
+wget -P data corpus https://www.gutenberg.org/cache/epub/16457/pg16457.txt

utils/spm_train_tokenizer.sh

@@ -0,0 +1 @@
+spm_train --input=corpus/wikipedia.txt --model_prefix=tokenizer_zh --vocab_size=32000 --num_threads=8

utils/tans_json_to_txt.py

@@ -0,0 +1,13 @@
+import json
+
+def extract_completions(json_file_path, output_file_path):
+    with open(json_file_path, 'r', encoding='utf-8') as json_file:
+        data = json.load(json_file)
+    
+    completions = [conversion["completion"] for conversion in data]
+
+    with open(output_file_path, 'w', encoding='utf-8') as output_file:
+        for completion in completions:
+            output_file.write(completion + '\n')
+
+extract_completions('corpus/wikipedia-cn-20230720-filtered.json', 'corpus/wikipedia.txt')