add tokenizer code

utils/download_corpus.sh

@@ -0,0 +1,3 @@
+mkdir corpus
+git clone https://huggingface.co/datasets/pleisto/wikipedia-cn-20230720-filtered corpus
+wget -P data corpus https://www.gutenberg.org/cache/epub/16457/pg16457.txt

utils/spm_train_tokenizer.sh

@@ -0,0 +1 @@
+spm_train --input=corpus/wikipedia.txt --model_prefix=tokenizer_zh --vocab_size=32000 --num_threads=8

utils/tans_json_to_txt.py

@@ -0,0 +1,13 @@
+import json
+
+def extract_completions(json_file_path, output_file_path):
+    with open(json_file_path, 'r', encoding='utf-8') as json_file:
+        data = json.load(json_file)
+    
+    completions = [conversion["completion"] for conversion in data]
+
+    with open(output_file_path, 'w', encoding='utf-8') as output_file:
+        for completion in completions:
+            output_file.write(completion + '\n')
+
+extract_completions('corpus/wikipedia-cn-20230720-filtered.json', 'corpus/wikipedia.txt')