add tokenizer code

2025-04-29 18:21:17 +08:00
parent df6dbd8702
commit 1b809eb64c
43 changed files with 546364 additions and 21 deletions
@@ -0,0 +1 @@
+spm_train --input=corpus/wikipedia.txt --model_prefix=tokenizer_zh --vocab_size=32000 --num_threads=8
				`@@ -0,0 +1 @@`
				`spm_train --input=corpus/wikipedia.txt --model_prefix=tokenizer_zh --vocab_size=32000 --num_threads=8`