add tokenizer code

2025-04-29 18:21:17 +08:00
parent c0bcb8947e
commit d5ebd9df0e
43 changed files with 546364 additions and 21 deletions
@@ -0,0 +1 @@
+spm_train --input=corpus/wikipedia.txt --model_prefix=tokenizer_zh --vocab_size=32000 --num_threads=8
				`@@ -0,0 +1 @@`
				`spm_train --input=corpus/wikipedia.txt --model_prefix=tokenizer_zh --vocab_size=32000 --num_threads=8`