add tokenizer code

2025-04-29 18:21:17 +08:00
parent c0bcb8947e
commit d5ebd9df0e
43 changed files with 546364 additions and 21 deletions
@@ -0,0 +1,3 @@
+mkdir corpus
+git clone https://huggingface.co/datasets/pleisto/wikipedia-cn-20230720-filtered corpus
+wget -P data corpus https://www.gutenberg.org/cache/epub/16457/pg16457.txt