add tokenizer code

2025-04-29 18:21:17 +08:00
parent c0bcb8947e
commit d5ebd9df0e
43 changed files with 546364 additions and 21 deletions
@@ -0,0 +1,3 @@
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")  
+print(tokenizer.vocab_size)