add LLM code

2025-04-29 18:59:54 +08:00
parent 392828cf83
commit 93d557256f
3 changed files with 21 additions and 0 deletions
@@ -0,0 +1,3 @@
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")  
+print(tokenizer.vocab_size)