update LLM

2025-04-29 19:05:11 +08:00
parent 58c964e134
commit b3bf9d7795
3 changed files with 0 additions and 21 deletions
@@ -1,3 +0,0 @@
-from transformers import AutoTokenizer
-tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")  
-print(tokenizer.vocab_size)