coffee
4 lines
142 B
Python
4 lines
142 B
Python
from transformers import AutoTokenizer
|
|
tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")
|
|
print(tokenizer.vocab_size)
|