coffee
21 lines
673 B
Python
21 lines
673 B
Python
from transformers import AutoTokenizer
|
|
wrapped_tokenizer = AutoTokenizer.from_pretrained("/mnt/han.luzhi/dolly/llm_tokenizer")
|
|
|
|
test_text = "深度学习英文是deep learning。"
|
|
|
|
encoded = wrapped_tokenizer.encode(test_text)
|
|
print("IDs:", encoded)
|
|
decoded_text = wrapped_tokenizer.decode(encoded)
|
|
print("Decoded:", decoded_text)
|
|
|
|
|
|
tokens = wrapped_tokenizer.tokenize(test_text)
|
|
print("自定义版tokens:", tokens)
|
|
encoded = wrapped_tokenizer.encode(test_text)
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained("/mnt/qwen2.5_vl_32B",use_fast=False)
|
|
tokens = tokenizer.tokenize(test_text)
|
|
print("Qwen3版tokens:", tokens)
|
|
encoded = tokenizer.encode(test_text)
|
|
|