[2025.06.05] add tokenzier test

2025-06-05 18:20:36 +08:00
parent 0e023c8bc1
commit dc4bbf43f6
1 changed files with 19 additions and 0 deletions
@@ -0,0 +1,19 @@
 from transformers import AutoTokenizer
 wrapped_tokenizer = AutoTokenizer.from_pretrained("/mnt/han.luzhi/dolly/llm_tokenizer")
 test_text = "深度学习英文是deep learning。"
 encoded = wrapped_tokenizer.encode(test_text)
 print("IDs:", encoded)           
 decoded_text = wrapped_tokenizer.decode(encoded)
 print("Decoded:", decoded_text)     
 tokens = wrapped_tokenizer.tokenize(test_text)
 print("自定义版tokens:", tokens) 
 encoded = wrapped_tokenizer.encode(test_text)
 tokenizer = AutoTokenizer.from_pretrained("/mnt/qwen2.5_vl_32B",use_fast=False)
 tokens = tokenizer.tokenize(test_text)
 print("Qwen3版tokens:", tokens) 
 encoded = tokenizer.encode(test_text)