[2025.06.05] add tokenzier test

2025-06-05 18:20:36 +08:00
parent 0e023c8bc1
commit dc4bbf43f6
1 changed files with 19 additions and 0 deletions
@@ -0,0 +1,19 @@
+from transformers import AutoTokenizer
+wrapped_tokenizer = AutoTokenizer.from_pretrained("/mnt/han.luzhi/dolly/llm_tokenizer")
+
+test_text = "深度学习英文是deep learning。"
+
+encoded = wrapped_tokenizer.encode(test_text)
+print("IDs:", encoded)           
+decoded_text = wrapped_tokenizer.decode(encoded)
+print("Decoded:", decoded_text)     
+
+
+tokens = wrapped_tokenizer.tokenize(test_text)
+print("自定义版tokens:", tokens) 
+encoded = wrapped_tokenizer.encode(test_text)
+
+tokenizer = AutoTokenizer.from_pretrained("/mnt/qwen2.5_vl_32B",use_fast=False)
+tokens = tokenizer.tokenize(test_text)
+print("Qwen3版tokens:", tokens) 
+encoded = tokenizer.encode(test_text)