[2025.06.05] add tokenzier test

2025-06-05 18:35:39 +08:00
parent edc60be53b
commit 628aa8c674
2 changed files with 69 additions and 13 deletions
@@ -0,0 +1,20 @@
+from transformers import AutoTokenizer
+wrapped_tokenizer = AutoTokenizer.from_pretrained("/mnt/han.luzhi/dolly/llm_tokenizer")
+
+test_text = "深度学习英文是deep learning。"
+
+encoded = wrapped_tokenizer.encode(test_text)
+print("IDs:", encoded)           
+decoded_text = wrapped_tokenizer.decode(encoded)
+print("Decoded:", decoded_text)     
+
+
+tokens = wrapped_tokenizer.tokenize(test_text)
+print("自定义版tokens:", tokens) 
+encoded = wrapped_tokenizer.encode(test_text)
+
+tokenizer = AutoTokenizer.from_pretrained("/mnt/qwen2.5_vl_32B",use_fast=False)
+tokens = tokenizer.tokenize(test_text)
+print("Qwen3版tokens:", tokens) 
+encoded = tokenizer.encode(test_text)
+