From dc4bbf43f6b5415813fd9c2ab4c0c597e7cb5f91 Mon Sep 17 00:00:00 2001 From: coffee Date: Thu, 5 Jun 2025 18:20:36 +0800 Subject: [PATCH] [2025.06.05] add tokenzier test --- test/test_tokenizer.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/test_tokenizer.py diff --git a/test/test_tokenizer.py b/test/test_tokenizer.py new file mode 100644 index 0000000..fea71ac --- /dev/null +++ b/test/test_tokenizer.py @@ -0,0 +1,19 @@ +from transformers import AutoTokenizer +wrapped_tokenizer = AutoTokenizer.from_pretrained("/mnt/han.luzhi/dolly/llm_tokenizer") + +test_text = "深度学习英文是deep learning。" + +encoded = wrapped_tokenizer.encode(test_text) +print("IDs:", encoded) +decoded_text = wrapped_tokenizer.decode(encoded) +print("Decoded:", decoded_text) + + +tokens = wrapped_tokenizer.tokenize(test_text) +print("自定义版tokens:", tokens) +encoded = wrapped_tokenizer.encode(test_text) + +tokenizer = AutoTokenizer.from_pretrained("/mnt/qwen2.5_vl_32B",use_fast=False) +tokens = tokenizer.tokenize(test_text) +print("Qwen3版tokens:", tokens) +encoded = tokenizer.encode(test_text)