From 628aa8c67430e20dd966f1fc689e52346502bca1 Mon Sep 17 00:00:00 2001
From: coffee <hanluzhi0429@gmail.com>
Date: Thu, 5 Jun 2025 18:35:39 +0800
Subject: [PATCH] [2025.06.05] add tokenzier test

---
 test/test_tokenizer.py         | 62 +++++++++++++++++++++++++++-------
 test/test_tokenizer_weights.py | 20 +++++++++++
 2 files changed, 69 insertions(+), 13 deletions(-)
 create mode 100644 test/test_tokenizer_weights.py

diff --git a/test/test_tokenizer.py b/test/test_tokenizer.py
index fea71ac..4ae3557 100644
--- a/test/test_tokenizer.py
+++ b/test/test_tokenizer.py
@@ -1,19 +1,55 @@
+
+import json
 from transformers import AutoTokenizer
-wrapped_tokenizer = AutoTokenizer.from_pretrained("/mnt/han.luzhi/dolly/llm_tokenizer")
 
-test_text = "深度学习英文是deep learning。"
+# 初始化tokenizer
+tokenizer = AutoTokenizer.from_pretrained("/mnt/han.luzhi/dolly/llm_tokenizer")
 
-encoded = wrapped_tokenizer.encode(test_text)
-print("IDs:", encoded)           
-decoded_text = wrapped_tokenizer.decode(encoded)
-print("Decoded:", decoded_text)     
+# 初始化统计变量
+total_characters = 0
+total_tokens = 0
+num_lines = 0
 
+# 处理单个文本的压缩率计算函数
+def calculate_compression(text):
+    char_count = len(text)
+    tokens = tokenizer.encode(text, add_special_tokens=False)
+    token_count = len(tokens)
+    return char_count, token_count
 
-tokens = wrapped_tokenizer.tokenize(test_text)
-print("自定义版tokens:", tokens) 
-encoded = wrapped_tokenizer.encode(test_text)
+# 处理JSONL文件
+with open('/mnt/han.luzhi/dolly/corpus/2020-40_zh_head_0001.jsonl', 'r', encoding='utf-8') as f:
+    for line in f:
+        try:
+            data = json.loads(line)
+            text = data.get('text', '')  # 获取text字段，若无则返回空字符串
+            
+            if text:  # 跳过空文本
+                chars, tokens = calculate_compression(text)
+                total_characters += chars
+                total_tokens += tokens
+                num_lines += 1
+                
+        except json.JSONDecodeError:
+            print(f"JSON解析错误: {line}")
+        except KeyError:
+            print(f"行缺少'text'字段: {line}")
 
-tokenizer = AutoTokenizer.from_pretrained("/mnt/qwen2.5_vl_32B",use_fast=False)
-tokens = tokenizer.tokenize(test_text)
-print("Qwen3版tokens:", tokens) 
-encoded = tokenizer.encode(test_text)
+# 计算整体压缩率
+if total_tokens > 0 and num_lines > 0:
+    overall_ratio = total_characters / total_tokens
+    avg_chars_per_line = total_characters / num_lines
+    avg_tokens_per_line = total_tokens / num_lines
+    
+    print("\n" + "="*50)
+    print(f"文件分析完成: ./corpus/test.jsonl")
+    print(f"处理行数: {num_lines}")
+    print(f"总字符数: {total_characters}")
+    print(f"总Token数: {total_tokens}")
+    print("-"*50)
+    print(f"整体压缩率: {overall_ratio:.2f} (字符/Token)")
+    print(f"平均每行字符数: {avg_chars_per_line:.1f}")
+    print(f"平均每行Token数: {avg_tokens_per_line:.1f}")
+    print("="*50)
+else:
+    print("未找到有效文本数据")
\ No newline at end of file
diff --git a/test/test_tokenizer_weights.py b/test/test_tokenizer_weights.py
new file mode 100644
index 0000000..abb8940
--- /dev/null
+++ b/test/test_tokenizer_weights.py
@@ -0,0 +1,20 @@
+from transformers import AutoTokenizer
+wrapped_tokenizer = AutoTokenizer.from_pretrained("/mnt/han.luzhi/dolly/llm_tokenizer")
+
+test_text = "深度学习英文是deep learning。"
+
+encoded = wrapped_tokenizer.encode(test_text)
+print("IDs:", encoded)           
+decoded_text = wrapped_tokenizer.decode(encoded)
+print("Decoded:", decoded_text)     
+
+
+tokens = wrapped_tokenizer.tokenize(test_text)
+print("自定义版tokens:", tokens) 
+encoded = wrapped_tokenizer.encode(test_text)
+
+tokenizer = AutoTokenizer.from_pretrained("/mnt/qwen2.5_vl_32B",use_fast=False)
+tokens = tokenizer.tokenize(test_text)
+print("Qwen3版tokens:", tokens) 
+encoded = tokenizer.encode(test_text)
+