admin/LLM-from-scratch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add tokenizer code

Browse Source
This commit is contained in:
coffee
2025-04-29 18:21:17 +08:00
parent df6dbd8702
commit 1b809eb64c
43 changed files with 546364 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tokenizer_code/spm_tokenizer_code/test_spm_tokenizer.py
+15
View File
@@ -0,0 +1,15 @@
import sentencepiece as spm
sp = spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer.model')
sp_en= spm.SentencePieceProcessor(model_file='tokenizer_part/dolly-tokenizer/tokenizer_en.model')
sentence = 'A Frenchman from Paris'
pieces = sp.encode_as_pieces(sentence)
pieces_en = sp_en.encode_as_pieces(sentence)
print(pieces) # ['▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'ar', 'is']
print(pieces_en) #'▁A', '▁F', 'ren', 'ch', 'man', '▁from', '▁P', 'aris'
tokens = sp.encode(sentence)
tokens_en = sp_en.encode(sentence)
print(tokens) # [338, 493, 587, 349, 807, 429, 373, 279, 282]
print(tokens_en) #364, 490, 5981, 398, 2036, 32046, 386, 9020]
# 解码后的结果应等于编码前的输入
# assert sp.decode(tokens) == sentence