add tokenizer code

requirements.txt

@@ -0,0 +1,19 @@
+sentencepiece==0.2.0
+protobuf==3.20.0 #important
+transformers==4.51.3
+certifi==2025.4.26
+charset-normalizer==3.4.1
+filelock==3.18.0
+fsspec==2025.3.2
+huggingface-hub==0.30.2
+idna==3.10
+numpy==2.2.5
+packaging==25.0
+PyYAML==6.0.2
+regex==2024.11.6
+requests==2.32.3
+safetensors==0.5.3
+tokenizers==0.21.1
+tqdm==4.67.1
+typing_extensions==4.13.2
+urllib3==2.4.0