add tokenizer code

tokenizer_code/hf_tokenizer_code/step1.1_hf_train_from_old_largefile.py

@@ -0,0 +1,23 @@
+from transformers import AutoTokenizer
+
+def get_training_corpus(file_path, batch_size=1000):
+    with open(file_path, "r", encoding="utf-8") as f:
+        batch = []
+        for line in f:
+            batch.append(line.strip())  
+            if len(batch) >= batch_size:
+                yield batch
+                batch = []
+        if batch:  
+            yield batch
+
+old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")  
+training_corpus = get_training_corpus("corpus/wikipedia.txt")  
+
+tokenizer = old_tokenizer.train_new_from_iterator(
+    training_corpus,
+    vocab_size=34000
+)
+
+
+tokenizer.save_pretrained("new_tokenizer2")

tokenizer_code/hf_tokenizer_code/step1_hf_train_from_old.py

@@ -0,0 +1,16 @@
+from datasets import load_dataset
+from transformers import AutoTokenizer
+
+dataset = load_dataset("text", data_files="corpus/wikipedia.txt", streaming=True)
+
+def batch_iterator(batch_size=1000):
+    for batch in dataset["train"].iter(batch_size):
+        yield batch["text"]
+
+old_tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/dolly_tokenizer_hf2")
+tokenizer = old_tokenizer.train_new_from_iterator(
+    batch_iterator(),
+    vocab_size=33000,
+)
+
+tokenizer.save_pretrained("new_tokenizer")

tokenizer_code/hf_tokenizer_code/step2_hf_train_from_scratch.py

@@ -0,0 +1,26 @@
+from tokenizers import Tokenizer
+from tokenizers.models import BPE  
+from tokenizers.trainers import BpeTrainer
+from tokenizers.pre_tokenizers import Whitespace
+from transformers import PreTrainedTokenizerFast
+
+tokenizer = Tokenizer(BPE(unk_token="[UNK]"))  
+tokenizer.pre_tokenizer = Whitespace()
+trainer = BpeTrainer(
+    vocab_size=30000,           
+    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],  
+    min_frequency=2,            
+    show_progress=True         
+)
+
+tokenizer.train(files=["corpus/wikipedia.txt"], trainer=trainer)
+tokenizer.save("tokenizer_file/tokenizer.json")
+fast_tokenizer = PreTrainedTokenizerFast(
+    tokenizer_file="tokenizer_file/tokenizer.json",
+    unk_token="[UNK]",
+    pad_token="[PAD]",
+    cls_token="[CLS]",
+    sep_token="[SEP]",
+    mask_token="[MASK]",
+)
+fast_tokenizer.save_pretrained("tokenizer_file/hf_tokenizer")

tokenizer_code/hf_tokenizer_code/step3_hf_add_new_token.py

@@ -0,0 +1,10 @@
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")
+
+new_tokens = ["<ACTION_1>", "<ACTION_2>"]
+tokenizer.add_tokens(new_tokens)
+
+text = "<ACTION_1>"
+print(tokenizer.tokenize(text))  
+tokenizer.save_pretrained("updated_tokenizer")