update LLM

test.py

@@ -1,3 +0,0 @@
-from transformers import AutoTokenizer
-tokenizer = AutoTokenizer.from_pretrained("tokenizer_file/hf_tokenizer")  
-print(tokenizer.vocab_size)

test_huggingface_transformer.py

@@ -1,18 +0,0 @@
-import transformers
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from transformers import LlamaForCausalLM
-tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
-from transformers import LlamaConfig
-config = LlamaConfig(
-    vocab_size=32000,  
-    hidden_size=4096,  
-    num_attention_heads=32,  
-    num_hidden_layers=32, 
-    intermediate_size=11008,  
-)
-
-model = LlamaForCausalLM(config)
-
-model = transformers.Qwen2ForCausalLM(config)
-model_size = sum(t.numel() for t in model.parameters())
-print(f"Model Size: {model_size/1000**2:.1f}M parameters")