add LLM code

This commit is contained in:
coffee
2025-04-29 18:59:54 +08:00
parent 1e63a5ef05
commit 58c964e134
3 changed files with 21 additions and 0 deletions
+18
View File
@@ -0,0 +1,18 @@
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers import LlamaForCausalLM
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
from transformers import LlamaConfig
config = LlamaConfig(
vocab_size=32000,
hidden_size=4096,
num_attention_heads=32,
num_hidden_layers=32,
intermediate_size=11008,
)
model = LlamaForCausalLM(config)
model = transformers.Qwen2ForCausalLM(config)
model_size = sum(t.numel() for t in model.parameters())
print(f"Model Size: {model_size/1000**2:.1f}M parameters")