admin/LLM-from-scratch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update dolly config

Browse Source
This commit is contained in:
coffee
2025-05-19 09:12:33 +08:00
parent 1ee644e8ae
commit c36f0911a7
2 changed files with 7 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weights_llm/config.json
+6 -9
View File
@@ -1,11 +1,9 @@
{ {
"architectures": [ "architectures": [
"Qwen3ForCausalLM" "DollyForCausalLM"
], ],
"attention_bias": false, "attention_bias": false,
"attention_dropout": 0.0, "attention_dropout": 0.0,
"bos_token_id": 151643,
"eos_token_id": 151645,
"head_dim": 128, "head_dim": 128,
"hidden_act": "silu", "hidden_act": "silu",
"hidden_size": 1024, "hidden_size": 1024,
@@ -13,18 +11,17 @@
"intermediate_size": 3072, "intermediate_size": 3072,
"max_position_embeddings": 40960, "max_position_embeddings": 40960,
"max_window_layers": 28, "max_window_layers": 28,
"model_type": "qwen3", "model_type": "dolly",
"num_attention_heads": 16, "num_attention_heads": 16,
"num_hidden_layers": 28, "num_hidden_layers": 28,
"num_key_value_heads": 8, "num_key_value_heads": 8,
"rms_norm_eps": 1e-06, "rms_norm_eps": 1e-06,
"rope_scaling": null, "rope_scaling": null,
"rope_theta": 1000000, "rope_theta": 1000000.0,
"sliding_window": null, "sliding_window": null,
"tie_word_embeddings": true, "torch_dtype": "float32",
"torch_dtype": "bfloat16", "transformers_version": "4.51.3",
"transformers_version": "4.51.0",
"use_cache": true, "use_cache": true,
"use_sliding_window": false, "use_sliding_window": false,
"vocab_size": 151936 "vocab_size": 151936
} }
weights_llm/generation_config.json
+1 -1
View File
@@ -9,5 +9,5 @@
"temperature": 0.6, "temperature": 0.6,
"top_k": 20, "top_k": 20,
"top_p": 0.95, "top_p": 0.95,
"transformers_version": "4.51.0" "transformers_version": "4.51.3"
} }