[2025.05.22] add MoE support

2025-05-22 16:47:52 +08:00
parent 1d698ba7c3
commit 28fe25780e
11 changed files with 1285 additions and 9 deletions
@@ -28,11 +28,11 @@ class DollyConfig(PretrainedConfig):
    def __init__(
        self,
        vocab_size=151936,
-        hidden_size=4096,
-        intermediate_size=22016,
-        num_hidden_layers=32,
-        num_attention_heads=32,
-        num_key_value_heads=32,
+        hidden_size=1024,
+        intermediate_size=3072,
+        num_hidden_layers=28,
+        num_attention_heads=16,
+        num_key_value_heads=8,
        head_dim=128,
        hidden_act="silu",
        max_position_embeddings=32768,
@@ -40,7 +40,7 @@ class DollyConfig(PretrainedConfig):
        rms_norm_eps=1e-6,
        use_cache=True,
        tie_word_embeddings=False,
-        rope_theta=10000.0,
+        rope_theta=1000000.0,
        rope_scaling=None,
        attention_bias=False,
        use_sliding_window=False,