From 28fe25780e5ebf2562260092be44b3ab973274c5 Mon Sep 17 00:00:00 2001 From: coffee Date: Thu, 22 May 2025 16:47:52 +0800 Subject: [PATCH] [2025.05.22] add MoE support --- .gitignore | 7 +- __init__.py | 6 +- src_llm/configuration_dolly.py | 12 +- src_llm_moe/__init__.py | 2 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 298 bytes .../configuration_dollyMoE.cpython-312.pyc | Bin 0 -> 3260 bytes .../modeling_dollyMoE.cpython-312.pyc | Bin 0 -> 49857 bytes src_llm_moe/configuration_dollyMoE.py | 95 ++ src_llm_moe/modeling_dollyMoE.py | 1034 +++++++++++++++++ src_pretrain/pretrain_deepspeed_MoE.py | 120 ++ test/compute_dollyMoE.py | 18 + 11 files changed, 1285 insertions(+), 9 deletions(-) create mode 100644 src_llm_moe/__init__.py create mode 100644 src_llm_moe/__pycache__/__init__.cpython-312.pyc create mode 100644 src_llm_moe/__pycache__/configuration_dollyMoE.cpython-312.pyc create mode 100644 src_llm_moe/__pycache__/modeling_dollyMoE.cpython-312.pyc create mode 100644 src_llm_moe/configuration_dollyMoE.py create mode 100644 src_llm_moe/modeling_dollyMoE.py create mode 100644 src_pretrain/pretrain_deepspeed_MoE.py create mode 100644 test/compute_dollyMoE.py diff --git a/.gitignore b/.gitignore index f0246d5..616afb3 100755 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ corpus/wikipedia-cn-20230720-filtered.json corpus/wikipedia.txt -corpus/*.jsonl +corpus/*/*.jsonl dolly_env_local llm_dolly.egg-info -src_llm/__pycache__ \ No newline at end of file +src_llm/__pycache__ +weights_MoE +weights_llm +weights_tokenizer/* \ No newline at end of file diff --git a/__init__.py b/__init__.py index b3c06d4..6713a8e 100755 --- a/__init__.py +++ b/__init__.py @@ -1 +1,5 @@ -__version__ = "0.0.1" \ No newline at end of file +__version__ = "0.0.1" +from src_llm_moe.configuration_dollyMoE import DollyMoEConfig +from src_llm_moe.modeling_dollyMoE import DollyMoEModel, DollyMoEPreTrainedModel +from src_llm.configuration_dolly import DollyConfig +from src_llm.modeling_dolly import DollyModel, DollyPreTrainedModel \ No newline at end of file diff --git a/src_llm/configuration_dolly.py b/src_llm/configuration_dolly.py index 1b76682..3367a96 100755 --- a/src_llm/configuration_dolly.py +++ b/src_llm/configuration_dolly.py @@ -28,11 +28,11 @@ class DollyConfig(PretrainedConfig): def __init__( self, vocab_size=151936, - hidden_size=4096, - intermediate_size=22016, - num_hidden_layers=32, - num_attention_heads=32, - num_key_value_heads=32, + hidden_size=1024, + intermediate_size=3072, + num_hidden_layers=28, + num_attention_heads=16, + num_key_value_heads=8, head_dim=128, hidden_act="silu", max_position_embeddings=32768, @@ -40,7 +40,7 @@ class DollyConfig(PretrainedConfig): rms_norm_eps=1e-6, use_cache=True, tie_word_embeddings=False, - rope_theta=10000.0, + rope_theta=1000000.0, rope_scaling=None, attention_bias=False, use_sliding_window=False, diff --git a/src_llm_moe/__init__.py b/src_llm_moe/__init__.py new file mode 100644 index 0000000..67e2dfd --- /dev/null +++ b/src_llm_moe/__init__.py @@ -0,0 +1,2 @@ +from .configuration_dollyMoE import DollyMoEConfig +from .modeling_dollyMoE import DollyMoEModel, DollyMoEPreTrainedModel \ No newline at end of file diff --git a/src_llm_moe/__pycache__/__init__.cpython-312.pyc b/src_llm_moe/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b72ac68e83a2511bcca4f1ff4bdb2cca57f3add6 GIT binary patch literal 298 zcmX@j%ge<81m~yeWmEv^#~=<2FhLog6@ZNC3@HpLj5!Rsj8Tk?AU0DDQ!aB9b1q91 z3nN1%vnI<+MxZiH##?+Y`8he2zWJ`s`FUxX>6%Qpcwr*G`6;P6x5Q!GfTGlpqQuO+ z)D*CUpC;=qv1G8O(xSwY%>2Cg6sS=}%s{hm3FdTrQC{-V9f_`yPa(qrsZhUTjs(yTYW?p7Ve7s&k7(25q``6KmAjGBt=oO=$MK{CAH<)t|2!+RooypQnc`amxmSiX|>jJmwCIC zER_cZFi-_(dTJ2|&B2Er5(EfxDUf3?Eef;;sx6?+Lk+lSFFF*Epa2Ql9y+sswpu%f zE{S>b&CJewJ3Bx7`@X&mf$JB)sjvU5kC4A12(Gx#oU8-$K2eAw>7*q!BnkNl@R3Hu z<)eDE6>G#K5+N*d$MqOE@m7}jMDn;sGD2P_O6(p{;!OG^>Pu5fT$LK>?L;kkh@zS# z2H)b$=CWomPy=j#^J*QjjUM*VHt=H{KJsy3dj+I+Uk^i-8_ua@4fv%77^ zZQNOvZRS~&pJCByf3pY0+%i0iu~Qx%d(NMd2mVf5hh9WlGuj=SYKkTLT-KtjwaO5# zYl^}QYT0mTOY}p)n1NYDTe7uLi;A3UNNsb28J0+R+lrWB@>a9tCl@YQDwu31XCs9C zTza2$r7oeUaj$L)kY50VbR&Sk&euf&`DFmQ7@%0FivtSEkaPzDB|_aGk93C&NN12H zL+LP}REQFQ(jiI$3X8Lw1{4$s>1F`sLdz^5Xb0Za?E};wqJBUFAsPTQ7@`qCLm?t} z$^rj)eP>>Q*gQ?MNqFh$8bz7}2hY$&u7S?320L^~&}t;o&BJ*{Ld__ke2B&XjfQ9( z&{&8j0F8%e63|453V`r9?$s^sMpR-*-6BW}fz*B};NG3;lK0vK8lDpE<}|5W`la+! zXrSAq7QZE8_I8_zwA;yCcEr{g-7>l2WvBwJ&ia_trbK4bY|5+D(sq~_Z^?IQ+q5({ zQ{4=84MjFcXS5p944d&5Q#33{_r@Wk)1tnOpUNXBdYty_Oop-;?pSaKbX3@$ZfORT zlIZ_0JcNhvbo5DIVTYoguZ+K++_dpXaAs)LJB(8<8!8hy-m)l`6J>2nq;cb>++1fO z;{|1%*)kYmtGAkPr=dPsrK_53iJ`~GSHRHh*c?knj3}&WDzKB)mN`C<&DvrVdMW+i z$hcwfdl-dIh7~We7#m+G`rSj?X4`F=t0E8kU{~R=f<4S7&9+3|hjQmGh%5`{6I&Al z&>^X5KqvPj6}LLDiAYg6ie^(rvRF^zZ;3APwMg=7ZFED}sTeg?@L63ly! ziObI-c@D`rBv^WU8c7974aq!`=aHO8vVf$HWD&^=NL~aY5*u4GSK)n7+SFxu3~Hgf z$~RrR|Gc``GVDcV@%Bw?UFK}j9bStTZ&F=v;SgK&rgewAlbX7E7&Vu4gVGt~j$7ah?9+ zg;zc>eiuW&7}z`tYzlL(%fLnH9GXo!ld~Z1cYJLiEuTN-#K{gEWJ~+mQZIY%quZa^ z2QS^&hyTJ2XW_+zh2{N)<0!0 z3ig`2jQSZTSgRL%)oUN{UoWD#^!HTXyUG33#4(AcGfw5gQ54ok{rQtPtS1mXQY&h^ z-;WTDz--Y7H2Omf>LGUSLmU^7f12LW$Sxu3kMf_s{@0zKU02`x$FClI_1n*uf4HK8 z3E90@fBDwRibmiregy=s6g7u9+cW}G>=0)c|2B}774_dAAOG>@>RZ=u)zX;jhTLM5 ziVTHZP&zvDIZEH|$a&sZek8sp4pGBLa=LqYr2@HL^! z#wl6blet+X`urqrGF^vKllUb#orNd<4#z&tad^4}8S4)~zP?ACq0;Uf&Y6q7*`;1= z`cbv+g#YtP&in-j=Z9r2xhuVit4{t*ue#J5x%ByT z@ACJ0!*4jlGrghN2edc0)Em5XoQ!#Q4U&Ap$xk?ivQsEIrLt3+az-YdN);uelg{v% zQ<`x`O306woMOcpFQR2(3N6bsNb{qpnwrJ`MCq$5FH7;vF+s{N2ak8FmJvDlqBg8G zXt?m5$nmp-;~Nu|C*^fFve^9H#@n$q-ST>BMrAhkI+byL9q&u{07EB|l*$-mj-%7t z7XvI(*Sr_L%-;lvk+-e_If_b>^pMmZlFCE!s*^76jUS|o`|0BS8-GeycN1SGZc9?8 WH(LFYAam4?kX)&kDZP_)eg6kavW+vf zOat7Ysmo;I0x5yift112uGB$umzjml1D3(Gt~BPi45SZQyR3s5T^WP6F595J%g*A{ z1{{OVE+_M+4`dFyx?F?qE;kEX2eP`dP;bUS)?jv5c0A5AnB9|N;s$fNa#@URAa5|g zE1&u80|kSHU4_i=7$_Pn?kZ+}=YV&xq^pGaGY3is%eu;#--WWiF5h5zS2+v22Py_D zyDFJKYoKbdx~qDyrmJSKwySoquB&dazN>z)p{s$VXAd+EHgz=(F6&x0*xc1T*wWQv z;!?Pvum6-fT>5XY^(~VU>S{~jc5(sFOI#o)nBCv2=TUucrN(ovWI1z@Gw(4u+gZ+h zZy*eKLu6zYsyrH$-Hma`r? z8y=H$6U*6%oK26(xtZl$hMdj8l3@8;SRwRuZDH{(h+iH|)mpHX#kL~0Ex1*S-Il@8 zzrdEjilE)N-$9#kJR1*dZz}s;iBVh2e2*T#k|p1=wl{^!_gVw(7}@QtwX4wDCxTmp zg<9Wsu=v%8Uz5;_4i>u>vFj3Ice2>^h~1D7yXy=W*!a4sYxhNK{nnq+8mTwQncIg4 z1}^R&-mz_XsJHLT&uAR$Q)JgcK6sSx?i&gQ_74Yw19JYBef#`djvn37d35)IPJhR- z&TZs5f-2HSy7_Ql_dvZ_wr=Yl80bDd5R|P4M#6o=L)`=@LA}q&-5JvX2E!&Q^ zb#&q-p1{SS?!mquKR-MY^pB1Ny2C-)bNIl)9sb>&yN~KG=|r)VAu84h8vcdfooM=ldvI-j;AUIK*PQL+8-73qd{{8M`}QX#c@K>SL)5S>L!sw_JgQHqK=vI8o*xYk^#r$} zm7%`gz964eKKJPGx!_Q8vYdxECRC)h1fRsRt-I%JPE6vZfAZ&l^5LKT@lU_`rJug`@`r!%yNLYo<$w2Izwt85{^6C+{rC@NfA-r~@CsHh z)x=VI)ZS5VE%;C2`@aL{4@}p%Fn*Qia%zP8JksBahfQ3Vae|bI@`?$TLQFQw1oH`|G>bYe{eY1 zGD!Oa=8~T!VJB%GxyaX|{8&NKhi_J!UWC+83!GnG;_h0Vm(xXS!JM_|ZeGE)fysfd zw7;?TwY4+nqLpi-dFv+3FWcvRl@sOzmW-jd$mLek^j}J0?KJ3RBt{Ghg(7e+ak0EgDX(74tDmsP z3aX`oInc!s9~Mvr<=3#9u*h!WnaUC0Ci~Dx0=PT}^_uiEl## zdg6-}US)$|o5ZsSV2jYw@M#jKCd-VerA@MU1b);H@+AQED5WiYJ;#^0FQ?*jrF4#^ zHTkeMH_%Sb4m>vQ*bL5gV>h4=al^#EL5SKBA6i@R`7qy&b8>_qD@q<+J&itdr~wXE zVRrtM`BnRMyHrpw7SuOK|NLth?E8)SJ$H{=fYD>0kWK z>_+TM*pP6F2mOB8=EsH*7#$$L!|y*o+C30YN%v!S=)s;v`vSk2oE>mvXMD%$2@VYS z{pblDF{~$>*(BjBDJwZSlu!WYmoIT2rI;<&pWDoqs*fG%mSz7f+ho~eVl87?dyq?e z)Gu+@X9+s{n}T5^5?&ic>1jMsY?ZCoUg&xH-xQwFc|0~$b-&3 zU*jjp@o=ez#8T|_J&6-f+b1t!nk1cU=H!ztf=TID2aaQ>5lwjZJmy?8obJX`JYae| zMTN!_Prv_VZ_rNE$G*W}L81?d;~5`~q&`^V(OfuN^+gz#fc|Mv0M7H&&W;-<*?hXM zJ5+DyYmir27|rm>Sr2_&<)lyB2WY&Tmov`LR2kui`{ne}5yg`MC~k;(c{*Fg5kpf-OG#kglvXniw>*FDdJdo!dDH(_A~VX&(%I9?s%3J62pXVV<)V z#LC;~xAb0N@vDW`3#V5~l`F)`6|+aB_Pt{JUg7YQbA?agEYG}`UwG}}(1?gXh}CAa8pqC78jhsL2v+OY=8~RS&qt@Ry2S(bNIAms)yRj# z@;H9@C&@WV&M}HjRgO^8^DH$)?ZWp^KF4MR#yp9iBYrk+LPBU2oJ-t%PQkU#$)Q5-AwqM&)oRT&F*N+=4j=XTid_0?_2wXLx;b8K&(6x%{e-e5pxv?)&gZ4Pv<*O z1HY4;UF7U0ho*qCMeiZsUO0(6@_vdWht8f5&2zRRr&_EJY<5fie=AD06e@c$W$M9y z?NJxdeL6W8?ZxJRDVW-u65o%VU1kt>Ed4ZtU$wBFLXsJexIt{HgcHur?PWCcuR8xybJX)Tr$TdM{pFXB=+~ zLw<_9sp^bVTadV?4y6aM%O_wMCL>rAIkXlh>|mVHhaieY;;x1jm8FRl5;x2Oa)=t^MCIQgteLs#fnaa< z=s;Mu_6=R|_wvE>9e{al0GvX7AtDgeI}>)_lX$6|ezrT*9S-xnpF*i;f?@s?`SJ(5 zpZAXphx!P2_XpL<6q0j>`94qyx(ED}BaW`}1j)x+$FQ~DAvp`xDl`ed+F%zSJVOoP z{ijEJdqJOw@7Js)X@TH{zMddYOK9o6S_4TAFP`PvzP$s*L;ok7#ryU`8kb)}xYb>I z_SIEW8|Ul|GZ&s*$!H6Sp3dwmvOxeOl-` z72Vn$Ejc~0{qvpkB{dV7Cic6T);;Wt3ka_%=%0-QplGurFbEu-IP$4@Z=$pa=vGdY5 zYZG%Fty0>i{64;g3Cy~@PhMik6tIk2!XCAy z{RFTlJrN6A!|}D>&Y8GK>Z86*#Qdn#F&roT`AM%pFpwIss(l_$OK1^c4@plXEt1xo z5=ePFLo1OUN$)RC%7&miU$CQjA5LZXw-qJJ9!~oNB{P&6tG2+QvH20jT-rmX6sm-^OLmy5~9R#H)wP$3s z10+(rvPb!q-8MKxq~8F)68Qk%rV?Gj#A70H^;v8ilP!#tsDe?r3t}+oST?Ii{?kDu!YVHbQk+Q{{kX-B1eR1fdfQ~KM9ap znF^^TZ^m(`?I|jK2oT044*OwA`K$Y{@1Lo9&osMoW|`<+EqOPJ-i=Z3X34u#^zOXv zkal&6ySjvDPKdis3jS`PClK`pCGT0$dsgV{m-^0&ednWIUa%I<`A#Z?j-6xm(*hTmJRBck8~s?A>L5?U}3Hal8C`b>FS~-m>p56P`JBXV0mt z-Pg`ep1n5syMsdSxe5C{XZDq6FF!kDzj^E*o!jPfYoy#JF}Eq2+bpzpNUi(C)_u{| zPBFJr%6VGM!CCN(aPrw`j(=juz3lvJ^^^5e$EJ5obDWv`gnHlLk)t#-2Z_v)tt zQt3(&f7vUSiv2|vm$}`vkj=S0S1w$>Ftz2?o!56xT!^}=rhDG#eXV!4_3KZ(`^0o_ zv}T>)S|?c7{lY^91n0K@9-=Ms+rF)qj_TCEskU@9rz;@0Eg8h6!$bjb6(ZNru>TC- z9pI__BcR?BluXWc{14G;U{G(CWru0%Ge76(ccF$$^ISUi+b3T<`5XS2rQo9!w`KkP zR4%QMoZJUy1Qo!T>RCimqBDpZcK?fT=u``UzK)#+q2wpPctwHKF)YU4Oi?&Tp!_L> zq~y#9C+4SfCte~YOz&fl5ndZfv4fV%c1Dec$*70T%1%Hloaf;Hdtr;129qaRB_i0e zOL@LkPIpOuUe9(9^zy%j+=KWZqWnN(GHeqo1b4;Na+)tw6_btASz>kt zVg!4Im{ECYYm9PSHC;=eOrJ`bv`?KE-9AxIzGrt|S#x>KRsL$vRNK|3r?<}8YXx(y z0z)L8^0c9`(f&htFxn}G7@}F-Re`~!J|+hs*hmVU0s5#$Gz>73N+~op;}%d>lJjX) zffY}PFqy<%P0r~a85y_;{v2UY z1pZP+{tL(!#{bYWaIlrxID6)mWtW#tIit3kOWR^LXF>>oraSBEip$TvvTpkPmp0C9 z6N_8K%;lGM#$0a0XuzX81$(hzE>>2{2E0VJ5Z7-+PE%Nai&y~EX!beZgl8#`@B}{k zc*=O{xS2jez;vz-^A&8CBed86003CjTs_olA&}1q2mw(HB>jjvVtFe*wZ>_|CQdVy zq({j~Q-I~B$%hA@Hg)tz@F<`5m6@G~w%oh<)h6FLeBuirEHW;Y!n68t_R76K`@;Ff zkxf2U^2e7i-TU)D0xL{md}%1gPrvrsPri2T!^@xl*(;ZR`o}LSsTjdvci4aK!jFIN z^Y{MZm3S5RzWL=Jzx?7)-hR7HVWHtY%6otErJFzb>gA@P4GO;8JTyA!*IwXf4fM6N zfb>a>W0ra@c+r2MdtekRoS1j4g$!!wRQHKdbXiX4gN%G#pT^V2<*!hrg|Yr*54|O+ z9x$xH0+g*(Db3svVFAiKWrr^iKecE4H6(ow|3kxYFgG)~woOv&b`gK(=1V&Sd+D@! z`fM!SanF;M$~Xw2u7zh3_Op&9d=-S1@y6>Vb93Mpr;n6vg~yRh|`@Z<^M+0#N` zAeu9HX@AUKAlZGQ-8Y>p)UCT26zjI#I(mC`)P7VjA7%4ZPE%;$vIUEgzbL1oGqQz^ zikztr-yrzXa=MBLrU!t!4v&U=G+F{$PkzFS?;pipzlhO*iEG4aP~pdMYXVb(_(~|B z{AhVI68KUyu1OCjxgjiy6H3G4)w&`U1_#?B<`*j?W*yHVOiaOfP!|DDNl$>$NO3lq zX}OKtBesa0?U%N&MglUzar(28Dnc+~3z!r5dPJW}dfv8Z_?r>M!qlIaQ~<%Z0JG6E z{e{t)=mbi|BSZJpZyQ81lj3+BZUtkLII^yWYdUj1q}g`6*wL#5<2Cb!MlfFYUSO@d7aJkg7I`Rhy!o&6oDw^AucLKe>L|GCd~Mt`lq5 z-Q05Xe6)5;)U);0lcHzerG4}1xk7sRd|uV-hu$>3*(x+|pS4NLH;ca{8Q@-#X#A=gGacVRD1ysS`bQGv1qRe|_vbC%<)4@YF>;M7CyPH^nXOIk_zG<>)Dl&5-n!WbY-udhjDZ5t8uAMQ> zv_`X+N!hE#?A1z=J#We(*sBC{l>&8PM6^ex2i#wUAzDQIB`K1WAf!ivmR>U>0h?cP zbfq!rEN8$HOgG|q2ht#SWtd_dtH8a1bjWlWmZF6lup-sYd>JfXf!2D>!%}TQXE2lb z>`S)d5${3WOSGNpr2h(42(oq#NQAi$Pd9kW^(97P{Pkuf(1 zgS>JEpG2K<#z6N)%-+7h^Twp_{|MM5i8YABW3kZ7Cu0o~gk?-RLaer>Yh%h0GNW#V zAO>rPpmA=*y7jWEkLp0#3^j}`mx!=QPatIsB<@9oYr_pckt7`;0=3rautpp*A_RS! zUK_WNL5imS(xiqX2yqhHsPUjZ+6erL?a|6BH7d0ljo2q_k9HLQf22Ly!T;FyXkYdd zj)VaKgRlO$@1+q&!-Oic$N2lBt_C%S{tKWb4#eTxYXHixp)QSpiuW$p->3TUHf<9` zd$B_#KmBdVF*uIb>MVBSQh~I9w&DRS8p^;Jm+U9nL(+2&6qS&bNG5t~FA|F=y_5fU z$a_Y~KmX06{6TmjXF|efa%M=8g7iPf#31X7lbAs~u}$Q3@PKgQ^NfqG;3ql&!0;UQ z4*vI%ng1hl2qH;j_r!sFIioM+S3nNN(dUm)z66|r0&01beL&eQUvNI{E%d(YHr9^rUpCS3K1FAT=f5 zh9dR{Rt~ZV|MrDyL`Tg`4hgS`j#W3)q}6-G)k`$5W;*c3;A?|7^Q1NV#5Mco^7d21 z@8(q-#?O~lNu?{q(iJo?#q$pLmA#kuN{)s*j)uGLxO|sT-98(Ry4MTtT`_mgwam%P z>8dvxUu%>qHi;FR?zlJ2SAd=Ofun4}!ewoaOI2;43FXn~P`7`ku%3&K^l&%zDSTQ)j6~GJu#mB2HCFf#t{;Cjvli#EBFLgTInGo(V>t z5vmO|AFb9-vxHV>E~?QsZVzjx(6|d_43KFfXe?l1Y|LjrMTs;<8LW$>8L{$!Vr#5L zJ}>N3Tij2Za?%sXP-l*b8!8D`smY(bErbY+yHdCaddx=J9!c%BpnQ#5m;M_{3D|~c zG(p!)eh_N_Q6RP{IVVsc`zxuDw6B=HU}E3WJ)VX4yZcFiEa}l#wu06Xd%AS&;s#=hkdL$KCP~1-_f&uAs~7KNg4u zFiX)VDLRM7D7kTbFDpUnF;F{j#Cz%=CcZ80@Qi%JzfV4`Rs8Q#@CG^Ggj1ibuxK|> z#NUwf3OU~)hdwepGgLWpMl6;cs^Bb1KFbb9U5!(Dc$#Yb+vI3mTG^qTLUEdqNBcfd zpMbGsuV&&Fb>!XziqxpxV z{Aa}cXM_`A{GN*DccVaV;k9QbpP9~iqxiMrnUm4-byE3ev3&C_Z?wEKntMRXeOk;_ z*|E9)37h(c4bl8oDSwriziKuT&EGENcZ&I)x6ccQk3{p2qFztUW$Q%h#F4v|Puz6e z>K6`!=G_&od`7hTCLCA$=k0}py?VZ~W+rrNNGv`!u`gEJF4e9VYuDfGy6qL7d`5aQ zC_Wh!&V-`1VbM`Fv1iJ0uWplIEsoh;SJqu#C)pcCd*jS=ckHWTF3*)0F25kTT0~dN z?5aDiH8*>|+5djOup3nNp1F;IyY4*6T`9UN7g9{Q`%-?MnwpjUz{b^XycziBx%baW z8=n>NSK}All|QO_BIa<%90f5)9&?Kp?6zv#f`ju_y*mEI@yTr@zP?Lr+ZBU|9QwNG%>3D!DB@q)m9CuAKuXkP6kXBRoU$syETPQ~0y z;6A4v4Gx8dd7e;8o+uYQp^>C-HgqPQMEDs`$Rod*oIT|1g_9_f`5r}*L+ds~%ZD+i zQ!Lhnv}#N3)Mpkr{M@YkIsGg=W#SrJV(n`_a<*IQ=kvYSB9oB?3;f{Jl~zu-zp?(c z^|QHO_rB{DDmREF8yC_k&B_&)Pv^ejea$NtG%sXOl#NAQcP-c{mp-JH!eF?uC(IWqOsbnuOV*9OFr6|+y>4Bpy7lB(jyLt^`3!F5>7 zIP!}u6kvnOy4HhUX^)DA-an40L|P)F9-3aVDilRyUPI7q6yIxx-kLcFSvDJxaZs=<|BPM`m*=GfeqmC#4{rV8c3@K&;kgi4K(RN4zhciAFC+B{=kv~k%Q*h+;c=d6eD&_n&`Ikh2HaF}DSHr7gAyf4rNMcWw9N?x|j>bh%i%e6~hv+b*_kzui8Ua|pRJ z?gDuRT7I_&vPgqc-$k+S;#|*I)bWDYGbTLifBBiIJgKlzENq-PE-l+3F57YIg0%A~ zapzMXIG+}J#ze;p_ft~}fQf(J@vpMu1;*6&GIOWtVwg0gjj?k>d;F%mj*Q%uUB2N8@y|MJPnAd{XV@_s?P}hiqIHoNSB%uM05>- z>#x;gRbLf9g_Jr?mfO-*%NBPt` zRdgH76fWXa_*eZcdabH&(TvUnx@M1Ls&!yK01a2=pc!}G1s=QsvKFu*r|Z+t2z9I= zGx|9*hq?_~$V6bYISew5HDY1N9;8d#(GA8`=3gft59erooB_QG0r;PwgaPuKluel_ zDF;IgJ!Imk^@R$`^B*ba{{=@jb?`G3s3#{(F=pDJ`FRTeh{CC%(LtT)rmnebUnA&UgEoZz67G5`>(KAbr&3IglX_|^Fk)4ZHI_IYs-Y~>VfGlrq$aZ7BtMX&5T70)(^^ECd(Sqd@&bvj$uhw6$pWZ)v{;w_Hv46`xSF~@!b+@VYojo`9%zpN^`FpnS z+JvXN=9+-E)K9pC!g?SdlD+hfy)@=1p8Bj*)+Uy<&GNTa3T184vR$HM*TjKXQN2{u zBo;NzoRpSr7ME>~7HvVTrIoLqyM8X_%%5rzo%J(o1(0b6CBUW|4+&joq^=>cYY0^2 zG8<5w2hgMNlKiZKHb{A`VqUAzwpBQIR62M{Ja|gTYmMf0OL=`_UZ2o^P8b{#M$Sj` z_zC;nqOxhT>%<wUH|DLMshYLSG>YC;ioEbm{?^*4w{v3u_p6rAmPD&I-D-iho$xdhTRMDN+}a~n z^$3pgn4=7|VA0Vq(>fCr+V@E9`^EPCw+E%pZn3jF+J5>2N6&(tD{ccQU)Cm-t`SSu z+-$ftEIipAEj>-xoU3Abzu;;WtgQ;NV@*bMXh&6+-1o`1nVdc3Bx5XhDUuvm8sb8g z@Bm|ZkYcp}E6cI8O&@#X#A_!OICy7+UmtjP;1}dokO3;yga6v&jr;CLP*)-{Ksse4 zSP&N~NQF>=MKSuKy8(B=985Dhjv;tpHBO;T8sZ0(YH$d%)O4h>%^V^HR!yWp_1UyI z`y*>{Ak|^q2IE-ekYFuvY9b1mtPX>tG+w(=N}3TrQ?Fg|_2va!5KnM3@q{dhCuGT* z=IHhyX+-n;Na8?eE-jKfAcTbgwEN8fOaA9kCmm2DA=|e#MZ87G69FX}coCmB__<)> z+mYhXV^m)t<=l2m)f%Ynq>dpHxeTNV#(=aTXB~28g*7U?PJ2fiG~{V$d03-`qc*ep zCjA$&Ml^9la7sOX6W^e7yqBzBT{HSiNxC2o9wwqlzlORrkQS|}NO)n~%NHQN>f!h} z?cNID$K=<>l(@R!>!!hckUp&x6%}ed|L@_*S-oWAN2l)k+v+_!Vy9?BzegD}$Pfyn zaLTU1fe}zJA?&O$MwJua6Hw-{Kg37?I38hAfp#FM!3>NfVQ@U{WBe^jbtwigSXGQQ zTL9)+0=-#z38gTSEiX{X{{Tm|k&(!GEm6K}Y1SP_#Z1QRmK#oDI35x4S9?Tw@~JzuPuI)u6y|v)2pP)cCoU3_JZ`p0r80gjJbMRJajsm-ved9U3bfxNtZp9diC24pcIk6d%Oa6TztrVu&-EutH(}8_NGC zIalHIByJ0|5fH9+zX*d%m?mUWjrGWNOwk0$+mJOGz^0=>z>w-s2ke^>Fuh`p*fb`K z2@;w1h>5lrvf;wsrBQPb8GrQcgyF5}YP|lNm?J>|!=z(W>JsPEA_{}YH0~UNbndt_ zu?4**uJ!{F&deFLkOPp7CEwQ{Z^7vMO{@kR|pPN8`uA6WQ`E?*bgUGr;B!D;O@x|+J z7Yo{F2cresVdn-gvGuxj8dh&w02AADuMU3Y*qpsF*0@@9)J^QazG2?$o7g{J;L~dn zvMVOie`NQ@93G*dUdV409gPz^zi%%?#1!6ID>`Z?pp20>kr{{TIy*Z%Iyyke&Q6B! z=(y%-1aulU&mcWUyOt?|pb1#E7#G1&lZ@ zxqPC_H(fW=KAS7GZWUX%-r6B;KPGNJCbS+C8;^?>PYSLl#f+zZQHVmyPDsPG2qs3D zJ@;uR(+##=G(k@&OR;3vG0dwb>PS1I0~r%o77o;))J?Pko@j~8xd2ayPTJp$CsT^? ztEoo))xlI4*)hR7TuN73AoUDv!v)N6tq6nW(q+ScJN`TH---X3`0ol>U`ft>F%7Cr znlXoDyMz|T4n4|djOvJ|swsCh+|{w0|i*AtTumkfycmIARI|)>9rMf}pm+ zJ`JcQ@kmk27{HiD@*yDVq`*85jWix8N*j{xU#vEcH-cpQiHVf-ycG|VIfaV}2JII0 z(tyOTwu|rmwfE8h@_n)GlfMNy4Qe2a=<)prT$$R*{{!*@xLk)H(mxK=I}}PhG>2jP z0fr_(DDo==e0Fltfx;yD6vJF*7_sb@(+RYLNeKQBRl{gYemP@@YBfT(kAO-edYA4eb@Wu>~-^w>?`{(?^i_TBnOjB+ctYdaMa8>*2G-7#H$rtH8FSIHT$Gp zB7IAr0D+r+lCCdsog2PAdHuVwHh@vW60koXX^1xFvX*Xdk2Ut2ssKlSC3Sr6BsE|F4Uu8G!5+<1C5Vm zNPHdXu)tWV5|1`2f+``Z84V+|BqWg%gkHT5pPx_K!ePOgoR<_3wLzQn~m<(Kxw z3d#g?Zmgz3FjvM3TZDq;6s%k+RJ04`vRFyEU@nT~7GCO%6_pC+{Fv7#mEZZV7gAG(S3m;MwrPkvM<31 zA4HhRSZWi1{s*NRY-y}45o;XGm%;L>?Z#lK4@)Hij;jBm`iNMg#5pxzCaYzsx4D9@ zpqtg_4rcWh1hQVTs2g%uHY^htla7l~PXgIX#AhxMk6s1r!OUJ8Y++lvJi(kLYt9Mg z26FK~4>dKZeb#>i`G_|V1`90p<^t43WH_{-Hyie_%~}r&7}=;0ScF%0s+$^^q`?8U z-!3}33A|)Sw*&LXm{g?WKJ2ih!#?Q&S%d0e4~_}qCMG?|p_NPE)fgPvOO!Vx4go9F zKnDGV34|N1B?<2|AS^B-8yOsOD$gf@uMSexV!pa=c?q0x_d}dV5xsR6+Y zu?A$rMMO7^>=Ulk>jFrFy@4K#wbnIn73S$QE3PWf8wt@s8@=UZC0Vh)2W1o1V;H?jprzpI z1{hssDa@2Z5UGry^7p8L(N{F$*&@~-BjY5{RfiNK=^=%5W>-4OT-rh?eE%hEr5K9z z7gHkXL%CSg37J%{A+`k)8Om6fEQ5NJAFz}&P$px^G6owklT21g5@j5c+c@SYmK&@YRSn*iQLEkk|5clU+u*3e@fK@#w*o7^x7% z(76a^UE~T{Rq!Yb}?u(6VBs{zO+lR$(_Ijdw)Mmu&jspl-_J z=7{Z52c-Vpko13q2`pwL%n>eE-_T#7`t)CD4M1D09 zeyUk#ec|*Qz(_E(4^(+0g1iy^yd7m=EHw)pv#<*0Fk=ZkofUBb77fD;F=~2!iNbsP zdwL(MbPMIBgG9C+C;KlucphTSPf+YvD3eIe9da1Wq?>#MKq?0|?HF<{IU?>!gR?7lXT(9t^mK1+TdLR*Y}e)2v3X?GFlFG5ZD~& z+o>3VyMP@T9cB|n=)=c`VKc=oJ}7zC=iAh)rEx4^osPnbxmHe<1IfTm*~xzVDmR19fFy2)5=`J}?V<(dN)U z0B`{_!^gF4GYPQi=3OQneM&m&7mxa-qvyq==gG2M)oP(?yXf%UuKHf%cN?WW{oCJr#YhVt09!@{;_Nj_+UShiuVY?BZO3J&kxl8RSPTt5+W`(O}g_N3IlOKjhD zI}4Vq#frVc(QfJJS@GyuG^sRmq7%#n2VO|b9jSN)q<3bwM2lBV?1Qd=l-DfgHOCx9 zQ~PH&-rOYYJE_3Tb`q$sVH~5f%2$W456=wVJP|G312LPA(zwbeu5Z1$9fSY+B|kD7`;kzF1CWET=k_Qy0rAiRDzpa%%7A+RL*i zogWo(<;}>pkc!}ZWpk{2S*)@xR`W!xenYIZHdfght6TSx2Z^NjQuh&+y6Rjg;z}B3 z%0+JzM%?9#xl5-v%x1w9IOs`e=haL(-tO70V$Bmm^%~KQzw(8>Ca$!8q0___mL$l1 zVK5xN7|j;}=E5DVm^$}d)v?mXn71)j)ELXJhWo&t=C&%M*>r&{nwgFOL5@RW8al7mtLx+eLP;G@4-@g4JrT1G zeX7)bh~g%uYv{U`P8zRj1qr`KP<=qrtK5XMe?ufAv@T)^tUFfJ)Z$^wWrAZCo)VU(#{d!rGgsu^B2f>y!QPced016G1!!EnzA z0n6dD05Ly;+%cXx)SB=WOktnaq6XEc|EgbxcY#1-03j>JActV$HsT~WI$$MtNywI< zyi7oV&^ukCFR9cQ<8Kr(!>De2ZZn--y_uTt5iJ7CuZ{bHu!1>g1zC|Q-JnR zcMz{(8QB{F9I}n9*~get{NJH%P!1tjM9xcs8sp-`is=VC>`W}S*k$EEMq$|rGmZpi zDYD19V6`Hp@e#5pfE~d+;1rvCMg!e)>hmL`5U?itnj)w4FI1JeFB}|%4lZ?SCk@{& zau{@1PC0$#(3T+=`1{U)GDfmD8nEUk7KtMI#Nb0s#9IOTED&q%9fph*4Ehr2Cwr*X zWV}M*nkbjwe4kRtY`maq!ay-Eob4YSBHLg1VMPML zfGW7n;J+i(B?jR!AdtXa*^RMPzajg?{C?yjYa94xOnh2n(G%~fFp>C8aw~3Y8qLn*td!^l7;_j{}v@}mXEAIA-l~BO+ zecw}gx2Ot=nuXn_`>CnLc?(=>ZXO^TcPZ7|GyAOAybVXR;I58&NV;OP>3~Vt)HSt9 zDqbTNubC@eC!FfWbrg3?YG+*ED2|rw5O51vb<3=4Zp)Eq^-;lEf!dr|iXDq4(a|)U zcE_>m`?=6EQ$jFI;~?DqAC#tpNZ?7M6bw@E{ZN z8wK9^f-0$iNty4Hnl_0|n7Ysin7PU;c zeq^r}>(>G1ki45<(LU*jN- zXIx_L6F1An+;!k&I*ZVT{9-BZpqO_M5FvwLYGc{?F;5|h1?5*t`SoIcJz%H&CXCm$ zb(8eQ1rL{h$aJ5xfzoORaP`n=pznga0j}h(7Twj7yFqj}%nXa}4KzYU%cY`iV$rr+ zdxb+?(V}N2_RhPCFv<$2j&=oQ#_21Nwt=2Rpo9=61~iSpB^vCD`PlL?1(P2QY*Eas znj;*DY`Ba=w~A{h?yB8fVZK9dPRd)ff8&AXj$SHGzEDyW z7bG4tG_$ZY=m9y&08|Q?K`M&pAR%DL<)s^uTwXkxAo6R~@6Axl^33Wl6C>XB>ur^B zd&bb;b0C+KRH2bB7!JVrw*#jv1C}XEdqn zQ)OFkAIJmDo~3?%@Z0e*6SSyU-Yms{C&88cNlJefj%@B79T?zuQ}}l&9K7m>Tkxh0|8to(O;9Lw-C?t^} zH6{l)o}D2>po;iHK+#t6s0OQ)^H_;wFI4rVh^faUXQ&lROLLyWJI1yposho@q%cZn z=trmxbdmzjQ3O*6GtE-Nda+@Bv|;0|@;eRN=8Nhk_AadD+{M^eA3Om91zn^!y(3!C zAQd!2U4CLWRDE7;xZV)U@&S=`FTeTp?VxbtG{^_VnG<`VLR3}HwCATy| z@=nu@rrGh^$LAW3#tLeb*U#{?t-XQj}- zYc6MZ%#km^Jh!7|VsEUddO9$({k`({4hc1@#G)spqK#tF#+zfeTW_Bh4j+vc9YZDt zLBvYyrP3C$v?W^Fdesi063J65dTM73S4gm3`Iz#Vubv|9biKiRp9GCj@V+=x!r23Vlg-Le|?L z4I6*zO&zeU(m@MhCpo*wp<_mohFwPsYa=eLzlZYfg~J$~n-y|jGo_I8d-xyvDI6v`>$Viq z9R>LLxXNTH|2WNLsX!;w^5C?#$M6R!T4oX=R#g{kS{`dzL4U9ik$=HtvCVsmr*=r* zCQx)l&$0yz#i8UPiCPx-kVq|ywQ)ri)796@7VIqO;2@lAk$kNp{_pMvS414@GWaJ)@o^$P_X|fd07nHo}xb9f6Pyii-S|2W>T}Wp^Q0q!rY6c6UQ|J)$LQ2?opH;AM zDuqkSUPx)M6vyIqrNTd7=v}bj7gZJ%FQk(X5Hb_3Tgaf0jmyfr=9qLW*eL`%8p@lT z6w2gs3l?1Da|60u$Rb}hUjDHMKI%yJ1A3nV&C#cq?d*C$MTL|pvGmZ9pgvZ3Xv*II zOT;o*&K*W6<%w|NIJ^TwQQQ|YT{Jy92fqy`EV~=Y2)e`VMVh99A^Zl$*A+|L4g}T$ z*o+n#-BCP?}MyGB1dXM7rl>q8CJ$R22@>p{Mvo3q}x`ixHtf*zDC9 zw?-^x$PgiqDph4PgFdUxpmBFX?|q+k93yV(OXB!ErZ#tC-Z4AI00gz$+)2kw_=gX ze&V+Lb1MI1a{e1RzoeS76s{pNE~Dt_%Xx8DAk5XBPKf6jP|ZN@6}02$#?|^4lf7i0 zS{m}|#yK3YNXHY)abP$U3K1Y;z)NJHgIxT5=dsZ`i37B)|0-mR>AW6f)8 z#Ao}W%lhF82fJqa-x<6yc(e0^=DotRed4lyXv@!4;)*NZy}aUw_3ZZu2M;q1`y-bQ z+*5$;Y5#2SR`y&GF8s*9v_(kICkysgU&tX&nmKSQYp#6T9c$@)%}S|et%yHs zCD|^Bc~?r_wW4?J%~sL7;a0=#&x%{06}^6x#fk4Mo?7{O*6UlQ!!t*PVsNn9e(3aE zJ}%_ez!dI*TgPti6>^@6I-mZbGw1S?R}W9+;@)v*(+{1l%ZIOKMxA8~kbEwE;O4-N z+djVB(t3DKdcHN-Nl1WbMIR)E-r!=>A_~sx6ithXxT;`~#_dCdiJb zkVa9!at8NSA#7o`z|vp~EG>`@JtCOygli3Cz|9EQ;MxLqxOUjs=eit0C$5~$3^)Us z0oU7Z^&HXV&KLUvWq~Y`-tMo^TB&*H2!@00f(+(z(bf*}Fq zFwiVO9U8q8IgFsh$KJN9em9b}R_vlBUX#nP%sdP<-%5_I|I~~w@SjDToD)BKx<{W^ zPKuu4V6Sp?{*W@QMy4BawI{|`B2Hb1M!~np=D4Pgn=n^2Dd>NqLuVXieyJw=5^Odp7W0I>zbkzvf8l~Mx*BccEWtccG>oUwd@h2OSs5VqpOajiA~3c1j2J^ ztl~BT5+uUwmm1cGvi{#_ZOEat76wgW&JuDvj~G$<1j>l|;Zq)U^!^VF9P|xI^bD2+ zgU=<%s3(r6oJ+&@P)o)P8*P~Nzfz-#uY)1ZuO_p5zgmkSXh`!@Z8PrJW~N>iHP%io zh$oC`h6)p>YeI&P6B?F_D+4~Si18RNs!#CQ`AM3P+tE`-9E)oK*y*D`vPlUUe!lnv zW8nalV#6h8lRk)Y20_pdN`k?WG4smi_T_Ts;#e@y(*a+HpqM9H>BA0n<366Hqzg%( z>-T$)lzWnIaCz zS{qxxcPS)>h+Fwpu2oWx{P8SEX=g(Vpn>(q;J8I^ve@NqdIvS1(cv3Nf%SBypV_F} z6S474x@LWW`trkK39*jVykt`8n4FFTbHmIfbU*n#G0UIZ-0Wh?5*4vg`90{Vgix&lS zr(dL1@L!s;xKzl<<*I%L3FT)8Dxac!@5`=||jn2T{VIDT#1HYtA zk-7yeAI3L+MUmmgb{Ii#xY4k{IT~$oizqOVLgJjY6oi?S@^<zsRtDk(uGWD6M;hFGU!AjxeIT%mL%Dv{D^iB=F zxn*Yj&$=aPR!@g=jcUZzQNg&g7~@qa)LV{6eI`Xsz9I+GO6mBQts3fLOhObL6)*$aoA z&~f(FXDAOj|40+<^YDOu6)S2G3L6D;Ud&xMwE<#9mvC=Q;gnzSE}ON^j0weSaTWNb zT@TC|U~lD_EVabmsw2lsP7OIFHp5-4uHvMHYdtG}WV+cNf5|Em*!l{lbzW6YULRcH=?`g-bbG z<;P|45qIf0*%RxC0WY2G}l3O?zb7bfhYGI$BvigCm&Lf*Tnh(E{A&7<96HcFpI|e7Fs= zHz$y(Q$7;zY;?<-pdR4gBXRSu+n{d~>g(+TeHf>RF)`z(SRYBs?gqpRAo&Im8pr?| ztf!OWbs`ATwNg6dWyCl`{RCqsJ%Bs3@_N#e#Ei#UUs1>mwgl~M8-ueIm{=z{ET%T# zVD5orDWfOwFJb-}&=49DPBi1u*C-0>f&Mbf7|9fc#Z-g@^1|KR{A*87K0S3oDr*(X zT4z^&ebc*}Zs*SB9w6caHi!&oT+dm-wa`TEAwR>ne?>Y0&+zW;s3if-Q3wR99^Ypp za={KXhyQJ~>k)|IuaP+L30g^5qN2>9r!Ki8&OjLmwf*7w;`oSdRB8-f8w@SXXKQ%CJMZ!lVdlPQ#rYoylgftoP+#xv{ zA6bl4GmTeRL-LI7<~By-`hD@PCH#De3#5NJK@}Xx@}oxp_TP=hSGR)?bzWsYspxPS zT2qti~_+s>tl(1>bNO}-kmlTB{VjCWgecPt0@GxbK8|jZx z&SYp3wT79MAKS34dx-A!I@>)IfU+)A-tdvyLddt#*Xrx*#jRhyxKN>@enASp0T_=O z3I>`LzH*AfO(q$vvEPBR2UI$!%0Mto+PI4flWwr@6x*{@6=GZ&b~qRs9SDbfgWVTN zuXl8a7_wx^o!+lp3HER;U?2C#?@Lz{Ocqz$e2U3_LL=lw+6-yClbs_lya;eVF8)Kb z^?D1>FhG`RF&YfySI2f|sFH0oY*dP0PC=&Fq2sW`O_wdOo3)~iiIK`_P$!`)+(jyJ zhZQ4-p&+kP1!Pem8SC|YdaPIWB;07ND!(eKwX9pj5SO!=3iA-mU|}c~9bDO=2H6$w zPR3%_u;Z^%3u$Z~$@=Cuqw+;$)+L~bIF`H@2TAB#s1;bUms`F5>tpYZ!Qd`Q_&P&@#enGxo1_BR|J*d_ zXTGpTDr^!9n`RD)g)I|%pggN6cvOn^O2OVZYrEO~p6f2D*W%{(lY--Jj$d4jWI-dl}3NsDh3OZ8hs{CT%X#XH2}9k+VFGx)8+xno_@u~Xu)Q_}IxDzy8*7x?_H>%P(~W#jfC-w|8QHtfFPMNvzl?m3N5c9k=*s`5wG& zzPwU!lN3I2>M}h@O@+cQY;u;pGWu%ddSqtL%?sZg ze}8-4<832GMvR4GKeH_#y_^{pJ{U#End6!$(kgfPqL+xg)U z@D7!mLXS`mLOOYN<`XkM@y;=&@N%RZ8KbXFKrIdCnTAXM4((m9qmoaKOWU)ru7dW# zY);g^N`Q7EUGu$G#2?NJXU^5*Ldmk3a1^%3oSRwNDzSVOJ5j3WL;-JqY;sJfSV>mu z!OTxa%~we)wu>vaM{{@lZ$-^lJZAii@bWKWEdLGh^2asN2r2(EHSyPulow!X{+lA@ zWadB4lfu;1o=5`P4!)ifytfkMBxZWj!;mD-2yIsPJlvnH?+kQDw#|^fW-~LDv6KQV3vT*~j_t#lb}Q6fluG z--Dsi9`*3P-++`vUIZ-9r!sy-3gkiKoQS2Z4C7KLy8LNrJHfwB8^dawwN7vx<1>(r z;iX$Z7e_6fO16j1&JHYUV#!&-~{+3qk7PKoF zIJ#S0HUVuysG&Mx|#%Q)31@)@E_>Q7)v5Py3s};%?H$IqD-UFb+|28vKENe zf)A|K%I34AAeRfuo49aMH8m2FoiNF+2iX-xit)IssC*GOf&dCT{@?$7bmS8NJ}`R< zUPOG-gXds3>U=K;w>(_!YO|MMoZ)F+*J^|f|1N9*3QTH5( zf4TlAN<*9rqF!k)7Qn15Zt+0B43I}?*t#%$+@Pu%3`g9A1n@2EG2q*GsLI>q=9S~6>qJ_tQc;&T&GOT*Qv*#L--P38KeM3HDCRfLY>no(Ncn5U{I${i z^%IU*ZkYs=F}cu4kLET?xvRun$cL|7$7RO%KfOx#bcLJu7

5eOh)0L=Ve(OROMe3j-g>=x6 zLbx)i=Ho*6;-DdX%I-{KftAWcirC<=lRcM)iHM5~Zidl#TC4c^2l@d%wGzLb^?T$z zkVYJ?>V@6xHW~&yCZt## zO$%QfK|Y?irHTNRjeN9O$_0z6gk{>k3+&Ayp7uA{wmncAXhI+)!G8x!y(1m zmQoY#qjIBhUhUeH_^l;M3F2bPrHe|igysM=e}amf49}#`NK=|8uvsoylpSi45PD>G zxRfgMAI_sSSix39Jc$D-f@y4tG15Fkyoh9%^kEd^V@lm@X_96d?<1$2a@z>@!LAO2 zwLIg2i4>c*pIs}%S5m(HI!sN`7^By^O$>C_uddOi@m`&yg z1=$4GW;ELZ7{elI4>|Hk374K#5LOO-4oFj{+Wl+e*<(4R~ zfbFjZJ4HFTOwWRoe3|gwr>Y-pHk~q=>Zel3I_U$B{13LLmYUk84!`>J^`{?j@FLGv zlPQ0y?bS8c*F50J|6mu!Xp?{|0{iEeeyH;P%C7E+O69wLL4lO+ol z_|c`rPEdLDE6qO9PW_zn2zIY%DY torch.Tensor: + ''' + 在GQA中, k和v的heads数量小于q的heads数量, + 所以在进行attention计算前需要进行repeat_kv操作, + 保持第2维度的数量相等,n=config.num_attention_heads // config.num_key_value_heads + ''' + #GQA中num_key_value_heads的数量小于attention heads数量 + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + #在第3维度[index=2]进行扩充 + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + scaling: float, + dropout: float = 0.0, + **kwargs, +): + #repeat kv，确保形状和query一致 + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + #计算attention score + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + #casual attention + if attention_mask is not None: + #causal_mask应当为下三角为0，上三角为负无穷 + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + # sofmax计算 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype) + # 训练时给attn_weights的随机位置置零 + attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training) + #乘value过程 + attn_output = torch.matmul(attn_weights, value_states) + #最后转置，形状变为batch_size*seq_len*num_heads*head_dim + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +class DollyMoEAttention(nn.Module): + """ + Attention in transformer + """ + def __init__(self, config: DollyMoEConfig, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + #num_key_value_groups即repeat kv的n + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads + #attention socre的缩放系数 + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = True + #Q权重矩阵 + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias + ) + #K权重矩阵 + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + #V权重矩阵 + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + #Attention的最后一步，线性连接层 + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias + ) + #给输入q和k进行归一化操作，使用RMSNom，注意仅在最后一个维度进行归一化，参数量为head_dim + self.q_norm = DollyMoERMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = DollyMoERMSNorm(self.head_dim, eps=config.rms_norm_eps) + #支持滑动窗口 + self.sliding_window = config.sliding_window + if not ( + self.config.use_sliding_window + and getattr(self.config, "sliding_window", None) is not None + and self.layer_idx >= self.config.max_window_layers + ): + self.sliding_window = None + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: Tuple[torch.Tensor, torch.Tensor], + attention_mask: Optional[torch.Tensor], + past_key_value: Optional[Cache] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + #input_shape: batch_size* seq_len + input_shape = hidden_states.shape[:-1] + #hidden_shape: batch_size* seq_len*-1* head_dim + hidden_shape = (*input_shape, -1, self.head_dim) + #将输入经过Q矩阵变为query，然后调整形状为batch_size* seq_len*-1* head_dim，然后进行归一化，再调整形状batch_size* -1*seq_len* head_dim + query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2) + #将输入经过K矩阵变为key，然后调整形状为batch_size* seq_len*-1* head_dim，然后进行归一化，再调整形状batch_size* -1*seq_len* head_dim + key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2) + #将输入经过V矩阵变为value，然后调整形状为batch_size* seq_len*-1* head_dim，再调整形状batch_size* -1*seq_len* head_dim + value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) + #得到嵌入旋转位置编码信息后的query和key + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # KVCache的cache更新位置信息和kv值 + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + attention_interface: Callable = eager_attention_forward + if self.config._attn_implementation != "eager": + if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False): + logger.warning_once( + "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to " + 'eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + else: + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + #attention的具体计算，返回结果和中间的attention score + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + attention_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + sliding_window=self.sliding_window, + **kwargs, + ) + #将batch_size*seq_len*num_heads*head_dim 变为 batch_size*seq_len*hiddent_size + attn_output = attn_output.reshape(*input_shape, -1).contiguous() + #attention的线性层 + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + +class DollyMoESparseMoeBlock(nn.Module): + def __init__(self, config): + super().__init__() + self.num_experts = config.num_experts + self.top_k = config.num_experts_per_tok + self.norm_topk_prob = config.norm_topk_prob + self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False) + self.experts = nn.ModuleList( + [DollyMoEMLP(config.hidden_size, config.intermediate_size, config.hidden_act) for _ in range(self.num_experts)] + ) + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + batch_size, sequence_length, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + router_logits = self.gate(hidden_states) + routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float) + routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1) + if self.norm_topk_prob: + routing_weights /= routing_weights.sum(dim=-1, keepdim=True) + routing_weights = routing_weights.to(hidden_states.dtype) + final_hidden_states = torch.zeros( + (batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device + ) + expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0) + for expert_idx in range(self.num_experts): + expert_layer = self.experts[expert_idx] + idx, top_x = torch.where(expert_mask[expert_idx]) + current_state = hidden_states[None, top_x].reshape(-1, hidden_dim) + current_hidden_states = expert_layer(current_state) * routing_weights[top_x, idx, None] + final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype)) + final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim) + return final_hidden_states, router_logits + +class DollyMoEDecoderLayer(nn.Module): + def __init__(self, config: DollyMoEConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = DollyMoEAttention(config=config, layer_idx=layer_idx) + if (layer_idx not in config.mlp_only_layers) and ( + config.num_experts > 0 and (layer_idx + 1) % config.decoder_sparse_step == 0 + ): + self.mlp = DollyMoESparseMoeBlock(config) + else: + self.mlp = DollyMoEMLP(config.hidden_size, config.intermediate_size, config.hidden_act) + self.input_layernorm = DollyMoERMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = DollyMoERMSNorm(config.hidden_size, eps=config.rms_norm_eps) + #仅flash_attention_2支持滑动窗口 + if config.sliding_window and config._attn_implementation != "flash_attention_2": + config.sliding_window = None + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: Optional[bool] = False, + output_router_logits: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + #保留最开始的值，后面做残差网络 + residual = hidden_states + #step1:归一化 + hidden_states = self.input_layernorm(hidden_states) + + #step2: Self Attention + hidden_states, self_attn_weights = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + #step3:残差 + hidden_states = residual + hidden_states + residual = hidden_states + #step4:归一化 + hidden_states = self.post_attention_layernorm(hidden_states) + #step5:MLP + hidden_states = self.mlp(hidden_states) + if isinstance(hidden_states, tuple): + hidden_states, router_logits = hidden_states + else: + router_logits = None + #step6:残差 + hidden_states = residual + hidden_states + outputs = (hidden_states,) + #是否输出attention weights + if output_attentions: + outputs += (self_attn_weights,) + if output_router_logits: + outputs += (router_logits,) + return outputs + +class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs): ... + +class DollyMoEPreTrainedModel(PreTrainedModel): + config_class = DollyMoEConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["DollyMoEDecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_flex_attn = True + _supports_cache_class = True + _supports_quantized_cache = True + _supports_static_cache = True + _supports_attention_backend = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + #初始线性层化权重参数 + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + #初始化embedding层参数 + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + +class DollyMoEModel(DollyMoEPreTrainedModel): + def __init__(self, config: DollyMoEConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + #embedding层 + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + #transformer层 + self.layers = nn.ModuleList( + [DollyMoEDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + #RMSNorm层 + self.norm = DollyMoERMSNorm(config.hidden_size, eps=config.rms_norm_eps) + #RoPE层 + self.rotary_emb = DollyMoERotaryEmbedding(config=config) + self.gradient_checkpointing = False + #进行模型初始化 + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + @can_return_tuple + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **flash_attn_kwargs: Unpack[FlashAttentionKwargs], + ) -> MoeModelOutputWithPast: + #是否输出attention的计算结果 + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_router_logits = ( + output_router_logits if output_router_logits is not None else self.config.output_router_logits + ) + #是否输出隐藏层状态 + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + #是否使用cache + use_cache = use_cache if use_cache is not None else self.config.use_cache + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if self.gradient_checkpointing and self.training and use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`." + ) + use_cache = False + + if not isinstance(past_key_values, (type(None), Cache)): + raise ValueError("The `past_key_values` should be either a `Cache` object or `None`.") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache() + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = self._update_causal_mask( + attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions + ) + + hidden_states = inputs_embeds + + #共享RoPE位置编码 + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + # 否是输出hidden state + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_router_logits = () if output_router_logits else None + #transformer的计算： + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + #如果启用了梯度检查点，并且处于训练模式下，模型将调用 self._gradient_checkpointing_func， + # 该函数使用 partial 函数来计算并重新计算部分前向过程，从而节省内存。 + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + partial(decoder_layer.__call__, **flash_attn_kwargs), + hidden_states, + causal_mask, + position_ids, + past_key_values, + output_attentions, + output_router_logits, + use_cache, + cache_position, + position_embeddings, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + output_router_logits=output_router_logits, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **flash_attn_kwargs, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + if output_router_logits: + all_router_logits += (layer_outputs[-1],) + #归一化 + hidden_states = self.norm(hidden_states) + + # 添加最后一层hidden_states + if output_hidden_states: + all_hidden_states += (hidden_states,) + #将结果打包为BaseModelOutputWithPast数据结构 + return MoeModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + hidden_states=all_hidden_states, + attentions=all_self_attns, + router_logits=all_router_logits, + ) + + def _update_causal_mask( + self, + attention_mask: torch.Tensor, + input_tensor: torch.Tensor, + cache_position: torch.Tensor, + past_key_values: Cache, + output_attentions: bool = False, + ): + #处理batch生成时padding问题 + if self.config._attn_implementation == "flash_attention_2": + if attention_mask is not None and past_key_values is not None: + is_padding_right = attention_mask[:, -1].sum().item() != input_tensor.size()[0] + if is_padding_right: + raise ValueError( + "You are attempting to perform batched generation with padding_side='right'" + " call `tokenizer.padding_side = 'left'` before tokenizing the input. " + ) + if attention_mask is not None and 0.0 in attention_mask: + return attention_mask + return None + + # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in + # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail + # to infer the attention mask. + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + using_static_cache = isinstance(past_key_values, StaticCache) + using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache) + + # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward + if ( + self.config._attn_implementation == "sdpa" + and not (using_static_cache or using_sliding_window_cache) + and not output_attentions + ): + if AttentionMaskConverter._ignore_causal_mask_sdpa( + attention_mask, + inputs_embeds=input_tensor, + past_key_values_length=past_seen_tokens, + sliding_window=self.config.sliding_window, + is_training=self.training, + ): + return None + + dtype, device = input_tensor.dtype, input_tensor.device + min_dtype = torch.finfo(dtype).min + sequence_length = input_tensor.shape[1] + # SlidingWindowCache or StaticCache + if using_sliding_window_cache or using_static_cache: + target_length = past_key_values.get_max_cache_shape() + # DynamicCache or no cache + else: + target_length = ( + attention_mask.shape[-1] + if isinstance(attention_mask, torch.Tensor) + else past_seen_tokens + sequence_length + 1 + ) + + # In case the provided `attention` mask is 2D, we generate a causal mask here (4D). + causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position( + attention_mask, + sequence_length=sequence_length, + target_length=target_length, + dtype=dtype, + device=device, + cache_position=cache_position, + batch_size=input_tensor.shape[0], + config=self.config, + past_key_values=past_key_values, + ) + + if ( + self.config._attn_implementation == "sdpa" + and attention_mask is not None + and attention_mask.device.type in ["cuda", "xpu"] + and not output_attentions + ): + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + + return causal_mask + + @staticmethod + def _prepare_4d_causal_attention_mask_with_cache_position( + attention_mask: torch.Tensor, + sequence_length: int, + target_length: int, + dtype: torch.dtype, + device: torch.device, + cache_position: torch.Tensor, + batch_size: int, + config: DollyMoEConfig, + past_key_values: Cache, + ): + if attention_mask is not None and attention_mask.dim() == 4: + # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing. + causal_mask = attention_mask + else: + min_dtype = torch.finfo(dtype).min + causal_mask = torch.full( + (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device + ) + diagonal_attend_mask = torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) + if config.sliding_window is not None: + # if we have sliding window, we should not attend to tokens beyond sliding window length, so we mask them out also + # the check is needed to verify is current checkpoint was trained with sliding window or not + if not isinstance(past_key_values, SlidingWindowCache) or sequence_length > target_length: + sliding_attend_mask = torch.arange(target_length, device=device) <= ( + cache_position.reshape(-1, 1) - config.sliding_window + ) + diagonal_attend_mask.bitwise_or_(sliding_attend_mask) + causal_mask *= diagonal_attend_mask + causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1) + if attention_mask is not None: + causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit + if attention_mask.shape[-1] > target_length: + attention_mask = attention_mask[:, :target_length] + mask_length = attention_mask.shape[-1] + padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to( + causal_mask.device + ) + padding_mask = padding_mask == 0 + causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( + padding_mask, min_dtype + ) + return causal_mask + +def load_balancing_loss_func( + gate_logits: Union[torch.Tensor, Tuple[torch.Tensor], None], + num_experts: Optional[int] = None, + top_k=2, + attention_mask: Optional[torch.Tensor] = None, +) -> Union[torch.Tensor, int]: + if gate_logits is None or not isinstance(gate_logits, tuple): + return 0 + + if isinstance(gate_logits, tuple): + compute_device = gate_logits[0].device + concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) for layer_gate in gate_logits], dim=0) + + routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1) + + _, selected_experts = torch.topk(routing_weights, top_k, dim=-1) + + expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) + + if attention_mask is None: + tokens_per_expert = torch.mean(expert_mask.float(), dim=0) + router_prob_per_expert = torch.mean(routing_weights, dim=0) + else: + batch_size, sequence_length = attention_mask.shape + num_hidden_layers = concatenated_gate_logits.shape[0] // (batch_size * sequence_length) + + expert_attention_mask = ( + attention_mask[None, :, :, None, None] + .expand((num_hidden_layers, batch_size, sequence_length, top_k, num_experts)) + .reshape(-1, top_k, num_experts) + .to(compute_device) + ) + + + tokens_per_expert = torch.sum(expert_mask.float() * expert_attention_mask, dim=0) / torch.sum( + expert_attention_mask, dim=0 + ) + + router_per_expert_attention_mask = ( + attention_mask[None, :, :, None] + .expand((num_hidden_layers, batch_size, sequence_length, num_experts)) + .reshape(-1, num_experts) + .to(compute_device) + ) + + router_prob_per_expert = torch.sum(routing_weights * router_per_expert_attention_mask, dim=0) / torch.sum( + router_per_expert_attention_mask, dim=0 + ) + + overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(0)) + return overall_loss * num_experts + +class DollyMoEForCausalLM(DollyMoEPreTrainedModel, GenerationMixin): + _tied_weights_keys = ["lm_head.weight"] + _tp_plan = {"lm_head": "colwise_rep"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + + def __init__(self, config): + super().__init__(config) + self.model = DollyMoEModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + self.router_aux_loss_coef = config.router_aux_loss_coef + self.num_experts = config.num_experts + self.num_experts_per_tok = config.num_experts_per_tok + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @can_return_tuple + @deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep") + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + logits_to_keep: Union[int, torch.Tensor] = 0, + **kwargs: Unpack[KwargsForCausalLM], + ) -> MoeCausalLMOutputWithPast: + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs: MoeModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + output_router_logits=output_router_logits, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs) + aux_loss = None + if output_router_logits: + aux_loss = load_balancing_loss_func( + outputs.router_logits, + self.num_experts, + self.num_experts_per_tok, + attention_mask, + ) + if labels is not None: + loss += self.router_aux_loss_coef * aux_loss.to(loss.device) + return MoeCausalLMOutputWithPast( + loss=loss, + aux_loss=aux_loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + router_logits=outputs.router_logits, + ) + +class DollyMoEForSequenceClassification(DollyMoEPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = DollyMoEModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @can_return_tuple + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + ) -> SequenceClassifierOutputWithPast: + + transformer_outputs: BaseModelOutputWithPast = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + hidden_states = transformer_outputs.last_hidden_state + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + last_non_pad_token = -1 + elif input_ids is not None: + # To handle both left- and right- padding, we take the rightmost token that is not equal to pad_token_id + non_pad_mask = (input_ids != self.config.pad_token_id).to(logits.device, torch.int32) + token_indices = torch.arange(input_ids.shape[-1], device=logits.device, dtype=torch.int32) + last_non_pad_token = (token_indices * non_pad_mask).argmax(-1) + else: + last_non_pad_token = -1 + logger.warning_once( + f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be " + "unexpected if using padding tokens in conjunction with `inputs_embeds.`" + ) + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), last_non_pad_token] + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config) + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) + +class DollyMoEForTokenClassification(DollyMoEPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = DollyMoEModel(config) + if getattr(config, "classifier_dropout", None) is not None: + classifier_dropout = config.classifier_dropout + elif getattr(config, "hidden_dropout", None) is not None: + classifier_dropout = config.hidden_dropout + else: + classifier_dropout = 0.1 + self.dropout = nn.Dropout(classifier_dropout) + self.score = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @can_return_tuple + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + ) -> TokenClassifierOutput: + + outputs: BaseModelOutputWithPast = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + sequence_output = outputs.last_hidden_state + sequence_output = self.dropout(sequence_output) + logits = self.score(sequence_output) + + loss = None + if labels is not None: + loss = self.loss_function(logits, labels, self.config) + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + +class DollyMoEForQuestionAnswering(DollyMoEPreTrainedModel): + base_model_prefix = "transformer" + + def __init__(self, config): + super().__init__(config) + self.transformer = DollyMoEModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, 2) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.transformer.embed_tokens + + def set_input_embeddings(self, value): + self.transformer.embed_tokens = value + + @can_return_tuple + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Cache] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + start_positions: Optional[torch.LongTensor] = None, + end_positions: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + **kwargs, + ) -> QuestionAnsweringModelOutput: + + outputs: BaseModelOutputWithPast = self.transformer( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + ) + + sequence_output = outputs.last_hidden_state + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1).contiguous() + end_logits = end_logits.squeeze(-1).contiguous() + + loss = None + if start_positions is not None and end_positions is not None: + loss = self.loss_function(start_logits, end_logits, start_positions, end_positions, **kwargs) + + return QuestionAnsweringModelOutput( + loss=loss, + start_logits=start_logits, + end_logits=end_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + +__all__ = [ + "DollyMoEForCausalLM", + "DollyMoEForQuestionAnswering", + "DollyMoEModel", + "DollyMoEPreTrainedModel", + "DollyMoEForSequenceClassification", + "DollyMoEForTokenClassification", +] \ No newline at end of file diff --git a/src_pretrain/pretrain_deepspeed_MoE.py b/src_pretrain/pretrain_deepspeed_MoE.py new file mode 100644 index 0000000..bfacf05 --- /dev/null +++ b/src_pretrain/pretrain_deepspeed_MoE.py @@ -0,0 +1,120 @@ +import torch +from src_llm_moe.modeling_dollyMoE import DollyMoEForCausalLM +from transformers import ( + Trainer, + TrainingArguments, + DataCollatorForLanguageModeling, + AutoTokenizer, + set_seed +) +from datasets import load_dataset +from accelerate import Accelerator +import os +import logging +import glob + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +accelerator = Accelerator() +set_seed(42) + +model_name = "/mnt/han.luzhi/dolly/weights_MoE" +resume_option = None +# resume_option = True +# resume_option = "/mnt/han.luzhi/dolly_llm/checkpoints_ds/checkpoint-500" +tokenizer = AutoTokenizer.from_pretrained(model_name) +if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + +if isinstance(resume_option, str) and os.path.exists(resume_option): + logger.info(f"从指定检查点 {resume_option} 加载模型...") + model = DollyForCausalLM.from_pretrained(resume_option) +else: + logger.info("未找到检查点，从预训练模型加载...") + logger.info(f"从预训练模型 {model_name} 加载...") + model = DollyMoEForCausalLM.from_pretrained(model_name) +model.gradient_checkpointing_enable() + +data_files = glob.glob("/mnt/han.luzhi/dolly/corpus/*.jsonl") +logger.info(f"找到训练语料: {data_files}") + +dataset = load_dataset('json', data_files=data_files, split='train') +# dataset = load_dataset('text', data_files={'train': '/mnt/han.luzhi/dolly_llm/corpus/wikipedia.txt'}) + +def preprocess_function(examples): + return tokenizer( + examples['text'], + truncation=True, + max_length=512, + padding='max_length', + return_tensors='pt' + ) + +tokenized_datasets = dataset.map( + preprocess_function, + batched=True, + # remove_columns=['text'], + remove_columns=dataset.column_names, + num_proc=16 +) + +data_collator = DataCollatorForLanguageModeling( + tokenizer=tokenizer, + mlm=False +) + +training_args = TrainingArguments( + output_dir="/mnt/han.luzhi/dolly/checkpoints_MoE", + deepspeed="/mnt/han.luzhi/dolly/deepspeed_config/ds_stage2.json", + do_eval=False, + save_strategy="steps", + save_steps=500, + learning_rate=1e-4, + per_device_train_batch_size=16, + num_train_epochs=2, + logging_dir="/mnt/han.luzhi/dolly/logs", + logging_steps=20, + save_total_limit=2, + fp16=True, + dataloader_num_workers=3, + gradient_accumulation_steps=2, + report_to="tensorboard", + optim="adamw_torch", + lr_scheduler_type="cosine", + warmup_steps=500, + weight_decay=0.01, + local_rank=int(os.environ.get("LOCAL_RANK", 0)), + disable_tqdm=False, + logging_first_step=True, + log_level="info", + ddp_find_unused_parameters=False, + overwrite_output_dir=True, +) + +trainer = Trainer( + model=model, + args=training_args, + train_dataset=tokenized_datasets, + processing_class=tokenizer, + data_collator=data_collator, +) + +logger.info("***** 开始训练 *****") +logger.info(f" 进程数 = {accelerator.num_processes}") +logger.info(f" 设备 = {accelerator.device}") +logger.info(f" 总批大小 = {training_args.per_device_train_batch_size * accelerator.num_processes * training_args.gradient_accumulation_steps}") + +if resume_option !=None: + trainer.train(resume_from_checkpoint=resume_option) +else: + trainer.train() + + +if accelerator.is_main_process: + trainer.save_model("/mnt/han.luzhi/dolly/chechpoints_MoE/final") + tokenizer.save_pretrained("/mnt/han.luzhi/dolly/chechpoints_MoE/final") + logger.info("***** 训练完成，模型已保存 *****") + +## deepspeed --num_gpus=4 --master_port=12345 pretrain_transformers.py +# deepspeed --include="localhost:1,2,3" --master_port=12345 pretrain_transformers.py \ No newline at end of file diff --git a/test/compute_dollyMoE.py b/test/compute_dollyMoE.py new file mode 100644 index 0000000..052de0b --- /dev/null +++ b/test/compute_dollyMoE.py @@ -0,0 +1,18 @@ +from transformers.modeling_utils import PreTrainedModel +import torch +from torch import nn +from src_llm_moe.configuration_dollyMoE import DollyMoEConfig +from src_llm_moe.modeling_dollyMoE import DollyMoEModel,DollyMoEForCausalLM +model = DollyMoEForCausalLM(DollyMoEConfig()) +model.save_pretrained("./weights_MoE") +model.config.save_pretrained("./weights_MoE") + +config = DollyMoEConfig.from_pretrained("./weights_MoE") +model = DollyMoEForCausalLM.from_pretrained("./weights_MoE", config=config) + +input_ids = torch.tensor([[101, 1024, 102]]).long() +outputs = model(input_ids) +print(outputs) + +model_size = sum(t.numel() for t in model.parameters()) +print(f"Model Size: {model_size/1000**2:.1f}M parameters") \ No newline at end of file