From 4fe0099f3f24b99a6ffa2db535efc69368b87735 Mon Sep 17 00:00:00 2001 From: seeker Date: Sat, 11 Jul 2026 20:04:18 +0800 Subject: [PATCH] Initial pi0 minimal validation --- .gitignore | 9 + README.md | 76 ++++ configs/overfit.yaml | 31 ++ outputs/overfit/evaluation.json | 18 + outputs/overfit/summary.json | 13 + outputs/overfit/training_history.json | 126 ++++++ outputs/overfit/trajectory_comparison.png | Bin 0 -> 80457 bytes pyproject.toml | 35 ++ src/pi0mini/__init__.py | 5 + src/pi0mini/data.py | 310 +++++++++++++++ src/pi0mini/flow.py | 72 ++++ src/pi0mini/infer.py | 78 ++++ src/pi0mini/model.py | 458 ++++++++++++++++++++++ src/pi0mini/runtime.py | 35 ++ src/pi0mini/train.py | 98 +++++ tests/test_data.py | 117 ++++++ tests/test_flow.py | 65 +++ tests/test_model.py | 65 +++ 18 files changed, 1611 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 configs/overfit.yaml create mode 100644 outputs/overfit/evaluation.json create mode 100644 outputs/overfit/summary.json create mode 100644 outputs/overfit/training_history.json create mode 100644 outputs/overfit/trajectory_comparison.png create mode 100644 pyproject.toml create mode 100644 src/pi0mini/__init__.py create mode 100644 src/pi0mini/data.py create mode 100644 src/pi0mini/flow.py create mode 100644 src/pi0mini/infer.py create mode 100644 src/pi0mini/model.py create mode 100644 src/pi0mini/runtime.py create mode 100644 src/pi0mini/train.py create mode 100644 tests/test_data.py create mode 100644 tests/test_flow.py create mode 100644 tests/test_model.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff92181 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +*.egg-info/ +.venv/ +data/ +outputs/**/*.pt +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..beba5ec --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +# π0 Minimal Validation + +这是一个 **Physical Intelligence π0 风格的最小、可运行过拟合实验**。它用小型视觉语言前缀和 action expert 学习连续动作 chunk，通过 flow matching 从噪声一次生成未来 `16×2` 轨迹。 + +项目不是官方代码，也不复制官方规模、数据和机器人能力；目标只是验证核心数学与信息流能否端到端训练。 + +## 最小架构 + +```text +图像 + 语言指令 → 小型 VLM prefix + │ blockwise attention +连续本体/历史 state ─────┤ +noisy action + timestep ─┘ + ▼ + action expert + │ + ▼ + flow velocity → action chunk +``` + +默认从相邻 Alpamayo 项目读取 1 个 PAI 驾驶样本，生成 4 个确定性轻微增强变体，输入 `64×64` 图像，预测未来 16 个 XY 动作点。 + +## π0 与 π0.5 的关键差异 + +| 方面 | 本项目对应的 π0 | π0.5 最小复现中的额外能力 | +|---|---|---| +| 控制层级 | 语言/视觉条件直接生成连续 action chunk | 先生成高层 subtask，再以 subtask 为条件生成动作 | +| 动作表示 | flow action expert 输出连续动作 | 可同时有 FAST 风格离散动作 token 和连续 flow 动作 | +| VLM 输出 | 主要作为条件语义前缀 | 自回归预测 subtask/离散动作 token | +| 训练目标 | 本实验仅使用 flow velocity MSE | 还可联合文本/动作 token CE | + +与 π0 一致，本项目把连续 state 放在 action suffix，而不是像 π0.5 那样先离散成语言 token。 + +因此，本项目故意不实现 π0.5 的高层 subtask 生成和 VLM 离散轨迹分支。效果图将对比 flow 生成轨迹与真值，而不是三方动作对比。 + +## 安装与测试 + +```bash +cd pi0-minimal-validation +python -m venv .venv +source .venv/bin/activate +python -m pip install -U pip +python -m pip install -e '.[dev]' +pytest -q +``` + +`tests/test_flow.py` 验证直线概率路径 `x_t=(1-t)·noise+t·action`、目标速度 `action-noise`、MSE 损失、随机可复现性和 Euler 积分。 + +## 过拟合训练与效果图 + +```bash +pi0mini-train --config configs/overfit.yaml +pi0mini-infer --checkpoint outputs/overfit/checkpoint.pt +``` + +也可用 `python -m pi0mini.train --config configs/overfit.yaml` 运行。默认训练 600 步，推理使用 10 步 Euler flow integration。训练后应生成 checkpoint、loss 历史、逐样本评估 JSON 和 `outputs/overfit/trajectory_comparison.png`。 + +默认数据源为： + +```text +../../alpamayo/alpamayo-qwen05b-validation/data/processed/overfit_manifest.jsonl +``` + +如本地路径不同，修改 `configs/overfit.yaml` 中的 `data.source_manifest`。 + +## 限制 + +- 小型自定义编码器代替预训练 PaliGemma，参数规模与官方模型不可比。 +- 仅对几个增强样本做记忆性训练，loss 下降不代表泛化、鲁棒性或安全性。 +- 二维驾驶轨迹替代多机器人、多相机、高维关节动作和异构数据联合训练。 +- 固定步长 Euler sampler 只用于演示，没有实时控制、碰撞检查、约束或故障降级。 +- 严禁将输出用于真实车辆、机械臂或其他会产生物理影响的设备。 + +## 成功标准 + +成功只表示：该缩小实现的视觉/语言条件、blockwise action expert 和 flow matching 能够端到端反向传播，并记住极小训练集。它不证明与官方 π0 数值等价。 diff --git a/configs/overfit.yaml b/configs/overfit.yaml new file mode 100644 index 0000000..013dae1 --- /dev/null +++ b/configs/overfit.yaml @@ -0,0 +1,31 @@ +seed: 7 +device: auto +output_dir: outputs/overfit + +data: + source_manifest: ../../alpamayo/alpamayo-qwen05b-validation/data/processed/overfit_manifest.jsonl + num_variants: 4 + image_size: 64 + action_horizon: 16 + +model: + d_model: 96 + n_heads: 4 + n_layers: 3 + mlp_ratio: 3 + patch_size: 16 + max_text_tokens: 64 + dropout: 0.0 + +train: + steps: 600 + batch_size: 4 + learning_rate: 0.001 + weight_decay: 0.0 + grad_clip_norm: 1.0 + flow_loss_weight: 1.0 + log_every: 20 + +inference: + flow_steps: 10 + noise_seed: 4 diff --git a/outputs/overfit/evaluation.json b/outputs/overfit/evaluation.json new file mode 100644 index 0000000..dbb532a --- /dev/null +++ b/outputs/overfit/evaluation.json @@ -0,0 +1,18 @@ +[ + { + "sample_id": "314f59c2-e420-4fcd-9df2-a51d572e0c20_1503899:aug0", + "trajectory_mse": 0.1524142026901245 + }, + { + "sample_id": "314f59c2-e420-4fcd-9df2-a51d572e0c20_1503899:aug1", + "trajectory_mse": 0.2658033072948456 + }, + { + "sample_id": "314f59c2-e420-4fcd-9df2-a51d572e0c20_1503899:aug2", + "trajectory_mse": 0.011374758556485176 + }, + { + "sample_id": "314f59c2-e420-4fcd-9df2-a51d572e0c20_1503899:aug3", + "trajectory_mse": 0.2792850136756897 + } +] \ No newline at end of file diff --git a/outputs/overfit/summary.json b/outputs/overfit/summary.json new file mode 100644 index 0000000..6f0cd69 --- /dev/null +++ b/outputs/overfit/summary.json @@ -0,0 +1,13 @@ +{ + "device": "mps", + "samples": 4, + "parameters": 662786, + "initial": { + "step": 1, + "flow_loss": 2.2617573738098145 + }, + "final": { + "step": 600, + "flow_loss": 0.16418831050395966 + } +} \ No newline at end of file diff --git a/outputs/overfit/training_history.json b/outputs/overfit/training_history.json new file mode 100644 index 0000000..89bc35c --- /dev/null +++ b/outputs/overfit/training_history.json @@ -0,0 +1,126 @@ +[ + { + "step": 1, + "flow_loss": 2.2617573738098145 + }, + { + "step": 20, + "flow_loss": 1.941188097000122 + }, + { + "step": 40, + "flow_loss": 1.479238510131836 + }, + { + "step": 60, + "flow_loss": 0.7427206635475159 + }, + { + "step": 80, + "flow_loss": 0.6053631901741028 + }, + { + "step": 100, + "flow_loss": 0.270697683095932 + }, + { + "step": 120, + "flow_loss": 0.11459995806217194 + }, + { + "step": 140, + "flow_loss": 0.4224633276462555 + }, + { + "step": 160, + "flow_loss": 0.14150527119636536 + }, + { + "step": 180, + "flow_loss": 0.15842989087104797 + }, + { + "step": 200, + "flow_loss": 0.34323638677597046 + }, + { + "step": 220, + "flow_loss": 0.1762494295835495 + }, + { + "step": 240, + "flow_loss": 0.3678283989429474 + }, + { + "step": 260, + "flow_loss": 0.10764116048812866 + }, + { + "step": 280, + "flow_loss": 0.17170406877994537 + }, + { + "step": 300, + "flow_loss": 0.20748832821846008 + }, + { + "step": 320, + "flow_loss": 0.12005910277366638 + }, + { + "step": 340, + "flow_loss": 0.1855630874633789 + }, + { + "step": 360, + "flow_loss": 0.16064667701721191 + }, + { + "step": 380, + "flow_loss": 0.11575131863355637 + }, + { + "step": 400, + "flow_loss": 0.1507677137851715 + }, + { + "step": 420, + "flow_loss": 0.23872917890548706 + }, + { + "step": 440, + "flow_loss": 0.25908488035202026 + }, + { + "step": 460, + "flow_loss": 0.15245740115642548 + }, + { + "step": 480, + "flow_loss": 0.3529265224933624 + }, + { + "step": 500, + "flow_loss": 0.22890940308570862 + }, + { + "step": 520, + "flow_loss": 0.18230678141117096 + }, + { + "step": 540, + "flow_loss": 0.14777007699012756 + }, + { + "step": 560, + "flow_loss": 0.06152040883898735 + }, + { + "step": 580, + "flow_loss": 0.18225648999214172 + }, + { + "step": 600, + "flow_loss": 0.16418831050395966 + } +] \ No newline at end of file diff --git a/outputs/overfit/trajectory_comparison.png b/outputs/overfit/trajectory_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..2e64cc26b26f4859bfe9c4b9b527ea77fd5e3bc1 GIT binary patch literal 80457 zcmeFZS6EZ+wgx&?6cq~!f`wv7q)AZ>T@(>%DhkqVG?8Mcp#&`0P^$D0Y!sz;LI(>y z5fCE16A(fR5ISc}{MX)V?R{}>&b~W7&ss`;lFa#)G2Zsgmshk@w{P9Q6+w{gYL_lt zLlE?z>%BK`g1@e?9kPJGWE@p)I_lV&JG$J)njxCE9qk|3IXtJPPD=8{2 zE_zD%prxauy@QOHn9YCvhNvCZLM)^r)($>pi~S{i2L#z0h5m1yn$wfF$T|d3yKwHh zD`B*Q)m6uAv0}0ysv(2fxj7->#kM`{?7LVtazB3c%A1k;p?Hj&^{grP$sN~!>2Y4S zKXQ2e`V7-&-WpeDJF^_6S4Z>hyYmtfJ0fs+bPO;#Sxo0x@p3!eNYtKg_e^`FXg2 zfBWN|d|{It>%YAK|5WO^ifsOmm%O+C|F8V_fc#&Mh6*dp#z8?rwc{G$2N^U<*zzJ% z-P}B`;t96Plc^S?;Cc2IT*;Yp33E4oxXql5oK|%&tJ3tQ>DJ!dV_!Mil4;hQq@VGy?;-}jly^Pp=G8BH zsiHG~nu4{WWz^boY>u5g`K5H3@jc6`@#yj67nSLmUv}`xolp-G4f*cNT{AUMa}0xN zX?`>R^W7JAwSLSK%9uJsf9TFB~9{ue5A*3R^E`^u={$a;8 zBQL@0v8w)jyLaun`gD8;!fsoMJLXHUh@pHbVT71}i@`6b;TP%BjPdB9Axo>@Y443( zdwd!a^~7Gjc_R`+MUbarSd*&5v=(bQ3jeqcLHwLoG3@RB!XOngTg+tPq4Uowb8Ol? z@Fj7YQPl&GW=(9JN&c`!U%*k)T$*XO*t*tN=ILBz#X#JnGg^_ZU;q#6HQna8TBsXct! zYBBTk{dqqa4%cO3yk?Ycw{?nk)#FVjMWLXS_J`vB{neiD3;1=&dw+u7 zR3&b9E@?s5)N{7Q!tYO0qPu4@TsX#g^!L@?)OE;aQy2qPawf$hJ8!(tAI7B+#^w8R z)SwW#4cmFf=m$efmY$^?y;?l&#&I;&#$Txh7_x)QC`~3F0pPQ8D#y$rGoEP z6-BRKZ~JUoQamh5g*B#=z4GILU1gWos>qW~+k>7yeR^1|CNX-rcy-0yVG#x>P?g_P zigM=j=g+qp91SUaf`Wom17W7saF<24x%aAe>*v}=w{*5)tM0=sVe3`6BXm11%lZtd z6s(w8Skx@cjMQm_`mk|Vr{Arpt$ndnxHMIRZPvHgGw1o%Svsg;E-5ZHHiDXHM9Ltz zbmGjPRRr#YCf?6Va+KM)o*79sQG0@Lizug?+> zdoSo0xn<9jR#z6su|M`%a9XV*$on@0y@g6#*`of8MX}o$iPjIBNIKUkXR^EON!{$j zhY!s!PS!?9YOXZbng;zd$PKDwwob|y#m)-;yN_4+^jMIm0$7vbBQ&2`*G?FS{?W?O zy661(6R?sba#?rq45fd~Ovz$W6j#G-pX9$8^T*qHElob3$ImQ|wPeQEMoDWModcO0CsaO-qvPYFn_(TBzd-R?IntD<=i+W;AyhouYCSlc zVqDN4tgRr}u$bqZ`~oVDy$hpGI?6iw<-5#vg>`$g&G+AIbIrFbE=n_U1Bd^zU1RCfib8{+8-;amZiQ8 zK~gGlkIwyF2vZ^E)3W1*jqSE=Imr>cNxX}4pKqprcq^zAcj2G={MWCpt(h{3l+8#Q zl(Aa@3}tt@)PrnX!U1m%NXwXN7i`a$dD#1OiCMCGIybxVU3}AWmQatnSW&-G&v=+A zUT_+Z_>}D~Roaa4n6nS`%`09UlWp|V)r^q%j=KXp7SESkGTopaYhD+bVL{=Gki%uP zKdj(YD4|kKy+rIvmUG&sinSHyKNZe28F8TqBzHoLgw)J<%j^!=yr zS8n7sy0uaoCT7~%=kRR%0rQ;kw%k;!bA&ec@gf0J&zTTXa*f^av0QEw*K_#g89fX= ztMT6G@APZe7nXML%2a+(@zvdLhf2#fNImB0v$gLxn^Dlo98Fgax6%iRjU-5eZ!58ao~JN5W*M=I7DtV{LE@{m`H zy)jE-Iy8N`VQUJVqC;W4fXtpZmOnyRq~Bkfo1B{Xy0ZqRzAtH-qRBK{U0LR1L$cW^ ze?D>V*)x!i*th3BjC+xGBT={4zvZX_k|4Q>h+nXJ`QpXI-ZPuF@4sj6@7A~A(3bC% z`1A`5NoyeaZ2HqF<`$&q83Af}0OPVEZO_}(q%G%VeCLDIMFkt&=l;h)ygslVS^wtm zZ$Gw^OMQ1^TXJt1eE6kY+m5LRnb3P_bhnv5I8PD!wFHi9FgZsZXk?LJu#!~F5O(G1 zdx4?@e_DOU=4mm~Op?j8g0F2NQ(pv|-D+a-=_c>+)Lr?Mj#(oMQO^D^i767}nA_zn zG*5%k!F>N#5`{ZIb^mThB~{;nO*?o_a0qC<<;TyJIg%3$vcK@0wO2|hnl+%Qoz`pS ze_*pGNyR%QC@4r{rdTCxP*YpmrR}2cNRsXk;>sL%v-i3UwOEsde22l>hwdnVAUBsv zcR1=4r+zeEO*%CXmAeL-`q5u+e^_LH@_?Nk_dC;Nyfe<4R%Nr3ERxBTeqD&TACg@!$ZrCFC#kj!Pyx}Lbn}M#?mE3t3e=^b>>*;hwQc|*U z_K_VTPO~}hTOS7Tg*eRdRM;*W6%l0Tp znN{iFQ%lu6U$Nrf*8>g73X2aMVuYx{7u_CzwrR^DrzV@lUyqh?n(|bnR%El8LY47s z)lPm)tqBz|BrvmWJ7g&Mh2An&en+lut8?$2e>m8Dmc3WWuA`B#(J8;(4n{?@JLjf~ zJ8%GV3ejfkMiB#_)QZdJU#y5XlYQLoq`=&gZIZN}$v7w(Zs87F8BNLS-8$dvE2hCL z^A*jnoDau)>FetsKMH-DG+*o|it`vr&P9=ukg6Y#c!i$q!TV+Ui?j6DhkZYMI&)e1 zT$Nz|z8yc)nsd{)#mLIgO#Ze$PrlUDQ2ob_F`+k<`)c*%x1=Iy+2$n*3qwbAei?8Y zY)-7XDzuzr&1QorSstVXGKBjJ8|rWy2ZVUHwHLUgk~)&~C2><#0s}1;F{eo#Ag`tr9o&h{Q_jl`;ohItBTdOFDEHRqj#zT`Nx7CD+ zg&nZXE9-N1Bews!nDFM7^jq=r}<(IDrsHqdU@UE$p+=~B3~LG z%gftBqY|5#kt*c;WZRLC8D5RcNrK7Qm#->=n!9~i5qPqV)zRToLwTFudOy9ncn9b<*0(Tti;&h%FE zO^-yy2nFY(xs3ETdZ#3%I+=Ux4jOLs)|Q`n>u`X+-?Hdg$@EbDS5k>ioJL^T6V|BM z7to)y8IqG{o*c}bn{#daaFeYji{o96Opf_pOTE$&wBEKkJ+1`n;e;}nD_+%)u6?FkzD#86d)-kj-(WjM`KoMXZlm( zZRG2of}LDQv}-KrR!Ozm!j@r*<%7kOoKWBl|*)& zT6MX|ZAKzBJ~(Ksh-nfb)I)dKKOwypS(lfdUpJp& zQZ%xErkLyGd?S#V=v?D4@!4tCoBSxS)!b!lYfvZ+YpW&8Dj3rf7v19MT*~jYbeovi zwA`HjljEVmg?LGh=2TO=t@%mRiLiu(6FU9O26eHB6fc8q=Gpt3{XAQ<(@sre-0GFm zu>+SK^b1|nhYx4CGi@2ma#NK@umD6E#7;}*F6yzDp0UPbuQCc9PI&4j--?M@SzfGZ z)K(PnS6E2UU$0f5G^qFce*He-{YuH_^ox?e6AyeaI!v#RpHh8-kW)E1>{!M(lYQPI z!wqS~&<|7JW+s)E(la_ueWy?Ar4Yt*pj|t;`w8)~_`ZJa7BUy-)QHEb^y63{CBBNo;*Cr}KL9J?S|^-gB+RWimn#UO^-0DE8w$Ik4r|38 zx@M?8BEAkOaJ5akEEjx>Keq75s{tgF})@3|dOay>#5 zCsw!;J-q}#dI>49dQIE2vk#|2OJ&6BGD{wZuGVShHuItGT>5-zx7qz$@Z*9izdAVQbwx@ch?UL)4#L^3q~)MaV?yFRQ?XyDqonfbHtV)>*gf! zOtt+|mH`$A=vw9*H9Hj8vxEp{3)90UY{<;c#O%}*%EQ=4N@1l<>(3gyId$Yana8QX z?M1f@@tfJyC3=0zo*OTHw1G?ueG0W$>*DLZhpeou2CeBDH?OH!ga(B>y;BU^Cuz=E zAW4i?Oe{9YvJ7OsksuU@Dr1Zf*|Q)&!Mgc{bl$bq8|DVz?pJpx#Ur@LLUIPON09*TbX! zg9?t%6!fkmhYy#_;c3w>ZXW(#%Oj@q#m~ghl|i#cF)Lp#ssY)8f;65d9~vD^w`|S& zj7!2VFZjWH<2>FhjulYm_e)xkr}OG2`zm;$*PBOyG8ZwzpeQO*oMA^gx5dG>Gw1ic zTq0oIrJHIJO`8N|P8LXPPxxu;&);_j6l3Kz8Es}(}xN+ZE`*#edIIZYE zPOE?&>6_2Q&gjqdL&2_NGLzC88XI||WnG+$2g`lfCMO2DBa!@Le(H_W{ zQYAAqA2OPXJr{X^5)aM;I1jF{nLa)8>+{{5`jOgNjS>{P0k0pc#LXX}ZM1Fw`7e!x zMH)+##hR>C;)*Ak^9@9K_ekoiSFf<6t=Tm_I{|JvycNCbuFTaFMR?FP`vE%9NBea2)hiNgM{vQ6#lwC4WBOfCI*u_IVXfU}z|?QE zTpwtZja})0nXo6lfkZw5ebPGp6%)V4^ zwoolJdZWX}2i768D&!hpU~x5KwD}{ZRxVK$eD1aFDP8HnlAEU=DPMDF$rXarNBn={;w0N4>^CWOFP&pgXH5kCHXq5foSk?SRy`}BY9t%o7K;E zkM4ebwtargH0q2EPe*}^*hGfMJXY1}`J)RP1$mSi!P9^WU$QVTcwuP!9;H(zvO6Fl3r`vbn0hTY%@+bLR=t1!KG6o0=s&{!4K+*i4BoPv zNJ0uF_0fU z_&qC23dYOHFqvc@-QLq0P6Kl8l0G8nUH;RXSPb4aY3VU?&!;qP_@8o}Wc$liRp zZJ8GSOW=15PI8GDgdcN~ud1z;O19p~bM|0}kX|TUklKR>57L>Er%&HN;f}*%QYKlV z#$+I_M!YuibWJ<#*I3Z?1-hueJeO!OUbWWLgsWdi6LZIfq>b;?7(BjVsO}E`Ul*fj zdhOb^S~{@jOCLUdj7@Odk_syf6UL!+>C%Z`jAZ9;@Z)L4uS*IXz?&txjrWs4@o&g` zc-Mgr!+hCLWQ2B<{neDdrj{0}a!0l;%r{UX1cdZbq*6!NYzErD+z|5jAGUA;AcnS+9)e@6>l+o0jY^WLP_xknQi*NQv zctvrBqRQ$m{{Q+d*FN0J-o9CoTuXlJfhvOvXt)K90LEizeKRvzy63F9UiCs}1CGq4 zrmiljzFGRg@3c#)8#q}3?EJ@{-&gLvbNP=~5QLNOfBhEw|3|M}aeU`e>L(!~aSz<8 zsS18C)Yqx?1fevvMzlZ!H8VV7q9P&$b>g)`06!c1K{=Zlp@u;#RZG-OzHBwVZtZHl zCo&(jWXSd57lD6u&w%GsA?I7@Bx~(1@xTVZKzxA^>M?kL&>HkosN?Qy|08cs0z_ruyYf) zK?#i`6#{p>mh(U#w>(4bAcjGI0Im@NyXZ2A|C*LbHxffplLA`a3FudazmoKI;CA@D zRy_V0;Itel(Jds0c(iHYelW87eEn)vR8*=Gu6}GM#xvp;nd5dcKB}yx$_;5Z7>zK^t^Za0ZFFDqQ@o#<_0A z)uq1Bop4EyZ@``_wT(P&J(xbH-2qbmpvUgU1yu6p||8z^eV;%V{cr$76dEwD?J^=|8JN$(e00*&QdN2#1T+w zUeaaZZH3n0Efaec_w3szctq7tD%YP^_IO*aos0WYpTFW6YU?m1drrFu5O-V+W2~b9 zMfyB=1i8f~ylzw#48TDdK)N z=$%FGQ3E}~t;c@se_PkFOu5qmH?VML9JEZ6IWLZXK53JrLJT0}c`7KymMH};^Rq_B7fb@T|8iBw4KorK$MT^FO{XQ5VO zr?GjyEyK)r%}1J)Gk)1OUa}ertcH?Q+k+CuK^aj|t&`Uix+iBGmo#~0920&k%Kl)C z`gnCY$UWH?qwQyLs*e$&_=E8Zd@~yf&~7ZqQL-*6pfg<5Qe1YVEpy4oSUWk+Iu~a` zH@A&QJJ?_Nvuri~EV1l0MPO|xwt0|0B55yX$2zP-=j?%*xZ~z}o->}JkisFE&Qnxh zc7ba4^q;*w=dbm_bs$7<>NTwhSk|>fU6I_yWN{gJaigEE#Ox_FL9_lm@NqIZ1vCE9 zUm*zeJ@;CiZYbkIRQ?GpA1orat%z>|aE9YjGCVsZ3szD2#Dm%4(g4GX8+!koU=w?z zCBuwOcuW?IJd(4q%vr^GC(m^Ah`0BcTrywnHJ|8;4F z5wn{2hPDjqF&4^eHLyLqAJf8|7{%*#ff^32K`K6 z3i$(tzyhO=;Z5D?r%7;!3{K{J5F<(BCOg(-sApB@pZVQaJLj=Uly()l`FbZEHEf+@ zL^rf&TkGM5W5fDi38e(UcXyGJt1Bx{*~BM8`KQf*csr0I0@sTES^caN?**Z#)lD!wxM&$MkFke+E(PAOG+`wKB;&dN+ZIJD9rrQ zv1NJ!8ArEqWUvwx_6|$({`H-1zqz%Z>f{M!n<4>yf9Mm)kx(*+W|W@l;5j zOZRz){r#QEVJ}|1P#T9Q+Qbno(WNJ79^B*D?_Rpxd-d`gDp08(Wn{U=(Ptl)1CP*z zxY|K!tck0+tX_X%=_@}nW)CqasJBqNLRZ#hOc+Hx!0uYx$wf=!AXL}V+xM{}2$UFJ zjM&5pM7aM}_wV$(0}CUml+4lR&&r_9>Yxx^oE@@7?VJc6j2A5Igv$^`BCJ-{SAZ$^ z3H%N;nz3g!ek-zBa3QxBWEt$z+(Qf3^3RWyjpGgxz|#I9taH2W8jLXzRt9~qkNO}m z&UV*MR5+3;9TF!mhx;`xPt}MGjpP~&Jq1&>Bz;`0bif* zRENG$`^^+Vi~<>QZ1E3huLyb)Qo+mMBBIv@NyQ?i0=L1r%8##IC+bS zi(j3kw+X=POGYpK3OIDWAA|xI_Y^Wqb9os>VNpZ0WC<&I>n1=e{XF~k5886-SQ-Im z7=HNhVQG8Ki-%ZfMq15eW(j`1v7F7nSg0@L<$%F zBx;hQr0s}RFaCt+nN&$WVp)q?bOtgaS(XZIT#wTaH}n>QUe@I})p$;!D6iqxJfO?) z!{@XWmL|?i{!YkLlTtE}y7I@fmTMEz9~ZrvHK(mDd?VGw7*J1b#x8cCTH6YgKCh+zjuJa*^sTb}*Z!Uqyv zMv`fXNzQ|o!_Rl{N~lkqw(B$k4k6{gu3#tLqRs z(cAB8R!)kn)D$hrVOiYHs91TW0z;Qn@s2z)Eg57w3+1NlB|*>iJrSLBRYry=BPulX zIP~HwNg_y9vsbN1IU z@0n|yZoMX9+twlP-W}UE{o!mni*NSmQ&F=xZn`C~hEX2^t2SJny6+N9)jd&_dT?JI zKr&M7mvaS!T;HIdny8n0SaFr$Ko;jlkg4W_A|j;0ueOn>S~pxg#aY$0Uj~QjZ|VK#|I-fruUFm=DUHK_-Aw4u(rC$J<{gbUQ&Tp2 zE0uaxYj^JpwEsG*{%0og{6qU^{D}9dPb$Fh@>GiTi* zH%)38v=S&fC6|Cf$D(*k05C%kc&F;mcJlKL8N-#`_9M_tQ6R`J3>uCK`ZZ;b;ib&- z=|T~81PnWKg>Yu>w1;Pk#?7oc6jf8*N)%GsLgE0m*YLWu9|BgnyI`LpzOWrfi+8{L&*34 zk^Ow4h=y~O_GbIzXX79Ht{M%`Mn~xhO%8a6i5j1TvT!OFl7h!Uems?{4a;`8H9OIv z0Pt%y*n5Yb@8k~!PNN3$1Ea7RFyXzy+K7f+JJZj`1v$XoC|NiKuu)@Ge7Ve{%xXfb zpptw7j)R^2&aDnGp$@6#)7dX50|ns+aTbe1A0s&QVp+G4P*RR{9EBmX0quZxJ@4EAcJu?UmCBttE!^e7*uK#l zLEc?|dWS5#IGTwX3t=!?GUG>Ru=wpcUSYsOL2rqVX0z^jX&2co20kwAwxP z-o2_lcWK;zFeR^tj1tT)W?d4Km+#<2px8w>#B1~CR%A1Lx#o=>>s82$j*Ae9)l^uS zy9;^cOz>vE(J?ZaH+`oeml@3P{PZcjx4b$;C@Q$Rc{YCuO)Y5zzCQ-;mE%gI@&%DO z2jglNV3~tXo$llE`{doSmQX1hRvsEirRT7mg6q8+z`$bLLB=wJRMy%il&HkLMkfqn z)nG(yKr4~b&qTxA6w=VO0@SvOUIuCDlM#t<_&l8@ya2HWp_dF?+4x>iaPRzx&E7qzJrnXgua+%q!>m(@5H;Rf>`WW~IX-D*X)__^!(B zLB0U-=Ilk;xYAW@CnRF0+W!_sJbFjS)8kkZdx*9m$bm1Wq$GWFs-EDmKDHxI#T?UB z{>uMlklI(-OCArdzn`VVpgN7iwYjWDt>8?ujCysLnAomeyU=LhJOo~j3JHxf_Z&Dd zGMn9`axo^j4H&(=Z5y;c)bXhBy#Mpvx%6TIxDsd{0}uWPF!&|AuTjO(+eP9>&DvB%_ zrr+@qT++#BnE;hYW8FKktN?B8<6gku%=dL?m^r-Sbb3k3?R2ZLX>%ZBR3UhJBrq^= z3NUR{8%ehQChx{%PwL@sZ{Ko4wp^^-1v*l|6GB)Yz=bm#y^9Z`VR!M^6cn}`Izl2?JOLvIfyRqGEBpC~-YjrF( z0~Nx!gl~2;M)ba1S4$xc?vDUPNboohc%Xu#jqA%#TQ`T>B=wem`*r~(HvM2F1V9;8 zgVhsjQsfqQ_956CgZHL^Vu?W^&2200a%|K@K$sQ}689v~XDK;WPj)_QK)vL!BD zJ8Yjby;It4+7jb96!7|hv=$1ma;cxekwZ7}{0y4cEB1iM)8L2V`N0EgTkP}Z?*t|^ z;u?@dJ@7l%E?eb&_}|MuND~Bq)pxxTw1&!`T5sxRG8L0+Tn5uXs4G*pUTk>R!Gykb zB*t?#0tIUg4Go{r@|X_GvL@eYq#w%DTM%(C0r(yGlb9O%>C;gX*qooBqmijrorNS) zDw?k9JkPo>OvLaw$8n8Q#;d^O%w^X7@%#d_K(=urbu&BNDV&>DpbBQ(t=Q3M90y$y z?Ug!bE$f-oM_M8)jVDQ~_KYG$OBzB);+p}Z?VFU8|tw8m6p;Z5br-Gz}L_ZKViCob1PlDcsZwq_l z?q7rUKWdH3C9GR0W98}Z6Qm4H<4lc9R&|`BC*>8soT^X>+YfGWAl!n?c;h2A;~W=g zbc3!1&a-S6i}9kDC3YT!q{ytYXb&pPG?wz{V2P`e{Ft2pqcXWy@K{u7p_KAzV@=8` zRMFsr$=O4+7B(+c3@V~n-13L4Nq67JN}TvQSvz$|ejw!mT+BCrT%O4_B>FsP4Le$8 z6%oYFEmHSIAadkNw$~~?Q}`wMOmm7SbSE_8TnTklz?z4tz{C{I{RspJy)opS{A# zJA_KMCz60LRlnLJ)Q^GygY5hB%Sc>&d=0cY5r9H_o`3BrECKA(0B%u92|v6;YoHKv zg*E#u2!}wzoLRhm2rYUnoBpc>@eHA$GA~^5;=L}nu3W0k4Cb6=Pq|OSN7bDXieAMP zYhHsdmq-veSfo7@^LHkDzc*xA=~#g|W(4iK0aPK{K!gQxve5;;uv_&|C_~BxrK^1W z0`O(tX}juWd%H5n(;}7g##+aJgr?Lb? zp!5>7MTVn~8dze0`HWF>0b;3x#XzPj;S|MDG+H&B4eq{43q%|!h8W- z$HWu%TAXpd!8v*wpE zqa6iQ!l=B*T-v-r!NjmcF@V0BOgdy7&49X_Z#i279-PLcDIBr62f30#5^SmvG(Qsb z_U%!qMb*q@Y1{S_XeJcY&6lrUNi2Q{L!m<6xWT==TV?C#A?qByt18_pc(R~a}e2jn#(M(?vN-b>Q6ahN>)S8@`lWpmzvj+8?m4*MJRr}o-e?|`e z;@SfEW*EM0;@0~_)F^j!wW|N%BN*3sxEPARHlN$jdykmh`%NbuJb?febntU<9D&so z2ufb0pLeGeNCTpU6Uq*Ttb#KbH&nr)`aP1dBteJ#eEoBfJ~j_KP@Mr`ZHE#QAyDoL^`xN`q-s;1z<31C<&&?-}aaMTaV%{d>{n`!SS} z_dSI+Yv(n+qmR+KTXM5>~e%{1T{`7Xp7D@opbXbJO>>|1@ z>AO2o4%i^J_uH0zXC8!)b`}|iiW=Wf0Tw9=N4F%P=hJcwe#UqC&aC9OzjWTjBYx+p zuztq3tbASFz=P;A@g}&uGFMvL0{dRmFn`IyRtyD%kcAH$CWz1)O+)bqC1@MFa#Nd? z`%}Khze2JnC^xEa2dZsg!$*tyzi{;YwOssl^~KV>P3I!pFdStGo~3KksU#ZFI`o?- zKozsjLR>?MmzO`Krm`{s4g@-2&q~wRTiW!AwUSa{u}lp{*Ko5xeI-ig6H=a4_4oJ7`J)_}^rmE}m;grtfj9W#Jpnsa zgZrZyrq~3r0-dLpPIHacB3mCce0nKm`Lk4C_^kWv1GEGQt_NPQ`81gHD5`_wpa1+_ z=$C`BQ(s%XmR)Xv+6$LXWxSe^>j*O~z5gpAkOc7231)RM$!6&g2wouNj!udfnHH?O zsG<^)%E7@gP{<%l`gpZY8H8;Z>ntk3X|S!!Mkc_SIa9-*xg-WXCKL)Ma3`t1tFT%5 z)ZbO8L}om9CAVdu^()7@^Gaaj;(W2^_0i8|<^9Wtw=`ix*DK^3_mAAhs3R46yGgw^ zfz|VzH6h87%;L*$u(R>ukcHAA4aY^bk42rj{g|ZC_gb=0Ar(#|Ox3IMSLHqI`*i;K z4)Ws6g-M8))ot73<&69q^FkdfP*>Ce7ZI4LkWAGDIe*l^vc9|+V!<^a*FPv81F3_7 zFB%6e7>ftZMTsn{QXuFB6874>?cYmrxP#XJr<_Ccb~OckSsnZY6gT&3L($8G6S|i~N&UNVuvqI>z+X3kQxzBnKDQm~ zTd}?rMDcpzT}IjWZ_t+Ezy|8@glj!QoLZ0%}7}w|$Z2ZdCFPGjXfpT-MJzrB%>xb?%)sYAi!qum;Gg z$ao>ck90F@?*wS7Gb8vN2hM2V=Lfjwq4R0>yiXKL$x|N7>)VmLNLF0+&l@`Rw9O&~fS-iU0r=LHE={k132_GT=~L_rvbeM?NyT2=MQ4hLk^w1C@TWmHox)2+am#XDD%)Cx4Jf z`7#7i>t_6TWtUc3p$|(^4Z0COt5dy_)sx#|DHazmK3mHc*wM?-~Ecyd0f8O55Y%QGTqAbeQG?0m-5Xm$!Xu?%b?v5M!%?!Ycwd zM;aq1Nt_i?I-9n2E^=ntW;XS-gKOjTHNh2Nf~epQw+mqRALM&wJ&Jh`8gb?XX!|1S z4?lxW@V|iZPCVD}I5!lb6R#i-a|sm2Pat*hC_!CkSN>1^HIp9>1tZ??ega|y!59wn z{KZ^=@6jLNAofn)|99Q@e=EWNcdz`TF`vLiL_~nlK|Cy<3)a_eUd?+j5}PlgHRIpF z|63OFAV|NP4o$t5BnLoc1VJwE_}3J`59oXx^I5M14dksK59TlgBR1N)K)*$^zj#mBa1N6ozNkaK{ay-; zD*N95ysX0RH59vfKb*G-2J=W2$m_|FlQ)6@B38pWJuwLQ*JdV*s-v#0+NDdU+MLGP z_0QOK3ZpOvgui^70tj@G{(MOU^Z)9X>!0rs8=;ZHTfHt%7Qn*UZ0b{o1B@*ZkqpPB zQTKXnLIc(+DN3P6kIDQ*HKA~@X;)4{4arvxcn2ebA-hQMBDEKPChx;v*3e=uS?C@_|Xl>vC<{%yXKk)Kq|#LICnaXU6xG z*+5Dnj=cxEN)SACpvuG6!wC&UDR|~<5askkh4l}&w6q+{Pld*Ep9~SB5e>2|C?QG4 z1)(T4Wq-OR#h~qb@ja@>vG||@xnr+8gg}qV$h30a9#%6op3)2(kCu1Oxp(hgxJ~wn zL{RCcfIUT-^sW25^-in!;pgag%XGvyLvLV$YS-l!HL)XpO?s^UH zJO=4Uzj+=LIN=>vI-OA?8Vu(=!;i(G@%B^-oTm)ib>w0=WdN1t(ZkGHc3md0!H+_m z$l)^{7~@lA@&QG>+}cX5Tmx;Wd^h`bcd5!WERAYNlO8K@ro$0`77RSgp>E4vQ;5!e zcMPfDaRDU&>TTaDD!85WVEt@aaJdoo7=p$*RGX{~uZirph5HZr{FNs*O@d{39H+W$n4Ja zZIX9?Y(o=zM%b!jgGbYx@l|wy6k(upl7(Fw!O>tC(OD7%p3I~}nTp&Y4&QzxI>gGU z|HYd(b&NdgFa_f=CK{N5bHSxc^4Uk(4UCEt zDR+egxg6gX`t%ZAI{7G#1T@Y8chF_5_R{sarfBqzN~c7>DAhF9XFZ)JsR%dU9B znWe!`DO@b~D?Q+(CPU5+%nkEJFmmQ0qFm=7b^ZRia)=w$-eSr#;c*SYdw~v0I1XKD zNx%EtalEsmw)Iy@vRlwi-%(Xam7;ssp)D~bzgoD;{`(%iO6W%YD2nHmH_A;P@?2o* z%N7iG-R+nK_d5vEu(gf|BVf?n}yN34f;M(-m;;V(8peuE(w$19SYa;ju$O8w=y z005&PhE$?HMmq)_7g>1}ayyTA+8poGBFeQ8Ha9OiTKxf8wQ(3DO@*^~7W?R9Nipin zNq^1~o!w^N?m`#FYucfoe6HQ=A|PeJ0|_&Y+2)sdZeustP_6YidWyJ%;Lxa%K(op& z2ux{@qo73<%~C8GUZ@Cmx*<(Xc?@fV1B;gVv;c5u$0qS~scCqgOl3CJ#9G`!9tMAm7yIdlb>|`*FsvaDJ$>(;jU}`hv#Q**Q@c8 ziXi$%-cpo*IZ8#(dJ$@N9nO?~IAIlkTksZCDikMe^u~YUMMB72RhkybLfVqe9J_aG z)W>j1Sh^^o;(1SHqzw*zZi(mt^!PE*T?X?;0kZ?_0kRFSySlDsl*DpUtj`q@&%#0U ztl{qWH7f>a1Q>2G810pg-O!IyuR~M=4P`;(^4-2)>!v=cv-|BoUENOZ*mqG;`*MNY zNYWkcY01x!g`qV*NlT9IX~37J3-d~*5Eef1CrlvRl5X}mH|xSxB9&J(9EASszy zIX>WjgJt^k<-4aJL*A?Mo{Y%)U&pc`8>2yP6c0FHS6ENl2?qp^cUi&DyX$Z=J?B@s zAOxcXWqz<-*FVO2mj^wYiYAn$y!dCv)160usvTTlwrB~73|W-**oRx!@w>g2duS}N z@6q^l5{G9e)-H>UaE&16g_!de=Im) zAIAkBme+he9m+8B&6xfcwIY)JDjQG@hql~pQKhSEOQshD%Cbv(Pp?86DWz>$=Ak9n zL45p~L(2w8w!D^gpcc4U^{0)QDP8S!t@CW-8(AVtT67;AD>*yXj zo6OgzJMzQr|Izj-_M_W$_M%UR8eWL6!3=eEZDiv*%eDmBMQ+E%5i!SSf*rkz%fiDJ z>b^*_GK*eo?4qf+rc2e;`*xX&6X>f z#F3}Z%1``ZU`w7UnpDJOUAiKh2(vrd7s&uskR852JJNb)u`%UHXR&tv>io`V;??V~ z!{)dvf`Q4v@m;YWjG<|x$dZEA8%FYQJlocv-jZ1{ZzpbSb)-c6to4llWBblCjsfax zu};*HpxH3s9MrbCSA3kvLK*m(1Z2E!SIP4{Bh z-G-*OA)Czsud2kWub4PB-7xpbL656%DNadpNN*K?r7*PsaA2S>@68a(-4HyEQ4|jmnk}3oB*j z4WHD|phH9~Pga{PXvF={dZjmdzwg}RLq2tE1bkI&i&eoU1GL*Gy$GKfhv5@rgz(Keo#EpRZ;>8Q|@{E52-~bc8!1svUAMQ*;)UETX^Pf|p9@^QdewZ7 zKR~-Gc<#i}waV8AlxmQWQthe54MyEfNFZN=>~}Djwy67pW&$N_+t2EB>#a4Drz*zf z7iBl)pkmN(sNr?HOY`|0Q%#DtuIhkzp_Ilx z)aZ;lN=PT_j55lY(M6Bm$r(g9LA0o07+uUL(ciUmUGH;WcX{vM`{jAQJokL@!_1z& zS2@TAmQabc*xQu-I zyI{c`>=&f)V-6=${XoK<3FZPv`3?m|9Sf0V#N#H|4@Cp+^M~Vt-n%pHsYcKPwoP1$9; zzn7%i?EVfCtupG`qKmVwbu6m+Y6=()S77*m@S-{1YvF-@MzOccqS_jwKchSVqNR6W zmr})i_i6C3GZ3t}a4s8LT3O9Od4$<1z@o7>_~qs`=3Ldm7A2$m%Kr+*C&oWo&?UAFAB#;smz*L6-hIAX1_VaPR&w z@*pn=X;**1`%wio9QCJBC;q_Ow_wGQr*DAvOzmoDINDooJ;Esf#Z3<-1%<_hP9QE~ zg{qM$d-?C4;l! zo3Ai`m$_kt0I;Y4dQVyy)E7R90F`$s0JG205@dV;bv@%PM0Wg_*Z)Qs{m&!*%UAvH zgzW$4yZ)u2|EEv?r7`|z)ctE?{GV-&e-ddZUI791hi;(T21x}M$msfki7A6zfbUDZ zha^)eM3n$^@i~w(>o&azgit_80{nOzRPz4$y7xPc;?s3Vdkmbp=3Csd)Zz$0K2BQv z11dagK0^$BrgZ_7(NAON%cR$;o3CwTWo5zE0g2Q9C1o(Eg+~LB$^dfN@D6fvDv$wQ z0wf@^h#h$WK9uSP*@1{)0nwKmWObXn2te-)8v+$B6{x!3i-zji9SA^e(#?2y z8HdZk_hAeRjl}@%gSRj6G(c|7-#KBb0&%b)YK1`EiZGBUmzFyYYt?$ME)A%ftzIB^ z@)bk1b^?TK0H{k1qu#aAbc1g1>|u+ zaY$tS0xAv<{Wcd^78e&uz|N=>EVwryQJ7h@dGvS2>HAe+#Ne0yi6kxDS<}{1j zbY26fyGM7+U$Q~@!_d~2feSqICmJaVx_(e(#k9VFTv5KRgKi7(TZ5F;kies+Fy+|u z+b>Y#jRiV5b%5B~tc}+IMOyNgHEPha2lvjN|7SJtWeY$LAsK zFCb=y2s;2n?gRk0&C$Uwr!vU0Q#Ti%j6=pB2gYririRy?>r(K5i{_A^* zT*^Yekv3N(_=1rW%oG=MK}Cb}(h(>j!56-<>;fy$)d4c_PO_~%&V}f(m#!^MXY}5# z0WQ88j`n88TGH*{qp6$r64yblaO%a|H0Ecq-bWt%M|zN}S|XzU9}3?jiye>@P&D?v zYJw^?o;E`z)eu0PLo76_GXTu^#cnS@1z%urMFKgVI%JMAk{SsS%5LBFR46LyP9vI& z)!9G735o+6((&GKQ@3-&kSXxFFo%l5i`NRRfY$>6jJSX1qFAYkt(xGh=FW7O-EaGY#sJuR25`d^z`kg~^kg;2 zo8ERpa~9&ULF&d#0B6h}4QNSga%yAm06wv8Rx~IR$4yUyarqmR=&~Rcp!jf6SHu(0 z4*=MLht?bdno``50JxFVg!0$n-f)AOx?`h){o!H@{02yxK3z4NH@wU&C5pEKgIIVX zBd+Kg=z5C%Pb83&PQ(*SKpaZ`a&N2oihS%|Be>2rC4kbsRe-ro2Ln6qwsAQC*~_?O z>^P2?mf2ob2L1Gz6IgCPFY+3lzCg?CJDv^DpLt_6^@h?5Gkd6DYp#uE*{A~auiuAa z&CG4Fe~@Na_TQ2C=jLFOmCbfuZ3sB<+JNM9-W!xQp{pMQ?4;Ch%i#qRxH)lm;1@6o zP@1yWsa$^Y5QrO`qC@fI0#JI!d3*vs7hwACViyKuY8$6p+!m&-&-{z(ba%GJZ4vnV zdI86U2)eU*v9YnbKR|t);u>220GOY0fX(Mdb%X%rO0DyKU`ob(;QtR~`yY(d10Y)6 zgW*l&9D(19$JP}K;Dmej7&r5$bnr)|D)I{!b@H>W8&q#&zI6*?tpn#IfOHNIQNzLa zoO_Y9{@B>q_@&Y-(8*hiVr1!;PM0}>7m}HpnrfqW^-Ka>V>KAh(o-|i5l03N61^t?wpV%Y#8Z1F!UCol1NLEF z$-oWUgBc990Y|u{pvjLyUD>n)2I{8Y>M9y*SpAz2xS5gkuN+1buZJUwwhOd>)J-GSrng6KNjXRLO$7@iq zV7lEi*6#{vTH?b%Q8x{7lSkeNoHc9=KzQr}ONw9B5&w|3dQE}v)$ug=3XIQPC$NQc z&IQik%}$gxzphh-uZ&N19@pJjwI$(3yulxzr z1i)Mo8@?6r>*WCeOI;XTfjA?3zyrud2>=F~PXLp3L=mj3kXOX~b~uBGjEmJo5Rjk5 zEkV;Obzx>^23q#+R@nj7t2^WJ5vbe2KY}67uL!*PzqA7GI%MBKf_%*ZRXr0dcD-$o zWq>UT;BOCsXX=OGH7?LNlKueCNl~viQr~f~L!Z4B@U14owgv={o?S0M^UeaLNfN-K z)j*ZH89?0Xr7twX?$Rr=PzpvDP|Ajs=2x$ic!k8S-%ji1+z>dT%k{H1R>v!@1HCPpp0;nJEgKqhpTT%zS zm_7FY*aL`(4O&a8gMc~y;oy%;DM2RXw?Bx{ogjBu>_+%e87@At06B|fH~1(WsH?mJ zCHOg*P(3gkf#h%GWgu5e{QSI6XBPz4gneTXnCcJ0C#&`sgwBKCgQum10%lr51FFCZJ`U<~Pl3@&zQ@`|J;*4`0f$Y^tr*I5O^rMdn)#E5PAkhd;~yUzzkkJqG#^0_XL697CV~tn-Qy$KCs3Fm9aN9;7Ont zHv-gwc}O0B=gR?ccrA9oM5Z`OixK;f2cTJ70A_)zP}rqSD)94PUcp~7AP&U8z8L># zl>V>y%Ll6OjM7Y0yCAc%`TG?mWH1jc?bg6X1Tfh$Ar&PMMnWkqJ1;M>{ z1nDF|lmR>{&~+mwx%3o8rZu`iQo&{MVmFIP#`o@&GoamYL9{opoP+K(Q**urb+hlk zXAJ#eHR(7|218_iAd-9ninWf7ks%?`U>WZK1Bnj|&uHMBmFN-#A)ONxh6PVpAe<4T z%j2ey;RYnF*1Ufk^w_IE<|_XtqyR9+?ebW)6VqVg&Oyl#z$O<5^Uja*;Zrl`O6a@JV&6VQ|mUfAm2AeAs_J`2tVe$Jib5y)-xV3 z75Rr8AA=)JAW8!?uwy}e-gO@&X~F4`fb)+q<$pE!8>}u5#T}OoTs}y$ey>)5OXqZ~kGI zzB2-Af4u3G?`*`&AD|9N5eu}f|JFo*2I!wlegeC^c`0C(u2Toz6-<5a-xK4&8uh6v zn3e)Mk7nKvKpp@6)qVkqWgM6=7LXhlM~Zm824-eWgV2sT>ft!jiKG3KM$l+|9xY9QjOu9-vjy4fB!Z& zz=ECk14!Y2^ERckOA(N;7=)5IsB{3y6(E4;F{m!+K+Hf2`TxN1{1cfxZ=0yzlJV66 z*@rctG0g)@EN%es+^jkV^ouNzK4mrjNn)7(>#_3-g5YO(d3ZXOx54%#>l2h1a|`^} zZ*~J=X~s=5VA8zq6tocj{{QG3FbQ7zL2z5_?f>FJO8>u(#EJt^4LULevS8X?2KbBy z#IpmQo?Bfnz=Nmy{^!B;C)xn;z}bGhx&#^&P(ejOJ_rnA(t6Ob2}q^6!Nx4MyStlu zR2!NpLP6o}ja!Yb#bZkz|2ore7YJdAv_z1a2N(8S=^W*?MN&n`w<2XydIDa z3j(0P0XKqBx(sP*AQO&CpwlP?VmzZ7cotA*C_?;hkemP`Sx{vGo?_Q>0ioA^MI4Om z4g1>|AzkaglUXGWxpfyNDd6YZB%?><0t{v_wQHZqD&G!Ez^ z)#1kR>@7;FzI%{!Pw8rTNxbf_%1JGr2O$nBqC4l!Amg z^XWk4+FjZNv~Rj#1_GxAr`z5aL3{l;KSqIx-J;_)X_I5cwME7&YVTV%|FsA(KJee= z(4RUt?ry&M(_~^;`sg+PL6dfZ$Rsk1?lLkj@%>zLNRG|Z78(*dqXCTeli^N!$c%HSG6!B?HAg}sFO_oc4nGslxQ4wpOD_HxUQw$6m| zyp1W{eXEwjGA=m#DDj%KabE3OSn0En%|D3J-D~MIvL7Cwem-#G0oeY7P%4roc7jc% zc70KDY5}&`FFJER@jCdB5|haAsR7*GD3hI4pIG!u)u?taMoXLLUgyt4u~z}-_{=;b z;+9HE7JgrhNSpiYL(Z_EV(`z=h2K!59ltEr`nPt5<}VR^dmHuem! z#bV_iCjDDX(T${ndvR0ybfh4fl$Y0#y8%168Obc@u`)4;F-X zcF0y6xOZyWtwHe*B1t66uN1KP5Imx(YK&;=be$DU<%n3@-ZHIA^syVJQ|eWN#B1}i zg*45SHfr+p(~Hv!ADrLM?P#9uu6U7RXCb%7UpgDaVs-$A{wu{`z#Oz?3)1*3IU2Yg zoKeq?U1=N^wR}(hiCOdaNt}^iZ6vN^vS!1iIt-aT*xRKU0y#P!eZ6*Ycw;BJMl$TU z-=p^A()J#d3_k5e)ZjBFfBm$qS=>>Z#Ao>V;ZDM_SdrD;J<|baVB?1$z00xtD26N( zxk&3SQ(O_{F; zG@)CrB*&9ei;Hn7{3!3o>~o*G=0a-Z+@2PI2^M=!HUHf4{*`V+-H9DhjchI=q2W8GzRD5rVpQ*4<}_LRU{ZaZ zhk@(;_ws!W;Pygz(H4{i_fzX7YcpNjP2JBOF8dkG0w(m!{$51ZNFv^KVn-`-VyQEIKAE;4v!!cl^puI;}cv~cD-yX0=CyoZ*n%F#U61lL|0M-}6TBo6Vt91YK z*07m`<7~z872s>TK>nx%EKAhs8!_FwJJ^GwdIF+>MG zwbGiRzmV+59oKJ9S^L2KLpiV3W&Y?#%QiJ$ZX^8>V_?u)(u<1CYt(x4m-WKYLPx?K z)f>`0Cd5lk*pFM^mul==9tiVn8T+c+BhJ!N9e9`i)k#SVvJr%%HzKR? zvCDlW#N=dG+kLU&IaTVJ^h`uVHfnnDFxJZ=zy_&~r3ocRw{92F?r!l^S;~8K$c;A4 zXZ~RAw{TVb5Wqs;{vtP_On%YGR@`HD!C9DHipTm4fDVFXJ6vujFqVX7fv+uFSz zi`_nb=`6Zi)^?3AujYK1hD(tn;uk`!vPCQcuEL`ocjy8qmJ4J0xa!u8%Gsv06IbW< zJ|>p>@2QQB9B_~iYZ0rLke*X@BxRhcyI(}KL6_U7~j;L(Gh_PQKfA$+s*yjSj7!b0^V=|xvnrr~hW z+9^9FnMWq9DZ8cp?7~|g+Z$h(1`@T5NaTP}WLe6@lRM4nFJikg=n{->N;PK$v~={| zDe8|5J2*1#U455AqN8}I7!nH`{qRGXmK}x<+@-os=cxda@i*@%d>l4>?qi8x>Tke* zq`GFU5D!00B??NbvkL23&rbf9iBWZl+%vkNcndY8&(xo8>7n{fOQ7cLr8sY&UE2he zs7I{yT|zpJenSu#yxjD?!vXJf-wg9&AHlAgq>I;w2B#!d-rbF?`o zkoYm!<0{Mg?q-x!E>2_Zu6fbgVDtVyA%+}ac2mOVYHArGjI=FRJ8R#@P8#txD)&t> z{i}p6XWv{A6@d17$%NK}X;08?y}lH2)KTz}j`3%s%qP8vrR|* zIC^)L#A?9elUUlcsIE^YUKJFBJ14J(L}#F$+J-R2De6u8v;?GZFEeeC>%FrTJMJsP za;e>A>~4!5o=130!}{TSx2I}JAoQcIb~5j(!E;Ro&9K@jPA|lW)*frU0k^g~c{;;1 zuffR1NO?}MwD<_G8&&B=sZ@Nof~PuJEh2m$V>{R-6c2ed*}OxUNE?z1jzl_(jyA3^ zcIg?7?J%01wO|K>Z5TBY$^Rfds8{;un{c2^8<~@lHIl0Q+fmrX*$Er|y-Aaq!Oyr> z^uRrsUy3v*=g!2(U`uNLwYhI&x{{BOA}^|buNb{dSX|J~oCz;Ml}PGY_D6lloZLROb-livFz0-FMm?`CqGaz{{v+mG3u&x2KjHIyq_!4*c!%SBz#NHNaCH|)Q z1j|{bxvV_>k%asaA?>SAjVBAhvZWT>BTb_9q#7)|E;y6mX(X!18K!>qDa%*#rhkWQ zQG7Dp=eQV}?7mL3#hvz%5eW@4-7vW_azV1A;QQm!rpB)D)y0;`kDD3Pi|xYsSrOkN zEVUcv`&S069~6@jJo}viIz}&Dipr| zW)4xgD-DlU44Jil{{0gue-}!6GE3ePjTm7$jtHCtCTLMRX{8Z?%bj3nsP-_#n?SDN z!|^Ig$d#oe#=0xMFsjtb796`x%c1&CnO(j0`RcdyM8Q!)1IaPQ#&^C)So~Rii~mwL z8p~e#+^KQuwYZV0w%m++w669f-3L0l`FI`u%qXN|tkb3yHw(il-LEPIU|;o)88h@!2pEU*Lv>f7U2EWSKoLvN)8^6;01? zetW0zIzP4holnfW@O{VZ_CDqu&grBs79xAZjrZFfxDw?oC&i@EgRO2p*+^omvb75O z<6zF3O6ReEwM)0CQS8vS)kd~QltdZ+`LUlLS@*M0PCmYO_+#6IU3w4=tl@J;vj|+9 z+_J%e9qnFCWLhoViBd>f&7lqs$*{19mr}=}!x643Un8beN2TmW^4M_2X1j1))u8Gv zlN&GiF6XoVK9KXPrxG~6?-KbVkTmYp3o~ZM=X;V&oqJ*9)}i6;yV+AcE6YMeCF^X} z?#Q<@e1U--V%a${0vAwoA<;dN#`BnUxeXb_PG5MMuDG~yY1d=a_m7IZ%R29_y_=U= zuC|TM7ITOEot97kgb6{KuMssfX_Prf?xk^34mPm?)c{&=1#yC2z3I@j9 z?2zuuNMO)*?kr*xLfPAH+p1=kN157bN!9^@*8sf_2m*_p2e%h6F&>apVG$m zz-F0iY7>}uXV}e9%o`-duYVzU<_~3Iv<{5aJ14Pao<;d9 ziFg&lTAcY5K9d#A;kL!@?w%!`&aQy=?Xt1=zPPLXwIX|Lnx9Mh&Mj~D_G6m^TnGAA zOQU7vv--@g=EY_iFm@j*br$_L&}ES`&imS?67&0d?lc_e;&oYuIi!RYY8wxd7qJUq z0UJ1vXlNYV$P~>t|E1~-@Hd{%FuYfNzQMyROk}{nPhoOyd_0X%mb0iMwW{Y?tH#R2 zxMy_0lY;VI^!4oedOyanbBRW;s5Z0nu70vs3lD(zqHS$NmP?maKFIJ)sqQCi&G7MG zxI>ll(Osic4MX7X1a92gcH`Xck)Abd^JjDaa#x^&QO|9RD08F*a4;Z#3-RCr6&>0# z+fhSo9CbhJg}-uC>3ep-g&rY$g@(QHM@#YKK;|y#>5sQ-=#Ej(Vea)9EwOwp z!%(3o*IujPO-N$)_y2x{q|;45=_4$nWIZcNL`T9aI5=bnYD|j+2Ggoc?9jAVats^t zxcqx|;!WPM=^EHb#82po<=BurIAG%@lEiYCp`ttXMF0vkP6(`B@p*i=xVSZ_=Ilxl zS=gjXL&I7t_>Kqs-Y1Q65!3l`dgr3qbhf!ZR|9p2hu%NKjx95^E?@W|s^JF-uakjO zPkLT?TtO!SEzezYzwUMyB5M$dH@R=VJfch>kMyCQj6?=VNhP>{;h0S^mhw4zX9jQ1 z6Cq!^-AGG*=lS6Y2dy~fSAEQn2-gk&oW{uTP7{hJA#-qksf^#z&CS#K`kua}W9C}K zh7ST&d;o#cY7cK3(DL5%sK&X|A(e&S*Uj+*2dw%nm)An@R+v|2EB+LR_*QCWfPZm7 z(6;30P5mjt;H-UOcDo=Czg}dWnVoNs=cI41UxoERBRj0lyfwPh7pk0`%Zv2CKZL~d zwtM(8;Pq-en&)d4`)hmL?Tcm~B(=$e&n0$+#-#ifsMpSP5UGm!I;`9iixk57O%w}X zW$5Q|h#)Up`BQZj-Vzk!dJ~nx)HN`+t{U$dNjlswq%XmPrI&7tzt_e7E3-%ZsHPiZ zlgKQB3vHv0td?QWE`*CBX-3`Wx4=esc%xouIE8&DCn*qx5-+7EnyB~?fcYp1}E{d@z$G53xf~dc{?zfGVgZ7#xl61 z`H0qa->uGOu8>-VS@#kWf9FN$PIj%ZIWn)AA@`Mf&_pY_&02Y+k-e_V)G@{)2WCmA zu#g~h7%E#`sQU15BINLtRrh>5bCZ1=)CslDQ`} zZ3D#7cUFD?Kac^$5<<5KJ{AIxfaZupTzvh!KgGR--UYYBlw`0KHS*dx9a=E?B{2A# zHkd(aY`7hr#pqI$t?e&4tb2a_HId%HGp_GcYy!*;0KaBk$TbnvrdzIo2I;qTB~jLb?q?H zCh^&XTx=|-_EDKZe_7c@+?M#)j`xgLR0cxcd)!o)XP=xYsa(`239# zl&eR4WCW)aWP&ue&)sWlc)A2^2NB4~I|B^b(iJi(LQk*}47eONTpZ)JByX2s$uTNv zJtxHa)=tv8X<%aiLxYyAq=L=m+Q4}^W(&S>Mdi|N^Cm95>L{B~D2l)x<2bXLE7Z^| z5CgC9T%Qu6$t*>0)YBoy2#cG4_s*K;2QcJ5N(+w-?T)XtNzc-3ZHb|ADWnpB(KuH) zPVwIojFU=KT5j}l@4~(?dgY_jxqyq z$>gesxxME&`E6BE0QSPh*`v%T%$<{)j>>ZHW09K}8+a=V-sTc2$Lc6Kht0y+ls0i< z`_yQK7NPTsiJIxWP$<50OtUyVL?-)DH%H_l#>*Ug@?MYC+pF}*XWJr!8l{qRpd!Zf zom|1iWnG<8C!4q%iK#EfV4@!h-BO+ae?%B+vE5lO9zn3Y}%4W@l&=2|IA_KCe)jz;q-tc;! zZ>{f=v=c$sz{e976t~g0p9xa-^HFo!=4}rNQ+d8lhr`w?yPUv`{ZVeiew0FB5H^1C z+spa`YsqwLBX5yW8!v(fyV@J;Ft*{~pRx=-s>TlshDVue9$QL)6Ukp&mDe#!8O{3JQy8QqQJmI%HiS{g!ZK7Za)b)*2+WAVYZd5cSuvM&{p- z$%qdclY%8QMcm(sJWV<)mXeyT zsI6Fx5w;erc?3H+m*dbhT*WaQ0$;LJ?no>L9+{%a(>Yzt?1CpW9ovndWk|*7mDMXu6u2$k>!Di(sQ`X=mr=XtY)42#0XD6^?9w zbydyHU-1)%`ll176r*I04N6CY(u<@>vTa8DQeu&-IQC9CgyHaHy zpK-pD$N;-r$a&h*9$)j##zK_Q!~^AMrS%Y}-9u{$G#e@f{^TaiPv;?8mB+2AVEiwh z(f&K|92Gg_-73obW0Y%|Bb|dnu{XgaI?zumCmyVP>Fm<@>WKyhpE-tEy}3?B^~413 z(IA-~{?|1igDUpyW_bzJN&@( zJ0jA*@x|1x*Bn`NTjY6Cd#nCft+sf{Hk$~)9o%{8YUt!)RHlKnIu%Dw7dC@O*k+W9 zoFt*7iPl2qYZ%JOqLVz<#CeY0>th6@w?7?-nc z#+A|z?Lw-t-xDrmKPlE2ZHeJaQc@os;YPq)-4J)2pS;|Ujv-IF(;Y9K4htRL>e+Rw zO16DbS<)sQm7?ZJ85pI$f@`CO#fRiL4FpNRM1p`jutgPE2F~|3JV+4;SPqm3gD{PB6@Z*p=J(#-_(fV!sP7~ul^V&p|j;u11KYmqzFyw&$DiYJvLYtJTvVviWNrR@ zlk4<34W977I$0QJTskEJ95%3G?KvWnlie$V9%Xp=Sau#$+#0frfJ_M0j+_6m`4s0xW zaEf7Y{ZiAxns4$;LJtkP!t+=e89};tA2`Op$Jhz7B&#dzCY+f$U8vs6w~4B5ZAT0_ z+C;4LKgvDJ(V#&)I_`-2?%K!c%5`1-GB^_chTLG^Y;eY_fO#p$>T8^c=ab=X*2x$j zjm}!yPxcQJnfA*W2p%-Fe7dvYm^Ghl#;Lv-Gc+@6jv1%$FOMqx;#K8(QQx{pX=nDv z$7>{s`$zi{$Yb-L1A1DLGa8R3ZS3(KEIjMmBb3Pex%!YTVfH<{vhW|zlwXagnB-lTz!inCD&D=FnD#d?IA61O@)jqOrLw}3Ag;pUJR(nVO*tPuu7tf0Z&Y9pspIiq;x$F0rP>jDXeZc}&Q?*nkt z&vYGyN{2FWw!bl_=AZXHF1|Sq!STJCv*&bDC6A*d+V-p4B@0%yQVaqfi@J@hF5b$rU|s6llv*vF(oc>|bNf7te40{R{)l!6E+` zh2B|TIzBoJCNGF}G7U|1k7%=Iv4W=f<3+-G z=B@#CR7;l-x}#_kZf>Ah&J>x;(>H2Qjq@^{Vaj!h!eRH)*wyRD0`baCf~PX<=%tDL zk#%|wicA}em_qNVk^p_#3{`YX4$API=31(wqG98pGz|jpEUc0ffoK+~SUxKy~n7+f>RGHCPj+ zUN4|=q~=3JONGzi&@4qNruN;o1aHdvxp)%|!_V28vYzU%3{R{2v#9*o^zFUH#&yG6 zu=+2&1s7twP@>CLF^B0cmrAjE&^W_Q(rGpMclpCPg(xA9O7)lkts>a*Jvyu1v4fLY zIiJ!B_GKE;NA_JBVy6Pww zK_N+atqe`F52E5=Q}FmLQ;z^!o2Px*_e-i>-SPApZ5_c2Q<>bRK{t1lwBtRM!`p1U zx4cX3xmRJocm2}0mwyv81GcYvzj}<)5d^kn(PSrYjZ^v|M3NMP zOH@;_PqIE^)tfv6E=8 zfk*mjFxj_pupU+?E~MSW-w?prR0}SO-G;~4U*Er<)>tf_yFQ<2y5YI04A<=P2rn1zBHRCq};rh`E{jvOhYn&t7S&A1V-Y8-GyUi{TRJgxAb7E%h}895K*udTedV^gvS4p%Q#n@eTmHCI7XGs81YM9% zYZo*3#TzPnEG51oZza~`?ie-+J?NCurQ84k4+LCqR z!%nx7bXh`5-4nt)o_X7c29+3z~GMy^<8sG=>w?BsBAo5;I%J0oCV>*Gam1)8L4 zg+Ls$?o_PoOry+r#G@t^bi2GpRlHT?X^+C!fwrM$sI@&Lo z@>FdYzkRhW+tcL^j0qo?7Y?|)y{GZC_R*}S~zdo;B70X{3@~4Kd9S! z!1-5n24+0^D8%|-*TI$>krFURkOYiUXS220!R(+>ZFElf89> zXY)H7`MA_g3k~K=y6?On?qH_`TZA7ki=&TmzihrqR|VMd@1KVWLcVyQ4W=+TZF+5F zDm(OsX*iKs)b-Y!dRRL~w%%)o>~j_CNQ^kp((;#bm)@Sd|8}1-Tm56oN@dRQ`r+RD z`uf@eYZcE%BW>w5XYcRv*E>)xA=h%RC-T}b<;3?+SX88FxbDAXSRd@#h#5XM_o<2P zA_c-*KZY|BnhPdfcD5ogg05d9w+{WUA&cyb+vOvTU3Scv#guCw7PT`pOB+?&MJz7* znj=)>iI#GrDaHW|c*gOPm*oZelcR6(yqg1#tHuRpzfE`Z*({#@&JeM9c67|3LE~I# z)o5>lsu{8xF7UHVi+TpKKS0B@11HWN6Y2>UG<%wIeb!{38iYAarN0RE>=5j633v;0 zU6MJg-#yQ>8=+1ZKJT_YoirJ?aUt5n+%-QI>1n1j6i&3?Ff&q(8;tyl;w(W-%KNl6 zymsGW7nU}&EYng+T(J)IA{4}C8>s6oqUF>|;e`Sl{^PQ`5;Kp9^REuLW{nw=2l1a6I_>XG5hyM8&cT#nk2dQ$6QrtGrm@i20V*O~#B(?TFED~gbnu#?h7SSkrB@VU5H z>K3Ay@+(k5`#z0!g8+KjkE-G?tFOB$%3Ge3;#1_0l$ff^@l0sb3%fmr&kQLR)XsNkTSy=bd#WOx*FBzbj}N2j*&J(cMM@&&F%IkPh%p~SXkq+of8d6>+#T8Z-a%GNN4M7q9AHPj?<+annJzAcNyjDbyRowYd z2TI0F+q+&^{B`bin=LPCI*V30=SEk<+{o*@Jozb3F*kZyBzOJ{lzG1I#7M3l;_d>1u ztJUg73rzAAYd4&YsxDSS14(D5b%bl! z4|XjbzbW|wC2khmuye+JhVH)3HA*GZg9-m-{qasO(_;Vi zSX3J$9K*Et-(`+EC(S#*n2Hi(sned~%ptWoYMC3UY^t4DKSbw> zJt*FMvO76uNcFL{b$D-g=5UE_BmP>5a@~slJeg_jq(maZ6~oMv<#^?F;(fsIYR~}Dv;#-LycUntqLRUog%{#dH zqPt5K$Z`7Sw;62SW{(IqUzO<-uO{3#T5vYZkaT^SQ^>URf_s5<-Y6q0N@Sbq+klO7 zSD;PK>H2t|(zUxvh@=tG|7!&85^4cDm z%KRvLpzvMMn~<;i((W6R14b$M=lr8Wgvsgw{eX-^EMm+vY&KoB?&Q5d9*e-OI2mxt z6ltUFR`PkZoB8@Z-9 zw2Jsscc@(1iKaP}igQWIopxrIjjkEoXZ5L$TJDzoeA?2!f)qX_{AH1jYL)YmikeAZ zjYH_8$3On`c(*qlRT=n>lT?X2Nb_7Q^O=5av>K;*Hm-9ufS)dRubKE-fHX2i(?ir(<&y?`pi4)NYCb2HC2h`Pd(^Hd6h4U zq~?L4YyJUn0bnVfvWk)=6x>XntGN~5f!TVwu1sYZxjC|b8{x8&-_I?`m+Nj|+=5`K z#76AZhkmHYlkxff<{2wa8dc$u<+SpM^S(jFDPwFGPuuzg%g>YzOVt~kaAC=zRq4m= zN;l_hY^B3Lzoj+6-O?k884y$-rpcu@(gqg9CaO*oSwFEl2YNXr)N@0ojy@T4vLS?;gZEkN1hS zS(Hm^Q#zfXZ(c`~pZ2VNd-K^v@|SvCQuP)xlv;ZxPh9F(RzE9^62?$|a6iCYHf+lw z37_BL(YAGH5Gu5_YrbpazRCaG5!PEKXIDNp5%Q`}7(c+`xAVzZpe28m-~B;r0`F3$ zd19R!6<*4<)o)Cxl%%vRm0{VN*3A|qXvD(RAft|rKAV|rWn&|!%>`#8lGg-fA7M9` z_!^AGVyP#Tn$0+w)#=*(BY)47H><5(xjwwq;uj`UIQN&2+fKZlX_N|M+nLe%o{OCm zMhIEiilc__V`@VxWe%-~E8*j2+l&1gG8yuq+0Xm$#+^1TYAQK77D|j%bCdjbm#d})T#+#VsV); z*Z<%(maICQ(cf0`n55~`c(TLev|#729mUZSlFn%83AC420cyRp80pJo2G3bM`6Nct zHKHnX#e($s+uW5JYw5I!V%-)KI*PVPYe>#^Ka6^Dyb@|@)&FW;Q- zurlW-vRy;uiy?=J&?_N6qyf%g&)BoxchB2+y-p!?UZ7s{C*%^ z`zB`5GvVojp!c5Y5jM#R+NndC7p*d7{Skx7|6i%<+uft(2 z?7MGlRO?n0{=%`;NHgE!#`5Bu0_s`pEg07z z#!60XJoKUtg#X-OUELP76HA#E*a@-g&p_=Tt_sPwsJV*3J`sZE1Aa^5Ibwvjk+hTX zrCBm2;&F~Y%_Bp1F7VDa&`#1do}W-u$%+ifSsQ-jSLwp-JfTjA?jWjpcf9M2#ILJq z)um|u#JQwdc?DEf24Ac)bIFtxbTVu_=WZ@s^)=A*iv?-nO4m~TDGOeHVU}L?M){-P z57xlp0I6?pLh7o6gM&9R$_>55!R9B$wY&9-t%*Xp!W)~DMcQ3nG!u{N)Y;P)_^*cf z<-agLa{C-63?GR$=4nx1iLZ{M^p8tH;&zv`cio3j2_(eOvZVa12t(XquIMj1wRaLE z>mp-zX6rXHPPH|^U9+~8hPL|0820@;^=#M(N2M{R1eOU;*l5a%QNG~ltU8UI?U3us z_H7OL&K>kzzgp!U?HWjJoFmVaO68XsEj%T8#ifK)Bq-p|S`_6lpm3|gRefEe+t)Xm zaNX74GubR;n$~|jk~uE4VxyCLa)2{Drs|d1wT zlG@~0BO}@|)1k-z7IgAyh#Vd{rM29ZVO&*o%dgtBN%`_<3W8S77O$@+yg>=YVe<&*UsLt=5;sC-fpn$JCBCuuEp#q%lfeKKg;o`aYh!tLapzov$U&q7HSKmr)Z)!G5N^AUVNo z*mudPDH_=*H4OSIW$<3P@g+O2mpuH1zaMymBN~8xG`jsf7v{DAn$OxU4$PU*wprw> zS0_8V#)PgWE&+dY%9`m{7rS?gfKizy{k^OcS#*?@z{Vtt>s=J%H)ZT{lUfqdH$b&{ zx1WX3^2s~Pk8N42EYRS~tI%34)4?wn2CFB}>3QaVs8ylc3!-RC`h0$jLGa8^-lGSy zJ|=Bv5MQ?B?Ps{7S?wALRE>?6$)d-rE{C-xzCJK5HJ_XIQt{W~4a&{8i;`Gm(|jKth2N`MSuv$1&&n8yE8P5uy6g1+koTTp zO?7R%E*A7r6qPDnK|s1R>0m>oBPG-XrFTrIp<9qDNbg0IPC!6<03Yc!KpR}>+rRg5to%(f<{Wd5dCz-{abMTTm$ukkn)GGBbkHMLry}>e#?Pf> zQtE<$L|py;dibT8k(s5TiE`QH<*8z~_1HNc;9etum0~i9IbR~*SEOUX`Y5S)f`^UOLz37UM9Q1Vb7n(tN`pw1#m|{CH(y( z$*p!a#f|`DYz~Rp6HTdUD+a%DzJ8YDjx!aW&Ch|dE{@$c*D;;3=Z+nxTy|^RJ&hgC zXrp>zUW=HJ6Wgy?Q{g&Y&bqlonyz?ARNw75Bj#eXp5n8piJE)~uLMY|doHCxbrVWH zaQ}c4B@k<&sHAvjQ*)&aaSYFNjR|3P3+?PK6rBw;##AnRm1#L!?fijv&`{7=_f8$c zdC894f_es(}t80 z&fn@Y7Sc|b=I=wBT18Zb$&c^co|q{J5vlGiixg5=dsLad&{UT6ZtAkMNjjtcM@Gx^ zq?J`fT$N;LQ&Q7LhXc}St|I>T_oT}a8lj|EB1rs_fa-5TqX{wsc%`%?N!KRJ;-2NA zF*4fcCec7DD`_Rl>lnkP-7%5s@{mRSaJ#Hy=C6ivu3^U^=K_bJUn3`OV#D(_aHm*a zv*?RXVGSdL4=#B2$2@M3{8*9WtXpdOlL1UhR9~0aOBpd#D~;A11bgmx%~U+-&)5ls zH&++2k%asAjGi{k6phYZ4&<;pQrSWDM-xQlVIgQ8MU6FnIfb8_Pb;8nh{jqjS3Khw zy-|KFmo?MX){Gbd$8czcR_e1tt9CY`g8w8eZ$5KrM`EX9(_JzK%(X`N&WC~Gkur*$ zDA`SJx%-@1j7)GC;*@rU0RE0@aQKVAqb6;${1k``?S zMtzHO)QS<;ma}{aDIuem8NPSmK4_E7wQUSqk=CW}#a#2%b30Rb7xO9+%Ip}zj4<`| z<(h8~^6)EMS@(*WyvQDnw*Ujp) zMEFg-GO^Z>ZQ~e?+Spla;G~-m-uEwy0L6^ef4~SNrpz7Lw(@(?`I&<1TT8IiEUwrd%I=(+mv5b=H~=T;f@2C`-nn;`q^RUuRNYCk{wyu(?_Z~Ch2P|u1Wk`Py`%$Xa!gl@2F zIF$AcvqFA5msPrUH-?o@)Hzrial*TDImAf8ARuze+(-9=;n@4jcRbqu^p%c%p3yd( zc6grY5&=icc|S5aH)Ip#Z1clZ?nY*s%tL0GvvLckvt+lau`Uf)sf}mK<)J)81e7W7 zo`{q{OZpfV-Cf4&27T`&A{=+^#EfYRU49&sJ*{JghqxvfI%{Y zRm&#b&=3x(Z$}QwvhdcQQGlCWyTUuHY)9NRKEu^T?5&TXo;_;9S!=I>yw zfyAw52ynU<$e}FuJum#|UQ^v;tG<%(z5Z6@zQGL=J-rL9t#P@PoF|>9y!m~NH9@)0 zlcrb`G+yIltmlC`+`f)3$>+!um`PY}*J!c+^hH(3H}xMyt)sf& z(w07bf@9N+9*-LDA-JCxEI_GtG(J&#qSCbNL`j|* z&1hke30dim57IL8Op)OECmwCu9hUm68EOJUH5litc6g$ zTT&V&Lki1DjE!S{@}AZdc)2W*Af?kUmh2@JdnHNL^m2}N_&ToX-C^l|-_K58r;I!U zzFU9z^5D3;F1sB*(;3O3p(V}?l`n4dO3jy>I(jg$%`45TCfDKPmL|^^OC><^ISx0| zv>qHVd@{Lm`zk`rAD%(q5bQT&vIgSP2x0H$)~-gfAFOtEw|==ZiJwZlI6_+F0~LkG zBg2Lng4LhG{7im;_Xs67ojNK9yMuKfs~?jhW7U91W5SMHxIq{z%Q^(Wu`|VE0LA!{ z2EPA!V0GPn`xQU$l?30mE9B$DydnMuiKW)(oHf4+ds+>03=7&xPmK(Ny$2zkMFj`b zA(hs3&o&@26&B59v26LWGNxo#SL5>dDuMtDjym=+`~KhOgNo{$VnU$W>a-o66BFf6 z%%Z+$Kbc)D629qQf~yo0sZ2@{^CM>1I1O+DXDK|CSft%a;2 z4l~_U#}8^7nPLmAJ)^$Y{3+^a>5powh$gvW?OhvY)jq5?VOck7H@SJ`iaG7^-E?K% zi&>Jm$BM3kN+-%HP23Gz(@GW;OzIDqh~2q06>G5!Q9_r@>-1UJ$QES(vSRuM&ntKT zcgmC5?iI(YeINH@`qkr~5IplNV5#&7?hLpL0EN6ucL#Fj$M)tnUsP>4f=1?Haugi) zt-qarkWHsZ(Y1CR!v{mQ_>H;MU8V#D#oq7*!0B@%$lmVxn&wxqC3S&Rw_n8p&tk6GCqS zDJe`be2K^*Pv2~sc+#^6V}G@3RA;*8k$e-CBZr8Jk9b@!J9@tDn?ZC(BkM^r>g_qh z1<0L?(>YncqT(la?vLWPn)^bou8Fc0gjc3^iCk-%)fkg4F1GvJ0)YN$>@DLV%x#49n>8iF?vQ*d0##%c?{Qd|5g@WNGNu5$bwIZ zFSa7~5uZTeo>yJNGx;^niAir>W1LMk;TAf#K;MN`>TyMPbq`l5Z*HMe+;m37qb1T7 z*RWERyKa|I#l67jaIMx{_R70+y^$gdJJR(9PFqVoumq^S0}Zj!!1HUQaKpWy6@Gm2 zDKf9)+D5wY+X%t9bLo}2y$K?*39bmQ%#sV7HV5DQ;7bzmz3F~keO-m)l#hAc1Zo;b zYVC-lLI`X=m0w2ad>8&gn}WvxT;vnYKn$2h=GXIgs&5;G`N6DOkzH;?R84A*67;LQ z^W?W6`2_&p<^`}_2C(1?{Bp#Q{iy}JR6a!MQPr&}`}8=Utg!?}+6+R|Lmo)ir;Ox{jHz`W;iYoEO&is$*dzy5yuW@xw%KzZNjG$w#`QIW-e9IiJ1iw)oQdW2be}WyrBHSR*s!)wYDOXBC^w>MjAa zU0sfM$#oMpO*`2KO%V+@wMVShzTKBwGcYXhGyLKQ>uk21Pp_%!nG4r{^v`?@cwX*u zXAQkBWq^Hs@tSbfZJ+Skd21y5;ae-B8qv~L>^llM&XV&6)Gg%&F}!lD*J zV_5a|k>o-M?CQ*4#SC-C&K0ON&lFu@jJ1MOy}O+l@zL(cW`kG$cf7D$gdkb9RgXs( z#y`i0l1J^;s5Ey^Ee)G4CDM3>u__gQ?)8!)HB$Sp?N++PwMV@P8pH3{bep-VV2?E4 zHn6G`e(T&&d*30{#&p%Xmwi38pJkH#Wp0?uP>`JDE)s=OhSyriOq=Y~q^onYPyTIH z!TXig08QIlzntUR<@WqsSGekQxT>+EwM*M*O+B|sl*ZPI@Or|v%a`w5l*=bidv;Va zud=c%B$W8(u=|EwWxd;olel71NDO2ydR8Wjo833Z_c`)KRnA{q%ArqvuRZ{s9^?9) z2u&DxP*E+r*+vTCdT8x_KDIFJo<_d3cFh|sD%>N@T$5pW;gvxW)HCJOpAzvmC1Yo@ z=-;dRdP4oCqstE_+}0JA@ARuVq~`}-Kz1vmW%=?dBiQR!<5D(9%hYr8ErTQb>StT^ zO}0E=A2`564KurZBz}`|-4|p$qInSH+fV_HGPBls(ejbJ55&It`_5Be=foOIU$Vqh zzi#F@VUp4e4+w0b!&!%_nDjiA5pjF7Aa}MVdvLDwTFT#B1^Ac9rh^@r7-D z?pzmvw0evt(c=k@*jkJk1U?g`|r%eQOXZ~+?#ol)ebo)ZY~bbYfsxZ*-|;mAqFs|KOanyVBmfBn{?ofa5K56(MKGY=) zb|aeGSAEKeYNG^4SD)Io*Vz8-d0b}fYSHhK{9vyozr4!1+0rXrrSj_=J_}z%c=s9B zTKCh7?nADssJ-B+;K~Y&)McbZZaY7`Bx-(d4!tm_To2$;R~tUInEdsRnXTlQncio- zT8JOaznjo@FhjRir|mU+mq}N+%x0Pxp@4BS=L+=kC1)P&SbD~6R*3PL?-X}UBqY`6 zqDwdf^Yc@iN%0MBpYb3c@To6txlwVu7-e?cE}{2V9_9!(>dZsEVK~-VoqvzJ#IzCv1jf2R~pS^H3_e^u^;WMn}43Idg4B%Uds%2 ztLz11mno55Pn}Muikm9{b|U7sqyf@-+l8D6;mh>_01=P~>}^vslz>H!!Rd9QYz)hk zL-o6b9Kc!>0yXi@pVv7(;;cf)YKnu~k@)(uxJ+N_*kbB+=kPU&ix3D*xYR%|7maWk z?HzO|v2F9Rww@Cj+v^v~U85n}jH;1~oUR$KIJ5lmc`uva$yJ)qJ@$(_wIO4bAjlr_ znAtWlqKB9$=o)5h2T#VU((*bOTKBk>UbKGJKAhZP?v`NKZY;>yt-L0%NQDVFlf<@q zXq?$uWYQ10JLM*8kPF$8UOq3XZd~8Pwj3stUWJHT+XdyQn z$cV3A;Ueq#8yQPyEW=1ea;_$E7UVk}NbLjeL((+1K zqhV;JVT1JqE zF6kZZZ^qHbj@2|0T0haExP?5EHmBHvtxdWz>zsoZoRXn3H;R+=H3pWCww1h$>zoee zIS7k)lWf(Z69vrX-(;FAxw8u@Jl}EePinNvOwo{}Ez)ZQS=(nTneE#O3AE+(=D4&i zv5dtnviv9ww!(rSMl1}+`6%eZU_8(hV*x2!hp;rm0Kr+!_tu9Do8%*Z_cAVY?PG)$P7&8G)52b+P*e> zyM3Lb*BUEWteaN$WC**G)hn;+h9rf5HW^2WO-z^H{c}d0UwbNdv-g@@eYueqBFr80 zG1>lXMNzmac5lH%q!$Bm%`71~MA&CU_9b#2ZOWA;F_JT8d5~w$Xd5v1X8mFpx+=TM z=bXBr3VTpP<0vqt!SpfV_T$6qyOqL>GWt_=`5$@U)xa(Gy)%&Z-8m-!#Y zwWJ2gx|Hp}odb*U+hW+<+*eR#9Y))GUN+6boBQ#(Jg=~G9Y#ELk{?-oI{)U3()KzZ z%UsKS*Vv9S6!JhOPkw8;;Om*(rYHwA!_|-5*D!-Eo}3T=GF8i$cHNk3%iGi5UYiE< z^pc(`ag8F9jg{(KD0K{Z2K^Nds~h9vh9$}4eePyACtIc(LNJ~8k`m;^tlS&Q^Grjq zzwYHvvf+C@uBm6`DDk_P-&i!rECVaf3qwO`+d;3e_-DL+}Que&h3>Q@@yZ(mRzR44-B#-&U0?Hanzmu9}L#?05zn`pJ zoRD)d+^!@Dk5P&3tt%-XQ?q08Znt~;RDRlgITl{4X%4fmQ&HS^F7NFPSG1q_{+u)4 zI>5GHdt>E}>tVhAsEa`2mHd`>RR^v|5BrUG9}ahH_`A;hys6e)W}nH9c+ftT%z>}8 zk62tbZ@yHk!T9zNYAyJgY_-hEw|(9v4%J~&vgOSeFKFBhokxGCq>fYYw<=ypxmh~i z>SoN-Uun@0egJ{t`(m9!_fjy0(ZM#<$pS_$RDO zz>Js67Di*3?&X(ID>f>vZ8)@4HCr`TM?;4vccd98hm+iuT|eHOM;C2aiCx~@61}kF zkg`R|z3=tJBb@Gq2!Rey&%n?<8))eF{Ti$Ey#?njgb>AW@U89JSv6g0B2r12TobxH z4blvt6yu3v##aL^T(g_+5VPc%~_)wX~BUZPC-d+)I$0C&_ z;qJsMWlL-P2%mjcACrYY`v|}N&Ruu!r`_`VvS|8z_A;~DWA?hd-%@po4O8xvgO{HB z8}Cn&o!{} z!52F3YbUOYDe(A8L*Dh_?LYm}a5ap34mpwWv7JEqTXtc(KSB~B9Io|uf5btVZp&Wh zp!wEW?Myh`HS$@W4b!G-D{@?5M=h2yVSjj9Y+~@MoxC%AIN!j#l7rUscVccbmKtGP zW!LCY+ICtZ4^9NEzJ#tc8M0Ej(C*2G#=KxWy~~V@X>1e2VBwv0^L_{`B9xEfa6UR* z9#UjFWBo3DZief(crMEFtvoR4>t$NTFK*{Drv5H!unZ2{R@|R+l(BulNGonXr91jH z1C*E?90y3sInle);FsEXc80vwo41-q$rfHIg2$GLEBFD0t54SW6bxM48*gxqI^Bv+ z%)FJs<_Naa6OfUpz(am)MIe0=Rsa?qkYFGm-N8j!=WGq_IoHctNVsW8X7&~K=@WDx z@dgo1Dx(H+obdd{y#2}^lH7!N)cz$DKC>1@S+UEZ-Nivw!M_CuL>ADh5E+P3`asP17 zwZVN{pn|A)j$27w?}xIYFa4?27WVQ)X{Z=7KvWwFsd%(vUe;F9!{r)Aubvd$A@R3WA31R{8Q7pn&nN`b_8seO%JWPoze;%_KH;@hvaq0M`UA;m%pLxZo526uNz2X6tzq&sm19SVoNZ{cENWqG9Z zj2tC&jpvwF{<(LKW7W4#BQ|pI{)n?bv?-r8@rLxL(zb2d3fC+Cv-f0jMQlss%j92E z-H7rijr&|w-s3)k`Fm5^GDn$M-94TVaZb1=99|3w6y_7Z)FEPn+JGjm3JzS1ND=10 zq^y^=F&C&c6!g*Tl+nLKFw$~3=tItjXXL~yh|T-nw& zs-L&b!Xvt4jx78@D<$SLLv;2v%rR=D;t*=(YMy}GhDwgpeYo9EV+`Xd1wKu+dS4nrbeibe# z2;DH#UFnO}ocWoqT4>dIuz2ncj+ppZ{P;==#y4k6+W4-s0FOuGM8yCO(odLG_W9N= z6~W&7$aJh}v&K<>k$lB|4xxtYrli?-HGD5_R}*1JMl)8j|8=)j&OR$@WO+Pqx9*i~ zO}Z@a4ySzai_f#LBgbLx(j>P!zo`c`6(5?$TTaNPcCa$!RoQR)ON#;H^_7CjH_hJt z)aK3iIOv+1dF#NPZ%9w&a>hu)6Ps+pSZuOkW&nJ{rSbvey@ZjF3Y9TjtF;)EJ77o-~n1X_j75`IAp%;VR& z4C60o+x24tZe@SfpqjI|1V2=w@|f8fk5uKv-La{JI9={Ow`%-83Tqs?w*My9}SAa&Z!Z_lAwkvUsN z#)8g^3&(P*_-diW%_W7!`gS4B!#uF`KEN9F`N^F>(nGatU#>SK8-pF)J|5WJD3>t4 z)`&z*F#&B~Yk{`Iut<2BeBk`9LuK-Kaf?&ft}2{@_9p%Y=1k0UisbXS8pAu@?!L8= zkpmn}r_>w|s&ClVSDIw6fO7p3q0$LUeLUP<`GG+_!<*}s54T-y6+YG?PL>1itY@=kiGQ%o=LAFxqhZ%58`zj_9F zuUD}pf#oyzQR#|~5`)(xo`_Yn{_3U=5im_pnTSnDc@D>Egs6@Z&O7v6$)8%Mue1Jf zmXEJ^FNHv4(+h~d@!zBi_t%6`+)iQj}pPE_|I)EJLzPymH^>2|;xYuHxjn zEtLeRaKXw1*IoLzvZ=UrZ}8yjwy6G^%zR+IDOP?H)0uoziyCTOBqrVxG9qlKJk+{O zm)zZ*vEenMS?z6!{v&E@(aMq)%wLa9RyQnIwlvrhrnA4JJm-GCNiE~9`JA(LecJCG z8Uxf;^}XWso>qJIO0-y_c>Y}A@)oKUr8b`i&G?(@d9^obVRWd{>UB+{tJv5oN*eV$ zhvr$t#)bOFbl3HVvU1vkQ?x>?Aeqd;uP^l`3_aRJw;^mtBY>Zie!5gO;*DnwbT^(_ zvqJOmclXo{xB5d1?UTR;4be=M{s7A;Is*o)wwj+7#S_Oaq?a2VCdcL0fD_Q}>2FkO z?A9n9aX;BxU`AS7?r*<){uwgu7SLCpl$eCH1Y8pxGcj~?Z8v7^pQp7uA1iys%*p50 zj2#$n`3jai6=dr!^sN!iXLfz+!|`=alei<9JB!&4@l5QYUcSzgS;DqI-tY|-uzTxP zhOvw+bm4~ZsQQ>6%r2}8kEFS7+QyPK;;ASylhdF>TOY&z*iu#?%P?lPSo&%L<{*hU z|B;M#?K$X{wYR9>G+5ZG(1?G=Pf_hUcJ9ZHP z6)`b4ZgBxqG3Z3>;*ct?+glLJchO3E!I3sXCbxG!=9gt&+MB)no~KWG@z9_961VOT z{UBHFI5Ess>bYevp0SOHsM3Lld`h6YZLwPKL_fE^TA4)WG;#Me3vUOH<-B~I)gXN= z{o{1QD%#o2Q4vGmmqAPFsr^T02z4p7hca3XJ?SEl%8Ec1^bDi- z%>D4Xmn z!KS}sR?N%_dQH*rOq;GBZFaxrN{!kYvd2umJl_sTeH#%Xtba+_vTmU5wFSa=G~LGCDX(SssC5o3YAFOMUh#-^p&E;)0HtCYHqDVrI~G}UU(qublYx+K<{ z`RD3OSi5?=X$?K2tnATA30&0pDA578hK3&Q^$DeOio}u1#g@;l$oWMxOyCzB`&-Xe z&oyV=Wd^)l3B4)o-N{P!SRrD(li&+%AN9BL+GRvg=qj zsEr0MtBb063_-HuZSw<9d$lSRU~Nxa-%In0jFllm)OVZ)b+%r07EImOu37eZ*%z~E zk5r=7Stiwd>_a3F&)Mte(uTVv<(O6<5~gp+v(}iZ+mA_`im}3w?FT8+>B4X;t8lIs zZYHLb*k9Kx{nG;o=Sw_bH3BL71V0qbmA#aZv>=6hmWk`WW7keL@auQ0sTwz4ZS!ot z7jeFU{bE1TI;{@nC>%YXc4B4!4S$8dwRq_~>g1jMuP@pbYXgYk?*SDkq9gs^W`pm; zRVN)Zl9#P)L9tRKS#opX)kxR#Blco(nUxdtftL%b-kW<9$qy*6Z#8{1j4{LRs#hKb z3nPnjva{#3F@R^$DiPF^bWo*E9jDy%TbO@yN?ilex3Q8DnfMilg?d*$`Y>!0S*pT#to zCAyX-=Q-XsU0CP_MPQAI9??K6ano2p(=V(g{HEp?>z1kToYL+<)BN0&hHl~hr*hRI zH)1|y`Y-MbvGgI!T5h-H!i*cHp>>5{A5V5(AlDH4vZLG#=FvEH%*Ts8$J&$k?O(*U zGtVR0=*`ybmHA{*)~RxGh+xN?C3hjsIcdAx-}IiGK&E3yVlQ>2=30Ir>hj)I)(4uT zJ?@)EsAUEDrYlwS+|{AOxXL+LqwnBPGxE_dchYPPsq_u0-yEBM zdBtSxcO0VkN8#A#i%kYCgO@jkHC47_0_41AHYnt_uQe#ENgGl-a+Aqq)#(dTlSCOr zS!vbGFPSM{kEA`XsHc)j2g?F-3sFPC837EZ`}?V9qXOZ&xw38+8SA2${cQgF7mb*(9%^!k|Q zJZ$vk6Cab1rQOonZUIbUUsUzrCdjJpJ;$A;&x`p64XJ?N)z@g+`)K58R8BS}YE=H5GSUBpvCo+K`0HbFj{;&&T|C!+ zVhBu?o0tMGw%z6_^Fo-*@K~{<8uhdE-mgxX<()_hl|GVh)N*??B9ty=I(ICoITLDG z9>Uj3+(cA0>-BxGPSB_Ol|hsHSJ|=~-swpBa;}eDc!8!|*6q4F8QmfBi@Is-?L2v0 z9<(S6qn=B7o#J}24ia4i&y);I6Q=TM5gG~|D>mM}0KNZRMto)UXkO7#+#GC-*7bZ( zPfUJvKdHt{o;t+4k(!Srd$<%(CH+-HR(6!qS&7J@SF(LyR4!Vgvr7z~AgLsf+CY-k zaKd-?(ul%gdEOaXGKB^#F};Gcc;|5#KL^G{gd<@$i7C@1@1Ay6LyA*OgqTv?dyoOj zscVzkkW$nH&PJ0ZGDBOx!2WP+DiiZ8+27j6M%5+{>$k3k504(UXyEV2ZQqq@fQgqo z7Z|MvMatxjuZ6|5oKt-J*tWtm(fjwjy%XO2Bcr3D_Pp5cPR2agVvx0ES(QOiUZbLB z5opksZ=R%bJYr0E+KmrRlR|Z&QIcN!?PWa+BV;a(F=fY@`;$}lnqc0Jr|^Y)a~b5` zzI}?W2B0a?Girdl(>ZbnBTtGn19hlkFTPWdfi%Q%%Wt&x)t9dMS6hm|s`()G-Cs)J z7bj-cW3#Uf=}k+i>P-l1+bZz)vEh0>%^2&;VuNQqmztQp31f}CD?E)RQesI+G13!> zmx~;hzTK_A26LLif12=G2`KWj&ApQCZT+KanfI)rB6suYDDgx(b^Hf!BE^1;4*XewnyzcI)o zJ~v(TY&cpHm#15e9~+*KcsuO^jJFYSznvamklN02yrG2~;#Lg| zI584P0fid)4|I+wgh%PlE~~l|(^?&dG{f&YxDzHhj1`LhrUVH{9kCv4g?Rs2l z+sp8srIC#AAaGZVL7);KtecVJZai$>8kmYTWvXrK?qTUgOwdW8H?jEqjwe3sw=pLf^m>Qe-@gkoP1t#UAR#|R zVfGVk)fp^qw2P;JdJzshA&Tu2gU2Z>3-K$MvHSH!_Pa_)TX8~j=xoeAr8M-_h1pzu zkFF~*=%%)Ine?}Ocb6Dfi@RPh#|;~)O(C-pY-9x|oW4woL!YM~@q4snbPgM?T*cU| zII>bY&{l2&HnQ(nq*fx-cryA10ZVPbFkBd~0+QQ{1+h-h@tF8g`I&bzH1r z*7jFZ6q)9_+{oS}f_PY+VGNzn&xL=9> z7490mvNOY><~;aamDb`dPsdE|V%(uGca5FD@(aTcLH((RX1mG@E`AT=a`(+3_X=M_ zCEqyumjwDK+zM_Mj4>$JN?s=2x%F{ugw1PZ%J}Iu93MgTk^f;6TiWol8^r_7Q=sbP zH+wG7xkP{|(R>=eWPzN~DNoF7n!pz_E4d10tQP_O<-4oZR|*o|eQM7r(>7+ta$X}=fkL)- z?YruE?It6kB#D5b@zs6||1lftx;A08VT-^sbcl7}T?05K-Zxwxb5zEmhAq_M4e&i^%oe{a)&%M)DhVUQNef zA2pn~opkI@90U<3vi)ZMK3Hd2Y&2Y!x+ z>A8kvA#MWMjj1kl7rnY^vornt?WM;4%Qg}es>7{c6KcbE$=l}?Z-fb~{?2=?E03AQ z*2&+-pdMx@j;6uoo0~>SK1svBk-jpJ8SsiF^F^9j(UQZ!fp6iI^oj?Vu+Dgn z3YV+XVx_yb9t{788CH=k3MaYhr6tcx=+&K!x9cR=x@&0OS6#;&UurR`X*1xx%Az~? ztL_3kv)!ehW={q)_+0LU(tQpqU3Ot3Loup7iutjFAVI{*DdiT#N#;1GNo*%xQT2mR zn82Cwhrnm{i!CPHIpxRds3?5yiCaA=HdHGrCi%gbJA6ELrV^$##Ue)2a%zk$HT+$m z?xv3ALG2C+Y=+-DwIWu3lr#ALCXUTDE-#U;e3*DY<+Ay9QuUt6p(IcSVs&P4$1qxF?n(?=z1{gW>-_o;jyw=&0Apr5oO3DRCgm}0m?iL!TCH{Q6e(#T7o zGJMq!S!vQuuwPp)g+iYCD?nWO7#&CMW#z+72b+?*-ezLoFo*{C6?5CoBpxyZ>Fe2= zQZUldg(`Xk1Oz;CNm_QPTV$vs^~-6h6F_-~8UzA)Q1NOY`=O1&1|S(fE|pMCfDCn9 z{mq5hHs|hhlZQJeeEjIl9B*z4(>!Et^mp&YxI&yA?medsz;Ix8U1Rl4{1|eIlu985 z9(4w7h9{!49sd@G?nCHDi$0v^>H5)F-PU)ZlPb`*qdLLp?4&?GwwmeNvbDS(Q-|%c z3sbBMd}h7*L>O=~v&gVx0733*KHe5_BLkT80ze~6Q7O2`gm|8fv?hk@hmuAUYZvtE ziu>nEzD1DuEm20{(bJ#fxfsuzvpgw1RS;$W94TVfYTtKWF`om#wx*>+AS;*` zU??f|$-oqcl*7&w{V&<$-P5b!Qmv2i< zYx({izHTkWF3QJk9v4>1YR+3mD9Kd+UH9EL9^jA1Ur&c4&yBc7WQ&20K>! zTJ3Sjb=4nnl?`mUI{M-J1svI*?4z#tDG>E!WFF0RH>1iGn>{0nH+C)^3+Q|Kn=oj& zt3W?FROf$%6pTuQqUKTj2w(Qw+CI3B`+W6Y+cl3PcB=@keQf@%PTV`hsoc|T2Wph* zJL)x>VVH5b^971`i7xO=t-s9O`Xv7;=dwv zB#9b{j1ycJdzebCsLe`kJ2d~#P?^u=s9s~;K%SFj*=3LaXykZ;b&caI3!9Q}fDZS2 zrQTyLywq~16`7iRm_K$-cdv)W*noSj*G*>eU8CFu-Yc}g$Y7D7sx^^f<<|5ZTA;M{ zhv1(9i)L|*yxysrgceV@vjM=5mM))>O{okgr%LL25HGf;ASDoEJ4RSARUvkC=GKW>qOArE!>0>zAoCCqnUTU3#66pLL&&LNdz>5YYh;PE&(6 zNl#0rjqU_2hnIj}R%u3@@H6;$3ovdyUE3oz)1&%t^T_tA7uR9CGrI3&W^_wr8NNl? z7B6j$`pRrjxBn%to5d z$x;-wPzN#7wbR@(sWEL1Xq0fBFD4`w6DKi6dS?H%=LOsb&n?-n)+1d;EjKV?jc@Rt)AITo zLYR~{1cj@nyk)tHoQf|s^w>J=PNxPwq&}%dPLz}~q+_-|%7aGQ%Opd(tkjcVePq0!BkOadE&u$Ul|0cj zJ!z9NNJd5`?}ZDfE~}np*RsF(VD<860X3k9bP4DLpg9Rt-nMU5OY7HEMgRKs3Om}C zu2jL2o}YO<7%28yqCWNV@>!1If43OdMSZDdp8;=NK0Hg^8-(#It`I;_CHKUb<0M2w|H+{LBJ1aM`knL5vSZ37+X%X_bPVI&t z`hECmM32xVvV23kPndU^_I9m6%D=-L?|g~|sKMSP3&=x%=HYbCLCUlK3Hq8(X9o*-)$RRBU9yO zz%-5pdmG|?o*tg*H%og)n+W?;6cqX@{vSvsMrv`0!o&R^^xS!y91Vn;$`*<9IKeZM&wIfdx4Re!2Lw)^t#H?WneCDK$4bN>;6Z>pP zw4bx-9K~?%7aKQ_SI^^UG|%x7D|AYTBY>}&{KfDuIqfR|OjQ3E*oRZ-DBZwMoFfAS z*L9IOG@F>1m~QDF=F#a_x}_94IJyJIox-A%>RK~ly?36yq8*(XUNb465ddykzIBS! zEXWE4=E4-qymO}N=QghMI!6G z8)yE%juER%FjzV&+xL)l+dSlDwF-2sB1iwJxB@ zT~y&cZnZSm@bp41|EvH1^{StCZt~YU7_+A&?PsT0K}t$W8;>y0GkzyWyOzrpIJgzt*(E6eJ&S(M7#U|ERW*Zz5nJ0D8-sk{C3@3=+Xs& z3}75l#Yy0&K_(CW_pthZZ}$$wmNG~IVBCxI!R>h<^q)PT@GY(c9(MNjg^?^$T0C0m zcZ(sY*w|Pay zV3mZP9AA{0hnK;>blCofLym(C-mM-e+P~Hq15P>} zBcr;DOR0p@_`R^QQ)Kpt^8au%|LGH(l#AIobAvyxMUvz?7C*s;fwt+T1IX$6s|IG;*+CZCw zz?e)C3gFew6eQs>j9`Cf6<3q=`rJh|km)S%FM<=#PW>-|@t^4RJ#!gNp<``pn>ISM z59q}~`~OWKSjpF_p*W3W|Hp@K>dc?2F<9K(+#It)JCkktOGrBS`j*Fkj^hVMaUiS$ ze5fu!%=k`H%e;MYy@q?)ZR~Ki>SGehf&+ zn(f%rgLnOB2eXgiquT%o`d>a9|2Nat|L@_-|NpJ0-6cfE7bbD9RppB$1N!e`1m*wf<2Kx7U2BhMZU$ixb_`f-kBjT)K4%pX=^$i;+g>GiKI z)Rvv6jAb2)o4i*kE?#A}dZQW$DmcRkO)6|2@c?Q(;x$oXTxkWM>6>8Q-?O(dpTY_P zMgbRCKt-_vua<{NjP&%%jb#jfMX|~ZX$T%@(`f_t@r?g2lnmZPU%-6qQeYoA-B2y6 z6N!Fo*NZ@~>-l{B08AXZ0l3?`FIDnPQuF?DF=7hd(FNR-xKdKS7=fs^`~zG}7YoV0 zmNv)!beW|lu&8*o_7F_=60zD!(-Hx2*$EAB&Bo6?S5FU*h{%P{x7!4lm0oW;-X~ht z&w`@V-tZd$+2ius+;1cJ)V*MHAx>NE^V1I+Du07(Q7x+k7>^Wpg2spmnF_w$r@Uky z1OUVIC$H6buA0cKe#2^P%!c0J>J7JSTsif|0IpDJ$?ITqWZo8pIUSX6X%X!X=Pq3N zcyfqWLJ)G3+TELf{CFe-UOt)|vn~y*HVl83b7F8V4VoaEINj_)U^S&i_K6^acwm<% z<(de5wAE&t{1=yt%M!s_lgJWz1|VtD=9Ky_D*oF5ug7ByYRlk-25%ppC6H2lrAo|i zCuF~+j@>4X9YFl#NxkBt(}1vjkOh`0wCVw|+dP1ryRZm2LXybwIyYNdt0*S%_B&T` zRgR47Js>rBaP=y?{J7H@r!dKWtnzKy!V>tj?|LJza8i$1X@nB+8L*z~kecysp`&Xb zl4J%~Da_;hnTwrZmFWvu@`JY_Cy%Vjn~OuA9$!q6hUb)hCfyEUZ!0y$7+O&7Ee$Nu5 zPKn@wP+|A}d8Q6X-oF@E+Eg9;gGPqw*eK?|6#Euq*Nl1JCR_-IY-B0dP>x=juc#0s z|7p24AK-NQv-Fl<1*1pbND=j5D&*f5$gnE*Iqcc0@Uwps3@P_w^Wr@b_DT9luu=$E z#c^7)U36y>G2&gc^xR)= z0U+_cA_?v~<0*suUQYYt90`aA3gKO|J!cm+})g zk)$QjVL%{e5F_qWet8S3k5jQzQ)XS=-IsAWlPA@K;LL8Xc%0_Urw(b8RgT<)po}^F zPltwAB;R~Wx3nS?M!$W7DTN_dCZlA#z!|cVn3TkulJe;>5y<0Gu+F&Zwyk8~CwFmO z?_X2qyuv6;iJQQN1^7F*C1HBHx-KDE98F&ks|Ud<5(ipJG_x>rK5j$9nl&rt2A*4~ zA6urAD(T2DCaRup82CrA?cYlcU_E5iKRxL_We|t|R*V@~{({ef7;;pW9mN7()@ya9N(QHhe7Y@gC+Hh;!3R4%cX)L5f+_b z53ZFzJ802mjw>$WItsJUOt*Au(+!Po%cp_}hNd&$ZiY!KB2XFepVv9|~fIa*%|fByW*wlTKEMv^tj zTUt0`7@MfZl$l@mmXeg+s7}uSLfo1CpdFqgitQbpfAHu_cLQo;J&Zz(OLm(F8V{Dh zE;99r#Om%0XSZ$EA4f$~>{5vGDZZrIwAiL~OtU9XB`r*~rU@a7O^yq1f8>oK!^@PS z5fty?(slsPZ!X0ufiwwt@oFt8`C30HY~9tVo3Qx{taE)ea#Qw%*iH_2=I2&K%IRTZ z<~GZBn*8ece#!R;QsRmpuU5K%p3N)VGKQA+edW)`F%Hl}=XO+JB3SBVM%`VwrCp5l z$z-M`Bj*;1jR|6z&jP}PW+RC-9oVh_Wi(D7aM5Ju|w85TY^K#@O+Jpr2Ya5 z$W`q`X*4BpwP=miT>T~GU?umZ80E-O9#Xm)5(vp@yXdVT31WEB3EKy0lty-&`U_{> zaF8J>PBYeMvYaRNALJ~@N#j{?(<{4=;p1GEXSnk8F`G!ab?j_^sQGUyq`Uj7-dGng zam9A`YA$DiaDBYMt^P+mDgI$$?7`9w?{w_HKX`|PIOdp71naNBHs)0(V|u$Y58y62 zI<8~eavjsNkX&V-ARLxuHJ|9oC-T5K1%>Mvvmm60SA6TIz@J}V);Rx1)9D_u*MjP{vk;$>Onf=|-2{`#e-yYmz$%4Jw7PfCz3H*27PT`;315`t|+^ zLhft6I)mu@>@%ZO#WanWfC@ChOFSX(*Qa&;3M{S~)3o-S_e54*e*|BTgH780?KWoS zxwSJ%$sLU1-$-4ioGuf!?rz|9)t44&ZeMY&c$;*clBE&w4Rf<|tOi2Pe#eddv>JOk z&H(*6F!)InUM*g0B}oigSm=P>$U~I2v@VMK36@FY1v^~DuYef**xAo9HAo@Tv~8b* zgM*>E81EKfjkhb;k`nbl-d$C&;;8o@FC25ChbGT$eZV1U?N3acB<3qV_0`nW+$YfR zkE?-vPUVb^^|8oiDjr-_dz{-h6Hpv z&z#VhP}&?Zs@v0&X&P!hRDaf2Zpj{;v&++|IU-12FvtXK4Hi$uzb!U2A?=+H1M1rG z<;xfSI6UCuhyKdYzEJamKg$O;aLMYhiQd{iR?yeiN08U6d+ELs{o4mS^PTE+%Brk>3@Na)kO|$T$v_KzTXklGmkS|2}8=!n?=Svp=&pi(Nz9B z2kTfVmJPG;YV9G$#aDz&iG3R!R!BA)N@<3A^aaMX%S!hj-+(~H5^8Q_`CG?VB|1TtFh?HpD zf|vu5Ox?GK-~ROH#fulOR^0$Y%;;*;nDivWq^X?SPkJ{|J85oUPw1h$ zOn2MjH}Yk^o!$6(D}=SRf0c0tW^C~)qCIMmyyZ7$;QL4hylek2b8ux21B@8#y^zS%hiToiy2 zlwTlT-?Fs%?6G0p(1IO^q2Rpm5-$QQ@>k7+DyeWb$G9%VFan3K-bN}po4Y(h8BRGC zh=T(f3L1(ae_c^w_~y-DaKNg+LhxgE@0S3@v`9(VGpMO2OvkY z9=Z0g9hVSqYctZ?ARUu(dL;U>JSr2HqW0gX)B|bYUGYJObjBJMs@8vrYqT036;cK4uzBLr3EY4chvL!-!J34d|vPHbjVS`>vKUfOv(HL?IUgo2crcX-uaHJ9 zW-Hf98ZKN+HJeOKuyt1hf-VqOW|;&>*37WV{m%*P)mgRB8DbnN>XaZlfi@lVhQ|w+ zm!zKXqgWyRDPZ0yZAC$#SgljwKsVGR>eH@Ny+Z%#48DyGs#s!@jR~l~{>!-%^V$^a z_-#x~!E{WKS16m;3Jf4>gflXYN}P?|I^O-`P^fp0E*O$!?MBuT6Q=t1wIT z5fLsRRuXfkxtdvfxQ zU@{N;fB7Z?&Pu(w8K_?$);972fu=32aH39OWr}v%)dvy{rmi-_-&DX$Z@b-g^C5j` zW9KyMC8=-X*Z<4^QB*tfjTpg5WjdYZNRQQ?*P4|!Y4W?>u`;>sG@9q!esvc8lZrWg zIxJ9}n43?S#o$P|*PRDHPS7PE2AjlhD%KIIn+Kw~I%EHZfWX9{Bh>h#G?3Tc^R6I> z3(TE5Zxa%8DTiIy>>U|!+jZa=-G>QfUHjhs*CTJ9HylLCPD-mabx=>yVV%C6VMIM0 zB5br}(v}$APt56Ol$oe`Y3iB{{X92U*8+)-OjGUl)XmsQ;fr9m$Gg4x zOI!+x!j(~W=G()pT1GdJg6Ut1Wl#$qBQGVflixT;Ft1L4HkPf2dg-WSsH9Co-dM>F zi!S8>gZXJ1*O+MQfm3!SRI{p$atm)KM%u)_Ir7-P?>9V=mboEQ@;EN`8%p{?QoX5t z7hF2oG!|aXIG!`sM>N#kiTY^YM-5e7UETVfwG=)b&kqSN`1Lq#M9`U^{3f%$-yoK! z$dnDxBjlx3HNu~JJ2bww_PL{Y|vjrA}q2V`+ybf7NMfjL?YYO=MK8z1ca{_Oel z;7eo+_I2HPbDsZ`GJ<=LyBY;z`!toeZFk>%&*jG}&+Y;})StVVNycN5B@P6_05jcX*`TVvs^c`v_-T{!tHthr zQ&xXjn85D6AMMV=s<4aSeo0tbKwDu<@p`9@i9zc1GXGugkR9JI(!kKOrzxf@Ee&Pn zy<)UCQ4k-_#KnK;W7bfcYe&oWOoT0Bb86vuIgjG%L1Y|F1WfHNX7eRPz~0b>hBvP= zB*ibcS^f32N^mj!%5sa=?VoLGj;(%>Z-)B%cfe-&=C*0!ToI$uXbjIuxYTEdBkH*87)}cMju*4w2g>v z&?T74cm665%2y+(yotiM{8=9#FnjJ9;X(HW#VhnGF?{(GSmskMY&(r#eM8>fMPxpT zk8VnWnHZksUS>Tkx_E9|Y7BlN%tDLA&>nyRR8-UdPqRuiu;i(ud@+xhFeuSBmq zjkVmAE9){BlzLJVRl8>ItMG^nx?Xm-Tmwq#_bPS>EBhj-zIVQN=bbFT!`)k-$>zV= z`U66iFwYrbhPbVtpFCPV4B0(f!UfssZVI(wFQL8=l(r#15pyNt&z?MXxkk&rn7DEe zn8(9y8a4NoP?LMGX1NbJ$mu;@pyJG`+`y{tGB+R>z|%H<0uPSkmh_TBL4Ik$4x8iT z;idOCmz47ncaJrJG*R4lKIlr7=OIRd8n6WRYqEB4>@EcEl(;$;S$t>?48-)^6dvgM z)pEY^{-MQRrkygau+&vMsoAAvo)$D2H3ZrA2R-a zM*A5)$;+u9Q)c@ko!{j_w~C2V_FhinJ+_;0f=D!s+!Z-JF^%k62HU}2A)!mYGE>c3 zP9wy(%?t2U8c%^nGg`Rp`hKs@;iSB-<}`yoxVP792ovc!Bt;MqQ>eob5BGO=Tbv9u ze^Q#9)p-aYNdEjZs2+MneAT2;9DPpINK~7FXlXR z`%-!n;v1`+Yu@bI>*(RBOVSkTD$ICGBXa$nH%f=Qz-JipY*su9u|VzJyLY|8HzGF2 z285i)b-hMCtHawn52sP3=uTys$!AG=g`Li-4X@=lUaGl~gZ>}@88eZn4n4vpy>&n_ zOuV*9^EAu7n+w13RBM4tu2-#aYOF;k+xOcwvfKFRp_izJlFn@N+CUq8vA4(={14iux#`nle?jL3oGy6F|T`SLJ z+V6};oJ(7K1|59p>DyJLp)X*)L0iupo;L6eFHbS&TQvEh>2i5Qza=76| zd+0Hi^%vwT^4N}byc6r#epZ}+|5QWjBceZ-;oe6|oJ86*+)E&mA&(Lx zq)FewOxS{cyj!_ZS_S;zUO%~|Y+`aU08Z}O2n;Z^b^tbswI*CS zEww#x%LIUZFJ>9oz%K6ZD}WcZiG%Pb$WKY8lzo1*wF}bNd(*dWvgU}+{RH&64%b^P~Vf4b!vcK++1NhHye|M|^7SL8pd@z1sR z&))dgwfJ9fHF|$J=66zDH=jvLN;-UypFfb$Je+UJaQaEuNXB!Z?$}y-;BQU&8Vbbc zu+2Se8&MjfN7=$mypA^i?*j7TNgT`iKjq@z=P7=g-}X;!T65ZhQ&nfuAEeG|B~uNE zaJz`qT-o142T5f<5> z4*;kC1iH4(o@as?D}67OY&|2A z$1g|-;eVt1zaIhP{eLaMoCl-(zhBShO{@OD`i=kR;Yr8gxl{lChVC;p3`DW@FF*gk z^2Y>cNt)LHGCdz6d=`%9LjXR{PHdF9JWi(%&s~pB->dMVjF@3Q+>{y!@shwPS1Dc4 z!h4DHT)=Rh`}%_S`mb*v>k?FX&fXw8K?kX^U!hS)r>9?*v>QE4n0Y{0$mg3MJ9ftz z{nO&^2yw zVV{K}bR9I~+4AAcbsSA_Ck4|96^z}oY}aP(~vt-tjIx^%t* zS8f>-9SF1XMJtK)&q>EyOF(S>c6wbr6Eed5{He^kyee*+28xxt-cK&hH|e>Fq39gwUSZ%D8+hk3z~LW=fLg-HRtMo_Sol2=wBCgv%(3l z**h>ER!cBP_}FVNhN_g-&+Sm|`3!G}f~4E1GtOGtY;M(R6nhq*fm`+uiM2a3yTn}8 zyHCjSTM+WzIEQ2m+q~^i{g+<)@2b6^i2GyNKm*^nLxa5R#<(3t4rAO^3he!+Te${| z%r3j7ig+~IYYv#ACT$8DuSX4?PdIAFSjE~Mos};kd?lYYMg$hPG5Pm?1hAcR*ohv7Z60>-@pX!?Vt5SzXtmxGnC&>zDHpmgNUKv7}DaS|)X!@1}t8 zH9t+u zfSuaEh{l+_!5>)f^$xe@+#X1shEU!u18e_Xvo2`dG;Q4s|c#V|#Ac+y0B)F!t#@cYEeKi>YGobPDUJP$#7xZgZ5VOj0pK-YFJd7mSPb~fjGVm4bIJ+NZw;I>*JQWl>8VzmFCZ3o`EnIqvBD=k4BU_@; z<5xXVZYDnl2C;9iUzO#sTXZUlv|XT9e+6d}lzmE04CYP6TT$47EyNi0F{S-*(jMuF zF>plqP(W9L;*BC$D$4QoZ4X%O;rk{=6k=>%PpT!xOYH#Nj3#R|JeRl)9>?bFSOZ<=T!M z*R8t2vT4U((J3u`IJqRVwL#Nht=6o~~_M=(L;e!-J1Lje>qJ~A+P1j>D^*Q zM(14R(4UtN96B^E3tF<%ZE3c2VIE#0@A#E3ZI$0kMiMkJL(^c7rktns@HIkZb3*q3 zdp+xk#K}~&Hgui)!z7P)@7}$gz8q2^!|B|wmlthB?l;^yA*KArAlfx(z1GM&muD zEz{u<7hQg99*tiS``1R_5<+p_yYD-QuB~O7)sc&~+X= zWfh(otTfPbAxow4rB>FV$CpWQf8I{Kc`++K#TaJRg5>XwDdb_r{S6!T*vy~F?}Gh~ z!#bRK@W-Bf-PC|hq(eC0>vYm8ceF-}WbJ^Zt?cCe44Fi?3*vb(2QHSL&ieE#%dnCS z;3G0rb#NdtL*pPhOJjq4*4xIk9Q4_(-~+U%c5%zPe@@aDb~T;UV)MbD1fSjW3r0AH zfr{X*X{29h5D{s@<<#Akn)S%d988osCEko-XC#?a$n)j6Z0gjH-78JJ-@`TbhmO4T zMH$Rb@pIO_N~9QE6=PrV)F}b;PX3 ztl7k|=1HsduqKnScNhGyT~m)K$t#UZ*goHGI~WNq^|1{TWirBS*VXWiA`z7E2s3UL zGwEalb}83XRceglN{?P72qEo^AGYZfCM#7!&kzq+(u5e)62s3iI3+@PFpl16Zn)vS z?X^1&UCesj*aK6W-v$a1ri9xjp9cnh1w7;*Jd?G(-ZT#+u?2NWCo7s!-mel{eAiE> zxJ-p6)16F_>O}DV^13Z~D02g&=Vz9CGi_)|Q(WgletGKFmNRl2PZT@O&BoD|DZC!8 zW@^#ekA_^-k~H))wNFjhSI<3`?taLOxYfv1HxM2#kc!KHgjXm7w1zrxtuz z$ol%PoMB=dFWRbH29fn*HE*ntgLYUkg zxw>8!O~kvZyt(L^U%cr_3t|1wA-kcOzOy$_wMlI=^lq>G%u~ND))$&9N+1 z=`yEfMy>T=&Gdr2rs3i*6^Uu=Lw6)mLY$Rn(YbvG`pXv_sCZ@R4=<0xv8dlt_T3<4 zsxQP@(}W@LCuD4xE#ACr8~=MbyRn?$-2(`>d3D{=;=Hq)H8r<|TQf=%5E)-x|Ogk^$U?+LD*EY8g?d&h$X!n$M{p=I_hb&*9 z0^A0b%~>W*N_Ca!2~lK-_I+hLN-*pdW6>Swl_dI}WtUJXX0WRpO}oAf_yCL~7~L^+ zb@Q-&g7psNXqtIz<=I?)k()I~iH;YdhhT7pILLkKh&fh#1Di7MR85=b)2g07@8fVL zojjU)H3Z(1LwR`GbtEdWtI$@HT1~~x(N$E)_Wtt75WDGlI1z8D zahTNR%f@8PaFZW%NHrE!BM_!r57zCCnLoMX(y*oC595J>1hRkKoqY*5(UcO4xp_>F zTk^4liWSn0#?;kwW?%n;1W$oX7p!0DojvQNc`|+-_u^*1lk&3sf@N6OS?QNku%)wD z=q?0hB?lAHw)ePAr|p-gdks6)LPf~1$NqMCG?e@B=@m$iSYqa;7itu)ZKU0Y6GJcB z+S)d^_4f7#T?$t@J~_B2%ys)IHgl7tjIEFMeMxeQBMA;nunS8&YRd7}6@^!Vy|13J-ciVh&><;G-} zNQZt_31E&ym$0FcI$qbtYlEul>_|R7wH2T^*bApzsu~c$NGUZjKZb~YeS@&Dr zQtOC0y|p7Y!ZE>XQn;Vmth}srVWD$n+IPFX4rK$yw3OV@BWYyJ5~!S_ss3|WnP;VU zN?_iIr#QcBMO$M#8g}-X7cOSfx$zPiWUfy>St5;EK}<#$`8W_OR*vKRw2!V#^Q^)Nx)^|jPc-0L?G z!7l3P9Oh9=^2dU<{^KdpYI@L+PP_HV-XrFe5!4q?;;2-@xn1hKUMS+p{(C2ClU=$T z1~{mzhJba213ovJvMTR%UFh7y(!;^!LsJs}{txs`JoS^EE6bY}ifKS!%Rm{1^;`pl zdv~*T_bf8D;=t9#^9}c-+&;3F8ucc-O`2zDWG$&WO38Rk(sFOQ_00OYQQ%mWFZ(J| zKvjC~n9sL17@Bhnvxddv75}w{T^)p*`voJ1WRQI-IxUBR0O_O#B|N&$i?M^b zelDv}#H2BZ3JF2*smL3HxDiL2^CuK7wG5IuO6nZu^~eX4z8v3RNyv9^+srjf6GhBa zO*`{i#5M<}l&0EVqIV71q5SXfB0N`Nz^gGk1scaD3i-lI=twQXJRc=(f8Lb*IYS*W zey*f7WHXoR)aft~{B?WvPzy?m3R<3-Nb=TCYBfea17H>^xlX^@S3@0}`NwV?o6grc zZ4hhmka^jtl%MSXlA~!mbttz##AL-%@|VxtzT%&(HR@N1ER89o_gwZ~lU(ICIN^{` z`W+mNS!PrNWACd>?F)?>L$(c#wjPo(TJEf7=aawM9J^$=Z;O-X7 z_fVz)EKbStixRcKpib1*5HJufHI0SL-A{giw_u;m>qVm5!-#g4%>%_A4sAt58(Br* zsewG=6w`pfI`Wo@`r`!>!bpV(+cMXJHmjlds5p7wHT!0_3*o21+3DP>OS^3)JtXhE zF<>)}3P@wl0U^(!Z_S%9wD65e+Ep-jsV=lf?+Emqo=OOB;<=vzN+ z8d__x=8qX7#ny{;e=549Hbo`EM1OcF z$irJPT}4su#-jN;K?@o&RRR7%VII##9P1edS$30%Yvb{GB#>Bgo)~6C(V42y(l)R| zJ~8fCHBN)_4QJL>E2gP0_GE-W9kLWj0zOf zM3zUkD)oHsP9qtC1m{T4WU-tZm{!@j4clh0<$dhgx)cnQltroM{yeO4 zKGaOvzlwG#HGe9kB}7z8s_k*@w~zO@J3AkJPpYm+81r;380zAgb4Uy?+Qd#9T*2t( z_;aw)nG&Lr71I!TDsD#8VCzUt5T7lQ?@OgeL|bAxl<7)`VbP^i8zf`qvb=rCaiwRb z3&%cb<~wHa-yz%7viyCuIL95HknuIgYffyOwH;s6G+XT^?Iquh1S#7~N9yGC*8n0t z5yrv_;O9fPzVn99J`+thI#2tbO}@`GUOGunbmXc$s~%2Gu13rU{j{E)R zO2OEe&Sv6Q+LMpwj>_q-WNk4T2u=fv9_U@7L9yQzZH@*XYzV163kqjHC9Bi%YpBwM zIQon<%Z9;CkQp(~B{Gt@z9H4>`J;%x6EQtq zTuW&W9}>#O9Xle? z54|sDl;{_mYzzzTf3Y}V+LWw)!?MP)137Z*!lM!F9M>X;sh*nU2Aj>hF9vdQw&o1M zjv^5Lp$c@-rGv6CGS9;3Rbh(N;RYr@MlP&>HjY$0-#P?z)^43bO@23;-NV>r`p5e1 zI|;J#1KLTuP^Iv}Yhzh=c7L$hUex)be(LbufKrYp2$!cyr~Np2iL7Am{K>Q_v7>O} zYou+DIDl9s(e2r8%~M+?)v9kuWWpOS+w7w+t^TkKw}T}qL5n1Fq4`~9vTJgBMzt!Y1n6;H|CLq{gNm4<6Hoo)lIc_fm(b8GglqJ zJyzPAc#kGnf2E-VO%eP$HzAPxCiPlb#mvIs0{NVGV*#c0J7P*7rH1b*I(~2Q#1%$L z@=5Lk>m$TrIblzomX}dOnZtHl&&BU2ao#ePr!L#VfbM`H<2+O*m8^6%$E2agr$nKz zT21)7A$n|V&a{O&!myzalr;R*?S{!UmkBD5Ha`~}A6Yn8klHjXngjq4JE)dq@dfUy zNHxzm*HNSWI?(*UxxTcoib~W-w5rgP^zPL5Z$!RK9H@xVw)(`ru$ zt-h7P4P&7Mt-2`d9#!r2-7fg4;6Pz*fayiJALR`aus>A*=7)J!xc@ zN-RlJ%ZEA3933zu`B^D?y64!0Z>rSTt1eP7)a34HYy%-uxt6s>!XD4iFhAH0GS^G; z#heYw2#>sFD{p$`B%LZ!R9hqcL8_JNm6IsL`+-jtye0@<6cre-xJ?q zabgxW7Df5-ge)4mxKvitt%u^8fvn#er#*fh1!0_Z2So>n}{ zK~mM!4Cd!cnuF`?#wwd^1r~~=3`C!wU3-A}k2TN#U3SmlT7_YP+pd`Hw3tw{wGM9G zPmUs)fz(?l8({b%AI2iromT`zomQ|guArAyprQxf{#OB9f%-H#0#s|KI1(MH2S;p~ z&^`M*E}d9?wCKFl$(vcLM;2c4Ok{jMn{9S$aFm#7uxcIi$-GLCQ0LWsRnmteZ8ho{ z^;?;a8V$gxDj0qR&cFHy6|9 zpR%)d$YtbNOyY_3VP9Jf?`t4M4W*Y+XH%1t&ASRLL8As$o$`10RUwAaXlA@lH2YNR z`v@nl`$H)T<+E)K9s|4h0zRIYk=w9`HCm5q=h>$XrbmxTSe9J4e1A7XPuk)JGAWz3BtMDAMEz7Mo$~Hd*K~%Hk>nhZ?0I#{ zO`<}iw(xCRoG*H$2#>I0?A4KNp;c*KCEMn*+F)EfFK;;_8_V-Bt&`$nFqEo@3M>m%I$MQz4Sg@oqN9*b#GU^O>UHwc7d`gUjDYT}k{4HYpl zT-sM#6)=~}zTt04iUYJ1Mq#L*&w6q*z2|lV<1%3U70~(snnT203IzERHIXN1eb}tr zX8KJxPSlU$Nky=9^R#AnoVd`G5$0>S&n+*Itme-etHe4=_%T#1>$z0um2R`;zqc7 z4t&!zyWO;_fQ%uY2MK?KUWO5}$~&Bng}vt|Y}THw7K!c>0_0=uiNQZ_w|3_>Zyq3q zG+o(W>ix2z|w}uH_AM)2(V6baIeSxlvb6+rYWC(76B*MY=xE6;QbH{&^ zTb}iEkb&T@bZ%2eq1*Da*-i$uKA6$E+jdPtN;?zbJbsoHo!i97$BZTCR@U)10ZLB+3(lfo zRvoT}EtA*}ox~S8Dh_ONnixd!3FZ>9Zt0hn`^Q76oebtNeX1@9nhmSblhhQa(?D~18O&ukiK zVIgKwJ-jboUGV!;W6tLk=5y79uUhf;IoflAmE6`b9~M~F#k|YeEZ52i8L3_uQM2{~ zIC>;d!m9s_g@r|PX#|nP*+Llw)j#yAQz^CYqC96###6-ENC?o%^EGe_Px+3teqv5nzHk(Pgnp z#&)18ywVi-XCNP1Sw14)M2Gp*&zAnQi3KpxTW%MEr?0ikMV8(Fje>7sj=lqj&Zwv; z_a@u9!PwB@g+foQwY_bHD73a2vH68@TJIVCkrTOuPT*)~l`+MCO<7hb4q#d@i+Mu3X2BP|R*rg$)v8 z(>`q9zcc`_lFwNAs@Qq>Tm9uXj3wNr{rTKC#aV=RX@iFIMAJi5045Ren4`cDxht8xpP}FmFfB3T<06MP@ zFn_fM@;s#Vc883kwInA773E+g?B2?J#D{hazpa~xlfc-rgKXEK8YK3jjg6@GYNt(% zjqBzyDIf{V{^0|&))ZG!$N+~RPkiYPrRrHp|s{w@--+)6cOWCc?Q&RBr z%k^+`akURkIZ`&igI+w`?NVCz4SqH)UXq%sYTw+%V8AeJMk#K~OF1Bxnajh{wxQU+ zE6nUxnD&qDh2~4Sn46URqdjdWYqei0aC`cE0fYOZa#aAmSDJJ{(J~@z{{X{Lqo??z z#70K;weCmvRfb6jt$KtInBStIW5ll)<9$7AZf@=qfrTtFc+!YC$AL%dt#@=DTt!E+ zoTf+j^MP zZhw37`#X}(1>h@xaUiLL;43Rv-CK@Wu~WtJm?g}_MoI8Z<$!EqUNm{OV4LRi)nN8+ z2LitqIiK&3AZMHiJ~a~04nM#^^oU!}WZH+uZ1(M4XmmPND`DWpZx+47+biuRiyl$^ zQ}(@By~AX9se8*;iv=`%nTj^n)cC8_w}q~_G64cS#stMsNQAi#p2wR9BNt;O9mPv5 z&cnNJ%0x1#H7B^Y5`#g0-%?cuZ`0@O>T4hOpr7`zI{hb+E-4iNs^vnO<`fv z8znZ9-YHg@C{@jSA{n&o8xYw)uy~^CUKP${cou{t@TP~*+8hx{ZaD)lze1#b_3G6w z91NP!Vy~#ESnB}FkS!rG@pGix66*^-a!4mS)XoyKp0;CFqT?wjg8Gu9&1u{n3doF~ zP>{jSgf@fwh{5+|UFPl+8+;6kqfkCFb3j!Kj77EcHwTvpM#(yFC?PS##|BD?4&af# zITAPT`5=K^5CZnqH#Wvh4-E|+$8F--Pe|u=Y3ixErVzrIb}nzBZ>W^@m@qZ6P2 zyn7&3^GTY60@3DJO=8r+E@mOFs3Wc@|5G~Ybtm*n8%oBp4sc>|5uawr2*B?t!2|$$i*hvIXQ1b4Gw; z*T<^q1Mv=qBqSuulX5=F%x?@mSIy8t(Ji@@Tee?e=$R(0242xdJ319oFr}WUBc})kk}9WkFUUR zbP)SC?@ZObMP-If2P9wuQz&)bl@f$Yl=CIWDP^KbjX^+f+_=T_5D8Ojks3~V z{|S{?#3L7>g$w!_Gs{qS6IcQe|TFENZl)jgwK+yB)r?I#g7 z-omoRyv}{B^#}Ax1?{ELLBO4L986z5XP+-$Wk?Ik4;JH3S;|60OXsCwn(=0gVdmt^ zpQy78S> z7eZ++(A3|UdhD2oZ1M@|uTY&1I`v39#x)y#ZHIg+QpuGi;M-S<+r|23%1nj91;LaL zGvFIMtC+n#32WIQ3rzDH3tyTgeJJEcl;V9tC+%#*o=Dwm$w|5vO053Ls(UU9#9lyr zSci7R3v&**o4@1xY(SUw1;V=YnXpa}`AJw5G5I)aD)%_g9E4kbjvXu{gZLy2H_OA8 zjc3mu7;>ux%S9c_8HiYi(M21GdexpN#^F9?#%wKB4aK*wyLHmuuqgPAS?Ph0eG9Qu z0NG!_vbXFls6@tUR&U3QjEuonXzGVAtExuNJ7Mn3rJOOe{v4F{oGN!)&hE3g@zu?6 ziIe1hi`dUEh>c@PY@98Bg{=U(*6x9i1<|-)-%u!&%Z}b{izU1!Jqa#p>u1mk{br2l zxibBjD>nY1P~(?(TiIyN^IM0z`m(Zk)ky3xgJfee8&$%>q{FIOMvNFP^c~T3BRbR- zdDe~kWQSWE?gyCTbw+E;_ooVce>OBU=;PoaqvFqKLbtK60u70Qqi=VikvHuNT@rSM zPY7*cnVh}@Xj3Zau+p_CW*VHqwe_fav0O2;z~Fw1xCgExk(S9Klkoh$Es$7=Uo~fy zPmMW~!n+X%7{Q*NNgQFKqENVe*PCaI7v$uv_H04xkya7Y(jDt=qN>rVEj;R#*a|~N z)?5qx!fQ}BubG^j{EiOfo1(gxE^RG9TbNOHS$FvzSx21)*7!q~6S>H2Sb?|C1 zU#H!ISz1j~?Yws+VfOSuL_0);*5?eK8@V{mvcG%$f`}61po|f-HJmU6oogzZcSZ~h z_3`775+!uZLDtgUJz%P8QyhA1VEkQ+#xSL3^n3jPArdf#>2Po6*`@7&=`M~ci+{f_ z%(=W#m|4Mp%%Sw@^@T+S(&~K@sp43PmF`Ia^hduTMw#Sazq(gtt-@D$o{41r6b<*1 z&$j@=jtL;$E6oo_nWW_GOHLO@7qrssd6N6`4(y4S!ieak=(AP#>Ryu89F4`4LG`$V zgtbz(Ki9yxU}Du!?3Re^m3&~kI)hHGt?7@>{CU?!X*`|h-=AL7S6IM6T5`iwg-=lX zw*cjR0$V>lKfnv!d}6diN*WqYxf_Wz{{)GXzy&1tx;+EOjNY!;`QL^0HGd`&`i}nk m*?Di?x&_DaWte{l5Tq0D@%z literal 0 HcmV?d00001 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..20a1f04 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pi0-minimal-validation" +version = "0.1.0" +description = "A tiny educational pi0-style VLA flow-matching overfit experiment" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +dependencies = [ + "matplotlib>=3.8", + "numpy>=1.26", + "Pillow>=10.0", + "PyYAML>=6.0", + "torch>=2.2", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0"] + +[project.scripts] +pi0mini-train = "pi0mini.train:main" +pi0mini-infer = "pi0mini.infer:main" + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra" diff --git a/src/pi0mini/__init__.py b/src/pi0mini/__init__.py new file mode 100644 index 0000000..d1a5b31 --- /dev/null +++ b/src/pi0mini/__init__.py @@ -0,0 +1,5 @@ +"""Minimal original-pi0 architecture validation package.""" + +from .model import Pi0Config, Pi0Mini, Pi0Output, flow_matching_sample + +__all__ = ["Pi0Config", "Pi0Mini", "Pi0Output", "flow_matching_sample"] diff --git a/src/pi0mini/data.py b/src/pi0mini/data.py new file mode 100644 index 0000000..1bcace6 --- /dev/null +++ b/src/pi0mini/data.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +import json +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Sequence + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image, ImageEnhance +from torch.utils.data import Dataset + + +DEFAULT_SOURCE_MANIFEST = ( + Path(__file__).resolve().parents[4] + / "alpamayo" + / "alpamayo-qwen05b-validation" + / "data" + / "processed" + / "overfit_manifest.jsonl" +) + + +@dataclass(frozen=True) +class SampleRecord: + clip_id: str + t0_us: int + image_path: Path + trajectory_path: Path + + @property + def sample_id(self) -> str: + return f"{self.clip_id}_{self.t0_us}" + + +@dataclass(frozen=True) +class ActionStatistics: + mean: torch.Tensor + scale: torch.Tensor + + def as_dict(self) -> dict[str, list[float]]: + return { + "mean": [float(value) for value in self.mean], + "scale": [float(value) for value in self.scale], + } + + +class ActionNormalizer: + """Invertibly normalize 2-D waypoint deltas for flow matching.""" + + def __init__(self, mean: torch.Tensor, scale: torch.Tensor) -> None: + self.mean = torch.as_tensor(mean, dtype=torch.float32).reshape(2) + # Tiny overfit subsets may have a nearly constant forward velocity. A + # floor prevents an ill-conditioned target without harming round trips. + self.scale = torch.as_tensor(scale, dtype=torch.float32).reshape(2).clamp_min(0.05) + + @staticmethod + def to_deltas(history_xy: torch.Tensor, future_xy: torch.Tensor) -> torch.Tensor: + history_xy = torch.as_tensor(history_xy, dtype=torch.float32) + future_xy = torch.as_tensor(future_xy, dtype=torch.float32) + if history_xy.ndim != 2 or history_xy.shape[-1] != 2 or len(history_xy) < 1: + raise ValueError("history_xy must have shape [history_steps, 2]") + if future_xy.ndim != 2 or future_xy.shape[-1] != 2 or len(future_xy) < 1: + raise ValueError("future_xy must have shape [action_horizon, 2]") + previous = torch.cat((history_xy[-1:], future_xy[:-1]), dim=0) + return future_xy - previous + + @classmethod + def fit( + cls, + histories: Sequence[torch.Tensor], + futures: Sequence[torch.Tensor], + ) -> ActionNormalizer: + if not histories or len(histories) != len(futures): + raise ValueError("histories and futures must be non-empty and equally sized") + deltas = [ + cls.to_deltas(history, future).reshape(-1, 2) + for history, future in zip(histories, futures, strict=True) + ] + flat = torch.cat(deltas, dim=0) + return cls(flat.mean(dim=0), flat.std(dim=0, unbiased=False)) + + @classmethod + def from_dict(cls, values: dict[str, Sequence[float]]) -> ActionNormalizer: + return cls(torch.tensor(values["mean"]), torch.tensor(values["scale"])) + + @property + def statistics(self) -> ActionStatistics: + return ActionStatistics(self.mean.clone(), self.scale.clone()) + + def encode(self, history_xy: torch.Tensor, future_xy: torch.Tensor) -> torch.Tensor: + return (self.to_deltas(history_xy, future_xy) - self.mean) / self.scale + + def decode(self, actions: torch.Tensor, history_xy: torch.Tensor) -> torch.Tensor: + actions = torch.as_tensor(actions, dtype=torch.float32) + history_xy = torch.as_tensor(history_xy, dtype=torch.float32) + if actions.ndim != 2 or actions.shape[-1] != 2: + raise ValueError("actions must have shape [action_horizon, 2]") + if history_xy.ndim != 2 or history_xy.shape[-1] != 2 or len(history_xy) < 1: + raise ValueError("history_xy must have shape [history_steps, 2]") + deltas = actions * self.scale.to(actions.device) + self.mean.to(actions.device) + return history_xy[-1:].to(actions.device) + torch.cumsum(deltas, dim=0) + + def as_dict(self) -> dict[str, list[float]]: + return self.statistics.as_dict() + + def save(self, path: str | Path) -> Path: + destination = Path(path).expanduser().resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(self.as_dict(), indent=2) + "\n", encoding="utf-8") + return destination + + +def _resolve_data_path(manifest_path: Path, raw_path: str | Path) -> Path: + path = Path(raw_path).expanduser() + if path.is_absolute(): + return path.resolve() + candidates = [manifest_path.parent / path] + if len(manifest_path.parents) >= 3: + candidates.insert(0, manifest_path.parents[2] / path) + for candidate in candidates: + if candidate.exists(): + return candidate.resolve() + return candidates[0].resolve() + + +def load_manifest(path: str | Path) -> list[SampleRecord]: + manifest_path = Path(path).expanduser().resolve() + records: list[SampleRecord] = [] + for line in manifest_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + records.append( + SampleRecord( + clip_id=str(row["clip_id"]), + t0_us=int(row["t0_us"]), + image_path=_resolve_data_path(manifest_path, row["image_path"]), + trajectory_path=_resolve_data_path(manifest_path, row["trajectory_path"]), + ) + ) + if not records: + raise ValueError(f"no samples found in {manifest_path}") + return records + + +def materialize_subset( + source_manifest: str | Path = DEFAULT_SOURCE_MANIFEST, + output_dir: str | Path = "data/overfit", + *, + max_samples: int = 1, +) -> Path: + """Copy one self-contained PAI sample from the neighboring Alpamayo project.""" + + if max_samples != 1: + raise ValueError("the pi0 minimal validation intentionally materializes exactly one sample") + records = load_manifest(source_manifest)[:1] + destination = Path(output_dir).expanduser().resolve() + image_dir = destination / "images" + trajectory_dir = destination / "trajectories" + image_dir.mkdir(parents=True, exist_ok=True) + trajectory_dir.mkdir(parents=True, exist_ok=True) + + record = records[0] + stem = f"sample_00_{record.sample_id}" + image_path = image_dir / f"{stem}{record.image_path.suffix.lower()}" + trajectory_path = trajectory_dir / f"{stem}.npz" + shutil.copy2(record.image_path, image_path) + shutil.copy2(record.trajectory_path, trajectory_path) + row = { + "clip_id": record.clip_id, + "t0_us": record.t0_us, + "image_path": str(image_path.relative_to(destination)), + "trajectory_path": str(trajectory_path.relative_to(destination)), + "history_steps": 8, + "action_horizon": 16, + } + manifest_path = destination / "overfit_manifest.jsonl" + manifest_path.write_text(json.dumps(row) + "\n", encoding="utf-8") + return manifest_path + + +def _resample_xy(xy: torch.Tensor, steps: int) -> torch.Tensor: + xy = torch.as_tensor(xy, dtype=torch.float32) + if xy.ndim != 2 or xy.shape[-1] != 2 or len(xy) < 2: + raise ValueError("trajectory must contain at least two XY waypoints") + if len(xy) == steps: + return xy + signal = xy.T.unsqueeze(0) + return F.interpolate(signal, size=steps, mode="linear", align_corners=True)[0].T + + +class Pi0OverfitDataset(Dataset[dict[str, Any]]): + """One PAI sample with four deterministic variants for pi0 overfitting.""" + + def __init__( + self, + manifest_path: str | Path, + *, + image_size: int = 64, + augmentation_factor: int = 4, + history_steps: int = 8, + action_horizon: int = 16, + seed: int = 7, + action_statistics: ActionStatistics | dict[str, Sequence[float]] | None = None, + ) -> None: + if image_size < 1: + raise ValueError("image_size must be positive") + if augmentation_factor < 1: + raise ValueError("augmentation_factor must be at least one") + if history_steps != 8: + raise ValueError("history_steps must be 8 so continuous state has dimension 16") + if action_horizon != 16: + raise ValueError("action_horizon must be 16 for this minimal validation") + self.records = load_manifest(manifest_path) + self.image_size = int(image_size) + self.augmentation_factor = int(augmentation_factor) + self.history_steps = int(history_steps) + self.action_horizon = int(action_horizon) + self.seed = int(seed) + self.state_dim = self.history_steps * 2 + self.action_dim = 2 + + if action_statistics is None: + geometries = [ + self._load_geometry(base_index, variant) + for base_index in range(len(self.records)) + for variant in range(self.augmentation_factor) + ] + self.action_normalizer = ActionNormalizer.fit( + [history for history, _ in geometries], + [future for _, future in geometries], + ) + elif isinstance(action_statistics, ActionStatistics): + self.action_normalizer = ActionNormalizer( + action_statistics.mean, action_statistics.scale + ) + else: + self.action_normalizer = ActionNormalizer.from_dict(action_statistics) + + @property + def action_statistics(self) -> ActionStatistics: + return self.action_normalizer.statistics + + def save_action_statistics(self, path: str | Path) -> Path: + return self.action_normalizer.save(path) + + def __len__(self) -> int: + return len(self.records) * self.augmentation_factor + + def _indices(self, index: int) -> tuple[int, int]: + if index < 0: + index += len(self) + if index < 0 or index >= len(self): + raise IndexError(index) + return divmod(index, self.augmentation_factor) + + def _rng(self, base_index: int, variant: int) -> np.random.Generator: + return np.random.default_rng(np.random.SeedSequence([self.seed, base_index, variant])) + + def _load_geometry(self, base_index: int, variant: int) -> tuple[torch.Tensor, torch.Tensor]: + record = self.records[base_index] + with np.load(record.trajectory_path) as arrays: + history = torch.from_numpy(np.asarray(arrays["history_xyz"], dtype=np.float32))[:, :2] + future = torch.from_numpy(np.asarray(arrays["future_xyz"], dtype=np.float32))[:, :2] + history = _resample_xy(history, self.history_steps) + future = _resample_xy(future, self.action_horizon) + if variant == 0: + return history, future + + rng = self._rng(base_index, variant) + angle = float(rng.normal(0.0, 0.008)) + scale = float(rng.normal(1.0, 0.006)) + rotation = history.new_tensor( + [[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]] + ) + history = scale * (history @ rotation.T) + future = scale * (future @ rotation.T) + progress = torch.linspace(0.0, 1.0, self.action_horizon, dtype=future.dtype) + future[:, 1] += float(rng.normal(0.0, 0.012)) * progress.square() + return history, future + + def _load_image(self, base_index: int, variant: int) -> torch.Tensor: + with Image.open(self.records[base_index].image_path) as source: + image = source.convert("RGB") + if variant: + rng = self._rng(base_index, variant) + image = ImageEnhance.Brightness(image).enhance(float(rng.uniform(0.97, 1.03))) + image = ImageEnhance.Contrast(image).enhance(float(rng.uniform(0.98, 1.02))) + image = image.resize( + (self.image_size, self.image_size), resample=Image.Resampling.BILINEAR + ) + array = np.asarray(image, dtype=np.float32).copy() / 255.0 + return torch.from_numpy(array).permute(2, 0, 1).contiguous() + + def __getitem__(self, index: int) -> dict[str, Any]: + base_index, variant = self._indices(index) + record = self.records[base_index] + history_xy, future_xy = self._load_geometry(base_index, variant) + actions = self.action_normalizer.encode(history_xy, future_xy) + return { + "image": self._load_image(base_index, variant), + "state": history_xy.reshape(-1), + "actions": actions, + "history_xy": history_xy, + "future_xy": future_xy, + "sample_id": f"{record.sample_id}:aug{variant}", + } diff --git a/src/pi0mini/flow.py b/src/pi0mini/flow.py new file mode 100644 index 0000000..18a4121 --- /dev/null +++ b/src/pi0mini/flow.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections.abc import Callable +import torch +from torch import Tensor + + +class FlowMatching: + def __init__(self, inference_steps: int = 10, time_cutoff: float = 0.999) -> None: + self.inference_steps = inference_steps + if not 0.0 < time_cutoff <= 1.0: + raise ValueError("time_cutoff must be in (0, 1]") + self.time_cutoff = time_cutoff + + def sample_training_times( + self, + batch: int, + *, + device: torch.device | str, + dtype: torch.dtype, + generator: torch.Generator | None = None, + ) -> Tensor: + """Sample pi0's shifted Beta(1.5, 1) flow times. + + The paper parameterizes time from clean action toward noise and uses + ``0.001 + 0.999 * Beta(1.5, 1)``. This project parameterizes the same + path in the reverse direction, from noise at tau=0 to action at tau=1, + hence ``tau = 0.999 * (1 - Beta(1.5, 1))``. Beta(a, 1) can be sampled + as ``U ** (1/a)``, which preserves ``torch.Generator`` support. + """ + uniform = torch.rand( + batch, device=device, dtype=dtype, generator=generator + ) + beta = uniform.pow(1.0 / 1.5) + return self.time_cutoff * (1.0 - beta) + + def training_batch( + self, target: Tensor, generator: torch.Generator | None = None + ) -> tuple[Tensor, Tensor, Tensor]: + batch = target.shape[0] + t = self.sample_training_times( + batch, + device=target.device, + dtype=target.dtype, + generator=generator, + ) + noise = torch.randn( + target.shape, device=target.device, dtype=target.dtype, generator=generator + ) + noisy = t[:, None, None] * target + (1 - t[:, None, None]) * noise + return noisy, t, target - noise + + @staticmethod + def loss(prediction: Tensor, target_velocity: Tensor) -> Tensor: + return torch.nn.functional.mse_loss(prediction.float(), target_velocity.float()) + + @torch.no_grad() + def sample( + self, + model_fn: Callable[[Tensor, Tensor], Tensor], + shape: tuple[int, ...], + device: torch.device | str, + steps: int | None = None, + generator: torch.Generator | None = None, + ) -> Tensor: + count = steps or self.inference_steps + x = torch.randn(shape, device=device, generator=generator) + grid = torch.linspace(0, 1, count + 1, device=device) + for index in range(count): + t = grid[index].expand(shape[0]) + x = x + (grid[index + 1] - grid[index]) * model_fn(x, t) + return x diff --git a/src/pi0mini/infer.py b/src/pi0mini/infer.py new file mode 100644 index 0000000..e22f9ed --- /dev/null +++ b/src/pi0mini/infer.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +import matplotlib.pyplot as plt +import numpy as np +import torch + +from .data import Pi0OverfitDataset +from .flow import FlowMatching +from .model import Pi0Config, Pi0Mini +from .runtime import choose_device, write_json + + +@torch.no_grad() +def evaluate(checkpoint_path: str) -> Path: + checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) + cfg = checkpoint["config"] + root = Path(__file__).resolve().parents[2] + device = choose_device(cfg.get("device", "auto")) + dataset = Pi0OverfitDataset( + root / "data/overfit/overfit_manifest.jsonl", + image_size=cfg["data"]["image_size"], + augmentation_factor=cfg["data"]["num_variants"], + action_horizon=cfg["data"]["action_horizon"], + action_statistics=checkpoint["action_statistics"] + ) + model = Pi0Mini(Pi0Config(**checkpoint["model_config"])) + model.load_state_dict(checkpoint["model"]) + model.to(device).eval() + flow = FlowMatching(cfg["inference"]["flow_steps"]) + figure, axes = plt.subplots(1, len(dataset), figsize=(4 * len(dataset), 4), squeeze=False) + metrics = [] + for index, sample in enumerate(dataset): + image = sample["image"][None].to(device) + state = sample["state"][None].to(device) + input_ids = torch.tensor([[1, 2]], device=device) + def velocity(x: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + return model(image, input_ids, state, x, t).vector_field + # Fixed noise makes the tiny overfit demonstration reproducible. + torch.manual_seed(cfg["inference"].get("noise_seed", cfg["seed"])) + normalized = flow.sample( + velocity, (1, dataset.action_horizon, 2), device, + cfg["inference"]["flow_steps"] + )[0].cpu() + predicted = dataset.action_normalizer.decode(normalized, sample["history_xy"]).numpy() + target = sample["future_xy"].numpy() + mse = float(np.mean((predicted - target) ** 2)) + metrics.append({"sample_id": sample["sample_id"], "trajectory_mse": mse}) + ax = axes[0, index] + ax.plot(target[:, 0], target[:, 1], "o-", label="target") + ax.plot(predicted[:, 0], predicted[:, 1], "x--", label="pi0 flow") + ax.scatter([0], [0], c="black", marker="s", label="ego") + ax.set_title(f"π0 flow MSE={mse:.3f}") + ax.set_xlabel("forward x (m)") + ax.set_ylabel("left y (m)") + ax.axis("equal") + ax.grid(True, alpha=.3) + ax.legend(fontsize=8) + figure.tight_layout() + output_dir = Path(checkpoint_path).parent + plot = output_dir / "trajectory_comparison.png" + figure.savefig(plot, dpi=160) + plt.close(figure) + write_json(output_dir / "evaluation.json", metrics) + print(metrics) + return plot + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint", default="outputs/overfit/checkpoint.pt") + args = parser.parse_args() + evaluate(args.checkpoint) + + +if __name__ == "__main__": + main() diff --git a/src/pi0mini/model.py b/src/pi0mini/model.py new file mode 100644 index 0000000..77aa261 --- /dev/null +++ b/src/pi0mini/model.py @@ -0,0 +1,458 @@ +"""A compact, pure-PyTorch approximation of the original pi0 model. + +This module keeps the architectural choices that distinguish pi0 from a +plain diffusion Transformer: + +* image patches and language tokens form the VLM prefix; +* continuous proprioception and noisy action tokens form distinct state and + action blocks inside the action-expert stream; +* the VLM and action experts have independent parameters at every layer; +* state queries can read the prefix and state, while action queries can read + the prefix, state, and all action tokens; +* noisy actions are concatenated with a flow-time embedding before an MLP; +* flow time is *not* injected with adaptive normalization (the pi0.5 change). + +The implementation is intentionally small and dependency-free so it can be +overfit on a few examples as an architecture/optimization smoke test. It is +not intended to reproduce the scale or robotics capability of pi0. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Optional + +import torch +from torch import Tensor, nn +import torch.nn.functional as F + + +@dataclass +class Pi0Config: + """Configuration for :class:`Pi0Mini`.""" + + vocab_size: int = 512 + action_dim: int = 2 + state_dim: int = 8 + image_channels: int = 3 + image_size: int = 64 + patch_size: int = 16 + d_model: int = 128 + n_heads: int = 4 + n_layers: int = 4 + mlp_ratio: int = 4 + max_text_tokens: int = 64 + max_action_tokens: int = 32 + time_feature_dim: int = 64 + dropout: float = 0.0 + rms_eps: float = 1e-6 + # The observation/language prefix is bidirectional in pi0. Keeping this + # configurable is useful only for small ablations. + prefix_causal: bool = False + + def __post_init__(self) -> None: + if self.d_model % self.n_heads: + raise ValueError("d_model must be divisible by n_heads") + if self.image_size % self.patch_size: + raise ValueError("image_size must be divisible by patch_size") + if self.time_feature_dim < 2 or self.time_feature_dim % 2: + raise ValueError("time_feature_dim must be a positive even number") + + @property + def max_image_tokens(self) -> int: + return (self.image_size // self.patch_size) ** 2 + + @property + def max_prefix_tokens(self) -> int: + return self.max_image_tokens + self.max_text_tokens + + @property + def max_suffix_tokens(self) -> int: + # One continuous proprioceptive-state token precedes action tokens. + return 1 + self.max_action_tokens + + +@dataclass +class Pi0Output: + """Outputs of the two-expert network.""" + + vector_field: Tensor + prefix_hidden: Tensor + suffix_hidden: Tensor + state_hidden: Tensor + action_hidden: Tensor + loss: Optional[Tensor] = None + flow_loss: Optional[Tensor] = None + + +class RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.eps = eps + + def forward(self, x: Tensor) -> Tensor: + # Accumulate in fp32 for stable mixed-precision training. + mean_square = x.float().pow(2).mean(dim=-1, keepdim=True) + normalized = x * torch.rsqrt(mean_square.to(x.dtype) + self.eps) + return normalized * self.weight + + +class FourierTimeFeatures(nn.Module): + """Fixed sinusoidal features for a scalar flow time in [0, 1].""" + + def __init__(self, dim: int) -> None: + super().__init__() + half = dim // 2 + frequencies = torch.exp(torch.linspace(math.log(1.0), math.log(1000.0), half)) + self.register_buffer("frequencies", frequencies, persistent=False) + + def forward(self, timesteps: Tensor) -> Tensor: + timesteps = timesteps.reshape(-1).float() + angles = 2 * math.pi * timesteps[:, None] * self.frequencies[None, :] + return torch.cat((angles.sin(), angles.cos()), dim=-1) + + +class ActionTimeProjection(nn.Module): + """pi0-style joint embedding of noisy action values and flow time.""" + + def __init__(self, action_dim: int, time_dim: int, model_dim: int) -> None: + super().__init__() + self.time_features = FourierTimeFeatures(time_dim) + self.mlp = nn.Sequential( + nn.Linear(action_dim + time_dim, model_dim), + nn.SiLU(), + nn.Linear(model_dim, model_dim), + ) + + def forward(self, noisy_actions: Tensor, timesteps: Tensor) -> Tensor: + time = self.time_features(timesteps).to( + device=noisy_actions.device, dtype=noisy_actions.dtype + ) + time = time[:, None, :].expand(-1, noisy_actions.shape[1], -1) + return self.mlp(torch.cat((noisy_actions, time), dim=-1)) + + +class AttentionWeights(nn.Module): + """Independent attention projections belonging to one expert.""" + + def __init__(self, dim: int) -> None: + super().__init__() + self.q = nn.Linear(dim, dim, bias=False) + self.k = nn.Linear(dim, dim, bias=False) + self.v = nn.Linear(dim, dim, bias=False) + self.out = nn.Linear(dim, dim, bias=False) + + +class MLP(nn.Module): + def __init__(self, dim: int, ratio: int, dropout: float) -> None: + super().__init__() + hidden = dim * ratio + self.net = nn.Sequential( + nn.Linear(dim, hidden), + nn.GELU(approximate="tanh"), + nn.Dropout(dropout), + nn.Linear(hidden, dim), + nn.Dropout(dropout), + ) + + def forward(self, x: Tensor) -> Tensor: + return self.net(x) + + +def _attention( + query: Tensor, + key: Tensor, + value: Tensor, + n_heads: int, + allowed: Optional[Tensor], + dropout: float, + training: bool, +) -> Tensor: + batch, query_length, width = query.shape + head_dim = width // n_heads + + def split_heads(x: Tensor) -> Tensor: + return x.reshape(batch, -1, n_heads, head_dim).transpose(1, 2) + + q = split_heads(query) + k = split_heads(key) + v = split_heads(value) + scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(head_dim) + if allowed is not None: + # allowed has shape [B, Q, K]; True denotes a visible attention edge. + scores = scores.masked_fill( + ~allowed[:, None, :, :], torch.finfo(scores.dtype).min + ) + probabilities = scores.float().softmax(dim=-1).to(scores.dtype) + probabilities = F.dropout(probabilities, p=dropout, training=training) + result = torch.matmul(probabilities, v) + return result.transpose(1, 2).contiguous().reshape(batch, query_length, width) + + +class DualExpertBlock(nn.Module): + """One pi0 block with separate VLM and action-expert parameters.""" + + def __init__(self, config: Pi0Config) -> None: + super().__init__() + dim = config.d_model + self.n_heads = config.n_heads + self.dropout = config.dropout + + self.vlm_norm1 = RMSNorm(dim, config.rms_eps) + self.vlm_attention = AttentionWeights(dim) + self.vlm_norm2 = RMSNorm(dim, config.rms_eps) + self.vlm_mlp = MLP(dim, config.mlp_ratio, config.dropout) + + # Unlike pi0.5, the original pi0 action expert uses ordinary norms. + # Time has already entered through ActionTimeProjection. + self.action_norm1 = RMSNorm(dim, config.rms_eps) + self.action_attention = AttentionWeights(dim) + self.action_norm2 = RMSNorm(dim, config.rms_eps) + self.action_mlp = MLP(dim, config.mlp_ratio, config.dropout) + + def forward( + self, + prefix: Tensor, + suffix: Tensor, + prefix_edges: Tensor, + suffix_edges: Tensor, + ) -> tuple[Tensor, Tensor]: + prefix_input = self.vlm_norm1(prefix) + vlm = self.vlm_attention + prefix_update = _attention( + vlm.q(prefix_input), + vlm.k(prefix_input), + vlm.v(prefix_input), + self.n_heads, + prefix_edges, + self.dropout, + self.training, + ) + prefix = prefix + vlm.out(prefix_update) + prefix = prefix + self.vlm_mlp(self.vlm_norm2(prefix)) + + suffix_input = self.action_norm1(suffix) + action = self.action_attention + # Prefix K/V are projected by the VLM expert while state/action K/V + # are projected by the action expert. Suffix queries use action-expert + # Q weights, giving one-way prefix -> suffix information flow. + keys = torch.cat((vlm.k(prefix_input), action.k(suffix_input)), dim=1) + values = torch.cat((vlm.v(prefix_input), action.v(suffix_input)), dim=1) + suffix_update = _attention( + action.q(suffix_input), + keys, + values, + self.n_heads, + suffix_edges, + self.dropout, + self.training, + ) + suffix = suffix + action.out(suffix_update) + suffix = suffix + self.action_mlp(self.action_norm2(suffix)) + return prefix, suffix + + +class Pi0Mini(nn.Module): + """Minimal original-pi0 dual-expert flow policy.""" + + def __init__(self, config: Optional[Pi0Config] = None) -> None: + super().__init__() + self.config = config or Pi0Config() + config = self.config + + self.patch_encoder = nn.Conv2d( + config.image_channels, + config.d_model, + config.patch_size, + stride=config.patch_size, + ) + self.token_embedding = nn.Embedding(config.vocab_size, config.d_model) + self.state_projection = nn.Sequential( + nn.Linear(config.state_dim, config.d_model), + nn.SiLU(), + nn.Linear(config.d_model, config.d_model), + ) + self.action_time_projection = ActionTimeProjection( + config.action_dim, config.time_feature_dim, config.d_model + ) + self.prefix_position = nn.Parameter( + torch.randn(1, config.max_prefix_tokens, config.d_model) * 0.01 + ) + self.suffix_position = nn.Parameter( + torch.randn(1, config.max_suffix_tokens, config.d_model) * 0.01 + ) + self.layers = nn.ModuleList( + DualExpertBlock(config) for _ in range(config.n_layers) + ) + self.prefix_norm = RMSNorm(config.d_model, config.rms_eps) + self.suffix_norm = RMSNorm(config.d_model, config.rms_eps) + self.vector_field_head = nn.Linear(config.d_model, config.action_dim) + + def encode_prefix(self, pixel_values: Tensor, input_ids: Tensor) -> Tensor: + if pixel_values.ndim != 4: + raise ValueError("pixel_values must have shape [batch, channels, height, width]") + if input_ids.ndim != 2: + raise ValueError("input_ids must have shape [batch, text_length]") + patches = self.patch_encoder(pixel_values).flatten(2).transpose(1, 2) + language = self.token_embedding(input_ids) + prefix = torch.cat((patches, language), dim=1) + if prefix.shape[1] > self.config.max_prefix_tokens: + raise ValueError("prefix exceeds configured maximum length") + return prefix + self.prefix_position[:, : prefix.shape[1]] + + def encode_suffix( + self, states: Tensor, noisy_actions: Tensor, timesteps: Tensor + ) -> Tensor: + if states.ndim != 2: + raise ValueError("states must have shape [batch, state_dim]") + state_token = self.state_projection(states)[:, None, :] + action_tokens = self.action_time_projection(noisy_actions, timesteps) + suffix = torch.cat((state_token, action_tokens), dim=1) + return suffix + self.suffix_position[:, : suffix.shape[1]] + + def _masks( + self, + batch: int, + prefix_length: int, + suffix_length: int, + image_tokens: int, + attention_mask: Optional[Tensor], + device: torch.device, + ) -> tuple[Tensor, Tensor]: + prefix_valid = torch.ones( + batch, prefix_length, dtype=torch.bool, device=device + ) + if attention_mask is not None: + expected = prefix_length - image_tokens + if attention_mask.shape != (batch, expected): + raise ValueError( + "attention_mask must have shape [batch, input_ids length]" + ) + prefix_valid[:, image_tokens:] = attention_mask.bool() + + prefix_edges = prefix_valid[:, None, :].expand( + batch, prefix_length, prefix_length + ).clone() + if self.config.prefix_causal: + causal = torch.ones( + prefix_length, prefix_length, dtype=torch.bool, device=device + ).tril() + prefix_edges &= causal[None, :, :] + + # Give padded queries a self edge so no softmax row is fully masked. + query_valid = prefix_valid[:, :, None] + identity = torch.eye(prefix_length, dtype=torch.bool, device=device)[None] + prefix_edges = torch.where(query_valid, prefix_edges, identity) + + visible_prefix = prefix_valid[:, None, :].expand( + batch, suffix_length, prefix_length + ) + # pi0 has three ordered attention blocks: + # [image + language] -> prefix only + # [state] -> prefix + state + # [actions] -> prefix + state + all actions + # The state and actions share action-expert weights, but they must not + # be collapsed into one visibility block: doing so leaks noisy future + # actions into the state representation. + visible_suffix = torch.ones( + batch, suffix_length, suffix_length, dtype=torch.bool, device=device + ) + if suffix_length > 1: + visible_suffix[:, 0, 1:] = False + suffix_edges = torch.cat((visible_prefix, visible_suffix), dim=-1) + return prefix_edges, suffix_edges + + def forward( + self, + pixel_values: Tensor, + input_ids: Tensor, + states: Tensor, + noisy_actions: Tensor, + timesteps: Tensor, + attention_mask: Optional[Tensor] = None, + target_velocity: Optional[Tensor] = None, + ) -> Pi0Output: + """Predict the conditional-flow vector field for an action chunk. + + Args: + pixel_values: Images with shape ``[B, C, H, W]``. + input_ids: Language prompt IDs with shape ``[B, L]``. + states: Continuous proprioception with shape ``[B, state_dim]``. + noisy_actions: Flow state with shape ``[B, horizon, action_dim]``. + timesteps: One scalar flow time per batch item. + attention_mask: Optional validity mask for language prompt tokens. + target_velocity: Optional flow-matching target, shaped like actions. + """ + if noisy_actions.ndim != 3: + raise ValueError( + "noisy_actions must have shape [batch, horizon, action_dim]" + ) + batch, horizon, action_dim = noisy_actions.shape + if action_dim != self.config.action_dim: + raise ValueError("noisy_actions has the wrong action dimension") + if horizon > self.config.max_action_tokens: + raise ValueError("action horizon exceeds configured maximum") + if states.shape != (batch, self.config.state_dim): + raise ValueError("states has the wrong batch or state dimension") + if timesteps.numel() != batch: + raise ValueError("timesteps must contain one scalar per batch item") + + prefix = self.encode_prefix(pixel_values, input_ids) + suffix = self.encode_suffix(states, noisy_actions, timesteps) + image_tokens = prefix.shape[1] - input_ids.shape[1] + prefix_edges, suffix_edges = self._masks( + batch, + prefix.shape[1], + suffix.shape[1], + image_tokens, + attention_mask, + prefix.device, + ) + for layer in self.layers: + prefix, suffix = layer(prefix, suffix, prefix_edges, suffix_edges) + + prefix_hidden = self.prefix_norm(prefix) + suffix_hidden = self.suffix_norm(suffix) + state_hidden = suffix_hidden[:, :1] + action_hidden = suffix_hidden[:, 1:] + vector_field = self.vector_field_head(action_hidden) + + flow_loss = None + if target_velocity is not None: + if target_velocity.shape != noisy_actions.shape: + raise ValueError("target_velocity must have the shape of noisy_actions") + flow_loss = F.mse_loss(vector_field, target_velocity) + + return Pi0Output( + vector_field=vector_field, + prefix_hidden=prefix_hidden, + suffix_hidden=suffix_hidden, + state_hidden=state_hidden, + action_hidden=action_hidden, + loss=flow_loss, + flow_loss=flow_loss, + ) + + +def flow_matching_sample(actions: Tensor) -> tuple[Tensor, Tensor, Tensor]: + """Sample the straight conditional-flow path used by pi-style policies. + + ``x_t = (1 - t) * noise + t * action`` and the target vector field is the + constant ``action - noise``. Sampling starts from noise at ``t=0`` and + integrates toward an action at ``t=1``. + """ + if actions.ndim < 2: + raise ValueError("actions must include batch and feature dimensions") + batch = actions.shape[0] + # In the paper's noise-time convention, t_noise is sampled as + # 0.001 + 0.999 * Beta(1.5, 1). This implementation integrates in the + # opposite, action-time direction, so tau = 1 - t_noise. + uniform = torch.rand(batch, device=actions.device, dtype=actions.dtype) + timesteps = 0.999 * (1.0 - uniform.pow(1.0 / 1.5)) + noise = torch.randn_like(actions) + broadcast_time = timesteps.reshape(batch, *([1] * (actions.ndim - 1))) + noisy_actions = (1 - broadcast_time) * noise + broadcast_time * actions + target_velocity = actions - noise + return noisy_actions, timesteps, target_velocity diff --git a/src/pi0mini/runtime.py b/src/pi0mini/runtime.py new file mode 100644 index 0000000..167255e --- /dev/null +++ b/src/pi0mini/runtime.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import json +import random +from pathlib import Path +import numpy as np +import torch +import yaml + + +def load_config(path: str | Path) -> dict: + with Path(path).open("r", encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +def seed_everything(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + +def choose_device(requested: str = "auto") -> torch.device: + if requested != "auto": + return torch.device(requested) + if torch.cuda.is_available(): + return torch.device("cuda") + if torch.backends.mps.is_available(): + return torch.device("mps") + return torch.device("cpu") + + +def write_json(path: str | Path, value: object) -> None: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(value, indent=2, ensure_ascii=False), encoding="utf-8") diff --git a/src/pi0mini/train.py b/src/pi0mini/train.py new file mode 100644 index 0000000..461782e --- /dev/null +++ b/src/pi0mini/train.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import torch +from torch.utils.data import DataLoader + +from .data import Pi0OverfitDataset, materialize_subset +from .flow import FlowMatching +from .model import Pi0Config, Pi0Mini +from .runtime import choose_device, load_config, seed_everything, write_json + + +def _collate(rows: list[dict]) -> dict: + keys = ("image", "state", "actions") + return {key: torch.stack([row[key] for row in rows]) for key in keys} + + +def train(config_path: str) -> Path: + cfg = load_config(config_path) + root = Path(__file__).resolve().parents[2] + seed_everything(cfg["seed"]) + device = choose_device(cfg.get("device", "auto")) + manifest = materialize_subset( + cfg["data"]["source_manifest"], root / "data/overfit", max_samples=1 + ) + dataset = Pi0OverfitDataset( + manifest, image_size=cfg["data"]["image_size"], + augmentation_factor=cfg["data"]["num_variants"], + action_horizon=cfg["data"]["action_horizon"] + ) + loader = DataLoader(dataset, batch_size=len(dataset), shuffle=True, collate_fn=_collate) + mc = cfg["model"] + model = Pi0Mini(Pi0Config( + vocab_size=16, action_dim=2, state_dim=dataset.state_dim, + image_size=cfg["data"]["image_size"], patch_size=mc["patch_size"], + d_model=mc["d_model"], n_heads=mc["n_heads"], n_layers=mc["n_layers"], + mlp_ratio=mc["mlp_ratio"], max_text_tokens=8, + max_action_tokens=cfg["data"]["action_horizon"], dropout=mc["dropout"] + )).to(device) + optimizer = torch.optim.AdamW( + model.parameters(), lr=cfg["train"]["learning_rate"], + weight_decay=cfg["train"]["weight_decay"] + ) + flow = FlowMatching(cfg["inference"]["flow_steps"]) + iterator = iter(loader) + history = [] + model.train() + for step in range(1, cfg["train"]["steps"] + 1): + try: + batch = next(iterator) + except StopIteration: + iterator = iter(loader) + batch = next(iterator) + image, state, actions = ( + batch["image"].to(device), batch["state"].to(device), + batch["actions"].to(device) + ) + # A tiny fixed language instruction stands in for the PaliGemma prompt. + input_ids = torch.tensor([[1, 2]], device=device).expand(image.shape[0], -1) + noisy, timestep, target_velocity = flow.training_batch(actions) + output = model( + image, input_ids, state, noisy, timestep, + target_velocity=target_velocity + ) + optimizer.zero_grad(set_to_none=True) + output.loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), cfg["train"]["grad_clip_norm"]) + optimizer.step() + if step == 1 or step % cfg["train"]["log_every"] == 0: + record = {"step": step, "flow_loss": float(output.loss.detach().cpu())} + history.append(record) + print(json.dumps(record)) + output_dir = root / cfg["output_dir"] + output_dir.mkdir(parents=True, exist_ok=True) + torch.save({ + "model": model.state_dict(), "model_config": model.config.__dict__, + "config": cfg, "action_statistics": dataset.action_normalizer.as_dict() + }, output_dir / "checkpoint.pt") + write_json(output_dir / "training_history.json", history) + write_json(output_dir / "summary.json", { + "device": str(device), "samples": len(dataset), + "parameters": sum(p.numel() for p in model.parameters()), + "initial": history[0], "final": history[-1] + }) + return output_dir / "checkpoint.pt" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", default="configs/overfit.yaml") + args = parser.parse_args() + train(args.config) + + +if __name__ == "__main__": + main() diff --git a/tests/test_data.py b/tests/test_data.py new file mode 100644 index 0000000..8bd0575 --- /dev/null +++ b/tests/test_data.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import torch +from PIL import Image + +from pi0mini.data import ActionNormalizer, Pi0OverfitDataset, load_manifest, materialize_subset + + +def _make_source_project(root: Path) -> Path: + processed = root / "data" / "processed" + image_dir = processed / "images" + trajectory_dir = processed / "trajectories" + image_dir.mkdir(parents=True) + trajectory_dir.mkdir(parents=True) + + pixels = np.zeros((48, 80, 3), dtype=np.uint8) + pixels[..., 0] = 64 + pixels[..., 1] = np.arange(80, dtype=np.uint8)[None, :] + Image.fromarray(pixels).save(image_dir / "sample.jpg") + + history_x = np.linspace(-7.0, 0.0, 8, dtype=np.float32) + history = np.stack((history_x, 0.02 * history_x, np.zeros(8)), axis=-1) + future_x = np.linspace(1.0, 32.0, 16, dtype=np.float32) + future = np.stack((future_x, 0.002 * future_x**2, np.zeros(16)), axis=-1) + np.savez_compressed( + trajectory_dir / "sample.npz", history_xyz=history, future_xyz=future + ) + + manifest = processed / "overfit_manifest.jsonl" + manifest.write_text( + json.dumps( + { + "clip_id": "clip-a", + "t0_us": 1234, + "image_path": "data/processed/images/sample.jpg", + "trajectory_path": "data/processed/trajectories/sample.npz", + } + ) + + "\n", + encoding="utf-8", + ) + return manifest + + +def test_action_normalizer_is_reversible() -> None: + history = torch.tensor([[-1.0, 0.0], [0.0, 0.1]]) + future = torch.tensor([[1.0, 0.2], [2.5, 0.4], [4.0, 0.7]]) + normalizer = ActionNormalizer.fit([history], [future]) + actions = normalizer.encode(history, future) + + assert actions.shape == (3, 2) + assert torch.allclose(normalizer.decode(actions, history), future, atol=1e-6) + restored = ActionNormalizer.from_dict(normalizer.as_dict()) + assert torch.allclose(restored.decode(actions, history), future, atol=1e-6) + + +def test_materialized_dataset_contract_and_determinism(tmp_path: Path) -> None: + source_manifest = _make_source_project(tmp_path / "source") + copied_manifest = materialize_subset(source_manifest, tmp_path / "target") + records = load_manifest(copied_manifest) + + assert len(records) == 1 + assert records[0].image_path.is_file() + assert records[0].trajectory_path.is_file() + + dataset = Pi0OverfitDataset(copied_manifest, seed=19) + repeated = Pi0OverfitDataset( + copied_manifest, + seed=19, + action_statistics=dataset.action_normalizer.as_dict(), + ) + assert len(dataset) == 4 + assert dataset.state_dim == 16 + + original = dataset[0] + augmented = dataset[1] + repeated_augmented = repeated[1] + assert set(original) == { + "image", + "state", + "actions", + "history_xy", + "future_xy", + "sample_id", + } + assert original["image"].shape == (3, 64, 64) + assert original["state"].shape == (16,) + assert original["actions"].shape == (16, 2) + assert original["history_xy"].shape == (8, 2) + assert original["future_xy"].shape == (16, 2) + assert original["sample_id"].endswith(":aug0") + assert augmented["sample_id"].endswith(":aug1") + assert not torch.equal(original["future_xy"], augmented["future_xy"]) + assert torch.equal(augmented["image"], repeated_augmented["image"]) + assert torch.equal(augmented["state"], repeated_augmented["state"]) + assert torch.equal(augmented["actions"], repeated_augmented["actions"]) + + decoded = dataset.action_normalizer.decode( + original["actions"], original["history_xy"] + ) + assert torch.allclose(decoded, original["future_xy"], atol=1e-5) + + +def test_statistics_can_be_saved(tmp_path: Path) -> None: + source_manifest = _make_source_project(tmp_path / "source") + manifest = materialize_subset(source_manifest, tmp_path / "target") + dataset = Pi0OverfitDataset(manifest) + path = dataset.save_action_statistics(tmp_path / "stats.json") + values = json.loads(path.read_text(encoding="utf-8")) + + assert len(values["mean"]) == 2 + assert len(values["scale"]) == 2 + assert min(values["scale"]) >= 0.05 diff --git a/tests/test_flow.py b/tests/test_flow.py new file mode 100644 index 0000000..36ebb7e --- /dev/null +++ b/tests/test_flow.py @@ -0,0 +1,65 @@ +import torch + +from pi0mini.flow import FlowMatching + + +def _time_view(t: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + return t.reshape(t.shape[0], *([1] * (target.ndim - 1))) + + +def test_training_batch_obeys_linear_path() -> None: + flow = FlowMatching() + action = torch.tensor([[[1.0, -2.0], [0.5, 3.0]], [[-1.0, 0.0], [2.5, -0.5]]]) + x_t, t, velocity = flow.training_batch( + action, generator=torch.Generator().manual_seed(123) + ) + + noise = action - velocity + expected = (1.0 - _time_view(t, action)) * noise + _time_view(t, action) * action + torch.testing.assert_close(x_t, expected) + assert x_t.shape == action.shape + assert torch.all((t >= 0.0) & (t <= 1.0)) + + +def test_training_batch_is_reproducible() -> None: + action = torch.randn(3, 16, 2) + flow = FlowMatching() + a = flow.training_batch(action, generator=torch.Generator().manual_seed(9)) + b = flow.training_batch(action, generator=torch.Generator().manual_seed(9)) + for left, right in zip(a, b): + torch.testing.assert_close(left, right) + + +def test_training_times_use_low_biased_shifted_beta() -> None: + flow = FlowMatching() + times = flow.sample_training_times( + 100_000, + device="cpu", + dtype=torch.float32, + generator=torch.Generator().manual_seed(17), + ) + + assert torch.all(times >= 0.0) + assert torch.all(times <= 0.999) + # E[0.999 * (1 - Beta(1.5, 1))] = 0.3996. This also guards + # against accidentally reverting to a uniform or high-time-biased draw. + assert abs(float(times.mean()) - 0.3996) < 0.005 + + +def test_loss_is_velocity_mse() -> None: + flow = FlowMatching() + target = torch.tensor([[[1.0, -1.0], [0.0, 2.0]]]) + prediction = target + torch.tensor([[[1.0, 1.0], [2.0, 0.0]]]) + torch.testing.assert_close(flow.loss(prediction, target), torch.tensor(1.5)) + + +def test_exact_vector_field_integrates_to_action() -> None: + flow = FlowMatching(inference_steps=10) + action = torch.tensor([[[0.25, -0.5], [1.0, 2.0]], [[-2.0, 0.75], [0.0, 0.5]]]) + + def exact_field(x_t: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + return (action.to(x_t) - x_t) / (1.0 - _time_view(t, x_t)) + + sample = flow.sample(exact_field, tuple(action.shape), action.device, + generator=torch.Generator().manual_seed(5)) + torch.testing.assert_close(sample, action, atol=1e-5, rtol=1e-5) diff --git a/tests/test_model.py b/tests/test_model.py new file mode 100644 index 0000000..5a14493 --- /dev/null +++ b/tests/test_model.py @@ -0,0 +1,65 @@ +import torch + +from pi0mini.model import Pi0Config, Pi0Mini + + +def test_pi0_three_block_attention_mask() -> None: + config = Pi0Config( + image_size=32, + patch_size=16, + max_text_tokens=4, + max_action_tokens=3, + d_model=32, + n_heads=4, + n_layers=1, + time_feature_dim=8, + ) + model = Pi0Mini(config) + + # Two image patches per axis -> 4 image tokens, plus 2 language tokens. + prefix_edges, suffix_edges = model._masks( + batch=1, + prefix_length=6, + suffix_length=4, # one state + three action tokens + image_tokens=4, + attention_mask=torch.tensor([[1, 1]]), + device=torch.device("cpu"), + ) + + assert prefix_edges.shape == (1, 6, 6) + assert torch.all(prefix_edges) + + # Suffix key order is [six prefix keys, state, action0, action1, action2]. + assert suffix_edges.shape == (1, 4, 10) + assert torch.all(suffix_edges[0, 0, :7]) + assert not torch.any(suffix_edges[0, 0, 7:]) + assert torch.all(suffix_edges[0, 1:, :]) + + +def test_pi0_mask_respects_padded_language_keys() -> None: + model = Pi0Mini( + Pi0Config( + image_size=16, + patch_size=16, + max_text_tokens=3, + max_action_tokens=1, + d_model=16, + n_heads=4, + n_layers=1, + time_feature_dim=8, + ) + ) + prefix_edges, suffix_edges = model._masks( + batch=1, + prefix_length=4, + suffix_length=2, + image_tokens=1, + attention_mask=torch.tensor([[1, 0, 0]]), + device=torch.device("cpu"), + ) + + # Padded prefix keys are hidden from valid prefix, state, and action rows. + assert not prefix_edges[0, 0, 2] + assert not prefix_edges[0, 0, 3] + assert not suffix_edges[0, :, 2].any() + assert not suffix_edges[0, :, 3].any()