From ffbcca8d2b6d3e2edb908c332ec95632e2fd028e Mon Sep 17 00:00:00 2001 From: seeker Date: Sat, 11 Jul 2026 20:04:18 +0800 Subject: [PATCH] Initial pi0.5 minimal validation --- .gitignore | 9 + README.md | 141 ++++++++ configs/overfit.yaml | 32 ++ outputs/overfit/evaluation.json | 34 ++ outputs/overfit/summary.json | 19 ++ outputs/overfit/training_history.json | 296 ++++++++++++++++ outputs/overfit/trajectory_comparison.png | Bin 0 -> 89369 bytes pyproject.toml | 38 +++ src/pi05mini/__init__.py | 5 + src/pi05mini/data.py | 316 +++++++++++++++++ src/pi05mini/flow.py | 54 +++ src/pi05mini/infer.py | 129 +++++++ src/pi05mini/model.py | 396 ++++++++++++++++++++++ src/pi05mini/runtime.py | 36 ++ src/pi05mini/tokenizer.py | 285 ++++++++++++++++ src/pi05mini/train.py | 193 +++++++++++ tests/test_action_tokens.py | 104 ++++++ tests/test_data.py | 121 +++++++ tests/test_flow.py | 85 +++++ tests/test_model.py | 61 ++++ 20 files changed, 2354 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 configs/overfit.yaml create mode 100644 outputs/overfit/evaluation.json create mode 100644 outputs/overfit/summary.json create mode 100644 outputs/overfit/training_history.json create mode 100644 outputs/overfit/trajectory_comparison.png create mode 100644 pyproject.toml create mode 100644 src/pi05mini/__init__.py create mode 100644 src/pi05mini/data.py create mode 100644 src/pi05mini/flow.py create mode 100644 src/pi05mini/infer.py create mode 100644 src/pi05mini/model.py create mode 100644 src/pi05mini/runtime.py create mode 100644 src/pi05mini/tokenizer.py create mode 100644 src/pi05mini/train.py create mode 100644 tests/test_action_tokens.py create mode 100644 tests/test_data.py create mode 100644 tests/test_flow.py create mode 100644 tests/test_model.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff92181 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +*.egg-info/ +.venv/ +data/ +outputs/**/*.pt +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..b40f44c --- /dev/null +++ b/README.md @@ -0,0 +1,141 @@ +# π0.5 Minimal Validation + +这是一个 **π0.5 风格的最小、可运行架构验证项目**。它从相邻 Alpamayo 验证项目准备的 PAI 子集中读取 1 个驾驶样本，生成 4 个确定性视觉增强变体做过拟合，目标是检查高层语义预测、blockwise attention、action expert 与 flow matching action chunk 能否组成完整训练和推理链路。 + +它不是 Physical Intelligence 官方代码，也不声称复现论文中的规模、数据或机器人能力。 + +## 验证内容 + +```text +图像 + 本体状态 + │ + ▼ +小型视觉/语言主干 ──→ 高层语义 subtask + │ │ + └──── semantic prefix┘ + │ + noisy action + time │ + └──────────▼ + action expert（双向 action attention） + │ + ▼ + flow velocity → action chunk +``` + +默认任务使用 `64×64` 前视相机图像、离散化历史 XY 状态和一句 subtask，预测未来 `16×2` 的二维轨迹。样本少且重复训练，因此成功标准是“能够记忆训练集”，而不是泛化。 + +## 与 π0.5 的对应关系 + +保留的关键思想： + +- 视觉/语言语义主干与独立 action expert； +- observation/text 与 action token 采用 blockwise attention 信息流； +- action token 内部采用双向 attention，一次预测整个 action chunk； +- flow matching 训练连续动作，推理时从高斯噪声做 Euler 积分； +- timestep 通过 adaptive RMSNorm 调制 action expert； +- 同时优化高层语义目标和低层连续动作目标。 +- VLM 自回归输出高层 subtask 以及 64-bin 离散动作 token； +- 推理时分别还原 VLM token 轨迹和 flow/diffusion 轨迹，与真值三方比较。 + +为了单机快速验证而简化的部分： + +- 小型自定义视觉编码器和 Transformer 取代 PaliGemma 2B； +- 隐藏宽度、层数、图像尺寸和 action horizon 大幅缩小； +- 使用固定小词表的 subtask 分类/生成，而非开放式语言理解； +- 使用二维驾驶轨迹，取代多机器人、多相机和高维关节控制； +- 默认使用 1 个 PAI 驾驶样本的 4 个视觉增强版本，没有机器人或网页联合训练数据； +- 使用正交 DCT-II、低频优先排列和 64-bin 量化实现 FAST-lite；保留全部频率系数，因此只验证表示与训练接口，不复现官方压缩率； +- Euler sampler 用固定步长，不包含面向部署的实时控制与安全系统。 + +因此，本项目适合验证架构接口和数学链路，不适合比较论文指标。 + +## 安装 + +建议使用 Python 3.10 或更新版本： + +```bash +cd pi05-minimal-validation +python -m venv .venv +source .venv/bin/activate +python -m pip install -U pip +python -m pip install -e '.[dev]' +``` + +默认实现只依赖 PyTorch、NumPy、Pillow、Matplotlib 和 PyYAML，不需要下载预训练权重。默认配置会读取： + +```text +../../alpamayo/alpamayo-qwen05b-validation/data/processed/overfit_manifest.jsonl +``` + +如果该路径不同，请先修改 `configs/overfit.yaml` 的 `data.source_manifest`。 + +## 运行测试 + +```bash +cd pi05-minimal-validation +pytest -q +``` + +`tests/test_flow.py` 单独验证： + +- 直线路径 `x_t=(1-t)·noise+t·action`； +- 目标速度 `v=action-noise`； +- velocity MSE； +- 固定目标的精确 vector field 经 Euler 积分后能到达目标； +- 使用相同 `torch.Generator` 时采样可复现。 + +## 过拟合训练 + +```bash +cd pi05-minimal-validation +pi05mini-train --config configs/overfit.yaml +``` + +也可以直接使用模块入口： + +```bash +python -m pi05mini.train --config configs/overfit.yaml +``` + +配置默认先进行 300 步离散 VLM/FAST-lite 预训练，再进行 500 步联合 post-training。Stage 1 不更新 flow action expert；Stage 2 联合优化 token CE 与 flow loss。CPU 可以运行；Apple Silicon 可自动选择 MPS，CUDA 可自动选择 GPU。 + +## 推理与效果图 + +```bash +pi05mini-infer --checkpoint outputs/overfit/checkpoint.pt +``` + +输出图 `outputs/overfit/trajectory_comparison.png` 会叠加 ground truth、VLM 离散动作 token 解码轨迹和 diffusion/flow 轨迹；逐样本指标写入 `evaluation.json`。训练过程另存为 `training_history.json`，训练摘要为 `summary.json`。 + +## 配置重点 + +`configs/overfit.yaml` 中最常调整的参数： + +- `data.num_variants`：默认 4，刻意保持很小； +- `model.d_model`、`n_layers`：控制最小模型容量； +- `train.steps`：过拟合不充分时增加； +- `train.semantic_loss_weight` 与 `flow_loss_weight`：两个训练目标的权重； +- `inference.flow_steps`：Euler 积分步数，越大通常越慢且更平滑。 + +## 安全限制 + +- 该模型只在合成样本上验证记忆能力，输出不具备现实世界有效性。 +- 不得用于车辆、机器人、机械臂或任何会造成物理影响的设备控制。 +- 不得把 loss 下降或单样本轨迹重合解释为安全性、鲁棒性或泛化能力。 +- 真实部署至少还需要独立验证集、分布外和扰动测试、控制约束、碰撞检查、故障降级、急停以及人工监督。 +- checkpoint 和配置可能加载本地路径；只运行可信来源的文件，不加载未知 PyTorch pickle checkpoint。 + +## 项目结构 + +```text +pi05-minimal-validation/ +├── configs/overfit.yaml # 4 样本过拟合配置 +├── src/pi05mini/ # 模型、flow、数据、训练与推理 +├── tests/test_flow.py # flow matching 数学测试 +├── pyproject.toml +└── README.md +``` + +## 预期结论 + +若最小实验成功，它只能说明：π0.5 风格的语义主干、blockwise 双专家交互和 flow action head 在此实现中能够端到端反向传播并记忆几个样本。它不能证明实现与官方模型数值等价，也不能证明在真实驾驶或机器人任务中有效。 diff --git a/configs/overfit.yaml b/configs/overfit.yaml new file mode 100644 index 0000000..7abdc7c --- /dev/null +++ b/configs/overfit.yaml @@ -0,0 +1,32 @@ +seed: 7 +device: auto +output_dir: outputs/overfit + +data: + source_manifest: ../../alpamayo/alpamayo-qwen05b-validation/data/processed/overfit_manifest.jsonl + num_variants: 4 + image_size: 64 + action_horizon: 16 + +model: + d_model: 96 + n_heads: 4 + n_layers: 3 + mlp_ratio: 3 + patch_size: 16 + max_text_tokens: 64 + dropout: 0.0 + +train: + pretrain_steps: 300 + posttrain_steps: 500 + batch_size: 4 + learning_rate: 0.001 + weight_decay: 0.0 + grad_clip_norm: 1.0 + semantic_loss_weight: 1.0 + flow_loss_weight: 2.0 + log_every: 20 + +inference: + flow_steps: 10 diff --git a/outputs/overfit/evaluation.json b/outputs/overfit/evaluation.json new file mode 100644 index 0000000..efb31eb --- /dev/null +++ b/outputs/overfit/evaluation.json @@ -0,0 +1,34 @@ +[ + { + "sample_id": "314f59c2-e420-4fcd-9df2-a51d572e0c20_1503899:aug0", + "target_subtask": "steer_left", + "predicted_subtask": "steer_left", + "vlm_action_tokens_valid": true, + "vlm_trajectory_mse": 0.07854792475700378, + "diffusion_trajectory_mse": 0.27604275941848755 + }, + { + "sample_id": "314f59c2-e420-4fcd-9df2-a51d572e0c20_1503899:aug1", + "target_subtask": "steer_left", + "predicted_subtask": "steer_left", + "vlm_action_tokens_valid": true, + "vlm_trajectory_mse": 0.025470610707998276, + "diffusion_trajectory_mse": 0.20049458742141724 + }, + { + "sample_id": "314f59c2-e420-4fcd-9df2-a51d572e0c20_1503899:aug2", + "target_subtask": "steer_left", + "predicted_subtask": "steer_left", + "vlm_action_tokens_valid": true, + "vlm_trajectory_mse": 0.013774838298559189, + "diffusion_trajectory_mse": 0.03129641339182854 + }, + { + "sample_id": "314f59c2-e420-4fcd-9df2-a51d572e0c20_1503899:aug3", + "target_subtask": "steer_left", + "predicted_subtask": "steer_left", + "vlm_action_tokens_valid": true, + "vlm_trajectory_mse": 0.059195056557655334, + "diffusion_trajectory_mse": 0.11321215331554413 + } +] \ No newline at end of file diff --git a/outputs/overfit/summary.json b/outputs/overfit/summary.json new file mode 100644 index 0000000..887b413 --- /dev/null +++ b/outputs/overfit/summary.json @@ -0,0 +1,19 @@ +{ + "device": "mps", + "samples": 4, + "parameters": 816482, + "initial": { + "stage": "pretrain", + "step": 1, + "loss": 78.87449645996094, + "text_loss": 78.87449645996094, + "flow_loss": null + }, + "final": { + "stage": "posttrain", + "step": 500, + "loss": 0.12353827804327011, + "text_loss": 3.1691649837739533e-06, + "flow_loss": 0.06176755577325821 + } +} \ No newline at end of file diff --git a/outputs/overfit/training_history.json b/outputs/overfit/training_history.json new file mode 100644 index 0000000..fdb3275 --- /dev/null +++ b/outputs/overfit/training_history.json @@ -0,0 +1,296 @@ +[ + { + "stage": "pretrain", + "step": 1, + "loss": 78.87449645996094, + "text_loss": 78.87449645996094, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 20, + "loss": 2.475086212158203, + "text_loss": 2.475086212158203, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 40, + "loss": 1.1025534868240356, + "text_loss": 1.1025534868240356, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 60, + "loss": 0.601564884185791, + "text_loss": 0.601564884185791, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 80, + "loss": 0.25571808218955994, + "text_loss": 0.25571808218955994, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 100, + "loss": 0.03314117714762688, + "text_loss": 0.03314117714762688, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 120, + "loss": 0.005310459993779659, + "text_loss": 0.005310459993779659, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 140, + "loss": 3.0251239877543412e-05, + "text_loss": 3.0251239877543412e-05, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 160, + "loss": 1.1354763955750968e-05, + "text_loss": 1.1354763955750968e-05, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 180, + "loss": 6.638075319642667e-06, + "text_loss": 6.638075319642667e-06, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 200, + "loss": 5.348983449948719e-06, + "text_loss": 5.348983449948719e-06, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 220, + "loss": 4.736786195280729e-06, + "text_loss": 4.736786195280729e-06, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 240, + "loss": 4.316162630857434e-06, + "text_loss": 4.316162630857434e-06, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 260, + "loss": 3.975574600190157e-06, + "text_loss": 3.975574600190157e-06, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 280, + "loss": 3.6877775073662633e-06, + "text_loss": 3.6877775073662633e-06, + "flow_loss": null + }, + { + "stage": "pretrain", + "step": 300, + "loss": 3.4408508327032905e-06, + "text_loss": 3.4408508327032905e-06, + "flow_loss": null + }, + { + "stage": "posttrain", + "step": 1, + "loss": 8.707193374633789, + "text_loss": 3.428078571232618e-06, + "flow_loss": 4.353594779968262 + }, + { + "stage": "posttrain", + "step": 20, + "loss": 3.533726453781128, + "text_loss": 0.14151602983474731, + "flow_loss": 1.6961052417755127 + }, + { + "stage": "posttrain", + "step": 40, + "loss": 3.0850143432617188, + "text_loss": 0.00014649856893811375, + "flow_loss": 1.5424339771270752 + }, + { + "stage": "posttrain", + "step": 60, + "loss": 1.3556954860687256, + "text_loss": 0.00011281041224719957, + "flow_loss": 0.6777913570404053 + }, + { + "stage": "posttrain", + "step": 80, + "loss": 1.4165724515914917, + "text_loss": 6.599419066333212e-06, + "flow_loss": 0.7082829475402832 + }, + { + "stage": "posttrain", + "step": 100, + "loss": 0.8609042167663574, + "text_loss": 8.11900736152893e-06, + "flow_loss": 0.430448055267334 + }, + { + "stage": "posttrain", + "step": 120, + "loss": 0.5642163753509521, + "text_loss": 7.576879397674929e-06, + "flow_loss": 0.28210440278053284 + }, + { + "stage": "posttrain", + "step": 140, + "loss": 0.7110214233398438, + "text_loss": 6.249698344618082e-06, + "flow_loss": 0.35550758242607117 + }, + { + "stage": "posttrain", + "step": 160, + "loss": 0.18725578486919403, + "text_loss": 2.876289045161684e-06, + "flow_loss": 0.09362645447254181 + }, + { + "stage": "posttrain", + "step": 180, + "loss": 0.33923929929733276, + "text_loss": 6.748496616637567e-06, + "flow_loss": 0.16961628198623657 + }, + { + "stage": "posttrain", + "step": 200, + "loss": 0.3003162443637848, + "text_loss": 9.495882295595948e-06, + "flow_loss": 0.15015336871147156 + }, + { + "stage": "posttrain", + "step": 220, + "loss": 0.23716998100280762, + "text_loss": 5.660519946104614e-06, + "flow_loss": 0.11858215928077698 + }, + { + "stage": "posttrain", + "step": 240, + "loss": 0.42717865109443665, + "text_loss": 8.400202204938978e-06, + "flow_loss": 0.21358512341976166 + }, + { + "stage": "posttrain", + "step": 260, + "loss": 0.11702287942171097, + "text_loss": 4.122769951209193e-06, + "flow_loss": 0.058509379625320435 + }, + { + "stage": "posttrain", + "step": 280, + "loss": 0.19245445728302002, + "text_loss": 5.941245035501197e-06, + "flow_loss": 0.09622425585985184 + }, + { + "stage": "posttrain", + "step": 300, + "loss": 0.18475297093391418, + "text_loss": 4.744139005197212e-06, + "flow_loss": 0.09237411618232727 + }, + { + "stage": "posttrain", + "step": 320, + "loss": 0.38481318950653076, + "text_loss": 3.1461640901397914e-06, + "flow_loss": 0.19240501523017883 + }, + { + "stage": "posttrain", + "step": 340, + "loss": 0.2163764089345932, + "text_loss": 4.1934181354008615e-06, + "flow_loss": 0.10818611085414886 + }, + { + "stage": "posttrain", + "step": 360, + "loss": 0.23533868789672852, + "text_loss": 3.3019753118423978e-06, + "flow_loss": 0.11766768991947174 + }, + { + "stage": "posttrain", + "step": 380, + "loss": 0.08397223055362701, + "text_loss": 8.292082384286914e-06, + "flow_loss": 0.04198196902871132 + }, + { + "stage": "posttrain", + "step": 400, + "loss": 0.13249839842319489, + "text_loss": 1.0871653103095014e-05, + "flow_loss": 0.06624376028776169 + }, + { + "stage": "posttrain", + "step": 420, + "loss": 0.4479130506515503, + "text_loss": 1.0128642315976322e-05, + "flow_loss": 0.22395145893096924 + }, + { + "stage": "posttrain", + "step": 440, + "loss": 0.22416232526302338, + "text_loss": 2.6114355478057405e-06, + "flow_loss": 0.11207985877990723 + }, + { + "stage": "posttrain", + "step": 460, + "loss": 0.06346385926008224, + "text_loss": 3.1035808660817565e-06, + "flow_loss": 0.031730376183986664 + }, + { + "stage": "posttrain", + "step": 480, + "loss": 0.385486900806427, + "text_loss": 2.184032382501755e-06, + "flow_loss": 0.19274236261844635 + }, + { + "stage": "posttrain", + "step": 500, + "loss": 0.12353827804327011, + "text_loss": 3.1691649837739533e-06, + "flow_loss": 0.06176755577325821 + } +] \ No newline at end of file diff --git a/outputs/overfit/trajectory_comparison.png b/outputs/overfit/trajectory_comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..dde41eaef5d18831cbd3dcb98a503d72536e3c27 GIT binary patch literal 89369 zcmeFZc{tSl`#wA>cSX@Al|o8FD6)*bBwA2OvR8IOb~Dx%6lKX42FX_0_nk_GFq3uc zlWhir!PwX5db_{B&*%Pre&6GG{&<$-`QvdM_i^9m9y9Ov>vg@Z>%7kMyxy;FsjKW| zI>v-Tq4r)=y>bVILjH4`&#u4VU$>Y~n!~@Oos{o8Y1)}Nxju9-MX5b>dh*!L>9OS_ zZWmJrM@u_fG2sgrgwLPjws3NK;wUX5V)L(e2-`WBi$v6>*ukglexj=5h(aBSMgC#9 z=KMSy#ehOxyK?ES8*ZX+XZXR9mD(9|nVc_o4o3=~u)K3Ohk=RbY`|HGbDg)cAJKfR zOCzga>a)r4aK9h?5x~fHi$7o&k7{;H!yWd>$@1j{%8&0t^|pDj5@9q74)>zo#7s2$ zu~!AXarG+tIEz1A!9Ra)ykeP0Zt;j%+t;&!4?4>X2l;TLxv3d7lR zMRnCE)Y%^cHU6eud8S$0Di!O0{dy}#kkvdatbc#7w6XD~{3&?1!YK;GX6HdPD+=`` zz7B(8O4_N2dScv>p_9c)xudDsrE+QECx_j;!Rn76&thil`AqvOFtfE>-tY8cC2SgH zwGEsjmOVJ#Coafyp-_Ls9Ski(=_r)q`w;r&K;*Kv`Qq4hhh@JU8)evs>XmXD4M|Tc z$Kj4I+z#=iH%XE=Ch`f9eDQAICS|1$<#=5^A?>tlwgWCo@D#=OFmhcxn9w2xaA_#+ zTXmR3W&HMxr<`!29-22q3X%J}dfxMhc&_O<_9 zsE#lDzMvk9NJ?s_s>Plmd2d!e?k=c*5gs0X=#x=(70&+WQBjNE+@B09%c3R4#kKHg z%JFMAZa7v}8C827U}x7hi7v}1bEZb#CoW9(jMT6ygeQ1!=qyZhoyyJ4ZOPCP8zWU$ z>$`NBl1ZlgG|^Oz1mR$XjS`HkUBB$7dzmqf(PAp8H=}u|#B530SY~nm+o`_t8|Rl+ zf>N9lm_38TVlZ9{N;)FEQ)FYg!9i?S)Q@y2*p{9F}oL7wZ zTj%3VjN8ercpUY>I9AfmrNEkqPxPLW?Gw=YJlvUU z>>usjQ)H8Zr&f%4{E%CpYYlqwVn=W`dShwmgmp&-zq7kw-s{~5&y%ub%I?Y+Uyuc`l;;Sh)axdUI_D|4nx8C|~o?E8-&5$H@`IeSQQ`1hUqeEERd=VK#o*WWwuHaFUw_0Vhm zn*V`~ktn0aA;Ij5yf%!plzv1+#K^Jn4x@&%QvrfmArF|pufhjD@yD_2BOkTz+;GJo z$W|zy2PY|v>}b}Pvbl-<5H8_=?LGF+aJp(`T=1xjONPfnm$}1NMt0R~!^+{;V)(NM zE;djJ3Y+22o^7MMwSBxjyU=47Tg&2~^xf0YFrMn{zA!OE6%C+v<|ggX*31Wn5gTNzD~}Ar^#t)4MyJUyqn7eZ{@INdaofs?@9HpLmGE!{U$5Yq zIUP9nVa}p;)l7i)}I9EU05B=yp=QGubuueSbpWBe$P;_>5JzNs#z)Kce^QYi9E38=eMr9 ztq$|_5^~jvSMkhy{?}DETeW{Sof;Y%3cRUt{knr(f3fT|2*eap4Ic3ka~(SA-qnH@8+_>Lg{E70G%p4sN~VQ>Vnhjp~U(JErzJ z6Afl3i%Hk9SZpW-cCcyRm)0~5$;~NJwRb25lAGPL(3bdZCs_Pz=VhnAY|k3-^=COD z5G*_Ua-zF1y-v3KMqiA$^>9j*p(uUlqoV`d?`TQJwL#>KC?l`Xy&MvEpESUVMTF3w zLBfRqc#329*jmj!5Pqg3O+v@HA ze0h2xn@j4+d+Ye_LTedmmkA4Udr!Qadq88kaV?8Vv29;KM)vKbt3fW4-RL`a?i_D! zZq6F(D|dA*uVgd$3E54bz$NSUd;cymv&JWx#_KEd3>dD@4{Q9$%d1o6grTBydPR-< zIXJAzfv;ZK7g)9bnqQV(8BOwXCqPD93!$%VUHvx(G+}dVvyT7$()*mwTkPE16yMKL zi5fp&A3X)BKCUrVQiFv{{;{pPO?SbC%C+L&P>N+y@3~Gj~6631JdFFfdIdE6`wAJ)P`Lu!xaHrN9xIyt(Sbh!AXUL z`0t-D&ps$s!ZG~k&llH2sBQmz0rlY0R+?0NGC`^~f?P60r+(H-KSL)sG1p1JU%wwj zRt2?i1vy^`2tau!q4uG8uGrdUAzM!&^78UBnz+Mz{%|4TxrG`3Mu9%nNF%VH2aVQ& z^3R%J1Uctm$RvEKk2i9@P!q5I1++E}k3ezsG7^B7|M9cy%yG+WnCdZ2WgL`oM`g7l38DRAwD+iR7f_nlUwsThdPyi z4dd=kcj=0bj((U+>RX0-lVa8sLoR~7Kd?$z?p>N0k`q=6p7H1Mx(~_PDu80u_VJ-1 zeLOw8jNI%w4;c44wIh=k!cCm;a{sB*Z55sgp=Xbf+Sk@R0GZEXHx#|J<2<@JQ5{$C zff+bTFp0~YJVSP+)PTn(4b{t*=h?#If3 zi8$p;lKpPRr5q0pF(Yh5Cn2vEOG<)w6`^OyBakYU*+;i zz`;|Mi-ww~Q!BrXWkMZss6F9YhmVs{s*2Z;Urw&`nk6sH6OGCZ?p?a{^k%fEG6d z+11^PI10^XZwKiNY5)o$ylL4RyY9VtP($|Q`s!j7e#=f+1ID?I@R*=41Dlm>=2MZ@<&z!jmNR(HS+V9GzEZJy!%frII70TTpj9P5lH>;`{gSQjd&{4K}lM@>7kWaXV>1;7jlrRK0#( zfPR>bZA7ukjaq;uBfEAY7}UBvz)llER-VCJ z3jLE*b3ASgyUr|9@uv=t7rsE49ceJ2BLy?)_3J$SArzsN3KxKIse}+)hEVtoP%;vx?E*i`%Ti<&1Of#{@y7 zw9G)4A_c82#iIFxvM294s>>MYNa0%nrZzBr)6v^q1!)v(|jL7Sr=q%FcT4*Ojc5Q%1|iM>pyM zrbU=g4`p1INj$!9c6;!fFLRTuZtrZi z_qw17qYqqbTKW;eMyiKvQrCr--ay-?kHC4?N@7-;g7QI~9FfgbTw~~Z< zGX3Xks)~STZ;}dYFr9>@W}0zU<_)~!i{EvP;r+t+twJlU{C?yLo`0b^Cp2-A`!z> zRM%Ix4x`@EWlvFbHp_MOH#gP|Y_R>%x4x9!3UoG`zYC+>Kt<*}6RIJW?Y39*w$9(s zC74eY);Z4(*Qemiz1FeITU~8~NVUFzcGmF_n=*)?ruDUDx+!b`Z#)5ykNl*Wb~sw{ zkQpIfXl!hZJi{BMKWJZdL!`2;u#e=mYI8qJMS%!lA-eYYPS!}sxl!-TmuE-FReEL`i#4vpNi&6mc0d)7#%#jonhH`scJJPO=hGeDF^JBJnV%fwcfx9rjN3AF zT6=#6a~Tbn4h6~JZ*ET4ur^g|XWSo%DPEd{OO~J~r=}i`?nkzD)@x;qkLX14a-jr@ zR6S`bh7)#xz=d}BUO`filfLHRQE@j@x4pu=ub)m(Fgl|aGc0|t0+Zm&cIZ%DIKO7H zMuPm?ckj+>eZG5vYSEhdMt14XbMiVS4>}T}^qkzsO*k|ymF})|*UJjA@aJ_(ta2v( z&kd^6`=Qvp9UeBJyFsTZU)>)|v$}fKk4js9f9ZKLv_5S8wg~1CRFD~w^dVcgkICO8-1#K7BXxoX#LNqlbEJfWlOL2G>FFcc+zdLOGnbr6A;AT&IBAn`7eo z!QPweuDin_)7SH-;dnG;XI?z+%=vm;#NOUM&tc^HQF)J&Jx4{88+jBp=H$lJqS8Jj zB^k5^kP1rFl$Dhu-0%r=ryoTL8#J!2!>X%2OgKDhInWP1#|SmMT7oebKX~8BqQ(v) zcO=F-E0oJ?>Fc2o%ZG`=dVx)FEKRGT94YvDD-5gbv=9G*4ak?L&z$*r6JSDY1puDZ zFXsgXZ%iy1%==X>BKTz~Yl&W7_TcL_G$37!{+eB~MBet7$A>Fs^EL5;G+MCPn+Ka_ zGNC~DxHMumIy;_Vt5Lw>YJ0gfV5)vID^+i9xPJnk%98r8_3v(ens4I_s7JnWCP9ne?uI~*{^cC%Y5gYSId=)J)D-uYiT zojbWaxU;KPK3l4G7++Edt8`!d1r@|9o@jUyFsDre?XzTo`S;#leL4an7RL(vy|fFb zTgH;Z)ZhLHWG}}ux5b|*%+X4#UK$J}(-|H>iJsn^+|_~gSYB^7tkx@fZ0S(6Fq|&O zvN+z}uvSI2b3Tvobps0q^REDDX z%kFYm;hN>Dg7pO;rq53%$FLSwg0hfh_JSQX{Q_DDbautR_A}EzV9+^>M-u%xMd_m#ChCEQH=ycPVhd}R5dbxFKg*!JC|dr} z_g(%Xzpx1i(?MMZHKp9(Coqo_FN1=FXHN(L`m#}8n1{=`GoITFdAyjQw_dIrSx|6M zCF$y$##uq_tO#!Z4bAN%H6VSCmApay3Wy?ZC z(xHM6e2vro7u>-2{B_DG!u99|KH#hZ7WKlW(4zS=UTSEMb~fWw8M_OM%qOKpY4<&Q zPEXo5cCl6E*D9n`?i<%S0US;gG)m3srehHY;{ELA7Y=i9=shaMrwbRT6y=Swaa?!5 zOV@f&9O|vcrUWc-Q(m!*jnDm7VvQbn*teGeibw^IQo~=*=x#UF)gx87Jll*#-o{Bg zA1MFbzY$)Y{hL$L7FSA)ABcCKLKo9eHWXOZMZ(dYzGy?meek{*o!oz2-1(CB`qwa^*mXfERLmU zgXr@7uf3uM4cmb@+1d@PT-DmfMeB~6@m?#IIwemMmotg|{LpINB;?@@LklGbp-(N1 zWZyL`yLpo0tGxwFjsq;2zJw2)z)=GEHB;*#C@s6a5CFKFZHt!twpjj|qJi^%J|{yd zaHif{hAX+LcN`TCedkeh_6Ov|?G=I&wpwBHF@&vVC!FnadAyB@Rj?cS-{1xj`78F8bc+FX4C5m?8JCF= zK*7OxyCof$q@IL=GgX0V}NiU$!ic6>(iwLk z_{9{8)&h-WCoOK#TaxRsxxPpqv=rL0!zV9C~X1?m2OJ=;>uYaMp+S{9cpF;pUKh6t1eoLwv$H6;=RtE0;j)&8h z%ZsHWVq>Y}a_U6An7sJnw`uX%U%fO?Dh@pYz(AZUSOg(Q8=%XbEK{p&JLs)NgCU_W z-JuBSSEOlYU|W)}9SkP0$qUD6yLOJ1kJpbjCz%e{g_37JrNs*%rIc`~J&I;EU_-pB zysq(H22aaqsm?`B1mf7Ex9AezJ^(zBh1ltC-36B902A+12;+pKujOVYdR3#ov7$vp z9pgnn{j+5Xnfo3abYPzV=wgP+UfR|`?Bh)>?9{t+pSd`_KjmJ+;kz5fbZQGPh-^js z34BkaQOTMS|M3tuC4XphVm6wcSbCsh+|<5(Yv7sUWU7{bz@Er&zFuW}@dF}-s7%93 ztf7bs^BaNg^mpa`h{R$%#mXhGKh7F{c+#Tz-T-llCxmM6Rn$$7O?#*p$i;^!Fk`nT z-rf)ng{&LnY-!&K(!v>Fmd!&Gqi;}CGAeMDmp*^1Ia{LHLBIEbG1tg*K7tzX7TDl- zgiG#6nmw};en8{yp=&n4BWq535`W06s=zk#1~oKyy8f1{k}6byal8zz!gwPrwIJ7^ zTETRZ8US@&s%;F4yohrZh=IplmyPw5mc1BWKsFj4ku@6q(T06kTvWS-@8r^!@A z?`LYmdaciELt=S@@%X`}?Y7=!m6Uk+*?9YzV-n809td$_*XN09`(q57Mw9#Qt?B_o zVh~*DTGtXj?~x!#*j% zOKX$6zGI;`A5tIiLn`yu>JsoSH!SNVv$e9BG$}idXjs`3f43Zy zW9LJG71a;zPc0b5>*O429Po~tGx=4|m+)Aiq7Ze>`>fNFrV+=xf?cUR#hsm<2O5O* zdy4>@2U9npdF!qEsrOVjIA2BC8I~@vTFU=G`@sj&YZlyCKPS#K(|O&)!j&;>w!`Np zO-hURD&RTU6+jv5s`4yHECt6TZ6C+CZd0@?yXi7@gA!7yd;k8i$-d@K%7KRu>_XOW zlXH}GbG;|UHy1=;fyH+IDlhGFSK1Udqh>AKj8x^xz>&qF=mr&yV&`8?93qgoTwNwQ zZ=1Mn`{%KDgJaKy7LoOq`8HrzRJEDceVZvbK2feqI*nbPf1DLL{w3s!2VUCw`S}LH z>&;3ssMS-Hng62UBEMsL>SXM!8BKn0JEHbK`2KRg?xeXb^cm$z&@Pj6`=das_|e#t zt}T*4q6?iLDbaiz@DMdCtWZl{V%T`%oW8ea^mc7+$~0oJd|WoN48&mj_@-FNa(Pxc7E~t}n$V9XBvdO* z4UdeJZrHuAR$HMda#JF%;PX|s7^v0tZ8!1`){&NZxMo9hhnomG<#cU^RF-2fvqVi8b@{^>}3m2K2Ly*S_H}$1?Lz0Yes%awBS`f)wX6 z)jNj80Z}_&g!UJ1B@9CJxRwjck->~(%L(bdoq=Op=Qv4gzEwk^9s;as<8j-^(4$qo zT{6a)@Wpa0p#Nz1N!?Yj_ne!lu=O6G;eRJ8HB`PqWjx23lK?%F67ur=Q3iuimxJTv>>uR{e^>zM?~R2YPW3&DkPQU|cP44}tV z)U8cU7YPN?i$-eTa99$Ucbfb-ul02B(w941INUk6A9w^cR-^ufL07;SGAx|3_sK$h ze6{JEsD=ZdIZ+V{@|22|4?yJyZEh@9r>3DNb*P0zLq~+;k#+60uSEmYp}8DgiI3}{ zSt_g=%lXdJNTf!tg9nFNFigWF^uQc-1H&pW5V=qJjPVvdw3N_oz9nlYKfs>ee84%zAt@6clSk&E> zXf)9kp>h$DMO97B`J7=z=LL_YCv=@&h#%WvQ_tRL2_P88caaCbRqp)qz*C!DSbtPk z9irlR_4tBjyp2jjG{aNVdb6i3ZZf?)c8*nHZQom(42sG_>7o_a7$Go8=oP&PS3K`p^?6iuEn-Wz;Cb|# zLo!R1;SZXE8)~QGls)?Unb?_h3Q@N}B>374NHUm~4^*47pNWu0$XqQJ2WP@E;MJUX zw{w<=ZRLE&9Rmv8_Dc5NBAcf(KApOsh&q3Sq$M;wH)8idr}n0D zRr+-eS(!kTTwlSG$f;8`6w~cF;MrJrh*$9P2Irbiwqg2SwvU0gc`EYF#0lwH*xCezFm;Xdd=tYYVedl(qJEaaIccNw>F-EVJlt_(r<)?*fLwXzr(@q59 z`WvnfGi{0yPi^=E4NtV^1@NpPts*$xz>Z0D1ZtR*m`J~yey?fBWpuS2gKL<*QaqWS zT_u`8Lf9t`N!yDJG%&C90ypD;NCgjF`fTjL7iPM;Fgd%vNNNCMVWYL{WVdNm67W8e zK__H>K_C6Ns(b{Ku%c4|oRQhfu9>0Q29RA$Y5kSt&Njlm#S~7#c)t(L&eFu&OYR0Z z7mX>L#v)R^@IycJOq=vVX`fqYshnafp6$lAh+@St0o_kMf`9y2e$&K!>bCZkUP~h& zX%(n3iLUF8U*OwKAl;lf)t0WUp)Ngkx^%APM#EYk!d4wZXU{=Lw@F;4Zu+4nd~xR0 zILeyAJTG;*TjKMi6n?~df&K!Znj>fMS2JJQ=lC0A5Ve+p8&JIsHWOOrdHMJncJtx5 z!P1pwSuK<~Df=6tK$P+MBmgS2PigZJ1WgXtJ!o(Q1DwdI7xmJ8g}@+Ok%V< z_L5dI#Bx;3?B+*Fu=k+6F8|A6f%1_9+v^n`CKSW_XJDIciV*A;>+;*naZ^hM-kGuG zsQHIO_vPKWl)Wgni(muOTw7ck3f5eMPkeCdKkOKa$sfRu``>=fK?VU^!#{p z@jIA4{@;A%edjGZ)SAeJ3%4B{9aBI6nlJ>X*Ig^C&tN*NZ%$HCs^p(Og*afiDP3De zW5|wME|NRXpY%s=Q{-qFtX^AN^Ep@k{{5F;6_r-Z=~c0{CmwksHEgwKRz(kOi(uRb z7GmoGn!;)ulazW_+PyFKx`)sN@FaG8C7oU{ml|3!G4c}(Qlo8swz<$+<)Z{$hFbou z_diddpG6g%G$(jNeq~kFvKnyqSwE^SCEpb_5MUhUd6WquQR=b#@Bd%>lF^@g|Jyd8 zm_cp&zvY#Hy@-eZ0V9jQ5)%_sW0N-cn8-kCN}}`*>+pCQT~$$u)w_H5?ng;R6sqT= zD&CQ=g?^p;$~Spl@WA}>E}-}ZtJXQjBNm=%g>L#cqtnCHqvvJpK!!x2Or^jMHPKV7 zk(ZbEt1uh)l??D^g39#BDB$kryVpR`O@Xs}TmwvXQj}69!WhK}AwNj9-dpFPQ-8;XI|s^#3DAOgpK9%o zAJ-5FQkgwFJKN}a`UOrgaL)77fIX;7WW(SlIC6g2Fcj+LD2}3@Oo>o@`NH!$@`>IS z1ZIWM5Q;3~3i_i^K+#cwHRy#kuqMgvMJY0l?S43URk!_!TJ>;GG(~nDNH9gEbJJ7b%p!F9YvR1d$@gh_P<-q z9&K67uH+?kVe)JV`_VCRBCmE|Xpv5dIeliDz6Zk0`9HGIy{o=(u_xavdxsr<97%|r z2uVUOXN|p@x!`b{=qEbYmYRCx{JVI!!QDCl7d=#rao&X1M_>4>Z6g-hZ6j*&)`B(Ii>%!)mNO)<2cEun(S6~uCA4-QCBgJ|MJkM5 zWhi9(h!dT?4V?|5GTZ6{_!@zi3$mL@aCmrxg#u8Fv5k%c zU5?kF%;`dK)HxjFIv8>0QVF0y@R?P06$H7OYCP?Y1KEt z)>mNDbI-PF+03#lF9l;rbZihRepic(Gg$Y41aS^bJf@0^N8xr3qO%9l*p%f9{rZgYPQIBMaSk|1 zrAT1|0YtQ{09h+*41>X>Cf?GRkcL)FJ2iCr8ME+tYExsQlU&-~K;YfN!RE$8ub%JF zn*}#Y-BEMgNtK}E_BJ%oXd2ac&zxCt-h>Sc_rp;VaEs7;gh7^5ybnfT5sF%Zya8U8 zezSY)C2mS{ytyRer@Y0YQ@Wftujr>uCy$V?nW0S~qd; zFkP+WJ2+Rmv}IBsNKa>B;zUet0qs25-QG)Wuk2QI1vdP8;9;R*$e6PO{#?Ic4&kQ# zgMrHm<^zAMvn$OaKgls#U5Y<^WW4=rxBDOHhlDG!^P;UA5E?O;zkLJm?71j?hgT*h zCUuB5NLEu-4FMz`{a1~3R_WD_Y$H4DHXOK@=O%l!k?|8vr;m?}UrE8POrtg69EH&f zB*y`imo#Cp<%|99YCpF;rn4X9L<-O}SdBC!-goSn#3VMsF8c1o2W^Mj$KF}OMUnq9 zC7F`R4Zhi%6Z2~%Qfn*ar4MBVd`kVtbe^IGbuwe%--Em)a z2gP8xjGE=B#vz9-Fflieedr6#tQT<(LD#y)wv*#eZ&|9gdsCmBl)mz^)rVnw$P_M7AKLAaG3FqDf{Ga>!0M-N7R)Je5 z8@hYXoE8%c(_jHh?z8`}bQrh#Z(w%-iTN5p9k8kNxQLS|cS#qOo-T zJ81czR2#uyfclQks`gk@pzR7RTdz(53bh24;%=6HSL5b4W|2DJ{BBgJCkP4yS=K9J zRHYBlIhlt5DtAU*HOEDZKV@%|!2qu8$8%fyZmzAy!>OVZ5J4em4NH9aqlxU{3MO*J zH{=?*_d%Uhb2l82sDuo}9<7et|Mbpo`sFA(4*r)Mvi4{FuiI}`3^+Yxa?I*iK8bKh z&GQ%oPUifht4c0 zw{?|^{elUe3(;*^20b?uK)u|ZFKi(jQb@Y#Z2&@sfu{(C#2mBJ25t%*f)5Z zHS5I@7$7>29(Zxk1kT4SLUm1a<(q=HBsEbzdqEW>S70jM9n6s?fK0A4{JAf0G^bKG z-1QfKnks;wbNVH>fU!J-&~bAUn8)ub!S7l|COQWlx$q4)+&-07m^hPNy&1|3 zSOux$#+^HF5r*~gBK*-Pr4kO;pQC7zX|x>}G!Tff&j~csk&0SQ_ty)^uM%5jMlC#P z=?@{N0xD^=d*$a}q=HrTluCuC>ql)7#J-VQ(_|&$`WYG8f&sj#k+T$PZBx%_f>(Z7 zTRf_eq9C-v(UDZTBkbokxzf)y8{=A~Kn*+Z-HD5iwq!!huaCA@bqoFhGp|~b5%jaX z(kq?Lj5nf$#g%qP-8=rVp$n$m#w>3!GXt+=(gm*np8g%qo=`_GHnjDx1GRn*+HPKQ z>tt(s5U5R$vt;FAq>A@#aDyV5;ROrG^L1w?GzDL_4Ye7ev;T%7DdGho^j=B@Sfk$I zD2?I1mQ}6G;9^IN-90>jMUyvN{$g4UTOYH79BqzAs4N}rnO-=Fx1sbN{Ktoe1Lkgo z4IlAq!sz*-{z7rr6@1J#r%H=rz8+q7s&_%hgwe3V7v!%BPj%PrS>?!x-((652zZMy z2zvI|5nHev1>Cd-7gs4;YRxmZ**;fNH20{!7cgV8U~Wc4%cC#_)LZn*fe8XP#I=6LOmg^HH$IST@8 zcj8rd!ygXud<7InF65ZK@HxCFC-?EWtjE+$p1in0kQwFLqstiupY_^%d)hxJ2Mz;G z5dpM!jG*^?z)OG>G4cAG*wMsWc94Y>eG9LD63{UGeUJ0R?;z;&;Tt3_yKh01FYy+H zQa`~Mi-BmCOJ z%aYKpwaWkWo#Lye|Rqx_k)k^W1CX1pd39?_x%7k9!mhsus5J|8v;? zJ9so_bt;Cd{m0M$lQYIY{IKw4IhD_3L+9>>b~3F^+*8~CYQUTWE-_=+8sDSoq-)rp z>u#6Ng)}eRYrhfX{XnCD@-_A2X?Wpxize?>`$2)C*Y5XTofe-X9!?AaqD-mCjwB2- zgvVE*%Urq|xM^smZE^>fy&8gJKaZ^CEn5g)apcy|NDl+m6@Xr1f zkm^(L;6CImU0>)K0W>7BxiOj)+*s`j+R```M;}v*8o&?!2sEHKCMb~7C%q7$@pUIx zctDK2XW45mg{p5bV};r!2{bx<&wrb7)iy++-M#^U$ipD~Nc?bGU8F5akwC5;1_~%@ zbkule09fsdoBidkcApwt@oB|iTP04)!VrQ}Q~skLOxZ=N2o49P4=XMLA$J-<#Q&I& zJ_{~o$h3&L7~0}fTprU;Yb!5+b?w^Co8f|4B_H(NCy?<3n2LB+GjLW^4yqM^CT294 z-Iif#QEUGZIf=558s=ew=7AtGN!<$yyV+U&6D%bq;5XDlW`KzXu5rNcW$IH)Mk4iw z0nkOk?Cmdv>y43r_-zNEcF_hy7$wwhz(UjxBZHA(?-6^P6K3lU(cd@(Uh>u(e}KK? zm67SWSv~yr1~SiRn5pD194~1XukSYC+fr<2BxZvY2C$UW1EC6?s(=GI+Wa~s$iNC) z@TAUy3o_c~+O=z;U@SKR^7Ax6oiK3eU59KQ*jNpt2g{K3O~K7f{u~d`{Cc(5dZRq7 z-p8{t!?I1z#eFWZh;3xv4%xVuPWKuxK;Rxi8Vr;KlF)qr#&!Gw<$yBPSbZQRN5E7G zdzav`1)Mf6JnC>mp>1FHo*wF+x7{4?7nr1w(QL3I5*u2!l_QS6T4}gzVHIY?N6WYU zuRS5)z0Itna(XR2T!zWt5oCDOd3kfudy@}Qy8w&mBlAP6H0`olQqSJF^UU^%_0k4%keiaFW2z zYNAw;yp&^xj}fw&!a_ssn7@J)-$SB z79JxJ+WM`9f7F(mKGP!^pHa1g={pZgY*IfZ!yZ zYOhMf8IUywQC?Asv`lI!qocIqHY|1Z0;i3jCey2>Rk-?{MkuMftmri~r zUeMBM^LuS=11Lp;(6oll%{eUtGpR=N1iu9)4mx=TOK{`8K62smfddDI;W9324~@D| z^CGIwfb6;_y%)S8^>uY22)JGc)N(F_-l8d3#pz%ejTTW*AhkD$A^GO+kx9%+OM*L) z;%c#{wi?U_fGlI_MgOZ#-2MK+o(G&H&XH2y69d<_2X2%TYk17%cs7!iy2j5JN}Ei6 zyq$D2H@0P4F(~@r)C|bhC=c|S$gRB@nz-;YO*;b7Z8KIuAe_dcrNgsqqZU{+p`D?e zuUyebta&*+0Cz67I#=pifSP|%^Y_QZq8yL_G|x$St~lj;mSw6RyO)%dg!K0O4iRsJ z>B!V;G0Cg^3fSmx(T9Q+%CLX6Jw#@6ty0mG;KZ3TOZxHDH&ef~h3W}m4}f{?8BW^_ z-^zUE5OUv3KyAQ%my=_xZ5#3f@)7aMfhTpYaN)_MWLS6UZp=K#ecf;!vobZRroKwKS(r5&-3x#mS2% zb^D?rGgQ#=GD!6!&cS(Fwef~9{~w0I}* zA3au5Fn^fyUI|!yc*&piOY?Np2jUquV5+%w`yWJe4>+_sPc>&^E4&Y1{>A8jD(384 zo&J74)`OM`AKP>sSSQ0vScS7Xy*Q-}lCAo@Hw<`nwk0A1Y|vQ0i=PHnjQ=2?`bA|d zq6(2Z>^pSKuR%T=LDU#9^WPos5Spv<`16UVIBBT?3z+rY5pStGs(l~vb-Bv}c1qFC zHk^f%p$gNO1_bIFAVml8lDYTBL-!d@l^dPr!)hXuHAoQZ@ZbnDtSB*hsa7dI6`! z!yl9k_;K8{hdfphc+sJ`3i06Rb>|O$(2rXD^I}(Q$|WvdZ?=E&0g! z)^{wV9z1~g;5uI^Snodg2h0pNH3Gr@ZWfb8Lkxs+=ivzvrzW05U3l@p#9)`VH()yialn2u$#FV8bq-i%vd#n4h|WfkAC(I z#${HHpU$qByJ^2JIKcu)r%Zk%Pj9a!sp7vO;V7&ycmdb)$nH#+0$Ru#~| zZ?TPE;?cl0{z0a2QHK&MDF(oF&4T?R*4tVHo3ac5NLHT0ue$QwmI%$4CDZg=Dm((g zglWOhV@4Sbk4!;`ztks$PcS<<3!-t%;%Gh6lPavwh4F8bh3>-?yk4n)G??j#HKp}O zem@Q|!*x&vE3sFYQNXzww+=lT7}Kqja+`jT5Cxg$0TGwRu4e&bNO0v}4B-MH#!&8L z=!u^NZc_v>_YHB$f^8%OH1^nM(ym!0gG`PGI>HXBOC{EMfL18%tU=vZff(G0+V3uc z9Ey0tndab9jI#uq2jxg$P2Tjv@MWH1JR zeIgX(+35b=h3{?=HM0%nvKJWE8U~)cBWN>=&UbG;+dD4*T#nxoZt~DOz->|LjQG$|0sa)E_lcOPt|cxzipU! zuRw_z;J-7A;{#6#3``x)+x_>SrpOU=d3ZNR{6J*DqILk>|J)JLE3*+^5^!kquG$Rc zFvAmLzaqiUwy~1fwIzh^^Mdy99A=vrAWpL@!5}Uz z&*`2hR92Z&y>>0vGjSX03WVVU%`G@@(VWZE&N0f%GGWdA+NQsvYm&_o2*6aY9HqBLeuEr$3&n_Dmz>K9t4MSdx7#<#eU&)11 z-1q%^z@yAhK=B3>ZRh=^Gz1=uWB3DQBT+Ei{WXIOIb>EP^5 zII6?&*o#nvRP^RMeYy@JM+B&Ui6mbX>M_U*op+r+N`Co#Q4z*Ulnx7lI}sidFr@7+ zsGwlfk!2tVk2{bsHUfmmJ$Icdya`?;$wjO;;&!JtD+3hmfein?CxM zw(wWdKrWPDho48y>!5-Ntmpuy<5nzSxD8Va<==nv+=x7PA(#N1baA$`x>3oKlL?SY z6n7n$_vq9wu!j?U1LpFR;cz;9+s*gR@L|<5IabuD+OqXbE3*@j5XggfTA&ldOcoMP z(`YBF;mqwjhD<6UHoHTpamEr5Y@=YP_^5(c1yZpN&B(z-E3aYNi9)F~h1t4O4u8Hd zI0m`GU9^1CiQC*4$Fy+&&GeUWd;iGt@P!6P!r*F%t*@pM&$u?yqVh(x+5 zFsdhm0>f*V&ftJ}p>q#2q$VfVLBbnB8pt=n5bsD25=Bpf^;3L*6DQT&OZDaUwH*3z znBg;YK_#A}yyF213Nj=5hSCD!WH7ToI9O4parp!w(X$oWqNzSEU#J zD@QdE2$GDw*hh6FIk&OA0i)17jrsd+&8OzPAn|8$z6=yRd-gng>NodpKgkDQP_3y% zn4BjvRw7S!Iyz|#CUdhF!SKAJ4|PKW_ySwfx% zg&4q*A_(-nmTM+=gkU1edNBfd9H z&X$VSb%n>K+%f6TJO9Do^UApD?x)}7O%qm_Phs!FKRdjyu>NG8NkVe(>J|ubIziU< zx7LJsaQO7&Io=dLmfMNk7;UURZuKx>p+gqHPB9(rzN2lkPKs`5kN3NLLKx&2)POQQ zUZB6#5rA+8^lP$!<%)ZV(rF@w6;gkC03D%dmdyzZ{b@BQb}q7{oLpb8wZ`vw_3ttw z!*UM8yUIa5xxY;gwUV1V4dviMFof(B4`H&mwAGpSRocO7yr7b7oYca_YMHXvFb{W4 zO^wYv>2kvQQT&$c1j~CZS$|;Po{0uyCA^h}BevR(Bp)Vn8UEP76Z!?@p>z4cz-4Rv z@cs-5>kP?qLl?&45Uz%oybJ}Z(Gwm`w5_ZUE_J#OIL(Js2);2{fvJYZO@#vOpX8?4 za!(tOCL7m*Q0<(DH$EH$g)$Ohy!NcaEX{iMmDNxAO?P`V8m#<A(Mn z7>E%zWd0b?)Hd8EJJ@}p=(Ef#FVXy@Q8ATn_w;#yBi0FahX=pc2YmU5U||tiRioc0 z)Ey@Ge1GgY<43S?W?b3v8_B!i`i#ush%vP3Q=kHQF!RHo(dyZZB#>G_iZgM}_keKN zP_i75h(b?K$TteY)t2opUi!NG<0QfE$6WR9;(o3s{_){Qy}wZ!h#7PN4zL#1f?D!4 z7@s)Ls{fNAJlpBGkWgi6TXa&YHFbUs#ZNVb}DI%t@6|%a__h~;5 zZF3LR;$GL-s0m|PDx;=^-$XrDnOKTviOh6yV#6<)C zcK}AHM+vK{0bx1@CelcVOW|og`{R{Jua3zk&|8LDc&zoyE~kB(cWvp!hsq!)qdtAV z_`)OMve1Y^!uxFNO#0%tGRv1Bbf|JLxOdAVX}7oA_2<)tK$z(4Im|HHI$=|~;e3Of zgqsc1#s-0_q}k)@+OK?oQB`FJVAp2>)j5Bq)(%X{t+JQ}5 zHBv}}9fuDDiqM;{UV&7a>qEG^0p}8iE$`Y2BrG!hfl1F@)+%O~Jh>D&qB!mTnAhe; z7h7cB?O22EX}$fd)*(Op@@}Y0wzkF+>u$Ci8D*y0R|m5uN~&MX7KMh_H=CHMlTnsU zTAb_@yd_5XCA_2L=g}M4uhiUK+6ET@hYAHoH$Ns1wpq4H3**nu(@ju4P>~SDr4T^C zVAkJB61<7lzurmmgC`{gbLiPxTrk9oMI9?Z@SVklh5JJ3SjFJ5FxGA1H~|Vj-#Q9R zy3y@^Ze^n&A8z)V6|ccso}DcNrSv6*mkygUp4&QXcJwzaphIo?wqX%ix9ZkgqBcFE zs}RVebiDfzqj(FMPu!yat?&udE{`m#%@mRR`yR(9S??@*S5cyulC-Ku!ENN)zuVD~ zm_?A{c389n>&GxE#i47c0T6{3U?hPFDEx?Idw_siRM zEc3>DMep0*nbPHRkFbzBI3yHOY(pr-E!D$`_w~Kp8Jp+6Y;)sa!jyfN&AWvGIqQqu zUe})%E)}7Y#G2rkdnjCcj!@n#+klDJ(_-Bq&^8-^PtKB_xV| zGu-i3P224{KjteGSseAQbZ@=TI$?$RO0r;BLQK7Uk* zUAq7J$RacZwE&y_ev>9QFTSfBgaoR$RnFz@pAA2}3b`>7_z|b}7az6lt!BItp+d)e z#r-kpo{m-=z#Q3eA(^@X*EQiwvm;m0=7y=119tU13jnNhr{8+c7WEC77W02d0B^3iprzs6Jx;q?(MSQSTBd zWEF0~((>c7OZg^EQu#~s9dDm0TAPCcIdLK%Kpq*S!bp>7Gy4O>d{%{qf9Ju7kTOv3 zN6{7Bbk{{n>TcYE%Qg2xJDOj`NSJKCJK+^eLocBPidy4ofK9c<+!j7^`n9)19>tbm z06%2OG1mgyg9n{DdlO`$SoD8E|1K8-j*I~`5|~l;U2)nM z0wq`2i8AHqn@wgD>FltB+`+9CZYQlQe(PO$iB&HyUAat8wed?;9P{@y}6A zS#OpX-XCUcOiYJ1ts1cIs-)c*VI-wPauwt6JMhCpL!xd=lVd-BwINppGy@C(Ogg&P zLT=N17xb;-P;i-GyO9eW)04B}7Jqz}7;Wt(m=?~l3Ekt)UHyWGlb;NHui|}?pX~}@RSF5THy(ny>5>htm z2hl<-Gy{WJ^14T~c47hikRNVoLV+8zEE<6Ox zJFh`+QE9EAtE=k|5M0AiYHH1)9MD=g-vc2`RT1ck9OL8TMUjUJa;Jn0NIxL^3PK78 z)4S~Hsw5u#JVVr_=zQX7xjV=$dX1Y%4URsQhC)!6SL2=rf|;u_l1U%fEq~zMS|{2e z`=%E;!?5nacZxz!SHA!bm^Xt4Ny}5+?fu|un|RUE*C1mIYxKq|w1$`yd>Qs0X0XdP zZ!AlO6I$5AcN?%{e>kn}bsFN)L0AAsnW1`G6WmMq!H3!@oD{aJaZS z;O5ECGge-&P-%xATr&x@>WyDv!Ogd@*9YdqHJbfW_a@9zX{sh?V@2%>if|JwDqebl zQ0~ZN33TQlvBWhJ-`Z;p;zg5Z*IIk;LO*`3qxFDvSV-p;--ZF{8Nsoamv;=R!mrbowY{#>%U>2~?y`>x;Nz zab5T5e5jl2sQI6IcLS+fj0+(ha2%0f+|+i5ke}6X()|uz#$09hwX?);aS-8g8kon@ ziLE=p5iE($07>jiHcEtdzM~KgmC+4E&^6P)J$$GLxD%3xlF~~Eg$t)Vh~aq#;43(y zp#3M^S<_c`XClkV>o=eA;C>I_KikX>h>u3!0T%(ufuw$3-p9DqPLEU`i7|F%`?tef zAbem7&o^ZYZI76g#M+%>UyhdDg$()tk_1%2iL;G`#xm$#kr*DQ;do=0Yju0Xaa=62Ej7Qp z3hJrZq(_e)G1HPqQ&W8N4m2A-Ui>}_6oBWz0?<=e4z_aW9a@KF4|z>g@K=s+ z?Z8}Cb=L*y+Um^II-|@U~=+aKCqdF z72Zk|JL4b#f~4b}MbZ3T+Ot8IoWxhMV&Z7zikcykW)G+T1Mx@AM%eHP_%Q3!qvxnVi-kbUQ^2AhprrPP4H*oNp0pp0=Z+MfnxqzlRp^Fw)3x5w5K~I)jlJ zdMk-1i>Y^$(tvRIQ@9E|kUJ3M{7CNeb|;^$V}7(_k61#(Aj4HIVPUO-+XIHLp6<;& ztdOddQ+MraVUtUBIf+SFww$$sy}CI`n)g+*(9-tf59=gUtZ(xBpn~!t_NbDcKY6mN z`SDeINDSaQKlS98+PPxlI2om8moHnkw}?NP0UyzC=T*cXFGGF=$6U1Io2)McPA;y` zJu{#ixq@xrt^=*)3dAkAH#FC!!1m|KhUhC9(4ek}LDa+Woqghn^xww<%Um2Rl@zB8 z(hh*;z9r2-U$5<+?s%%)E1HqS07nW3<>Ompj(0DJ%l2MemK0R=#l#uR#&cdJWvdwC zOP_bPE*67wM6N7mPadSB%WczlnUGll{?Y)eH%p@;U{Yo#0Zgn5kkKgD`xYz)Vift1 z#MWb_|2e`7^R^99ICWI==XZUX6-QZ0OmPs2w*zMj2!enEwx3>XW;+`7NCOfHGs}Svpf`uy{XdBo+wTMPy zI5SXt@6aUzSiouIG52&~yjA%bWDD=!z5C8a*9)Bvcru343a16SlVqQvhX=F=W@S(I zgS6DvX1WD$WW-l6%1mNfPznWeSJ%P;6hnlh;&cVd&tJoV&AK$uci-g^BUDC1pK>vT z1i%tRZG&pk;2jT0iF z3M|?N3Q)>~(3NTAvqfn;eeFrL9*8SkLxKN-spzq-{D$^pAtSJn$3yV%Jj4f5C`|=c<(YNbpECE6jAI#{O3=hczTpQy;KyuEgN%| zamOmc4vYA*2d7`>-Nr^%c*m+8Cj62TF)zk!0`(;TPcRDBfIa^;mISga(8k!DFCGfqj6gH#cCGNFSs1u@G=O&{>m%18)6&Dz=7dB=HQ zg>~>3#1u#Z7|q@81&G5^qHd~PgH>?H_rane3BvZ15F8YAM)l1F&LC2Q)cU>che8~z z2oWSyW28Gkg~Yk73>Y#ekc@ymXdz*TSf?xFws+sAOLxX?f|}3uWvB!Z zvd1%nNGVV6l}Y71Qjk=$AY_FrRD0)xS0@}vr1w!7QLHZPayaL<2{)ij#Xo|BJ5+%p zfu;K2Un~AQW&Zu|ARc+~|N5_R2)BTwPKKqbwl+l4eJOMz2au2yd^@}Vy!}4PH;py=z3Z4uZ8`(b+i{R|1DJ2_NFY>3_)LW@#ib0RKe*$fF3G;V|$85wKnUl8~CpZbn49 z+>!>UOOd=y0~lJQhD!jp0Vx!=cjmh>PT`2qkQ;$a!gv;Tb|5tjhmepCG!ivX+wcL} zX@YY0ckS5btHJpP9~L66AM4KJ0{nRmNWx74Ov?n0fZ1J8P_VfOOR=jIhzq1ThNx~81Rx&RySNBI7;hYW zV3`O1c$Pw;UIqI8;IHK3J$Ht7oYonOu5b-y6z}-cN~=_eZb!;83u^P#^SN&KHYkFA$MFVOAO~ng4QS zJ)eJzL=|7$rki8eV33N2Nx<>r$6vqm(1aVNZUc@9Z48_O$P9gLs1NN4%LR3CqU23X zObiIBC?jqj-uE|*D1rZ>;W>ZGJ%=ytI1O?qz>I1^FgFZF=?EYuYa!T^8QM#MVsff( z)>R17;5mEMFA#P>HND(>oB+l~#CfeLA|C*vsDa5t)`cMuiwV06a&gu4Dk< zRfz6o9vGaGl5AzUh(gW=5DgX;(tw`;5Ke0Q7hwK$x%{2yzVdRX49G&vAEFpaelHjv z5D4p%64bKDU6uxbzeQV0q?J1Ah|mBaL=!VZEg1~>Ze;HUfUS38@!%V>LJ5|&aBFW1 z6rWPiD#6kL%Al*DS!4r8o9XsreBvLEF@9TkE$22|2()B<>IG=@6W7OJJN#6&fSSp5 zKEs0Jkyub);uad^JDV|3z5nC({7tc~%_YEYcp6R4K{C2Tkcs^hd?V3+^lv{ z^^J&a0~Vg<^wa-&EA5rraT@OV#PY%Mb6?;V=Wd_K`KlIiR1H#Jc|eO24xN176?Jd% z7Jjto|MW}LwFa5wu$v_MD5oX1eI?r>>NNCOPXCn7&B(}v=b;n4b@`7!@|}$aY~b=H znWWx4(Fc_f3rHdluDe4$r3CPWY%yBVF1@3(u6RkE3FvNn_TM|l#GgCIzpduf*j6B< zJ4984En5^O<6%V9JW!d>-H=qEjElIcfjN=3{kWlO57R+Cg-ic13O?=se9bbV)#Kk^ z|4S_Yc2K|TT9Dr+@vZ?%iN-T23J8+;roi*(3ldx6kVz zuH{qE@g)82TBhtGrLa=X8)09^6zDp38p;fSht%m=Ucg|SH5&inS1yIdga2>8 zvJv=|Wx=of8Z5=zo^5Q?so8n*i-~~Jm%hHYjkJ%fQ^CRA`17+)-T?U)7AdOd8DPys zT73Fd{*|(%<46DfVuLaH)lQlJ{L23a%C7(C3d27OIyjHMkom2fY3fr46K8-(Wrq-$ zFyL#hUkzX~9Ig8j3w zCvly8eY@7b4!Mb*dI2}WFR+1}X;8XULDmO&=ALaJcKpxR*FRT9%;xmB8Q7;CniSa; zL71g_<~LYsYkNeH01E_5Q3Itw`&1!=Ds(j>lKYTN6S+~9pwJkGmeW`owp{eufsE*U zO|3^IVY5Z62DWq>nYPZlU9joVfpD{WG^U5=`OdEnNH+&s8_KkI{Rr{^cGWHtUt3=v z1~5D=-O1_qzI?$z=Z3q$y@vU{7wL##gH(piv91(i$9eJCB)Lb3WI!D;2-+qsStKC5 zM^$zZtg`h-%HYqGhhD(X8q289TDoJQlS+W{5nE~l`5(Y?IT}H)wzJ?BdUp3G)6|F3Yl(T40 zi$Ti;nk?a1whX)ZGS?NPt61Q0tY(d<0w#(FsZJ!IYC%Gx<{RiDTuy?mEp0&BNF_$R zwWkj_Yt{qg?Trv@)-YMF8tS?JJZn@LE@zt~8K?uuZiz&*>Lqy!&a=$E4Vc3v0O?1_IZ;2#1oZXXS`MlxP^ET!3)d*Z99utgDg;V+xJj9Raoh>~)Z4PW^pCHeqKF($ zrWWSQ&&$uiaRs9dRU+;T`fyhSV}&n#hVarbS1kUdv)UFgT#rjU;h&D62A6ZQnfWVh z*RLt0ixMQu3}DDwrg8b68!Jf_kGsMV9af7{uK_4BriG(&nas`x6bf875bXlQy^W6*Y-wQj9r(eR`C#K zi#6g*sqjW3eRgjY)097!7^PDF;qd;XY>sPq1tc918vR*{1~1qHbJNo>7_CNm9*+Lq zJ0Q`IPJj0~W(J(1fWr{X4a>7^ivYAo`if_h!ck%W{zE{F03P(-bW>+xgp0uyqzm3{HvwBKnY}Hb_T|2l z>d^-8Zck4mg-qyS z*?~ih;SC{h1d(SHK}f}VZD#jmv)vE-8mXL}{x_B^-a-od7qx3;7vUijr0rj`1T|PN z@{8X00>_U*bJPWk?Wlox=poSnF|=}tt)e^F3K-01`K6!>@5!x4&nATpNZ3d0e;b+EwCuUPE6MNL1kT5z)1L`Tg10Ms0C z*3Xq%g3gjP7Os7ymFbbqulSlyQK^I7OBzJI3R%-lE9()TlCQ83#Q|kJ)gc^gM(0nSu*3!v9LMEmF(K3 z=>9u^X`Z@=xB90MQj=>-|WpLftFYP;w?wpS<*U>=KJFtDibq@$x7 ziGA14e9U{HBMxQYY|y2N3fKPlY4@!es?|ss-G!h+)&m$64E5~k4d^;Fk;fyVTb)W)mnyB$T$x$r0lXnkH@vTA{=R( z9UhT*m#kYarfLf;BS%Ib7PKjJ;3)yHq!t=3v;MND4&dErR``e?gVg2qldu6^9vB=< z0Axf?4MR(eMq=Q-PTlBD&qo+KN9KDYpINc^MNjCc68C<6@?=*&n8!L>;U}q3N=(oI z0#zT{!z2u(7|!1btFH)P1qdTT019~7FBgA;_d99@Oe?HNM&lCf(RP=8ET990J(eGY zBY6O9|1d2?W^T9=n3OSy-aTJ@X~M_Yx-?lY=kr89Y-k1mR-3s6LMqy|?F)Q57#NnS z*4X9kU^KKv1`!$qsZW8C=$q-tFP{bjy zdyvlI)4u>CGdJ60MlsG z>W-D`&1&~RJz=7D5Dn;IRXb>qKy?QHRQO(q zM2Ath1ih8@f7`ohAfb>QsF+z1pZ`%1zr5WHu4=$5bt$~@9VifQyIO1pox|IMJIK%p zRGUQ@vH`d{>;}p?m>b<*Bh+qi7FlyDp1^s6A%g$9sDDj$l)l7?X(^NRhVd5R&=U3V z*TUn0BFW(U|N0gfbPRu4|L>vl56(^PNasO~G8D3kAgr#$0VB>0E9a$OeDEVT5B?|O zhi-z{KpIQ~3=n0)4IM=*6+pM4;%IMV1x&}h8`b#pD9z~K4EXR)ZPb}NzvepA7Fgi-d#2(ly!$eY?!c|MF8bnV$>59Ps1#7D!cZNRek{!OeW>qcGIu7_WcL z%KZC2=M}%njRlG^-y4=e4#Z@oAN}uBW_^YmqoSlFundJ7MJwPCeu8UC&d%4p0k%zvMc+(2p!47hzfVi^o`4j_YC{&Eg1tC(5A|K%5FywkHs$x~o2 zi9qI5xbN6eh5(vX0gql!gEt5u6)s=C{ByfzD)|R@0dLzzletqy2%#8|xQr$d!*5iM zae=e4y)NqP`tKu5gCi^f)%Bkh5kyf(fYRyBnTL`vd{g5{9>91|^8fdm!E3tvfo;iZ zVty$MutKp9OQH*aqM~`jus#1PM+2^%j{g0RFnWq~$I%<2ZT|OIOej#gE^A?s7vGWa zvjpA5p^{*4-3PEbmEuoIX?E5Fk z4exCIXFw4A>))^67q@@)-+zZw`S$H`3>2*`wzYbY>knwhsx<9Rk*1FR={OC2Qe{n1iUIfOm2j_k;5ixuQ@;i9YQ?J~L zb!69i^czCy0zo>6bz2xc0|h%U8h_F2ia^+Px5FAy1svM;ZI6;DAV``S*?tFKiJ&Bk z#~GdQt9qMny)H>IaZj;)si1OgAoAZY1M{O?VsT5FN$r=rVG0_u83)iUksh|M&T$%1 z?az08gB@`Q8je7i+@b*7MWhy*z6Jt+5F01}HK_Wq4|*}wz+?inqz#OKCnUSi*9j67 z3Z1FRKbd8at^j3FY?Mp_``~DlXJc5(e~~H!Nc6v&t+KU_mSkCpC$;+(TjTFuP0X&U zCvh3rNMv6hv5Dhucljd4Lz1zFsnH-?RMy$5&*a!PRiW%t7;^mQX5bS0*JcoYQMkPS zkshVB8#a&l$z7}ZsyxjU&dxU)a!|uF)l(k~hI*cK?lCR!Twez8xvTmi;eJB&2d7Hw z25uPQ$VIe%+*Y9iF%f1xU+fWJg7^|sDBiYd9+0*6G@Jr#4gAu9pajuDJPxe4huaci zci=u?{J6hBQhtaeW;Jw)^2~f`j%CgF_;CTR_E-V>Lqb=2XMY`XI6d)T)kUt%G#y=P znfN<3`zMkV386Mk@7L_HW=h@{PVCuEhP`59%PW;fWb9=YAnyUcaIWTa~) z=rv_sSI?kAH^+Rp*m@}#!h(t*5Ah_yLh=4_$BaNlOwj&k_B9#J?;hZ@9@JwvcIMr~P?r z3Q95c%#HX-pkNsff;OXWzN_PX`j7UB6kUx;NIoA#( zpqa})XOnVOu<3GX7qc>1yXUv*w5DD59}Qe2Nvj^zw(p)5EeMY};7GN_)9`cJ6KZ5pR^|n9T0pgLd{`ry=~Epkt`@-!Dv4KnU`ijk6K1gwS%f9_zln!h_USR)Tc=l}W9uEwK^D zBUlF4i~5BJ#lls?$dbvd#44|9X*U@Lw@0F+!0)D~g1!!YT&t3I>f=ta>UU8*xHkPXX zQFA(vl@}_CpZOv)=z=>2SV`n>_c9FT;!wi!gMAZWw#_B?#b_FLiez84`0K2e6(Lr+IGHnQe6PX6J-J_Pc{#p6Rh@lzb$mtv!? z$_DJleS?qu{A&3D+L5q~z9QZBvQoUt;VW}-!rXVR$SHb-Drh|XP?hWVD@XpB@JDvK z3_6EpaIP)aty1>25Ug7E>|fH*-=RGW{=buZK*7mroTvj+H8JT z`G{1_kQA}>c_L#_Z9qJ&?p9leXnryFmquRWOKaS=nSal>Cd2F4w{FAk%TZ4=PoJ@q zwQ_A1j8iY3&j(N5n;7;u{!X@5mhzs@@@B`!9^${`%-K6|i0j#izh=X?e!f0n`AhV< ztj7HO(5n_o)zdwh4go`nPTZM(!mTdV7Y{88E^a!u-qK~2F0INQUmY$y3Dv#M^L&$) z)YxJED(*J(6IIEsz7$tKypyIN^`vrJZSGXGh~?PQ^>}W@IM_hPzDPAu9s9wZh4p;G zce^vvHk(+j86(!QvBDv92VTX1m%$68=8hLXA9_Br*`XQ};&t=t%r7HTeZ66@PRm@p zaAGIsme?5vrgo$9p*;!OB^rFAyuO4f+*;3cEAyubmNQO;Lw6X0FUaQ}H)WubA35Vw zBhGSOcW->b^F<3fY z!@Lf*qzi^G+h@-6itBx*J#Jl_oQTUfNY~j*jg&i9Lj~8CYqYhd9hPcbwB+Swx;yGB zXmw4*O;_hbwyKCeS;K)tO0OQ7)8%u^2lt+2k#`7CNFbl3(*7}@_2J54xk|EOwo6Lt zRF~{?vwQU{c|>dVsv6<$__G<*TKRMC^NwZMRb3_i-;PSH{hq%TX79MS(UQgc_JQS=`=Zu*94#o>eF zaH8I-1IvhxjLOzU9;s+;{aAiyX02bb8uHGSZc@SyAKsffl}VqHN$7a2Q4}&mF1m75 zjHbzx6O*Eyf;o4UfojT5;ZjCO`N+npUS?i0`4n*B%(1j$&9=pJy4j}*%>_Ff&A(Vr zSmI1to3W|oo!z||WrZA{VvJ5JR-7CauKZEcu6b-Y+m9zQQl~RT`HgyDAyNGt!D^tU zPJ-oe=@%Ua?@IiR*4~i1Xbwd}O-Q0eX>}>36qjI#Rc%Bo)s|GPeooyok!Kf=O43bE zJB6z%)yy&w2*DZM1%uJ^Z9dvB%^3-%@<+p#-)>BoBn?9?q?E>djNVu(tQ)b%EZ!*2I( zJH9B-ex+#|v%7PhMK9A#ugt~aCPQ#&9KXw{4tDti3VdCzFB;Oin)9TDAK`AiuK6+{r-{+N`5Ce3>)9Xuf!`nP zbd1f1$xbsi={<7AEE*UY*L}^&V&q18Be$sg`9-V4Hbm}J4*rhf`LJC0KmTG=7 z7#KfnNYIrR+?;;2+VPX6^81!2Hz#kBYiI9yP0ym>v*)E`gDthf$HN9#J~>3+#4~UQ zyC=R-30Is^$3t%ZbjrJp}>|chcPlDe@jkcrVu^S?~p>-eM>&|}YYxh07)luDl?ESnP)fFnnl3_WQSI^%p zI=ojFPS!&^$gTPw>X?$5j_h5_Q)j)Tb?M)M0q-PRo`lC#*v#U}VGZ{st9keK=Wv2U z>A05dw9?|uhLi}c>IzAoMup+2^4~9M>-T1aJ+?<(#BKs1@cWyya z7<)G^XrczrZTJM2o}G43w6^Puo?AIYv35GYrK05b(xJ=r)1leXJ3n1N!ulbXF?2|q z5Pn|BHlp)-Qk8MNYh=ELm}6d@ue9^S)ItgYf2Z_$_1=q|-1>VM>NzsiEY4qA2-dyp zYoMp%C!x!p-!fGqJf+~XQ^ttU%9M7~(7%_Q>IW6q2d+xR93^|IV~nhMb>y2~o%t1Y z#~7)t=M8dRb7@TdrU@Z_rY?*L+#t+;Qxl>%fU1N{LE)=5EzI)uF*wbkQOIh1q;|;E zqIWBjK(MrSLkpzRlNtvRnzgy{!*Teo7rtTn6G_#J?7s~AouVGFZBD$xyT?r15J~IT z^1Yyz$Zv3QnL7V4$@OgPvn*xGQ$aIQB^sIU8QBhcxw(FfXXIRCx~RFS`a|TGf%(fM zuf3AZ%VJ4Na}$2bN!^2LLEKJG^2e=Y{P^t@X8aB+iuLP#GW_6k|8uuuLE?R0fq-)R z_dR1@)J@(8I8jRukcI_@+r9OUPE>oXY9|mRZKAr3uWfEJ7{=KzovD4Qh!dgo z_bbl-tfs7v^^-(47+k>q^9`}V0lRC|li6wlG`Y4UC8jfaR~Isb+3w_X`dLMUS)bfX zL0@Y8wC8n+_EKdy!!y|))#hhEJ_gq=<=7EkG*GP;VrE|eYwXs4H7nX6#KkF6lY(I+ zW7Xwh`GpfSvDiW#enLY^g!ad8v#s2PRTQ<;xu(#Re7$^NIIT&&e0Rw{{?IAh^e!-{ zoGu@7E8t+`*Jl}gpfGYNsO}GqbX(bXJ8WvPNx74~2GVkee|%S1Bzxo!em&EfF(c?x zQyURQ8^g0c9OujN-h{GN`<1Z5@Uuk&_J_Vke%E`dT8ihd{Suc;G<1fFm1B6oZ(!+a z;YaHTiL`IQd;&$4fO+uZNff0*I*U2#t$W=MR+$9g{Lao#1>`W#rK41B>lv7 zPmSb*5yhb)=)=5~!NdE?>M+aBxhj+FEw1^g(e=%JwR;BdT?(7PtxNKxk&>UJ;62Ba z_9jZW=wqommbTQKMM@moVAG{9F02y#(S#t08xN@rr!`Wg8l(!4#N(z}XZTf`Sf|$t zR|t`{f_VmfWBHr{WWQ!sHshj+*BKg7Sb(=6s+Av%ec_;lQh;Hhl&L#0N6&^D@@z43 zg>X07xeOkRjLOZue5rwWWbB@({%V>+Li~YLefJ@H5%*#P-D>aRInCNnBhH4K3|!ie ztN&QsV!aq)8^kLguHf}!N3~=DOgC9ly3^qGDJ)leT1}kmdSfKjv5+%^CZCnFMTeec z2Wxlq8sPnT(r#8yY390_EKj|WE4-T)x*Va9vGZ%s&gGd-?FPQr6v}qr6~aF5s;>8^ zt`4A;zhqWTz%$#go_DZl;kStp##=ld$ied}=LbXwa+ssk%sUp7A5+~ZBRnfU`h zpEaWC!e3$IZi{r`T4e1ZL7T<~2jeD2z#3L!r#_ZpN|lHV`qBPG;btYyBhsa zr&J@jyyN*ok<`0VqnF3!v9s$}!92x0>sfwT|2xxRAM7&{kMK)!=?T0KI27sb;q0GX zUC$e&7@KT30VNmZtZ;Jj!$J94_wQ%@V^sH?F=jtb=I1JgIuQBkPEkLRG9Y6`+58j%mDd&A|||Ke-hkU)46dzP3m|Hj0=@s;OHhIcdr# zy&SQ>&*wDg<(rSlJ)99)=sBIr`?P6vq3TN_en|LeWW;bF<$U)6B2^>R?cirLEv*U* zCZ*;JE)*hP2gA;%rkX7JkYk5myfAhwV!qKX9$y#3PD-J#!HacQ?U9vNvR8{eZrYu| zlB}`$YfYWAhBG{5-Z!CYBsk_SOZybNL|gNdciMYu)h4WY=x^S+Q+AGKKmt?c$J!j+ zKy2j=r1R~q>F~hv{0bsW)HrKqlYhP^@=*oUaxR4nnb@X=F|_MdJGc9&M_-E3TXFW1Xf5Rc!Rp zgvVIXdOWrGfrN8~3QChWW(w~B*E;(gr%WT+4;OTJ9OM-knVsF zG$YjfeTEIMT|HXVybV{$fp${(0F`mphCd$k(sw7gk9Dim6D^K~FYXa}LNnxaTZ%B4 z8cw^Ef;->yBSKwsl49HnqF9UcB$ma#txRVWd*~Nu zVd1YY7np}wZ``N$_ok&)@^9z9i~kh1qv^;R#%QQ1m$7j}@oJi+_5p_78Xwz}M|KrV zTRbne>%Ndv^+7ozsyb)wwPloMLrRKHgnE>B=FM6RCnM>BXt9%kRW*;Gm>79RQ}Y_D zp^%7Dw99>~e6rchT;}HK4Tfvx4Ph#^V+pAklw%)?__qd#ZxkE<_+bcTsrnp~y*P!D zDLpuAakY*cK`nfkcX-$pLoV&*!H{w8q26F8>o+kc z<1Nm7z`lIsU2r(_@VP%q8xLTLgL|mxovm(sw-$>1*;)EcQ0ziFh0V8=ET_`?YMbu9 z({4Pr*bJpu!%UR}YOXg9)c#Ot?qUmJN%{EkMc2a_N-^6jztWfjW(P{mvz$V7A8G&b zwRBRbm#4oc%+$DQRcN?CCMiy3Kih{h4S)Q!O89Vs|J$&j_~Z*(vsFJ8zVFJer*rAW ze>*wHY2@%**EZ_yWF+1HdGi=9DeY5OxANX6KX6lj{G?PV)_$hrJDjbZR9p5;u4&XJ zm`&YR)bmn>=k+^-vBhqwe#wV;F)b(5*(F&9Nvp~g-CZUwlyw zn2R5wp=%uvmWU&eRT$4T%5N+0biGqTWLS8K<-wwE#w@W7MZ=%=Mii%4rHPSDe zW%(3N^{MCgnNKdMVF7z-f2>BS>MbWUoQQ2)eA0hOpE8Vmuh6f7DJE!txy}m`#`Fm` zl%T9lTM}DC+^wuF#UE=rkE`$>_~{Ybuy)V-c%#4l6!+!qf?|!nf#C4nMm{{`n+8@i z)m0qbjQtL^(%LD(S+e3e*>jZ#^?H``<~2UKc$t`#7f2+x(3(e&2D(ki%sm+<9+1SY zx*F@}McjgMvO|L`T8%olIBAsK@7|3ux*j?a_`XL&H{>_Tu^8)^pODQmP*VAlVz?tG zN1$oSqIP4-JMZ|x6KB%)L>%v+Fdh0G6nwi@x4l~KHXeU=Yijam57kO{OGPzt_@K95 zxX8@yfwY^7TygF4Y>QOH9iusJ_V?QRJFG)H_^6+{j*WkhP>@oubj$0?WF0Bu{xlH{LR?RY9RyfTnd9i(KkK}LxU-1}QQxJb6T~U&dU)YwUV~EG}=P(-W)NHq? z3fB7h#7TDFlk7^KS;`Vq-Qta#!DNAZi46BX9o!Wp&JjK8cW^5Mca~e->Sb~9hL8zc%k5A!nf8VU!+WWjvkd>_IbaPwA^Y^YHA{(d2xmXw;7jL z^)xrhbU08+H#)2}Awf8pw?{B#VgS!3n03YUu~x)n>(3K9OnKH82KccbO}S5O^y|J{ z-J7#F{3%bQb3BQEfPNz-C0D45U52sFpm9ag+F-AQ3yID1c47oKaRVGb#^vYD% ze-3m{>gB&%;%yngesEFUV^a96fVlAq3VWspem|AxadeF_2^6bEc4IX}C$qkrvn2+( z*@_2d#BP56b!cEo$VXq#G@rojU0b4C;&>=boR{oqAu$}3x@SZAM>jG1&@Rf8Cj`#* z9J#w{uw{>9l~%{{sCYc(+319%JAwQnj-n4Pq4EJ@S}z?Y6VGOro|swV_ZKoWAS(~k z6c}i%%@=WQ`SR+n57U@$xY-ZYlchZMvzT@+)8!O@qAn~Dz0O*$iQGO&~#Im zXJ(kV=K%R6Z}?%~WoL_fFY{GwLSzGca+(d7sb0-^Ncr9z(yIAgcA#^)s#fJ7?Y)rv z=T-AWqx~1m<|^;IRWGhG|9J5s?f%i0NPJq3+NQt+|A?)A@G&vE_+c4SbEhrQE+Ql496Ek%lX9 zK04axrf)U4>rvrvGpj;D`w0Ib!zQ1?x8ItW)O&V2l}sqL7snS&t@geB;524IoOexV z91H)U6_CHgyQw7MR9n$%z~37V@n|Vw1AQa4x?-vtw{*GtE@swYtZ%qKNU9BI)ayIp z)c5F*i&J(<%GL%}>zA*vOFj|O;Y``kX%n_)aL~`+ zQ#nfcd9Ax#QfUk|L>A6;3kg-T=Zmpn_~u=QUTP-Sznzw&9?tx6C*v8P!(d{nfi9pvE)sy+PytDjATb2AvmbiX>{w8-*}B0jR2pdzD>_j>n3~3xQE>A6 z^yiS2C0BKJ>}U>9b5Ewa&ufMqU6m+Q!_K-6y?J>@5YuhiIvIhz;%sXkD&%Wrn!xXx zL*Yx-$yD*mX2BNGzDf%WI>`8-jrr^Q^e=X`G)boG9`8HzTt0kvtzZf;$jj6dwzQ_6 z7&B5ft|(|=`^MKZU-pweJ4Cs*p!&2Tk+e_}?^V9iJsLjkzIO7%?Y)KaW!lBfQ8KR z-7wDjdVvz5g{;SFWhZ{my`Jm*DT?Q%O3Bh4=BIuawrhR-yOy%KjaAq$wo+E1RbjV> zajFjAxr36Eb~mpyOXrszO=zwj)rv6C$@ zXW~0`l0n84_0bA%OO4g5%&!<{l$p5=<39@;$=hS?_Lju6H(&K3zPGS+svUa6CU(D( zB2%+7Tq9>@jahfeN&0Mgr8TdVQBjGQWckEm%>=%b4;XlS5ez~cn9HU}qeyy}~#5CBg06J(3wtw?`v+^aR6UF@{aEOHQ1%SdnuvGErJ% z-yRj}tT#{94qWX_lMYldU5b0--Zb&B)0?O&&cB&^o1EsuB6XRm+2@u`Z;z2_%k7Ym zF6Zx_g}+W^o#Q#VC6FebHP9TLMb(u%-fBTC{4|8R_NBf!g8DhlkkATfQplpRFX-&rry65uN(p*N=}XT$c^w(Y>!$W&ONE zDwOUvcFOLK-GdVA=k>yKry|0OXTq21y@GeFQfqvSh~B~0;iwSH;5oAupTVm9J&H9f zlWjahZQM5XY3%v%n%4IXlAB^@e@60m^$CxW@;%+3ranE2kLtSAbUbK{Ij?zY3;P(? z)0FMzL&i3{0U zd;hZUTUWt%17CLC?(xrE9|{#2`uHnXvFHT9&lmAF=RBKy!^1t+uJek8d^Y-wI$~En zmJ)|<*LJZE6q`ND*5gUUq_IwRS>#l$rKaAQ6>#nT<^Z6=ufVB#4XXNSHVYf;P__u{ zoA`=f%=Os3A+eW3_$Wz^`cx(h2i*7Tq~uWk^=oMk#b-%8*0mk11EXDT`L2s^_$qLp zExvIMSe#^}+9DUG%cSw}Pu+|rI?+~z`?A%R&!>}oU>buXJ4{RR#b*_K+{~_uu+?syhux<0TP(qMa z8eyPxBMKs|ATd&5G$RJ1VPerBwb3XkQc@d(0ZNb2F=B)QqsE9$y59MF-{W}yg&jMd z&-2{(b)Dw{^-W{@sa(oCu=@hWnpeunpeU7iknMvj85|oMaSHYH-=XT##?El_jm4cjxCUiq;FL@Zj6F24VH`a}&DCV?5Duu@sI4JK|+Ox|&z zgfS{af-^EZ>b5Qg&T&kY<$+FGq6m2PKdyAgUG3ni?mmHw1+RO#jb6RD|1thaGH!%8 z`qx5f`;Ponvn*mM2zxZnauwQDo@z?RfNT%q3S56du!LKCxlaP=O$=1gr4M;$xc9LOo_|*Xmaa>h%Gd_ zM;v><_?W)Wrsrd-HI7doyPq*-uf%o6IS;{!)IlMishgBK&l6|@4;5xtR~ctjA@-@| zRVAEnDwAzLG%=YAik^zc)ObZZKE{+5UamzcRr+Yq8u$kKB7j9mOHh_5KrVUn3zibY0fD?R)R0;3t)Q|uSI2*I) z_c8{U+_wz9yWC+e+lX7Z-K1gI9_T_kcemfQu7`#j1&h<-^D|+BWm$o>c3xq`C|Am3z4O1s0Ty@l`B6 zo3z`|tbUf!r*{caS^mm1Z9st4B%s}4r#lXRp3DWDdiU#5xwwKo7~Ja-taA}pBYGm` z_1@{>%8fWAkUd8*l#e!Va8D5fuCF*dOB)%dupzNO_BDFwn7dt=_; zwgJDx{?&6XXjzL7u1+Vq+C7a4|q5e7?Xl(5OLP5ap6pxD-B&b-Iylz zFg9u-#>zW87GPs5D=wCr^^OrM`XKS$aY*u)+ED-*!aLwg%nY*Cf#c#>NiL`qdMAnn zqyZyl89>*(vEIC#Ajecl=xgtXohbCcr4K3PvBW{#18paBYi33eU$Hu;9FN-0c9hAKg zVqOvmx#4S)utgibFJ^3DkEatMzf(#i3%@;yhHyDfPUmq(OjB|2`Q195uN{KKmKKig zKL^ynn=CQYcScKy6-3cKNw+_+gkaM$MuoNE_S2?DKC_AE9S=C7R$i$Zq7+DZNbM%0 zG%pm(3Bra-W~z3ae4)^>DupJ-D{euZ_EZ471ow$^s|k_Rpj)PYS2QnsuSV9{wWrR^ z-Yev=SAoejzk<%|@-Gd0{mwwCeGX<-7VgWL;EK_|3lHwJEqv_3%mdy2V*-N+4@x3`xw?MO?y~b)nAR6i?V$hJS>L zhXGpd{0z~J0hYUSW#7)+-z-03HV|qP?0H~{ts(YIzqf^3 zm&(F(uV3TudXu~sIEOVHK&DMp0bs0(HkP+j#&Ohx#l5>s?wQY&MGag&n=Lgl9&M7N zYced~Z6gL`gaxb*jE2)Iy)-oR%&d*MhOE}NOi^X~r!JHCU`#HPeg)E(<&2_+!zGd% zHg~Qx6`dO#AVRSJ@MSdSNXpt8l8IMI$&#IegP0S%@d$D8mhy{#oPH62{_88D612l^ zvzvzZ1kh?0RkcWD#-&WZPw2Jpr3DyFvh|LUu&Gnp*2cr^Yt9r%A06t|^X1(Ug@-v6 z;Jr_FZ}lFuU!QbjMTqPA5*{q-2lhZ6Ez9+x^JPFWKp%A8y^Pi0r>Pbx9a&sxqYSv| zh>kTiYm&Dl6%^LJeBf$|WiK_+X}wU`;g^ZYLk-S2X>t3dV3V<(SfMeaTv?p^c3D+tWxNbu1mFqb-5BU*Ui`1CwX zsu3?fm97V^u&FuI>_KY%G2ik{8?*btqL#W|4}R9elnc1wOC)X3yrIH|E>6vZkDC{M zO$88++{c_4F2B{G3pfkeq93lMp7APf=^IvGoO8}>GQ8QdD;O%%@8;nF58a*YVm_nA zbPfK2cT2v1Px|M(h?AQY{#OZ)WtZt`4{ow9zS22$eZbh{61?IU+P8}}J6h@;awhw- z=dT7Ig^5$s*kaFwDG3e4SL<=M0oUf{FOG3mmZt2?J}Ug|>LAPLC&)b_MS}Qo&gP+J z;1olpCZZ84qg_)v8M5gs6Xmx5`SG|)oUtRoim^&UE+w_wiHjK**e-IGeKhxTukzRv zZMCbBBX4cvvJ*h0?^1K1J7TSNU;I%JAHHFX06t}M?BBh55B-u67L-ZfuoMuQM51PV zu&Ee#s({W%*_O#y()LF+u2z!MsRFXP@e^W!cBrM8@t5~RuM2kHYD-8tuJU6rt4Rr_ z_8FmFtx9oLc|+ijtY-&jGb*(wJCG$-n9TC7zFSTbosd6ydP8`2M1AxX0Kp-FzYcr= zuy`~3V}@zUK#NHwK0_Hpz1nsOeq)IEL~;F)(;oY|xva7?@E0xc?ir>9E2cFKW`PhN zpF+9k{JG<+RvD+3y1zcm*1D@X|EL`ysBPDXS9&B?C_8%gE*Fxh&x*vNL^qUOY`mgF zPW7|bUC%TM>RuMq*!^XN5pZ~w7LX#$0Vb1#5-Jm=Da3|3TL57m0qeEHvW~F zRE)n!gVLLKh#tb!a#3LIS&*~S&)>gXRc3h>wVM5w1DdKmL-R4C)d`$6ZJ_iQ3eT0} zxUv3=xNqNIN|rs8?4R-LRRP;Nqmm_ZZlVXy{L42x9Uw|FU*!x=fAD4ZEZ4zo#`oPe zqnq;8%&&XpVft-;EPT|4{wBF4iv=a0-dtb?jK+(mP{5ptY*txv0U3_(!uNn@}&h@*S4i2 zz7a+_?cjyaJ!UrOMNUJJVs6Ln?a&5J4`ZB#i%WUx5PR`wc`frSxZwdNwr;|_{LE7| zl`)VkDz_0(u2(iS;cMDow;V0-%m8w<#NxB+e(V4SAgt-E#OE3+$-7sPI+<1Lia{x{ zVx-v+n}8gEtE!=`S*y)RQ~&NW|y#OIjg= zvi;M_K0QlV4dy~()*2=fkHxL~B@4%D!aDQHe!o-9H7_flzy9Ey`<+IRz@&=dUY*ps zY78jGx*7mXvd1Qs3&tmcO9i#VkbV4-l$!N<-*`sgcKnTQ(ZtsEkSa^a;=B_qb;&d$ zB{*|2uuG;$Dh6l+C`0IC1Z23`Uhv;qGU-`SXX+!1T7x&;es#d2n29?ev*v>20ON}5 zA+KY{)CEkPnhP3f&jv%pUTnJ9E8pLwy2GHX;-iX-t+Z5v?A4*_YDu3X+9yrqpFHfJ zxXYtqRq$XW_msK561CR*Gp*T!)5_NaC8QT*=OsR~t(kW;XiYExHSPf#n|*LeJM(tR zaW%f_Q%GIHxJB9$OHA1&M03?@n0Cmnes97PbZyQ~b{?P4@oSR$=zrK$teNFe>~12d zZjc~(udy%k>%6Wc9;mblLa(D;z0s!KV=_)ye>8vs33Lu&xbH0JQTDEJt3r6Ywmy4! zc%ELQL3h{b(xzeAJEiOokAAJDNE%vJ+1QwG&ni`vFIpxDoX?1M_Tv zR#{d0lP3479>?+;rKYk^!vY8F>-vy^t?y~elQ!h>LmMaX)|G4Lk|#~K-1?q5wA~#q>6zPzJBVh|J>->XtXfp#oeSNMl|jF zPmJV?|Lhxs7$6ymmp3sPn}PoBHm2$7EhB^NRVA`><&t`ct4Z}#dWHn4o?+Mq9+{_< zYBp5@Hmqdlmn%0foRq4WdJoI&yH1Zd#H&-t(~ZQhnmlRqDt(uc&}lczd0F{lCrF5b zsX`OlH2wymh~FP`sI}~?7S0wm{o)$!PWjrL z8rV0RNEyX=77iv#Xii=Zk6~FQDp2_6h$+Wr{c?};m#7qsYKd>966zRc54aW!9|%U> zGo-k6^^|1tI3QWgFI4}?3%*VXY7n)hJ7|j+e3mvouQ1R<|6Wknr2lsM%n<0mnIaY? zJS~^t7AE;9=PW?5RL%dlE{b+I1}bwO9L=4HqoyG-Q3uDe6R$5y`8BrD&&V~8jmb*{ zk5^Z8m}!T>*Ww=8G{kI4x?uLd-4g#S=ova?M%kju8%d#MhsdQ%Jx=Yeo+PAl$?xGo z@@2q#k{g)W>5rF-hhO@zylwDlkJ>Z0`I;{t-Rx%jEpw&KFHenM-A;Tu3dUEa^LJq& z$QyNl>#GHvUp({GJqs7I1Sr$P-SnHDEQ{E8{HAg^YSd!JyGAHlN;^+oM{A+(QI}UJ zFAy$KU#I*8*8j0(%An0{p}K7LD|ki@mc|n+t6y zJX1Jnry(q6J37M^YMI0P8+a z$9XSVP(O)gbDg~8xN4&VczK;g?6cH>^2l*KjdJ_l%BN4p@!USG&yL7XP_+mHcKuqbyG;YLbd3jNTTN`Bic4rL!55rLV543(^snfL5q(_1Lr{EiTge)^Lhr`ykJA+62)++2l*# zk*>?gjGy+Ls-@;CvQ>xfc6$~M^<^$Rqu|C^U$i18v-sZtqJ)f0by%q-CDaD11comn z;u#_UiNF0Z?vi#P-GJOU9#=&Qu<+R@i-!7wXBTQ*jGJG1yt6*w4j|`8<_=lw5be2 z4CQTPx$C`O6ATcRvnMC|XfnaFUB%t&OSd746rWfI*$<)Y9BV#OLIf0Oa-WIR-sR8m z3$v(2}I`_E0PZJuMx zkkebA473}-vIhG#gO-=Dqw~L#tRMbtXGYU!F3OkLRjq)+uk(X$)qN2Qcetw*-J}EO z7LeRM5NDkis4jn9BHXR3ZPv_>8ZT_c2uESJ1fH(^MA)2&^c!cOM&`ynP?=Y`r&fVN zp|lbyVH$xz2+`p1xqlt4dkSS__BP4>j}Q~7;PFbmiW(2ktm$Oh6JFJ@*xr3+X$EKZ zypH$?c=CIJ68F@8oW)za6u2?wG8B?&a&0=Z`=$aS!(sGqXvq&~q1CH2ANk@|?gGHB&W!{{awH-{GjD{GQymEqJ%6y^%<1pE=o+K7v!a_Pg#;F;@;*Q!P@!DOF7ShqBO= zIoPSw4VI)6*oCap<4lqOPAUjSI6^0gJ}CX_+QevC&nl2MT*eXt@GB|HWQrvDUZ^-D zZKW>!-Q2fOCuk(uC1`nAy#y%CC)trbdYzfUmg2ii5*!@0V5|nk7sH3NE#L9A9E5%^XxvLsd1x^ zX;`l!V?9s&s@=m-y`jijkGWwBI8wtz5jDfqMf1y$mZf~`nC4Ky@bZTj8)0Q-54B}( zs>Yie^X95T6GIkV;5tjd z&00Kp&~n{N<3_ruIF`vr8~h|>|@7M5Q@(~HLq-tf<45a_1G$x3q~RA2NIV| zEA^TWxm?_L(ptNUefpy({4nZ&NI0E7@cS8Vh8deD9k8JTCCWz$AYsfSzL>tYVy(v)v0aO)ko? z=hj<{qN)98jM5C+^B#L6_P{o?*5kCwkfl47&V6*@XzML~iUMwNC;^5L-JKh3zj2s* zRyTdtWK#C>dOL?03-v{A$5{$;Po6BlJ8-9TF>AL_p-#7}Pt-L19d$&M?xXK&z%r~{ z((m_^?L*(F2bJ2NN#im*GD{?OOZ{$g(4=&!GB5IgHuQ9D>|=IL4`oao@kLi8brTzE zUsOPe@swYCrjs)3h5IGBxsrsf0F@>Gy?tdc0c;~WdtP&CIjEi2M%U7NCMQn+J}!B! z>dbA&6%Bd~1cfZrMU4aRkB0%-9yv`u5HjPO44m)ACM)}wzfwV3N3$uaUE-!N|90E5nD6pKd$}Z zV%pbrjW!~adHCDXHwsfwOV)ca2xOE2pIUby;+1|TN2|N>j zaN~GZKVMMSCUA|?ZB6PP|NKI=vJsBeFXFSM{c>myP-k^QBltiIRPpb;|3FW(ZH+f{ zaR>M6t-sLM*N05JiAa}!!GQ{)Q6%hcS{hW#ZHfE)DKrOBs2?Gt9#4l^7NF3hrZ=rm z746GGNWg}2Nshmrj?jxP0xB3${@Hj61$#1f4;|zX=i--Fnz?$(KB7yUqf$IsU|^2d z^X9BZv-pg0hUJW4v*rA;0UWKGd^6KAW7{H*Je_(+y(+to+;DRA%L9c$1_0ZuuJY7C zEi`A@e-FG|+jm>JyMDz0F#(860#q1C_;AEXU@L-$I$g(T(|!d+?Si*Qu)lJ#xt`O{ z?m|zW#j%DqV!^5qma0DEE29Oj(ujB9R|@w7r;MjgUyg|J?}dTlq+~ApM{Veq?URN} zl(79An-lg2nBRGrS5V@JGJ$nyzESu<2HvANb?w~QP9=NjqDcUI!YRA8NZx5x{ zMh*Tft*9`lege;2pYdIa8D^9G;}%USOQOwdY#e*j-f)i-uAft~zOe+sVb-p&nPfGS z=eEWe52!2NF50gOeFzNFe-@yb^!`f?C-Wn}EK#}`4@3Z7T=xBg?8%K5cx4J>g4rx= zNn*ZsvYqH1jUzmmyP;^(Zsc;s4mxRBOc1p!57S!<;fd0EFXyuzt%aFKUb2=ZI_^Zd z4L(jTYlN7@z4~Z!(vpiPcJQTLzE0$&#lDSrg!nI=aPKz{qAlS4SFN2<6H~lu6;8Ze2H|37bBnFnol(p;6QAJZ1D&?7v5EI3Axwl16f}*}EZyN< z!u{f(F=fywrBVsY)8oMiNXi}$gPMOm0e~tucL-~yWe0Vqn~Xn?`}LQ@UiYS7`Uw2W z<4mb3hy2Shk!%pO#Fg#E0=HwXgALeDev02hRc*Qr9DBWqwoaXkW^xvmJ_iSz5S;oWa?_S^;`5%H_k1-fo5ZSRT zb)AmWR_0sN3-XmfpE}LikZC?9L!i%8; z-*vwtZ?U=D&q2eXUJ6DC{(W?BtV-DJp61zEXT&D%zlK67o!(LZ3QN~4+a2qAigtKd~yuyzeN&nDOzItkKn%5YqkGy3dNb#q*IYXlfY#5F5r_~{e z9|N{mK78)jUCGw{JNN4`O6ywa>|^ijRo`(N_hN+E3?ZsqGr4qAQ~B)7bSzE$eRzBT zdBD=82HSDRA)G&h1fNDf zJeh@0jTQe3A%Ico+D#2>5q=Trvfzxrj@f;MC9f8#lF*|rjGPv70+i1!(W*=2&je8; z8_k<(X*U}|;(|`c0;K1ARfJ9qC~Bk-tqf@J(cZu~6}}!@9{#SNmN4S`8b6)ewakb> z*E$?nIr2v9?v5AMw>Eqdt0Fjzz**8$Gg%o(B#o;PEK_TbE9ti^S`hceQSSSGRm$ZD zz4tC^HSq%+4olRjf!jVo@0C3!%S#l3#0TsiF@YH0)F|+I=#HK9PHwci2C#$c;*i92 zk}{xKii<)ixF>3Ot%h1Ctcz&vL&Q<9L@7g_cMp7n1TT_5>VnWS?9il$GtZWLxcdkR z%lkazn*xDAVl}bbs3s|5E-mTN5@T{r73l!OZyNdB)PIR;#JEuB>EFo>(-rHCVT=)& zNi~#kC}AspJU2WykNKxix8k9B2*c;q3TLu;!uH?LoM!d^IL80PG&|9SZ;iQZO9&%( zmkV6~@8K_GB-B8@AA@2>`G+f?4Yz=fhCUko!6lDfl!ePEwCRNuFVv}(pXgbvg(aSM zYkBCM4gB?VpGsQY1gZSg{)8`Xl1=rIE)AXw8-`bTK3yL)lYWD{n zd~oA;t}h-J=4$T@P!~PM55tc*3)OV`zpzolwMCHSCqpl@iV3=x$rrx)wogjfB;V<~ zb=bH#8ER94B=89yuYH(>-wDQ<3PfNd^Flfv6cv|3WpHf!4 zOnrywCPQgIZv#buaqfKBt}oI=mDbR(qfhbEurzc^Iy?+Yr)h=`f*Q$r{*MOxs#3*qrt;9iKBD1bP>yWhUBgYDQg z20aV!V8#amPDx0Ehe4w<4~KkHgSYN&M{7P%=0@9heE$a9+tNs%^&7o?cJ8;?{Yq&( zdO#bz(|kX{R?WXcSzc29jboG|i5IgXPFpV|Xd9;x#qG8=fwOg31Xqoyj#yB$kPf0z zhMWSR#}3EH=t9aAk{l#Hzo*X-=8MtFD8zBz#A)O(PF9p?Myr$(E_Ut8;-)maT~kGI zy818kPdOT$4z79cYZzRk-_i;O{r!Oi1~!7jd|Hp{2M*l|7PUOb8U3NxdnxqRFUhsxi94FW@_=CT`wrdHEAw)^Ppx5ZmB2A)uo_fl> zyxQxSnwjY)K9rJD3z#X8`pPskRJKNw@?pQIzgMi|L<-UphradEoS`a~%sUEI(bLe? z;ZKp`b-yGQ4^;_bYGOdXjRvTR9m1hS6n^aJ0=RZG%5MKGKi6;fro_hFcQnvZaLD>8 zT|Dd<2@o`c)wBo`rfgZ+Ced*5)4g}bjZU)JkH!>{@pyl@Bo51oFV zy&g)P$?Lg2{kwAQriyyg+JDPq@R^myF8AUpL%=PXrIq9+UHcS}hUmFh2R4;aFY7qG zo5t9yLMmAPi}y!Hk0tZj1_?$Ea7}$QzkaT(RyJM0wQfbcvB`V@Hna(~ed@uQD^^P& zF8&BuR2mRQzlk(vU(dMP3zu5c(n)2G8HWqxv1%bBRV>AJIn!U%A()>hcWdow9UF|W zXYzWC>@N?0nQ!?lAj*=(8_2i8glaNqhS&hK$RWnl`>o%-9XPk`#AkN>(5zT@oW5X% z!k-fI0U4%07Kr+m8b#>3Oc4%6?q2cr2VtGTFInwg{-q#3jc5mCTux63N;KSA8 zKE}YNrgJ|;nRL#Q^R&yYXPec4<2hzG+H;3YFL>c&YDb}gM+*+kn6rME!(Sv{J83&| ze1UN@gME9Iu?a`%AbDpJLT#G-blZYP+QB#mUF)aa{2k7OX8Qx3WEi#5=4)WDo2e=8 z#a_eQNxB4!Hc%Zd9q0;b^+ZJVSvwW=yUva-x5!$5Bebw4n8#ex)AM;AYTv$_z;4B+ zP}RAdoLDZ7MwZovj>U{SAS0D`OEu{$JX0EH#Z%dVX5lX0qtAb((M?f8y+`hVOW(Lp z+98Hbv!B%VPm4uha(P1YB0EDxTN_+mBM{0_Vhv1{`2(hPehp19pR>_WyB2Zjn3AH48Rl>R_`cyT!oRuESQo!;Zhx}D6C4PxwYE*tCZGC_-Z&1o*tr4}&%T7GG9JsB>e^KR2OGlt*(+(U> zFMdiDm5`7V4Z{3m{daPB{++t;vqJWU_BiPME6*0GqZjj~2wGO&sy|n;0bGVBu_QE- zhhNp`OmCQaSN1ZYP$rxfn?+}o(_o{C| zM&JNwVqc>iQ=_|W9e`4XDKn~foy_=QDN_RDMfSB|-8n~~4}TY26nWpC`2J>b_s&J3 z?sJb5;c3Qo%2W_b)#$m$0#a=3! z1AVkHY!Gq^&OJCS5fN$C<ps37Pka4pmY?>~s1R zE=gkss9KGf&6S>rQ;Tx}9e2#0!>XKL65f7q-lpeZ34QCm*p0lEbe;9t=DRV=k94oJ zE?sMxa;B_(_+lW!D}f_Z@u7%4H)Pp9;ICC0=s>d?-)I@u9bf%wEpp}Jyu}s^_o(x* z9$miyH|Oz{v$*5N-ORKW!v&|ndGOT}j+2skb##lECI7mAjL>o15;dGk6w;blFLo~d zg{+YXGsKrYkj3Sl7KVD&<6ToTd#0lQQyjLd>k<73#`$l|4c?2PEp8xO0q$}wt%9D* z5pnYVUsTs`*3@uqy5k2ma%sbGTV5^1?#*Is4`_d#Wzv7-m*a0;1YktB#Q$x`OKL@I zt#owTrBGlJz_i4Q8afDZB`G$njB_YtKf0)`Lgxvr(hh1oWUR5Ar|ddwzbcc#Vr1Exh2G<$0h0VtvkV5Z(0CNcK%<_lh84zly&u*Q(*;1 zF8yfdm@47@BQ-ko6beX~BN|Km$NC;uR!n(NSWwFjfvF3^s z1I0H2Vn_}hptpv zSX8R)AGY2z_YW@G`h0k0vIvbX=ZOgnIVwB}4n7(fERj8I5*)}oaQ>6@IOIzQ>~#OP z{XhWP{&|!1RFkBdsO{4s#%6dNf!_>UJq%5PX2j-g@lLm=s4nJ}Nk3v^id0?iZ2LZtG`R zJYY(=S~b;SdHaa^J#3wmY4?$&%t_QB&rY$tC#_5rew8y5%+m1gZLUGZ;Dw(&u-Fz~ zL(ie1b_1>~Wp8kHFSbXL6&5w~_P`0ZaONK4vZF#U3P=ett}aH8=f-=>H4A2rDZ2{R zAzgA6OL4=jOLT(6-`m#Kh}gW=NHU4P>tu}s(Mka^9#_oZs-)QXx4_Nctmx}7{gR51RGbE}bF@{pCaHZ{qBR*Wso*?6$Su)c6C3c%Zz{kQ zz;m zh%3yRagT0&uS5XRXh|u5>tkQ9ZIk(Q&s379+Yy@$z>g_L&#S>YTkooWb!r2$K+X05ZI&4YLL>3AE)pA^A$qZ} z+eU-~&YHgG$W;lhN=Uf_ziPBv0e7#`jyDHsEJ%b;7*y8%H_(#zNj!oBz&$j;s<`@$ z$KQ*1mmxTL)Dog$nkAlV8{pfuZ3-rM59e3=49HBdosm*PFv(xyi{J;agPs|zTC&e;NIAvq@ zE#wf&o61!&CY*M68YeEWMs$e##mrba#b|zC3zNnEI{)%Y$-m~?l8hU1Xr7m8{mRI> z6P{92-Cwm4ga;5A_RT-K9(jvz}Jr&--2{dMYU>$vG2-8h-EtzQyQlog>Rfm{jXP z3KVo^=ME)kRgUuBA(-- z>8tkB+w8h-Zl(VQ!wjffLv@8knzkEx`QcUet|ZVGtq~}%T7Xuhmb+1wJ!`}m@6PUd z%F{o)M%A+c*t#2%@2fJ(ENgvzs*dp8t91r7%I`XXvbmO5(1^=<$&c5^!V0Ba^E-U! zmO!}6GHwOKWH`HRqKIdWB?00H|5*s;xV4zW-zRC#nMw*9SYNUOE{I85$3u-Bk5tcm zj*ob&dp(jlbaV^E#p3q{`27^FgFrQg8`i_lr>7R}UrEwsz;4ZlHJb zx@pzNTKZ}8b@Fz$2*$_rfo`c5?zScT#y}5ari$*mU&KT;qZus?%d)uD#GuwRMR| zdrf2(ibNyy&dPsQCV4epbyrGEBbCvr>9S1ER@}t`>(~GML^{b0z_Iy4U8jF&#igNl zN>U5R`c}-+DpING7<>WtK4515_-MZPX!Or!<-qFix(1&nt)cY2mgdO2l}`I~AE6?R z!^6Yh5=y?iivIpyQPKY`#HP$Dhu&cJuFt#+fmBDIM^^QxmHe(|w<~+KH&Ps3%6d%} zf3MVjdKQ>}Pg)h9s#+AG#~)u=JtSZ&{-5;~+)GI_xS1f5)BVTRUaM_27X@lC&*ba| zru}fK=5FqHOWVA~{xCh9GB@6m@|AoJIlwCF!YA|)%TUw-ChDfh%4WrZYNKJJoYt&b zFA*Nq2cU!+__aS8rVpYUlj**l&~;lp6h4}Dlm!aEOm!aMY!;GmjU5CAe-|ud+ZO59kW(fwDx4bl} zy!?l-L2Fje^25%K^+Zw@AyK~w2EnQWk;ne9>3yc!?BcBIY^|_L7x*)T8HLLfSSdEn z%>FI)wDW2ISa=0)@rzN+Li$}wpo%>WZXSNMi5&)IrVY!hJRD+XF?m{L`v#fXRu6^o zi`CHF_nUZL8sdKuIfsYV`Q6yJf_%?G?AaAbSTey~gSk6^;jP}eJ!g7%lLA7PmSMJsvPzaLAE9gppx7~PmW9cr*pj?X$L2+? z(EN^6#jmn#<%%U8P7J|E8TSsEYRIOvpkVu2U(yrXQuz=TpDI#a)UxlpkMRprn#D;s zT&y2tqBQ&*@vRt9+RV6WI&1e%p9g^J-p-#Lh7BthtAM0{?Ds^*s-CA>@ra1;@461g znG!>pxJ&hsd^Y&4V29oLZLUt`ZPVoEpv+UwJeKpO@xj7>#sKqRr6Kdp^>Wl_@c5kY zpH!K7uv}t@&6X?6sfo1j1@Pz{zad@V=Gnr0S0LH1fhv1HIdkgXoO9?wjM878{F4e6&CFiRW8D^LiTg;mr8Y*LC!a zr1iDq5H=Oe^PZ-1UUDF)(XgP0Ec?DoQNH}^!7L79PEgg@neMOR(D(cPb zhj7fUeH=#4sZQQ1SI4JKSfQbA+E-)J!M$W%=BwVblxJb}BOjeTT&;L>rF*sYs~>)g zxyEID&KI=x`&YQZ+YIHuw=QUjuEhA+da`)FUZ2cs{TbZW2PoDJHT@V8=7(|kZu9W$ zR-o4SyE03`_UQ-?rHKeS0^PEB<(V5{_lz<0i0;|Rs!|%*lf#~HO13qB=@ETq2VHf} zfh9=?33e~DYIzWHpZ_OrJ}JGpM?Yh;?>_dMmg@KZ)P!YjK7rbvQ5c4t%7q|hOmUv{ z-#GaIyMhOsgL8{3F0fSZM~HIa_{pZ=nbQx@u8nq8P0>!$S}a~8X1jmdBYL9RxF$=J z_YuNR58sc?_`WuLV~U^2e+Z6S@x5=80v6Ii&8z}@yd7@})hct~s?VRq8*IIp95Vgp zjL0P5zvq{T1gXEUEf|Wf%RG$qbDSqN#J~S#B$aGJ^}>3kb7so%H%+9^i5zpG#pcA3 z)9z7Y{_U=iD8)&Q>$f#27z4K7I~sA`zLPUvQ-1<=e3DZFYh$9jTdW(Ozps4m$1U~Z zSuB?XYQ6a*h#;2aFuu{FxY{YIhM#>63AONAZjpD7|GWt{wTw_`y6=JN9e|{TuRB-m zl0F`&=gP^Zh;=VdG z4wUT-m#D|Oh1y3CLl@@FK=sgf$}R)42P)a-B-Wfc-9S|d`V&i!yGB;$zO+!fJWkEU z+0Q~x1tmhW<#e7h+tyw?l~?0q=Ni%bq!xX3CU=xskN>D*<1D*%VGacA&k(-M^Zq)e z0BbCGpjwQc2w|zJi<`Pqv7<)*sZtLu5m_5FQEQF!1gG-3k`@bFFSyVX;^PT9bwTCR zbH1^!f^qFcT;d3dN7Tn}RwEt>B=QFn>b(gCyYhh@mpa{`Jhxn(Z?r!)GrcL<0b{7j zX#zqb)ZDKvKtfE4gj=)}=j&+sox1Uzi5~W~buj+U{LW|(yO5`f`EKT@Ec(Gvs?58E z2J<4Moc@7I=XaJrrN9-WQBxA#+X>-w>*qB!HN)Q;A;PE<7g<%`^VKuG_9WdHdFh{K z-tl@0N**X3@BaJ+x;EgU@-%+hCGUnDmOW+ytWfsGtD-wI; z_^x-rC%_P6m&NtP7||Z$RB9>6`Dhn<-6=>S1xG)p4&kD&vZ&*T5nPh?s)_D!P5kq* zO<*9=$zvovpM7%_=>dqqAM?g{?cQW29VqqCW~F_8zESkn8#|qM#)@7WIF0NlYo5x@ z*69u)Ziy?KU-PLZIOav0d^a=6tW-L-ynK`1EI^rp&nvv&HN(NDjqLGACc#ZI!M62# z&ywU`H2UqYK0+AWyxk>wr5;rfVEyBfpUx=pmSW-i3a%)@&Hmiba%FijERo;NU9My? zd$eP-pV283E4aZub$Uf%dqJ0x)Ia=|aOE2jj^nL2`<4FR@_81T z_qLD}ztlrVyKok5tfYxGTb7dZe0#0CZKJOqmzQP&SP5k6PutKGcpXG)AUTvRWm9pc zzJ9rfI*)8HX{7&0;pr)Rf}xMzV0=(dJ&S!6StXXqgY!QZDOlV@&0OlS^H$8hpU{`` z@77{dkBsjlnUV&r(rF(#69sK#p9ocsQx+vreJ835C22QvwXaa2-JycOO_2S%^B&W! z-t0r$TuGt`zIT9fZ}38;p_=o92CXarpX(={r$1DPALbM7#;h`z_RCm+!xnrk}4XY#SXXD1F*YQQ|^hRAitI`H^QBCW<( zR}=E5ij>KyY_*p0!8c(EAvR}E!)ompsX?D$S*Bg?_S;==x%98B>Wc>Pv(I%<>cD&= zST;U;I_9*yh6`my>sCOHUD%k^1qW2pP6xSL?op26G$nAvF=^T5bBJNnE7#a@Zri$(US#f!b6t1r zv=tNe619tYA-fzxOQ=$K_UMP_afyIuH&TY*P4A|O>vEbl)+)}}p%e?g{pgOhKdYSq zDeyPGpdaW@7K_{FWO&6?N!YlQ$_E=6egEE z7ZWZ@cLpDY#MSM_o^?e@{d>|jn|$_+KFjL}+)H8B^&#IYapQ7A2Dkcc3wm`8b)gY|pz(4W;P{G4RMyECS09DJf+u3=$FF&fupF;{gh2m14wJ54(Dm zwX~9{T(=UBitu$-GwhIl;md`_WUY7)Ttr>VE`M|Nv*SBFM~B~8+8=O^=*3h29fCB~ z+C!KgnX*Syt1%4F=vfo>L8E-8SUR%RJZ0s;Pjsw{F(e5T;PZ0LdeXda=)E0e{Zl%P!K-6r?Eue<)MB`Dl^A zhW0K?&nAcFSsI3mpNU`TVbORq-c`Xblfi{R$CX8JStwbZY;MwSc5QmLRKb;*^rRj% zAB*`I5ktco1ecMlUnAFBjX_&iZWV(XxBnK~VdAfC(+VS(J>@nwS&OOJ7PTRT>1a|n zAd;H|MgCX$K2Pb|#yQZ1v+5R$#bPG^6Q+>&2Z#?bbsAot%%?dpSg<&Gk$(QIHO9XO zb_NN^3Ymp5vj4c@uGV?3aMsVPefJFhbN7ARxTab@J^T>GQ%bV(V)#7-y$K1%%?@Xn z-1pC_^7tZcfmrUIYb1wS$FS{PUwcP^nY`EAwyg!*&gFE8{kL|l$s;Mw8bU_1DWQdi zPF5=hII^u7_jqh`1o&`@JS0J-uY1^Rj~jX`|A)Qz42o*o+D01{-6j-JP$Y>0B1v)% z3W`V)5s-}JoO3pkLCF~;XC!AZ5F}?BXo8Y6LK7tRH; z?OMIonscr(pYe=mj3IM7;=Et=fmFT>mA%FHRgqvydc)-jYqkUnV>^}H)$w>Sj|-xf zTuJVHBM}iK#%5+=2M6D{ETDZZ#m11d|60Vz$_Dy{{)>}z=)ByB_b)Ctjj-&OsM7|j z9G~g1bqu?UPn#aQW_?!#Ju~)Tbi5eTx?xB2wy|h+X(UZ&ckV6Q&UL{~hMzv8W^Y1j%VjQn_#S{`LDhKSOn`16OQc91;azkYPii3V)Rm#Nzg*GGUN|wrL zjer*;sb|(p{^&B0J90F;aQm?39z(vi7YS;T7u(JuvpxZd6Rb${+g;5(mEs3M{EN3Z z6qI#qAQi0&P1PoQK491^glCqoP=C$x%*WoM3k+A{mL5e0jSO-zFus*O`8HrLBx z-jS31ayxoOli!|x*m;UVp=iuQJ|Gh6Zj9zTjXi!n7rA2hEsDsX{FrD3vS>CHKjd;DjYjis|i$%t^ zoKrZYCik#Dr<<$fb%conJDmsZCi_~d)^&vApT=ghbp5#1bll7SK@hny|Neq_aH8iC z-{R0RBt7Wmrvxr+(-n@ZHCK&pSarKD&|aP~-@(6}uOftR)15x0Z)CkGd7dEWt9tSo z6RY0gD^go_*VVKAfA(HAr4@LgQoXWLV!e}}65fAe3P0oF?%sv;n*}isiNn7``Kb+W zC}@^m-t1VM(%_f^WzC+b_+(Jv9w==d(deUupf5M24QPg9QoloP3mx>@e5t3uaL~og zVbJ~#3cmEV#|?RU%IIx+#NR%tRz{PD?=;IcgFjZc@@(H~cbRee^Yf)GvpGK8n#dB3 zvz2Oy)Wr77mGs(+1+S(xmb$vWJ#20&a(R+g(!kIp#qUh2n-n3-fa(@7hl+nlqwzK* zf7tV2ifRU|YGiiyZH~vwiTB;sYIhu}u$FR*{hy*#R_lcI?`sRO^P;a+*`+2e6)=c1 zQik-tEO)}3XQ^1q|J?FyKccYF>GY)^3_3!nc@0hn3}oBPKy$w^=uXX$jCxG(TywOy zG`mFMI4bW_Ej!oyg!2QpLb;`Ndd44e^&+ONL)$FPW5r_$Aa#X=hUaWrR@XI?r6z@= zmP0Dobm|t=a+ewm%WlwIBM3Ouo$ow`8tXDBZh`*l`p~d&16n)Zzl2)!vmiU3Z4@f% z;-BB3d|M;*{1d+6>+$!U76M8`#>zZf#-7R~t(xR!c6(>SvHWQ=0@B1V(hD~ zNWZGejoGvi!KYZ@kI%HpU)e~qsw#``oZx$}Qlb_i8xk>S<7_x!(qj8ezqsdB3Mf=f zNX!FJYbAM|QYJf+<9MPI>GwTfoje}`&81~YSk#SJOmgzd*fsFzW#gh&HNsk3A6@0K zh&zC`m|1Ezb%*Yrh*C<0@L9(8^HzKIlmDNnL?LIBuFSLsbzC_^3@v z2}AkX0<{P(Whr;}SGx|X&oQW_q9kw5a+lQ4G4!(K z;w%aSntF*FG&pK&-QbsZ0tIa9$=PRnn_Pzs;4bld1}eA>QKE=E_>Io zCY%v7DdbxycpjDHx>e3Acj~P@doz8an@&|d(wZ39x1&{0w5BUGX{q3W1WRUCbc5@m z{r;QF{_Nbr!(jg!mYYaZtI}0tG&HlYGvZn#QcG zmu-A=al}5@64YZ2mxVNTZP7J#&A!6tchQqMBYT-Sw>R|ycxwGIa*l+Wbb+LaLi(n| zm~s^px;xD+tPk39Sk9V$CmIb5wpK$ur10;WTN-)#MYeaikPDL1pdmZfUYRPAq2zdy zgHJS^^NXM19=UIWo=^hSrhKH#PyK?zsGh%lejeO5N z?Htf4$%lM=OgvefB3b)ubBu_WioASJmN|XN*cHsGKM{Yz_h;#Nb9Bw@6jekQ>=Zo% zvFwLyt)_~L*3x|G(O%dv-I&w7a$-Al0ywFX)^*&rtGh z5_@Lm!{nFOr~Wt`d)odfmw;svrDdqxpQ%A()AoAPtlk}y{ktN7k2!s8mw!a_@~r}N znxH`(xq$>mMw#+G+9}A;vazydUHa5yO~MxVl`WdCE3}{k_N`be>5~mbbdJ({D}mMt zO-{<1Y!%q`@Wtb735)&0MueJ=U%h{=h3g|UrTy4Zfsw)TvaMwSHvZ(?%C>UWTCdCP;ag{Rk5pg{-|-J$ zk}&)JUggNcmEP?sJNkt|hc6UnvD{54FX=%v6x&9jHfv!q9wA*dcj0BbN4L2O?Xub3 zoY2(~c0eXCMQAB^Ml!@m+-CY?VPjN2!?4f31DABk&NU%nT2%j08C6}?eZ#VeVe{q| zU57CA(i2LK7X)q7#CRx;p2*1>Hr=Lgkg7xvJFwluxQv`ibEuWni&5VESXwZ@p2MPt zsY{lq;k9&8R*HB?)8v2IFwi2nxT@rkckGR>xd83h8yF9J9sKhsQCxpcO7VSvlIWMk zxw%XbG?U(0-MmI7-K3fy)M|)Ay*tx@G&*!w&aG?-Uq~BvV4~X=uB4|36;!>EI#cbWJ)`>qBv>vrIFtxWk<`=qx((@hd^- zV*>Ndv^cpbeIg_HCsk%i(_ z=gSM-)RHV|kcOS}uDL>mg#L{>`R>A6-^-w1vr>Z7A7L-yeRMehli01X2KKHZ@q-bSL3X{oSuF1^yy6)m(iVH zQ^8Vq@7^6vv|cW3e&pcbaABDlexB$*4M}h#X$^}dPc)BZf*A}+k=;VCEG_{0O-njE zJ3rdVDInvw%dFI_ak6gp+=kAf9ri%MwR$>uHmpxOZ7apal6Q^!*%muC9RCbegh;G4 z4SsO>|287O?Fr{4YfYHDNIoZ-NvCatxJ2?cIGls+&-!TwKeqh-I<{nPs)lj_#S0y{ zch{S#W-8|rLf7hXnn_YhR^0M6;37l2s^Y_K^=d7~_bH!gn z>xjh5vs--^9Y(MsvKV!Pg_IVTmO2i>$*%_zu9Z~F?S1DMbixx2`NwMcz22y8p$>gF zY$Z^CjhScE>ea4Cu+X;8{2q_};!~AEoi~Eu#u$%C!9Cpc{}1<|25B=e)*j9fG}ecn z&KqNn6D`*)Yp&zt<6jR(#QK#U&~&x0Bocn%A<4hL9(rX+Eo~_BHVbt!3nF?_Bw`Dg zNu<8%!dq>S?|HbT%z;JP0@qUg59fBq3!v3O`!~o&6&f0r;9Lr@QT8 zBdJyc6AH=d?AUsJn)uiYdbm9oo_}8+-^>u~1=zQJ3)CzUg6J+>`JWcn^1_G}v}&!{9x=6<|KQz<)T3%;EI2pn2+f;gmcS0RLK6a}<0sEO{D1fr zSe}{L+1YyFC8S^c3A_6H36K2_rb7X}Zx@Hbtj=Vn1cl`0=Gv{&z%%i`_zw$3S@kR` zyiwk4XUXy}%3Vu6%MN>UQF{yxmBSO5QH z|0DvgTn8Mbv3C{5J7i;HBW#5jey=L%-|uNJfsz5m#K^d`(e+VPCGQGY-*PDW3)c7F z&8p|_Ok~0k(puf+Kic?`0XXk(AL`@s&{X;FAJLOYOUA!{yv~W^QUCo56zVF}wEyQf z>=*I9{`rj__5Zym@i)=meX0L;*t-`9V%LJ4Yc+|dY9ZbK z1+OK@(NthEpi-Cg|;@bD2RNiap%^^2z#|*F$f%~Ix-=1`y z^rfevb6Ey_FI2Sl=M;)c0G}<6_r17Fn32T)KKHO5u|iG8*a4CCuOHrRNaIluAYYgk7F1RQxt0 z;|x3rH(PRVg~gZGrwz1!fyeXCwv$idk(5;R;SQWGn#pB47{zOu63ETYCqyyo4!g%B zC0$0J@yVdlUvl(zah^xJFMKJhV!V$6-rtaD)sNI@x(=)6;VbYVT40Ybt`ao$U~!r$ zbHexH&)Uf??hH9a<@UHn)AIW^!amd_Ik*Ah{&p!>jHAg6E~)M;DSr29L@GGgas&d) zA3uGvNal2U3RP%KlAzKf{b#@x@{b-p(gXi@1A>b5)d(zk_yXX!=(-krg2$e*Z63bI z54K4_ftGadI6=>R`Z^pk@2qB4_=ZWVPQ4Chq!NB#ABr@b7J_$ZH57I-nUm+q^s2tZ zSvOrpSj1MK+?bG#9?tFdXbr_ExVw;%I31-ML>SzB1ED5mR^eP~ z;@>oQv6hBL{H{x%3{T)*lnnQ9gE|hdzfj@t@x+s1Dg$+TL37|W(k-?l%v^i)S2mc~ zZx{K8_5u7v6&w;fzGCZv4UhV85FnAlW(K3xO|j~e*$aIcZL8I5loew@D?;5r2EhF3 zIn;q_u%6OCg6Z0OwYX?v8y_C7AF&sho}JyQj_&Wjxyj*8KtYDBgY2m2G)bj=b%v}m zhn0~qT>DYLRLlBu9xo4HLha=_gUK4rf@gW{Tk{rB?R%D=hX<{m%)rbnvv@$;xg_A! zF&SlWtWzP#nC`pa`TKEb4ictk7afzEPhK$Bo+~JyMs7Jy{dt!Ro<=(Q`*w_wxaEjWSr>Cc* zOf|TX4Yq}LIlp#$B=y3uieWr0KyJ@Sg3`7bc662cLnZ?68`Fhw(6_iR3h~qLBO}p2 z#4q4d-#JCi_wYOpYhPdP;1%JEK%PtozT${r(IDfchlWtt%;28OV(80h&@!o$Pko=7Nf!P-9@uo7IObJg_7 z7FsKD6Wb|>l|~GUt68$QNCB(`uW?H#m-pgXJ}?D~x+gDCn&lH~m~lB*T|3; ziqX5wH;fsr(-8~7vf!VC*E>|?;Kv0?ku|VqYc_b=u)e-9mfJtQc_jb_^TIy|ljrwf zriV7_uyH)$e7+0&@q30$!BJK3M_g1+fUfine;P z*r2XKG+OHU^XCn^2U56*urNDkS2$7#q^guQ2kXtTH1$BdNd*aQKO45|lR3Vl0UIj9 zWcCRefHygxr2zvXdIqtt>A%(x_uO^0k{-$lz?Q2U#Mg9_7}mV)utBUjFJb_%iev5I zv?$-LFMux|4~g`M(Z7c8sjWll<+vm5t^l%8nJfo)*#O>Bi2m9)=z1KiOW4*zj4So| z1y~^=5_sD5qhO|Aly#3Jt1>_PD0;MpJ7WDA1qefZ5YG9u<4xfZHR?PU&#v2at`G0$ zw;RD>pI@DdU7PU4xnrXMLyVt@gR4Hgu<|{lMWj@(g~;|0c7|{LSqjI4d!j7;nalog zD`;s&w#}p6F3(mlxE4D6D^IcYxWle?n$EViP{iijy@5d&HIUhav=W5Kp>+C=$G4Cg z+P*?I1tBzuxpK5PwuUh*j4Q(BZWf+M3>v6|D8L2;E1-RJm3scCR`aSa*WgF!l29Ls zOLVn#b#=d)16GJdECMtZ#F!OBng+#BcZfy(E+gnT?koX@xbBlX@M{LtyA@MGOQTxI z^fG)P=J+yIGmxW<)O@=3tFEewf6|#}%r10cHn-6eJKMoQ%4OVpljHQw3T?z+5n6*b zJLjY~pOqNBuRMZfDps|l;l4FM=Yj3}{JppnDs|T%I`HjIhaM*u+JySpD)bvBr;L6h z9X*ox_7C`8NcJROBZd}UNOtI!QT+O$zP`Sp1GFwjF>Hbt>w{TY{mE7pXIT>c6!}m4Vqq@%;$+Ea5OF&(Zn%&vh7i(QYJEJ~x#*uS=erGDzo371(ThB$B z7LQv6P&y28BNy#8SV9;59ZUxE-A%yfeGx9S3Q*ybZpk}2TYC?9uS%k)*JmQNVkD_X z7KZC>Adjg=o6>eb5sDCc2%iGtQSZbVl)w)^#{Bu-yA!am+mV~DZIcyP=}((8)Yp%e z?S?PrH8jC&R1ga#+nW#iQhFXPY0Y$b9u>O*y9gl*-KKzrrdl{tG`SNZ4nDq=BDjNM zuos@N@TB%mlA=b;xQh?gkRpB%iLD&9`g;EQj`0EMfg56?gN}*9HIR?{2rSy_%>&?* zGVx!+7Y_@drj-f+U4FOrpFh6du*iWz$iX5<&&FMr@?h5~x%I9Eo`SmX@!!4JZ@dW2 zDsZU2vRw{ZzgkrjN$LUQ-b9>iC-|8?Muoo!(*AElvk-+bSX zMz?_xoJo2`5Yi_nqOt67ME;LE@@jZ&9g&JOU=|Y-Yghq*P%8-g-ETsbSWfUd;QUE8 z5PKvngRkE^G<2oHe08)Ep=e|0kexg|3fV{()`1(pA3ty1Kq&ETs8CnHEnjxkX|qS- z2AvDw-qPl6Dmd!B2ky|(86c&u?7Yy?uc=0Z;zmKc-?bViaEDsragKm9AxeYi{jYm8 zzK^Gk@SJ*Z0}-&bDLe=yqg7N?q%;17^vA=SKrjdQsFheHs2{`<$_+Zm6v#6hJwdrp zd;lH}PggvjZMwSEXMBW&_?CFINnZ(Dv%>BUvhln`96aElWgAw=CN!)YF9YC+MB=(v zw<<*rH$#EjkpaEM6`Os4yR^;ZwgHGPPp4FM7koZc>SrLxQ`6FDr^|q2=TK2*MQ~ls zMk}K{{p``42#ziE?x=<1!(~TcQQ80jSlYP47B94cz)R%Q^B5JCN-Gs1&NLuk(0*n3 zxi`g49v)>x7~u?r1Puge0DP#_x|Cjh=CM+WzDZBtJenV`_m+U-w(D}Cx8SeuuWW|( zBcqWWy}Ou%S@mYDMU^r5(Sbb>M&v*`EJ*Xr!>kKUJM%}?bP4ZAO^O7r*x7h{CO0Ds z1UAKrU1wEs!2Vtd;9D=XAP%nmy|wc^qtsMxKVL-W=4hQ#t8=SdAb?TzCb#in*uyqE zBeioGP9V{f8Nh|JU1z{|pd-sMUS+B2!vcz;VoSefdjH*b*)IN;@{*DgiERqlZxw-R z+xD`*cnc_j!gQ1sFdg!SXLA5z(my_!%Q*cFVXI(CsX4Xu4bWdYrhRjvuS2`2 z>BUV1HM@+%(_XQTmF=wsEYyyM_vrUR2bm?JeNgET!#Y5hgs0T36$s-6(N}svo`DBY znwk5}ixi3A)ejSZbx|IO0IO!RSE>lSc%N6v&UJ7xOoK&F|KMOuxIhW9-5v8^$)PTC z0xO&ZoZuf{$rX#ZS7qlzCg@}(t{gR7Y#3YM2-B9?wKZ`BI4j)-2_L;nGHUc})hUFE z)|btFwLP0j-2>`>b;$`d&1+jhR-@zta5It|`F?&n-m&F(nUQ9-&$nt3cqhN8OEp0H z#Ddh~tRMmWl8_YE20&#IP&X~W;?RRe;l1(D6LobnrTrmTQPU8m@ZVEk`3ZI1w(eZR{pT`y+^7ekOT_^RNb{)6Q8E~3FzHn`Bd zMp1EbhrOhvWCPI+keR8_3`@(kFz0-Dp^_OaD<0BlM6`3YFo*a+qH1&ZXi6uk-6sWo2dP9C1<- zV0ZuAl62Vj<|Yoi`R=G4X0V9m?Rn_aS=YP#>9kJ~mHJB~b^CgR^E+1mbkt+}$A!M647{f!i1 zv`CM#?mZZ!%^GmZ1Q%D1pCSm$DX1HpKHzbgXIM`j<%DR-d}|Pu;2OolDpLEmoa16I}a6moI*L{f9SIOhKvW zoj3=4q<^Ul;dkJQWfE8)g7n1`FQumOZY34fM_ zuTTTOyydE4T+a_(#9!TA(!!Yts|n%`ut=DD(_v4c9Rv^Wj7wY-N1*j8P1CbOVqT!( zLo>U^V1PTefb8#I**^`(oI&OZ_Ex_?eiF$)c|wG(gKDm-2w9+TumgI@-#p8f_0bC* z<|8$5GF9k!<5(Dp4u0R#gUG3FnooKZR-f=F#tYhBErTCsK;%t=TLF_o7`%Sr5)Thg z^8w$UTR0T^>GojjNm=_~6#X}G0Pb?UF-ynPy4|8PMI|u4)4-I3Lem0biJ)sdER$?U zl2v4L3&OvY*ZxpaRwnH|eTQqZZw1$2y?1-6(QMWDJCZ7b(uuQT-~dnw7)GMp^1e`;J_BYn;9BxG;Kg!zXitG z?4(WUFH3Oy*{IY&e{n-1C~Oxy*9MX`qEeLAV6Mt6LeI(!3nNSYltPAVIA9Bbg*o$& zpHH%_^+m}Y_pULa5v7J>v; z??^Q*ERXYyJBfk1Ft(y=7_i8+7H%(c6oSTtzHQ*Nsvy!RtbIu*JKU|0%-N+UD(X8( za2p6aLn614T~?*_#NHQEcVC@s0h=z}{K=?r6r6p+B0D6$N|b`{}a%qih_Nc zU*(x@1<-M7EVY}xQH|~(4Vdj(Cp|qoJ4;uhW#_W$I{&Pd4WTrD&+Le8d;$YR#bkKK zZj!27y|_e5+6LW+8Pv>674LQRMK>bh1BOl5AFxQcozu4g`Z~!&P~#3qtW)`zgRVA2 zo2dg!im?zRU}A}sLexEcP>yd(*bo|4rb}s;=)a6))nY-48ttkIoPkw9Eo8UA)6u*z z9SHDDj!ru2;tn68@j$$O1NNV=kCBGt4zy6gm8`#8!`M6=rHeJAfy(~Qz23PglOi2o z7oIIa>IC&~r81CUfhyves%mN-8?SpJ@CFmpT3cH~#*y){OG4c_4Z$&PsOzFXZfC_y zB6IAdWAarFg8&A~{$V<#55vOAOsV6*-bd|o?K&Y!=Ob2l+r3y2WFqr9QMTO)WLDFl zT$G%5HjJ}=%Wwpw-(36aV4xN|Mm3>+Hi8zTWpXv16D^Q|G4bl($0p*KV6~(?RaA_g ztUu2^ME&D*ZOqp`!9)_2CNv_LH8hg=1VM(_W^%jkyXgwl@RT7?u3IYG=u{m5&;h%n zKE+WB2tq~$Q$Tc0WhSq#ZrqM$6mTOAB^K6o?9%#FbA}le>`GM%J3CjF#q91375r&^ ziLlA|FMn`;cLQmJ9>hn}a{@?kK=igiXp|Y*yaUMjio88ZBY3A0GfY7YZ)TI7p~r zEGbBE3>`hY{*ZHWw42$6TfBYU1UCQG^jd`EW#&iG-97b+lvy>bTi2f3it`k?l~<+F z&!O&aZ4U#v+%1G}AF{J~>v^mhLYFc>G?~bn-}K3Ljr8n-QRb19sLcef{&y4yqG$Nt z;G9sOcN5n7hNcXr|IoZg0o5eaf36{-9hG8(XtxB1z#fYp&wb29D$x2E1kR!;_F;!@ zpFS7=%^H}=Q*<5;DPwUkl8VJK-ud4N@Eg{V2lVWkJXH=~b#ij@ryYn%eqPJyRe=J3E>X8_Nh#sCeN7vUxIm^MEt| z_lNxfByFgB?9QAP;;#86cmB#9QBCKW%RpIL>cMtkaQ}kFX^Vo|?GPx=U@@c%L$xY~ zNBzIghC4_(gVJT*jST>VG^!5Yvqy7{3H~O%AfF{`L7k zs#+pR?3$C>k)IBImReo?yo- zQ2ZNABv(i(|6%VtH5_a0VI%0u*eMf4|L(L?-rr){Gf~_4b|@t5{MYCIsIChfW^9@( z{a;r@{w8WO|M#E(^3DIZ(zfM=aXh^sguz34ecRjHvzraw83w6F8ps33ss~I#{}(;A z1rDSSdMO+>7Ud?8GVC~0221?b;R96yQi4u}NE)K&EHdh$v6Y4>)$dj) zpl7V;(x|jas{y=)L^DB?@eU%J_fHa#-#Uz@Yy)l6eG*npCdkVu`tkN!tCE^It&-_M z%ugig9sCpl>*}#tw{gH84F<(q-#iRoLeNAO>5EtgyF;k{Coenz|5{qC;FaX>3LqE~ zTntvk6*zK{1u1Y2h43sQt^$i6wF9nK*9HKKN4mNx0aw_*)JV+C&N_S@KZZ1vo`W_w zYRJ9?tOlJhJmpuuagCoy5Ic^~7NJ0Ax!e#XkF^yITPx^wEsjX`RP^V&Kd^($S2)Kf zN;j(z^Ai7T!yL9HQ$9rx5LD>Ffm@LA8QyqVq&ZoKVFh>$d_P_j^SjkF`MM+w0+oY( zIu-uq%e{pHU;uC7z*i7(3#iLqTry=MSD+T2teHj{nRZ1W(8<{yUDFJSW>dm)6NX#P zWL@)U=oiPcR_WWo=B{XOU+M-MJRJcD+}YOLU8DhtP6{hi74Q+e$D=FmW0hQ__lprh z61Iq=IrPA&86;@aP-vB1D;NY)1c03gsBj=@Y6StaDU*{NXuk%?-O6CG1V$O4!#ju9R3_ikAd^s&THkF!DpC9+`ay_rbaOI z#z5Ci692TXa|?491LZe4eCwJQC+;>=?qdx@p3TJ%65PtW7R76=<6Ur-EN7lKz3DbW zPWviO9y0G00e8{VZ`P2Us1U?E=|iB!BmC?8aRuCmQc9NtxG2_Q1mp4TR720u(7XV+ zcJ3}!C$!F-J?jp*{!TM%JJ8WTFyIQfgJXKVc_}L+!Ot}pRvYmIOe;;$7Oc^kSRPBw zdVxVC=4&}xVNKX9#tPKSrqE`$7+d1fmWw;28)kJ`lzR)x)z^co>vq@!>8gv@I1Hrw zr}#i@I05%dh^=%vT5!H{Bq*e{Rr98c$8?Y>alqs}c`TppY=a=v&V*+@C{?H@ohTvv zTbG`m?%~Kw1*~S~oc_-lU;zB~MqxsKA#>-`92pU@mjJCHBgwK8 zP%ZhMrX%Okl#^p8!c9+fe-KNlJA!E+*#T(OcCg+QP$L?Tt|Ffw6cQ2)RHA4Lz_T&p%CXjJFmC>TU!J z_VQ?d^2#j0AqaU55OjdEH24U5lZ`*;&R^qh-vNL0*Io`xFq=udmxx5RwFA@MIG^=o zgU?)E{n1_>WYZQ&cm|K>f@2lnyC2*IBsmeb5CQf5{P>5<8HsrNj1;F9ZXLz4>pyCr zTdcP{bfi3rqi`7dT*H#IbnC+dbtHg}l;;JhJoY#uh&7MnC@#c-b94qVs~6s?^8t7u{;!nk<4mz$cl=V&nu3zb13b!xR4%iw|USvXCi@+DP491 zZa}oPpW_g+gr#AN<=D@eoi(r(Ud+`JlBIBl3eYl+qKKkm7((kE(x-(If`#`oqlFH@ z$UO2)9lN#V=0`UY;1J&dF72skPc(m@szBbNb6&ycy)_NPsCjF|H*(cpd(jU0FoN+t zdypL(Q@~Z%9i0bW3R`paSxUE`cZn^H%1^CSumKIE0>OwH5H`%ZD~h~AG!F(33_t)P z&}sL5VEFSMZ(xcD7zARTM7v(e+2TjZ@mKeWR>bq0@9Gk}dk(BtKwm4aa#M7a2yVBF zE~sugM2aS%jZG$?sUGpo`3$ETJ;~@z&SlPJd35%5aUHEbL4qC89eD-Khv5#N!C<{g z87Ixzr)s|B6U@uX7h~G%B!*2KQIx5kH|f0bp2aZ+*SAW(SE*=AH*$3nf=O(A%pDo? zOSN^RJESAm#aOG3dCw+oma1c|_Qo?32P~^sxNU2WfE^GQR2H~vl%S%V&>bX*M6mdp zM)ajwUEB=Yqk0OeqM!j9!k8enXQr~&<7YqA9c)P13c`0vp{(wpilRx7^57H}x=`vb zSra8&T|d8@F+~%zgC-99 zD$q0+#w{4~9l9m!Uf+vZpgOx?#C0)Ar6x|-u?xvH zlU~l%RVQ0o+)u8HT-^UK`=}Hf=dg1Yr(R$kPWps12UuAgYay{H4Y&Qv& zTGiBV9Ets4(vICo}MQ*=zcIhnm@c|Uh=+j4d~|c zEjtK(@*?~azbauWYmAUIJPu8zx+}#&Mz5iNC!$QQ(oW zTMc^Q;ZZ1WS-rP+W94Rt%pSq z`jxoA&lKHDsARFD3k8V6yup3%Lf^#g%KEYjIdK|%uNdl0Rthq6<9CRP^0C?QM93e7 zMo$8fDT9E}ayd;v`MJzO=|KhXHNxj25 zwh8f~bSujjb+33?xrmri(|~@E>x$-nz#sn%$nXys3D38EJ1T*un@5N$D}K~vB2`k% zvnKg?>u26{-^%YV&q{xJOz%05x+j6}g>bEvT9MfVb9Vg4L&cDk0xi$J;bWqIt~NC7 z6I3Rj7jz=VR?H=O3Qu}4zwFybVwZYKB|SPVMz72hJ@&FHh3M(&O@bDnKNiQ%l8yB< zdZl*6o-%U|NT%`Q9YPkt{6~}(cn|R3&R1l+ln)*AcRWFxez>@~ERB~myL=G3BLb1U zOc}gQVU06_2OQL>$(pBBW7)I6uSOi{WOuKs zUPbDTdqMw>bgkTk*w+Q9(>bhQbzRLfC9(NIz?J88W=6vKDaS1 zkBE4{$*_2VUjdbgOW5MuTN6;l(z^?F?{974Y4mxz%%t?SX=Q>=-(+9Lyf4dc__19A zi{%INREa<`{gJ)5ecR0=K`u}t+&~hqgqb)kuPU4}R{IFcHrU&LG?T__DTWhb)M&ah z1R*a^SI1+P3R^f&xA%n`TpnMKUuAa;Cwjt8``XR_O!yNc@zir^cf;yro@8cP#!$X_ z^g;5i-lHetVSM_Jn!YU`P_50>cdebbYTQl9}y{3z<5+q-#1q>S`?8RIK5MsHA7YUv^3&4 znUG5f^C=4dN9H*dCOjT9 zW!4cL{sD(ZeGMgUKM-w3mBWh3BDx^a3g=`V!>G(s?G`t&pA~`YDGEaHsRogHoBRA8 zFUiQIhXEBggi5AEBQyu1{32farRsN;9LyXY?sq``Vee*s4O=>?rC@<(t&2lt@iSdI zRU(UtI(K8|0?l{yatUEm{OkKy`avSI!tP4emGi7nXViUQqJK@NG-EGkFE0_JAhl=0 zB)1+nqBzx3Xe21tc4)FN!Ee7PZtmD5Q+nW7Q+W4Nfe*IVdkR+=Ja@mzRn^A6(eLu} z!iSlU!f!_h1O$An3;MNVjUeByXnse2UFi>Y`2&2)bK{_O*(>97-CESgGF>V#5@ey& zs9qq;H|j~=j$sMe(B0hW?rl>g7%G6AB4sQv<+0DdzKw>ULM+6}8%EK-?{gG$X31Oy znvYXZftF{7_*Z_hFcFHdZ?+?L<%JYjN;kB7Tyyx%*X_0=tV#knWWzftRk9@yed?9* z6%=xxv4KZ0m)f7OiLrV#eoffT>CSL}9&>nH46By#2**7bs*c5hoDRebJ69JxQ@kp8 z1TXE$;Chc_l>FUs5HFY#GY(LaYC3(W%_1@erT#1s=iFDPYp|WR?n1&#qGk=w3-?_O z6F7z(K>%@yp!c<&M8WpS+BWT4ea>IeOR=yNDrHH<&W3jn zJ?iNYV}6y(B&;?=4f4BQ?+dJ4S{hFYVGhqFt16(fUmYFxuwNSND6tqD3A-i#1<{R{ zImu?PCkCCWNrt6y`z6p~~l}?Kb4DYTn~swx&zrjLN-(xcWua zt^^_T>dmzYk!6FAA;;{+cr{)3YA3Rj-)3(ae|pMuQ_);Z(k$>A4>!2;R_S}`R2T)G z+gv$!kE@g2k$#Z8ZTIq8hS1r|xAhM0`MPD_Qb~Js*%mwu%xAZb2ffQzJ3h+0dY_z? zRWm6IRpYuZvLi!%_0O3O%6@R}%AHR~`;Zu5ZQl0{Yj~^kxP&DEQf}rmcPv1H!`m8F zCC=d38uwj7r~vK<@=WecR}F8Dij*YUm=nbs;BIH#@H{$j2x$(u%CQ_6K_?Rv1qp|v zni<{cq8l*h-fW@$LJ-E{zN>k;jz;;eWPd_7b;=`CvbK#W`?~BhqzufFnxrBbYmf7j zxh|m9Sedw=yK&YbWjmIKi<@QV&r>J|H7>+1dcKEfn@8PtH}5#49GNLUln*J`OSc#W zTCLgu_^dL&pl!*mp-NEYYn9PoykM>AVtvp>K8A*gacl>0VZ4@DS<0|OPFx`T0h_Tp zW#YC8l4mxA>Z=6(nK{ul^O%uV-A7OQh7_iSem)CX$v8{OvT2^1JgEpcrR;vp)61Tp zU*dBI&p^-7Qe78>#fq;9H;J_vDQPJ;cd2=Y@~LS1`7pQPj5T}97`yX(!4ZDvGY%Ft z>kqJCE-{~xG$VoR+ukQDC$}^*CQQCd7t`I76ZJ^C7C~lpK8;(ET}>}sWG`LdJdue~ zjzJ0mIq#i?GP(W)GVy*hoh=y=3T4F^Ii5bQ$5&LUYBZ`H92RUl$gHzZ{NWB-3%9qu z{@b!+AVxHg% zft&%ANoK-~M3dxKd$$ic<8(xrm(uSJJyu;6L(}^mB4Yk)^!#YE5{7pNh2HBm#KC2< zfl7%^GvUh1M0yZ_B3p^3VHP8~iZ*`YZShe;2@NyIO)KRAO2*mUxkR~&&4O25E@s)O znQ5D0(?7b0YWDOaSv1Vpzk}1TEKW8;%1HnN)u&t%JTDZT%{$}xp6}a`TcN8REsGsiR0TeTx306wyDla_Z1MCE z$T!&y`!+3-aj8UjPIl&zh9LiZ?5DyX9ATOzj*Bgx9kD#+QlCtRGlAjaH;7%VS&%;R zX((;&S@iMS=bTbC&$?|I^FE7~iSw3NES<=F!HYq?$fLmJjK!1~CvlPU%IVV4nbRUY zH?NIFDfdptaG5T*JHm!aX)UqWJ=oHBo|?8N=VeU9_uHx^w_qa1K0Z~?U~Qz$u1J4$ zhwR1xucF?Bo?Cw{f7#xTv`9^p=KG54=ewr~iFVTU9)~X_BTMM5+^hs2 zv5p9EWzPdUctdSr2~u-7RxsrmG8|d5@%-GgOXIcGl^_LNeiWFkp6hD-;SKwI#=1L_ z=>%vrF;McU&rKD#?WyWXKgQ&USfv-&WX{Q_nO{E=a6X9KeVKZhMV&XwF0J)iCr(0> z(?2mHOOP`;e~r*q*2E8{kWpoeu3}clmJI?ZYdKiHOEe-WU4G+rW~G zyuxjp;_Rf3F{u=#?gA@pHepAEA79pQsNp`8mmM@AvLq__+bq({NqqqlvYqbW*0w@X zt=gOVZd2lTkueJzTgp4z=UB;NBQ@l^C89aUhrl(?+K#i)yZ9x=i<3rfNAZ{<2k&K3P(%8h${^S9~XI#0aF(o0yvn){nyNOAh=Z zd=*M5I$qn4Jn6?h-EGLb9X}CtJf=4FHj7jwOQ5O?ZNGw?(B+!v%QsQkf#B<2#RA;P ze___)*DNoa*4ceRiuTx|HU8|L8gI*4kJVVw+kpGPwgg#Z0_oPkLY#Jjixy|Ropi0k zZD7|1Q)a&Hv=)K-=KLu>WC&>@%>mTh{!E;haoWIFSZ;h>u`xGvX$f&3 zgU3@(J1JCZVmit3-gHr&28x9#b<-nA&e|U7HxEB8LvwBnZ5gJQ78}k_b$eINS2>!3 zN|ETjg~#p6yeDao=$D0D0b{A_@H?)WSsHnO%KGyOQLvc~(mUWhHkm1mJ$BULr^kDI zD03*Q+aGgjauDaUFxhNFCS{-fgl;c)$u{4CV2t#W{GYSQ7jq@cPy5lLf^_l_E6H3% zzWCkCj<=qD)b1CKQ%GU(m9Z|(9$xUs8>s;$_;d5@XM9S9=jzL{bVM32s}^xboF=9l zVx?8*KkenKb6I1b@3e)7XPVDP!E(w^-GHT zKAVgn_vs%|A^k*lr~mW~n~Qf>a<>PiRl;|+h-pPMAth`U)SE?z%b=#V&pRM}w9f#~ zq{I<$W5L=`ggosisTJkP=~PqTip&N`vxZqre<-QkD3Y-!=vJQ3eP&4gz&gc> z(#HXTPir5nsm#e@Z}V5v+VdXIjmb=n&ZQ7=C<&hFN))NkI9z*mLHOe#erR6MffWy1hsHE+(i*Ol=>mkz0ob|>QQaz)U3j}e)!gHnV~YFAD+DuPkDQyuh9 z&oP`NdFw(eDIP-^a)}>T77G^3KOA16<7Gki3GI!<%%bA_Eza?Ba8EMXgqVr*U61jf ztdRMIe-OmW>dt#aU#9)}m_W2Lv7NnTF61;@@1UKPc7~n?h)XJih)Zf{8&K`A@8>Uk zTyb=gbDi;+gd1X0b>LlKY||$RJyxU>D+P%&Y|9D-aIpen&==|seB z#oNnKXTIYu&QVZ-15(r+ySKH}JQlniv;vW|3r{m2e=eJXC>$BD+b^pp<^{ZRT|K*4 zDDe6RhFkj+sya{m^GojZ&Dbo3jH(1RfKd!DfL6t9BSq)Fl=Y-%U4!zM=50W+qV4w~ zDP!58boXEv`?X96XsFN644Q)|@Io#aiTMskG8Za8y6RKXjX%aeL@`vuZ>|Cy#;U zWKqrqs(@Ti(`C*ZH@;d_LkwGPe*aXBAMe_)E4)@oLW3pd817XMa-8n}YVXS5q2B-Y zQ0j0hrPQ~gwBXQ!WY2PFGxo-iB~xLn6T;ZCw46fWSYxb>EHk!jW8aD>l*&32BgsA_ z88Qve>*GAv_5D8Q`~%N(T@OF>X|9^(Gw=6nxnK8vzme`o^sTYlHc{2nleFNkO`BPs z=*K5TmyuE;%H`HDjEssIUbr?lFU{j-AM=+-pouCb!~z78tTw#+;e-g)lVS^9Iompq z04TtjPYrh;D=$?%*s^z|x(>iqVol*>cjUGR`DwZl1JZ!V*JQ9d;Oi~08m~53qNQ>^ zB31*Lv_44qH(Um*z59JLA~&cWaI{)m<-)sOy6d`PMAg0fRnS||Uc-f*j4vZjbTuU; znzdvh?zXe?FqHj}rS~|NYi6{UdOurWdnC|K=v(FiB6x@Y^zhX4IT!#v8<+ZAh17LB zxc~%|$_p*A7!ELKR9D+chDA9BCF{4Ik_;+u=H6~gA`RCNRM`F&4Zr{^KiZhO?tGz^ z`2_lqY>3k|@}q>AOIZ=KgEfqX2;78xo*vX->XRD*e7G1pZQ9h9bm-G&)i;x+1p|#FZ03Ndvrc(B9_upc)LCZ2hmMbq za-9QbK8AfBmS8gen}fw6hp}>v1;c?Gs82u;asx&tds!R+gjB>5HsT!Aw}pxG{4ACs zwh+tp?<6S@_u{YvTMM@BK({AP`p)f^aWU1e?h|_%Q}KY+pFfBPTg2oY-aN9zCn*|4P&rb)#@Niu?U!>8$KkK#X-BP4Mnt(M zQl#TyfJ>h2El8fNAf*Gs6`Y(7ZJVBD5^l7+|KgimpR>OmcorPWv(SJN4o`u`zTXiVLcMQlmWe# z@ul4BK+oERm#-g1We>Sw6BIx#3%F4r$I_*CM2IPL@p4j!-=_9t4FT%DQV4&p| z2m@~P3OBc+W8FsHCulBxCj-)TY(2@J$r`23rN+(|A|@%}(Pn4fQ{}4_yX&?`U|i1a z8~jzurgm9(g%yI?<6$#4QIE8Uch-uEj}Gqtz&$nC;61TiI4`xib*=xHl2igl`1GJp z_m9Si*@GY~QkapuGB>SJNe!7)E@GQ^FG16zkazdPcZB(;qt2@9aK>ZIyP;KCnq z=9JjC&oH%@2=TV=Fy^HP#DoW*jx10%?hzJVTV~aVN*{CXuwaqr9j@e1f}Qkjx`P2? zVB}I+^r4L3H|$@KDy#>M*^J!TzE9N$`V=BY8rslbFTQ*AMyr|P27Bu^m<64=-1GX! zYd{`{fv=%D;5HYRi4Kk?2>-y6&+_+;8{%%5J2ZiMYkVl!Lwgi)fT z_*6)Fx$N#ov0c&+gPi2Cf0{1pR@74l)_~Mx-!TrG{h7&@1$$;cXZxm zw^s0GOF&ba$kxDN%fz31^&!YWk6OIdC0Ya`BzCg`4tMgy1B>aGBW6Q1jl|qbx z+IPKi%T&*3OFk_JVc{G)al$<5l>LlC#zBgK=_0)#3g4d4X}0uh!OULWym!jSzWfn{ z1(Y;sllx3AD)eb*nmXeR{=@N;_Nf)F1Mht-I$s&nD)3n^GY1m6oF$*ZY#j_f)<(K5OuAA5!1@`t8 z;oKbL9q~2zj+J^Z{HL7MHIr8 z+1FWS`p5f@my%^ZzPIL~@Q~=4oqtn6`aRc~7@p-FL3R;#$j-+(v9qYp9nGlJ=3H{X z9|FRzA9E^|chFC(P`51tqla!WO%)K1x-PtkX;CE|$_t_n6a?+28!gTOKaDQ1Wh%$e zp5^=3vKh_g4U;aBAY*raEHuH2TE``bohq}X(rzJvq zk5Qs*Q{;}^oA;#upzzv|=WpkuZADda&em?gD`|?~1=Wc|gXHsN$Lp3nx}UwM=Y2p1 z)gpTLJySc-x@X$-lcB9~kY%}K%*$|WvF_(i=Yv9VWeY4to~M zJ(c94b@V*17((a#_s@aRw^E+}{ws3jE5aZ0w@0@BAAH0(GM2juHGFv^FHfoeqI0SD zw3+8<+YU_CO?ym+7SFz(7H|>mgix;!h_68wI^NKKKQgcY0I50dxW)UR)QeC*pGLrA z%B8IESLvE-{9FqZYIs@fjotwd_(h>?wBhR>fnOA=_MWl~KrhXHjq~UEv`}ea^mQ6j zXsv>&zvYLiAre5%B`Ob%Oe~jXXyG~jrkv27XD;I@?F$5z8O}hN6UOq+1hl)~<9-bC zptNo^`%wdOtII8Fhy_xY zo`zB^O0x}kO}*7pR-kOQMff0=G*_pS)f42Ms|%)kO9%`a9x~h*GVT4O3D#hC2Rza$ z5VzEj%b^Rk5r|!8vv$fjq1jKXR|}Sk5eY^lTP5PZR|IkW7oapt0$NnBncoYF8xUEM z0TZ-38cv*~x_~9c2Z9G_@tb*2jg*JVZyAW~tk&^nzoGMp9r5);Zx$3s&WO2)@oW!~ zq0w_vKh3qvr7crA`wO-*{pG1&+6GPbtSzNs{T^o z`Ifp4oGNu=)`OcFAcXWj6MNNxhTw)Dz`5vzpfR%m-eFfO^1`iikuHvT_s-Lc2-3&+ z7(^7?F1ugRRaW!o*Sp+Qz^-E^m-bA7l&x6PoFRDNjN>VtqRF2&&_se@)RKWXhd#wE zLGM;lG*}-P(8sDFvcmm40c1)7fe(G|2Gm^+jMf6K&L9-V?72lawEid;q#;`V+sAi| zS0R?9jhxVh(1nkPW$JJsva)W3E`-D$MWGHLN8E4pnz+fS2EFwQb}IBTfyb?Au1$d; z*V_pUU!AK^i<=Z(fo81cEhy-Cp`Q#!n)9ad6IZIc>l0Shp^beu-kG8zU&$P9y8D7A zdd64QeJsg=o?hcKH}q_Y2uj2O!jlG_+q`_dz@p`!VN}R`S(5ZeMveR1pxbh{g6@3- zVnU8sGM26}$>$zEWz;|(VyhrHLpKd(Dw_m$A9rLdK?!ZNq5yr%O%4f@t?x0o3Imn_ z7Tk}mu-}dv1A_8Qu^JF%UU*^}XUc~Pb3}Ir5N&r)Ku?P_|Aj>b4XJEIJQN$$0MZwA z3fzTFxC>!E(L!|%UH8;;2q!Ct?_4Ecg)-ujA?hQI zU^NVw2prX+?(g;0L%e@-d}_WvkR!qM_fH$Dh}i@x#vMvLvPX^_F@_3G3(l?C0Q+Jq z=r>iG>E3?ZN0_`o#R4Ot0HW!AxwJ#c3>>avQ801P|It%nU5b^*c#5-_xS&;I=? zMRh%L0Qt9!r$t9leOI|bJUhisRlJ5UP$FJQfM9qV5X=Dr03kjG&oGyUo?Uyvdm&PU zpRo!mqkh)beW!m3_z7%~Qe6W2*~9$rKrTgbQb5FdYDPt-_N(^QpUd+!?W@0GH+c!o znp()lFCp5_&3Ph-_BLO=+IxDpex#l_PZx!hb58|MXUuAX-bd!Lx7;_MPzP93Y`+yz zTMr==NXGC6R`Xp?6kkE>KbWBAGphwyE+K|&yZrT2(`XxL;93_KVZS>snOJQ=g8+VgLa$8}RKfXdjk9@dPjn+3Mr`98m6dVz8!Tad%tESg2 z7@WoFJ>@KL6`_%B@It6BFp(^NTVPf|BH*wiAKpKQ%3p8N8(}AMRH^!jzwsD{1wlp# zqZdBaB$#8hDzaj*QpSw&x9M{$D-Z|E8JfWMjvE3&*G0jRTE}L;3rMUUhVc9@bRiNk zjch6RCf;yp=BGW3IKD=D? zhSL+Akiv&O$8-aL-RU3)s;~}#xeO5S^s#=@sg@vSsY6tt4B%Bt3eMfX1QY-Wkd&q# zKstObJn@wM>wcE^RCh7%#RI_#jw|e&#lU9&U@+D!;0A$f>RCd2ntDwY5K#^A0MNCU zL8*J(rnVmdm|hJ<|L=|%swpMT#sTgLI{?<)&RHv%Ycmw1rA>3MHYhoFw~a!29%BRj zxP8$Zrr*1K;5I0G?||sYzy1v)-*p-i9RFNILMA-uKC=E5gJ|d>*QU_ADv^V1nroc< zFy;2@7{3De0-FP-STE|Q=fS+IOM<4tAA%M-<31eX)Rl2t1yI5}MNgVvATw^@Q8c+S zXq_68fFyMTM$=t*MQyo$(OT)ooDpXgR2FRJ4PDPL`jKeYfs!O(GjCjjBS49Yhh(vS z-$sj>fU`VyXI^g!uhN)3$`z*vO>`aVV?lx_XM&_Nd9((Eia7#E?SRC3v1ugP??eiT z5;z)Q{XxSsLMR?P6=-Bin5p+bknS_Pg`rgP9!K7&CHcyTKZH8s0}4xqbO0c0rvYrt zG?@9-VB3~OpgZ;toJ3zu29RaYJbv+eB6WEY8r#{qarWxX(Y9-`-$%GZR8`;OrhHv; z9wTQdWj|D>Ia0+jl2&8~vjTRCr{>%PTgi&pUU8iC~o_wa^SHK`v5+OdfbC{-<{Q39H^9=Cf;e`8V>8*>_;)6(^7Q_bnCEiYGdW~7J=9s=rb?} z%`_MRjp}uJum&t{MI^}=ujhN%;{UjWBXB}cB`JtEaT&lbt2N-cI~FFHCW{N^mj8g9 zq7Sdv38~nn0x@@Pva2;!7UqHs%Aw(b1WKBcZ%Vl|1_G(FSGS{k)}F2>IS`0+0Mpz- z!YIuZmc1RqD)$5v7QQ_GY;;l9QFU?ksqr8{NAX~Zhli+}4NRUfC>W#~vV|7uG=fXI z8o2buMn_*`GQlc=5W64e_a*+S+R^qO-11S6Y8A~7eo$35S|O_ks=&WO);}&IWsrrW z3wAOzT{p{FbG=@wb9xL9XkQF!>YyGW@(*>m3$4Zt=h)jFc^rfNmRed{V8weC$Zs8R z4wvK7yZ6;lOei3SG3^B|CO}M;1;-QjTE4j)<_A0I6yl8nRM%9Bc~ROVy*#hLSJ|;E zBMw&zr8uN2Zn*H}R5bw~ow)4r4H`GG+EPW|qJfUZG^Q?3Dg77PnMOKlib;BU9C zFE}L4j^9V>=D8!VfjxCnHjCg?A{VB4v@DS(0-^@PjNyraAwlxS9P}+1i=Kv%Cl7j= zvi5mt9v0|RZl>NQQ;BdWV)g0E&=Y!#+)0Nu(6iy2LwDdI;w$cb2BW|+5S%CiUAiW45D@cZ zJMf(9(YP=NjZm}+V?WGR()_2S7%$Oww+Y=cRu{a^RIFvqz_70t{M3uMIV(3p`oL zCTSEnG5p8R@b9L76J$2tMc(Fr{`~(V|EuqY@7uKviCNaHW~XP`$Q#u?XLz=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pi05-minimal-validation" +version = "0.1.0" +description = "A tiny, educational pi0.5-style hierarchical VLA overfitting experiment" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "pi05 minimal validation contributors" }] +dependencies = [ + "matplotlib>=3.8", + "numpy>=1.26", + "Pillow>=10.0", + "PyYAML>=6.0", + "torch>=2.2", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", +] + +[project.scripts] +pi05mini-train = "pi05mini.train:main" +pi05mini-infer = "pi05mini.infer:main" + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra" diff --git a/src/pi05mini/__init__.py b/src/pi05mini/__init__.py new file mode 100644 index 0000000..9be415e --- /dev/null +++ b/src/pi05mini/__init__.py @@ -0,0 +1,5 @@ +"""Minimal pi0.5 architecture validation package.""" + +from .model import Pi05Config, Pi05Mini, Pi05Output, flow_matching_sample + +__all__ = ["Pi05Config", "Pi05Mini", "Pi05Output", "flow_matching_sample"] diff --git a/src/pi05mini/data.py b/src/pi05mini/data.py new file mode 100644 index 0000000..6080d5d --- /dev/null +++ b/src/pi05mini/data.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +import json +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Sequence + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image, ImageEnhance +from torch.utils.data import Dataset + +from .tokenizer import MiniTokenizer + + +@dataclass(frozen=True) +class SampleRecord: + clip_id: str + t0_us: int + reasoning: str + image_path: Path + trajectory_path: Path + + @property + def sample_id(self) -> str: + return f"{self.clip_id}_{self.t0_us}" + + +@dataclass(frozen=True) +class ActionStatistics: + mean: torch.Tensor + scale: torch.Tensor + + def as_dict(self) -> dict[str, list[float]]: + return { + "mean": [float(value) for value in self.mean], + "scale": [float(value) for value in self.scale], + } + + +class ActionNormalizer: + """Normalize 2-D waypoint deltas and retain invertible dataset statistics.""" + + def __init__(self, mean: torch.Tensor, scale: torch.Tensor) -> None: + self.mean = torch.as_tensor(mean, dtype=torch.float32).reshape(2) + self.scale = torch.as_tensor(scale, dtype=torch.float32).reshape(2).clamp_min(0.05) + + @classmethod + def fit(cls, histories: Sequence[torch.Tensor], futures: Sequence[torch.Tensor]) -> ActionNormalizer: + if not histories or len(histories) != len(futures): + raise ValueError("histories and futures must be non-empty sequences of equal length") + chunks = [] + for history, future in zip(histories, futures, strict=True): + chunks.append(cls.to_deltas(history, future).reshape(-1, 2)) + flat = torch.cat(chunks, dim=0).float() + return cls(flat.mean(dim=0), flat.std(dim=0, unbiased=False)) + + @classmethod + def from_dict(cls, values: dict[str, Sequence[float]]) -> ActionNormalizer: + return cls(torch.tensor(values["mean"]), torch.tensor(values["scale"])) + + @property + def statistics(self) -> ActionStatistics: + return ActionStatistics(self.mean.clone(), self.scale.clone()) + + @staticmethod + def to_deltas(history_xy: torch.Tensor, future_xy: torch.Tensor) -> torch.Tensor: + previous = torch.cat((history_xy[-1:].float(), future_xy[:-1].float()), dim=0) + return future_xy.float() - previous + + def encode(self, history_xy: torch.Tensor, future_xy: torch.Tensor) -> torch.Tensor: + return (self.to_deltas(history_xy, future_xy) - self.mean) / self.scale + + def decode(self, actions: torch.Tensor, history_xy: torch.Tensor) -> torch.Tensor: + deltas = actions.float() * self.scale + self.mean + return history_xy[-1:].float() + torch.cumsum(deltas, dim=0) + + def as_dict(self) -> dict[str, list[float]]: + return self.statistics.as_dict() + + def save(self, path: str | Path) -> Path: + output = Path(path).expanduser().resolve() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(self.as_dict(), indent=2) + "\n", encoding="utf-8") + return output + + +def _resolve_data_path(manifest_path: Path, raw_path: str | Path) -> Path: + path = Path(raw_path).expanduser() + if path.is_absolute(): + return path + candidates = [manifest_path.parent / path] + if len(manifest_path.parents) >= 3: + candidates.insert(0, manifest_path.parents[2] / path) + for candidate in candidates: + if candidate.exists(): + return candidate.resolve() + return candidates[0].resolve() + + +def load_manifest(path: str | Path) -> list[SampleRecord]: + manifest_path = Path(path).expanduser().resolve() + records: list[SampleRecord] = [] + for line in manifest_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + records.append( + SampleRecord( + clip_id=str(row["clip_id"]), + t0_us=int(row["t0_us"]), + reasoning=str(row.get("reasoning", "")), + image_path=_resolve_data_path(manifest_path, row["image_path"]), + trajectory_path=_resolve_data_path(manifest_path, row["trajectory_path"]), + ) + ) + if not records: + raise ValueError(f"no samples found in {manifest_path}") + return records + + +def materialize_subset( + source_manifest: str | Path, + output_dir: str | Path, + max_samples: int = 3, + num_variants: int | None = None, +) -> Path: + """Copy a self-contained PAI overfit subset out of the Alpamayo project.""" + + # ``num_variants`` is accepted as a compatibility alias used by the + # training entry point. Actual variants stay virtual and deterministic in + # Pi05OverfitDataset, avoiding redundant copies of the large JPEG. + if num_variants is not None: + if num_variants < 1: + raise ValueError("num_variants must be at least one") + max_samples = min(max_samples, num_variants) + if max_samples < 1: + raise ValueError("max_samples must be at least one") + records = load_manifest(source_manifest)[:max_samples] + destination = Path(output_dir).expanduser().resolve() + image_dir = destination / "images" + trajectory_dir = destination / "trajectories" + image_dir.mkdir(parents=True, exist_ok=True) + trajectory_dir.mkdir(parents=True, exist_ok=True) + + rows: list[dict[str, Any]] = [] + for index, record in enumerate(records): + stem = f"sample_{index:02d}_{record.sample_id}" + image_path = image_dir / f"{stem}{record.image_path.suffix.lower()}" + trajectory_path = trajectory_dir / f"{stem}.npz" + shutil.copy2(record.image_path, image_path) + shutil.copy2(record.trajectory_path, trajectory_path) + rows.append( + { + "clip_id": record.clip_id, + "t0_us": record.t0_us, + "reasoning": record.reasoning, + "image_path": str(image_path.relative_to(destination)), + "trajectory_path": str(trajectory_path.relative_to(destination)), + "history_steps": 8, + "expert_waypoints": 16, + } + ) + + manifest_path = destination / "overfit_manifest.jsonl" + manifest_path.write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), + encoding="utf-8", + ) + return manifest_path + + +def _resample_waypoints(xy: torch.Tensor, steps: int = 16) -> torch.Tensor: + if xy.shape[0] == steps: + return xy.float() + if xy.shape[0] < 2: + raise ValueError("future trajectory must contain at least two waypoints") + signal = xy.float().T.unsqueeze(0) + return F.interpolate(signal, size=steps, mode="linear", align_corners=True)[0].T + + +class Pi05OverfitDataset(Dataset[dict[str, Any]]): + """Tiny deterministic PAI dataset for validating a pi0.5-style training loop.""" + + def __init__( + self, + manifest_path: str | Path, + *, + image_size: int | tuple[int, int] = (128, 224), + augmentation_factor: int = 4, + action_horizon: int = 16, + seed: int = 7, + tokenizer: MiniTokenizer | None = None, + action_statistics: ActionStatistics | dict[str, Sequence[float]] | None = None, + ) -> None: + if augmentation_factor < 1: + raise ValueError("augmentation_factor must be at least one") + if isinstance(image_size, int): + image_size = (image_size, image_size) + if len(image_size) != 2 or min(image_size) < 1: + raise ValueError("image_size must be a positive (height, width) pair") + if action_horizon < 2: + raise ValueError("action_horizon must be at least two") + self.records = load_manifest(manifest_path) + self.image_size = (int(image_size[0]), int(image_size[1])) + self.augmentation_factor = int(augmentation_factor) + self.action_horizon = int(action_horizon) + self.seed = int(seed) + self.tokenizer = tokenizer or MiniTokenizer() + self.state_dim = 16 + + if action_statistics is None: + geometries = [ + self._load_augmented_geometry(base_index, variant) + for base_index in range(len(self.records)) + for variant in range(self.augmentation_factor) + ] + histories = [geometry[0] for geometry in geometries] + futures = [geometry[1] for geometry in geometries] + self.action_normalizer = ActionNormalizer.fit(histories, futures) + elif isinstance(action_statistics, ActionStatistics): + self.action_normalizer = ActionNormalizer( + action_statistics.mean, action_statistics.scale + ) + else: + self.action_normalizer = ActionNormalizer.from_dict(action_statistics) + + @property + def action_statistics(self) -> ActionStatistics: + return self.action_normalizer.statistics + + @property + def action_mean(self) -> torch.Tensor: + return self.action_normalizer.mean.clone() + + @property + def action_std(self) -> torch.Tensor: + return self.action_normalizer.scale.clone() + + def save_action_statistics(self, path: str | Path) -> Path: + return self.action_normalizer.save(path) + + def __len__(self) -> int: + return len(self.records) * self.augmentation_factor + + def _indices(self, index: int) -> tuple[int, int]: + if index < 0: + index += len(self) + if index < 0 or index >= len(self): + raise IndexError(index) + return divmod(index, self.augmentation_factor) + + def _rng(self, base_index: int, variant: int) -> np.random.Generator: + return np.random.default_rng( + np.random.SeedSequence([self.seed, base_index, variant]) + ) + + def _load_augmented_geometry( + self, base_index: int, variant: int + ) -> tuple[torch.Tensor, torch.Tensor]: + record = self.records[base_index] + with np.load(record.trajectory_path) as arrays: + history = torch.from_numpy(np.asarray(arrays["history_xyz"], dtype=np.float32))[:, :2] + future = torch.from_numpy(np.asarray(arrays["future_xyz"], dtype=np.float32))[:, :2] + future = _resample_waypoints(future, steps=self.action_horizon) + if variant == 0: + return history, future + + rng = self._rng(base_index, variant) + angle = float(rng.normal(0.0, 0.008)) + scale = float(rng.normal(1.0, 0.006)) + rotation = history.new_tensor( + [[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]] + ) + history = scale * (history @ rotation.T) + future = scale * (future @ rotation.T) + # A tiny smooth lateral perturbation adds distinct trajectories while retaining the label. + curvature = float(rng.normal(0.0, 0.012)) + progress = torch.linspace(0.0, 1.0, future.shape[0], dtype=future.dtype) + future[:, 1] += curvature * progress.square() + return history, future + + def _load_image(self, base_index: int, variant: int) -> torch.Tensor: + with Image.open(self.records[base_index].image_path) as source: + image = source.convert("RGB") + if variant: + rng = self._rng(base_index, variant) + brightness = float(rng.uniform(0.97, 1.03)) + image = ImageEnhance.Brightness(image).enhance(brightness) + height, width = self.image_size + image = image.resize((width, height), resample=Image.Resampling.BILINEAR) + array = np.asarray(image, dtype=np.float32).copy() / 255.0 + return torch.from_numpy(array).permute(2, 0, 1).contiguous() + + def __getitem__(self, index: int) -> dict[str, Any]: + base_index, variant = self._indices(index) + record = self.records[base_index] + history_xy, future_xy = self._load_augmented_geometry(base_index, variant) + reasoning = record.reasoning + target = self.tokenizer.encode_subtask(reasoning) + actions = self.action_normalizer.encode(history_xy, future_xy) + return { + "image": self._load_image(base_index, variant), + "prefix_tokens": self.tokenizer.build_prefix(history_xy), + "subtask_target": target, + "subtask_label": self.tokenizer.subtasks.decode(target), + "actions": actions, + "action_tokens": self.tokenizer.encode_actions(actions), + "state": history_xy.reshape(-1), + "history_xy": history_xy, + "future_xy": future_xy, + "reasoning": reasoning, + "sample_id": f"{record.sample_id}:aug{variant}", + } diff --git a/src/pi05mini/flow.py b/src/pi05mini/flow.py new file mode 100644 index 0000000..ad24cb9 --- /dev/null +++ b/src/pi05mini/flow.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from collections.abc import Callable + +import torch +from torch import Tensor + + +class FlowMatching: + """Rectified-flow objective used by the minimal action expert.""" + + def __init__(self, inference_steps: int = 10, cutoff: float = 0.999) -> None: + self.inference_steps = inference_steps + self.cutoff = cutoff + + def training_batch( + self, target: Tensor, generator: torch.Generator | None = None + ) -> tuple[Tensor, Tensor, Tensor]: + batch = target.shape[0] + uniform = torch.rand( + batch, device=target.device, dtype=target.dtype, generator=generator + ) + # Paper sampling: z~Beta(1.5,1), tau=s*(1-z), emphasizing noisy/low tau. + timestep = self.cutoff * (1.0 - uniform.pow(1.0 / 1.5)) + noise = torch.randn( + target.shape, + device=target.device, + dtype=target.dtype, + generator=generator, + ) + t = timestep[:, None, None] + noisy = t * target + (1.0 - t) * noise + return noisy, timestep, target - noise + + @staticmethod + def loss(prediction: Tensor, target_velocity: Tensor) -> Tensor: + return torch.nn.functional.mse_loss(prediction.float(), target_velocity.float()) + + @torch.no_grad() + def sample( + self, + model_fn: Callable[[Tensor, Tensor], Tensor], + shape: tuple[int, ...], + device: torch.device | str, + steps: int | None = None, + generator: torch.Generator | None = None, + ) -> Tensor: + count = steps or self.inference_steps + x = torch.randn(shape, device=device, generator=generator) + grid = torch.linspace(0.0, 1.0, count + 1, device=device) + for index in range(count): + timestep = grid[index].expand(shape[0]) + x = x + (grid[index + 1] - grid[index]) * model_fn(x, timestep) + return x diff --git a/src/pi05mini/infer.py b/src/pi05mini/infer.py new file mode 100644 index 0000000..6dd9b2f --- /dev/null +++ b/src/pi05mini/infer.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import argparse +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import torch + +from .data import Pi05OverfitDataset +from .flow import FlowMatching +from .model import Pi05Config, Pi05Mini +from .runtime import choose_device, write_json + + +@torch.no_grad() +def evaluate(checkpoint_path: str, manifest: str | None = None) -> Path: + checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) + cfg = checkpoint["config"] + root = Path(__file__).resolve().parents[2] + device = choose_device(cfg.get("device", "auto")) + data_manifest = Path(manifest) if manifest else root / "data/overfit/overfit_manifest.jsonl" + dataset = Pi05OverfitDataset( + data_manifest, image_size=cfg["data"]["image_size"], + action_horizon=cfg["data"]["action_horizon"], + action_statistics=checkpoint["action_statistics"] + ) + model = Pi05Mini(Pi05Config(**checkpoint["model_config"])) + model.load_state_dict(checkpoint["model"]) + model.to(device).eval() + flow = FlowMatching(cfg["inference"]["flow_steps"]) + rows = [] + figure, axes = plt.subplots(1, len(dataset), figsize=(4 * len(dataset), 4), squeeze=False) + for index, sample in enumerate(dataset): + image = sample["image"][None].to(device) + tokens = sample["prefix_tokens"][None].to(device) + # Autoregressively predict semantic subtask followed by the complete + # discretized action stream (ACTION_START, 32 scalar tokens, ACTION_END). + zero_action = torch.zeros(1, dataset.action_horizon, 2, device=device) + generated: list[int] = [] + expected_tokens = 1 + 2 + dataset.action_horizon * 2 + current_tokens = tokens + for _ in range(expected_tokens): + semantic = model( + image, current_tokens, zero_action, + torch.zeros(1, device=device), + generated_token_start=tokens.shape[1] + ) + next_token = int(semantic.text_logits[0, -1].argmax().cpu()) + generated.append(next_token) + current_tokens = torch.cat( + (current_tokens, torch.tensor([[next_token]], device=device)), dim=1 + ) + predicted_token = generated[0] + vlm_action_tokens = torch.tensor(generated[1:], dtype=torch.long) + try: + vlm_normalized = dataset.tokenizer.decode_actions(vlm_action_tokens) + vlm_xy = dataset.action_normalizer.decode( + vlm_normalized, sample["history_xy"] + ).numpy() + vlm_valid = True + except ValueError: + vlm_xy = np.full_like(sample["future_xy"].numpy(), np.nan) + vlm_valid = False + conditioned_tokens = torch.cat( + (tokens, torch.tensor([[predicted_token]], device=device)), dim=1 + ) + def velocity(x: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + return model( + image, conditioned_tokens, x, t, + generated_token_start=tokens.shape[1] + ).vector_field + normalized = flow.sample( + velocity, (1, dataset.action_horizon, 2), device, + steps=cfg["inference"]["flow_steps"], + generator=None, + )[0].cpu() + predicted_xy = dataset.action_normalizer.decode( + normalized, sample["history_xy"] + ).numpy() + target_xy = sample["future_xy"].numpy() + diffusion_mse = float(np.mean((predicted_xy - target_xy) ** 2)) + vlm_mse = float(np.mean((vlm_xy - target_xy) ** 2)) if vlm_valid else None + target_label = dataset.tokenizer.subtasks.decode(sample["subtask_target"]) + try: + predicted_label = dataset.tokenizer.subtasks.decode(predicted_token) + except ValueError: + predicted_label = f"token_{predicted_token}" + rows.append({ + "sample_id": sample["sample_id"], "target_subtask": target_label, + "predicted_subtask": predicted_label, + "vlm_action_tokens_valid": vlm_valid, + "vlm_trajectory_mse": vlm_mse, + "diffusion_trajectory_mse": diffusion_mse, + }) + ax = axes[0, index] + ax.plot(target_xy[:, 0], target_xy[:, 1], "o-", label="target") + ax.plot(vlm_xy[:, 0], vlm_xy[:, 1], "s--", label="VLM tokens") + ax.plot(predicted_xy[:, 0], predicted_xy[:, 1], "x--", label="diffusion") + ax.scatter([0], [0], c="black", marker="s", label="ego") + vlm_text = "invalid" if vlm_mse is None else f"{vlm_mse:.3f}" + ax.set_title( + f"{predicted_label}\nVLM={vlm_text}, diffusion={diffusion_mse:.3f}" + ) + ax.set_xlabel("forward x (m)") + ax.set_ylabel("left y (m)") + ax.axis("equal") + ax.grid(True, alpha=.3) + ax.legend(fontsize=8) + figure.tight_layout() + output_dir = Path(checkpoint_path).parent + plot_path = output_dir / "trajectory_comparison.png" + figure.savefig(plot_path, dpi=160) + plt.close(figure) + write_json(output_dir / "evaluation.json", rows) + print(rows) + return plot_path + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint", default="outputs/overfit/checkpoint.pt") + parser.add_argument("--manifest") + args = parser.parse_args() + evaluate(args.checkpoint, args.manifest) + + +if __name__ == "__main__": + main() diff --git a/src/pi05mini/model.py b/src/pi05mini/model.py new file mode 100644 index 0000000..0496397 --- /dev/null +++ b/src/pi05mini/model.py @@ -0,0 +1,396 @@ +"""A compact, pure-PyTorch approximation of the pi0.5 dual-expert model. + +The important architectural detail is that prefix and action tokens have +different parameters at every Transformer layer. Prefix queries only attend +to prefix keys, while action queries attend to the complete prefix and to all +action tokens. This is deliberately implemented explicitly (instead of +concatenating the streams into an ordinary Transformer) so the information +flow is easy to inspect in a minimal reproduction. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Optional + +import torch +from torch import Tensor, nn +import torch.nn.functional as F + + +@dataclass +class Pi05Config: + vocab_size: int = 512 + action_dim: int = 2 + image_channels: int = 3 + image_size: int = 64 + patch_size: int = 16 + d_model: int = 128 + n_heads: int = 4 + n_layers: int = 4 + mlp_ratio: int = 4 + max_text_tokens: int = 64 + max_action_tokens: int = 32 + dropout: float = 0.0 + rms_eps: float = 1e-6 + + def __post_init__(self) -> None: + if self.d_model % self.n_heads: + raise ValueError("d_model must be divisible by n_heads") + if self.image_size % self.patch_size: + raise ValueError("image_size must be divisible by patch_size") + + @property + def max_image_tokens(self) -> int: + return (self.image_size // self.patch_size) ** 2 + + @property + def max_prefix_tokens(self) -> int: + return self.max_image_tokens + self.max_text_tokens + + +@dataclass +class Pi05Output: + text_logits: Tensor + vector_field: Tensor + prefix_hidden: Tensor + action_hidden: Tensor + text_token_offset: int + loss: Optional[Tensor] = None + text_loss: Optional[Tensor] = None + flow_loss: Optional[Tensor] = None + + +class RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.eps = eps + + def forward(self, x: Tensor) -> Tensor: + # Accumulating the variance in fp32 avoids overflows under autocast. + rms = x.float().pow(2).mean(dim=-1, keepdim=True) + normalized = x * torch.rsqrt(rms.to(x.dtype) + self.eps) + return normalized * self.weight + + +class AdaRMSNorm(nn.Module): + """RMSNorm modulated by a flow-time embedding.""" + + def __init__(self, dim: int, cond_dim: int, eps: float = 1e-6) -> None: + super().__init__() + self.norm = RMSNorm(dim, eps) + self.modulation = nn.Linear(cond_dim, 2 * dim) + # Start as an ordinary RMSNorm, which makes tiny-data training stable. + nn.init.zeros_(self.modulation.weight) + nn.init.zeros_(self.modulation.bias) + + def forward(self, x: Tensor, condition: Tensor) -> Tensor: + shift, scale = self.modulation(condition).chunk(2, dim=-1) + return self.norm(x) * (1 + scale[:, None, :]) + shift[:, None, :] + + +class TimeEmbedding(nn.Module): + def __init__(self, dim: int) -> None: + super().__init__() + fourier_dim = max(8, dim // 2) + if fourier_dim % 2: + fourier_dim += 1 + self.fourier_dim = fourier_dim + self.mlp = nn.Sequential( + nn.Linear(fourier_dim, dim), nn.SiLU(), nn.Linear(dim, dim) + ) + + def forward(self, timestep: Tensor) -> Tensor: + timestep = timestep.reshape(-1).float() + half = self.fourier_dim // 2 + frequencies = torch.exp( + torch.linspace(math.log(1.0), math.log(1000.0), half, + device=timestep.device, dtype=timestep.dtype) + ) + angles = 2 * math.pi * timestep[:, None] * frequencies[None, :] + embedding = torch.cat((angles.sin(), angles.cos()), dim=-1) + return self.mlp(embedding) + + +class AttentionWeights(nn.Module): + """One expert's independent Q/K/V/output projections.""" + + def __init__(self, dim: int) -> None: + super().__init__() + self.q = nn.Linear(dim, dim, bias=False) + self.k = nn.Linear(dim, dim, bias=False) + self.v = nn.Linear(dim, dim, bias=False) + self.out = nn.Linear(dim, dim, bias=False) + + +class MLP(nn.Module): + def __init__(self, dim: int, ratio: int, dropout: float) -> None: + super().__init__() + hidden = dim * ratio + self.net = nn.Sequential( + nn.Linear(dim, hidden), + nn.GELU(approximate="tanh"), + nn.Dropout(dropout), + nn.Linear(hidden, dim), + nn.Dropout(dropout), + ) + + def forward(self, x: Tensor) -> Tensor: + return self.net(x) + + +def _attention( + query: Tensor, + key: Tensor, + value: Tensor, + n_heads: int, + allowed: Optional[Tensor], + dropout: float, + training: bool, +) -> Tensor: + batch, q_len, width = query.shape + k_len = key.shape[1] + head_dim = width // n_heads + + def split(x: Tensor) -> Tensor: + return x.reshape(batch, -1, n_heads, head_dim).transpose(1, 2) + + q, k, v = split(query), split(key), split(value) + scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(head_dim) + if allowed is not None: + # allowed is [B or 1, Q, K]; True means the edge exists. + scores = scores.masked_fill(~allowed[:, None, :, :], torch.finfo(scores.dtype).min) + probabilities = scores.float().softmax(dim=-1).to(scores.dtype) + probabilities = F.dropout(probabilities, p=dropout, training=training) + output = torch.matmul(probabilities, v) + return output.transpose(1, 2).contiguous().reshape(batch, q_len, width) + + +class DualExpertBlock(nn.Module): + def __init__(self, config: Pi05Config) -> None: + super().__init__() + d = config.d_model + self.n_heads = config.n_heads + self.dropout = config.dropout + + # The two streams never share Transformer parameters. + self.vlm_norm1 = RMSNorm(d, config.rms_eps) + self.vlm_attention = AttentionWeights(d) + self.vlm_norm2 = RMSNorm(d, config.rms_eps) + self.vlm_mlp = MLP(d, config.mlp_ratio, config.dropout) + + self.action_norm1 = AdaRMSNorm(d, d, config.rms_eps) + self.action_attention = AttentionWeights(d) + self.action_norm2 = AdaRMSNorm(d, d, config.rms_eps) + self.action_mlp = MLP(d, config.mlp_ratio, config.dropout) + + def forward( + self, + prefix: Tensor, + action: Tensor, + time_condition: Tensor, + prefix_edges: Tensor, + action_edges: Tensor, + ) -> tuple[Tensor, Tensor]: + prefix_input = self.vlm_norm1(prefix) + p = self.vlm_attention + prefix_update = _attention( + p.q(prefix_input), p.k(prefix_input), p.v(prefix_input), + self.n_heads, prefix_edges, self.dropout, self.training + ) + prefix = prefix + p.out(prefix_update) + prefix = prefix + self.vlm_mlp(self.vlm_norm2(prefix)) + + action_input = self.action_norm1(action, time_condition) + a = self.action_attention + # Prefix KV uses the VLM expert's parameters; action KV uses the action + # expert's parameters. Thus action can read prefix without allowing a + # reverse information path from action into the prefix stream. + key = torch.cat((p.k(prefix_input), a.k(action_input)), dim=1) + value = torch.cat((p.v(prefix_input), a.v(action_input)), dim=1) + action_update = _attention( + a.q(action_input), key, value, self.n_heads, action_edges, + self.dropout, self.training + ) + action = action + a.out(action_update) + action = action + self.action_mlp(self.action_norm2(action, time_condition)) + return prefix, action + + +class Pi05Mini(nn.Module): + """Minimal pi0.5-style model for architecture validation/overfitting.""" + + def __init__(self, config: Optional[Pi05Config] = None) -> None: + super().__init__() + self.config = config or Pi05Config() + c = self.config + self.patch_encoder = nn.Conv2d( + c.image_channels, c.d_model, c.patch_size, stride=c.patch_size + ) + self.token_embedding = nn.Embedding(c.vocab_size, c.d_model) + self.action_projection = nn.Linear(c.action_dim, c.d_model) + self.prefix_position = nn.Parameter( + torch.randn(1, c.max_prefix_tokens, c.d_model) * 0.01 + ) + self.action_position = nn.Parameter( + torch.randn(1, c.max_action_tokens, c.d_model) * 0.01 + ) + self.time_embedding = TimeEmbedding(c.d_model) + self.layers = nn.ModuleList(DualExpertBlock(c) for _ in range(c.n_layers)) + self.prefix_norm = RMSNorm(c.d_model, c.rms_eps) + self.action_norm = AdaRMSNorm(c.d_model, c.d_model, c.rms_eps) + self.text_head = nn.Linear(c.d_model, c.vocab_size, bias=False) + self.vector_field_head = nn.Linear(c.d_model, c.action_dim) + self.text_head.weight = self.token_embedding.weight + + def encode_prefix(self, pixel_values: Tensor, input_ids: Tensor) -> tuple[Tensor, int]: + patches = self.patch_encoder(pixel_values).flatten(2).transpose(1, 2) + text_tokens = self.token_embedding(input_ids) + # pi0.5 represents state in the VLM vocabulary. There is deliberately + # no separate continuous-state projection in this model. + prefix = torch.cat((patches, text_tokens), dim=1) + if prefix.shape[1] > self.config.max_prefix_tokens: + raise ValueError("prefix exceeds configured maximum length") + prefix = prefix + self.prefix_position[:, : prefix.shape[1]] + return prefix, patches.shape[1] + + def _masks( + self, + batch: int, + prefix_length: int, + action_length: int, + text_offset: int, + attention_mask: Optional[Tensor], + action_prefix_mask: Optional[Tensor], + generated_token_start: Optional[int], + device: torch.device, + ) -> tuple[Tensor, Tensor]: + valid = torch.ones(batch, prefix_length, dtype=torch.bool, device=device) + if attention_mask is not None: + valid[:, text_offset:] = attention_mask.bool() + # Image plus observation/instruction/state tokens form a fully + # bidirectional observation block. Tokens generated after that block + # (subtask and FAST actions) read the observation and earlier generated + # tokens causally; observation queries cannot read generated tokens. + generated_start = prefix_length if generated_token_start is None else text_offset + generated_token_start + if not text_offset <= generated_start <= prefix_length: + raise ValueError("generated_token_start must index input_ids") + q = torch.arange(prefix_length, device=device)[:, None] + k = torch.arange(prefix_length, device=device)[None, :] + structural = torch.where(q < generated_start, k < generated_start, k <= q) + prefix_edges = structural[None] & valid[:, None, :] + # Padded prefix queries retain a harmless self edge to avoid all -inf + # rows; their outputs are not used by the text loss. + query_valid = valid[:, :, None] + identity = torch.eye(prefix_length, dtype=torch.bool, device=device)[None] + prefix_edges = torch.where(query_valid, prefix_edges, identity) + + action_valid = valid + if action_prefix_mask is not None: + supplied = action_prefix_mask.bool() + if supplied.shape[1] == prefix_length - text_offset: + supplied = torch.cat( + (torch.ones(batch, text_offset, dtype=torch.bool, device=device), supplied), + dim=1, + ) + if supplied.shape != valid.shape: + raise ValueError("action_prefix_mask must cover text tokens or the full prefix") + action_valid = action_valid & supplied + prefix_keys = action_valid[:, None, :].expand(batch, action_length, prefix_length) + action_keys = torch.ones( + batch, action_length, action_length, dtype=torch.bool, device=device + ) + action_edges = torch.cat((prefix_keys, action_keys), dim=-1) + return prefix_edges, action_edges + + def forward( + self, + pixel_values: Tensor, + input_ids: Tensor, + noisy_actions: Tensor, + timesteps: Tensor, + attention_mask: Optional[Tensor] = None, + action_prefix_mask: Optional[Tensor] = None, + generated_token_start: Optional[int] = None, + text_labels: Optional[Tensor] = None, + target_velocity: Optional[Tensor] = None, + text_loss_weight: float = 1.0, + flow_loss_weight: float = 1.0, + ) -> Pi05Output: + """Run both experts. + + ``timesteps`` is a scalar per batch item in [0, 1]. ``text_labels`` + corresponds only to ``input_ids`` (not image prefix positions) + and may contain -100. ``target_velocity`` has the same shape as + ``noisy_actions``. + """ + if noisy_actions.ndim != 3: + raise ValueError("noisy_actions must have shape [batch, horizon, action_dim]") + if noisy_actions.shape[1] > self.config.max_action_tokens: + raise ValueError("action horizon exceeds configured maximum") + + prefix, text_offset = self.encode_prefix(pixel_values, input_ids) + horizon = noisy_actions.shape[1] + action = self.action_projection(noisy_actions) + action = action + self.action_position[:, :horizon] + condition = self.time_embedding(timesteps).to(action.dtype) + prefix_edges, action_edges = self._masks( + prefix.shape[0], prefix.shape[1], horizon, text_offset, + attention_mask, action_prefix_mask, generated_token_start, prefix.device + ) + for layer in self.layers: + prefix, action = layer( + prefix, action, condition, prefix_edges, action_edges + ) + + prefix_hidden = self.prefix_norm(prefix) + action_hidden = self.action_norm(action, condition) + # Return logits only for actual text positions, which keeps downstream + # next-token loss/generation code uncomplicated. + text_logits = self.text_head(prefix_hidden[:, text_offset:]) + vector_field = self.vector_field_head(action_hidden) + + text_loss = None + if text_labels is not None: + text_loss = F.cross_entropy( + text_logits.reshape(-1, text_logits.shape[-1]), + text_labels.reshape(-1), ignore_index=-100 + ) + flow_loss = None + if target_velocity is not None: + flow_loss = F.mse_loss(vector_field, target_velocity) + loss = None + if text_loss is not None or flow_loss is not None: + loss = sum( + component for component in ( + None if text_loss is None else text_loss_weight * text_loss, + None if flow_loss is None else flow_loss_weight * flow_loss, + ) if component is not None + ) + return Pi05Output( + text_logits=text_logits, + vector_field=vector_field, + prefix_hidden=prefix_hidden, + action_hidden=action_hidden, + text_token_offset=text_offset, + loss=loss, + text_loss=text_loss, + flow_loss=flow_loss, + ) + + +def flow_matching_sample(actions: Tensor) -> tuple[Tensor, Tensor, Tensor]: + """Construct pi-style conditional-flow training targets. + + The path is x_t = (1-t) * noise + t * action, with constant target vector + field ``action - noise``. + """ + batch = actions.shape[0] + timestep = torch.rand(batch, device=actions.device, dtype=actions.dtype) + noise = torch.randn_like(actions) + broadcast_t = timestep.reshape(batch, *([1] * (actions.ndim - 1))) + noisy_actions = (1 - broadcast_t) * noise + broadcast_t * actions + target_velocity = actions - noise + return noisy_actions, timestep, target_velocity diff --git a/src/pi05mini/runtime.py b/src/pi05mini/runtime.py new file mode 100644 index 0000000..d015228 --- /dev/null +++ b/src/pi05mini/runtime.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import json +import random +from pathlib import Path + +import numpy as np +import torch +import yaml + + +def load_config(path: str | Path) -> dict: + with Path(path).open("r", encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +def seed_everything(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + +def choose_device(requested: str = "auto") -> torch.device: + if requested != "auto": + return torch.device(requested) + if torch.cuda.is_available(): + return torch.device("cuda") + if torch.backends.mps.is_available(): + return torch.device("mps") + return torch.device("cpu") + + +def write_json(path: str | Path, value: object) -> None: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(value, indent=2, ensure_ascii=False), encoding="utf-8") diff --git a/src/pi05mini/tokenizer.py b/src/pi05mini/tokenizer.py new file mode 100644 index 0000000..461b644 --- /dev/null +++ b/src/pi05mini/tokenizer.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Iterable + +import torch + + +@dataclass(frozen=True) +class CoordinateRange: + minimum: float + maximum: float + + def __post_init__(self) -> None: + if self.maximum <= self.minimum: + raise ValueError("coordinate maximum must be greater than minimum") + + +class CoordinateTokenizer: + """Uniformly discretize XY state values into the literal token range 0..255.""" + + bins = 256 + + def __init__( + self, + x_range: tuple[float, float] = (-10.0, 10.0), + y_range: tuple[float, float] = (-5.0, 5.0), + ) -> None: + self.ranges = ( + CoordinateRange(*x_range), + CoordinateRange(*y_range), + ) + + def encode(self, coordinates: torch.Tensor | Iterable[Iterable[float]]) -> torch.Tensor: + values = torch.as_tensor(coordinates, dtype=torch.float32) + if values.shape[-1] != 2: + raise ValueError("coordinates must end in an XY dimension of size 2") + low = values.new_tensor([axis.minimum for axis in self.ranges]) + high = values.new_tensor([axis.maximum for axis in self.ranges]) + unit = ((values - low) / (high - low)).clamp(0.0, 1.0) + return torch.round(unit * (self.bins - 1)).to(torch.long) + + def decode(self, tokens: torch.Tensor | Iterable[Iterable[int]]) -> torch.Tensor: + encoded = torch.as_tensor(tokens, dtype=torch.long) + if encoded.shape[-1] != 2: + raise ValueError("coordinate tokens must end in an XY dimension of size 2") + if encoded.numel() and (encoded.min() < 0 or encoded.max() >= self.bins): + raise ValueError("coordinate tokens must be in the inclusive range 0..255") + low = torch.tensor([axis.minimum for axis in self.ranges], dtype=torch.float32) + high = torch.tensor([axis.maximum for axis in self.ranges], dtype=torch.float32) + return low + encoded.float() / (self.bins - 1) * (high - low) + + +class SubtaskVocabulary: + """Small semantic vocabulary standing in for pi0.5's generated language subtask.""" + + labels = ( + "keep_lane", + "steer_left", + "steer_right", + "slow_down", + "stop", + "accelerate", + "unknown", + ) + + def __init__(self, first_token_id: int = 260) -> None: + self.first_token_id = int(first_token_id) + self._label_to_token = { + label: self.first_token_id + index for index, label in enumerate(self.labels) + } + self._token_to_label = {token: label for label, token in self._label_to_token.items()} + + @property + def size(self) -> int: + return len(self.labels) + + @property + def token_ids(self) -> tuple[int, ...]: + return tuple(self._label_to_token[label] for label in self.labels) + + def classify(self, reasoning: str) -> str: + text = reasoning.casefold() + # More specific commands precede generic lane/follow language. + if any(term in text for term in ("stop", "halt", "stationary")): + return "stop" + if any(term in text for term in ("slow", "deceler", "reduce speed", "brake")): + return "slow_down" + if any(term in text for term in ("accelerat", "speed up")): + return "accelerate" + if any(term in text for term in ("left", "counterclockwise")): + return "steer_left" + if any(term in text for term in ("right", "clockwise")): + return "steer_right" + if any(term in text for term in ("lane", "straight", "follow", "continue", "proceed")): + return "keep_lane" + return "unknown" + + def encode(self, label_or_reasoning: str) -> int: + label = ( + label_or_reasoning + if label_or_reasoning in self._label_to_token + else self.classify(label_or_reasoning) + ) + return self._label_to_token[label] + + def decode(self, token_id: int | torch.Tensor) -> str: + token = int(token_id) + try: + return self._token_to_label[token] + except KeyError as error: + raise ValueError(f"unknown subtask token: {token}") from error + + +class MiniTokenizer: + """Compose image/state, semantic, and FAST-lite action tokens. + + Actions are z-score-normalized by the dataset before reaching this + tokenizer. A complete ``[horizon, 2]`` chunk is transformed along time + with an orthonormal DCT-II. All coefficients are retained and serialized + in frequency-major XY order, so low-frequency motion appears first in the + token stream. The coefficients are divided by ``sqrt(horizon)`` before + being uniformly quantized over ``[-4, 4]`` into 64 shared tokens. This + keeps the fixed quantization interval valid for any horizon when the input + actions have first been clipped to the same interval. + + This intentionally implements only the central transform/order idea from + FAST. It does not learn dataset-specific coefficient bins or drop trailing + high-frequency coefficients. + """ + + BOS = 256 + IMAGE = 257 + STATE = 258 + SUBTASK_QUERY = 259 + ACTION_START = SUBTASK_QUERY + 1 + len(SubtaskVocabulary.labels) + ACTION_END = ACTION_START + 1 + ACTION_TOKEN_START = ACTION_END + 1 + ACTION_BINS = 64 + ACTION_TOKEN_END = ACTION_TOKEN_START + ACTION_BINS - 1 + ACTION_MINIMUM = -4.0 + ACTION_MAXIMUM = 4.0 + + def __init__( + self, + coordinate_tokenizer: CoordinateTokenizer | None = None, + subtasks: SubtaskVocabulary | None = None, + ) -> None: + self.coordinates = coordinate_tokenizer or CoordinateTokenizer() + self.subtasks = subtasks or SubtaskVocabulary(first_token_id=self.SUBTASK_QUERY + 1) + + @property + def vocab_size(self) -> int: + return max(max(self.subtasks.token_ids), self.ACTION_TOKEN_END) + 1 + + @property + def action_token_ids(self) -> range: + """Contiguous vocabulary interval reserved for quantized DCT coefficients.""" + + return range(self.ACTION_TOKEN_START, self.ACTION_TOKEN_END + 1) + + def build_prefix(self, history_xy: torch.Tensor) -> torch.Tensor: + coordinate_tokens = self.coordinates.encode(history_xy).reshape(-1) + specials = torch.tensor([self.BOS, self.IMAGE, self.STATE], dtype=torch.long) + query = torch.tensor([self.SUBTASK_QUERY], dtype=torch.long) + return torch.cat((specials, coordinate_tokens.cpu(), query), dim=0) + + def encode_subtask(self, reasoning: str) -> torch.Tensor: + return torch.tensor(self.subtasks.encode(reasoning), dtype=torch.long) + + @staticmethod + def _orthonormal_dct_matrix( + horizon: int, + *, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + """Return the orthonormal DCT-II matrix ``[frequency, time]``.""" + + if horizon <= 0: + raise ValueError("DCT horizon must be positive") + time = torch.arange(horizon, device=device, dtype=dtype) + frequency = torch.arange(horizon, device=device, dtype=dtype).unsqueeze(1) + matrix = torch.cos(math.pi / horizon * (time + 0.5) * frequency) + matrix[0] *= math.sqrt(1.0 / horizon) + if horizon > 1: + matrix[1:] *= math.sqrt(2.0 / horizon) + return matrix + + def dct_actions( + self, actions: torch.Tensor | Iterable[Iterable[float]] + ) -> torch.Tensor: + """Apply an orthonormal DCT-II along an action chunk's time axis.""" + + values = torch.as_tensor(actions, dtype=torch.float32) + if values.ndim != 2 or values.shape[-1] != 2: + raise ValueError("actions must have shape [horizon, 2]") + if values.shape[0] == 0: + raise ValueError("actions must contain at least one timestep") + transform = self._orthonormal_dct_matrix( + values.shape[0], device=values.device, dtype=values.dtype + ) + return transform @ values + + def idct_actions( + self, coefficients: torch.Tensor | Iterable[Iterable[float]] + ) -> torch.Tensor: + """Invert :meth:`dct_actions` using the transpose of its basis.""" + + values = torch.as_tensor(coefficients, dtype=torch.float32) + if values.ndim != 2 or values.shape[-1] != 2: + raise ValueError("DCT coefficients must have shape [horizon, 2]") + if values.shape[0] == 0: + raise ValueError("DCT coefficients must contain at least one frequency") + transform = self._orthonormal_dct_matrix( + values.shape[0], device=values.device, dtype=values.dtype + ) + return transform.transpose(0, 1) @ values + + def encode_actions( + self, actions: torch.Tensor | Iterable[Iterable[float]] + ) -> torch.Tensor: + """DCT-transform and encode normalized actions as a marked token stream.""" + + values = torch.as_tensor(actions, dtype=torch.float32) + if values.ndim != 2 or values.shape[-1] != 2: + raise ValueError("actions must have shape [horizon, 2]") + if values.shape[0] == 0: + raise ValueError("actions must contain at least one timestep") + values = values.clamp(self.ACTION_MINIMUM, self.ACTION_MAXIMUM) + coefficients = self.dct_actions(values) / math.sqrt(values.shape[0]) + # Exact zero is useful for a stable shared center bin. Tiny non-zero + # values here are only floating-point residue from the DCT basis. + coefficients = torch.where( + coefficients.abs() < 1e-6, + torch.zeros_like(coefficients), + coefficients, + ) + unit = ( + ( + coefficients.clamp(self.ACTION_MINIMUM, self.ACTION_MAXIMUM) + - self.ACTION_MINIMUM + ) + / (self.ACTION_MAXIMUM - self.ACTION_MINIMUM) + ) + quantized = torch.round(unit * (self.ACTION_BINS - 1)).to(torch.long) + # Row-major flattening is frequency-major: DC XY, frequency-1 XY, ... + action_tokens = quantized.reshape(-1) + self.ACTION_TOKEN_START + boundaries = torch.tensor( + [self.ACTION_START, self.ACTION_END], + dtype=torch.long, + device=action_tokens.device, + ) + return torch.cat((boundaries[:1], action_tokens, boundaries[1:])) + + def decode_actions(self, tokens: torch.Tensor | Iterable[int]) -> torch.Tensor: + """Decode FAST-lite tokens back to normalized ``[horizon, 2]`` actions.""" + + encoded = torch.as_tensor(tokens, dtype=torch.long) + if encoded.ndim != 1: + raise ValueError("action tokens must be a one-dimensional sequence") + if ( + encoded.numel() < 4 + or encoded[0] != self.ACTION_START + or encoded[-1] != self.ACTION_END + ): + raise ValueError("action tokens must be enclosed by ACTION_START and ACTION_END") + payload = encoded[1:-1] + if payload.numel() % 2: + raise ValueError("action token payload must contain complete XY pairs") + if payload.min() < self.ACTION_TOKEN_START or payload.max() > self.ACTION_TOKEN_END: + raise ValueError( + "action token payload is outside the reserved action token interval" + ) + quantized = payload - self.ACTION_TOKEN_START + normalized_coefficients = ( + quantized.float() + / (self.ACTION_BINS - 1) + * (self.ACTION_MAXIMUM - self.ACTION_MINIMUM) + + self.ACTION_MINIMUM + ) + horizon = payload.numel() // 2 + coefficients = normalized_coefficients.reshape(horizon, 2) * math.sqrt(horizon) + return self.idct_actions(coefficients) diff --git a/src/pi05mini/train.py b/src/pi05mini/train.py new file mode 100644 index 0000000..2524517 --- /dev/null +++ b/src/pi05mini/train.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import torch +from torch.utils.data import DataLoader + +from .data import Pi05OverfitDataset, materialize_subset +from .flow import FlowMatching +from .model import Pi05Config, Pi05Mini +from .runtime import choose_device, load_config, seed_everything, write_json + + +def _collate(rows: list[dict]) -> dict: + tensor_keys = ( + "image", "prefix_tokens", "subtask_target", "actions", "action_tokens" + ) + result = {key: torch.stack([row[key] for row in rows]) for key in tensor_keys} + result["sample_id"] = [row["sample_id"] for row in rows] + return result + + +def build_model(config: dict, dataset: Pi05OverfitDataset) -> Pi05Mini: + model_cfg = config["model"] + return Pi05Mini(Pi05Config( + vocab_size=dataset.tokenizer.vocab_size, + action_dim=2, + image_size=config["data"]["image_size"], + patch_size=model_cfg["patch_size"], + d_model=model_cfg["d_model"], + n_heads=model_cfg["n_heads"], + n_layers=model_cfg["n_layers"], + mlp_ratio=model_cfg["mlp_ratio"], + max_text_tokens=model_cfg["max_text_tokens"], + max_action_tokens=config["data"]["action_horizon"], + dropout=model_cfg["dropout"], + )) + + +def train(config_path: str) -> Path: + cfg = load_config(config_path) + root = Path(__file__).resolve().parents[2] + seed_everything(cfg["seed"]) + device = choose_device(cfg.get("device", "auto")) + manifest = materialize_subset( + source_manifest=cfg["data"]["source_manifest"], + output_dir=root / "data" / "overfit", + num_variants=cfg["data"]["num_variants"], + ) + dataset = Pi05OverfitDataset( + manifest, image_size=cfg["data"]["image_size"], + action_horizon=cfg["data"]["action_horizon"] + ) + loader = DataLoader( + dataset, batch_size=min(cfg["train"]["batch_size"], len(dataset)), + shuffle=True, collate_fn=_collate + ) + model = build_model(cfg, dataset).to(device) + flow = FlowMatching(cfg["inference"]["flow_steps"]) + output_dir = root / cfg["output_dir"] + output_dir.mkdir(parents=True, exist_ok=True) + history = [] + + def is_action_parameter(name: str) -> bool: + return any(part in name for part in ( + "action_", "time_embedding", "vector_field_head" + )) + + # Stage 1 follows pi0.5 pre-training: train the VLM/token pathway with + # discrete state and FAST-lite actions, while the flow expert is untouched. + stage1_params = [p for name, p in model.named_parameters() if not is_action_parameter(name)] + optimizer = torch.optim.AdamW( + stage1_params, lr=cfg["train"]["learning_rate"], + weight_decay=cfg["train"]["weight_decay"] + ) + + def next_batch(iterator): + try: + return next(iterator), iterator + except StopIteration: + iterator = iter(loader) + return next(iterator), iterator + + iterator = iter(loader) + model.train() + for step in range(1, cfg["train"]["pretrain_steps"] + 1): + batch, iterator = next_batch(iterator) + image = batch["image"].to(device) + tokens = batch["prefix_tokens"].to(device) + actions = batch["actions"].to(device) + subtask = batch["subtask_target"].to(device)[:, None] + action_tokens = batch["action_tokens"].to(device) + targets = torch.cat((subtask, action_tokens), dim=1) + # Teacher-forced autoregressive stream. The final target is predicted + # from the preceding token and therefore is not part of model_tokens. + model_tokens = torch.cat((tokens, targets[:, :-1]), dim=1) + model_labels = torch.full_like(model_tokens, -100) + query_index = tokens.shape[1] - 1 + model_labels[:, query_index:] = targets + # Flow tokens may read observation/state and the generated semantic + # subtask, but not teacher-forced discrete action tokens. + zero_action = torch.zeros_like(actions) + output = model( + image, model_tokens, zero_action, + torch.zeros(image.shape[0], device=device), + generated_token_start=tokens.shape[1], + text_labels=model_labels, + ) + optimizer.zero_grad(set_to_none=True) + output.loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), cfg["train"]["grad_clip_norm"]) + optimizer.step() + if step == 1 or step % cfg["train"]["log_every"] == 0: + record = { + "stage": "pretrain", + "step": step, + "loss": float(output.loss.detach().cpu()), + "text_loss": float(output.text_loss.detach().cpu()), + "flow_loss": None, + } + history.append(record) + print(json.dumps(record)) + + torch.save(model.state_dict(), output_dir / "stage1_vlm.pt") + + # Stage 2 follows post-training: add the still-random action expert and + # jointly retain token prediction while optimizing continuous flow. + optimizer = torch.optim.AdamW( + model.parameters(), lr=cfg["train"]["learning_rate"], + weight_decay=cfg["train"]["weight_decay"] + ) + iterator = iter(loader) + for step in range(1, cfg["train"]["posttrain_steps"] + 1): + batch, iterator = next_batch(iterator) + image = batch["image"].to(device) + tokens = batch["prefix_tokens"].to(device) + actions = batch["actions"].to(device) + subtask = batch["subtask_target"].to(device)[:, None] + action_tokens = batch["action_tokens"].to(device) + targets = torch.cat((subtask, action_tokens), dim=1) + model_tokens = torch.cat((tokens, targets[:, :-1]), dim=1) + model_labels = torch.full_like(model_tokens, -100) + model_labels[:, tokens.shape[1] - 1:] = targets + action_prefix_mask = torch.zeros_like(model_tokens, dtype=torch.bool) + action_prefix_mask[:, : tokens.shape[1] + 1] = True + noisy, timestep, velocity = flow.training_batch(actions) + output = model( + image, model_tokens, noisy, timestep, + action_prefix_mask=action_prefix_mask, + generated_token_start=tokens.shape[1], + text_labels=model_labels, target_velocity=velocity, + text_loss_weight=cfg["train"]["semantic_loss_weight"], + flow_loss_weight=cfg["train"]["flow_loss_weight"], + ) + optimizer.zero_grad(set_to_none=True) + output.loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), cfg["train"]["grad_clip_norm"]) + optimizer.step() + if step == 1 or step % cfg["train"]["log_every"] == 0: + record = { + "stage": "posttrain", "step": step, + "loss": float(output.loss.detach().cpu()), + "text_loss": float(output.text_loss.detach().cpu()), + "flow_loss": float(output.flow_loss.detach().cpu()), + } + history.append(record) + print(json.dumps(record)) + checkpoint = { + "model": model.state_dict(), "config": cfg, + "model_config": model.config.__dict__, + "action_statistics": dataset.action_normalizer.as_dict(), + } + torch.save(checkpoint, output_dir / "checkpoint.pt") + write_json(output_dir / "training_history.json", history) + write_json(output_dir / "summary.json", { + "device": str(device), "samples": len(dataset), + "parameters": sum(p.numel() for p in model.parameters()), + "initial": history[0], "final": history[-1], + }) + return output_dir / "checkpoint.pt" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", default="configs/overfit.yaml") + args = parser.parse_args() + train(args.config) + + +if __name__ == "__main__": + main() diff --git a/tests/test_action_tokens.py b/tests/test_action_tokens.py new file mode 100644 index 0000000..9025dbe --- /dev/null +++ b/tests/test_action_tokens.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import pytest +import torch + +from pi05mini.tokenizer import MiniTokenizer + + +def test_orthonormal_dct_round_trip() -> None: + tokenizer = MiniTokenizer() + actions = torch.tensor( + [ + [-4.0, -2.75], + [-0.25, 0.0], + [1.375, 3.9], + [4.0, 2.125], + ] + ) + + coefficients = tokenizer.dct_actions(actions) + reconstructed = tokenizer.idct_actions(coefficients) + + assert coefficients.shape == actions.shape + torch.testing.assert_close(reconstructed, actions, atol=1e-5, rtol=1e-5) + + +def test_fast_lite_tokens_round_trip_with_bounded_quantization_error() -> None: + tokenizer = MiniTokenizer() + actions = torch.tensor( + [ + [-3.5, -2.75], + [-0.25, 0.0], + [1.375, 3.5], + [3.75, 2.125], + ] + ) + + tokens = tokenizer.encode_actions(actions) + reconstructed = tokenizer.decode_actions(tokens) + + bin_width = ( + tokenizer.ACTION_MAXIMUM - tokenizer.ACTION_MINIMUM + ) / (tokenizer.ACTION_BINS - 1) + assert reconstructed.shape == actions.shape + # Orthonormal IDCT preserves total coefficient error. Coefficients are + # quantized after division by sqrt(horizon), yielding this RMSE bound. + rmse = torch.sqrt(torch.mean((reconstructed - actions).square())) + assert float(rmse) <= bin_width / 2 * actions.shape[0] ** 0.5 + 1e-6 + + +def test_action_tokens_are_serialized_low_frequency_first() -> None: + tokenizer = MiniTokenizer() + horizon = 4 + coefficients = torch.zeros(horizon, 2) + coefficients[0, 0] = 1.5 * horizon**0.5 # positive X DC coefficient + coefficients[1, 1] = -1.5 * horizon**0.5 # negative Y first harmonic + actions = tokenizer.idct_actions(coefficients) + + payload = tokenizer.encode_actions(actions)[1:-1] + zero_token = tokenizer.ACTION_TOKEN_START + round( + (0.0 - tokenizer.ACTION_MINIMUM) + / (tokenizer.ACTION_MAXIMUM - tokenizer.ACTION_MINIMUM) + * (tokenizer.ACTION_BINS - 1) + ) + + # Frequency-major XY order is [DC-x, DC-y, f1-x, f1-y, ...]. + assert payload[0].item() > zero_token + assert payload[1].item() == zero_token + assert payload[2].item() == zero_token + assert payload[3].item() < zero_token + assert torch.all(payload[4:] == zero_token) + + +def test_action_token_ids_are_contiguous_reserved_vocabulary() -> None: + tokenizer = MiniTokenizer() + edge_actions = torch.tensor([[-4.0, 4.0], [0.0, 1.0]]) + tokens = tokenizer.encode_actions(edge_actions) + + assert tokens[0].item() == tokenizer.ACTION_START + assert tokens[-1].item() == tokenizer.ACTION_END + assert tokens[1:-1].min().item() >= tokenizer.ACTION_TOKEN_START + assert tokens[1:-1].max().item() <= tokenizer.ACTION_TOKEN_END + assert tuple(tokenizer.action_token_ids) == tuple( + range(tokenizer.ACTION_TOKEN_START, tokenizer.ACTION_TOKEN_END + 1) + ) + assert tokenizer.vocab_size == tokenizer.ACTION_TOKEN_END + 1 + + +def test_decode_actions_rejects_missing_boundaries_and_non_action_ids() -> None: + tokenizer = MiniTokenizer() + + with pytest.raises(ValueError, match="enclosed"): + tokenizer.decode_actions(torch.tensor([tokenizer.ACTION_TOKEN_START] * 2)) + with pytest.raises(ValueError, match="reserved"): + tokenizer.decode_actions( + torch.tensor( + [ + tokenizer.ACTION_START, + tokenizer.BOS, + tokenizer.ACTION_TOKEN_START, + tokenizer.ACTION_END, + ] + ) + ) diff --git a/tests/test_data.py b/tests/test_data.py new file mode 100644 index 0000000..7b23be7 --- /dev/null +++ b/tests/test_data.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import torch +from PIL import Image + +from pi05mini.data import Pi05OverfitDataset, load_manifest, materialize_subset +from pi05mini.tokenizer import CoordinateTokenizer, MiniTokenizer, SubtaskVocabulary + + +def _make_source_project(root: Path) -> Path: + processed = root / "data" / "processed" + image_dir = processed / "images" + trajectory_dir = processed / "trajectories" + image_dir.mkdir(parents=True) + trajectory_dir.mkdir(parents=True) + + image_path = image_dir / "sample.jpg" + pixels = np.zeros((48, 80, 3), dtype=np.uint8) + pixels[..., 0] = 64 + pixels[..., 1] = np.arange(80, dtype=np.uint8)[None, :] + Image.fromarray(pixels).save(image_path) + + history_x = np.linspace(-7.0, 0.0, 8, dtype=np.float32) + history = np.stack((history_x, 0.02 * history_x, np.zeros(8)), axis=-1) + future_x = np.linspace(1.0, 32.0, 16, dtype=np.float32) + future = np.stack((future_x, 0.002 * future_x**2, np.zeros(16)), axis=-1) + trajectory_path = trajectory_dir / "sample.npz" + np.savez_compressed(trajectory_path, history_xyz=history, future_xyz=future) + + manifest = processed / "overfit_manifest.jsonl" + row = { + "clip_id": "clip-a", + "t0_us": 1234, + "reasoning": "Steer left through the construction lane.", + "image_path": "data/processed/images/sample.jpg", + "trajectory_path": "data/processed/trajectories/sample.npz", + } + manifest.write_text(json.dumps(row) + "\n", encoding="utf-8") + return manifest + + +def test_coordinate_tokens_are_literal_bytes_and_round_trip() -> None: + tokenizer = CoordinateTokenizer(x_range=(-10.0, 10.0), y_range=(-5.0, 5.0)) + coordinates = torch.tensor([[-10.0, -5.0], [0.0, 0.0], [10.0, 5.0]]) + tokens = tokenizer.encode(coordinates) + + assert tokens.dtype == torch.long + assert tokens.tolist()[0] == [0, 0] + assert tokens.tolist()[-1] == [255, 255] + assert 0 <= int(tokens.min()) <= int(tokens.max()) <= 255 + decoded = tokenizer.decode(tokens) + assert torch.allclose(decoded, coordinates, atol=10.1 / 255.0) + + +def test_subtask_vocabulary_and_prefix_contract() -> None: + tokenizer = MiniTokenizer() + history = torch.zeros(8, 2) + prefix = tokenizer.build_prefix(history) + + assert prefix.shape == (20,) + assert prefix[:3].tolist() == [tokenizer.BOS, tokenizer.IMAGE, tokenizer.STATE] + assert prefix[-1].item() == tokenizer.SUBTASK_QUERY + assert prefix[3:-1].min() >= 0 and prefix[3:-1].max() <= 255 + target = tokenizer.encode_subtask("Steer left around the cones") + assert tokenizer.subtasks.decode(target) == "steer_left" + assert SubtaskVocabulary().classify("Slow down for a pedestrian") == "slow_down" + + +def test_materialize_and_dataset_are_self_contained_and_deterministic(tmp_path: Path) -> None: + source_manifest = _make_source_project(tmp_path / "source") + copied_manifest = materialize_subset(source_manifest, tmp_path / "target", max_samples=1) + records = load_manifest(copied_manifest) + + assert len(records) == 1 + assert records[0].image_path.is_file() + assert records[0].trajectory_path.is_file() + + dataset = Pi05OverfitDataset( + copied_manifest, + image_size=(32, 48), + augmentation_factor=3, + seed=19, + ) + same_dataset = Pi05OverfitDataset( + copied_manifest, + image_size=(32, 48), + augmentation_factor=3, + seed=19, + action_statistics=dataset.action_normalizer.as_dict(), + ) + + assert len(dataset) == 3 + original = dataset[0] + augmented = dataset[1] + repeated = same_dataset[1] + assert original["image"].shape == (3, 32, 48) + assert original["prefix_tokens"].shape == (20,) + assert original["actions"].shape == (16, 2) + assert original["subtask_target"].shape == () + assert original["subtask_label"] == "steer_left" + assert original["reasoning"].startswith("Steer left") + assert original["sample_id"].endswith(":aug0") + assert augmented["sample_id"].endswith(":aug1") + assert not torch.equal(original["future_xy"], augmented["future_xy"]) + assert torch.equal(augmented["future_xy"], repeated["future_xy"]) + assert torch.equal(augmented["image"], repeated["image"]) + + reconstructed = dataset.action_normalizer.decode( + original["actions"], original["history_xy"] + ) + assert torch.allclose(reconstructed, original["future_xy"], atol=1e-5) + + statistics_path = dataset.save_action_statistics(tmp_path / "action_stats.json") + statistics = json.loads(statistics_path.read_text(encoding="utf-8")) + assert len(statistics["mean"]) == 2 + assert len(statistics["scale"]) == 2 + assert min(statistics["scale"]) >= 0.05 diff --git a/tests/test_flow.py b/tests/test_flow.py new file mode 100644 index 0000000..a537242 --- /dev/null +++ b/tests/test_flow.py @@ -0,0 +1,85 @@ +import torch + +from pi05mini.flow import FlowMatching + + +def _expand_batch_time(t: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Expand a per-example time tensor over non-batch dimensions.""" + return t.reshape(t.shape[0], *([1] * (target.ndim - 1))) + + +def test_training_batch_obeys_linear_flow_equations() -> None: + flow = FlowMatching() + target = torch.tensor( + [ + [[1.0, -2.0], [0.5, 3.0]], + [[-1.0, 0.0], [2.5, -0.5]], + ] + ) + generator = torch.Generator().manual_seed(123) + + x_t, t, target_velocity = flow.training_batch(target, generator=generator) + + assert x_t.shape == target.shape + assert target_velocity.shape == target.shape + assert t.shape[0] == target.shape[0] + assert torch.all((t >= 0.0) & (t <= 1.0)) + + t_view = _expand_batch_time(t, target) + # v = target - noise, so the sampled noise is exactly recoverable. + noise = target - target_velocity + expected_x_t = (1.0 - t_view) * noise + t_view * target + torch.testing.assert_close(x_t, expected_x_t) + + +def test_training_batch_is_reproducible_with_generator() -> None: + flow = FlowMatching() + target = torch.randn(3, 5, 2) + + batch_a = flow.training_batch( + target, generator=torch.Generator().manual_seed(99) + ) + batch_b = flow.training_batch( + target, generator=torch.Generator().manual_seed(99) + ) + + for tensor_a, tensor_b in zip(batch_a, batch_b): + torch.testing.assert_close(tensor_a, tensor_b) + + +def test_loss_is_mean_squared_velocity_error() -> None: + flow = FlowMatching() + target_velocity = torch.tensor([[[1.0, -1.0], [0.0, 2.0]]]) + prediction = target_velocity + torch.tensor([[[1.0, 1.0], [2.0, 0.0]]]) + + # Squared errors are [1, 1, 4, 0], whose mean is 1.5. + torch.testing.assert_close(flow.loss(prediction, target_velocity), torch.tensor(1.5)) + torch.testing.assert_close( + flow.loss(target_velocity, target_velocity), torch.tensor(0.0) + ) + + +def test_euler_sampler_reaches_fixed_target_for_exact_vector_field() -> None: + flow = FlowMatching() + target = torch.tensor( + [ + [[0.25, -0.5], [1.0, 2.0]], + [[-2.0, 0.75], [0.0, 0.5]], + ] + ) + + def exact_model(x_t: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + t_view = _expand_batch_time(t, x_t) + # Straight conditional path to target. The sampler evaluates t < 1. + return (target.to(x_t) - x_t) / (1.0 - t_view) + + torch.manual_seed(5) + sampled = flow.sample( + exact_model, + shape=tuple(target.shape), + device=target.device, + steps=8, + ) + + torch.testing.assert_close(sampled, target, atol=1e-5, rtol=1e-5) + diff --git a/tests/test_model.py b/tests/test_model.py new file mode 100644 index 0000000..511734f --- /dev/null +++ b/tests/test_model.py @@ -0,0 +1,61 @@ +import torch + +from pi05mini.model import Pi05Config, Pi05Mini + + +def tiny_model() -> Pi05Mini: + return Pi05Mini(Pi05Config( + vocab_size=300, + image_size=16, + patch_size=16, + d_model=32, + n_heads=4, + n_layers=1, + max_text_tokens=16, + max_action_tokens=4, + )) + + +def test_pi05_has_no_continuous_state_projection() -> None: + model = tiny_model() + assert not hasattr(model, "state_projection") + output = model( + torch.randn(2, 3, 16, 16), + torch.randint(0, 300, (2, 7)), + torch.randn(2, 4, 2), + torch.rand(2), + generated_token_start=4, + ) + assert output.text_logits.shape == (2, 7, 300) + assert output.vector_field.shape == (2, 4, 2) + + +def test_observation_generated_and_flow_attention_partition() -> None: + model = tiny_model() + # One image patch followed by four observation tokens and three generated + # tokens. Flow may read observation plus the first generated (subtask) + # token, but not the two teacher-forced FAST tokens. + action_prefix_mask = torch.tensor([[1, 1, 1, 1, 1, 1, 0, 0]], dtype=torch.bool) + prefix, action = model._masks( + batch=1, + prefix_length=8, + action_length=2, + text_offset=1, + attention_mask=torch.ones(1, 7, dtype=torch.bool), + action_prefix_mask=action_prefix_mask, + generated_token_start=4, + device=torch.device("cpu"), + ) + + generated_start = 5 # image patch + four observation text tokens + # Observation block is fully bidirectional and cannot see generated tokens. + assert prefix[0, 0, :generated_start].all() + assert not prefix[0, 0, generated_start:].any() + assert prefix[0, 4, :generated_start].all() + # Generated tokens see all observation tokens and only prior/self generated tokens. + assert prefix[0, 5, :6].all() and not prefix[0, 5, 6:].any() + assert prefix[0, 6, :7].all() and not prefix[0, 6, 7] + # Flow reads allowed prefix keys and all flow-action keys bidirectionally. + assert action[0, :, :6].all() + assert not action[0, :, 6:8].any() + assert action[0, :, 8:].all()