pi05-minimal-validation/src/pi05mini/infer.py

from __future__ import annotations

import argparse
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
import torch

from .data import Pi05OverfitDataset
from .flow import FlowMatching
from .model import Pi05Config, Pi05Mini
from .runtime import choose_device, write_json


@torch.no_grad()
def evaluate(checkpoint_path: str, manifest: str | None = None) -> Path:
    checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
    cfg = checkpoint["config"]
    root = Path(__file__).resolve().parents[2]
    device = choose_device(cfg.get("device", "auto"))
    data_manifest = Path(manifest) if manifest else root / "data/overfit/overfit_manifest.jsonl"
    dataset = Pi05OverfitDataset(
        data_manifest, image_size=cfg["data"]["image_size"],
        action_horizon=cfg["data"]["action_horizon"],
        action_statistics=checkpoint["action_statistics"]
    )
    model = Pi05Mini(Pi05Config(**checkpoint["model_config"]))
    model.load_state_dict(checkpoint["model"])
    model.to(device).eval()
    flow = FlowMatching(cfg["inference"]["flow_steps"])
    rows = []
    figure, axes = plt.subplots(1, len(dataset), figsize=(4 * len(dataset), 4), squeeze=False)
    for index, sample in enumerate(dataset):
        image = sample["image"][None].to(device)
        tokens = sample["prefix_tokens"][None].to(device)
        # Autoregressively predict semantic subtask followed by the complete
        # discretized action stream (ACTION_START, 32 scalar tokens, ACTION_END).
        zero_action = torch.zeros(1, dataset.action_horizon, 2, device=device)
        generated: list[int] = []
        expected_tokens = 1 + 2 + dataset.action_horizon * 2
        current_tokens = tokens
        for generation_step in range(expected_tokens):
            semantic = model(
                image, current_tokens, zero_action,
                torch.zeros(1, device=device),
                generated_token_start=tokens.shape[1]
            )
            if generation_step == 0:
                candidates = torch.tensor(
                    dataset.tokenizer.subtasks.token_ids, device=device
                )
                candidate_logits = semantic.text_logits[0, -1, candidates]
                next_token = int(candidates[candidate_logits.argmax()].cpu())
            else:
                next_token = int(semantic.text_logits[0, -1].argmax().cpu())
            generated.append(next_token)
            current_tokens = torch.cat(
                (current_tokens, torch.tensor([[next_token]], device=device)), dim=1
            )
        predicted_token = generated[0]
        vlm_action_tokens = torch.tensor(generated[1:], dtype=torch.long)
        try:
            vlm_normalized = dataset.tokenizer.decode_actions(vlm_action_tokens)
            vlm_xy = dataset.action_normalizer.decode(
                vlm_normalized, sample["history_xy"]
            ).numpy()
            vlm_valid = True
        except ValueError:
            vlm_xy = np.full_like(sample["future_xy"].numpy(), np.nan)
            vlm_valid = False
        conditioned_tokens = dataset.tokenizer.build_low_level_prefix(
            sample["history_xy"], predicted_token
        )[None].to(device)
        def velocity(x: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
            return model(
                image, conditioned_tokens, x, t
            ).vector_field
        normalized = flow.sample(
            velocity, (1, dataset.action_horizon, 2), device,
            steps=cfg["inference"]["flow_steps"],
            generator=None,
        )[0].cpu()
        predicted_xy = dataset.action_normalizer.decode(
            normalized, sample["history_xy"]
        ).numpy()
        target_xy = sample["future_xy"].numpy()
        diffusion_mse = float(np.mean((predicted_xy - target_xy) ** 2))
        vlm_mse = float(np.mean((vlm_xy - target_xy) ** 2)) if vlm_valid else None
        target_label = dataset.tokenizer.subtasks.decode(sample["subtask_target"])
        try:
            predicted_label = dataset.tokenizer.subtasks.decode(predicted_token)
        except ValueError:
            predicted_label = f"token_{predicted_token}"
        rows.append({
            "sample_id": sample["sample_id"], "target_subtask": target_label,
            "task_instruction": sample["task_instruction"],
            "predicted_subtask": predicted_label,
            "vlm_action_tokens_valid": vlm_valid,
            "vlm_trajectory_mse": vlm_mse,
            "diffusion_trajectory_mse": diffusion_mse,
        })
        ax = axes[0, index]
        ax.plot(target_xy[:, 0], target_xy[:, 1], "o-", label="target")
        ax.plot(vlm_xy[:, 0], vlm_xy[:, 1], "s--", label="VLM tokens")
        ax.plot(predicted_xy[:, 0], predicted_xy[:, 1], "x--", label="diffusion")
        ax.scatter([0], [0], c="black", marker="s", label="ego")
        vlm_text = "invalid" if vlm_mse is None else f"{vlm_mse:.3f}"
        ax.set_title(
            f"{predicted_label}\nVLM={vlm_text}, diffusion={diffusion_mse:.3f}"
        )
        ax.set_xlabel("forward x (m)")
        ax.set_ylabel("left y (m)")
        ax.axis("equal")
        ax.grid(True, alpha=.3)
        ax.legend(fontsize=8)
    figure.tight_layout()
    output_dir = Path(checkpoint_path).parent
    plot_path = output_dir / "trajectory_comparison.png"
    figure.savefig(plot_path, dpi=160)
    plt.close(figure)
    write_json(output_dir / "evaluation.json", rows)
    print(rows)
    return plot_path


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--checkpoint", default="outputs/overfit/checkpoint.pt")
    parser.add_argument("--manifest")
    args = parser.parse_args()
    evaluate(args.checkpoint, args.manifest)


if __name__ == "__main__":
    main()