seeker
723 lines
30 KiB
Python
723 lines
30 KiB
Python
#!/usr/bin/env python3
|
|
"""Build one artifact set comparing ground truth, VLM, and diffusion outputs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import textwrap
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import matplotlib
|
|
import numpy as np
|
|
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt # noqa: E402
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
METRIC_NAMES = ("action_mse", "ade_m", "fde_m", "max_error_m")
|
|
|
|
|
|
def _resolve(path: Path) -> Path:
|
|
return path.expanduser().resolve() if path.is_absolute() else (PROJECT_ROOT / path).resolve()
|
|
|
|
|
|
def _require_file(path: Path) -> Path:
|
|
resolved = _resolve(path)
|
|
if not resolved.is_file():
|
|
raise FileNotFoundError(f"Required input does not exist: {resolved}")
|
|
return resolved
|
|
|
|
|
|
def _read_json(path: Path) -> dict[str, Any]:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(value, dict):
|
|
raise ValueError(f"Expected a JSON object in {path}")
|
|
return value
|
|
|
|
|
|
def _read_manifest_row(path: Path, index: int) -> dict[str, Any]:
|
|
rows = [line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
if not 0 <= index < len(rows):
|
|
raise IndexError(f"Manifest index {index} is outside 0..{len(rows) - 1}: {path}")
|
|
value = json.loads(rows[index])
|
|
if not isinstance(value, dict):
|
|
raise ValueError(f"Expected a JSON object at manifest row {index}: {path}")
|
|
return value
|
|
|
|
|
|
def _load_npz(path: Path) -> dict[str, np.ndarray]:
|
|
with np.load(path, allow_pickle=False) as archive:
|
|
return {key: np.asarray(archive[key]) for key in archive.files}
|
|
|
|
|
|
def _require_array(
|
|
arrays: dict[str, np.ndarray], key: str, source: Path, *, allow_batch: bool = True
|
|
) -> np.ndarray:
|
|
if key not in arrays:
|
|
raise KeyError(f"Missing NPZ key {key!r} in {source}")
|
|
value = arrays[key]
|
|
if allow_batch and value.ndim == 3 and value.shape[0] == 1:
|
|
value = value[0]
|
|
if value.ndim != 2 or value.shape[1] != 2:
|
|
raise ValueError(f"{source}:{key} must have shape (N, 2) or (1, N, 2), got {value.shape}")
|
|
if not np.issubdtype(value.dtype, np.number) or not np.isfinite(value).all():
|
|
raise ValueError(f"{source}:{key} must contain only finite numeric values")
|
|
return value.astype(np.float64, copy=False)
|
|
|
|
|
|
def _require_xy_prefix(arrays: dict[str, np.ndarray], key: str, source: Path) -> np.ndarray:
|
|
if key not in arrays:
|
|
raise KeyError(f"Missing NPZ key {key!r} in {source}")
|
|
value = arrays[key]
|
|
if value.ndim != 2 or value.shape[1] < 2:
|
|
raise ValueError(f"{source}:{key} must have shape (N, >=2), got {value.shape}")
|
|
if not np.issubdtype(value.dtype, np.number) or not np.isfinite(value).all():
|
|
raise ValueError(f"{source}:{key} must contain only finite numeric values")
|
|
return value[:, :2].astype(np.float64, copy=False)
|
|
|
|
|
|
def _require_token_values(arrays: dict[str, np.ndarray], key: str, source: Path) -> np.ndarray:
|
|
if key not in arrays:
|
|
raise KeyError(f"Missing NPZ key {key!r} in {source}")
|
|
value = np.asarray(arrays[key]).reshape(-1)
|
|
if not np.issubdtype(value.dtype, np.integer):
|
|
raise ValueError(f"{source}:{key} must contain integer token values")
|
|
return value.astype(np.int64, copy=False)
|
|
|
|
|
|
def _assert_same_truth(
|
|
name: str,
|
|
vlm_truth: np.ndarray,
|
|
diffusion_truth: np.ndarray,
|
|
vlm_path: Path,
|
|
diffusion_path: Path,
|
|
) -> float:
|
|
if vlm_truth.shape != diffusion_truth.shape:
|
|
raise ValueError(
|
|
f"Ground-truth {name} shape mismatch: {vlm_path} has {vlm_truth.shape}, "
|
|
f"{diffusion_path} has {diffusion_truth.shape}"
|
|
)
|
|
max_abs_difference = float(np.max(np.abs(vlm_truth - diffusion_truth), initial=0.0))
|
|
if not np.array_equal(vlm_truth, diffusion_truth):
|
|
raise ValueError(
|
|
f"Ground-truth {name} mismatch between {vlm_path} and {diffusion_path}; "
|
|
f"max_abs_difference={max_abs_difference:.9g}"
|
|
)
|
|
return max_abs_difference
|
|
|
|
|
|
def _extract_component(text: str, component: str) -> str:
|
|
start = f"<|{component}_start|>"
|
|
end = f"<|{component}_end|>"
|
|
if text.count(start) != 1 or text.count(end) != 1:
|
|
raise ValueError(f"Structured VLM text must contain exactly one {start} and one {end}")
|
|
start_index = text.find(start)
|
|
content_start = start_index + len(start)
|
|
end_index = text.find(end, content_start)
|
|
if start_index < 0 or end_index < content_start:
|
|
raise ValueError(f"Structured VLM markers are out of order for {component}")
|
|
value = text[content_start:end_index].strip()
|
|
if not value:
|
|
raise ValueError(f"Structured VLM component {component!r} is empty")
|
|
return value
|
|
|
|
|
|
def _prediction_metrics(
|
|
predicted_action: np.ndarray,
|
|
predicted_xy: np.ndarray,
|
|
target_action: np.ndarray,
|
|
target_xy: np.ndarray,
|
|
) -> dict[str, float]:
|
|
position_errors = np.linalg.norm(predicted_xy - target_xy, axis=-1)
|
|
return {
|
|
"action_mse": float(np.mean(np.square(predicted_action - target_action))),
|
|
"ade_m": float(np.mean(position_errors)),
|
|
"fde_m": float(position_errors[-1]),
|
|
"max_error_m": float(np.max(position_errors)),
|
|
}
|
|
|
|
|
|
def _reported_metrics(metrics: dict[str, Any], section: str | None = None) -> dict[str, float]:
|
|
source: Any = metrics if section is None else metrics.get(section)
|
|
if not isinstance(source, dict):
|
|
raise KeyError(f"Missing metrics section {section!r}")
|
|
result: dict[str, float] = {}
|
|
for name in METRIC_NAMES:
|
|
if name not in source:
|
|
raise KeyError(f"Missing reported metric {name!r} in section {section!r}")
|
|
result[name] = float(source[name])
|
|
return result
|
|
|
|
|
|
def _metric_validation(
|
|
computed: dict[str, float], reported: dict[str, float], tolerance: float = 1e-5
|
|
) -> dict[str, Any]:
|
|
absolute_differences = {name: abs(computed[name] - reported[name]) for name in METRIC_NAMES}
|
|
consistent = all(difference <= tolerance for difference in absolute_differences.values())
|
|
if not consistent:
|
|
raise ValueError(
|
|
"Computed metrics do not match the metrics JSON: "
|
|
f"absolute_differences={absolute_differences}, tolerance={tolerance}"
|
|
)
|
|
return {
|
|
"consistent": True,
|
|
"absolute_tolerance": tolerance,
|
|
"absolute_differences": absolute_differences,
|
|
}
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _display_path(path: Path) -> str:
|
|
try:
|
|
return str(path.relative_to(PROJECT_ROOT))
|
|
except ValueError:
|
|
return str(path)
|
|
|
|
|
|
def _source(path: Path, *, keys: list[str] | None = None) -> dict[str, Any]:
|
|
result: dict[str, Any] = {"path": _display_path(path), "sha256": _sha256(path)}
|
|
if keys is not None:
|
|
result["keys"] = keys
|
|
return result
|
|
|
|
|
|
def _decode_action(
|
|
action: np.ndarray,
|
|
history_xy: np.ndarray,
|
|
mean: np.ndarray,
|
|
scale: np.ndarray,
|
|
) -> np.ndarray:
|
|
deltas = action * scale + mean
|
|
return history_xy[-1] + np.cumsum(deltas, axis=0)
|
|
|
|
|
|
def _validate_decoded_xy(
|
|
name: str,
|
|
decoded_xy: np.ndarray,
|
|
stored_xy: np.ndarray,
|
|
tolerance: float = 1e-5,
|
|
) -> float:
|
|
max_abs_difference = float(np.max(np.abs(decoded_xy - stored_xy), initial=0.0))
|
|
if not np.allclose(decoded_xy, stored_xy, rtol=0.0, atol=tolerance):
|
|
raise ValueError(
|
|
f"Decoded {name} action does not reproduce stored XY; "
|
|
f"max_abs_difference={max_abs_difference:.9g}, tolerance={tolerance}"
|
|
)
|
|
return max_abs_difference
|
|
|
|
|
|
def _write_waypoints(path: Path, waypoints: list[dict[str, Any]]) -> None:
|
|
fieldnames = [
|
|
"waypoint_index",
|
|
"source_future_index",
|
|
"time_s",
|
|
"gt_action_x",
|
|
"gt_action_y",
|
|
"vlm_action_x",
|
|
"vlm_action_y",
|
|
"diffusion_action_x",
|
|
"diffusion_action_y",
|
|
"vlm_action_error_l2",
|
|
"diffusion_action_error_l2",
|
|
"gt_x_m",
|
|
"gt_y_m",
|
|
"vlm_x_m",
|
|
"vlm_y_m",
|
|
"diffusion_x_m",
|
|
"diffusion_y_m",
|
|
"vlm_position_error_m",
|
|
"diffusion_position_error_m",
|
|
]
|
|
with path.open("w", encoding="utf-8", newline="") as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=fieldnames, lineterminator="\n")
|
|
writer.writeheader()
|
|
for waypoint in waypoints:
|
|
writer.writerow(
|
|
{
|
|
"waypoint_index": waypoint["index"],
|
|
"source_future_index": waypoint["source_future_index"],
|
|
"time_s": waypoint["time_s"],
|
|
"gt_action_x": waypoint["action"]["ground_truth"][0],
|
|
"gt_action_y": waypoint["action"]["ground_truth"][1],
|
|
"vlm_action_x": waypoint["action"]["vlm"][0],
|
|
"vlm_action_y": waypoint["action"]["vlm"][1],
|
|
"diffusion_action_x": waypoint["action"]["diffusion"][0],
|
|
"diffusion_action_y": waypoint["action"]["diffusion"][1],
|
|
"vlm_action_error_l2": waypoint["action_error_l2"]["vlm"],
|
|
"diffusion_action_error_l2": waypoint["action_error_l2"]["diffusion"],
|
|
"gt_x_m": waypoint["xy_m"]["ground_truth"][0],
|
|
"gt_y_m": waypoint["xy_m"]["ground_truth"][1],
|
|
"vlm_x_m": waypoint["xy_m"]["vlm"][0],
|
|
"vlm_y_m": waypoint["xy_m"]["vlm"][1],
|
|
"diffusion_x_m": waypoint["xy_m"]["diffusion"][0],
|
|
"diffusion_y_m": waypoint["xy_m"]["diffusion"][1],
|
|
"vlm_position_error_m": waypoint["position_error_m"]["vlm"],
|
|
"diffusion_position_error_m": waypoint["position_error_m"]["diffusion"],
|
|
}
|
|
)
|
|
|
|
|
|
def _save_plot(
|
|
path: Path,
|
|
history_xy: np.ndarray,
|
|
target_xy: np.ndarray,
|
|
vlm_xy: np.ndarray,
|
|
diffusion_xy: np.ndarray,
|
|
target_action: np.ndarray,
|
|
vlm_action: np.ndarray,
|
|
diffusion_action: np.ndarray,
|
|
waypoint_times_s: np.ndarray,
|
|
vlm_errors: np.ndarray,
|
|
diffusion_errors: np.ndarray,
|
|
ground_truth_reasoning: str,
|
|
vlm_reasoning: str,
|
|
) -> None:
|
|
figure, axes = plt.subplots(2, 2, figsize=(14, 10))
|
|
trajectory_axis, action_x_axis, action_y_axis, error_axis = axes.reshape(-1)
|
|
trajectory_axis.plot(history_xy[:, 0], history_xy[:, 1], "o-", color="#555555", label="history")
|
|
trajectory_axis.plot(
|
|
target_xy[:, 0], target_xy[:, 1], "o-", color="#d62728", label="ground truth"
|
|
)
|
|
trajectory_axis.plot(
|
|
vlm_xy[:, 0],
|
|
vlm_xy[:, 1],
|
|
"o--",
|
|
color="#1f77b4",
|
|
label=f"VLM discrete (ADE {vlm_errors.mean():.3f} m)",
|
|
)
|
|
trajectory_axis.plot(
|
|
diffusion_xy[:, 0],
|
|
diffusion_xy[:, 1],
|
|
"o-.",
|
|
color="#ff7f0e",
|
|
label=f"diffusion (ADE {diffusion_errors.mean():.3f} m)",
|
|
)
|
|
trajectory_axis.scatter([0.0], [0.0], marker="x", color="black", label="current ego")
|
|
trajectory_axis.set_xlabel("ego-relative x (m)")
|
|
trajectory_axis.set_ylabel("ego-relative y (m)")
|
|
trajectory_axis.set_title("XY trajectory (axes independently scaled)")
|
|
trajectory_axis.grid(alpha=0.25)
|
|
trajectory_axis.legend()
|
|
|
|
action_series = (
|
|
("ground truth", target_action, "#d62728", "o-"),
|
|
("VLM discrete", vlm_action, "#1f77b4", "o--"),
|
|
("diffusion", diffusion_action, "#ff7f0e", "o-."),
|
|
)
|
|
for label, action, color, style in action_series:
|
|
action_x_axis.plot(waypoint_times_s, action[:, 0], style, color=color, label=label)
|
|
action_y_axis.plot(waypoint_times_s, action[:, 1], style, color=color, label=label)
|
|
for axis, component in ((action_x_axis, "x"), (action_y_axis, "y")):
|
|
axis.set_xlabel("future time (s)")
|
|
axis.set_ylabel(f"normalized action {component}")
|
|
axis.set_title(f"Normalized action {component}")
|
|
axis.grid(alpha=0.25)
|
|
action_x_axis.legend()
|
|
|
|
error_axis.plot(waypoint_times_s, vlm_errors, "o--", color="#1f77b4", label="VLM discrete")
|
|
error_axis.plot(waypoint_times_s, diffusion_errors, "o-.", color="#ff7f0e", label="diffusion")
|
|
error_axis.set_xlabel("future time (s)")
|
|
error_axis.set_ylabel("position error (m)")
|
|
error_axis.set_title("Per-waypoint position error")
|
|
error_axis.grid(alpha=0.25)
|
|
error_axis.legend()
|
|
|
|
match = "yes" if vlm_reasoning == ground_truth_reasoning else "no"
|
|
text_block = (
|
|
f"Ground truth text: {ground_truth_reasoning}\n"
|
|
f"VLM text: {vlm_reasoning}\n"
|
|
f"Exact match: {match}"
|
|
)
|
|
figure.suptitle("Ground truth vs VLM logits vs diffusion")
|
|
figure.text(0.02, 0.015, textwrap.fill(text_block, width=150), va="bottom")
|
|
figure.tight_layout(rect=(0.0, 0.10, 1.0, 0.97))
|
|
figure.savefig(path, dpi=180)
|
|
plt.close(figure)
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--vlm-npz",
|
|
type=Path,
|
|
default=Path("outputs/stage1_vlm_trajectory/trajectory_logits.npz"),
|
|
)
|
|
parser.add_argument(
|
|
"--vlm-metrics",
|
|
type=Path,
|
|
default=Path("outputs/stage1_vlm_trajectory/trajectory_logits_metrics.json"),
|
|
)
|
|
parser.add_argument(
|
|
"--diffusion-npz",
|
|
type=Path,
|
|
default=Path("outputs/stage2_expert/reload_prediction.npz"),
|
|
)
|
|
parser.add_argument(
|
|
"--diffusion-metrics",
|
|
type=Path,
|
|
default=Path("outputs/stage2_expert/stage2_metrics.json"),
|
|
)
|
|
parser.add_argument(
|
|
"--expert-config",
|
|
type=Path,
|
|
default=Path("outputs/stage2_expert/expert_config.json"),
|
|
)
|
|
parser.add_argument(
|
|
"--manifest", type=Path, default=Path("data/processed/overfit_manifest.jsonl")
|
|
)
|
|
parser.add_argument("--manifest-index", type=int, default=0)
|
|
parser.add_argument("--diffusion-seed", type=int, default=12)
|
|
parser.add_argument("--time-step-s", type=float, default=0.1)
|
|
parser.add_argument("--output-dir", type=Path, default=Path("outputs/complete_comparison"))
|
|
args = parser.parse_args()
|
|
if args.time_step_s <= 0:
|
|
raise ValueError("--time-step-s must be positive")
|
|
|
|
vlm_npz_path = _require_file(args.vlm_npz)
|
|
vlm_metrics_path = _require_file(args.vlm_metrics)
|
|
diffusion_npz_path = _require_file(args.diffusion_npz)
|
|
diffusion_metrics_path = _require_file(args.diffusion_metrics)
|
|
expert_config_path = _require_file(args.expert_config)
|
|
manifest_path = _require_file(args.manifest)
|
|
output_dir = _resolve(args.output_dir)
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
vlm_arrays = _load_npz(vlm_npz_path)
|
|
diffusion_arrays = _load_npz(diffusion_npz_path)
|
|
vlm_metrics = _read_json(vlm_metrics_path)
|
|
diffusion_metrics = _read_json(diffusion_metrics_path)
|
|
expert_config = _read_json(expert_config_path)
|
|
manifest = _read_manifest_row(manifest_path, args.manifest_index)
|
|
trajectory_value = manifest.get("trajectory_path")
|
|
if not isinstance(trajectory_value, str):
|
|
raise KeyError("Manifest row lacks string field 'trajectory_path'")
|
|
trajectory_path = _require_file(Path(trajectory_value))
|
|
trajectory_arrays = _load_npz(trajectory_path)
|
|
|
|
vlm_target_action = _require_array(vlm_arrays, "target_action", vlm_npz_path)
|
|
vlm_target_xy = _require_array(vlm_arrays, "target_xy", vlm_npz_path)
|
|
vlm_action = _require_array(vlm_arrays, "generated_action", vlm_npz_path)
|
|
vlm_xy = _require_array(vlm_arrays, "generated_xy", vlm_npz_path)
|
|
diffusion_target_action = _require_array(diffusion_arrays, "target_action", diffusion_npz_path)
|
|
diffusion_target_xy = _require_array(diffusion_arrays, "target_xy", diffusion_npz_path)
|
|
diffusion_action = _require_array(diffusion_arrays, "predicted_action", diffusion_npz_path)
|
|
diffusion_xy = _require_array(diffusion_arrays, "predicted_xy", diffusion_npz_path)
|
|
history_xy = _require_array(
|
|
diffusion_arrays, "history_xy", diffusion_npz_path, allow_batch=False
|
|
)
|
|
target_token_values = _require_token_values(vlm_arrays, "quantized_target_values", vlm_npz_path)
|
|
generated_token_values = _require_token_values(vlm_arrays, "generated_values", vlm_npz_path)
|
|
dataset_history_xy = _require_xy_prefix(trajectory_arrays, "history_xyz", trajectory_path)
|
|
dataset_target_xy = _require_xy_prefix(trajectory_arrays, "future_xyz", trajectory_path)
|
|
if "waypoint_indices" not in trajectory_arrays:
|
|
raise KeyError(f"Missing NPZ key 'waypoint_indices' in {trajectory_path}")
|
|
source_future_indices = np.asarray(trajectory_arrays["waypoint_indices"]).reshape(-1)
|
|
if not np.issubdtype(source_future_indices.dtype, np.integer):
|
|
raise ValueError(f"{trajectory_path}:waypoint_indices must contain integers")
|
|
|
|
action_truth_difference = _assert_same_truth(
|
|
"action", vlm_target_action, diffusion_target_action, vlm_npz_path, diffusion_npz_path
|
|
)
|
|
xy_truth_difference = _assert_same_truth(
|
|
"XY", vlm_target_xy, diffusion_target_xy, vlm_npz_path, diffusion_npz_path
|
|
)
|
|
dataset_xy_difference = _assert_same_truth(
|
|
"dataset XY", vlm_target_xy, dataset_target_xy, vlm_npz_path, trajectory_path
|
|
)
|
|
history_xy_difference = _assert_same_truth(
|
|
"history XY", history_xy, dataset_history_xy, diffusion_npz_path, trajectory_path
|
|
)
|
|
target_action = vlm_target_action
|
|
target_xy = vlm_target_xy
|
|
expected_shape = target_action.shape
|
|
for name, value in {
|
|
"target_xy": target_xy,
|
|
"vlm_action": vlm_action,
|
|
"vlm_xy": vlm_xy,
|
|
"diffusion_action": diffusion_action,
|
|
"diffusion_xy": diffusion_xy,
|
|
}.items():
|
|
if value.shape != expected_shape:
|
|
raise ValueError(f"{name} shape {value.shape} does not match target {expected_shape}")
|
|
if source_future_indices.shape != (target_xy.shape[0],):
|
|
raise ValueError(
|
|
"waypoint_indices length does not match the compared trajectory: "
|
|
f"{source_future_indices.shape} vs {target_xy.shape[0]}"
|
|
)
|
|
if target_token_values.shape != generated_token_values.shape:
|
|
raise ValueError(
|
|
"VLM generated and target trajectory token shapes differ: "
|
|
f"{generated_token_values.shape} vs {target_token_values.shape}"
|
|
)
|
|
if target_token_values.size != target_action.size:
|
|
raise ValueError(
|
|
f"Expected {target_action.size} trajectory tokens, got {target_token_values.size}"
|
|
)
|
|
token_matches = generated_token_values == target_token_values
|
|
waypoint_times_s = (source_future_indices.astype(np.float64) + 1.0) * args.time_step_s
|
|
|
|
action_mean = np.asarray(expert_config.get("action_mean"), dtype=np.float64)
|
|
action_scale = np.asarray(expert_config.get("action_scale"), dtype=np.float64)
|
|
if action_mean.shape != (2,) or action_scale.shape != (2,):
|
|
raise ValueError("expert_config action_mean and action_scale must both have shape (2,)")
|
|
if not np.isfinite(action_mean).all() or not np.isfinite(action_scale).all():
|
|
raise ValueError("expert_config action statistics must be finite")
|
|
if np.any(action_scale <= 0):
|
|
raise ValueError("expert_config action_scale must be positive")
|
|
action_decode_differences = {
|
|
"ground_truth": _validate_decoded_xy(
|
|
"ground-truth",
|
|
_decode_action(target_action, history_xy, action_mean, action_scale),
|
|
target_xy,
|
|
),
|
|
"vlm": _validate_decoded_xy(
|
|
"VLM",
|
|
_decode_action(vlm_action, history_xy, action_mean, action_scale),
|
|
vlm_xy,
|
|
),
|
|
"diffusion": _validate_decoded_xy(
|
|
"diffusion",
|
|
_decode_action(diffusion_action, history_xy, action_mean, action_scale),
|
|
diffusion_xy,
|
|
),
|
|
}
|
|
|
|
untruncated = vlm_metrics.get("untruncated_vlm_generation")
|
|
if not isinstance(untruncated, dict) or not isinstance(untruncated.get("raw_text"), str):
|
|
raise KeyError("VLM metrics lack untruncated_vlm_generation.raw_text")
|
|
vlm_structured_text = untruncated["raw_text"]
|
|
target_structured_text = vlm_metrics.get("structured_target")
|
|
if not isinstance(target_structured_text, str):
|
|
raise KeyError("VLM metrics lack structured_target")
|
|
vlm_reasoning = _extract_component(vlm_structured_text, "cot")
|
|
vlm_meta_action = _extract_component(vlm_structured_text, "meta_action")
|
|
vlm_target_reasoning = _extract_component(target_structured_text, "cot")
|
|
vlm_target_meta_action = _extract_component(target_structured_text, "meta_action")
|
|
ground_truth_reasoning = manifest.get("reasoning")
|
|
if not isinstance(ground_truth_reasoning, str):
|
|
raise KeyError("Manifest row lacks string field 'reasoning'")
|
|
diffusion_target_reasoning = diffusion_metrics.get("target_reasoning")
|
|
if vlm_target_reasoning != ground_truth_reasoning:
|
|
raise ValueError("VLM target reasoning does not match the manifest ground truth")
|
|
if diffusion_target_reasoning != ground_truth_reasoning:
|
|
raise ValueError("Diffusion target reasoning does not match the manifest ground truth")
|
|
if vlm_metrics.get("target_meta_action") != vlm_target_meta_action:
|
|
raise ValueError("Structured target meta-action does not match VLM metrics")
|
|
|
|
vlm_position_errors = np.linalg.norm(vlm_xy - target_xy, axis=-1)
|
|
diffusion_position_errors = np.linalg.norm(diffusion_xy - target_xy, axis=-1)
|
|
vlm_action_errors = np.linalg.norm(vlm_action - target_action, axis=-1)
|
|
diffusion_action_errors = np.linalg.norm(diffusion_action - target_action, axis=-1)
|
|
|
|
vlm_computed_metrics = _prediction_metrics(vlm_action, vlm_xy, target_action, target_xy)
|
|
diffusion_computed_metrics = _prediction_metrics(
|
|
diffusion_action, diffusion_xy, target_action, target_xy
|
|
)
|
|
vlm_reported_metrics = _reported_metrics(vlm_metrics, section="untruncated_vlm_generation")
|
|
diffusion_reported_metrics = _reported_metrics(diffusion_metrics)
|
|
vlm_metric_validation = _metric_validation(vlm_computed_metrics, vlm_reported_metrics)
|
|
diffusion_metric_validation = _metric_validation(
|
|
diffusion_computed_metrics, diffusion_reported_metrics
|
|
)
|
|
|
|
waypoints: list[dict[str, Any]] = []
|
|
for index in range(target_xy.shape[0]):
|
|
waypoints.append(
|
|
{
|
|
"index": index + 1,
|
|
"source_future_index": int(source_future_indices[index]),
|
|
"time_s": float(waypoint_times_s[index]),
|
|
"action": {
|
|
"ground_truth": target_action[index].tolist(),
|
|
"vlm": vlm_action[index].tolist(),
|
|
"diffusion": diffusion_action[index].tolist(),
|
|
},
|
|
"action_error_l2": {
|
|
"vlm": float(vlm_action_errors[index]),
|
|
"diffusion": float(diffusion_action_errors[index]),
|
|
},
|
|
"xy_m": {
|
|
"ground_truth": target_xy[index].tolist(),
|
|
"vlm": vlm_xy[index].tolist(),
|
|
"diffusion": diffusion_xy[index].tolist(),
|
|
},
|
|
"position_error_m": {
|
|
"vlm": float(vlm_position_errors[index]),
|
|
"diffusion": float(diffusion_position_errors[index]),
|
|
},
|
|
}
|
|
)
|
|
|
|
comparison = {
|
|
"schema_version": 1,
|
|
"sample": {
|
|
"manifest_index": args.manifest_index,
|
|
"clip_id": manifest.get("clip_id"),
|
|
"t0_us": manifest.get("t0_us"),
|
|
"camera": manifest.get("camera"),
|
|
"history_steps": manifest.get("history_steps"),
|
|
"expert_waypoints": manifest.get("expert_waypoints"),
|
|
"source_time_step_s": args.time_step_s,
|
|
"image_path": _display_path(_resolve(Path(str(manifest.get("image_path"))))),
|
|
"trajectory_path": _display_path(trajectory_path),
|
|
},
|
|
"prediction_protocol": {
|
|
"vlm": {
|
|
"decoding": "greedy untruncated structured generation",
|
|
"trajectory_representation": "32 discrete normalized-XY-delta tokens",
|
|
"trajectory_vocabulary_bins": int(vlm_metrics["vocabulary"]["size"]),
|
|
},
|
|
"diffusion": {
|
|
"decoding": "flow-matching sample",
|
|
"seed": args.diffusion_seed,
|
|
"inference_steps": expert_config.get("inference_steps"),
|
|
},
|
|
"units": {
|
|
"action_mse": "normalized_action_squared",
|
|
"trajectory": "metres",
|
|
},
|
|
},
|
|
"text": {
|
|
"ground_truth_reasoning": ground_truth_reasoning,
|
|
"vlm_reasoning": vlm_reasoning,
|
|
"vlm_reasoning_exact_match": vlm_reasoning == ground_truth_reasoning,
|
|
"vlm_meta_action": vlm_meta_action,
|
|
"target_meta_action": vlm_target_meta_action,
|
|
"vlm_meta_action_exact_match": vlm_meta_action == vlm_target_meta_action,
|
|
"vlm_structured_output": vlm_structured_text,
|
|
"target_structured_output": target_structured_text,
|
|
"vlm_structured_exact_match": vlm_structured_text == target_structured_text,
|
|
"diffusion_conditioning_reasoning": diffusion_metrics.get("generated_reasoning"),
|
|
"diffusion_reasoning_exact_match": diffusion_metrics.get("generated_reasoning")
|
|
== ground_truth_reasoning,
|
|
},
|
|
"arrays": {
|
|
"ground_truth": {
|
|
"action": target_action.tolist(),
|
|
"xy_m": target_xy.tolist(),
|
|
"discrete_token_values": target_token_values.tolist(),
|
|
},
|
|
"vlm": {
|
|
"action": vlm_action.tolist(),
|
|
"xy_m": vlm_xy.tolist(),
|
|
"discrete_token_values": generated_token_values.tolist(),
|
|
},
|
|
"diffusion": {
|
|
"action": diffusion_action.tolist(),
|
|
"xy_m": diffusion_xy.tolist(),
|
|
"seed": args.diffusion_seed,
|
|
},
|
|
"history_xy_m": history_xy.tolist(),
|
|
"source_future_indices": source_future_indices.tolist(),
|
|
"waypoint_times_s": waypoint_times_s.tolist(),
|
|
},
|
|
"waypoints": waypoints,
|
|
"metrics": {
|
|
"vlm": {
|
|
"computed_from_npz": vlm_computed_metrics,
|
|
"reported_in_metrics_json": vlm_reported_metrics,
|
|
"trajectory_token_matches": int(token_matches.sum()),
|
|
"trajectory_token_count": int(token_matches.size),
|
|
"trajectory_token_accuracy": float(token_matches.mean()),
|
|
},
|
|
"diffusion": {
|
|
"computed_from_npz": diffusion_computed_metrics,
|
|
"reported_in_metrics_json": diffusion_reported_metrics,
|
|
"seed": args.diffusion_seed,
|
|
},
|
|
"diffusion_minus_vlm": {
|
|
name: diffusion_computed_metrics[name] - vlm_computed_metrics[name]
|
|
for name in METRIC_NAMES
|
|
},
|
|
},
|
|
"validation": {
|
|
"ground_truth_action_exact_match": True,
|
|
"ground_truth_action_max_abs_difference": action_truth_difference,
|
|
"ground_truth_xy_exact_match": True,
|
|
"ground_truth_xy_max_abs_difference": xy_truth_difference,
|
|
"dataset_target_xy_exact_match": True,
|
|
"dataset_target_xy_max_abs_difference": dataset_xy_difference,
|
|
"dataset_history_xy_exact_match": True,
|
|
"dataset_history_xy_max_abs_difference": history_xy_difference,
|
|
"ground_truth_text_matches_manifest": True,
|
|
"vlm_discrete_tokens_exact_match": bool(token_matches.all()),
|
|
"action_decode_max_abs_differences": action_decode_differences,
|
|
"vlm_reported_metrics_match_npz": vlm_metric_validation,
|
|
"diffusion_reported_metrics_match_npz": diffusion_metric_validation,
|
|
},
|
|
"data_sources": {
|
|
"manifest": _source(manifest_path),
|
|
"vlm_npz": _source(vlm_npz_path, keys=sorted(vlm_arrays)),
|
|
"vlm_metrics": _source(vlm_metrics_path),
|
|
"diffusion_npz": _source(diffusion_npz_path, keys=sorted(diffusion_arrays)),
|
|
"diffusion_metrics": _source(diffusion_metrics_path),
|
|
"expert_config": _source(expert_config_path),
|
|
"trajectory_npz": _source(trajectory_path, keys=sorted(trajectory_arrays)),
|
|
},
|
|
"artifacts": {
|
|
"comparison_json": _display_path(output_dir / "comparison.json"),
|
|
"waypoints_csv": _display_path(output_dir / "waypoints.csv"),
|
|
"trajectory_plot": _display_path(output_dir / "trajectory_comparison.png"),
|
|
},
|
|
}
|
|
|
|
comparison_path = output_dir / "comparison.json"
|
|
waypoints_path = output_dir / "waypoints.csv"
|
|
plot_path = output_dir / "trajectory_comparison.png"
|
|
comparison_path.write_text(
|
|
json.dumps(comparison, indent=2, ensure_ascii=False, allow_nan=False) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
_write_waypoints(waypoints_path, waypoints)
|
|
_save_plot(
|
|
plot_path,
|
|
history_xy,
|
|
target_xy,
|
|
vlm_xy,
|
|
diffusion_xy,
|
|
target_action,
|
|
vlm_action,
|
|
diffusion_action,
|
|
waypoint_times_s,
|
|
vlm_position_errors,
|
|
diffusion_position_errors,
|
|
ground_truth_reasoning,
|
|
vlm_reasoning,
|
|
)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"comparison_json": str(comparison_path),
|
|
"waypoints_csv": str(waypoints_path),
|
|
"trajectory_plot": str(plot_path),
|
|
"vlm_metrics": vlm_computed_metrics,
|
|
"diffusion_metrics": diffusion_computed_metrics,
|
|
"diffusion_seed": args.diffusion_seed,
|
|
"vlm_trajectory_tokens": {
|
|
"matches": int(token_matches.sum()),
|
|
"count": int(token_matches.size),
|
|
},
|
|
"vlm_reasoning_exact_match": vlm_reasoning == ground_truth_reasoning,
|
|
"ground_truth_consistent": True,
|
|
},
|
|
indent=2,
|
|
ensure_ascii=False,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|