vee
676 lines
27 KiB
Python
676 lines
27 KiB
Python
"""Run the repository-owned front-end seat suite against a connected T2G bench."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_SUITE = ROOT / "tests" / "fixtures" / "seat_frontend_hil_suite.json"
|
|
DEFAULT_OUTPUT = ROOT / "artifacts" / "hil" / "seat_frontend_hil_latest.json"
|
|
MOTION_AXES = {
|
|
"longitudinal",
|
|
"cushion_tilt",
|
|
"backrest_recline",
|
|
"legrest_height",
|
|
}
|
|
EXPECTED_CAPABILITIES = {
|
|
"heating": "seat.set_heating",
|
|
"ventilation": "seat.set_ventilation",
|
|
"longitudinal": "seat.set_longitudinal_position",
|
|
"cushion_tilt": "seat.set_cushion_tilt",
|
|
"backrest_recline": "seat.set_backrest_recline",
|
|
"legrest_height": "seat.set_legrest_height",
|
|
}
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def request_json(
|
|
url: str,
|
|
*,
|
|
payload: dict[str, Any] | None = None,
|
|
timeout_s: int = 180,
|
|
) -> tuple[int, dict[str, Any]]:
|
|
data = None
|
|
method = "GET"
|
|
headers: dict[str, str] = {}
|
|
if payload is not None:
|
|
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
method = "POST"
|
|
headers["Content-Type"] = "application/json; charset=utf-8"
|
|
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout_s) as response:
|
|
return response.status, json.loads(response.read().decode("utf-8"))
|
|
except urllib.error.HTTPError as exc:
|
|
body = exc.read().decode("utf-8", errors="replace")
|
|
try:
|
|
detail = json.loads(body)
|
|
except json.JSONDecodeError:
|
|
detail = {"message": body}
|
|
return exc.code, detail
|
|
|
|
|
|
def dig(value: dict[str, Any], *path: str) -> Any:
|
|
current: Any = value
|
|
for item in path:
|
|
if not isinstance(current, dict) or item not in current:
|
|
return None
|
|
current = current[item]
|
|
return current
|
|
|
|
|
|
def failed_hardware_result(result: dict[str, Any]) -> dict[str, Any] | None:
|
|
items = dig(result, "execution_report", "spec", "results") or []
|
|
for item in reversed(items):
|
|
if item.get("status") != "succeeded":
|
|
return item
|
|
return None
|
|
|
|
|
|
def expand_case_entries(
|
|
entries: list[dict[str, Any]], *, group: str | None = None
|
|
) -> list[dict[str, Any]]:
|
|
"""Expand compact input-equivalence families into individually reported cases."""
|
|
|
|
expanded: list[dict[str, Any]] = []
|
|
for entry in entries:
|
|
base = {key: value for key, value in entry.items() if key != "variants"}
|
|
if group is not None:
|
|
base["group"] = group
|
|
variants = entry.get("variants")
|
|
if not variants:
|
|
expanded.append(base)
|
|
continue
|
|
for variant in variants:
|
|
case = {**base, **variant}
|
|
if "id" not in case or "utterance" not in case:
|
|
raise ValueError("each case variant requires id and utterance")
|
|
expanded.append(case)
|
|
return expanded
|
|
|
|
|
|
def expected_operation_errors(
|
|
result: dict[str, Any],
|
|
expected: dict[str, int],
|
|
expected_motion: dict[str, Any] | None = None,
|
|
expected_dynamic_mode: dict[str, Any] | None = None,
|
|
) -> list[str]:
|
|
errors: list[str] = []
|
|
seat_config = (result.get("function_configs") or {}).get("seat") or result.get(
|
|
"function_config"
|
|
) or {}
|
|
operations = dig(seat_config, "spec", "operations") or []
|
|
by_capability = {item.get("capability"): item for item in operations}
|
|
for field, expected_value in expected.items():
|
|
capability = EXPECTED_CAPABILITIES.get(field)
|
|
if capability is None:
|
|
continue
|
|
operation = by_capability.get(capability)
|
|
if operation is None:
|
|
errors.append(f"expected capability was not generated: {capability}")
|
|
continue
|
|
if dig(operation, "target", "instance") != "passenger" or dig(
|
|
operation, "target", "zone"
|
|
) != "front_right":
|
|
errors.append(f"{capability} did not target passenger/front_right")
|
|
parameter = "level" if field in {"heating", "ventilation"} else "position_pct"
|
|
actual_value = dig(operation, "parameters", parameter)
|
|
if field in {"heating", "ventilation"} and expected_value == 0:
|
|
if dig(operation, "parameters", "state") != "off":
|
|
errors.append(f"{capability} state is not off")
|
|
elif actual_value != expected_value:
|
|
errors.append(
|
|
f"{capability} {parameter}={actual_value}, expected={expected_value}"
|
|
)
|
|
if expected_motion:
|
|
axis = str(expected_motion["axis"])
|
|
capability = EXPECTED_CAPABILITIES[axis]
|
|
operation = by_capability.get(capability)
|
|
motion = operation.get("motion") if operation else None
|
|
if not isinstance(motion, dict):
|
|
errors.append(f"{capability} did not preserve motion metadata")
|
|
else:
|
|
for field, expected_value in expected_motion.items():
|
|
if field == "axis":
|
|
continue
|
|
actual_value = motion.get(field)
|
|
if actual_value != expected_value:
|
|
errors.append(
|
|
f"{capability} motion.{field}={actual_value}, "
|
|
f"expected={expected_value}"
|
|
)
|
|
if expected_dynamic_mode:
|
|
expected_mode_id = int(expected_dynamic_mode["mode_id"])
|
|
capability = (
|
|
"seat.stop_dynamic_mode"
|
|
if expected_mode_id == 0
|
|
else "seat.start_dynamic_mode"
|
|
)
|
|
operation = by_capability.get(capability)
|
|
if operation is None:
|
|
errors.append(f"expected capability was not generated: {capability}")
|
|
elif expected_mode_id and dig(operation, "parameters", "mode_id") != expected_mode_id:
|
|
errors.append(
|
|
f"{capability} mode_id={dig(operation, 'parameters', 'mode_id')}, "
|
|
f"expected={expected_mode_id}"
|
|
)
|
|
|
|
hardware_results = dig(result, "execution_report", "spec", "results") or []
|
|
telemetry = next(
|
|
(
|
|
item.get("telemetry", {})
|
|
for item in hardware_results
|
|
if item.get("domain") == "seat"
|
|
and item.get("telemetry", {}).get("dynamic_mode_id") is not None
|
|
),
|
|
None,
|
|
)
|
|
if telemetry is None:
|
|
errors.append("dynamic-mode execution did not include physical 0x306 telemetry")
|
|
else:
|
|
if telemetry.get("dynamic_mode_id") != expected_mode_id:
|
|
errors.append(
|
|
f"dynamic telemetry mode_id={telemetry.get('dynamic_mode_id')}, "
|
|
f"expected={expected_mode_id}"
|
|
)
|
|
dynamic_state = telemetry.get("dynamic_state")
|
|
accepted_states = {0} if expected_mode_id == 0 else {1, 2}
|
|
if dynamic_state not in accepted_states:
|
|
errors.append(
|
|
f"dynamic telemetry state={dynamic_state}, "
|
|
f"expected one of {sorted(accepted_states)}"
|
|
)
|
|
return errors
|
|
|
|
|
|
def physical_fingerprint(state_result: dict[str, Any]) -> dict[str, Any]:
|
|
physical = state_result.get("physical_state", {})
|
|
return {
|
|
"source": physical.get("source"),
|
|
"complete": physical.get("complete"),
|
|
"heating": dig(physical, "heating", "level"),
|
|
"ventilation": dig(physical, "ventilation", "level"),
|
|
"position": {
|
|
axis: dig(physical, "position", axis, "position_pct")
|
|
for axis in sorted(MOTION_AXES)
|
|
},
|
|
}
|
|
|
|
|
|
def physical_state_request(
|
|
base_url: str, vehicle_id: str, conversation_id: str
|
|
) -> tuple[int, dict[str, Any]]:
|
|
query = urllib.parse.urlencode(
|
|
{"conversation_id": conversation_id, "vehicle_id": vehicle_id}
|
|
)
|
|
return request_json(
|
|
f"{base_url}/api/hardware/seat-state?{query}", timeout_s=20
|
|
)
|
|
|
|
|
|
def validate_physical_state(
|
|
state_result: dict[str, Any], expected: dict[str, int], tolerance: int
|
|
) -> list[str]:
|
|
errors: list[str] = []
|
|
physical = state_result.get("physical_state", {})
|
|
synchronized = state_result.get("actuator_state", {}).get("passenger", {})
|
|
if physical.get("source") != "physical_can":
|
|
errors.append("physical state source is not physical_can")
|
|
if not physical.get("complete"):
|
|
errors.append("physical state is incomplete: " + ",".join(physical.get("missing", [])))
|
|
for feature in ("heating", "ventilation"):
|
|
if feature not in expected:
|
|
continue
|
|
actual = dig(physical, feature, "level")
|
|
twin = dig(synchronized, feature, "level")
|
|
if actual != expected[feature]:
|
|
errors.append(f"{feature} physical={actual}, expected={expected[feature]}")
|
|
if twin != actual:
|
|
errors.append(f"{feature} twin={twin}, physical={actual}")
|
|
if dig(synchronized, feature, "feedback_source") != "physical_can":
|
|
errors.append(f"{feature} twin feedback source is not physical_can")
|
|
for axis in MOTION_AXES:
|
|
if axis not in expected:
|
|
continue
|
|
actual = dig(physical, "position", axis, "position_pct")
|
|
twin = dig(synchronized, "position", axis, "position_pct")
|
|
if not isinstance(actual, (int, float)) or abs(actual - expected[axis]) > tolerance:
|
|
errors.append(f"{axis} physical={actual}, expected={expected[axis]}±{tolerance}")
|
|
if twin != actual:
|
|
errors.append(f"{axis} twin={twin}, physical={actual}")
|
|
if dig(synchronized, "position", axis, "feedback_source") != "physical_can":
|
|
errors.append(f"{axis} twin feedback source is not physical_can")
|
|
return errors
|
|
|
|
|
|
def safety_block_for_case(
|
|
case: dict[str, Any], safety_blocks: dict[str, Any]
|
|
) -> dict[str, Any] | None:
|
|
"""Refuse only motions covered by a documented, evidence-backed bench hazard."""
|
|
|
|
expected = case.get("expected", {})
|
|
for axis, policy in safety_blocks.items():
|
|
target = expected.get(axis)
|
|
if target is None:
|
|
continue
|
|
if policy.get("block_all_targets") is True:
|
|
return {
|
|
"axis": axis,
|
|
"target_pct": target,
|
|
"block_all_targets": True,
|
|
"reason": policy.get("reason", "documented hardware safety block"),
|
|
"evidence": policy.get("evidence", {}),
|
|
}
|
|
minimum = policy.get("minimum_safe_target_pct")
|
|
if isinstance(minimum, (int, float)) and target < minimum:
|
|
return {
|
|
"axis": axis,
|
|
"target_pct": target,
|
|
"minimum_safe_target_pct": minimum,
|
|
"reason": policy.get("reason", "documented hardware safety boundary"),
|
|
"evidence": policy.get("evidence", {}),
|
|
}
|
|
return None
|
|
|
|
|
|
def run_case(
|
|
base_url: str,
|
|
vehicle_id: str,
|
|
conversation_id: str,
|
|
case: dict[str, Any],
|
|
tolerance: int,
|
|
) -> dict[str, Any]:
|
|
started = time.monotonic()
|
|
record: dict[str, Any] = {
|
|
"id": case["id"],
|
|
"group": case.get("group", "mode"),
|
|
"utterance": case["utterance"],
|
|
"started_at": utc_now(),
|
|
"status": "failed",
|
|
"errors": [],
|
|
}
|
|
payload = {
|
|
"utterance": case["utterance"],
|
|
"provider": case.get("provider", "local"),
|
|
"conversation_id": conversation_id,
|
|
"execution_mode": "hardware",
|
|
"vehicle_id": vehicle_id,
|
|
}
|
|
try:
|
|
status_code, result = request_json(
|
|
f"{base_url}/api/requests", payload=payload, timeout_s=900
|
|
)
|
|
record["http_status"] = status_code
|
|
record["request_id"] = result.get("request_id")
|
|
record["provider_used"] = result.get("provider_used")
|
|
if status_code != 201:
|
|
record["errors"].append(result.get("message", f"HTTP {status_code}"))
|
|
elif result.get("status") != "succeeded":
|
|
failed = failed_hardware_result(result)
|
|
record["errors"].append(
|
|
failed.get("message", "request failed") if failed else "request failed"
|
|
)
|
|
if failed:
|
|
record["failure_telemetry"] = failed.get("telemetry", {})
|
|
elif dig(result, "execution_report", "spec", "status") != "succeeded":
|
|
failed = failed_hardware_result(result)
|
|
record["errors"].append(
|
|
failed.get("message", "hardware execution failed")
|
|
if failed
|
|
else "hardware execution failed"
|
|
)
|
|
if failed:
|
|
record["failure_telemetry"] = failed.get("telemetry", {})
|
|
else:
|
|
expected_provider_prefix = case.get("expected_provider_prefix")
|
|
if expected_provider_prefix and not str(result.get("provider_used", "")).startswith(
|
|
str(expected_provider_prefix)
|
|
):
|
|
record["errors"].append(
|
|
f"provider_used={result.get('provider_used')}, "
|
|
f"expected prefix={expected_provider_prefix}"
|
|
)
|
|
record["errors"].extend(
|
|
expected_operation_errors(
|
|
result,
|
|
case.get("expected", {}),
|
|
case.get("expected_motion"),
|
|
case.get("expected_dynamic_mode"),
|
|
)
|
|
)
|
|
seat_config = (result.get("function_configs") or {}).get(
|
|
"seat"
|
|
) or result.get("function_config") or {}
|
|
operations = dig(seat_config, "spec", "operations") or []
|
|
seat_operations = [
|
|
item for item in operations if str(item.get("capability", "")).startswith("seat.")
|
|
]
|
|
invalid_targets = [
|
|
item.get("target")
|
|
for item in seat_operations
|
|
if dig(item, "target", "instance") != "passenger"
|
|
or dig(item, "target", "zone") != "front_right"
|
|
]
|
|
if invalid_targets:
|
|
record["errors"].append(f"non-passenger targets generated: {invalid_targets}")
|
|
visible_notices = [
|
|
*[str(item) for item in result.get("decision_summary", [])],
|
|
*[
|
|
str(item.get("message", ""))
|
|
for item in dig(result, "execution_report", "spec", "results") or []
|
|
],
|
|
*[
|
|
str(item.get("telemetry", {}).get("user_notice", ""))
|
|
for item in dig(result, "execution_report", "spec", "results") or []
|
|
],
|
|
]
|
|
visible_notice_text = "\n".join(visible_notices)
|
|
for expected_notice in case.get("expected_notice_contains", []):
|
|
if expected_notice not in visible_notice_text:
|
|
record["errors"].append(
|
|
f"visible execution notice does not contain {expected_notice!r}"
|
|
)
|
|
expected_mode = case.get("mode_id")
|
|
matched_modes = [item.get("id") for item in result.get("matched_modes", [])]
|
|
record["matched_modes"] = matched_modes
|
|
if expected_mode and expected_mode not in matched_modes:
|
|
record["errors"].append(
|
|
f"mode {expected_mode} was not matched: {matched_modes}"
|
|
)
|
|
state_code, state = physical_state_request(
|
|
base_url, vehicle_id, conversation_id
|
|
)
|
|
record["state_http_status"] = state_code
|
|
if state_code != 200:
|
|
record["errors"].append(
|
|
state.get("message", f"physical state HTTP {state_code}")
|
|
)
|
|
else:
|
|
record["errors"].extend(
|
|
validate_physical_state(state, case.get("expected", {}), tolerance)
|
|
)
|
|
expected_dynamic_mode = case.get("expected_dynamic_mode")
|
|
if expected_dynamic_mode:
|
|
actual_dynamic_mode = dig(
|
|
state, "actuator_state", "passenger", "dynamic_mode"
|
|
) or {}
|
|
for field in ("mode_id", "state"):
|
|
expected_value = expected_dynamic_mode[field]
|
|
if actual_dynamic_mode.get(field) != expected_value:
|
|
record["errors"].append(
|
|
f"dynamic_mode.{field}={actual_dynamic_mode.get(field)}, "
|
|
f"expected={expected_value}"
|
|
)
|
|
record["physical_state"] = state.get("physical_state")
|
|
record["status"] = "passed" if not record["errors"] else "failed"
|
|
except Exception as exc: # continue the suite after transport and parse errors
|
|
record["errors"].append(f"{type(exc).__name__}: {exc}")
|
|
record["finished_at"] = utc_now()
|
|
record["duration_ms"] = round((time.monotonic() - started) * 1000)
|
|
return record
|
|
|
|
|
|
def run_rejection_case(
|
|
base_url: str,
|
|
vehicle_id: str,
|
|
conversation_id: str,
|
|
case: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
started = time.monotonic()
|
|
record: dict[str, Any] = {
|
|
"id": case["id"],
|
|
"group": case.get("group", "safety"),
|
|
"utterance": case["utterance"],
|
|
"started_at": utc_now(),
|
|
"status": "failed",
|
|
"errors": [],
|
|
}
|
|
try:
|
|
before_code, before = physical_state_request(base_url, vehicle_id, conversation_id)
|
|
if before_code != 200:
|
|
record["errors"].append(f"pre-state HTTP {before_code}")
|
|
payload = {
|
|
"utterance": case["utterance"],
|
|
"provider": case.get("provider", "local"),
|
|
"conversation_id": conversation_id,
|
|
"execution_mode": "hardware",
|
|
"vehicle_id": vehicle_id,
|
|
}
|
|
status_code, result = request_json(
|
|
f"{base_url}/api/requests", payload=payload, timeout_s=60
|
|
)
|
|
record["http_status"] = status_code
|
|
record["request_id"] = result.get("request_id")
|
|
expected_http = int(case.get("expected_http_status", 201))
|
|
if status_code != expected_http:
|
|
record["errors"].append(f"HTTP {status_code}, expected {expected_http}")
|
|
if expected_http == 201:
|
|
expected_status = case.get("expected_status", "needs_confirmation")
|
|
if result.get("status") != expected_status:
|
|
record["errors"].append(
|
|
f"status={result.get('status')}, expected={expected_status}"
|
|
)
|
|
if result.get("function_config") is not None:
|
|
record["errors"].append("rejected request generated a function_config")
|
|
if dig(result, "execution_report", "spec", "mode") == "hil":
|
|
record["errors"].append("rejected request produced a HIL execution report")
|
|
else:
|
|
expected_error = case.get("expected_error")
|
|
if expected_error and result.get("error") != expected_error:
|
|
record["errors"].append(
|
|
f"error={result.get('error')}, expected={expected_error}"
|
|
)
|
|
after_code, after = physical_state_request(base_url, vehicle_id, conversation_id)
|
|
if after_code != 200:
|
|
record["errors"].append(f"post-state HTTP {after_code}")
|
|
before_fingerprint = physical_fingerprint(before)
|
|
after_fingerprint = physical_fingerprint(after)
|
|
record["physical_state_before"] = before_fingerprint
|
|
record["physical_state_after"] = after_fingerprint
|
|
if before_fingerprint != after_fingerprint:
|
|
record["errors"].append("physical state changed for a rejected request")
|
|
record["status"] = "passed" if not record["errors"] else "failed"
|
|
except Exception as exc:
|
|
record["errors"].append(f"{type(exc).__name__}: {exc}")
|
|
record["finished_at"] = utc_now()
|
|
record["duration_ms"] = round((time.monotonic() - started) * 1000)
|
|
return record
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--suite", type=Path, default=DEFAULT_SUITE)
|
|
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8000")
|
|
parser.add_argument("--start-at")
|
|
parser.add_argument(
|
|
"--case",
|
|
action="append",
|
|
dest="case_ids",
|
|
help="Run only this case ID; repeat the option to select an ordered subset.",
|
|
)
|
|
parser.add_argument(
|
|
"--only-group",
|
|
choices=("thermal", "motion", "combined", "mode", "model", "safety"),
|
|
)
|
|
parser.add_argument(
|
|
"--confirm-hardware",
|
|
action="store_true",
|
|
help="Required acknowledgement that a secured, clear T2G bench is connected.",
|
|
)
|
|
args = parser.parse_args()
|
|
if not args.confirm_hardware:
|
|
parser.error("--confirm-hardware is required")
|
|
|
|
suite = json.loads(args.suite.read_text(encoding="utf-8"))
|
|
runtime_code, runtime = request_json(f"{args.base_url}/api/runtime", timeout_s=10)
|
|
if runtime_code != 200 or not runtime.get("hardware_execution_allowed"):
|
|
raise SystemExit("hardware execution is not enabled by the running front-end service")
|
|
vehicle_id = suite.get("vehicle_id", "denza_n7")
|
|
conversation_id = "conv-hil-full-" + datetime.now(UTC).strftime("%Y%m%d%H%M%S")
|
|
tolerance = int(suite["execution_policy"].get("position_tolerance_pct", 2))
|
|
cases = (
|
|
expand_case_entries(list(suite.get("cases", [])))
|
|
+ expand_case_entries(list(suite.get("mode_cases", [])), group="mode")
|
|
+ expand_case_entries(list(suite.get("model_cases", [])), group="model")
|
|
+ expand_case_entries(list(suite.get("rejection_cases", [])), group="safety")
|
|
)
|
|
if args.only_group:
|
|
cases = [item for item in cases if item.get("group") == args.only_group]
|
|
if args.case_ids:
|
|
if args.start_at:
|
|
parser.error("--case and --start-at cannot be used together")
|
|
by_id = {item["id"]: item for item in cases}
|
|
missing = [item for item in args.case_ids if item not in by_id]
|
|
if missing:
|
|
raise SystemExit("unknown --case ID(s): " + ", ".join(missing))
|
|
cases = [by_id[item] for item in args.case_ids]
|
|
if args.start_at:
|
|
ids = [item["id"] for item in cases]
|
|
if args.start_at not in ids:
|
|
raise SystemExit(f"unknown --start-at case {args.start_at}")
|
|
cases = cases[ids.index(args.start_at) :]
|
|
|
|
report: dict[str, Any] = {
|
|
"suite": suite["suite"],
|
|
"started_at": utc_now(),
|
|
"base_url": args.base_url,
|
|
"vehicle_id": vehicle_id,
|
|
"conversation_id": conversation_id,
|
|
"runtime": {
|
|
"hardware_execution_allowed": runtime.get("hardware_execution_allowed"),
|
|
"firmware_profiles": dig(runtime, "hardware", "firmware_profiles"),
|
|
},
|
|
"results": [],
|
|
}
|
|
sync_case = {
|
|
"id": "PREFLIGHT-POSITION-MEMORY-SYNC",
|
|
"group": "preflight",
|
|
**suite["final_state"],
|
|
}
|
|
print(
|
|
"[PREFLIGHT] Four-axis physical homing and Hall-memory synchronization",
|
|
flush=True,
|
|
)
|
|
sync_result = run_case(
|
|
args.base_url, vehicle_id, conversation_id, sync_case, tolerance
|
|
)
|
|
report["position_memory_sync"] = sync_result
|
|
print(
|
|
f" {sync_result['status'].upper()} {sync_result['duration_ms']} ms"
|
|
+ (
|
|
""
|
|
if sync_result["status"] == "passed"
|
|
else ": " + " | ".join(sync_result["errors"])
|
|
),
|
|
flush=True,
|
|
)
|
|
if sync_result["status"] != "passed":
|
|
report["final_state"] = sync_result
|
|
report["finished_at"] = utc_now()
|
|
report["summary"] = {
|
|
"total": 0,
|
|
"passed": 0,
|
|
"failed": 0,
|
|
"safety_blocked": 0,
|
|
"executed": 0,
|
|
"position_memory_sync_passed": False,
|
|
"final_state_passed": False,
|
|
}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
)
|
|
print(json.dumps(report["summary"], ensure_ascii=False), flush=True)
|
|
print(str(args.output.resolve()), flush=True)
|
|
return 1
|
|
|
|
for index, case in enumerate(cases, start=1):
|
|
print(f"[{index:02d}/{len(cases):02d}] {case['id']} {case['utterance']}", flush=True)
|
|
safety_block = safety_block_for_case(
|
|
case, suite["execution_policy"].get("known_safety_blocks", {})
|
|
)
|
|
if safety_block is not None:
|
|
result = {
|
|
"id": case["id"],
|
|
"group": case.get("group", "mode"),
|
|
"utterance": case["utterance"],
|
|
"status": "safety_blocked",
|
|
"started_at": utc_now(),
|
|
"finished_at": utc_now(),
|
|
"duration_ms": 0,
|
|
"errors": [],
|
|
"safety_block": safety_block,
|
|
}
|
|
report["results"].append(result)
|
|
print(
|
|
" SAFETY_BLOCKED: "
|
|
+ (
|
|
f"all {safety_block['axis']} motion is blocked by current bench evidence"
|
|
if safety_block.get("block_all_targets")
|
|
else f"{safety_block['axis']} target {safety_block['target_pct']}% is below "
|
|
f"the verified {safety_block['minimum_safe_target_pct']}% boundary"
|
|
),
|
|
flush=True,
|
|
)
|
|
continue
|
|
if case.get("expected_status") or case.get("expected_http_status"):
|
|
result = run_rejection_case(
|
|
args.base_url, vehicle_id, conversation_id, case
|
|
)
|
|
else:
|
|
result = run_case(
|
|
args.base_url, vehicle_id, conversation_id, case, tolerance
|
|
)
|
|
report["results"].append(result)
|
|
suffix = "" if result["status"] == "passed" else ": " + " | ".join(result["errors"])
|
|
print(f" {result['status'].upper()} {result['duration_ms']} ms{suffix}", flush=True)
|
|
|
|
final_case = {
|
|
"id": "FINAL-THERMAL-OFF",
|
|
"group": "cleanup",
|
|
**suite["final_state"],
|
|
}
|
|
final_result = run_case(
|
|
args.base_url, vehicle_id, conversation_id, final_case, tolerance
|
|
)
|
|
report["final_state"] = final_result
|
|
report["finished_at"] = utc_now()
|
|
report["summary"] = {
|
|
"total": len(report["results"]),
|
|
"passed": sum(item["status"] == "passed" for item in report["results"]),
|
|
"failed": sum(item["status"] == "failed" for item in report["results"]),
|
|
"safety_blocked": sum(
|
|
item["status"] == "safety_blocked" for item in report["results"]
|
|
),
|
|
"executed": sum(
|
|
item["status"] != "safety_blocked" for item in report["results"]
|
|
),
|
|
"position_memory_sync_passed": sync_result["status"] == "passed",
|
|
"final_state_passed": final_result["status"] == "passed",
|
|
}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
)
|
|
print(json.dumps(report["summary"], ensure_ascii=False), flush=True)
|
|
print(str(args.output.resolve()), flush=True)
|
|
return 0 if report["summary"]["failed"] == 0 and final_result["status"] == "passed" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|