"""Run the repository-owned front-end seat suite against a connected T2G bench.""" from __future__ import annotations import argparse import json import sys import time import urllib.error import urllib.parse import urllib.request from datetime import UTC, datetime from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] DEFAULT_SUITE = ROOT / "tests" / "fixtures" / "seat_frontend_hil_suite.json" DEFAULT_OUTPUT = ROOT / "artifacts" / "hil" / "seat_frontend_hil_latest.json" MOTION_AXES = { "longitudinal", "cushion_tilt", "backrest_recline", "legrest_height", } EXPECTED_CAPABILITIES = { "heating": "seat.set_heating", "ventilation": "seat.set_ventilation", "longitudinal": "seat.set_longitudinal_position", "cushion_tilt": "seat.set_cushion_tilt", "backrest_recline": "seat.set_backrest_recline", "legrest_height": "seat.set_legrest_height", } def utc_now() -> str: return datetime.now(UTC).isoformat().replace("+00:00", "Z") def request_json( url: str, *, payload: dict[str, Any] | None = None, timeout_s: int = 180, ) -> tuple[int, dict[str, Any]]: data = None method = "GET" headers: dict[str, str] = {} if payload is not None: data = json.dumps(payload, ensure_ascii=False).encode("utf-8") method = "POST" headers["Content-Type"] = "application/json; charset=utf-8" request = urllib.request.Request(url, data=data, headers=headers, method=method) try: with urllib.request.urlopen(request, timeout=timeout_s) as response: return response.status, json.loads(response.read().decode("utf-8")) except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") try: detail = json.loads(body) except json.JSONDecodeError: detail = {"message": body} return exc.code, detail def dig(value: dict[str, Any], *path: str) -> Any: current: Any = value for item in path: if not isinstance(current, dict) or item not in current: return None current = current[item] return current def failed_hardware_result(result: dict[str, Any]) -> dict[str, Any] | None: items = dig(result, "execution_report", "spec", "results") or [] for item in reversed(items): if item.get("status") != "succeeded": return item return None def expand_case_entries( entries: list[dict[str, Any]], *, group: str | None = None ) -> list[dict[str, Any]]: """Expand compact input-equivalence families into individually reported cases.""" expanded: list[dict[str, Any]] = [] for entry in entries: base = {key: value for key, value in entry.items() if key != "variants"} if group is not None: base["group"] = group variants = entry.get("variants") if not variants: expanded.append(base) continue for variant in variants: case = {**base, **variant} if "id" not in case or "utterance" not in case: raise ValueError("each case variant requires id and utterance") expanded.append(case) return expanded def expected_operation_errors( result: dict[str, Any], expected: dict[str, int], expected_motion: dict[str, Any] | None = None, ) -> list[str]: errors: list[str] = [] seat_config = (result.get("function_configs") or {}).get("seat") or result.get( "function_config" ) or {} operations = dig(seat_config, "spec", "operations") or [] by_capability = {item.get("capability"): item for item in operations} for field, expected_value in expected.items(): capability = EXPECTED_CAPABILITIES.get(field) if capability is None: continue operation = by_capability.get(capability) if operation is None: errors.append(f"expected capability was not generated: {capability}") continue if dig(operation, "target", "instance") != "passenger" or dig( operation, "target", "zone" ) != "front_right": errors.append(f"{capability} did not target passenger/front_right") parameter = "level" if field in {"heating", "ventilation"} else "position_pct" actual_value = dig(operation, "parameters", parameter) if field in {"heating", "ventilation"} and expected_value == 0: if dig(operation, "parameters", "state") != "off": errors.append(f"{capability} state is not off") elif actual_value != expected_value: errors.append( f"{capability} {parameter}={actual_value}, expected={expected_value}" ) if expected_motion: axis = str(expected_motion["axis"]) capability = EXPECTED_CAPABILITIES[axis] operation = by_capability.get(capability) motion = operation.get("motion") if operation else None if not isinstance(motion, dict): errors.append(f"{capability} did not preserve motion metadata") else: for field, expected_value in expected_motion.items(): if field == "axis": continue actual_value = motion.get(field) if actual_value != expected_value: errors.append( f"{capability} motion.{field}={actual_value}, " f"expected={expected_value}" ) return errors def physical_fingerprint(state_result: dict[str, Any]) -> dict[str, Any]: physical = state_result.get("physical_state", {}) return { "source": physical.get("source"), "complete": physical.get("complete"), "heating": dig(physical, "heating", "level"), "ventilation": dig(physical, "ventilation", "level"), "position": { axis: dig(physical, "position", axis, "position_pct") for axis in sorted(MOTION_AXES) }, } def physical_state_request( base_url: str, vehicle_id: str, conversation_id: str ) -> tuple[int, dict[str, Any]]: query = urllib.parse.urlencode( {"conversation_id": conversation_id, "vehicle_id": vehicle_id} ) return request_json( f"{base_url}/api/hardware/seat-state?{query}", timeout_s=20 ) def validate_physical_state( state_result: dict[str, Any], expected: dict[str, int], tolerance: int ) -> list[str]: errors: list[str] = [] physical = state_result.get("physical_state", {}) synchronized = state_result.get("actuator_state", {}).get("passenger", {}) if physical.get("source") != "physical_can": errors.append("physical state source is not physical_can") if not physical.get("complete"): errors.append("physical state is incomplete: " + ",".join(physical.get("missing", []))) for feature in ("heating", "ventilation"): if feature not in expected: continue actual = dig(physical, feature, "level") twin = dig(synchronized, feature, "level") if actual != expected[feature]: errors.append(f"{feature} physical={actual}, expected={expected[feature]}") if twin != actual: errors.append(f"{feature} twin={twin}, physical={actual}") if dig(synchronized, feature, "feedback_source") != "physical_can": errors.append(f"{feature} twin feedback source is not physical_can") for axis in MOTION_AXES: if axis not in expected: continue actual = dig(physical, "position", axis, "position_pct") twin = dig(synchronized, "position", axis, "position_pct") if not isinstance(actual, (int, float)) or abs(actual - expected[axis]) > tolerance: errors.append(f"{axis} physical={actual}, expected={expected[axis]}±{tolerance}") if twin != actual: errors.append(f"{axis} twin={twin}, physical={actual}") if dig(synchronized, "position", axis, "feedback_source") != "physical_can": errors.append(f"{axis} twin feedback source is not physical_can") return errors def safety_block_for_case( case: dict[str, Any], safety_blocks: dict[str, Any] ) -> dict[str, Any] | None: """Refuse only motions covered by a documented, evidence-backed bench hazard.""" expected = case.get("expected", {}) for axis, policy in safety_blocks.items(): target = expected.get(axis) if target is None: continue if policy.get("block_all_targets") is True: return { "axis": axis, "target_pct": target, "block_all_targets": True, "reason": policy.get("reason", "documented hardware safety block"), "evidence": policy.get("evidence", {}), } minimum = policy.get("minimum_safe_target_pct") if isinstance(minimum, (int, float)) and target < minimum: return { "axis": axis, "target_pct": target, "minimum_safe_target_pct": minimum, "reason": policy.get("reason", "documented hardware safety boundary"), "evidence": policy.get("evidence", {}), } return None def run_case( base_url: str, vehicle_id: str, conversation_id: str, case: dict[str, Any], tolerance: int, ) -> dict[str, Any]: started = time.monotonic() record: dict[str, Any] = { "id": case["id"], "group": case.get("group", "mode"), "utterance": case["utterance"], "started_at": utc_now(), "status": "failed", "errors": [], } payload = { "utterance": case["utterance"], "provider": case.get("provider", "local"), "conversation_id": conversation_id, "execution_mode": "hardware", "vehicle_id": vehicle_id, } try: status_code, result = request_json( f"{base_url}/api/requests", payload=payload, timeout_s=900 ) record["http_status"] = status_code record["request_id"] = result.get("request_id") record["provider_used"] = result.get("provider_used") if status_code != 201: record["errors"].append(result.get("message", f"HTTP {status_code}")) elif result.get("status") != "succeeded": failed = failed_hardware_result(result) record["errors"].append( failed.get("message", "request failed") if failed else "request failed" ) if failed: record["failure_telemetry"] = failed.get("telemetry", {}) elif dig(result, "execution_report", "spec", "status") != "succeeded": failed = failed_hardware_result(result) record["errors"].append( failed.get("message", "hardware execution failed") if failed else "hardware execution failed" ) if failed: record["failure_telemetry"] = failed.get("telemetry", {}) else: expected_provider_prefix = case.get("expected_provider_prefix") if expected_provider_prefix and not str(result.get("provider_used", "")).startswith( str(expected_provider_prefix) ): record["errors"].append( f"provider_used={result.get('provider_used')}, " f"expected prefix={expected_provider_prefix}" ) record["errors"].extend( expected_operation_errors( result, case.get("expected", {}), case.get("expected_motion"), ) ) seat_config = (result.get("function_configs") or {}).get( "seat" ) or result.get("function_config") or {} operations = dig(seat_config, "spec", "operations") or [] seat_operations = [ item for item in operations if str(item.get("capability", "")).startswith("seat.") ] invalid_targets = [ item.get("target") for item in seat_operations if dig(item, "target", "instance") != "passenger" or dig(item, "target", "zone") != "front_right" ] if invalid_targets: record["errors"].append(f"non-passenger targets generated: {invalid_targets}") visible_notices = [ *[str(item) for item in result.get("decision_summary", [])], *[ str(item.get("message", "")) for item in dig(result, "execution_report", "spec", "results") or [] ], *[ str(item.get("telemetry", {}).get("user_notice", "")) for item in dig(result, "execution_report", "spec", "results") or [] ], ] visible_notice_text = "\n".join(visible_notices) for expected_notice in case.get("expected_notice_contains", []): if expected_notice not in visible_notice_text: record["errors"].append( f"visible execution notice does not contain {expected_notice!r}" ) expected_mode = case.get("mode_id") matched_modes = [item.get("id") for item in result.get("matched_modes", [])] record["matched_modes"] = matched_modes if expected_mode and expected_mode not in matched_modes: record["errors"].append( f"mode {expected_mode} was not matched: {matched_modes}" ) state_code, state = physical_state_request( base_url, vehicle_id, conversation_id ) record["state_http_status"] = state_code if state_code != 200: record["errors"].append( state.get("message", f"physical state HTTP {state_code}") ) else: record["errors"].extend( validate_physical_state(state, case.get("expected", {}), tolerance) ) record["physical_state"] = state.get("physical_state") record["status"] = "passed" if not record["errors"] else "failed" except Exception as exc: # continue the suite after transport and parse errors record["errors"].append(f"{type(exc).__name__}: {exc}") record["finished_at"] = utc_now() record["duration_ms"] = round((time.monotonic() - started) * 1000) return record def run_rejection_case( base_url: str, vehicle_id: str, conversation_id: str, case: dict[str, Any], ) -> dict[str, Any]: started = time.monotonic() record: dict[str, Any] = { "id": case["id"], "group": case.get("group", "safety"), "utterance": case["utterance"], "started_at": utc_now(), "status": "failed", "errors": [], } try: before_code, before = physical_state_request(base_url, vehicle_id, conversation_id) if before_code != 200: record["errors"].append(f"pre-state HTTP {before_code}") payload = { "utterance": case["utterance"], "provider": case.get("provider", "local"), "conversation_id": conversation_id, "execution_mode": "hardware", "vehicle_id": vehicle_id, } status_code, result = request_json( f"{base_url}/api/requests", payload=payload, timeout_s=60 ) record["http_status"] = status_code record["request_id"] = result.get("request_id") expected_http = int(case.get("expected_http_status", 201)) if status_code != expected_http: record["errors"].append(f"HTTP {status_code}, expected {expected_http}") if expected_http == 201: expected_status = case.get("expected_status", "needs_confirmation") if result.get("status") != expected_status: record["errors"].append( f"status={result.get('status')}, expected={expected_status}" ) if result.get("function_config") is not None: record["errors"].append("rejected request generated a function_config") if dig(result, "execution_report", "spec", "mode") == "hil": record["errors"].append("rejected request produced a HIL execution report") else: expected_error = case.get("expected_error") if expected_error and result.get("error") != expected_error: record["errors"].append( f"error={result.get('error')}, expected={expected_error}" ) after_code, after = physical_state_request(base_url, vehicle_id, conversation_id) if after_code != 200: record["errors"].append(f"post-state HTTP {after_code}") before_fingerprint = physical_fingerprint(before) after_fingerprint = physical_fingerprint(after) record["physical_state_before"] = before_fingerprint record["physical_state_after"] = after_fingerprint if before_fingerprint != after_fingerprint: record["errors"].append("physical state changed for a rejected request") record["status"] = "passed" if not record["errors"] else "failed" except Exception as exc: record["errors"].append(f"{type(exc).__name__}: {exc}") record["finished_at"] = utc_now() record["duration_ms"] = round((time.monotonic() - started) * 1000) return record def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--suite", type=Path, default=DEFAULT_SUITE) parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) parser.add_argument("--base-url", default="http://127.0.0.1:8000") parser.add_argument("--start-at") parser.add_argument( "--case", action="append", dest="case_ids", help="Run only this case ID; repeat the option to select an ordered subset.", ) parser.add_argument( "--only-group", choices=("thermal", "motion", "combined", "mode", "model", "safety"), ) parser.add_argument( "--confirm-hardware", action="store_true", help="Required acknowledgement that a secured, clear T2G bench is connected.", ) args = parser.parse_args() if not args.confirm_hardware: parser.error("--confirm-hardware is required") suite = json.loads(args.suite.read_text(encoding="utf-8")) runtime_code, runtime = request_json(f"{args.base_url}/api/runtime", timeout_s=10) if runtime_code != 200 or not runtime.get("hardware_execution_allowed"): raise SystemExit("hardware execution is not enabled by the running front-end service") vehicle_id = suite.get("vehicle_id", "denza_n7") conversation_id = "conv-hil-full-" + datetime.now(UTC).strftime("%Y%m%d%H%M%S") tolerance = int(suite["execution_policy"].get("position_tolerance_pct", 2)) cases = ( expand_case_entries(list(suite.get("cases", []))) + expand_case_entries(list(suite.get("mode_cases", [])), group="mode") + expand_case_entries(list(suite.get("model_cases", [])), group="model") + expand_case_entries(list(suite.get("rejection_cases", [])), group="safety") ) if args.only_group: cases = [item for item in cases if item.get("group") == args.only_group] if args.case_ids: if args.start_at: parser.error("--case and --start-at cannot be used together") by_id = {item["id"]: item for item in cases} missing = [item for item in args.case_ids if item not in by_id] if missing: raise SystemExit("unknown --case ID(s): " + ", ".join(missing)) cases = [by_id[item] for item in args.case_ids] if args.start_at: ids = [item["id"] for item in cases] if args.start_at not in ids: raise SystemExit(f"unknown --start-at case {args.start_at}") cases = cases[ids.index(args.start_at) :] report: dict[str, Any] = { "suite": suite["suite"], "started_at": utc_now(), "base_url": args.base_url, "vehicle_id": vehicle_id, "conversation_id": conversation_id, "runtime": { "hardware_execution_allowed": runtime.get("hardware_execution_allowed"), "firmware_profiles": dig(runtime, "hardware", "firmware_profiles"), }, "results": [], } sync_case = { "id": "PREFLIGHT-POSITION-MEMORY-SYNC", "group": "preflight", **suite["final_state"], } print( "[PREFLIGHT] Four-axis physical homing and Hall-memory synchronization", flush=True, ) sync_result = run_case( args.base_url, vehicle_id, conversation_id, sync_case, tolerance ) report["position_memory_sync"] = sync_result print( f" {sync_result['status'].upper()} {sync_result['duration_ms']} ms" + ( "" if sync_result["status"] == "passed" else ": " + " | ".join(sync_result["errors"]) ), flush=True, ) if sync_result["status"] != "passed": report["final_state"] = sync_result report["finished_at"] = utc_now() report["summary"] = { "total": 0, "passed": 0, "failed": 0, "safety_blocked": 0, "executed": 0, "position_memory_sync_passed": False, "final_state_passed": False, } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text( json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8" ) print(json.dumps(report["summary"], ensure_ascii=False), flush=True) print(str(args.output.resolve()), flush=True) return 1 for index, case in enumerate(cases, start=1): print(f"[{index:02d}/{len(cases):02d}] {case['id']} {case['utterance']}", flush=True) safety_block = safety_block_for_case( case, suite["execution_policy"].get("known_safety_blocks", {}) ) if safety_block is not None: result = { "id": case["id"], "group": case.get("group", "mode"), "utterance": case["utterance"], "status": "safety_blocked", "started_at": utc_now(), "finished_at": utc_now(), "duration_ms": 0, "errors": [], "safety_block": safety_block, } report["results"].append(result) print( " SAFETY_BLOCKED: " + ( f"all {safety_block['axis']} motion is blocked by current bench evidence" if safety_block.get("block_all_targets") else f"{safety_block['axis']} target {safety_block['target_pct']}% is below " f"the verified {safety_block['minimum_safe_target_pct']}% boundary" ), flush=True, ) continue if case.get("expected_status") or case.get("expected_http_status"): result = run_rejection_case( args.base_url, vehicle_id, conversation_id, case ) else: result = run_case( args.base_url, vehicle_id, conversation_id, case, tolerance ) report["results"].append(result) suffix = "" if result["status"] == "passed" else ": " + " | ".join(result["errors"]) print(f" {result['status'].upper()} {result['duration_ms']} ms{suffix}", flush=True) final_case = { "id": "FINAL-THERMAL-OFF", "group": "cleanup", **suite["final_state"], } final_result = run_case( args.base_url, vehicle_id, conversation_id, final_case, tolerance ) report["final_state"] = final_result report["finished_at"] = utc_now() report["summary"] = { "total": len(report["results"]), "passed": sum(item["status"] == "passed" for item in report["results"]), "failed": sum(item["status"] == "failed" for item in report["results"]), "safety_blocked": sum( item["status"] == "safety_blocked" for item in report["results"] ), "executed": sum( item["status"] != "safety_blocked" for item in report["results"] ), "position_memory_sync_passed": sync_result["status"] == "passed", "final_state_passed": final_result["status"] == "passed", } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text( json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8" ) print(json.dumps(report["summary"], ensure_ascii=False), flush=True) print(str(args.output.resolve()), flush=True) return 0 if report["summary"]["failed"] == 0 and final_result["status"] == "passed" else 1 if __name__ == "__main__": sys.exit(main())