feat: validate real-bench seat orchestration and self-repair

This commit is contained in:
vee
2026-08-09 19:00:16 +08:00
parent 3bceb0d5a1
commit 3701192a11
41 changed files with 10403 additions and 3392 deletions
@@ -4,7 +4,8 @@ The original HIL demo mixed HTTP hosting, business state, firmware tooling and
ZCANPRO process control in one script. This module keeps the proven state
machine and diagnostic rules while exposing them through the repository's
existing server. Every machine-specific path is supplied through environment
configuration; importing the module never launches hardware tooling.
configuration or discovered standard installations; importing the module never
launches hardware tooling.
"""
from __future__ import unicode_literals
@@ -29,6 +30,9 @@ from .signal_diagnostic import (
from .zcan_bridge import (
ZcanBridgeManager, run_self_test as run_bridge_manager_self_test,
)
from cognitive_agent.infrastructure.firmware.t2g_seat import (
EXPECTED_FIRMWARE_REVISION,
)
REPOSITORY_ROOT = Path(__file__).resolve().parents[4]
@@ -39,6 +43,17 @@ def _env_path(name, default):
return Path(value).expanduser() if value else Path(default)
def _env_or_installed_path(name, candidates, fallback):
value = os.getenv(name, "").strip()
if value:
return Path(value).expanduser()
for candidate in candidates:
path = Path(candidate)
if path.exists():
return path
return Path(fallback)
def _env_flag(name, default=False):
value = os.getenv(name)
if value is None:
@@ -71,16 +86,28 @@ HARDWARE_LOG_ROOT = _env_path(
REPOSITORY_ROOT / "artifacts" / "self-repair" / "hardware",
)
PROJECT_DIR = _env_path(
"SELF_REPAIR_FIRMWARE_ROOT", REPOSITORY_ROOT / "__unconfigured_firmware__")
"SELF_REPAIR_FIRMWARE_ROOT",
REPOSITORY_ROOT / "firmware" / "t2g" / "Seat_Controller_demo")
IAR_PROJECT_PATH = PROJECT_DIR / "IAR" / "Seat_Controller_demo.ewp"
FIRMWARE_OUT_PATH = PROJECT_DIR / "IAR" / "rev_d" / "Exe" / \
"Seat_Controller_demo.out"
FIRMWARE_HEX_PATH = PROJECT_DIR / "IAR" / "rev_d" / "Exe" / \
"Seat_Controller_demo.hex"
IAR_BUILD_PATH = _env_path(
"SELF_REPAIR_IAR_BUILD_PATH", REPOSITORY_ROOT / "__unconfigured_iar_build__")
CSPY_BAT_PATH = _env_path(
"SELF_REPAIR_CSPY_BAT_PATH", REPOSITORY_ROOT / "__unconfigured_cspy__")
IAR_INSTALL_ROOTS = (
Path(r"D:\Program Files\IAR Systems\Embedded Workbench 9.2"),
Path(r"C:\Program Files\IAR Systems\Embedded Workbench 9.2"),
Path(r"C:\Program Files\IAR Systems\Embedded Workbench 9.4"),
)
IAR_BUILD_PATH = _env_or_installed_path(
"SELF_REPAIR_IAR_BUILD_PATH",
tuple(root / "common" / "bin" / "iarbuild.exe"
for root in IAR_INSTALL_ROOTS),
REPOSITORY_ROOT / "__unconfigured_iar_build__")
CSPY_BAT_PATH = _env_or_installed_path(
"SELF_REPAIR_CSPY_BAT_PATH",
tuple(root / "common" / "bin" / "CSpyBat.exe"
for root in IAR_INSTALL_ROOTS),
REPOSITORY_ROOT / "__unconfigured_cspy__")
CSPY_GENERAL_PATH = _env_path(
"SELF_REPAIR_CSPY_GENERAL_PATH",
PROJECT_DIR / "IAR" / "settings" / "Seat_Controller_demo.rev_d.general.xcl",
@@ -96,13 +123,19 @@ FIRMWARE_LOG_ROOT = _env_path(
FIRMWARE_SOURCE_ROOTS = tuple(
PROJECT_DIR / name for name in ("App", "Bsp", "Config", "Driver"))
FIRMWARE_SOURCE_SUFFIXES = frozenset((".c", ".h", ".s", ".asm"))
REQUIRED_BRIDGE_VERSION = "2026-08-07-reset-home-far-target-v15"
ZCANPRO_EXE = _env_path(
"SELF_REPAIR_ZCANPRO_EXE", REPOSITORY_ROOT / "__unconfigured_zcanpro__")
ZCANPRO_SCRIPT_CONFIG = _env_path(
"SELF_REPAIR_ZCANPRO_SCRIPT_CONFIG",
REPOSITORY_ROOT / "__unconfigured_zcanpro_script_config__",
REQUIRED_BRIDGE_VERSION = "2026-08-09-real-self-heal-v32"
ZCANPRO_ROOTS = (
Path(r"C:\Program Files (x86)\ZCANPRO"),
Path(r"C:\Program Files\ZCANPRO"),
)
ZCANPRO_EXE = _env_or_installed_path(
"SELF_REPAIR_ZCANPRO_EXE",
tuple(root / "ZCANPRO.exe" for root in ZCANPRO_ROOTS),
REPOSITORY_ROOT / "__unconfigured_zcanpro__")
ZCANPRO_SCRIPT_CONFIG = _env_or_installed_path(
"SELF_REPAIR_ZCANPRO_SCRIPT_CONFIG",
tuple(root / "config" / "extend_script.ini" for root in ZCANPRO_ROOTS),
REPOSITORY_ROOT / "__unconfigured_zcanpro_script_config__")
BRIDGE_MANAGER = ZcanBridgeManager(
STATUS_PATH, BRIDGE_SCRIPT_PATH,
required_version=REQUIRED_BRIDGE_VERSION,
@@ -268,6 +301,13 @@ class ConsoleState(object):
"CAN链路有流量但缺少0x301~0x305,判定为未烧录/固件不匹配类问题" if
bridge.get("bus_traffic") else
"桥接在线但座椅协议无报文,判定为未烧录/固件未运行类问题")
elif not bridge.get("firmware_revision_ok"):
reason = "FIRMWARE_REVISION_MISMATCH"
detail = (
"Seat_Controller_demo固件revision=%s,期望revision=%s,"
"需要自动重新构建并烧录" %
(bridge.get("firmware_revision", "未知"),
EXPECTED_FIRMWARE_REVISION))
else:
return {
"ready": True, "auto_flash": False,
@@ -334,7 +374,8 @@ class ConsoleState(object):
last_bridge = self._bridge_summary(last_status, time.time())
if (last_bridge.get("online") and
(not require_protocol or
last_bridge.get("protocol_online"))):
(last_bridge.get("protocol_online") and
last_bridge.get("firmware_revision_ok")))):
return last_status, last_bridge
time.sleep(0.25)
return last_status, last_bridge or {}
@@ -385,7 +426,8 @@ class ConsoleState(object):
trigger_reason=qualification.get("reason_code"))
status, bridge = self._wait_bridge(require_protocol=True,
timeout_s=12.0)
if not bridge.get("protocol_online"):
if (not bridge.get("protocol_online") or
not bridge.get("firmware_revision_ok")):
detail = ("自动烧录已执行,但仍未收到0x301~0x305;"
"硬件诊断未启动,请检查ECU供电或烧录日志")
with self.lock:
@@ -430,6 +472,7 @@ class ConsoleState(object):
online = age is not None and age < 3.0 and version_ok
system = {} if not isinstance(status, dict) else (
status.get("system") or {})
firmware_revision = system.get("firmware_revision")
frame_epochs = {} if not isinstance(status, dict) else (
status.get("frame_epochs") or {})
recent_frame_ids = []
@@ -460,6 +503,10 @@ class ConsoleState(object):
"current_request_id": None if not isinstance(status, dict) else status.get(
"current_request_id"),
"position_valid_mask": int(system.get("position_valid_mask", 0)),
"firmware_revision": firmware_revision,
"expected_firmware_revision": EXPECTED_FIRMWARE_REVISION,
"firmware_revision_ok":
firmware_revision == EXPECTED_FIRMWARE_REVISION,
"active_mask": int(system.get("active_mask", 0)),
"faulted_mask": int(system.get("faulted_mask", 0)),
"bus": {} if not isinstance(status, dict) else (
@@ -618,6 +665,16 @@ class ConsoleState(object):
self.hardware_active = False
if not self.hardware_run_id or self.hardware_report_path is not None:
return
active_test = (self.hardware_result or {}).get("active_test") or {}
test_phase = active_test.get("phase")
if test_phase in ("PASSED", "FAILED"):
self.hardware_prepare.update({
"state": "DIAGNOSTIC_%s" % test_phase,
"detail": active_test.get("detail") or
active_test.get("title") or
"21针主动检测已结束",
"finished_epoch": now,
})
report = {
"schema": "seat-hardware-diagnostic-report-v1",
"run_id": self.hardware_run_id,
@@ -625,8 +682,7 @@ class ConsoleState(object):
"events": list(self.hardware_events),
"diagnosis": self.hardware_result,
"startup_rule_check": self.hardware_self_test,
"active_pin_test": (self.hardware_result or {}).get(
"active_test"),
"active_pin_test": active_test,
"bridge": self._bridge_summary(status, now),
}
path = HARDWARE_LOG_ROOT / self.hardware_run_id / \
@@ -839,10 +895,11 @@ class ConsoleState(object):
"firmware": self._firmware_snapshot(),
}
def start(self, case_id, execution_mode="shadow"):
def start(self, case_id, execution_mode="shadow", allow_batch=False):
with self.lock:
if execution_mode == "bench":
if self.hardware_active or self.batch.get("active"):
if (self.hardware_active or
(self.batch.get("active") and not allow_batch)):
raise ValueError("硬件诊断或批量验证正在运行,不能启动实物故障用例")
scenario = SCENARIO_BY_ID.get(case_id) or {}
if not (scenario.get("bench") or {}).get("supported", False):
@@ -850,8 +907,9 @@ class ConsoleState(object):
"该用例没有安全的实物注入手段,请使用影子仿真")
self.bridge_manager.ensure_async()
now = time.time()
status = _read_json(STATUS_PATH)
bridge = self._bridge_summary(status, now)
status, bridge = self._wait_bridge(
require_protocol=True, timeout_s=12.0)
now = time.time()
if not bridge.get("online"):
raise ValueError("ZCANPRO桥接未在线,不能向实物ECU写入故障")
if not bridge.get("protocol_online"):
@@ -888,7 +946,8 @@ class ConsoleState(object):
self.bench_request_id = None
return self.engine.start(case_id, execution_mode="shadow")
def reset(self, execution_mode="shadow"):
def reset(self, execution_mode="shadow", allow_batch=False,
force_all_axes=True, home_nonzero_axes=False):
with self.lock:
if (self.engine.execution_mode == "bench" and
self.engine.phase not in ("IDLE", PHASE_PASSED,
@@ -907,12 +966,14 @@ class ConsoleState(object):
})
return {"healing": self.engine.snapshot(),
"bench_reset": dict(self.bench_reset)}
if self.hardware_active or self.batch.get("active"):
if (self.hardware_active or
(self.batch.get("active") and not allow_batch)):
raise ValueError("硬件诊断或批量验证正在运行,不能执行四轴归零")
self.bridge_manager.ensure_async()
now = time.time()
status = _read_json(STATUS_PATH)
bridge = self._bridge_summary(status, now)
status, bridge = self._wait_bridge(
require_protocol=True, timeout_s=12.0)
now = time.time()
if not bridge.get("online") or not bridge.get("protocol_online"):
raise ValueError("CAN桥接或座椅ECU协议未在线,不能执行四轴归零")
if (status or {}).get("current_request_id"):
@@ -925,7 +986,10 @@ class ConsoleState(object):
"armed_until_epoch": now + 600.0,
"source": "seat_self_healing_console",
"action": "initialize_known_ranges",
"payload": {"force_all_axes": True},
"payload": {
"force_all_axes": bool(force_all_axes),
"home_nonzero_axes": bool(home_nonzero_axes),
},
}
zero = self._bench_zero_snapshot(status)
self.bench_reset = {
@@ -938,13 +1002,64 @@ class ConsoleState(object):
return {"healing": self.engine.snapshot(),
"bench_reset": dict(self.bench_reset)}
def _wait_bench_reset(self, timeout_s=620.0):
"""Wait for the bridge-owned four-axis homing request to finish."""
deadline = time.time() + float(timeout_s)
last = None
while time.time() < deadline:
last = self.snapshot()
reset = last.get("bench_reset") or {}
if reset.get("phase") in ("PASSED", "FAILED"):
return reset
time.sleep(0.2)
with self.lock:
self.bench_reset.update({
"phase": "FAILED",
"finished_epoch": time.time(),
"summary": "四轴归零等待超时,批量验证已停止",
})
return dict(self.bench_reset)
@staticmethod
def _wait_bench_bridge_idle(timeout_s=8.0):
"""Wait until the bridge clears its just-published request result."""
deadline = time.time() + float(timeout_s)
while time.time() < deadline:
status = _read_json(STATUS_PATH) or {}
if not status.get("current_request_id"):
return True
time.sleep(0.05)
return False
def _batch_worker(self, execution_mode):
try:
for scenario in public_scenarios():
with self.lock:
self.batch["current_case_id"] = scenario["id"]
self.latest_report_path = None
self.start(scenario["id"], execution_mode)
if execution_mode == "bench":
if not self._wait_bench_bridge_idle():
raise ValueError(
"%s前台架桥接任务未在安全窗口内结束" % scenario["id"])
reset = {}
for reset_attempt in range(2):
self.reset(
"bench", allow_batch=True,
force_all_axes=False,
home_nonzero_axes=True)
reset = self._wait_bench_reset()
if (reset.get("phase") == "PASSED" and
reset.get("all_zero")):
break
if reset_attempt == 0:
time.sleep(1.0)
if (reset.get("phase") != "PASSED" or
not reset.get("all_zero")):
raise ValueError(
"%s前四轴自动归零失败:%s" %
(scenario["id"], reset.get("summary", "未知错误")))
self.start(
scenario["id"], execution_mode, allow_batch=True)
while True:
value = self.snapshot()
phase = value["healing"]["phase"]
@@ -961,9 +1076,12 @@ class ConsoleState(object):
"duration_s": value["healing"].get("elapsed_s"),
"diagnosed_code": value["healing"].get("diagnosed_code"),
"execution_mode": execution_mode,
"snapshot_path": (None if self.latest_report_path is None else
str(self.latest_report_path.parent /
"diagnostic_snapshot_success.json")),
"report_path": (None if self.latest_report_path is None else
str(self.latest_report_path)),
"snapshot_path": (
None if self.latest_report_path is None or not passed else
str(self.latest_report_path.parent /
"diagnostic_snapshot_success.json")),
}
self.batch["results"].append(result)
self.batch["completed"] += 1
@@ -990,8 +1108,9 @@ class ConsoleState(object):
if execution_mode not in ("shadow", "bench"):
raise ValueError("未知批量验证方式")
if execution_mode == "bench":
raise ValueError(
"实物台架需逐项验证:每个用例前请人工点击重置并确认四轴归零")
if self.hardware_active:
raise ValueError("硬件诊断正在运行,不能启动实物批量验证")
self.bridge_manager.ensure_async()
self.batch = self._new_batch_state(execution_mode)
self.batch["active"] = True
self.batch["started_epoch"] = time.time()
@@ -1376,7 +1495,10 @@ class SelfRepairService(object):
return self.state.start_batch(payload.get("execution_mode", "shadow"))
def reset_healing(self, payload):
return self.state.reset(payload.get("execution_mode", "shadow"))
return self.state.reset(
payload.get("execution_mode", "shadow"),
force_all_axes=bool(payload.get("force_all_axes", True)),
home_nonzero_axes=bool(payload.get("home_nonzero_axes", False)))
def start_hardware(self):
return self.state.start_hardware()
@@ -24,6 +24,7 @@ class ZcanBridgeManager(object):
def __init__(self, status_path, script_path, heartbeat_stale_s=3.0,
poll_s=1.0, retry_cooldown_s=15.0,
busy_heartbeat_grace_s=45.0,
required_version=None, zcanpro_exe=None,
script_config_path=None, auto_start=False):
self.status_path = Path(status_path)
@@ -34,6 +35,7 @@ class ZcanBridgeManager(object):
self.heartbeat_stale_s = float(heartbeat_stale_s)
self.poll_s = float(poll_s)
self.retry_cooldown_s = float(retry_cooldown_s)
self.busy_heartbeat_grace_s = float(busy_heartbeat_grace_s)
self.required_version = required_version
self.lock = threading.RLock()
self.stop_event = threading.Event()
@@ -79,6 +81,7 @@ class ZcanBridgeManager(object):
"last_attempt_epoch": self.last_attempt_epoch,
"last_online_epoch": self.last_online_epoch,
"heartbeat_age_s": None if age is None else round(age, 2),
"busy_heartbeat_grace_s": self.busy_heartbeat_grace_s,
"zcan_window_found": self.zcan_window_found,
"retry_armed": self.retry_requested,
"zcan_exe": str(self.zcanpro_exe),
@@ -120,6 +123,20 @@ class ZcanBridgeManager(object):
return False
return True
def _bridge_busy_within_grace(self):
try:
import json
with self.status_path.open("r", encoding="utf-8") as stream:
status = json.load(stream)
heartbeat = float(status.get("heartbeat_epoch", 0) or 0)
age = max(0.0, time.time() - heartbeat)
return bool(
status.get("current_request_id") and
status.get("bridge_state") == "RUNNING" and
heartbeat > 0 and age <= self.busy_heartbeat_grace_s)
except (OSError, ValueError, TypeError):
return False
def _take_retry_request(self):
with self.lock:
if not self.retry_requested:
@@ -130,15 +147,22 @@ class ZcanBridgeManager(object):
def _monitor(self):
while not self.stop_event.is_set():
if self._heartbeat_online():
with self.lock:
# A defensive ensure request is already satisfied by a
# healthy bridge. Do not carry it into a later live task.
self.retry_requested = False
self._set_state(
"ONLINE", "CAN桥接在线,由ZCANPRO扩展脚本持续提供数据",
last_online_epoch=time.time())
elif self._bridge_busy_within_grace():
self._set_state(
"BUSY", "CAN bridge request is active; restart is deferred")
else:
with self.lock:
# A bridge that was genuinely online gets one automatic
# recovery attempt. A failed recovery is not repeated
# until an explicit Start action calls ``ensure_async``.
if self.state == "ONLINE":
if self.state in ("ONLINE", "BUSY"):
self.retry_requested = True
if self._take_retry_request():
self._attempt_start()