feat: validate real-bench seat orchestration and self-repair
This commit is contained in:
@@ -4,7 +4,8 @@ The original HIL demo mixed HTTP hosting, business state, firmware tooling and
|
||||
ZCANPRO process control in one script. This module keeps the proven state
|
||||
machine and diagnostic rules while exposing them through the repository's
|
||||
existing server. Every machine-specific path is supplied through environment
|
||||
configuration; importing the module never launches hardware tooling.
|
||||
configuration or discovered standard installations; importing the module never
|
||||
launches hardware tooling.
|
||||
"""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
@@ -29,6 +30,9 @@ from .signal_diagnostic import (
|
||||
from .zcan_bridge import (
|
||||
ZcanBridgeManager, run_self_test as run_bridge_manager_self_test,
|
||||
)
|
||||
from cognitive_agent.infrastructure.firmware.t2g_seat import (
|
||||
EXPECTED_FIRMWARE_REVISION,
|
||||
)
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[4]
|
||||
@@ -39,6 +43,17 @@ def _env_path(name, default):
|
||||
return Path(value).expanduser() if value else Path(default)
|
||||
|
||||
|
||||
def _env_or_installed_path(name, candidates, fallback):
|
||||
value = os.getenv(name, "").strip()
|
||||
if value:
|
||||
return Path(value).expanduser()
|
||||
for candidate in candidates:
|
||||
path = Path(candidate)
|
||||
if path.exists():
|
||||
return path
|
||||
return Path(fallback)
|
||||
|
||||
|
||||
def _env_flag(name, default=False):
|
||||
value = os.getenv(name)
|
||||
if value is None:
|
||||
@@ -71,16 +86,28 @@ HARDWARE_LOG_ROOT = _env_path(
|
||||
REPOSITORY_ROOT / "artifacts" / "self-repair" / "hardware",
|
||||
)
|
||||
PROJECT_DIR = _env_path(
|
||||
"SELF_REPAIR_FIRMWARE_ROOT", REPOSITORY_ROOT / "__unconfigured_firmware__")
|
||||
"SELF_REPAIR_FIRMWARE_ROOT",
|
||||
REPOSITORY_ROOT / "firmware" / "t2g" / "Seat_Controller_demo")
|
||||
IAR_PROJECT_PATH = PROJECT_DIR / "IAR" / "Seat_Controller_demo.ewp"
|
||||
FIRMWARE_OUT_PATH = PROJECT_DIR / "IAR" / "rev_d" / "Exe" / \
|
||||
"Seat_Controller_demo.out"
|
||||
FIRMWARE_HEX_PATH = PROJECT_DIR / "IAR" / "rev_d" / "Exe" / \
|
||||
"Seat_Controller_demo.hex"
|
||||
IAR_BUILD_PATH = _env_path(
|
||||
"SELF_REPAIR_IAR_BUILD_PATH", REPOSITORY_ROOT / "__unconfigured_iar_build__")
|
||||
CSPY_BAT_PATH = _env_path(
|
||||
"SELF_REPAIR_CSPY_BAT_PATH", REPOSITORY_ROOT / "__unconfigured_cspy__")
|
||||
IAR_INSTALL_ROOTS = (
|
||||
Path(r"D:\Program Files\IAR Systems\Embedded Workbench 9.2"),
|
||||
Path(r"C:\Program Files\IAR Systems\Embedded Workbench 9.2"),
|
||||
Path(r"C:\Program Files\IAR Systems\Embedded Workbench 9.4"),
|
||||
)
|
||||
IAR_BUILD_PATH = _env_or_installed_path(
|
||||
"SELF_REPAIR_IAR_BUILD_PATH",
|
||||
tuple(root / "common" / "bin" / "iarbuild.exe"
|
||||
for root in IAR_INSTALL_ROOTS),
|
||||
REPOSITORY_ROOT / "__unconfigured_iar_build__")
|
||||
CSPY_BAT_PATH = _env_or_installed_path(
|
||||
"SELF_REPAIR_CSPY_BAT_PATH",
|
||||
tuple(root / "common" / "bin" / "CSpyBat.exe"
|
||||
for root in IAR_INSTALL_ROOTS),
|
||||
REPOSITORY_ROOT / "__unconfigured_cspy__")
|
||||
CSPY_GENERAL_PATH = _env_path(
|
||||
"SELF_REPAIR_CSPY_GENERAL_PATH",
|
||||
PROJECT_DIR / "IAR" / "settings" / "Seat_Controller_demo.rev_d.general.xcl",
|
||||
@@ -96,13 +123,19 @@ FIRMWARE_LOG_ROOT = _env_path(
|
||||
FIRMWARE_SOURCE_ROOTS = tuple(
|
||||
PROJECT_DIR / name for name in ("App", "Bsp", "Config", "Driver"))
|
||||
FIRMWARE_SOURCE_SUFFIXES = frozenset((".c", ".h", ".s", ".asm"))
|
||||
REQUIRED_BRIDGE_VERSION = "2026-08-07-reset-home-far-target-v15"
|
||||
ZCANPRO_EXE = _env_path(
|
||||
"SELF_REPAIR_ZCANPRO_EXE", REPOSITORY_ROOT / "__unconfigured_zcanpro__")
|
||||
ZCANPRO_SCRIPT_CONFIG = _env_path(
|
||||
"SELF_REPAIR_ZCANPRO_SCRIPT_CONFIG",
|
||||
REPOSITORY_ROOT / "__unconfigured_zcanpro_script_config__",
|
||||
REQUIRED_BRIDGE_VERSION = "2026-08-09-real-self-heal-v32"
|
||||
ZCANPRO_ROOTS = (
|
||||
Path(r"C:\Program Files (x86)\ZCANPRO"),
|
||||
Path(r"C:\Program Files\ZCANPRO"),
|
||||
)
|
||||
ZCANPRO_EXE = _env_or_installed_path(
|
||||
"SELF_REPAIR_ZCANPRO_EXE",
|
||||
tuple(root / "ZCANPRO.exe" for root in ZCANPRO_ROOTS),
|
||||
REPOSITORY_ROOT / "__unconfigured_zcanpro__")
|
||||
ZCANPRO_SCRIPT_CONFIG = _env_or_installed_path(
|
||||
"SELF_REPAIR_ZCANPRO_SCRIPT_CONFIG",
|
||||
tuple(root / "config" / "extend_script.ini" for root in ZCANPRO_ROOTS),
|
||||
REPOSITORY_ROOT / "__unconfigured_zcanpro_script_config__")
|
||||
BRIDGE_MANAGER = ZcanBridgeManager(
|
||||
STATUS_PATH, BRIDGE_SCRIPT_PATH,
|
||||
required_version=REQUIRED_BRIDGE_VERSION,
|
||||
@@ -268,6 +301,13 @@ class ConsoleState(object):
|
||||
"CAN链路有流量但缺少0x301~0x305,判定为未烧录/固件不匹配类问题" if
|
||||
bridge.get("bus_traffic") else
|
||||
"桥接在线但座椅协议无报文,判定为未烧录/固件未运行类问题")
|
||||
elif not bridge.get("firmware_revision_ok"):
|
||||
reason = "FIRMWARE_REVISION_MISMATCH"
|
||||
detail = (
|
||||
"Seat_Controller_demo固件revision=%s,期望revision=%s,"
|
||||
"需要自动重新构建并烧录" %
|
||||
(bridge.get("firmware_revision", "未知"),
|
||||
EXPECTED_FIRMWARE_REVISION))
|
||||
else:
|
||||
return {
|
||||
"ready": True, "auto_flash": False,
|
||||
@@ -334,7 +374,8 @@ class ConsoleState(object):
|
||||
last_bridge = self._bridge_summary(last_status, time.time())
|
||||
if (last_bridge.get("online") and
|
||||
(not require_protocol or
|
||||
last_bridge.get("protocol_online"))):
|
||||
(last_bridge.get("protocol_online") and
|
||||
last_bridge.get("firmware_revision_ok")))):
|
||||
return last_status, last_bridge
|
||||
time.sleep(0.25)
|
||||
return last_status, last_bridge or {}
|
||||
@@ -385,7 +426,8 @@ class ConsoleState(object):
|
||||
trigger_reason=qualification.get("reason_code"))
|
||||
status, bridge = self._wait_bridge(require_protocol=True,
|
||||
timeout_s=12.0)
|
||||
if not bridge.get("protocol_online"):
|
||||
if (not bridge.get("protocol_online") or
|
||||
not bridge.get("firmware_revision_ok")):
|
||||
detail = ("自动烧录已执行,但仍未收到0x301~0x305;"
|
||||
"硬件诊断未启动,请检查ECU供电或烧录日志")
|
||||
with self.lock:
|
||||
@@ -430,6 +472,7 @@ class ConsoleState(object):
|
||||
online = age is not None and age < 3.0 and version_ok
|
||||
system = {} if not isinstance(status, dict) else (
|
||||
status.get("system") or {})
|
||||
firmware_revision = system.get("firmware_revision")
|
||||
frame_epochs = {} if not isinstance(status, dict) else (
|
||||
status.get("frame_epochs") or {})
|
||||
recent_frame_ids = []
|
||||
@@ -460,6 +503,10 @@ class ConsoleState(object):
|
||||
"current_request_id": None if not isinstance(status, dict) else status.get(
|
||||
"current_request_id"),
|
||||
"position_valid_mask": int(system.get("position_valid_mask", 0)),
|
||||
"firmware_revision": firmware_revision,
|
||||
"expected_firmware_revision": EXPECTED_FIRMWARE_REVISION,
|
||||
"firmware_revision_ok":
|
||||
firmware_revision == EXPECTED_FIRMWARE_REVISION,
|
||||
"active_mask": int(system.get("active_mask", 0)),
|
||||
"faulted_mask": int(system.get("faulted_mask", 0)),
|
||||
"bus": {} if not isinstance(status, dict) else (
|
||||
@@ -618,6 +665,16 @@ class ConsoleState(object):
|
||||
self.hardware_active = False
|
||||
if not self.hardware_run_id or self.hardware_report_path is not None:
|
||||
return
|
||||
active_test = (self.hardware_result or {}).get("active_test") or {}
|
||||
test_phase = active_test.get("phase")
|
||||
if test_phase in ("PASSED", "FAILED"):
|
||||
self.hardware_prepare.update({
|
||||
"state": "DIAGNOSTIC_%s" % test_phase,
|
||||
"detail": active_test.get("detail") or
|
||||
active_test.get("title") or
|
||||
"21针主动检测已结束",
|
||||
"finished_epoch": now,
|
||||
})
|
||||
report = {
|
||||
"schema": "seat-hardware-diagnostic-report-v1",
|
||||
"run_id": self.hardware_run_id,
|
||||
@@ -625,8 +682,7 @@ class ConsoleState(object):
|
||||
"events": list(self.hardware_events),
|
||||
"diagnosis": self.hardware_result,
|
||||
"startup_rule_check": self.hardware_self_test,
|
||||
"active_pin_test": (self.hardware_result or {}).get(
|
||||
"active_test"),
|
||||
"active_pin_test": active_test,
|
||||
"bridge": self._bridge_summary(status, now),
|
||||
}
|
||||
path = HARDWARE_LOG_ROOT / self.hardware_run_id / \
|
||||
@@ -839,10 +895,11 @@ class ConsoleState(object):
|
||||
"firmware": self._firmware_snapshot(),
|
||||
}
|
||||
|
||||
def start(self, case_id, execution_mode="shadow"):
|
||||
def start(self, case_id, execution_mode="shadow", allow_batch=False):
|
||||
with self.lock:
|
||||
if execution_mode == "bench":
|
||||
if self.hardware_active or self.batch.get("active"):
|
||||
if (self.hardware_active or
|
||||
(self.batch.get("active") and not allow_batch)):
|
||||
raise ValueError("硬件诊断或批量验证正在运行,不能启动实物故障用例")
|
||||
scenario = SCENARIO_BY_ID.get(case_id) or {}
|
||||
if not (scenario.get("bench") or {}).get("supported", False):
|
||||
@@ -850,8 +907,9 @@ class ConsoleState(object):
|
||||
"该用例没有安全的实物注入手段,请使用影子仿真")
|
||||
self.bridge_manager.ensure_async()
|
||||
now = time.time()
|
||||
status = _read_json(STATUS_PATH)
|
||||
bridge = self._bridge_summary(status, now)
|
||||
status, bridge = self._wait_bridge(
|
||||
require_protocol=True, timeout_s=12.0)
|
||||
now = time.time()
|
||||
if not bridge.get("online"):
|
||||
raise ValueError("ZCANPRO桥接未在线,不能向实物ECU写入故障")
|
||||
if not bridge.get("protocol_online"):
|
||||
@@ -888,7 +946,8 @@ class ConsoleState(object):
|
||||
self.bench_request_id = None
|
||||
return self.engine.start(case_id, execution_mode="shadow")
|
||||
|
||||
def reset(self, execution_mode="shadow"):
|
||||
def reset(self, execution_mode="shadow", allow_batch=False,
|
||||
force_all_axes=True, home_nonzero_axes=False):
|
||||
with self.lock:
|
||||
if (self.engine.execution_mode == "bench" and
|
||||
self.engine.phase not in ("IDLE", PHASE_PASSED,
|
||||
@@ -907,12 +966,14 @@ class ConsoleState(object):
|
||||
})
|
||||
return {"healing": self.engine.snapshot(),
|
||||
"bench_reset": dict(self.bench_reset)}
|
||||
if self.hardware_active or self.batch.get("active"):
|
||||
if (self.hardware_active or
|
||||
(self.batch.get("active") and not allow_batch)):
|
||||
raise ValueError("硬件诊断或批量验证正在运行,不能执行四轴归零")
|
||||
self.bridge_manager.ensure_async()
|
||||
now = time.time()
|
||||
status = _read_json(STATUS_PATH)
|
||||
bridge = self._bridge_summary(status, now)
|
||||
status, bridge = self._wait_bridge(
|
||||
require_protocol=True, timeout_s=12.0)
|
||||
now = time.time()
|
||||
if not bridge.get("online") or not bridge.get("protocol_online"):
|
||||
raise ValueError("CAN桥接或座椅ECU协议未在线,不能执行四轴归零")
|
||||
if (status or {}).get("current_request_id"):
|
||||
@@ -925,7 +986,10 @@ class ConsoleState(object):
|
||||
"armed_until_epoch": now + 600.0,
|
||||
"source": "seat_self_healing_console",
|
||||
"action": "initialize_known_ranges",
|
||||
"payload": {"force_all_axes": True},
|
||||
"payload": {
|
||||
"force_all_axes": bool(force_all_axes),
|
||||
"home_nonzero_axes": bool(home_nonzero_axes),
|
||||
},
|
||||
}
|
||||
zero = self._bench_zero_snapshot(status)
|
||||
self.bench_reset = {
|
||||
@@ -938,13 +1002,64 @@ class ConsoleState(object):
|
||||
return {"healing": self.engine.snapshot(),
|
||||
"bench_reset": dict(self.bench_reset)}
|
||||
|
||||
def _wait_bench_reset(self, timeout_s=620.0):
|
||||
"""Wait for the bridge-owned four-axis homing request to finish."""
|
||||
deadline = time.time() + float(timeout_s)
|
||||
last = None
|
||||
while time.time() < deadline:
|
||||
last = self.snapshot()
|
||||
reset = last.get("bench_reset") or {}
|
||||
if reset.get("phase") in ("PASSED", "FAILED"):
|
||||
return reset
|
||||
time.sleep(0.2)
|
||||
with self.lock:
|
||||
self.bench_reset.update({
|
||||
"phase": "FAILED",
|
||||
"finished_epoch": time.time(),
|
||||
"summary": "四轴归零等待超时,批量验证已停止",
|
||||
})
|
||||
return dict(self.bench_reset)
|
||||
|
||||
@staticmethod
|
||||
def _wait_bench_bridge_idle(timeout_s=8.0):
|
||||
"""Wait until the bridge clears its just-published request result."""
|
||||
deadline = time.time() + float(timeout_s)
|
||||
while time.time() < deadline:
|
||||
status = _read_json(STATUS_PATH) or {}
|
||||
if not status.get("current_request_id"):
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return False
|
||||
|
||||
def _batch_worker(self, execution_mode):
|
||||
try:
|
||||
for scenario in public_scenarios():
|
||||
with self.lock:
|
||||
self.batch["current_case_id"] = scenario["id"]
|
||||
self.latest_report_path = None
|
||||
self.start(scenario["id"], execution_mode)
|
||||
if execution_mode == "bench":
|
||||
if not self._wait_bench_bridge_idle():
|
||||
raise ValueError(
|
||||
"%s前台架桥接任务未在安全窗口内结束" % scenario["id"])
|
||||
reset = {}
|
||||
for reset_attempt in range(2):
|
||||
self.reset(
|
||||
"bench", allow_batch=True,
|
||||
force_all_axes=False,
|
||||
home_nonzero_axes=True)
|
||||
reset = self._wait_bench_reset()
|
||||
if (reset.get("phase") == "PASSED" and
|
||||
reset.get("all_zero")):
|
||||
break
|
||||
if reset_attempt == 0:
|
||||
time.sleep(1.0)
|
||||
if (reset.get("phase") != "PASSED" or
|
||||
not reset.get("all_zero")):
|
||||
raise ValueError(
|
||||
"%s前四轴自动归零失败:%s" %
|
||||
(scenario["id"], reset.get("summary", "未知错误")))
|
||||
self.start(
|
||||
scenario["id"], execution_mode, allow_batch=True)
|
||||
while True:
|
||||
value = self.snapshot()
|
||||
phase = value["healing"]["phase"]
|
||||
@@ -961,9 +1076,12 @@ class ConsoleState(object):
|
||||
"duration_s": value["healing"].get("elapsed_s"),
|
||||
"diagnosed_code": value["healing"].get("diagnosed_code"),
|
||||
"execution_mode": execution_mode,
|
||||
"snapshot_path": (None if self.latest_report_path is None else
|
||||
str(self.latest_report_path.parent /
|
||||
"diagnostic_snapshot_success.json")),
|
||||
"report_path": (None if self.latest_report_path is None else
|
||||
str(self.latest_report_path)),
|
||||
"snapshot_path": (
|
||||
None if self.latest_report_path is None or not passed else
|
||||
str(self.latest_report_path.parent /
|
||||
"diagnostic_snapshot_success.json")),
|
||||
}
|
||||
self.batch["results"].append(result)
|
||||
self.batch["completed"] += 1
|
||||
@@ -990,8 +1108,9 @@ class ConsoleState(object):
|
||||
if execution_mode not in ("shadow", "bench"):
|
||||
raise ValueError("未知批量验证方式")
|
||||
if execution_mode == "bench":
|
||||
raise ValueError(
|
||||
"实物台架需逐项验证:每个用例前请人工点击重置并确认四轴归零")
|
||||
if self.hardware_active:
|
||||
raise ValueError("硬件诊断正在运行,不能启动实物批量验证")
|
||||
self.bridge_manager.ensure_async()
|
||||
self.batch = self._new_batch_state(execution_mode)
|
||||
self.batch["active"] = True
|
||||
self.batch["started_epoch"] = time.time()
|
||||
@@ -1376,7 +1495,10 @@ class SelfRepairService(object):
|
||||
return self.state.start_batch(payload.get("execution_mode", "shadow"))
|
||||
|
||||
def reset_healing(self, payload):
|
||||
return self.state.reset(payload.get("execution_mode", "shadow"))
|
||||
return self.state.reset(
|
||||
payload.get("execution_mode", "shadow"),
|
||||
force_all_axes=bool(payload.get("force_all_axes", True)),
|
||||
home_nonzero_axes=bool(payload.get("home_nonzero_axes", False)))
|
||||
|
||||
def start_hardware(self):
|
||||
return self.state.start_hardware()
|
||||
|
||||
@@ -24,6 +24,7 @@ class ZcanBridgeManager(object):
|
||||
|
||||
def __init__(self, status_path, script_path, heartbeat_stale_s=3.0,
|
||||
poll_s=1.0, retry_cooldown_s=15.0,
|
||||
busy_heartbeat_grace_s=45.0,
|
||||
required_version=None, zcanpro_exe=None,
|
||||
script_config_path=None, auto_start=False):
|
||||
self.status_path = Path(status_path)
|
||||
@@ -34,6 +35,7 @@ class ZcanBridgeManager(object):
|
||||
self.heartbeat_stale_s = float(heartbeat_stale_s)
|
||||
self.poll_s = float(poll_s)
|
||||
self.retry_cooldown_s = float(retry_cooldown_s)
|
||||
self.busy_heartbeat_grace_s = float(busy_heartbeat_grace_s)
|
||||
self.required_version = required_version
|
||||
self.lock = threading.RLock()
|
||||
self.stop_event = threading.Event()
|
||||
@@ -79,6 +81,7 @@ class ZcanBridgeManager(object):
|
||||
"last_attempt_epoch": self.last_attempt_epoch,
|
||||
"last_online_epoch": self.last_online_epoch,
|
||||
"heartbeat_age_s": None if age is None else round(age, 2),
|
||||
"busy_heartbeat_grace_s": self.busy_heartbeat_grace_s,
|
||||
"zcan_window_found": self.zcan_window_found,
|
||||
"retry_armed": self.retry_requested,
|
||||
"zcan_exe": str(self.zcanpro_exe),
|
||||
@@ -120,6 +123,20 @@ class ZcanBridgeManager(object):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _bridge_busy_within_grace(self):
|
||||
try:
|
||||
import json
|
||||
with self.status_path.open("r", encoding="utf-8") as stream:
|
||||
status = json.load(stream)
|
||||
heartbeat = float(status.get("heartbeat_epoch", 0) or 0)
|
||||
age = max(0.0, time.time() - heartbeat)
|
||||
return bool(
|
||||
status.get("current_request_id") and
|
||||
status.get("bridge_state") == "RUNNING" and
|
||||
heartbeat > 0 and age <= self.busy_heartbeat_grace_s)
|
||||
except (OSError, ValueError, TypeError):
|
||||
return False
|
||||
|
||||
def _take_retry_request(self):
|
||||
with self.lock:
|
||||
if not self.retry_requested:
|
||||
@@ -130,15 +147,22 @@ class ZcanBridgeManager(object):
|
||||
def _monitor(self):
|
||||
while not self.stop_event.is_set():
|
||||
if self._heartbeat_online():
|
||||
with self.lock:
|
||||
# A defensive ensure request is already satisfied by a
|
||||
# healthy bridge. Do not carry it into a later live task.
|
||||
self.retry_requested = False
|
||||
self._set_state(
|
||||
"ONLINE", "CAN桥接在线,由ZCANPRO扩展脚本持续提供数据",
|
||||
last_online_epoch=time.time())
|
||||
elif self._bridge_busy_within_grace():
|
||||
self._set_state(
|
||||
"BUSY", "CAN bridge request is active; restart is deferred")
|
||||
else:
|
||||
with self.lock:
|
||||
# A bridge that was genuinely online gets one automatic
|
||||
# recovery attempt. A failed recovery is not repeated
|
||||
# until an explicit Start action calls ``ensure_async``.
|
||||
if self.state == "ONLINE":
|
||||
if self.state in ("ONLINE", "BUSY"):
|
||||
self.retry_requested = True
|
||||
if self._take_retry_request():
|
||||
self._attempt_start()
|
||||
|
||||
Reference in New Issue
Block a user