vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(web-console): rebuild intelligent cockpit sync interface

Browse Source
This commit is contained in:
vee
2026-08-18 13:24:36 +08:00
parent 344f662970
commit 9b24d5c8c9
30 changed files with 4100 additions and 12594 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cognitive_agent/application/calibration/service.py
+51 -19
View File
@@ -12,7 +12,6 @@ from typing import Any
from . import protocol
from .mqtt_client import MqttSeatPoseClient
from .transports import PythonCanTransport, SimulationTransport, ZlgBridgeTransport
from .video import VideoBridge
AXES = (
{"id": 0, "name": "前后", "title": "前后电机", "unit": "cm", "physical_range": 21.5, "nominal": 480},
@@ -33,8 +32,14 @@ DEFAULT_SETTINGS: dict[str, Any] = {
"dll_path": r"C:\Program Files (x86)\ZCANPRO\zlgcan.dll",
"resistance": 1,
},
"mqtt": {"host": "172.20.10.12", "port": 1883, "topic": "zed/seatpose"},
"video": {"host": "172.20.10.12", "port": 8554, "path": "zed"},
"mqtt": {"host": "172.20.10.6", "port": 1883, "topic": "zed/seatpose"},
"video": {
"scheme": "http",
"host": "172.20.10.6",
"port": 8889,
"path": "zed",
"transport": "webrtc",
},
}
@@ -60,7 +65,6 @@ class CalibrationService:
self._mqtt: MqttSeatPoseClient | None = None
self._saved_job = -1
self.settings = self._load_settings()
self.video = VideoBridge(self._on_video_state)
self.state: dict[str, Any] = self._initial_state()
def start(self, *, auto_connect: bool = False) -> None:
@@ -84,7 +88,6 @@ class CalibrationService:
if self._mqtt:
self._mqtt.close()
self._mqtt = None
self.video.stop()
if self._transport:
self._transport.close()
self._transport = None
@@ -103,7 +106,6 @@ class CalibrationService:
mode = str(self.state["transport"].get("mode", "hardware"))
if mode != "simulation":
raise ApiError("真实台架会话不能通过虚拟重启接口复位", 409)
self.video.stop()
if self._mqtt:
self._mqtt.close()
self._mqtt = None
@@ -285,12 +287,24 @@ class CalibrationService:
mqtt = self.settings["mqtt"]
video = self.settings["video"]
self._validate_host_port(mqtt.get("host"), mqtt.get("port"), "MQTT")
self._validate_host_port(video.get("host"), video.get("port"), "RTSP")
self._validate_host_port(video.get("host"), video.get("port"), "WebRTC")
scheme = str(video.get("scheme", "http")).strip().lower()
if scheme not in {"http", "https"}:
raise ApiError("WebRTC 信令协议必须是 http 或 https")
path = str(video.get("path", "")).strip().strip("/")
if not path or any(char in path for char in " ?#\\"):
raise ApiError("RTSP 流路径无效")
raise ApiError("WebRTC 流路径无效")
video["scheme"] = scheme
video["path"] = path
video["transport"] = "webrtc"
self.state["settings"] = json.loads(json.dumps(self.settings))
self.state["video"].update(
{
"transport": "webrtc",
"url": self._video_url(),
"detail": "WebRTC 由浏览器直接接收",
}
)
self._save_settings()
self._touch()
return self.snapshot()
@@ -331,19 +345,19 @@ class CalibrationService:
return self.snapshot()
def start_video(self) -> dict[str, Any]:
config = self.settings["video"]
uri = f"rtsp://{config['host']}:{int(config['port'])}/{str(config['path']).strip('/')}"
try:
self.video.start(uri)
except Exception as error:
self._on_video_state("error", str(error))
raise ApiError(str(error), 503) from error
self._on_video_state("ready", f"浏览器可直接连接 {self._video_url()}")
return self.snapshot()
def stop_video(self) -> dict[str, Any]:
self.video.stop()
self._on_video_state("idle", "浏览器 WebRTC 接收已停止")
return self.snapshot()
def _video_url(self) -> str:
config = self.settings["video"]
scheme = str(config.get("scheme", "http")).strip().lower()
path = str(config["path"]).strip("/")
return f"{scheme}://{config['host']}:{int(config['port'])}/{path}/"
def _initial_state(self) -> dict[str, Any]:
return {
"version": 0,
@@ -399,9 +413,10 @@ class CalibrationService:
"areas": {},
},
"video": {
"state": "idle",
"detail": "视频接收尚未启动",
"ffmpeg_available": self.video.available,
"state": "ready",
"detail": "WebRTC 由浏览器直接接收",
"transport": "webrtc",
"url": self._video_url(),
},
"settings": json.loads(json.dumps(self.settings)),
"last_result": self._load_result(),
@@ -704,6 +719,23 @@ class CalibrationService:
for section in settings:
if isinstance(stored.get(section), dict):
settings[section].update(stored[section])
stored_video = stored.get("video")
legacy_video_endpoint = (
isinstance(stored_video, dict)
and str(stored_video.get("host", "")).strip() == "172.20.10.12"
and str(stored_video.get("port", "")).strip() == "8554"
)
if isinstance(stored_video, dict) and (
"scheme" not in stored_video or legacy_video_endpoint
):
# Settings written by the retired RTSP/FFmpeg bridge are not
# valid WHEP endpoints. Migrate that schema to the known ZED
# MediaMTX endpoint instead of trying port 8554 as WebRTC. The
# explicit endpoint check also catches legacy settings that a
# newer UI already decorated with scheme/transport fields.
settings["video"] = json.loads(json.dumps(DEFAULT_SETTINGS["video"]))
if settings["mqtt"].get("host") == "172.20.10.12":
settings["mqtt"]["host"] = DEFAULT_SETTINGS["mqtt"]["host"]
except (OSError, ValueError):
pass
return settings
src/cognitive_agent/server.py
+169 -26
View File
@@ -10,6 +10,8 @@ import mimetypes
import os
import re
import socket
import threading
import time
from http.cookies import CookieError, SimpleCookie
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -63,6 +65,82 @@ PUBLIC_AUTH_PATHS = {
}
class VehicleDisplaySync:
"""Process-local snapshot consumed by passive vehicle display clients."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._revision = 0
self._snapshot: dict[str, object] = {
"revision": 0,
"updated_at": None,
"task": {
"status": "idle",
"utterance": "",
"source": "head_unit",
"result": None,
"error": None,
},
"extension": {
"catalog": [],
"last_action": None,
"result": None,
},
}
@staticmethod
def _safe_copy(value: object) -> object:
return json.loads(json.dumps(value, ensure_ascii=False, default=str))
def update_task(
self,
*,
status: str,
utterance: str,
source: str = "head_unit",
result: object = None,
error: object = None,
) -> None:
with self._lock:
self._revision += 1
self._snapshot["revision"] = self._revision
self._snapshot["updated_at"] = time.time()
self._snapshot["task"] = {
"status": status,
"utterance": utterance,
"source": source or "head_unit",
"result": self._safe_copy(result),
"error": self._safe_copy(error),
}
def update_extension(
self,
*,
catalog: object | None = None,
action: object | None = None,
result: object | None = None,
) -> None:
with self._lock:
self._revision += 1
current = dict(self._snapshot.get("extension", {}))
if catalog is not None:
current["catalog"] = self._safe_copy(catalog)
if action is not None:
current["last_action"] = self._safe_copy(action)
if result is not None:
current["result"] = self._safe_copy(result)
self._snapshot["revision"] = self._revision
self._snapshot["updated_at"] = time.time()
self._snapshot["extension"] = current
def snapshot(self, health: dict[str, object]) -> dict[str, object]:
with self._lock:
payload = self._safe_copy(self._snapshot)
assert isinstance(payload, dict)
payload["extension_health"] = self._safe_copy(health)
return payload
def _lan_ipv4_address() -> str | None:
"""Return the IPv4 address selected by the host's default route."""
probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
@@ -86,6 +164,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
)
repair = SelfRepairService()
auth = AuthRuntime(AuthConfig.from_env())
vehicle_display_sync = VehicleDisplaySync()
server_version = "CognitiveAgentCabinDemo/2.0"
def do_GET(self) -> None: # noqa: N802 - stdlib handler API
@@ -151,6 +230,9 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
if path == "/api/runtime":
self._json(self.service.runtime_info())
return
if path == "/api/vehicle-display-sync":
self._json(self.vehicle_display_sync.snapshot(self.extension.health()))
return
if path in {"/api/hardware/seat-state", "/api/hardware/vehicle-state"}:
query = parse_qs(parsed_url.query)
conversation_id = str(query.get("conversation_id", [""])[0])
@@ -277,6 +359,19 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
self._json({"error": "not_found"}, HTTPStatus.NOT_FOUND)
return
utterance = ""
source = "head_unit"
def fail_display_sync(code: str, error: Exception) -> None:
if not utterance:
return
self.vehicle_display_sync.update_task(
status="failed",
utterance=utterance,
source=source,
error={"code": code, "message": str(error)},
)
try:
username = self._session_username() or "local"
retry_after = self.auth.api_requests.retry_after(username)
@@ -293,8 +388,15 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
if self.auth.config.enabled:
self.auth.api_requests.record(username)
payload = self._read_json_body()
utterance = str(payload.get("utterance", ""))
source = str(payload.get("client_type", "head_unit"))
self.vehicle_display_sync.update_task(
status="running",
utterance=utterance,
source=source,
)
result = self.service.process(
str(payload.get("utterance", "")),
utterance,
str(payload.get("provider", "auto")),
str(payload["conversation_id"]) if payload.get("conversation_id") else None,
str(payload.get("execution_mode", "dry_run")),
@@ -319,8 +421,15 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
else None
),
)
self.vehicle_display_sync.update_task(
status=str(result.get("status", "succeeded")),
utterance=utterance,
source=source,
result=result,
)
self._json(result, HTTPStatus.CREATED)
except HardwareAccessRequiredError as exc:
fail_display_sync("hardware_access_required", exc)
self._json(
{
"error": "hardware_access_required",
@@ -330,6 +439,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
HTTPStatus.FORBIDDEN,
)
except HardwareBusyError as exc:
fail_display_sync("hardware_busy", exc)
self._json(
{
"error": "hardware_busy",
@@ -340,13 +450,16 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
{"Retry-After": str(exc.retry_after_seconds)},
)
except HardwareSafetyError as exc:
fail_display_sync("hardware_safety_gate", exc)
self._json(
{"error": "hardware_safety_gate", "message": str(exc)},
HTTPStatus.FORBIDDEN,
)
except (ValueError, json.JSONDecodeError) as exc:
fail_display_sync("invalid_request", exc)
self._json({"error": "invalid_request", "message": str(exc)}, HTTPStatus.BAD_REQUEST)
except OpenAIIntentError as exc:
fail_display_sync("openai_request_failed", exc)
self.log_error("OpenAI intent request failed: %s", exc)
self._json(
{
@@ -356,6 +469,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
HTTPStatus.BAD_GATEWAY,
)
except Exception as exc: # noqa: BLE001 - keep the demo API responsive
fail_display_sync("internal_error", exc)
self.log_error("request failed: %s", exc)
self._json(
{
@@ -412,8 +526,31 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
if action:
self._handle_extension_gateway(action, payload)
return
if path == "/api/extension/display-sync":
catalog = payload.get("peripherals", payload.get("catalog", []))
if not isinstance(catalog, list) or len(catalog) > 248:
raise ValueError("外设能力目录必须是至多 248 项的数组。")
sync_result = {"status": "ready", "count": len(catalog)}
self.vehicle_display_sync.update_extension(
catalog=catalog,
action={
"type": "hardware_detected",
"device": payload.get("device"),
},
result=sync_result,
)
self._json({"success": True, **sync_result})
return
if path == "/api/extension/assistant/plan":
self._json(self.extension.plan(payload))
self.vehicle_display_sync.update_extension(
catalog=payload.get("peripherals", payload.get("catalog", [])),
action={"type": "plan", "command": payload.get("utterance", "")},
)
result = self.extension.plan(payload)
self.vehicle_display_sync.update_extension(
result=result,
)
self._json(result)
return
if path == "/api/extension/vehicle/execute":
result = self.extension.execute_vehicle(
@@ -421,6 +558,17 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
self.service,
hardware_access_bypass=self._is_unproxied_loopback_request(),
)
self.vehicle_display_sync.update_task(
status=str(result.get("status", "succeeded")),
utterance=str(payload.get("command", payload.get("utterance", ""))),
source=str(payload.get("client_type", "head_unit")),
result=result,
)
self.vehicle_display_sync.update_extension(
catalog=payload.get("catalog") if "catalog" in payload else None,
action={"type": "vehicle_execute"},
result=result,
)
self._json(result)
return
self._json({"error": "not_found"}, HTTPStatus.NOT_FOUND)
@@ -694,31 +842,23 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
pass
def _serve_calibration_video(self) -> None:
self.send_response(HTTPStatus.OK)
self.send_header("Content-Type", "multipart/x-mixed-replace; boundary=n7frame")
self.send_header("Cache-Control", "no-store")
self.send_header("Connection", "close")
self._send_security_headers()
self.end_headers()
sequence = -1
try:
while True:
sequence, frame = self.calibration.video.wait_frame(sequence)
if frame is None:
continue
header = (
b"--n7frame\r\nContent-Type: image/jpeg\r\nContent-Length: "
+ str(len(frame)).encode("ascii")
+ b"\r\n\r\n"
)
self.wfile.write(header + frame + b"\r\n")
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError):
pass
self._json(
{
"error": "legacy_video_transport_removed",
"message": "ZED 画面已改为浏览器直连 WebRTC/WHEP。",
},
HTTPStatus.GONE,
)
def _handle_extension_gateway(self, action: str, payload: dict[str, object]) -> None:
try:
self._json(self.extension.execute_gateway(action, dict(payload)))
result = self.extension.execute_gateway(action, dict(payload))
if action != "info":
self.vehicle_display_sync.update_extension(
action={"type": action, **dict(payload)},
result=result,
)
self._json(result)
except GatewayError as exc:
self._json(
{"success": False, "error": "gscu_unavailable", "message": str(exc)},
@@ -986,7 +1126,9 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
cache_control = "public, max-age=31536000, immutable"
elif relative_path.startswith("assets/"):
cache_control = "no-cache"
elif candidate.name in {"index.html", "login.html", "service-worker.js"}:
elif candidate.name == "service-worker.js":
cache_control = "no-store"
elif candidate.name in {"index.html", "login.html"}:
cache_control = "no-cache"
self.send_header("Cache-Control", cache_control)
if candidate.name == "service-worker.js":
@@ -1050,7 +1192,8 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
"script-src 'self' 'wasm-unsafe-eval'; "
"style-src 'self'; "
"img-src 'self' data: blob:; "
"connect-src 'self' blob:; "
"media-src 'self' blob:; "
"connect-src 'self' blob: http://172.20.10.6:8889; "
"object-src 'none'; base-uri 'none'; frame-ancestors 'none'",
)
if self._is_https():