vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完善 N7 智能座舱交互与控制链路

Browse Source
This commit is contained in:
vee
2026-09-07 15:34:59 +08:00
parent 32d179143f
commit c96970f068
100 changed files with 30088 additions and 1174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/test_seat_web_api.py
+138 -63
View File
@@ -16,6 +16,7 @@ from unittest.mock import patch
from cognitive_agent.application.mode_profile import ModeProfileRegistry
from cognitive_agent.application.seat_service import REPOSITORY_ROOT, SeatDemoService
from cognitive_agent.application.self_extension import (
GatewayError,
SelfExtensionService,
SimulationGateway,
)
@@ -223,6 +224,8 @@ operations:
result = {
"request_id": request_id,
"utterance": instruction,
"status": "succeeded",
"vehicle": {"id": "denza_n7"},
"execution_mode": "dry_run",
"control_transport": "wireless_mqtt",
"function_configs": {
@@ -617,40 +620,52 @@ operations:
def test_static_console_and_runtime_endpoint(self) -> None:
with self.opener.open(f"{self.base_url}/", timeout=3) as response:
html = response.read().decode("utf-8")
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-114", timeout=3) as response:
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-126", timeout=3) as response:
app_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/gesture-control.js?v=android-pad-9", timeout=3
) as response:
gesture_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/console-v2.js?v=delivery-state-55", timeout=3
f"{self.base_url}/console-v2.js?v=delivery-state-62", timeout=3
) as response:
console_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/console-v2.css?v=delivery-state-50", timeout=3
f"{self.base_url}/console-v2.css?v=delivery-state-65", timeout=3
) as response:
console_styles = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-input.js?v=sensevoice-7", timeout=3
f"{self.base_url}/speech-input.js?v=sensevoice-12", timeout=3
) as response:
speech_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-input.css?v=sensevoice-5", timeout=3
f"{self.base_url}/speech-input.css?v=sensevoice-7", timeout=3
) as response:
speech_styles = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-audio-worklet.js?v=sensevoice-4", timeout=3
f"{self.base_url}/speech-audio-worklet.js?v=sensevoice-5", timeout=3
) as response:
speech_worklet = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/self-extension.js?v=gscu-tcp-41", timeout=3
f"{self.base_url}/self-extension.js?v=gscu-tcp-43", timeout=3
) as response:
extension_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/self-extension.css?v=gscu-tcp-42", timeout=3
f"{self.base_url}/self-extension.css?v=gscu-tcp-43", timeout=3
) as response:
extension_styles = response.read().decode("utf-8").replace("\r\n", "\n")
with self.opener.open(
f"{self.base_url}/assistant-chat.js?v=codex-assistant-20", timeout=3
) as response:
assistant_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/energy-monitor.js?v=energy-monitor-4", timeout=3
) as response:
energy_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/energy-monitor.css?v=energy-monitor-4", timeout=3
) as response:
energy_styles = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/service-worker.js", timeout=3
) as response:
@@ -681,26 +696,31 @@ operations:
self.assertIn("CMD 0x431 · FB 0x41A · CH1", html)
self.assertIn("CMD 0x431 · FB 0x388 · CH1", html)
self.assertNotIn("透明车体 · 实体设备", html)
self.assertIn('src="/app.js?v=dual-seat-114"', html)
self.assertIn('href="/styles.css?v=dual-seat-21"', html)
self.assertIn('src="/app.js?v=dual-seat-126"', html)
self.assertIn('href="/styles.css?v=dual-seat-23"', html)
self.assertIn('src="/gesture-control.js?v=android-pad-9"', html)
self.assertNotIn('class="vehicle-system-hud"', html)
self.assertIn('src="/console-v2.js?v=delivery-state-55"', html)
self.assertIn('href="/console-v2.css?v=delivery-state-50"', html)
self.assertIn('src="/speech-input.js?v=sensevoice-7"', html)
self.assertIn('href="/speech-input.css?v=sensevoice-5"', html)
self.assertIn('src="/console-v2.js?v=delivery-state-62"', html)
self.assertIn('href="/console-v2.css?v=delivery-state-65"', html)
self.assertIn('src="/speech-input.js?v=sensevoice-12"', html)
self.assertIn('href="/speech-input.css?v=sensevoice-7"', html)
self.assertIn('id="speechDictationButton"', html)
self.assertIn('class="speech-waveform"', html)
self.assertIn('<body class="console-v2">', html)
self.assertIn('id="v2BootScreen"', html)
self.assertIn('正在加载车机控制与实时状态', html)
self.assertIn('src="/self-extension.js?v=gscu-tcp-41"', html)
self.assertIn('href="/self-extension.css?v=gscu-tcp-42"', html)
self.assertIn('src="/self-extension.js?v=gscu-tcp-43"', html)
self.assertIn('href="/self-extension.css?v=gscu-tcp-43"', html)
self.assertNotIn("self-calibration", html)
self.assertNotIn("self-repair", html)
self.assertNotIn("self-evolution", html)
self.assertNotIn("滚动查看更多", html)
self.assertIn('aria-label="执行车舱控制指令"', html)
self.assertIn('aria-label="发送消息"', html)
self.assertIn('id="assistantTranscript"', html)
self.assertIn('href="/assistant-chat.css?v=codex-assistant-22"', html)
self.assertIn('src="/assistant-chat.js?v=codex-assistant-20"', html)
self.assertIn('href="/energy-monitor.css?v=energy-monitor-4"', html)
self.assertIn('src="/energy-monitor.js?v=energy-monitor-4"', html)
self.assertNotIn('"admin"', console_script)
self.assertNotIn("v2RoleSelect", console_script)
self.assertIn('sidebarToggle.id = "v2SidebarToggle"', console_script)
@@ -709,14 +729,24 @@ operations:
'panelHeading.classList.add("v2-operation-heading", "v2-module-heading")',
console_script,
)
self.assertIn('if (title) title.textContent = "一句话远程编排"', console_script)
self.assertIn('if (title) title.textContent = "AI助手"', console_script)
self.assertIn('id="v2PanelControlDock"', console_script)
self.assertIn('sidebarCollapsed: true', console_script)
self.assertIn('setSidePanelVisibility("devices", false)', console_script)
self.assertIn('setSidePanelVisibility("extensions", false)', console_script)
self.assertIn('setDrawerOpen(true)', assistant_script)
self.assertIn('($("#v2PanelControlDock") || document.body).append(trigger)', assistant_script)
self.assertIn('trigger.id = "energyMonitorToggle"', energy_script)
self.assertIn('document.body.append(layer)', energy_script)
self.assertIn('.energy-monitor-layer', energy_styles)
self.assertNotIn('assistantDrawerClose', assistant_script)
self.assertIn('fetch("/api/vehicle-display-sync"', console_script)
self.assertIn('document.body.classList.add("v2-passive-display", "v2-interactive-sync")', console_script)
self.assertIn('input.readOnly = false', console_script)
self.assertIn('function buildTwinDeviceInteractions()', console_script)
self.assertIn('id="v2SyncLogList"', console_script)
self.assertIn('data-panel-toggle="devices"', console_script)
self.assertIn('data-panel-toggle="extensions"', console_script)
self.assertIn('deviceToggle.dataset.panelToggle = "devices"', console_script)
self.assertIn('extensionToggle.dataset.panelToggle = "extensions"', console_script)
self.assertIn('id="vehicleExtensionSync"', console_script)
self.assertNotIn(
'if (ui.activeModule === "self-extension" && ui.role !== "user")',
@@ -827,7 +857,9 @@ operations:
self.assertIn('id="modeRailContent"', html)
self.assertIn('role="region"', html)
self.assertIn('aria-label="功能模式列表"', html)
self.assertIn('<span>自定义</span>', html)
self.assertIn('<span>AI 新建</span>', html)
self.assertIn('class="mode-rail-identity"', html)
self.assertIn('aria-expanded="true"', html)
self.assertIn('模拟下发', html)
self.assertIn('真实下发', html)
self.assertNotIn('id="simulationMode"', html)
@@ -841,7 +873,7 @@ operations:
self.assertNotIn('id="v2HeadUnitState"', console_script)
self.assertIn('.v2-sync-execution-field .execution-mode-options', console_styles)
self.assertIn(
'.v2-sync-execution-field .execution-mode-options {\n grid-template-columns: repeat(2, minmax(0, 1fr))',
'grid-template-columns: repeat(2, minmax(0, 1fr))',
console_styles,
)
self.assertNotIn('class="v2-sync-protocol"', console_script)
@@ -959,28 +991,42 @@ operations:
self.assertIn("CAN 指令已提交，等待车辆状态确认", console_script)
self.assertIn('canvas.addEventListener("webglcontextlost"', app_script)
self.assertIn('"cognitive-agent:module-change"', app_script)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v228"', service_worker)
self.assertIn('"/app.js?v=dual-seat-114"', service_worker)
self.assertIn('"/speech-input.js?v=sensevoice-7"', service_worker)
self.assertIn('"/speech-input.css?v=sensevoice-5"', service_worker)
self.assertIn('"/speech-audio-worklet.js?v=sensevoice-4"', service_worker)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v269"', service_worker)
self.assertIn('/console-v2.css?v=delivery-state-65', service_worker)
self.assertIn('/console-v2.js?v=delivery-state-62', service_worker)
self.assertIn('/assistant-chat.css?v=codex-assistant-22', service_worker)
self.assertIn('/assistant-chat.js?v=codex-assistant-20', service_worker)
self.assertIn('/energy-monitor.css?v=energy-monitor-4', service_worker)
self.assertIn('/energy-monitor.js?v=energy-monitor-4', service_worker)
self.assertIn('"/app.js?v=dual-seat-126"', service_worker)
self.assertIn('"/speech-input.js?v=sensevoice-12"', service_worker)
self.assertIn('"/speech-input.css?v=sensevoice-7"', service_worker)
self.assertIn('"/speech-audio-worklet.js?v=sensevoice-5"', service_worker)
self.assertIn('fetch("/api/speech/preview"', speech_script)
self.assertIn('fetch("/api/speech/transcribe"', speech_script)
self.assertIn('"X-Speech-Utterance-Id"', speech_script)
self.assertIn("result.commands", speech_script)
self.assertIn('"X-Speech-Preview-Sequence"', speech_script)
self.assertIn(
'"X-Speech-Wake-Word-Required": inputMode === "wake" ? "1" : "0"',
speech_script,
)
self.assertIn("result.assistant_text", speech_script)
self.assertIn("AudioWorkletNode", speech_script)
self.assertIn('registerProcessor("speech-capture-processor"', speech_worklet)
self.assertIn("navigator.mediaDevices.getUserMedia", speech_script)
self.assertIn("PREFERRED_MICROPHONE", speech_script)
self.assertIn("ENDPOINT_SILENCE_MS", speech_script)
self.assertIn("speechExecutionEligible", speech_script)
self.assertIn("speechSemanticStatus", speech_script)
self.assertIn('"recognized_unsupported"', speech_script)
self.assertIn('"incomplete"', speech_script)
self.assertNotIn("input.dataset.speechExecutionEligible =", speech_script)
self.assertIn("lastAppliedPreviewSequence", speech_script)
self.assertIn("result.stale === true", speech_script)
self.assertIn("wakeWord.detected === true", speech_script)
self.assertIn("wakeWord.remainder ?? result.live_text", speech_script)
self.assertIn("status.wake_word.primary", speech_script)
self.assertIn("speechExecutionEligible", app_script)
self.assertIn("speech_utterance_id", app_script)
self.assertIn("speechUtteranceId", speech_script)
self.assertIn("result.peripheral_routing?.execution_eligible", speech_script)
self.assertIn("result.routing_text", speech_script)
self.assertIn('const DEFAULT_WAKE_WORD = "你好，双十一"', speech_script)
self.assertIn('replaceInput(liveText, "partial")', speech_script)
self.assertIn('speech_semantic: "语音语义"', console_script)
self.assertIn("语音意图已识别，但当前车辆能力不支持执行", app_script)
self.assertIn("语音控制参数不完整，请补充后再执行", app_script)
@@ -1015,27 +1061,26 @@ operations:
self.assertIn('data-active-seat="front-right"', html)
self.assertNotIn('placeholder="请输入服务端配置令牌"', html)
self.assertIn('data-callout="backrest_recline"', html)
self.assertIn('data-callout="legrest_extension"', html)
self.assertIn('data-callout="backrest_included_angle"', html)
self.assertIn('data-callout="legrest_included_angle"', html)
self.assertNotIn('class="twin-callout-layer"', html)
self.assertNotIn('class="twin-callout', html)
self.assertNotIn('data-callout=', html)
self.assertIn('data-twin-function="ventilation"', html)
self.assertIn('data-twin-function="heating"', html)
self.assertIn('class="twin-level-meter"', html)
self.assertIn('id="presetIndicator"', html)
self.assertIn('id="vehicleSelect"', html)
self.assertIn('id="saveModeToggle"', html)
self.assertIn("根据指令创建", html)
self.assertIn('aria-label="快捷功能模式"', html)
self.assertIn("AI 新建", html)
self.assertIn('aria-label="快捷功能模式，可横向滚动"', html)
self.assertIn('id="modeRailContent"', html)
self.assertIn('aria-label="功能模式列表"', html)
self.assertNotIn('id="modeRailPrev"', html)
self.assertNotIn('id="modeRailNext"', html)
self.assertLess(html.index('id="saveModeToggle"'), html.index('id="modeRailContent"'))
self.assertGreater(html.index('id="saveModeToggle"'), html.index('id="modeRailContent"'))
self.assertIn('id="modeGalleryToggle"', html)
self.assertIn('id="modeGallery"', html)
self.assertIn('id="modeGalleryGroups"', html)
self.assertIn("自定义模式", html)
self.assertIn("管理功能模式", html)
self.assertIn('id="modeEditorBackdrop"', html)
self.assertIn('id="modeEditorName"', html)
self.assertIn('id="modeEditorInstruction"', html)
@@ -1064,11 +1109,21 @@ operations:
self.assertNotIn("function requestCommandAndWait", app_script)
self.assertIn("已永久保存到模式卡片", app_script)
self.assertLess(
app_script.index('id: "personal"'),
app_script.index('id: "frequent"'),
app_script.index('id: "static"'),
app_script.index('id: "dynamic"'),
)
gallery_groups = app_script[
app_script.index("const MODE_GALLERY_GROUPS")
: app_script.index("const MODE_DOMAIN_LABELS")
]
self.assertNotIn('id: "personal"', gallery_groups)
self.assertNotIn('id: "frequent"', gallery_groups)
self.assertNotIn('id: "comfort"', gallery_groups)
self.assertIn("function isDynamicModeProfile", app_script)
self.assertIn('if (group.id === "static") grid.append(createModeGalleryAddCard())', app_script)
self.assertIn('button.dataset.modeOrigin === "user"', app_script)
self.assertIn('userDivider.dataset.modeDivider = "user"', app_script)
self.assertIn('appendGroupLabel("我的模式", "user")', app_script)
self.assertIn('appendDivider("user")', app_script)
self.assertIn('deleteUserModeProfile(deleteButton.dataset.modeDelete)', app_script)
self.assertIn('method: "DELETE"', app_script)
self.assertIn('deleteButton.dataset.modeDelete = mode.id', app_script)
@@ -1182,6 +1237,16 @@ operations:
"zero_gravity",
{mode["id"] for mode in runtime["modes"]},
)
modes_by_id = {mode["id"]: mode for mode in runtime["modes"]}
self.assertEqual(modes_by_id["rocking_horse"]["behavior"], "cyclic")
self.assertEqual(modes_by_id["dynamic_cradle"]["behavior"], "cyclic")
self.assertEqual(modes_by_id["office"]["behavior"], "static")
self.assertTrue(
all(
mode["behavior"] in {"static", "cyclic"}
for mode in runtime["modes"]
)
)
self.assertEqual(
runtime["vehicle"]["seat"]["motors"]["longitudinal"]["range"],
215,
@@ -1234,8 +1299,13 @@ operations:
@classmethod
def transcribe_wav(
cls, content: bytes, *, utterance_id: str | None = None
cls,
content: bytes,
*,
utterance_id: str | None = None,
require_wake_word: bool = False,
) -> dict[str, object]:
self.assertFalse(require_wake_word)
cls.received = content
return {
"utterance_id": utterance_id,
@@ -1385,7 +1455,7 @@ operations:
) as response:
extension_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/app.js?v=dual-seat-114", timeout=3
f"{self.base_url}/app.js?v=dual-seat-126", timeout=3
) as response:
app_script = response.read().decode("utf-8")
@@ -1792,7 +1862,10 @@ operations:
content: bytes,
*,
utterance_id: str | None = None,
require_wake_word: bool = False,
) -> dict[str, object]:
if require_wake_word:
raise AssertionError("manual transcription unexpectedly required wake word")
if content != b"RIFF-speech-peripheral":
raise AssertionError("unexpected audio payload")
return {**understanding, "utterance_id": utterance_id}
@@ -2274,22 +2347,24 @@ operations:
self.extension_temporary_directory.name, "vehicle-light-not-ir.sqlite3"
),
)
extension.sync_peripheral_catalog(
[
{
"id": "toy-001-light-on",
"status": "learned",
"device_id": "toy-001",
"device_name": "玩具 001",
"device_aliases": ["氛围灯"],
"card_uid": "2CC8B103",
"action": "开启闪灯",
"aliases": ["打开"],
"risk": "low",
"slot": 1,
}
]
)
with self.assertRaisesRegex(GatewayError, "车端设备保留名称"):
extension.sync_peripheral_catalog(
[
{
"id": "toy-001-light-on",
"status": "learned",
"device_id": "toy-001",
"device_name": "玩具 001",
"device_aliases": ["氛围灯"],
"card_uid": "2CC8B103",
"action": "开启闪灯",
"aliases": ["打开"],
"risk": "low",
"slot": 1,
}
]
)
self.assertEqual(extension.list_peripheral_capabilities(), [])
SeatDemoHandler.extension = extension
SeatDemoHandler.nfc = SimpleNamespace(
snapshot=lambda: {