vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: integrate remote speech capabilities into feature/N7Web

Browse Source
This commit is contained in:
minghanchen
2026-09-01 17:01:50 +08:00
parent cbf230182d aa12135b22
commit 00c7d6b874
39 changed files with 6869 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/test_seat_web_api.py
+199 -4
View File
@@ -13,16 +13,19 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from cognitive_agent.auth import AuthConfig, AuthRuntime, hash_password
from cognitive_agent.application.seat_service import REPOSITORY_ROOT, SeatDemoService
from cognitive_agent.application.mode_profile import ModeProfileRegistry
from cognitive_agent.application.seat_service import REPOSITORY_ROOT, SeatDemoService
from cognitive_agent.application.self_extension import (
SelfExtensionService,
SimulationGateway,
)
from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError
from cognitive_agent.auth import AuthConfig, AuthRuntime, hash_password
from cognitive_agent.infrastructure.can.access import HardwareAccessController
from cognitive_agent.infrastructure.can.gateway import CanGatewayError
from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError
from cognitive_agent.infrastructure.speech.understanding import (
UnderstandingTranscriptProcessor,
)
from cognitive_agent.server import SeatDemoHandler
@@ -599,6 +602,18 @@ operations:
f"{self.base_url}/console-v2.css?v=delivery-state-43", timeout=3
) as response:
console_styles = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-input.js?v=sensevoice-6", timeout=3
) as response:
speech_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-input.css?v=sensevoice-4", timeout=3
) as response:
speech_styles = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-audio-worklet.js?v=sensevoice-4", timeout=3
) as response:
speech_worklet = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/self-extension.js?v=gscu-tcp-40", timeout=3
) as response:
@@ -621,6 +636,8 @@ operations:
nfc_status = json.loads(response.read().decode("utf-8"))
with self.opener.open(f"{self.base_url}/api/runtime", timeout=3) as response:
runtime = json.loads(response.read().decode("utf-8"))
with self.opener.open(f"{self.base_url}/api/speech/status", timeout=3) as response:
speech_status = json.loads(response.read().decode("utf-8"))
self.assertIn("511工程-智能座舱", html)
self.assertIn('id="seatTwinCanvas"', html)
@@ -641,6 +658,10 @@ operations:
self.assertNotIn('class="vehicle-system-hud"', html)
self.assertIn('src="/console-v2.js?v=delivery-state-50"', html)
self.assertIn('href="/console-v2.css?v=delivery-state-43"', html)
self.assertIn('src="/speech-input.js?v=sensevoice-6"', html)
self.assertIn('href="/speech-input.css?v=sensevoice-4"', html)
self.assertIn('id="speechDictationButton"', html)
self.assertIn('class="speech-waveform"', html)
self.assertIn('<body class="console-v2">', html)
self.assertIn('id="v2BootScreen"', html)
self.assertIn('正在加载车机控制与实时状态', html)
@@ -891,8 +912,30 @@ operations:
self.assertIn("CAN 指令已提交，等待车辆状态确认", console_script)
self.assertIn('canvas.addEventListener("webglcontextlost"', app_script)
self.assertIn('"cognitive-agent:module-change"', app_script)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v220"', service_worker)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v221"', service_worker)
self.assertIn('"/app.js?v=dual-seat-109"', service_worker)
self.assertIn('"/speech-input.js?v=sensevoice-6"', service_worker)
self.assertIn('"/speech-input.css?v=sensevoice-4"', service_worker)
self.assertIn('"/speech-audio-worklet.js?v=sensevoice-4"', service_worker)
self.assertIn('fetch("/api/speech/transcribe"', speech_script)
self.assertIn('"X-Speech-Utterance-Id"', speech_script)
self.assertIn("result.commands", speech_script)
self.assertIn("AudioWorkletNode", speech_script)
self.assertIn('registerProcessor("speech-capture-processor"', speech_worklet)
self.assertIn("navigator.mediaDevices.getUserMedia", speech_script)
self.assertIn("PREFERRED_MICROPHONE", speech_script)
self.assertIn("ENDPOINT_SILENCE_MS", speech_script)
self.assertIn("speechExecutionEligible", speech_script)
self.assertIn("speechSemanticStatus", speech_script)
self.assertIn('"recognized_unsupported"', speech_script)
self.assertIn('"incomplete"', speech_script)
self.assertIn("speechExecutionEligible", app_script)
self.assertIn("speech_utterance_id", app_script)
self.assertIn("speechUtteranceId", speech_script)
self.assertIn("语音意图已识别，但当前车辆能力不支持执行", app_script)
self.assertIn("语音控制参数不完整，请补充后再执行", app_script)
self.assertIn(".speech-dictation.is-listening", speech_styles)
self.assertEqual(speech_status["model"], "SenseVoiceSmall")
self.assertIn('"/assets/pets/self-extension-cat.png?v=3"', service_worker)
self.assertIn(".then(() => self.skipWaiting())", service_worker)
self.assertNotIn("const cached = await caches.match(request)", service_worker)
@@ -919,6 +962,7 @@ operations:
self.assertIn("BENCH BUSY", app_script)
self.assertIn("FBXLoader.js", html)
self.assertIn('data-seat-layout="five-seat"', html)
self.assertIn('data-active-seat="front-right"', html)
self.assertNotIn('placeholder="请输入服务端配置令牌"', html)
self.assertIn('data-callout="backrest_recline"', html)
@@ -1126,6 +1170,157 @@ operations:
)
)
def test_speech_transcription_endpoint_forwards_wav_without_persisting_it(self) -> None:
class SpeechStub:
received = b""
@classmethod
def transcribe_wav(
cls, content: bytes, *, utterance_id: str | None = None
) -> dict[str, object]:
cls.received = content
return {
"utterance_id": utterance_id,
"text": "打开空调到二十四度。",
"language": "zh",
"duration_ms": 800,
"inference_ms": 92,
"model": "SenseVoiceSmall",
}
original_speech = SeatDemoHandler.speech
SeatDemoHandler.speech = SpeechStub()
try:
wav_content = b"RIFF-speech-test"
request = urllib.request.Request(
f"{self.base_url}/api/speech/transcribe",
method="POST",
headers={
"Content-Type": "audio/wav",
"Accept": "application/json",
"X-Speech-Utterance-Id": "e2e-speech-1",
},
data=wav_content,
)
with self.opener.open(request, timeout=3) as response:
payload = json.loads(response.read().decode("utf-8"))
permissions_policy = response.headers["Permissions-Policy"]
finally:
SeatDemoHandler.speech = original_speech
self.assertEqual(SpeechStub.received, wav_content)
self.assertEqual(payload["text"], "打开空调到二十四度。")
self.assertEqual(payload["utterance_id"], "e2e-speech-1")
self.assertIn("microphone=(self)", permissions_policy)
def test_speech_utterance_id_preserves_structured_slots_into_intent_ir(self) -> None:
understanding = UnderstandingTranscriptProcessor(REPOSITORY_ROOT).process(
"把副驾腿托向上调整到最大位置"
).to_dict()
understanding["utterance_id"] = "e2e-speech-bridge"
class SpeechStub:
@staticmethod
def cached_result(utterance_id: str) -> dict[str, object]:
if utterance_id != "e2e-speech-bridge":
raise AssertionError("unexpected speech utterance id")
return understanding
original_speech = SeatDemoHandler.speech
SeatDemoHandler.speech = SpeechStub()
try:
request = urllib.request.Request(
f"{self.base_url}/api/requests",
method="POST",
headers={"Content-Type": "application/json", "Accept": "application/json"},
data=json.dumps(
{
"utterance": understanding["clean_text"],
"speech_utterance_id": "e2e-speech-bridge",
"client_type": "web_console",
"provider": "openai",
"conversation_id": "conv-e2e-speech-bridge",
"execution_mode": "dry_run",
},
ensure_ascii=False,
).encode("utf-8"),
)
with self.opener.open(request, timeout=3) as response:
payload = json.loads(response.read().decode("utf-8"))
finally:
SeatDemoHandler.speech = original_speech
self.assertEqual(payload["status"], "succeeded")
self.assertEqual(payload["provider_used"], "speech_semantic")
self.assertEqual(
payload["speech_understanding"]["raw_text"],
"把副驾腿托向上调整到最大位置",
)
action = payload["intent_ir"]["spec"]["actions"][0]
self.assertEqual(action["capability"], "seat.set_legrest_height")
self.assertEqual(action["target"]["instance"], "passenger")
self.assertEqual(action["parameters"], {"position_pct": 100})
self.assertEqual(action["provenance"]["parser"], "speech_semantic")
def test_roof_shade_speech_crosses_server_bridge_as_native_can(self) -> None:
cases = (
("把遮阳帘打开", {"motion": "open"}),
("遮阳帘打开一半", {"motion": "half"}),
)
for index, (utterance, parameters) in enumerate(cases):
with self.subTest(utterance=utterance):
utterance_id = f"e2e-roof-shade-{index}"
understanding = UnderstandingTranscriptProcessor(
REPOSITORY_ROOT
).process(utterance).to_dict()
understanding["utterance_id"] = utterance_id
def cached_result(
requested_id: str,
*,
expected_id: str = utterance_id,
result: dict[str, object] = understanding,
) -> dict[str, object]:
if requested_id != expected_id:
raise AssertionError("unexpected speech utterance id")
return result
original_speech = SeatDemoHandler.speech
SeatDemoHandler.speech = SimpleNamespace(cached_result=cached_result)
try:
request = urllib.request.Request(
f"{self.base_url}/api/requests",
method="POST",
headers={
"Content-Type": "application/json",
"Accept": "application/json",
},
data=json.dumps(
{
"utterance": understanding["clean_text"],
"speech_utterance_id": utterance_id,
"client_type": "web_console",
"provider": "openai",
"conversation_id": f"conv-e2e-roof-shade-{index}",
"execution_mode": "dry_run",
},
ensure_ascii=False,
).encode("utf-8"),
)
with self.opener.open(request, timeout=3) as response:
payload = json.loads(response.read().decode("utf-8"))
finally:
SeatDemoHandler.speech = original_speech
self.assertEqual(payload["status"], "succeeded")
self.assertEqual(payload["provider_used"], "speech_semantic")
self.assertEqual(payload["deferred_requests"], [])
action = payload["intent_ir"]["spec"]["actions"][0]
self.assertEqual(action["capability"], "roof.set_shade_motion")
self.assertEqual(action["target"]["zone"], "vehicle")
self.assertEqual(action["parameters"], parameters)
self.assertEqual(action["provenance"]["parser"], "speech_semantic")
def test_self_extension_vehicle_success_contract_is_strict(self) -> None:
with self.opener.open(
f"{self.base_url}/self-extension.js?v=extension-44", timeout=3