vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate Chinese speech understanding and capability routing

Browse Source
This commit is contained in:
vee
2026-08-28 15:19:44 +08:00
parent 2529b4896a
commit aa12135b22
39 changed files with 6864 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/test_seat_web_api.py
+202 -7
View File
@@ -13,15 +13,18 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from cognitive_agent.auth import AuthConfig, AuthRuntime, hash_password
from cognitive_agent.application.seat_service import REPOSITORY_ROOT, SeatDemoService
from cognitive_agent.application.mode_profile import ModeProfileRegistry
from cognitive_agent.application.seat_service import REPOSITORY_ROOT, SeatDemoService
from cognitive_agent.application.self_extension import (
SelfExtensionService,
SimulationGateway,
)
from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError
from cognitive_agent.auth import AuthConfig, AuthRuntime, hash_password
from cognitive_agent.infrastructure.can.access import HardwareAccessController
from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError
from cognitive_agent.infrastructure.speech.understanding import (
UnderstandingTranscriptProcessor,
)
from cognitive_agent.server import SeatDemoHandler
@@ -332,7 +335,7 @@ operations:
def test_static_console_and_runtime_endpoint(self) -> None:
with self.opener.open(f"{self.base_url}/", timeout=3) as response:
html = response.read().decode("utf-8")
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-103", timeout=3) as response:
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-104", timeout=3) as response:
app_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/gesture-control.js?v=android-pad-9", timeout=3
@@ -346,6 +349,18 @@ operations:
f"{self.base_url}/console-v2.css?v=sync-layout-41", timeout=3
) as response:
console_styles = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-input.js?v=sensevoice-6", timeout=3
) as response:
speech_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-input.css?v=sensevoice-4", timeout=3
) as response:
speech_styles = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-audio-worklet.js?v=sensevoice-4", timeout=3
) as response:
speech_worklet = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/self-extension.js?v=nfc-toy-33", timeout=3
) as response:
@@ -368,6 +383,8 @@ operations:
nfc_status = json.loads(response.read().decode("utf-8"))
with self.opener.open(f"{self.base_url}/api/runtime", timeout=3) as response:
runtime = json.loads(response.read().decode("utf-8"))
with self.opener.open(f"{self.base_url}/api/speech/status", timeout=3) as response:
speech_status = json.loads(response.read().decode("utf-8"))
self.assertIn("511工程-智能座舱", html)
self.assertIn('id="seatTwinCanvas"', html)
@@ -382,12 +399,16 @@ operations:
self.assertIn("CMD 0x431 · FB 0x41A · CH1", html)
self.assertIn("CMD 0x431 · FB 0x388 · CH1", html)
self.assertNotIn("透明车体 · 实体设备", html)
self.assertIn('src="/app.js?v=dual-seat-103"', html)
self.assertIn('src="/app.js?v=dual-seat-104"', html)
self.assertIn('href="/styles.css?v=dual-seat-16"', html)
self.assertIn('src="/gesture-control.js?v=android-pad-9"', html)
self.assertNotIn('class="vehicle-system-hud"', html)
self.assertIn('src="/console-v2.js?v=sync-layout-43"', html)
self.assertIn('href="/console-v2.css?v=sync-layout-41"', html)
self.assertIn('src="/speech-input.js?v=sensevoice-6"', html)
self.assertIn('href="/speech-input.css?v=sensevoice-4"', html)
self.assertIn('id="speechDictationButton"', html)
self.assertIn('class="speech-waveform"', html)
self.assertIn('<body class="console-v2">', html)
self.assertIn('id="v2BootScreen"', html)
self.assertIn('正在加载车机控制与实时状态', html)
@@ -622,7 +643,29 @@ operations:
self.assertIn('client_type: "web_console"', app_script)
self.assertIn('canvas.addEventListener("webglcontextlost"', app_script)
self.assertIn('"cognitive-agent:module-change"', app_script)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v210"', service_worker)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v216"', service_worker)
self.assertIn('"/speech-input.js?v=sensevoice-6"', service_worker)
self.assertIn('"/speech-input.css?v=sensevoice-4"', service_worker)
self.assertIn('"/speech-audio-worklet.js?v=sensevoice-4"', service_worker)
self.assertIn('fetch("/api/speech/transcribe"', speech_script)
self.assertIn('"X-Speech-Utterance-Id"', speech_script)
self.assertIn("result.commands", speech_script)
self.assertIn("AudioWorkletNode", speech_script)
self.assertIn('registerProcessor("speech-capture-processor"', speech_worklet)
self.assertIn("navigator.mediaDevices.getUserMedia", speech_script)
self.assertIn("PREFERRED_MICROPHONE", speech_script)
self.assertIn("ENDPOINT_SILENCE_MS", speech_script)
self.assertIn("speechExecutionEligible", speech_script)
self.assertIn("speechSemanticStatus", speech_script)
self.assertIn('"recognized_unsupported"', speech_script)
self.assertIn('"incomplete"', speech_script)
self.assertIn("speechExecutionEligible", app_script)
self.assertIn("speech_utterance_id", app_script)
self.assertIn("speechUtteranceId", speech_script)
self.assertIn("语音意图已识别，但当前车辆能力不支持执行", app_script)
self.assertIn("语音控制参数不完整，请补充后再执行", app_script)
self.assertIn(".speech-dictation.is-listening", speech_styles)
self.assertEqual(speech_status["model"], "SenseVoiceSmall")
self.assertIn('"/assets/pets/self-extension-cat.png?v=3"', service_worker)
self.assertIn(".then(() => self.skipWaiting())", service_worker)
self.assertNotIn("const cached = await caches.match(request)", service_worker)
@@ -649,6 +692,7 @@ operations:
self.assertIn("BENCH BUSY", app_script)
self.assertIn("FBXLoader.js", html)
self.assertIn('data-seat-layout="five-seat"', html)
self.assertIn('data-active-seat="front-right"', html)
self.assertNotIn('placeholder="请输入服务端配置令牌"', html)
self.assertIn('data-callout="backrest_recline"', html)
@@ -856,13 +900,164 @@ operations:
)
)
def test_speech_transcription_endpoint_forwards_wav_without_persisting_it(self) -> None:
class SpeechStub:
received = b""
@classmethod
def transcribe_wav(
cls, content: bytes, *, utterance_id: str | None = None
) -> dict[str, object]:
cls.received = content
return {
"utterance_id": utterance_id,
"text": "打开空调到二十四度。",
"language": "zh",
"duration_ms": 800,
"inference_ms": 92,
"model": "SenseVoiceSmall",
}
original_speech = SeatDemoHandler.speech
SeatDemoHandler.speech = SpeechStub()
try:
wav_content = b"RIFF-speech-test"
request = urllib.request.Request(
f"{self.base_url}/api/speech/transcribe",
method="POST",
headers={
"Content-Type": "audio/wav",
"Accept": "application/json",
"X-Speech-Utterance-Id": "e2e-speech-1",
},
data=wav_content,
)
with self.opener.open(request, timeout=3) as response:
payload = json.loads(response.read().decode("utf-8"))
permissions_policy = response.headers["Permissions-Policy"]
finally:
SeatDemoHandler.speech = original_speech
self.assertEqual(SpeechStub.received, wav_content)
self.assertEqual(payload["text"], "打开空调到二十四度。")
self.assertEqual(payload["utterance_id"], "e2e-speech-1")
self.assertIn("microphone=(self)", permissions_policy)
def test_speech_utterance_id_preserves_structured_slots_into_intent_ir(self) -> None:
understanding = UnderstandingTranscriptProcessor(REPOSITORY_ROOT).process(
"把副驾腿托向上调整到最大位置"
).to_dict()
understanding["utterance_id"] = "e2e-speech-bridge"
class SpeechStub:
@staticmethod
def cached_result(utterance_id: str) -> dict[str, object]:
if utterance_id != "e2e-speech-bridge":
raise AssertionError("unexpected speech utterance id")
return understanding
original_speech = SeatDemoHandler.speech
SeatDemoHandler.speech = SpeechStub()
try:
request = urllib.request.Request(
f"{self.base_url}/api/requests",
method="POST",
headers={"Content-Type": "application/json", "Accept": "application/json"},
data=json.dumps(
{
"utterance": understanding["clean_text"],
"speech_utterance_id": "e2e-speech-bridge",
"client_type": "web_console",
"provider": "openai",
"conversation_id": "conv-e2e-speech-bridge",
"execution_mode": "dry_run",
},
ensure_ascii=False,
).encode("utf-8"),
)
with self.opener.open(request, timeout=3) as response:
payload = json.loads(response.read().decode("utf-8"))
finally:
SeatDemoHandler.speech = original_speech
self.assertEqual(payload["status"], "succeeded")
self.assertEqual(payload["provider_used"], "speech_semantic")
self.assertEqual(
payload["speech_understanding"]["raw_text"],
"把副驾腿托向上调整到最大位置",
)
action = payload["intent_ir"]["spec"]["actions"][0]
self.assertEqual(action["capability"], "seat.set_legrest_height")
self.assertEqual(action["target"]["instance"], "passenger")
self.assertEqual(action["parameters"], {"position_pct": 100})
self.assertEqual(action["provenance"]["parser"], "speech_semantic")
def test_roof_shade_speech_crosses_server_bridge_as_native_can(self) -> None:
cases = (
("把遮阳帘打开", {"motion": "open"}),
("遮阳帘打开一半", {"motion": "half"}),
)
for index, (utterance, parameters) in enumerate(cases):
with self.subTest(utterance=utterance):
utterance_id = f"e2e-roof-shade-{index}"
understanding = UnderstandingTranscriptProcessor(
REPOSITORY_ROOT
).process(utterance).to_dict()
understanding["utterance_id"] = utterance_id
def cached_result(
requested_id: str,
*,
expected_id: str = utterance_id,
result: dict[str, object] = understanding,
) -> dict[str, object]:
if requested_id != expected_id:
raise AssertionError("unexpected speech utterance id")
return result
original_speech = SeatDemoHandler.speech
SeatDemoHandler.speech = SimpleNamespace(cached_result=cached_result)
try:
request = urllib.request.Request(
f"{self.base_url}/api/requests",
method="POST",
headers={
"Content-Type": "application/json",
"Accept": "application/json",
},
data=json.dumps(
{
"utterance": understanding["clean_text"],
"speech_utterance_id": utterance_id,
"client_type": "web_console",
"provider": "openai",
"conversation_id": f"conv-e2e-roof-shade-{index}",
"execution_mode": "dry_run",
},
ensure_ascii=False,
).encode("utf-8"),
)
with self.opener.open(request, timeout=3) as response:
payload = json.loads(response.read().decode("utf-8"))
finally:
SeatDemoHandler.speech = original_speech
self.assertEqual(payload["status"], "succeeded")
self.assertEqual(payload["provider_used"], "speech_semantic")
self.assertEqual(payload["deferred_requests"], [])
action = payload["intent_ir"]["spec"]["actions"][0]
self.assertEqual(action["capability"], "roof.set_shade_motion")
self.assertEqual(action["target"]["zone"], "vehicle")
self.assertEqual(action["parameters"], parameters)
self.assertEqual(action["provenance"]["parser"], "speech_semantic")
def test_self_extension_vehicle_success_contract_is_strict(self) -> None:
with self.opener.open(
f"{self.base_url}/self-extension.js?v=extension-44", timeout=3
) as response:
extension_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/app.js?v=dual-seat-103", timeout=3
f"{self.base_url}/app.js?v=dual-seat-104", timeout=3
) as response:
app_script = response.read().decode("utf-8")