优化语音输入/普通文本输入

2026-09-03 13:36:40 +08:00
parent 009d8ca73e
commit 32d179143f
26 changed files with 2318 additions and 53 deletions
@@ -624,7 +624,7 @@ operations:
        ) as response:
            gesture_script = response.read().decode("utf-8")
        with self.opener.open(
-            f"{self.base_url}/console-v2.js?v=delivery-state-54", timeout=3
+            f"{self.base_url}/console-v2.js?v=delivery-state-55", timeout=3
        ) as response:
            console_script = response.read().decode("utf-8")
        with self.opener.open(
@@ -632,7 +632,7 @@ operations:
        ) as response:
            console_styles = response.read().decode("utf-8")
        with self.opener.open(
-            f"{self.base_url}/speech-input.js?v=sensevoice-6", timeout=3
+            f"{self.base_url}/speech-input.js?v=sensevoice-7", timeout=3
        ) as response:
            speech_script = response.read().decode("utf-8")
        with self.opener.open(
@@ -685,9 +685,9 @@ operations:
        self.assertIn('href="/styles.css?v=dual-seat-21"', html)
        self.assertIn('src="/gesture-control.js?v=android-pad-9"', html)
        self.assertNotIn('class="vehicle-system-hud"', html)
-        self.assertIn('src="/console-v2.js?v=delivery-state-54"', html)
+        self.assertIn('src="/console-v2.js?v=delivery-state-55"', html)
        self.assertIn('href="/console-v2.css?v=delivery-state-50"', html)
-        self.assertIn('src="/speech-input.js?v=sensevoice-6"', html)
+        self.assertIn('src="/speech-input.js?v=sensevoice-7"', html)
        self.assertIn('href="/speech-input.css?v=sensevoice-5"', html)
        self.assertIn('id="speechDictationButton"', html)
        self.assertIn('class="speech-waveform"', html)
@@ -959,9 +959,9 @@ operations:
        self.assertIn("CAN 指令已提交，等待车辆状态确认", console_script)
        self.assertIn('canvas.addEventListener("webglcontextlost"', app_script)
        self.assertIn('"cognitive-agent:module-change"', app_script)
-        self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v227"', service_worker)
+        self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v228"', service_worker)
        self.assertIn('"/app.js?v=dual-seat-114"', service_worker)
-        self.assertIn('"/speech-input.js?v=sensevoice-6"', service_worker)
+        self.assertIn('"/speech-input.js?v=sensevoice-7"', service_worker)
        self.assertIn('"/speech-input.css?v=sensevoice-5"', service_worker)
        self.assertIn('"/speech-audio-worklet.js?v=sensevoice-4"', service_worker)
        self.assertIn('fetch("/api/speech/transcribe"', speech_script)
@@ -979,6 +979,9 @@ operations:
        self.assertIn("speechExecutionEligible", app_script)
        self.assertIn("speech_utterance_id", app_script)
        self.assertIn("speechUtteranceId", speech_script)
+        self.assertIn("result.peripheral_routing?.execution_eligible", speech_script)
+        self.assertIn("result.routing_text", speech_script)
+        self.assertIn('speech_semantic: "语音语义"', console_script)
        self.assertIn("语音意图已识别，但当前车辆能力不支持执行", app_script)
        self.assertIn("语音控制参数不完整，请补充后再执行", app_script)
        self.assertIn(".speech-dictation.is-listening", speech_styles)
@@ -1741,6 +1744,143 @@ operations:
        self.assertEqual(operation["domain"], "self_extension")
        self.assertTrue(operation["telemetry"]["gateway_acknowledged"])

+    def test_speech_routes_exact_asr_text_to_learned_peripheral_and_ui_config(
+        self,
+    ) -> None:
+        original_extension = SeatDemoHandler.extension
+        original_nfc = SeatDemoHandler.nfc
+        original_speech = SeatDemoHandler.speech
+        extension = SelfExtensionService(
+            SimulationGateway(),
+            catalog_db_path=Path(
+                self.extension_temporary_directory.name,
+                "speech-nfc-route.sqlite3",
+            ),
+        )
+        extension.sync_peripheral_catalog(
+            [
+                {
+                    "id": "cap-third-party-light-on",
+                    "status": "learned",
+                    "enabled": True,
+                    "device_id": "third-party-light",
+                    "device_name": "第三方氛围灯",
+                    "object_names": ["第三方氛围灯"],
+                    "card_uid": "1A53DD95",
+                    "action": "开启",
+                    "action_names": ["打开", "开启"],
+                    "slot": 9,
+                }
+            ]
+        )
+        SeatDemoHandler.extension = extension
+        SeatDemoHandler.nfc = SimpleNamespace(
+            snapshot=lambda: {
+                "revision": 1,
+                "reader": {"online": True},
+                "card": {"action": "present", "uid": "1A53DD95"},
+            }
+        )
+        understanding = UnderstandingTranscriptProcessor(REPOSITORY_ROOT).process(
+            "打开第三方氛围灯"
+        ).to_dict()
+        understanding["utterance_id"] = "e2e-speech-peripheral"
+
+        class SpeechStub:
+            @staticmethod
+            def transcribe_wav(
+                content: bytes,
+                *,
+                utterance_id: str | None = None,
+            ) -> dict[str, object]:
+                if content != b"RIFF-speech-peripheral":
+                    raise AssertionError("unexpected audio payload")
+                return {**understanding, "utterance_id": utterance_id}
+
+            @staticmethod
+            def cached_result(utterance_id: str) -> dict[str, object]:
+                if utterance_id != "e2e-speech-peripheral":
+                    raise AssertionError("unexpected speech utterance id")
+                return dict(understanding)
+
+        SeatDemoHandler.speech = SpeechStub()
+        try:
+            transcription_request = urllib.request.Request(
+                f"{self.base_url}/api/speech/transcribe",
+                method="POST",
+                headers={
+                    "Content-Type": "audio/wav",
+                    "X-Speech-Utterance-Id": "e2e-speech-peripheral",
+                },
+                data=b"RIFF-speech-peripheral",
+            )
+            with self.opener.open(transcription_request, timeout=3) as response:
+                transcription = json.loads(response.read().decode("utf-8"))
+
+            wrong_target_request = urllib.request.Request(
+                f"{self.base_url}/api/requests",
+                method="POST",
+                headers={"Content-Type": "application/json"},
+                data=json.dumps(
+                    {
+                        "utterance": understanding["clean_text"],
+                        "speech_utterance_id": "e2e-speech-peripheral",
+                        "provider": "local",
+                        "conversation_id": "conv-e2e-speech-wrong-target",
+                        "execution_mode": "dry_run",
+                    },
+                    ensure_ascii=False,
+                ).encode("utf-8"),
+            )
+            with self.assertRaises(urllib.error.HTTPError) as rejected:
+                self.opener.open(wrong_target_request, timeout=3)
+            self.assertEqual(rejected.exception.code, 409)
+            rejected_payload = json.loads(
+                rejected.exception.read().decode("utf-8")
+            )
+            self.assertEqual(rejected_payload["error"], "speech_text_changed")
+
+            control_request = urllib.request.Request(
+                f"{self.base_url}/api/requests",
+                method="POST",
+                headers={"Content-Type": "application/json"},
+                data=json.dumps(
+                    {
+                        "utterance": transcription["routing_text"],
+                        "speech_utterance_id": "e2e-speech-peripheral",
+                        "provider": "openai",
+                        "conversation_id": "conv-e2e-speech-peripheral",
+                        "execution_mode": "dry_run",
+                        "client_type": "web_console",
+                    },
+                    ensure_ascii=False,
+                ).encode("utf-8"),
+            )
+            with self.opener.open(control_request, timeout=3) as response:
+                result = json.loads(response.read().decode("utf-8"))
+        finally:
+            SeatDemoHandler.extension = original_extension
+            SeatDemoHandler.nfc = original_nfc
+            SeatDemoHandler.speech = original_speech
+
+        self.assertEqual(transcription["routing_text"], "打开第三方氛围灯")
+        self.assertTrue(
+            transcription["peripheral_routing"]["execution_eligible"]
+        )
+        self.assertEqual(result["status"], "succeeded")
+        self.assertEqual(result["orchestration_route"], "nfc_peripheral")
+        self.assertEqual(set(result["function_configs"]), {"self_extension"})
+        operation = result["function_configs"]["self_extension"]["spec"][
+            "operations"
+        ][0]
+        self.assertEqual(operation["capability"], "infrared.emit")
+        self.assertEqual(operation["parameters"]["slot"], 9)
+        self.assertEqual(operation["parse_source"], "local_rule")
+        self.assertEqual(
+            result["speech_understanding"]["routing"],
+            "validated_peripheral_text",
+        )
+
    def test_main_request_combines_explicit_ir_clause_with_existing_can_parser(self) -> None:
        original_extension = SeatDemoHandler.extension
        original_nfc = SeatDemoHandler.nfc