feat: integrate Chinese speech understanding and capability routing

src/cognitive_agent/application/capability_registry.py

@@ -2,23 +2,66 @@
 
 from __future__ import annotations
 
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
 import yaml
 
 
+@dataclass(frozen=True)
+class CapabilityDefinition:
+    """One capability together with its authoritative catalog provenance."""
+
+    name: str
+    domain: str
+    availability: str
+    catalog_id: str
+    catalog_path: Path
+    executor_binding: str
+    spec: dict[str, Any]
+
+    @property
+    def target_devices(self) -> tuple[str, ...]:
+        return tuple(str(value) for value in self.spec.get("target_devices", ()))
+
+    @property
+    def target_scopes(self) -> tuple[str, ...]:
+        return tuple(str(value) for value in self.spec.get("target_scopes", ()))
+
+    @property
+    def parameters(self) -> tuple[dict[str, Any], ...]:
+        return tuple(dict(value) for value in self.spec.get("parameters", ()))
+
+
 class CapabilityRegistry:
     """Expose domain catalogs without embedding domain semantics in prompts or code."""
 
     def __init__(self, repository_root: Path, executable_domains: set[str]) -> None:
         self.root = repository_root
         self.executable_domains = set(executable_domains)
+        self._definitions: list[CapabilityDefinition] = []
         self._snapshot = self._load()
 
     def snapshot(self) -> dict[str, Any]:
         return self._snapshot
 
+    def definitions(self, *, domain: str | None = None) -> tuple[CapabilityDefinition, ...]:
+        """Return catalog entries without inventing aliases or execution support."""
+
+        return tuple(
+            definition
+            for definition in self._definitions
+            if domain is None or definition.domain == domain
+        )
+
+    def find(self, name: str) -> tuple[CapabilityDefinition, ...]:
+        """Return every authoritative definition for a canonical capability id."""
+
+        return tuple(
+            definition for definition in self._definitions if definition.name == name
+        )
+
     def _load(self) -> dict[str, Any]:
         domains = []
         catalog_paths = [
@@ -31,16 +74,37 @@ class CapabilityRegistry:
             catalog = yaml.safe_load(catalog_path.read_text(encoding="utf-8"))
             domain = catalog["spec"]["domain"]
             native_can_catalog = catalog_path.parent.name == "native_can_capabilities"
+            availability = (
+                "native_can"
+                if native_can_catalog
+                else "executable"
+                if domain in self.executable_domains
+                else "catalog_only"
+            )
+            executor_binding = (
+                "NativeCanRuntime/NativeCanExecutor"
+                if native_can_catalog
+                else "SeatCompiler/SeatExecutor"
+                if domain == "seat"
+                else f"ConfiguredDomainRuntime[{domain}]"
+            )
+            catalog_id = str(catalog.get("metadata", {}).get("id", catalog_path.stem))
+            self._definitions.extend(
+                CapabilityDefinition(
+                    name=str(capability["name"]),
+                    domain=str(domain),
+                    availability=availability,
+                    catalog_id=catalog_id,
+                    catalog_path=catalog_path,
+                    executor_binding=executor_binding,
+                    spec=dict(capability),
+                )
+                for capability in catalog["spec"]["capabilities"]
+            )
             domains.append(
                 {
                     "domain": domain,
-                    "availability": (
-                        "native_can"
-                        if native_can_catalog
-                        else "executable"
-                        if domain in self.executable_domains
-                        else "catalog_only"
-                    ),
+                    "availability": availability,
                     "capabilities": catalog["spec"]["capabilities"],
                 }
             )

src/cognitive_agent/application/chinese_values.py

@@ -0,0 +1,128 @@
+"""Shared deterministic parsing for Chinese scalar control values.
+
+This module only normalizes values that the user actually spoke. Capability
+validation remains responsible for deciding whether a target accepts the
+result and whether its type/range is executable.
+"""
+
+from __future__ import annotations
+
+import re
+
+_ARABIC_PERCENT = re.compile(r"(-?\d+(?:\.\d+)?)\s*[%％]")
+_FRACTION = re.compile(
+    r"([零〇一二两三四五六七八九十百\d]+)分之"
+    r"([零〇一二两三四五六七八九十百点\d.]+)"
+)
+_SLASH_FRACTION = re.compile(r"(?<!\d)(\d+)\s*/\s*(\d+)(?!\d)")
+_DIGITS = {
+    "零": 0,
+    "〇": 0,
+    "一": 1,
+    "二": 2,
+    "两": 2,
+    "三": 3,
+    "四": 4,
+    "五": 5,
+    "六": 6,
+    "七": 7,
+    "八": 8,
+    "九": 9,
+}
+
+
+def _chinese_integer(token: str) -> int | None:
+    if not token:
+        return 0
+    if token.isdigit():
+        return int(token)
+    if "百" in token:
+        left, right = token.split("百", 1)
+        hundreds = _DIGITS.get(left[-1], 1) if left else 1
+        tail = _chinese_integer(right)
+        return None if tail is None else hundreds * 100 + tail
+    if "十" in token:
+        left, right = token.split("十", 1)
+        tens = _DIGITS.get(left[-1], 1) if left else 1
+        ones = _DIGITS.get(right[-1], 0) if right else 0
+        return tens * 10 + ones
+    if all(char in _DIGITS for char in token):
+        return int("".join(str(_DIGITS[char]) for char in token))
+    return None
+
+
+def _number(token: str) -> int | float | None:
+    try:
+        value = float(token)
+    except ValueError:
+        if "点" in token:
+            whole, fraction = token.split("点", 1)
+            whole_value = _chinese_integer(whole)
+            if (
+                whole_value is None
+                or not fraction
+                or any(char not in _DIGITS for char in fraction)
+            ):
+                return None
+            value = float(
+                f"{whole_value}."
+                + "".join(str(_DIGITS[char]) for char in fraction)
+            )
+        else:
+            integer = _chinese_integer(token)
+            if integer is None:
+                return None
+            value = float(integer)
+    return int(value) if value.is_integer() else value
+
+
+def parse_percentage_value(text: str) -> int | float | None:
+    """Return an explicitly spoken percentage/ratio/extreme position.
+
+    Bare numbers intentionally do not become percentages. Fractions are
+    rounded to the nearest whole percentage because every current continuous
+    vehicle position capability is integer-valued; the resolver still checks
+    the final range and type.
+    """
+
+    match = _ARABIC_PERCENT.search(text)
+    if match:
+        value = float(match.group(1))
+        return int(value) if value.is_integer() else value
+
+    match = _FRACTION.search(text)
+    if match:
+        denominator = _number(match.group(1))
+        numerator = _number(match.group(2))
+        if denominator not in {None, 0} and numerator is not None:
+            return round(float(numerator) * 100 / float(denominator))
+
+    match = _SLASH_FRACTION.search(text)
+    if match and int(match.group(2)) != 0:
+        return round(int(match.group(1)) * 100 / int(match.group(2)))
+
+    if re.search(r"一半|半开|开一半", text):
+        return 50
+
+    extrema = (
+        (
+            (
+                r"最大(?:位置|开度)|最高(?:位置)?|"
+                r"(?:调|调整|升|抬|开)(?:到|至)(?:最大|最高)|到顶|"
+                r"全开|完全打开|打开到底"
+            ),
+            100,
+        ),
+        (
+            (
+                r"最小(?:位置|开度)|最低(?:位置)?|"
+                r"(?:调|调整|降|收|关)(?:到|至)(?:最小|最低)|"
+                r"全关|完全关闭|关闭到底"
+            ),
+            0,
+        ),
+    )
+    for pattern, value in extrema:
+        if re.search(pattern, text):
+            return value
+    return None

src/cognitive_agent/application/native_can_intent.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 import re
 from typing import Any
 
+from cognitive_agent.application.chinese_values import parse_percentage_value
 
 WINDOW_TARGETS = {
     "全部": "all",
@@ -595,8 +596,20 @@ def _parse_chinese_driver_seat_commands(compact: str) -> list[dict[str, Any]]:
         ("加热", "seat.set_driver_heating_native"),
     )
     present_thermal = [item for item in thermal_features if item[0] in compact]
-    if len(present_thermal) == 1:
-        label, capability = present_thermal[0]
+    coordinated_thermal = re.search(
+        r"(?:座椅)?(?:通风|加热)(?:和|与|以及|及|、)"
+        r"(?:打开|开启|启动|关闭|关掉|停止)?"
+        r"(?:座椅)?(?:通风|加热)",
+        compact,
+    )
+    shared_thermal_state: str | None = None
+    if coordinated_thermal is not None:
+        shared_prefix = compact[: coordinated_thermal.start()]
+        if re.search(r"关闭|关掉|停止|取消|不要|别开", shared_prefix):
+            shared_thermal_state = "off"
+        elif re.search(r"打开|开启|启动|启用", shared_prefix):
+            shared_thermal_state = "on"
+    for label, capability in present_thermal:
         thermal_text = _feature_context(compact, label)
         thermal_grade = _grade_alias(thermal_text, include_numeric=False)
         parameters: dict[str, Any] | None = None
@@ -608,9 +621,11 @@ def _parse_chinese_driver_seat_commands(compact: str) -> list[dict[str, Any]]:
             parameters = {"state": "on", "level": 1}
         elif thermal_grade == "low" or re.search(
             r"低速|(?:2|二)(?:档|级)", thermal_text
-        ):
+        ) or re.search(r"打开|开启|启动|启用", thermal_text):
             parameters = {"state": "on", "level": 2}
-        elif re.search(r"打开|开启|启动|启用", thermal_text):
+        elif shared_thermal_state == "off":
+            parameters = {"state": "off"}
+        elif shared_thermal_state == "on":
             parameters = {"state": "on", "level": 2}
         if parameters is not None:
             actions.append(_action(
@@ -850,24 +865,11 @@ def _parse_chinese_window_commands(compact: str) -> list[dict[str, Any]]:
             window_text = f"{window_text}，{clauses[index + 1]}"
 
         targets, explicit_window_target = _window_targets(window_text)
-        position = re.search(
-            r"(?:位置|开度|调整到|调到|设为|设置为|开启到|打开到|开到)"
-            r"(\d{1,3})%",
-            window_text,
-        )
-        half_position = bool(
-            re.search(
-                r"(?:开启|打开|开到|开度)?(?:一半|半开)",
-                window_text,
-            )
-        )
-        position_pct = (
-            int(position[1]) if position else 50 if half_position else None
-        )
+        position_pct = parse_percentage_value(window_text)
         if (
             position_pct is not None
             and all(target not in WINDOW_GROUP_MEMBERS for target in targets)
-            and position_pct <= 100
+            and 0 <= position_pct <= 100
         ):
             reason = (
                 "用户指定了单个车窗的绝对开度。"

src/cognitive_agent/application/seat_service.py

@@ -1445,6 +1445,139 @@ class SeatDemoService:
             _direct_can_batch=direct_can_batch,
         )
 
+    def _adapt_speech_understanding(
+        self,
+        understanding: dict[str, Any],
+        context: dict[str, Any],
+        vehicle_profile: Any,
+    ) -> dict[str, Any]:
+        """Revalidate a server-owned speech result into the normal control IR.
+
+        Speech is allowed to skip a second natural-language parse, but it never
+        skips the authoritative capability catalog or ModelPlanAdapter.  Native
+        CAN actions and configured-domain/seat actions are routed through the
+        same structures used by the rest of one-sentence orchestration.
+        """
+
+        if understanding.get("execution_eligible") is not True:
+            raise ValueError("speech understanding is not execution eligible")
+        commands = understanding.get("commands")
+        if not isinstance(commands, list) or not commands:
+            raise ValueError("speech understanding has no executable commands")
+
+        native_actions: list[dict[str, Any]] = []
+        model_requests: list[dict[str, Any]] = []
+        detected_domains: list[str] = []
+
+        def add_payload(payload: object, source_text: str) -> None:
+            if not isinstance(payload, dict):
+                raise ValueError("speech command has no validated executor payload")
+            domain = str(payload.get("domain", ""))
+            capability = str(payload.get("capability", ""))
+            target = str(payload.get("target", ""))
+            operation = str(payload.get("operation", "set"))
+            parameters = payload.get("parameters")
+            if not domain or not capability or not target or not isinstance(parameters, dict):
+                raise ValueError("speech executor payload is incomplete")
+
+            definitions = tuple(
+                definition
+                for definition in self.capability_registry.find(capability)
+                if definition.domain == domain
+            )
+            if len(definitions) != 1:
+                raise ValueError(
+                    f"speech capability is not uniquely registered: {domain}.{capability}"
+                )
+            if domain not in detected_domains:
+                detected_domains.append(domain)
+            # Catalog availability describes where a capability is declared,
+            # not necessarily which executor owns it. Roof, climate and
+            # ambient capabilities are declared in domain catalogs but some
+            # are byte-exact NativeCanRuntime operations. Keep this routing in
+            # lockstep with CapabilityResolver._executor_binding().
+            if capability in self.native_can_runtime.supported_capabilities:
+                native_actions.append(
+                    {
+                        "domain": domain,
+                        "capability": capability,
+                        "target": target,
+                        "operation": operation,
+                        "parameters": deepcopy(parameters),
+                        "parse_source": "speech_semantic",
+                        "source_span": source_text,
+                        "reason": "语音语义层已按真实能力目录解析并由服务端重新校验。",
+                    }
+                )
+                return
+            model_requests.append(
+                {
+                    "domain": domain,
+                    "capability": capability,
+                    "target_scope": target,
+                    "operation": operation,
+                    "parameters": [
+                        {"name": name, "value": value}
+                        for name, value in parameters.items()
+                    ],
+                    "reason": "语音语义层已按真实能力目录解析并由服务端重新校验。",
+                }
+            )
+
+        for command in commands:
+            if not isinstance(command, dict):
+                raise ValueError("speech command must be an object")
+            if (
+                command.get("status") != "active"
+                or command.get("negated") is True
+                or command.get("resolution_status") != "executable"
+                or command.get("validation_issues")
+            ):
+                raise ValueError("speech command failed execution invariants")
+            source_text = str(command.get("source_text") or understanding.get("raw_text") or "")
+            payload = command.get("executor_payload")
+            if command.get("mode_profile_id"):
+                if not isinstance(payload, dict) or not isinstance(payload.get("operations"), list):
+                    raise ValueError("speech mode command has no validated operations")
+                for operation_payload in payload["operations"]:
+                    add_payload(operation_payload, source_text)
+            else:
+                add_payload(payload, source_text)
+
+        plan = {
+            "detected_domains": detected_domains,
+            "requests": model_requests,
+            "ambiguities": [],
+            "decision_summary": [
+                "复用服务端语音理解结果；未对 cleanText 进行第二次自然语言解析。"
+            ],
+            "confidence": min(
+                float(command.get("confidence", 0.0))
+                for command in commands
+                if isinstance(command, dict)
+            ),
+        }
+        parsed = self.model_plan_adapter.adapt(
+            plan,
+            context=context,
+            vehicle_profile=vehicle_profile,
+            utterance=str(understanding.get("raw_text") or ""),
+        )
+        parsed["native_can_actions"] = native_actions
+        parsed.setdefault("semantic_constraints", [])
+        parsed.setdefault("unresolved_spans", [])
+        if parsed.get("ambiguities") or parsed.get("deferred_requests"):
+            raise ValueError("speech command no longer passes capability resolution")
+        mapped_count = (
+            len(parsed.get("actions", ()))
+            + len(parsed.get("domain_actions", ()))
+            + len(parsed.get("native_can_actions", ()))
+        )
+        expected_count = len(model_requests) + len(native_actions)
+        if mapped_count != expected_count:
+            raise ValueError("speech command mapping lost a validated capability mutation")
+        return parsed
+
     def process(
         self,
         utterance: str,
@@ -1465,6 +1598,7 @@ class SeatDemoService:
         _allow_empty_control_result: bool = False,
         _on_commands_validated: Callable[[dict[str, Any]], None] | None = None,
         _direct_can_batch: tuple[list[str], dict[str, Any]] | None = None,
+        _speech_understanding: dict[str, Any] | None = None,
     ) -> dict[str, Any]:
         control_transport = self._validate_control_transport(control_transport)
         # Creating a reusable scene is a configuration workflow.  Synthesize
@@ -1490,6 +1624,7 @@ class SeatDemoService:
                 _allow_empty_control_result,
                 _on_commands_validated,
                 _direct_can_batch,
+                _speech_understanding,
             )
         if control_transport == WIRELESS_MQTT_TRANSPORT:
             # The head unit owns both physical CAN channels in wireless mode.
@@ -1511,6 +1646,7 @@ class SeatDemoService:
                 _allow_empty_control_result,
                 _on_commands_validated,
                 _direct_can_batch,
+                _speech_understanding,
             )
         if hardware_access_bypass or self._driver_only_vehicle_execution_active(
             control_transport
@@ -1532,6 +1668,7 @@ class SeatDemoService:
                     _allow_empty_control_result,
                     _on_commands_validated,
                     _direct_can_batch,
+                    _speech_understanding,
                 )
         authorized_client_id = self.hardware_access.ensure_authorized(hardware_client_id)
         with self.hardware_execution.reserve():
@@ -1552,6 +1689,7 @@ class SeatDemoService:
                     _allow_empty_control_result,
                     _on_commands_validated,
                     _direct_can_batch,
+                    _speech_understanding,
                 )
 
     def _process_once(
@@ -1571,6 +1709,7 @@ class SeatDemoService:
         allow_empty_control_result: bool = False,
         on_commands_validated: Callable[[dict[str, Any]], None] | None = None,
         direct_can_batch: tuple[list[str], dict[str, Any]] | None = None,
+        speech_understanding: dict[str, Any] | None = None,
     ) -> dict[str, Any]:
         control_transport = self._validate_control_transport(control_transport)
         utterance = utterance.strip()
@@ -1641,7 +1780,7 @@ class SeatDemoService:
         context["mode_profiles"] = self.mode_profiles.public_modes(vehicle_profile.id)
         matched_modes = (
             []
-            if direct_can_batch is not None
+            if direct_can_batch is not None or speech_understanding is not None
             else self.mode_profiles.match_text(
                 utterance,
                 vehicle_id=vehicle_profile.id,
@@ -1691,6 +1830,16 @@ class SeatDemoService:
             "can_commands": (
                 list(direct_can_batch[0]) if direct_can_batch is not None else []
             ),
+            "speech_understanding": (
+                {
+                    "utterance_id": speech_understanding.get("utterance_id"),
+                    "raw_text": speech_understanding.get("raw_text"),
+                    "clean_text": speech_understanding.get("clean_text"),
+                    "semantic_status": speech_understanding.get("semantic_status"),
+                }
+                if speech_understanding is not None
+                else None
+            ),
         }
 
         started = time.perf_counter()
@@ -1733,6 +1882,19 @@ class SeatDemoService:
             parsed = deepcopy(direct_can_batch[1])
             provider_used = "direct_can_control"
             provider_notice = None
+        elif speech_understanding is not None:
+            parsed = self._adapt_speech_understanding(
+                speech_understanding,
+                context,
+                vehicle_profile,
+            )
+            parsed = self._mark_control_parse_source(
+                parsed,
+                "speech_semantic",
+                overwrite=True,
+            )
+            provider_used = "speech_semantic"
+            provider_notice = "使用服务端缓存并重新校验的语音结构化意图。"
         else:
             parsed, provider_used, provider_notice = self._interpret(
                 utterance,

src/cognitive_agent/infrastructure/speech/__init__.py

@@ -0,0 +1,29 @@
+"""Local speech-recognition infrastructure."""
+
+from cognitive_agent.infrastructure.speech.assembler import TranscriptAssembler
+from cognitive_agent.infrastructure.speech.contracts import (
+    SpeechCommandIntent,
+    UnderstandingTranscript,
+)
+from cognitive_agent.infrastructure.speech.semantic import (
+    SpeechIntentNormalization,
+    normalize_speech_intent,
+)
+from cognitive_agent.infrastructure.speech.sensevoice import (
+    SpeechRecognitionError,
+    SpeechTranscriber,
+)
+from cognitive_agent.infrastructure.speech.understanding import (
+    UnderstandingTranscriptProcessor,
+)
+
+__all__ = [
+    "SpeechCommandIntent",
+    "SpeechIntentNormalization",
+    "SpeechRecognitionError",
+    "SpeechTranscriber",
+    "TranscriptAssembler",
+    "UnderstandingTranscript",
+    "UnderstandingTranscriptProcessor",
+    "normalize_speech_intent",
+]

src/cognitive_agent/infrastructure/speech/assembler.py

@@ -0,0 +1,156 @@
+"""Idempotent assembly of partial and final ASR segment callbacks."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+
+from cognitive_agent.infrastructure.speech.contracts import (
+    AssembledTranscript,
+    SpeechSegmentResult,
+)
+
+_PUNCTUATION = re.compile(r"^[，,。！？!?；;\s]+|[，,。！？!?；;\s]+$")
+
+
+class TranscriptAssembler:
+    """Maintain ordered segment state with replace/commit and overlap semantics."""
+
+    def __init__(self, utterance_id: str) -> None:
+        self.utterance_id = utterance_id
+        self._segments: dict[str, SpeechSegmentResult] = {}
+        self._duplicate_callbacks = 0
+        self._out_of_order_callbacks = 0
+        self._conflicts: list[str] = []
+        self._received = 0
+        self._last_sequence = -1
+        self._complete = False
+
+    def update(self, segment: SpeechSegmentResult) -> AssembledTranscript:
+        if segment.utterance_id != self.utterance_id:
+            raise ValueError("segment utterance_id does not match assembler")
+        if self._complete:
+            self._duplicate_callbacks += 1
+            return self.snapshot()
+
+        self._received += 1
+        segment = replace(segment, received_index=self._received)
+        if segment.sequence < self._last_sequence:
+            self._out_of_order_callbacks += 1
+        self._last_sequence = max(self._last_sequence, segment.sequence)
+
+        previous = self._segments.get(segment.segment_id)
+        if previous is not None:
+            if previous.is_final:
+                if segment.is_final and self._same_commit(previous, segment):
+                    self._duplicate_callbacks += 1
+                else:
+                    self._conflicts.append(
+                        f"segment {segment.segment_id} changed after final commit"
+                    )
+                return self.snapshot()
+            if self._same_callback(previous, segment):
+                self._duplicate_callbacks += 1
+                return self.snapshot()
+
+        self._segments[segment.segment_id] = segment
+        return self.snapshot()
+
+    def finish(self) -> AssembledTranscript:
+        self._complete = True
+        return self.snapshot()
+
+    def reset(self, utterance_id: str) -> None:
+        self.__init__(utterance_id)
+
+    def snapshot(self) -> AssembledTranscript:
+        ordered = sorted(
+            self._segments.values(),
+            key=lambda item: (item.sequence, item.start_ms, item.received_index),
+        )
+        finals = tuple(item for item in ordered if item.is_final)
+        partials = tuple(item for item in ordered if not item.is_final)
+        text, overlaps = self._assemble_text(finals, partials)
+        return AssembledTranscript(
+            utterance_id=self.utterance_id,
+            text=text,
+            final_segments=finals,
+            partial_segments=partials,
+            duplicate_callbacks=self._duplicate_callbacks,
+            out_of_order_callbacks=self._out_of_order_callbacks,
+            conflicts=tuple(dict.fromkeys(self._conflicts)),
+            overlaps_removed=tuple(overlaps),
+            complete=self._complete,
+        )
+
+    @staticmethod
+    def _same_commit(first: SpeechSegmentResult, second: SpeechSegmentResult) -> bool:
+        return (
+            first.sequence == second.sequence
+            and first.start_ms == second.start_ms
+            and first.end_ms == second.end_ms
+            and first.text == second.text
+        )
+
+    @classmethod
+    def _same_callback(cls, first: SpeechSegmentResult, second: SpeechSegmentResult) -> bool:
+        return first.is_final == second.is_final and cls._same_commit(first, second)
+
+    @classmethod
+    def _assemble_text(
+        cls,
+        finals: tuple[SpeechSegmentResult, ...],
+        partials: tuple[SpeechSegmentResult, ...],
+    ) -> tuple[str, list[str]]:
+        display = [*finals]
+        final_ids = {item.segment_id for item in finals}
+        display.extend(item for item in partials if item.segment_id not in final_ids)
+        display.sort(key=lambda item: (item.sequence, item.start_ms, item.received_index))
+        if not display:
+            return "", []
+
+        assembled = display[0].text.strip()
+        previous = display[0]
+        overlaps: list[str] = []
+        for current in display[1:]:
+            addition = current.text.strip()
+            overlap = cls._safe_overlap(assembled, addition, previous, current)
+            if overlap:
+                overlaps.append(overlap)
+                addition = addition[len(overlap) :]
+            assembled = cls._join(assembled, addition)
+            previous = current
+        return assembled.strip(), overlaps
+
+    @staticmethod
+    def _safe_overlap(
+        left: str,
+        right: str,
+        previous: SpeechSegmentResult,
+        current: SpeechSegmentResult,
+    ) -> str:
+        """Remove only boundary overlap supported by audio overlap or a long phrase."""
+
+        left_body = _PUNCTUATION.sub("", left)
+        right_body = _PUNCTUATION.sub("", right)
+        maximum = min(len(left_body), len(right_body), 24)
+        for length in range(maximum, 1, -1):
+            candidate = left_body[-length:]
+            if candidate != right_body[:length]:
+                continue
+            audio_evidence = current.audio_overlap_ms > 0 or (
+                current.start_ms < previous.end_ms
+            )
+            if length >= 4 or (audio_evidence and length >= 2):
+                return candidate
+        return ""
+
+    @staticmethod
+    def _join(left: str, right: str) -> str:
+        if not right:
+            return left
+        if not left:
+            return right
+        if left[-1] in "，,。！？!?；;" or right[0] in "，,。！？!?；;":
+            return f"{left}{right}"
+        return f"{left}，{right}"

src/cognitive_agent/infrastructure/speech/capability.py

@@ -0,0 +1,396 @@
+"""Resolve speech slots against the application's authoritative capability catalogs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from cognitive_agent.application.capability_registry import (
+    CapabilityDefinition,
+    CapabilityRegistry,
+)
+from cognitive_agent.application.mode_profile import ModeProfile, ModeProfileRegistry
+from cognitive_agent.application.native_can_runtime import NativeCanRuntime
+
+ResolutionStatus = Literal[
+    "executable",
+    "unsupported",
+    "ambiguous",
+    "incomplete",
+]
+
+
+@dataclass(frozen=True)
+class CapabilityResolution:
+    """Catalog-backed outcome for one semantic mutation."""
+
+    status: ResolutionStatus
+    definition: CapabilityDefinition | None
+    issues: tuple[str, ...] = ()
+    executor_binding: str | None = None
+    executor_payload: dict[str, Any] | None = None
+    intent_ir_action: dict[str, Any] | None = None
+
+
+@dataclass(frozen=True)
+class ModeResolution:
+    """A repository mode profile and whether it applies to the requested target."""
+
+    status: ResolutionStatus
+    profile: ModeProfile | None
+    issues: tuple[str, ...] = ()
+    executor_payloads: tuple[dict[str, Any], ...] = ()
+
+
+# These are lexical-to-schema selectors, not a second capability registry. A
+# selector can only resolve a capability that is present in CapabilityRegistry.
+_PROPERTY_SELECTORS: dict[str, tuple[str, ...]] = {
+    "hvac.power": ("set_power",),
+    "hvac.temperature": ("temperature",),
+    "hvac.fan_level": ("fan_level",),
+    "seat.ventilation": ("ventilation",),
+    "seat.heating": ("heating",),
+    "seat.longitudinal_position": ("longitudinal_position",),
+    "seat.backrest_recline": ("backrest",),
+    "seat.cushion_tilt": ("cushion_tilt",),
+    "seat.legrest_height": ("legrest_height",),
+    "seat.legrest_extension": ("legrest_extension",),
+    "window.motion": ("set_motion",),
+    "window.position": ("set_position",),
+    "roof.shade_motion": ("shade_motion",),
+    "roof.shade_position": ("shade_position",),
+    "roof.transmittance": ("transmittance",),
+    "ambient.power": ("set_power",),
+    "ambient.color": ("illumination", "color_index"),
+    "ambient.brightness": ("illumination", "brightness_level"),
+    "reading_light.power": ("reading_power",),
+    "indoor_light.power": ("indoor_power",),
+    "front_light.power": ("front_light", "headlight", "head_lamp"),
+}
+
+
+class CapabilityResolver:
+    """Adapt natural-language slots to the capability/executor facts in code."""
+
+    def __init__(self, repository_root: Path) -> None:
+        self.root = Path(repository_root)
+        self.registry = CapabilityRegistry(
+            self.root,
+            {"seat", "climate", "roof", "ambient_light"},
+        )
+        self.native_runtime = NativeCanRuntime(self.root)
+        self.native_supported = self.native_runtime.supported_capabilities
+        self.mode_profiles = ModeProfileRegistry(self.root / "config" / "modes")
+
+    def definitions(self) -> tuple[CapabilityDefinition, ...]:
+        return self.registry.definitions()
+
+    def executor_binding_for(self, definition: CapabilityDefinition) -> str:
+        """Return the runtime that actually owns an executable definition."""
+
+        return self._executor_binding(definition)
+
+    @staticmethod
+    def semantic_property_for(capability: str, parameters: dict[str, Any] | None = None) -> str:
+        """Project an authoritative capability into the speech display vocabulary.
+
+        This is deliberately a projection, not an availability table: whether the
+        capability exists and can execute still comes exclusively from the catalog
+        and executor runtime.
+        """
+
+        values = parameters or {}
+        if capability == "seat.set_driver_axis_native":
+            return {
+                "horizontal": "seat.longitudinal_position",
+                "backrest": "seat.backrest_recline",
+                "basin": "seat.cushion_tilt",
+                "height": "seat.height",
+            }.get(str(values.get("axis")), capability)
+        projections = (
+            ("set_driver_ventilation", "seat.ventilation"),
+            ("seat.set_ventilation", "seat.ventilation"),
+            ("set_driver_heating", "seat.heating"),
+            ("seat.set_heating", "seat.heating"),
+            ("set_backrest_recline", "seat.backrest_recline"),
+            ("set_longitudinal_position", "seat.longitudinal_position"),
+            ("set_cushion_tilt", "seat.cushion_tilt"),
+            ("set_legrest_height", "seat.legrest_height"),
+            ("set_legrest_extension", "seat.legrest_extension"),
+            ("window.set_motion", "window.motion"),
+            ("window.set_position", "window.position"),
+            ("roof.set_shade_motion", "roof.shade_motion"),
+            ("roof.set_shade_position", "roof.shade_position"),
+            ("roof.set_transmittance", "roof.transmittance"),
+            ("ambient_light.set_power", "ambient.power"),
+            ("ambient_light.set_illumination", "ambient.color"),
+            ("ambient_light.set_color_index", "ambient.color_index"),
+            ("ambient_light.set_brightness_level", "ambient.brightness"),
+            ("climate.set_power", "hvac.power"),
+            ("climate.set_temperature", "hvac.temperature"),
+            ("climate.set_fan_level", "hvac.fan_level"),
+            ("interior_light.set_reading_power", "reading_light.power"),
+            ("interior_light.set_indoor_power", "indoor_light.power"),
+        )
+        return next(
+            (property_name for token, property_name in projections if token in capability),
+            capability,
+        )
+
+    def resolve_action(self, action: dict[str, Any], *, source_text: str) -> CapabilityResolution:
+        """Validate an action produced by the application's existing intent parser."""
+
+        capability = str(action.get("capability", ""))
+        return self.resolve(
+            domain=str(action.get("domain", capability.partition(".")[0])),
+            semantic_property=self.semantic_property_for(
+                capability, dict(action.get("parameters", {}))
+            ),
+            target_scope=str(action.get("target", action.get("target_scope", "vehicle"))),
+            target_device=None,
+            operation=str(action.get("operation", "set")),
+            parameters=dict(action.get("parameters", {})),
+            source_text=source_text,
+            preferred_capability=capability,
+        )
+
+    def match_mode(self, text: str) -> ModeProfile | None:
+        matches = self.mode_profiles.match_text(text, vehicle_id="denza_n7")
+        return matches[0] if len(matches) == 1 else None
+
+    def resolve_mode(
+        self,
+        text: str,
+        *,
+        target_scope: str | None,
+        target_explicit: bool,
+    ) -> ModeResolution:
+        matches = self.mode_profiles.match_text(text, vehicle_id="denza_n7")
+        if not matches:
+            return ModeResolution("unsupported", None, ("mode_profile_not_registered",))
+        if len(matches) > 1:
+            return ModeResolution("ambiguous", None, ("multiple_mode_profiles_matched",))
+        profile = matches[0]
+        operation_scopes = {operation.target_scope for operation in profile.operations}
+        if target_explicit and target_scope not in operation_scopes:
+            scopes = ",".join(sorted(operation_scopes))
+            return ModeResolution(
+                "unsupported",
+                profile,
+                (f"mode_profile.{profile.id}仅支持范围{scopes}",),
+            )
+
+        payloads: list[dict[str, Any]] = []
+        issues: list[str] = []
+        for operation in profile.operations:
+            resolution = self.resolve(
+                domain=operation.domain,
+                semantic_property="",
+                target_scope=operation.target_scope,
+                target_device=None,
+                operation=operation.operation,
+                parameters=operation.parameters,
+                source_text=text,
+                preferred_capability=operation.capability,
+            )
+            if resolution.status != "executable" or resolution.executor_payload is None:
+                issues.extend(resolution.issues or ("mode_operation_not_executable",))
+                continue
+            payloads.append(resolution.executor_payload)
+        if issues or len(payloads) != len(profile.operations):
+            return ModeResolution("unsupported", profile, tuple(dict.fromkeys(issues)))
+        return ModeResolution("executable", profile, executor_payloads=tuple(payloads))
+
+    def resolve(
+        self,
+        *,
+        domain: str,
+        semantic_property: str,
+        target_scope: str,
+        target_device: str | None,
+        operation: str,
+        parameters: dict[str, Any],
+        source_text: str,
+        preferred_capability: str | None = None,
+    ) -> CapabilityResolution:
+        candidates = list(self.registry.definitions(domain=domain))
+        if preferred_capability:
+            candidates = [item for item in candidates if item.name == preferred_capability]
+        else:
+            selectors = _PROPERTY_SELECTORS.get(semantic_property, ())
+            candidates = [
+                item for item in candidates if any(selector in item.name for selector in selectors)
+            ]
+        if not candidates:
+            return CapabilityResolution(
+                "unsupported",
+                None,
+                (f"capability_not_registered:{domain}.{semantic_property}",),
+            )
+
+        parameter_names = set(parameters)
+        ranked = sorted(
+            candidates,
+            key=lambda item: self._score(
+                item,
+                target_scope=target_scope,
+                target_device=target_device,
+                parameter_names=parameter_names,
+            ),
+            reverse=True,
+        )
+        definition = ranked[0]
+        top_score = self._score(
+            definition,
+            target_scope=target_scope,
+            target_device=target_device,
+            parameter_names=parameter_names,
+        )
+        equally_ranked = [
+            item
+            for item in ranked
+            if self._score(
+                item,
+                target_scope=target_scope,
+                target_device=target_device,
+                parameter_names=parameter_names,
+            )
+            == top_score
+        ]
+        if len(equally_ranked) > 1 and not preferred_capability:
+            return CapabilityResolution(
+                "ambiguous",
+                None,
+                ("multiple_capabilities_match:" + ",".join(item.name for item in equally_ranked),),
+            )
+
+        issues = self._validate(definition, target_scope, target_device, parameters)
+        missing = [issue for issue in issues if issue.startswith("required_parameter_missing:")]
+        if missing:
+            status: ResolutionStatus = "incomplete"
+        elif issues:
+            status = "unsupported"
+        elif definition.availability == "catalog_only":
+            status = "unsupported"
+            issues.append(f"capability_catalog_only:{definition.name}")
+        elif (
+            definition.availability == "native_can" and definition.name not in self.native_supported
+        ):
+            status = "unsupported"
+            issues.append(f"native_executor_not_implemented:{definition.name}")
+        else:
+            status = "executable"
+
+        binding = self._executor_binding(definition) if status == "executable" else None
+        payload = None
+        intent_action = None
+        if status == "executable":
+            executor_operation = "set" if operation in {"on", "off", "open", "close"} else operation
+            payload = {
+                "domain": domain,
+                "capability": definition.name,
+                "target": target_scope,
+                "operation": executor_operation,
+                "parameters": dict(parameters),
+                "parse_source": "local_rule",
+            }
+            intent_action = {
+                "domain": domain,
+                "capability": definition.name,
+                "target": {
+                    "device": definition.target_devices[0] if definition.target_devices else domain,
+                    "zone": target_scope,
+                },
+                "operation": executor_operation,
+                "parameters": dict(parameters),
+                "source_span": source_text,
+                "risk_level": str(definition.spec.get("risk_level", "high")),
+                "confirmation_policy": str(
+                    definition.spec.get("confirmation_policy", "on_ambiguity")
+                ),
+            }
+        return CapabilityResolution(
+            status,
+            definition,
+            tuple(dict.fromkeys(issues)),
+            binding,
+            payload,
+            intent_action,
+        )
+
+    @staticmethod
+    def _score(
+        definition: CapabilityDefinition,
+        *,
+        target_scope: str,
+        target_device: str | None,
+        parameter_names: set[str],
+    ) -> tuple[int, int, int, int, int]:
+        specs = {str(item["name"]): item for item in definition.parameters}
+        accepted = set(specs)
+        missing_required = sum(
+            1
+            for name, spec in specs.items()
+            if spec.get("required") and name not in parameter_names
+        )
+        return (
+            int(target_scope in definition.target_scopes),
+            int(not parameter_names - accepted),
+            -missing_required,
+            int(target_device is not None and target_device in definition.target_devices),
+            int(definition.availability == "native_can"),
+        )
+
+    @staticmethod
+    def _validate(
+        definition: CapabilityDefinition,
+        target_scope: str,
+        target_device: str | None,
+        parameters: dict[str, Any],
+    ) -> list[str]:
+        issues: list[str] = []
+        if definition.target_scopes and target_scope not in definition.target_scopes:
+            issues.append(f"{definition.name}不支持范围{target_scope}")
+        if (
+            target_device
+            and definition.target_devices
+            and target_device not in definition.target_devices
+        ):
+            issues.append(f"{definition.name}不支持设备{target_device}")
+        specs = {str(item["name"]): item for item in definition.parameters}
+        for name in parameters.keys() - specs.keys():
+            issues.append(f"unknown_parameter:{definition.name}:{name}")
+        for name, spec in specs.items():
+            if spec.get("required") and name not in parameters:
+                issues.append(f"required_parameter_missing:{definition.name}:{name}")
+                continue
+            if name not in parameters:
+                continue
+            value = parameters[name]
+            expected = str(spec.get("type"))
+            numeric = isinstance(value, (int, float)) and not isinstance(value, bool)
+            valid_type = {
+                "number": numeric,
+                "integer": numeric and float(value).is_integer(),
+                "string": isinstance(value, str),
+                "boolean": isinstance(value, bool),
+                "object": isinstance(value, dict),
+                "array": isinstance(value, list),
+            }.get(expected, False)
+            if not valid_type:
+                issues.append(f"invalid_parameter_type:{definition.name}:{name}:{expected}")
+                continue
+            if numeric and spec.get("minimum") is not None and value < spec["minimum"]:
+                issues.append(f"{definition.name}.{name}低于工程下限{spec['minimum']}")
+            if numeric and spec.get("maximum") is not None and value > spec["maximum"]:
+                issues.append(f"{definition.name}.{name}高于工程上限{spec['maximum']}")
+            allowed = spec.get("allowed_values")
+            if allowed is not None and value not in allowed:
+                issues.append(f"{definition.name}.{name}不在工程允许值中")
+        return issues
+
+    def _executor_binding(self, definition: CapabilityDefinition) -> str:
+        if definition.name in self.native_supported:
+            return "NativeCanRuntime/NativeCanExecutor"
+        return definition.executor_binding

src/cognitive_agent/infrastructure/speech/contracts.py

@@ -0,0 +1,233 @@
+"""Stable data contracts for the speech-understanding pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+CommandStatus = Literal["active", "cancelled"]
+ResolutionStatus = Literal["executable", "unsupported", "ambiguous", "incomplete"]
+TranscriptStatus = Literal[
+    "recognized_executable",
+    "recognized_unsupported",
+    "ambiguous",
+    "incomplete",
+    "no_command",
+]
+
+
+@dataclass(frozen=True)
+class SpeechCommandIntent:
+    """A conservative speech intent aligned with the project's capability catalog."""
+
+    domain: str
+    capability: str | None
+    target_scope: str
+    target_device: str
+    property_name: str
+    operation: str
+    parameters: dict[str, Any]
+    value: Any
+    unit: str | None
+    qualifiers: dict[str, Any]
+    negated: bool
+    confidence: float
+    source_text: str
+    status: CommandStatus = "active"
+    resolution_status: ResolutionStatus = "executable"
+    validation_issues: tuple[str, ...] = ()
+    executor_binding: str | None = None
+    executor_payload: dict[str, Any] | None = None
+    intent_ir_action: dict[str, Any] | None = None
+    mode_profile_id: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "domain": self.domain,
+            "capability": self.capability,
+            "target_scope": self.target_scope,
+            "target_device": self.target_device,
+            "property": self.property_name,
+            "operation": self.operation,
+            "parameters": dict(self.parameters),
+            "value": self.value,
+            "unit": self.unit,
+            "qualifiers": dict(self.qualifiers),
+            "negated": self.negated,
+            "confidence": round(self.confidence, 3),
+            "source_text": self.source_text,
+            "status": self.status,
+            "resolution_status": self.resolution_status,
+            "validation_issues": list(self.validation_issues),
+            "executor_binding": self.executor_binding,
+            "executor_payload": self.executor_payload,
+            "intent_ir_action": self.intent_ir_action,
+            "mode_profile_id": self.mode_profile_id,
+        }
+
+
+@dataclass(frozen=True)
+class CancelledSpeechFragment:
+    """Text that was heard but must never become an active control."""
+
+    text: str
+    reason: str
+    marker: str = ""
+    commands: tuple[SpeechCommandIntent, ...] = ()
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "text": self.text,
+            "reason": self.reason,
+            "marker": self.marker,
+            "commands": [command.to_dict() for command in self.commands],
+        }
+
+
+@dataclass(frozen=True)
+class UnderstandingTranscript:
+    """Raw ASR evidence plus display text and non-executable semantic analysis."""
+
+    raw_text: str
+    clean_text: str
+    commands: tuple[SpeechCommandIntent, ...] = ()
+    cancelled_fragments: tuple[CancelledSpeechFragment, ...] = ()
+    semantic_status: TranscriptStatus = "no_command"
+    uncertain: bool = False
+    reasons: tuple[str, ...] = ()
+    removed_fillers: tuple[str, ...] = ()
+    repair_resolved: bool = False
+    intent_domains: tuple[str, ...] = ()
+    changed: bool = False
+    processing_ms: float = 0.0
+    command_resolution_ms: float = 0.0
+    semantic_trace: tuple[dict[str, Any], ...] = ()
+
+    @property
+    def text(self) -> str:
+        """Backward-compatible display-text alias."""
+
+        return self.clean_text
+
+    @property
+    def execution_eligible(self) -> bool:
+        return bool(self.commands) and not self.uncertain and all(
+            command.status == "active"
+            and not command.negated
+            and command.resolution_status == "executable"
+            and not command.validation_issues
+            for command in self.commands
+        )
+
+    def metadata(self) -> dict[str, object]:
+        return {
+            "changed": self.changed,
+            "removed_fillers": list(self.removed_fillers),
+            "repair_resolved": self.repair_resolved,
+            "intent_domains": list(self.intent_domains),
+            "uncertain": self.uncertain,
+            "semantic_status": self.semantic_status,
+            "reason": list(self.reasons),
+            "execution_eligible": self.execution_eligible,
+            "processing_ms": round(self.processing_ms, 3),
+            "command_resolution_ms": round(self.command_resolution_ms, 3),
+        }
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "raw_text": self.raw_text,
+            "clean_text": self.clean_text,
+            "commands": [command.to_dict() for command in self.commands],
+            "cancelled_fragments": [
+                fragment.to_dict() for fragment in self.cancelled_fragments
+            ],
+            "semantic_status": self.semantic_status,
+            "uncertain": self.uncertain,
+            "reason": list(self.reasons),
+            "execution_eligible": self.execution_eligible,
+            "normalization": self.metadata(),
+            "semantic_trace": [dict(item) for item in self.semantic_trace],
+        }
+
+
+@dataclass(frozen=True)
+class SpeechSegmentResult:
+    """One ordered partial or final ASR callback."""
+
+    utterance_id: str
+    segment_id: str
+    sequence: int
+    start_ms: int
+    end_ms: int
+    text: str
+    is_final: bool
+    audio_overlap_ms: int = 0
+    inference_ms: float = 0.0
+    received_index: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "utterance_id": self.utterance_id,
+            "segment_id": self.segment_id,
+            "sequence": self.sequence,
+            "start_ms": self.start_ms,
+            "end_ms": self.end_ms,
+            "raw_text": self.text,
+            "state": "final" if self.is_final else "partial",
+            "audio_overlap_ms": self.audio_overlap_ms,
+            "inference_ms": round(self.inference_ms, 3),
+        }
+
+
+@dataclass(frozen=True)
+class AssembledTranscript:
+    utterance_id: str
+    text: str
+    final_segments: tuple[SpeechSegmentResult, ...]
+    partial_segments: tuple[SpeechSegmentResult, ...]
+    duplicate_callbacks: int = 0
+    out_of_order_callbacks: int = 0
+    conflicts: tuple[str, ...] = ()
+    overlaps_removed: tuple[str, ...] = ()
+    complete: bool = False
+
+    @property
+    def uncertain(self) -> bool:
+        return bool(self.conflicts)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "utterance_id": self.utterance_id,
+            "text": self.text,
+            "complete": self.complete,
+            "duplicate_callbacks": self.duplicate_callbacks,
+            "out_of_order_callbacks": self.out_of_order_callbacks,
+            "conflicts": list(self.conflicts),
+            "overlaps_removed": list(self.overlaps_removed),
+            "segments": [segment.to_dict() for segment in self.final_segments],
+        }
+
+
+@dataclass
+class SpeechPipelineTimings:
+    audio_read_ms: float = 0.0
+    vad_ms: float = 0.0
+    asr_ms: float = 0.0
+    assembly_ms: float = 0.0
+    semantic_ms: float = 0.0
+    recovery_ms: float = 0.0
+    total_ms: float = 0.0
+    extras: dict[str, float] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, float]:
+        result = {
+            "audio_read_ms": round(self.audio_read_ms, 3),
+            "vad_ms": round(self.vad_ms, 3),
+            "asr_ms": round(self.asr_ms, 3),
+            "assembly_ms": round(self.assembly_ms, 3),
+            "semantic_ms": round(self.semantic_ms, 3),
+            "recovery_ms": round(self.recovery_ms, 3),
+            "total_ms": round(self.total_ms, 3),
+        }
+        result.update({key: round(value, 3) for key, value in self.extras.items()})
+        return result

src/cognitive_agent/infrastructure/speech/recovery.py

@@ -0,0 +1,46 @@
+"""Optional recovery recognizer contract for uncertain fast-path results."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Protocol
+
+from cognitive_agent.infrastructure.speech.contracts import UnderstandingTranscript
+
+
+@dataclass(frozen=True)
+class RecoveryResult:
+    raw_text: str
+    model: str
+    confidence: float
+    elapsed_ms: float
+
+
+class RecoveryRecognizer(Protocol):
+    @property
+    def available(self) -> bool: ...
+
+    def recognize(
+        self,
+        samples: Sequence[float],
+        sample_rate: int,
+        fast_path: UnderstandingTranscript,
+    ) -> RecoveryResult | None: ...
+
+
+class DisabledRecoveryRecognizer:
+    """Default recovery implementation; never downloads or guesses a model."""
+
+    @property
+    def available(self) -> bool:
+        return False
+
+    def recognize(
+        self,
+        samples: Sequence[float],
+        sample_rate: int,
+        fast_path: UnderstandingTranscript,
+    ) -> RecoveryResult | None:
+        del samples, sample_rate, fast_path
+        return None

src/cognitive_agent/infrastructure/speech/semantic.py

@@ -0,0 +1,61 @@
+"""Backward-compatible entry point for structured speech understanding."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from cognitive_agent.infrastructure.speech.understanding import (
+    UnderstandingTranscriptProcessor,
+)
+
+
+@dataclass(frozen=True)
+class SpeechIntentNormalization:
+    text: str
+    removed_fillers: tuple[str, ...]
+    repair_resolved: bool
+    intent_domains: tuple[str, ...]
+    changed: bool
+    commands: tuple[object, ...] = ()
+    cancelled_fragments: tuple[object, ...] = ()
+    uncertain: bool = False
+    reasons: tuple[str, ...] = ()
+
+    def metadata(self) -> dict[str, object]:
+        return {
+            "changed": self.changed,
+            "removed_fillers": list(self.removed_fillers),
+            "repair_resolved": self.repair_resolved,
+            "intent_domains": list(self.intent_domains),
+            "uncertain": self.uncertain,
+            "reason": list(self.reasons),
+        }
+
+
+_DEFAULT_PROCESSOR: UnderstandingTranscriptProcessor | None = None
+
+
+def normalize_speech_intent(
+    text: str, repository_root: Path | None = None
+) -> SpeechIntentNormalization:
+    global _DEFAULT_PROCESSOR
+    if repository_root is not None:
+        processor = UnderstandingTranscriptProcessor(repository_root)
+    else:
+        if _DEFAULT_PROCESSOR is None:
+            repository = Path(__file__).resolve().parents[4]
+            _DEFAULT_PROCESSOR = UnderstandingTranscriptProcessor(repository)
+        processor = _DEFAULT_PROCESSOR
+    result = processor.process(text)
+    return SpeechIntentNormalization(
+        text=result.clean_text,
+        removed_fillers=result.removed_fillers,
+        repair_resolved=result.repair_resolved,
+        intent_domains=result.intent_domains,
+        changed=result.changed,
+        commands=result.commands,
+        cancelled_fragments=result.cancelled_fragments,
+        uncertain=result.uncertain,
+        reasons=result.reasons,
+    )

src/cognitive_agent/infrastructure/speech/sensevoice.py

@@ -0,0 +1,493 @@
+"""Low-latency local Chinese speech understanding backed by SenseVoiceSmall."""
+
+from __future__ import annotations
+
+import copy
+import hashlib
+import importlib.metadata
+import importlib.util
+import math
+import os
+import re
+import sys
+import threading
+import time
+import uuid
+import wave
+from array import array
+from collections import OrderedDict
+from collections.abc import Sequence
+from dataclasses import replace
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+
+from cognitive_agent.infrastructure.speech.assembler import TranscriptAssembler
+from cognitive_agent.infrastructure.speech.contracts import (
+    SpeechPipelineTimings,
+    SpeechSegmentResult,
+)
+from cognitive_agent.infrastructure.speech.recovery import DisabledRecoveryRecognizer
+from cognitive_agent.infrastructure.speech.tracing import SpeechTrace, env_flag
+from cognitive_agent.infrastructure.speech.understanding import (
+    UnderstandingTranscriptProcessor,
+)
+from cognitive_agent.infrastructure.speech.vad import SpeechVadSegmenter
+
+SENSEVOICE_MODEL_NAME = "SenseVoiceSmall"
+SENSEVOICE_SAMPLE_RATE = 16_000
+SENSEVOICE_MAX_DURATION_SECONDS = 30.0
+SENSEVOICE_MIN_DURATION_SECONDS = 0.2
+SENSEVOICE_MAX_WAV_BYTES = 2 * 1024 * 1024
+_SENSEVOICE_TAG = re.compile(r"<\|[^|>]+\|>")
+_UTTERANCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,95}$")
+
+
+class SpeechRecognitionError(RuntimeError):
+    """A safe, user-facing speech-recognition failure."""
+
+    def __init__(self, code: str, message: str, *, status: int = 422) -> None:
+        super().__init__(message)
+        self.code = code
+        self.status = status
+
+
+class SpeechTranscriber:
+    """One recognizer, one VAD, and idempotent per-utterance orchestration."""
+
+    def __init__(self, repository_root: Path) -> None:
+        self.repository_root = repository_root
+        default_dir = repository_root / "artifacts" / "models" / "sensevoice-small"
+        self.model_dir = Path(os.getenv("SPEECH_MODEL_DIR", str(default_dir))).expanduser()
+        configured_model = os.getenv("SPEECH_MODEL_FILE", "model.onnx").strip() or "model.onnx"
+        self.model_path = self.model_dir / configured_model
+        self.tokens_path = self.model_dir / "tokens.txt"
+        self.language = os.getenv("SPEECH_LANGUAGE", "zh").strip() or "zh"
+        self.use_itn = env_flag("SPEECH_USE_ITN", True)
+        default_threads = min(8, max(2, (os.cpu_count() or 4) // 2))
+        self.num_threads = max(
+            1, min(16, int(os.getenv("SPEECH_NUM_THREADS", str(default_threads))))
+        )
+        self.provider = os.getenv("SPEECH_PROVIDER", "cpu").strip().lower() or "cpu"
+        self.hr_lexicon = Path(os.getenv("SPEECH_HR_LEXICON", "")).expanduser()
+        self.hr_rule_fsts = Path(os.getenv("SPEECH_HR_RULE_FSTS", "")).expanduser()
+        self.processor = UnderstandingTranscriptProcessor(repository_root)
+        self.segmenter = SpeechVadSegmenter(repository_root)
+        self.recovery = DisabledRecoveryRecognizer()
+        self._recognizer: Any | None = None
+        self._load_lock = threading.Lock()
+        self._decode_lock = threading.Lock()
+        self._cache_lock = threading.Lock()
+        self._cache: OrderedDict[str, tuple[str, dict[str, object]]] = OrderedDict()
+        self._inflight: dict[str, threading.Event] = {}
+
+    def status(self) -> dict[str, object]:
+        runtime_installed = importlib.util.find_spec("sherpa_onnx") is not None
+        model_installed = self.model_path.is_file() and self.tokens_path.is_file()
+        try:
+            runtime_version = importlib.metadata.version("sherpa-onnx")
+        except importlib.metadata.PackageNotFoundError:
+            runtime_version = ""
+        homophone_enabled = self.hr_lexicon.is_file() and self.hr_rule_fsts.is_file()
+        return {
+            "enabled": runtime_installed and model_installed,
+            "runtime_installed": runtime_installed,
+            "runtime_version": runtime_version,
+            "model_installed": model_installed,
+            "loaded": self._recognizer is not None,
+            "model": SENSEVOICE_MODEL_NAME,
+            "model_file": self.model_path.name,
+            "language": self.language,
+            "use_itn": self.use_itn,
+            "sample_rate": SENSEVOICE_SAMPLE_RATE,
+            "max_duration_seconds": SENSEVOICE_MAX_DURATION_SECONDS,
+            "provider": self.provider,
+            "num_threads": self.num_threads,
+            "decoder": "offline-sensevoice-final-only",
+            "vad": self.segmenter.status(),
+            "homophone_replacer": {
+                "enabled": homophone_enabled,
+                "lexicon": str(self.hr_lexicon) if homophone_enabled else "",
+                "rule_fsts": str(self.hr_rule_fsts) if homophone_enabled else "",
+            },
+            "recovery_recognizer": {"available": self.recovery.available},
+            "debug_logging": env_flag("SPEECH_DEBUG"),
+        }
+
+    def transcribe_wav(
+        self, content: bytes, *, utterance_id: str | None = None
+    ) -> dict[str, object]:
+        resolved_id = self._resolve_utterance_id(utterance_id)
+        digest = hashlib.sha256(content).hexdigest()
+        cached = self._claim_or_wait(resolved_id, digest)
+        if cached is not None:
+            cached["idempotent_replay"] = True
+            return cached
+        try:
+            result = self._transcribe_wav(content, resolved_id)
+            with self._cache_lock:
+                self._cache[resolved_id] = (digest, copy.deepcopy(result))
+                self._cache.move_to_end(resolved_id)
+                while len(self._cache) > 64:
+                    self._cache.popitem(last=False)
+            return result
+        finally:
+            with self._cache_lock:
+                event = self._inflight.pop(resolved_id, None)
+                if event is not None:
+                    event.set()
+
+    def cached_result(self, utterance_id: str) -> dict[str, object]:
+        """Return a server-owned recognition result for a later control request.
+
+        The browser submits only the opaque utterance id.  Executor payloads are
+        recovered from this bounded server cache so client-side JSON can never be
+        promoted into a trusted control plan.
+        """
+
+        resolved_id = self._resolve_utterance_id(utterance_id)
+        with self._cache_lock:
+            cached = self._cache.get(resolved_id)
+            if cached is None:
+                raise SpeechRecognitionError(
+                    "speech_result_expired",
+                    "语音理解结果已过期，请重新听写后再执行。",
+                    status=409,
+                )
+            self._cache.move_to_end(resolved_id)
+            return copy.deepcopy(cached[1])
+
+    def _transcribe_wav(self, content: bytes, utterance_id: str) -> dict[str, object]:
+        trace = SpeechTrace(utterance_id)
+        timings = SpeechPipelineTimings()
+        total_started = time.perf_counter()
+        read_started = time.perf_counter()
+        sample_rate, original_samples = self._read_pcm16_wav(content)
+        samples = self._resample(original_samples, sample_rate, SENSEVOICE_SAMPLE_RATE)
+        timings.audio_read_ms = (time.perf_counter() - read_started) * 1000
+        duration_seconds = len(samples) / SENSEVOICE_SAMPLE_RATE
+        self._validate_duration(duration_seconds)
+        rms = math.sqrt(math.fsum(sample * sample for sample in samples) / len(samples))
+        trace.event(
+            "audio_preprocessed",
+            sampleRate=SENSEVOICE_SAMPLE_RATE,
+            sourceSampleRate=sample_rate,
+            durationMs=round(duration_seconds * 1000),
+            rms=round(rms, 6),
+        )
+        if rms < 0.001:
+            raise SpeechRecognitionError(
+                "no_speech", "未检测到清晰的说话声，请靠近麦克风重试。"
+            )
+
+        segmentation = self.segmenter.segment(samples)
+        timings.vad_ms = segmentation.elapsed_ms
+        trace.event(
+            "vad_complete",
+            usedVad=segmentation.used_vad,
+            fallbackReason=segmentation.fallback_reason,
+            segmentCount=len(segmentation.segments),
+            segments=[
+                {
+                    "segmentId": item.segment_id,
+                    "startMs": item.start_ms,
+                    "endMs": item.end_ms,
+                    "audioOverlapMs": item.audio_overlap_ms,
+                }
+                for item in segmentation.segments
+            ],
+            stageMs=round(segmentation.elapsed_ms, 3),
+        )
+        if segmentation.fallback_reason == "vad_found_no_segment":
+            raise SpeechRecognitionError(
+                "no_speech", "VAD 未检测到可靠语音，请靠近麦克风重试。"
+            )
+
+        recognizer = self._get_recognizer()
+        assembler = TranscriptAssembler(utterance_id)
+        labels: dict[str, str] = {}
+        asr_started = time.perf_counter()
+        with self._decode_lock:
+            for segment in segmentation.segments:
+                segment_started = time.perf_counter()
+                try:
+                    stream = recognizer.create_stream()
+                    stream.accept_waveform(SENSEVOICE_SAMPLE_RATE, segment.samples)
+                    recognizer.decode_stream(stream)
+                    native_result = stream.result
+                except Exception as exc:  # pragma: no cover - native errors vary
+                    raise SpeechRecognitionError(
+                        "inference_failed",
+                        "语音模型暂时无法完成转写，请稍后重试。",
+                        status=503,
+                    ) from exc
+                inference_ms = (time.perf_counter() - segment_started) * 1000
+                raw_segment = self._clean_text(str(getattr(native_result, "text", "")))
+                labels = {
+                    "language": self._clean_label(
+                        getattr(native_result, "lang", self.language)
+                    ),
+                    "emotion": self._clean_label(getattr(native_result, "emotion", "")),
+                    "event": self._clean_label(getattr(native_result, "event", "")),
+                }
+                callback = SpeechSegmentResult(
+                    utterance_id=utterance_id,
+                    segment_id=segment.segment_id,
+                    sequence=segment.sequence,
+                    start_ms=segment.start_ms,
+                    end_ms=segment.end_ms,
+                    text=raw_segment,
+                    is_final=True,
+                    audio_overlap_ms=segment.audio_overlap_ms,
+                    inference_ms=inference_ms,
+                )
+                before = assembler.snapshot().text
+                assembled = assembler.update(callback)
+                trace.event(
+                    "asr_callback",
+                    segmentId=segment.segment_id,
+                    sequence=segment.sequence,
+                    segmentStartMs=segment.start_ms,
+                    segmentEndMs=segment.end_ms,
+                    state="final",
+                    rawAsrResult=raw_segment,
+                    beforeAssembler=before,
+                    assembledText=assembled.text,
+                    inferenceMs=round(inference_ms, 3),
+                )
+        timings.asr_ms = (time.perf_counter() - asr_started) * 1000
+        assembly_started = time.perf_counter()
+        assembled = assembler.finish()
+        timings.assembly_ms = (time.perf_counter() - assembly_started) * 1000
+        raw_text = assembled.text.strip()
+        if not raw_text:
+            raise SpeechRecognitionError(
+                "no_transcript", "没有识别出有效文字，请重新说一次。"
+            )
+
+        semantic_started = time.perf_counter()
+        understanding = self.processor.process(raw_text)
+        timings.semantic_ms = (time.perf_counter() - semantic_started) * 1000
+        timings.extras["command_resolution_ms"] = understanding.command_resolution_ms
+        if assembled.uncertain:
+            understanding = replace(
+                understanding,
+                uncertain=True,
+                reasons=tuple(
+                    dict.fromkeys((*understanding.reasons, "segment_assembly_conflict"))
+                ),
+            )
+        trace.event(
+            "understanding_complete",
+            rawText=raw_text,
+            cleanText=understanding.clean_text,
+            commands=[command.to_dict() for command in understanding.commands],
+            cancelledFragments=[
+                fragment.to_dict() for fragment in understanding.cancelled_fragments
+            ],
+            semanticStatus=understanding.semantic_status,
+            semanticTrace=list(understanding.semantic_trace),
+            uncertain=understanding.uncertain,
+            reason=list(understanding.reasons),
+            stageMs=round(timings.semantic_ms, 3),
+        )
+
+        if understanding.uncertain and self.recovery.available:
+            recovery_started = time.perf_counter()
+            recovered = self.recovery.recognize(
+                samples, SENSEVOICE_SAMPLE_RATE, understanding
+            )
+            timings.recovery_ms = (time.perf_counter() - recovery_started) * 1000
+            trace.event(
+                "recovery_complete",
+                recovered=bool(recovered),
+                stageMs=round(timings.recovery_ms, 3),
+            )
+
+        timings.total_ms = (time.perf_counter() - total_started) * 1000
+        payload: dict[str, object] = {
+            "utterance_id": utterance_id,
+            "text": understanding.clean_text,
+            "clean_text": understanding.clean_text,
+            "raw_text": raw_text,
+            "commands": [command.to_dict() for command in understanding.commands],
+            "cancelled_fragments": [
+                fragment.to_dict() for fragment in understanding.cancelled_fragments
+            ],
+            "semantic_status": understanding.semantic_status,
+            "semantic_trace": list(understanding.semantic_trace),
+            "uncertain": understanding.uncertain,
+            "reason": list(understanding.reasons),
+            "execution_eligible": understanding.execution_eligible,
+            "semantic_cleaned": understanding.changed,
+            "normalization": understanding.metadata(),
+            "assembly": assembled.to_dict(),
+            "segments": [item.to_dict() for item in assembled.final_segments],
+            **labels,
+            "duration_ms": round(duration_seconds * 1000),
+            "inference_ms": round(timings.asr_ms),
+            "timings": timings.to_dict(),
+            "model": SENSEVOICE_MODEL_NAME,
+            "provider": self.provider,
+            "idempotent_replay": False,
+        }
+        debug_events = trace.response_events()
+        if debug_events:
+            payload["debug_trace"] = debug_events
+        return payload
+
+    def _claim_or_wait(
+        self, utterance_id: str, digest: str
+    ) -> dict[str, object] | None:
+        while True:
+            with self._cache_lock:
+                cached = self._cache.get(utterance_id)
+                if cached is not None:
+                    cached_digest, payload = cached
+                    if cached_digest != digest:
+                        raise SpeechRecognitionError(
+                            "utterance_id_conflict",
+                            "同一 utteranceId 收到了不同录音，已拒绝重复提交。",
+                            status=409,
+                        )
+                    self._cache.move_to_end(utterance_id)
+                    return copy.deepcopy(payload)
+                event = self._inflight.get(utterance_id)
+                if event is None:
+                    self._inflight[utterance_id] = threading.Event()
+                    return None
+            if not event.wait(timeout=50):
+                raise SpeechRecognitionError(
+                    "duplicate_request_timeout", "重复语音请求等待超时。", status=503
+                )
+
+    def _get_recognizer(self) -> Any:
+        if self._recognizer is not None:
+            return self._recognizer
+        with self._load_lock:
+            if self._recognizer is not None:
+                return self._recognizer
+            if importlib.util.find_spec("sherpa_onnx") is None:
+                raise SpeechRecognitionError(
+                    "runtime_unavailable",
+                    "语音识别运行时未安装，请安装 speech 可选依赖。",
+                    status=503,
+                )
+            if not self.model_path.is_file() or not self.tokens_path.is_file():
+                raise SpeechRecognitionError(
+                    "model_unavailable", "SenseVoiceSmall 模型尚未部署。", status=503
+                )
+            try:
+                import sherpa_onnx
+
+                kwargs: dict[str, object] = {
+                    "model": str(self.model_path),
+                    "tokens": str(self.tokens_path),
+                    "num_threads": self.num_threads,
+                    "sample_rate": SENSEVOICE_SAMPLE_RATE,
+                    "provider": self.provider,
+                    "language": self.language,
+                    "use_itn": self.use_itn,
+                    "debug": env_flag("SPEECH_ASR_DEBUG"),
+                }
+                if self.hr_lexicon.is_file() and self.hr_rule_fsts.is_file():
+                    kwargs["hr_lexicon"] = str(self.hr_lexicon)
+                    kwargs["hr_rule_fsts"] = str(self.hr_rule_fsts)
+                self._recognizer = sherpa_onnx.OfflineRecognizer.from_sense_voice(
+                    **kwargs
+                )
+            except Exception as exc:  # pragma: no cover - native failures vary
+                raise SpeechRecognitionError(
+                    "model_load_failed",
+                    "SenseVoiceSmall 模型加载失败，请检查模型文件与运行时。",
+                    status=503,
+                ) from exc
+        return self._recognizer
+
+    @staticmethod
+    def _resolve_utterance_id(value: str | None) -> str:
+        if value is None or not value.strip():
+            return f"server-{uuid.uuid4()}"
+        candidate = value.strip()
+        if not _UTTERANCE_ID.fullmatch(candidate):
+            raise SpeechRecognitionError(
+                "invalid_utterance_id", "utteranceId 格式无效。", status=400
+            )
+        return candidate
+
+    @staticmethod
+    def _validate_duration(duration_seconds: float) -> None:
+        if duration_seconds < SENSEVOICE_MIN_DURATION_SECONDS:
+            raise SpeechRecognitionError("audio_too_short", "录音时间太短，请重新说一次。")
+        if duration_seconds > SENSEVOICE_MAX_DURATION_SECONDS + 0.5:
+            raise SpeechRecognitionError(
+                "audio_too_long",
+                f"单次听写不能超过 {int(SENSEVOICE_MAX_DURATION_SECONDS)} 秒。",
+                status=413,
+            )
+
+    @staticmethod
+    def _clean_text(text: str) -> str:
+        return re.sub(r"\s+", " ", _SENSEVOICE_TAG.sub("", text)).strip()
+
+    @staticmethod
+    def _clean_label(value: object) -> str:
+        return str(value or "").removeprefix("<|").removesuffix("|>").strip()
+
+    @staticmethod
+    def _resample(
+        samples: Sequence[float], input_rate: int, output_rate: int
+    ) -> list[float]:
+        if input_rate == output_rate:
+            return list(samples)
+        output_length = max(1, round(len(samples) * output_rate / input_rate))
+        if output_length == 1:
+            return [float(samples[0])]
+        ratio = input_rate / output_rate
+        result: list[float] = []
+        for output_index in range(output_length):
+            position = min(len(samples) - 1, output_index * ratio)
+            left = int(position)
+            right = min(len(samples) - 1, left + 1)
+            fraction = position - left
+            result.append(
+                float(samples[left] * (1.0 - fraction) + samples[right] * fraction)
+            )
+        return result
+
+    @staticmethod
+    def _read_pcm16_wav(content: bytes) -> tuple[int, list[float]]:
+        if not content or len(content) > SENSEVOICE_MAX_WAV_BYTES:
+            raise SpeechRecognitionError(
+                "invalid_audio_size", "录音数据为空或超过大小限制。", status=413
+            )
+        try:
+            with wave.open(BytesIO(content), "rb") as source:
+                channels = source.getnchannels()
+                sample_width = source.getsampwidth()
+                sample_rate = source.getframerate()
+                frame_count = source.getnframes()
+                compression = source.getcomptype()
+                frames = source.readframes(frame_count)
+        except (EOFError, wave.Error) as exc:
+            raise SpeechRecognitionError("invalid_audio", "录音格式无效，请重新录制。") from exc
+        if compression != "NONE" or sample_width != 2:
+            raise SpeechRecognitionError("unsupported_audio", "仅支持 16 位 PCM WAV 录音。")
+        if channels < 1 or channels > 2 or sample_rate < 8_000 or sample_rate > 96_000:
+            raise SpeechRecognitionError(
+                "unsupported_audio", "录音声道数或采样率不受支持。"
+            )
+        pcm = array("h")
+        pcm.frombytes(frames)
+        if sys.byteorder != "little":
+            pcm.byteswap()
+        if channels == 2:
+            mono = [
+                (pcm[index] + pcm[index + 1]) / (2 * 32768.0)
+                for index in range(0, len(pcm) - 1, 2)
+            ]
+        else:
+            mono = [sample / 32768.0 for sample in pcm]
+        if not mono:
+            raise SpeechRecognitionError("invalid_audio", "录音中没有音频采样。")
+        return sample_rate, mono

src/cognitive_agent/infrastructure/speech/tracing.py

@@ -0,0 +1,59 @@
+"""Opt-in structured diagnostics for one speech utterance."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+LOGGER = logging.getLogger("cognitive_agent.speech")
+_HANDLER_LOCK = threading.Lock()
+
+
+def env_flag(name: str, default: bool = False) -> bool:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+@dataclass
+class SpeechTrace:
+    utterance_id: str
+    enabled: bool = field(default_factory=lambda: env_flag("SPEECH_DEBUG"))
+    include_response: bool = field(
+        default_factory=lambda: env_flag("SPEECH_DEBUG_RESPONSE")
+    )
+    started_at: float = field(default_factory=time.perf_counter)
+    events: list[dict[str, Any]] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        if not self.enabled or LOGGER.handlers:
+            return
+        with _HANDLER_LOCK:
+            if LOGGER.handlers:
+                return
+            handler = logging.StreamHandler()
+            handler.setFormatter(logging.Formatter("%(message)s"))
+            LOGGER.addHandler(handler)
+            LOGGER.setLevel(logging.INFO)
+            LOGGER.propagate = False
+
+    def event(self, stage: str, **details: Any) -> None:
+        if not self.enabled:
+            return
+        payload = {
+            "utteranceId": self.utterance_id,
+            "stage": stage,
+            "elapsedMs": round((time.perf_counter() - self.started_at) * 1000, 3),
+            **details,
+        }
+        self.events.append(payload)
+        LOGGER.info("speech_trace %s", json.dumps(payload, ensure_ascii=False))
+
+    def response_events(self) -> list[dict[str, Any]]:
+        return list(self.events) if self.enabled and self.include_response else []

src/cognitive_agent/infrastructure/speech/vad.py

@@ -0,0 +1,173 @@
+"""Sherpa-ONNX Silero VAD with conservative boundary padding."""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import threading
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from cognitive_agent.infrastructure.speech.tracing import env_flag
+
+
+@dataclass(frozen=True)
+class VadAudioSegment:
+    segment_id: str
+    sequence: int
+    start_sample: int
+    end_sample: int
+    samples: tuple[float, ...]
+    audio_overlap_ms: int = 0
+
+    @property
+    def start_ms(self) -> int:
+        return round(self.start_sample / 16)
+
+    @property
+    def end_ms(self) -> int:
+        return round(self.end_sample / 16)
+
+
+@dataclass(frozen=True)
+class VadSegmentation:
+    segments: tuple[VadAudioSegment, ...]
+    elapsed_ms: float
+    used_vad: bool
+    fallback_reason: str = ""
+
+
+class SpeechVadSegmenter:
+    """Reuse one native VAD instance and reset its state per utterance."""
+
+    sample_rate = 16_000
+
+    def __init__(self, repository_root: Path) -> None:
+        default_model = repository_root / "artifacts" / "models" / "silero-vad" / "silero_vad.onnx"
+        self.model_path = Path(os.getenv("SPEECH_VAD_MODEL", str(default_model))).expanduser()
+        self.enabled = env_flag("SPEECH_VAD_ENABLED", True)
+        self.threshold = float(os.getenv("SPEECH_VAD_THRESHOLD", "0.45"))
+        self.min_speech_seconds = float(os.getenv("SPEECH_VAD_MIN_SPEECH", "0.16"))
+        self.min_silence_seconds = float(os.getenv("SPEECH_VAD_MIN_SILENCE", "0.65"))
+        self.max_speech_seconds = float(os.getenv("SPEECH_VAD_MAX_SPEECH", "12.0"))
+        self.pre_roll_seconds = float(os.getenv("SPEECH_VAD_PRE_ROLL", "0.25"))
+        self.post_roll_seconds = float(os.getenv("SPEECH_VAD_POST_ROLL", "0.35"))
+        self.buffer_seconds = float(os.getenv("SPEECH_VAD_BUFFER", "32"))
+        self.num_threads = max(1, min(4, int(os.getenv("SPEECH_VAD_THREADS", "1"))))
+        self._vad: object | None = None
+        self._lock = threading.Lock()
+
+    @property
+    def available(self) -> bool:
+        return (
+            self.enabled
+            and self.model_path.is_file()
+            and importlib.util.find_spec("sherpa_onnx") is not None
+        )
+
+    def status(self) -> dict[str, object]:
+        return {
+            "enabled": self.enabled,
+            "available": self.available,
+            "loaded": self._vad is not None,
+            "model_path": str(self.model_path),
+            "threshold": self.threshold,
+            "min_speech_seconds": self.min_speech_seconds,
+            "min_silence_seconds": self.min_silence_seconds,
+            "max_speech_seconds": self.max_speech_seconds,
+            "pre_roll_seconds": self.pre_roll_seconds,
+            "post_roll_seconds": self.post_roll_seconds,
+        }
+
+    def segment(self, samples: Sequence[float]) -> VadSegmentation:
+        started = time.perf_counter()
+        if not self.available:
+            return self._fallback(samples, started, "vad_unavailable")
+        with self._lock:
+            vad = self._get_vad()
+            try:
+                vad.reset()
+                chunk_size = 512
+                for offset in range(0, len(samples), chunk_size):
+                    vad.accept_waveform(samples[offset : offset + chunk_size])
+                vad.flush()
+                raw_boundaries: list[tuple[int, int]] = []
+                while not vad.empty():
+                    segment = vad.front
+                    start = max(0, int(segment.start))
+                    end = min(len(samples), start + len(segment.samples))
+                    if end > start:
+                        raw_boundaries.append((start, end))
+                    vad.pop()
+            finally:
+                vad.reset()
+        if not raw_boundaries:
+            return self._fallback(samples, started, "vad_found_no_segment")
+
+        pre_roll = round(self.pre_roll_seconds * self.sample_rate)
+        post_roll = round(self.post_roll_seconds * self.sample_rate)
+        padded = [
+            (max(0, start - pre_roll), min(len(samples), end + post_roll))
+            for start, end in raw_boundaries
+        ]
+        # Padding must not create overlapped audio submitted to two ASR streams.
+        merged: list[tuple[int, int]] = []
+        for start, end in padded:
+            if merged and start <= merged[-1][1]:
+                merged[-1] = (merged[-1][0], max(merged[-1][1], end))
+            else:
+                merged.append((start, end))
+        segments = tuple(
+            VadAudioSegment(
+                segment_id=f"segment-{sequence}",
+                sequence=sequence,
+                start_sample=start,
+                end_sample=end,
+                samples=tuple(samples[start:end]),
+            )
+            for sequence, (start, end) in enumerate(merged)
+        )
+        return VadSegmentation(
+            segments,
+            (time.perf_counter() - started) * 1000,
+            used_vad=True,
+        )
+
+    def _get_vad(self):
+        if self._vad is not None:
+            return self._vad
+        import sherpa_onnx
+
+        silero = sherpa_onnx.SileroVadModelConfig(
+            model=str(self.model_path),
+            threshold=self.threshold,
+            min_silence_duration=self.min_silence_seconds,
+            min_speech_duration=self.min_speech_seconds,
+            window_size=512,
+            max_speech_duration=self.max_speech_seconds,
+        )
+        config = sherpa_onnx.VadModelConfig(
+            silero_vad=silero,
+            sample_rate=self.sample_rate,
+            num_threads=self.num_threads,
+            provider="cpu",
+            debug=env_flag("SPEECH_VAD_DEBUG"),
+        )
+        self._vad = sherpa_onnx.VoiceActivityDetector(config, self.buffer_seconds)
+        return self._vad
+
+    @staticmethod
+    def _fallback(
+        samples: Sequence[float], started: float, reason: str
+    ) -> VadSegmentation:
+        segment = VadAudioSegment(
+            "segment-0", 0, 0, len(samples), tuple(samples)
+        )
+        return VadSegmentation(
+            (segment,),
+            (time.perf_counter() - started) * 1000,
+            used_vad=False,
+            fallback_reason=reason,
+        )

src/cognitive_agent/infrastructure/speech/vocabulary.py

@@ -0,0 +1,283 @@
+"""Chinese lexical aliases used before authoritative capability resolution."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class DeviceTerm:
+    canonical: str
+    domain: str
+    semantic_key: str
+    target_device: str
+    aliases: tuple[str, ...]
+
+
+DEVICE_TERMS: tuple[DeviceTerm, ...] = (
+    DeviceTerm("空调", "climate", "hvac.power", "hvac", ("空调", "空调系统")),
+    DeviceTerm(
+        "空调温度",
+        "climate",
+        "hvac.temperature",
+        "hvac",
+        ("空调温度", "温度"),
+    ),
+    DeviceTerm(
+        "空调风量",
+        "climate",
+        "hvac.fan_level",
+        "hvac",
+        ("空调风量", "风量", "风速"),
+    ),
+    DeviceTerm(
+        "座椅靠背",
+        "seat",
+        "seat.backrest_recline",
+        "seat_backrest_motor",
+        ("座椅靠背", "座椅的靠背", "靠背", "椅背"),
+    ),
+    DeviceTerm(
+        "座椅",
+        "seat",
+        "seat.longitudinal_position",
+        "seat_longitudinal_motor",
+        ("座椅", "坐椅", "座位"),
+    ),
+    DeviceTerm(
+        "座椅坐垫",
+        "seat",
+        "seat.cushion_tilt",
+        "seat_cushion_tilt_motor",
+        ("座椅坐垫", "坐垫", "坐盆"),
+    ),
+    DeviceTerm(
+        "座椅腿托",
+        "seat",
+        "seat.legrest_height",
+        "seat_legrest_lift_motor",
+        ("座椅腿托", "腿托", "脚托", "小腿托"),
+    ),
+    DeviceTerm(
+        "座椅通风",
+        "seat",
+        "seat.ventilation",
+        "seat",
+        ("座椅通风", "通风座椅"),
+    ),
+    DeviceTerm(
+        "座椅加热",
+        "seat",
+        "seat.heating",
+        "seat",
+        ("座椅加热", "加热座椅"),
+    ),
+    DeviceTerm(
+        "车窗",
+        "window",
+        "window.motion",
+        "window",
+        ("车窗", "窗户", "玻璃窗"),
+    ),
+    DeviceTerm(
+        "遮阳帘",
+        "roof",
+        "roof.shade_motion",
+        "roof_shade",
+        ("遮阳帘", "天幕帘", "天窗遮阳帘"),
+    ),
+    DeviceTerm(
+        "天幕",
+        "roof",
+        "roof.transmittance",
+        "roof_glass",
+        ("天幕", "天窗玻璃"),
+    ),
+    DeviceTerm(
+        "氛围灯",
+        "ambient_light",
+        "ambient.power",
+        "ambient_light",
+        ("氛围灯", "环境灯"),
+    ),
+    DeviceTerm(
+        "阅读灯",
+        "interior_light",
+        "reading_light.power",
+        "reading_light",
+        ("阅读灯",),
+    ),
+    DeviceTerm(
+        "车内灯",
+        "interior_light",
+        "indoor_light.power",
+        "indoor_light",
+        ("车内灯", "室内灯", "顶灯"),
+    ),
+    DeviceTerm(
+        "前大灯",
+        "front_light",
+        "front_light.power",
+        "front_light",
+        ("前大灯", "大灯", "前照灯"),
+    ),
+)
+
+SCOPE_ALIASES: dict[str, tuple[str, ...]] = {
+    "driver": ("主驾", "主驾驶", "驾驶位", "司机位", "左前"),
+    "passenger": ("副驾", "副驾驶", "乘客位", "右前"),
+    "front_row": ("前排",),
+    "rear_row": ("后排",),
+    "rear_left": ("左后", "后排左侧"),
+    "rear_right": ("右后", "后排右侧"),
+    "vehicle": ("全车", "整车", "车内"),
+    "all": ("全部", "所有", "四个"),
+}
+
+ACTION_ALIASES: dict[str, tuple[str, ...]] = {
+    "on": ("打开", "开启", "启动", "开"),
+    "off": ("关闭", "关掉", "关上", "停止", "关"),
+    "set": (
+        "调整到",
+        "调整为",
+        "调节到",
+        "调节为",
+        "设置到",
+        "设置为",
+        "调到",
+        "调成",
+        "变成",
+        "设为",
+        "开到",
+    ),
+    "forward": ("向前", "往前", "朝前", "前移", "前倾"),
+    "backward": ("向后", "往后", "朝后", "后移", "后仰"),
+    "up": ("向上", "往上", "升高", "抬高", "抬起"),
+    "down": ("向下", "往下", "降低", "降下"),
+    "adjust": ("调整", "调节", "移动", "调", "移"),
+}
+
+CORRECTION_MARKERS: tuple[str, ...] = (
+    "重新说",
+    "说错了",
+    "准确地说",
+    "应该是",
+    "我是说",
+    "更正为",
+    "更正成",
+    "不对",
+    "口误",
+    "改成",
+    "改为",
+    "重来",
+)
+
+CANCELLATION_MARKERS: tuple[str, ...] = ("算了", "取消", "作废")
+POSITIVE_CONNECTORS: tuple[str, ...] = (
+    "然后",
+    "同时",
+    "并且",
+    "接着",
+    "随后",
+    "再",
+    "并",
+)
+NEGATION_PREFIXES: tuple[str, ...] = ("不要", "别", "不想", "无需", "不必", "禁止")
+UNITS: tuple[str, ...] = ("摄氏度", "百分比", "毫米", "厘米", "毫秒", "分钟", "度", "%", "档", "秒")
+
+COLOR_ALIASES: dict[str, tuple[str, ...]] = {
+    "#FF0000": ("红色", "红灯", "正红"),
+    "#00FF00": ("绿色", "绿灯"),
+    "#0000FF": ("蓝色", "蓝灯"),
+    "#FFFFFF": ("白色", "白灯"),
+    "#FFFF00": ("黄色", "黄灯"),
+    "#FFA500": ("橙色", "橘色"),
+    "#800080": ("紫色", "紫灯"),
+}
+
+
+def _alias_pattern(values: Iterable[str]) -> re.Pattern[str]:
+    ordered = sorted(set(values), key=len, reverse=True)
+    return re.compile("|".join(re.escape(value) for value in ordered))
+
+
+DEVICE_ALIAS_PATTERN = _alias_pattern(
+    alias for term in DEVICE_TERMS for alias in term.aliases
+)
+ACTION_ALIAS_PATTERN = _alias_pattern(
+    alias for aliases in ACTION_ALIASES.values() for alias in aliases
+)
+SCOPE_ALIAS_PATTERN = _alias_pattern(
+    alias for aliases in SCOPE_ALIASES.values() for alias in aliases
+)
+
+
+def device_terms_in(text: str) -> list[tuple[int, int, DeviceTerm, str]]:
+    """Return longest non-overlapping device terms in source order."""
+
+    alias_map = {
+        alias: term
+        for term in DEVICE_TERMS
+        for alias in term.aliases
+    }
+    return [
+        (match.start(), match.end(), alias_map[match.group(0)], match.group(0))
+        for match in DEVICE_ALIAS_PATTERN.finditer(text)
+    ]
+
+
+def action_terms_in(text: str) -> list[tuple[int, int, str, str]]:
+    alias_map = {
+        alias: action
+        for action, aliases in ACTION_ALIASES.items()
+        for alias in aliases
+    }
+    device_spans = [(start, end) for start, end, _, _ in device_terms_in(text)]
+    matches: list[tuple[int, int, str, str]] = []
+    for match in ACTION_ALIAS_PATTERN.finditer(text):
+        # Single-character actions such as ``调`` are useful in ``往后调``, but
+        # must not turn the ``调`` inside ``空调`` into a later action callback.
+        starts_inside_device = any(
+            start <= match.start() < end for start, end in device_spans
+        )
+        if starts_inside_device:
+            continue
+        matches.append(
+            (match.start(), match.end(), alias_map[match.group(0)], match.group(0))
+        )
+    return matches
+
+
+def scope_in(text: str, *, default: str, domain: str | None = None) -> str:
+    matches = [
+        (match.start(), scope)
+        for scope, aliases in SCOPE_ALIASES.items()
+        for alias in aliases
+        for match in re.finditer(re.escape(alias), text)
+    ]
+    scope = min(matches)[1] if matches else default
+    if domain == "window":
+        return {
+            "driver": "front_left",
+            "passenger": "front_right",
+            "front_row": "front",
+            "rear_row": "rear",
+        }.get(scope, scope)
+    return scope
+
+
+def color_in(text: str) -> tuple[str, str] | None:
+    """Return a canonical RGB value and the exact spoken color alias."""
+
+    matches = [
+        (match.start(), color, alias)
+        for color, aliases in COLOR_ALIASES.items()
+        for alias in aliases
+        for match in re.finditer(re.escape(alias), text)
+    ]
+    if not matches:
+        return None
+    _, color, alias = min(matches)
+    return color, alias

src/cognitive_agent/server.py

@@ -12,28 +12,28 @@ import re
 import socket
 import threading
 import time
-from http.cookies import CookieError, SimpleCookie
 from http import HTTPStatus
+from http.cookies import CookieError, SimpleCookie
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from urllib.parse import ParseResult, parse_qs, quote, unquote, urlparse
 
 from cognitive_agent.config_loader import load_local_env
 
-
 REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
 # Repository-local launch configuration is deliberate and must not be shadowed
 # by stale variables inherited from an IDE or an earlier model experiment.
 load_local_env(REPOSITORY_ROOT / ".env", override=True)
 load_local_env(REPOSITORY_ROOT / ".env.auth", override=True)
 
-from cognitive_agent.auth import AuthConfig, AuthRuntime, verify_password  # noqa: E402
-from cognitive_agent.application.seat_service import SeatDemoService  # noqa: E402
-from cognitive_agent.application.gesture_bridge import GestureCapabilityBridge  # noqa: E402
 from cognitive_agent.application.calibration.service import (  # noqa: E402
     ApiError as CalibrationApiError,
+)
+from cognitive_agent.application.calibration.service import (
     CalibrationService,
 )
+from cognitive_agent.application.gesture_bridge import GestureCapabilityBridge  # noqa: E402
+from cognitive_agent.application.seat_service import SeatDemoService  # noqa: E402
 from cognitive_agent.application.self_extension import (  # noqa: E402
     ExtensionIntegrationError,
     GatewayError,
@@ -42,16 +42,20 @@ from cognitive_agent.application.self_extension import (  # noqa: E402
     resolve_active_peripheral_command,
 )
 from cognitive_agent.application.self_repair import SelfRepairService  # noqa: E402
+from cognitive_agent.auth import AuthConfig, AuthRuntime, verify_password  # noqa: E402
+from cognitive_agent.infrastructure.can.access import (  # noqa: E402
+    HardwareAccessRequiredError,
+)
 from cognitive_agent.infrastructure.can.hardware import (  # noqa: E402
     HardwareBusyError,
     HardwareSafetyError,
 )
-from cognitive_agent.infrastructure.can.access import (  # noqa: E402
-    HardwareAccessRequiredError,
-)
 from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError  # noqa: E402
 from cognitive_agent.infrastructure.nfc import NfcReaderService  # noqa: E402
-
+from cognitive_agent.infrastructure.speech import (  # noqa: E402
+    SpeechRecognitionError,
+    SpeechTranscriber,
+)
 
 STATIC_ROOT = REPOSITORY_ROOT / "apps" / "web-console"
 mimetypes.add_type("application/manifest+json", ".webmanifest")
@@ -176,6 +180,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
         repository_root=REPOSITORY_ROOT,
     )
     repair = SelfRepairService()
+    speech = SpeechTranscriber(REPOSITORY_ROOT)
     auth = AuthRuntime(AuthConfig.from_env())
     vehicle_display_sync = VehicleDisplaySync()
     server_version = "CognitiveAgentCabinDemo/2.0"
@@ -342,6 +347,9 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
                 return
             self._json({"error": "not_found"}, HTTPStatus.NOT_FOUND)
             return
+        if path == "/api/speech/status":
+            self._json(self.speech.status())
+            return
         if path.startswith("/api/requests/"):
             self._serve_request(path.removeprefix("/api/requests/"))
             return
@@ -357,6 +365,9 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
             return
         if not self._authorize(path):
             return
+        if path == "/api/speech/transcribe":
+            self._handle_speech_transcription()
+            return
         if path in {
             "/api/hardware/access/request",
             "/api/hardware/access/release",
@@ -482,6 +493,32 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
                     payload.get("control_transport", "wired_can")
                 ),
             }
+            speech_utterance_id = str(
+                payload.get("speech_utterance_id", "")
+            ).strip()
+            if speech_utterance_id:
+                if path != "/api/requests":
+                    raise ValueError(
+                        "speech semantic execution is only accepted by /api/requests"
+                    )
+                speech_understanding = self.speech.cached_result(
+                    speech_utterance_id
+                )
+                if str(speech_understanding.get("clean_text", "")).strip() != utterance:
+                    raise SpeechRecognitionError(
+                        "speech_text_changed",
+                        "语音文本已被修改，请确认修改后的内容再执行。",
+                        status=409,
+                    )
+                if speech_understanding.get("execution_eligible") is not True:
+                    raise SpeechRecognitionError(
+                        "speech_not_executable",
+                        "语音意图尚未通过能力或参数校验，已拒绝执行。",
+                        status=422,
+                    )
+                provider = "local"
+                common_arguments["provider"] = provider
+                common_arguments["_speech_understanding"] = speech_understanding
             if (
                 path != "/api/can/commands"
                 and "application/x-ndjson" in self.headers.get("Accept", "")
@@ -596,6 +633,12 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
                 {"error": "extension_request_ambiguous", "message": str(exc)},
                 HTTPStatus.UNPROCESSABLE_ENTITY,
             )
+        except SpeechRecognitionError as exc:
+            fail_display_sync(exc.code, exc)
+            self._json(
+                {"error": exc.code, "message": str(exc)},
+                HTTPStatus(exc.status),
+            )
         except (ValueError, json.JSONDecodeError) as exc:
             fail_display_sync("invalid_request", exc)
             self._json({"error": "invalid_request", "message": str(exc)}, HTTPStatus.BAD_REQUEST)
@@ -1340,6 +1383,41 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
             raise ValueError("request body must be a JSON object")
         return payload
 
+    def _handle_speech_transcription(self) -> None:
+        content_type = self.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
+        if content_type not in {"audio/wav", "audio/wave", "audio/x-wav"}:
+            self._json(
+                {
+                    "error": "unsupported_media_type",
+                    "message": "语音听写接口仅接受 16 位 PCM WAV 音频。",
+                },
+                HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
+            )
+            return
+        try:
+            content_length = int(self.headers.get("Content-Length", "0"))
+            if content_length <= 0 or content_length > 2 * 1024 * 1024:
+                raise SpeechRecognitionError(
+                    "invalid_audio_size",
+                    "录音数据为空或超过大小限制。",
+                    status=413,
+                )
+            result = self.speech.transcribe_wav(
+                self.rfile.read(content_length),
+                utterance_id=self.headers.get("X-Speech-Utterance-Id"),
+            )
+            self._json(result)
+        except SpeechRecognitionError as exc:
+            self._json(
+                {"error": exc.code, "message": str(exc)},
+                HTTPStatus(exc.status),
+            )
+        except (TypeError, ValueError):
+            self._json(
+                {"error": "invalid_audio", "message": "录音请求格式无效。"},
+                HTTPStatus.BAD_REQUEST,
+            )
+
     def _authorize(self, path: str) -> bool:
         if not self.auth.config.enabled or path in PUBLIC_AUTH_PATHS:
             return True
@@ -1612,7 +1690,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
         self.send_header("Referrer-Policy", "no-referrer")
         self.send_header(
             "Permissions-Policy",
-            "camera=(self), microphone=(), geolocation=(), payment=(), usb=()",
+            "camera=(self), microphone=(self), geolocation=(), payment=(), usb=()",
         )
         self.send_header(
             "Content-Security-Policy",