diff --git a/.gitignore b/.gitignore
index 304a00e..c1441f3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@ artifacts/mode-profiles/
artifacts/server/
artifacts/tmp/
artifacts/self-repair/
+artifacts/models/
artifacts/*.log
tmp/*.log
.codex_tmp/
diff --git a/README.md b/README.md
index 06b6b75..387b42b 100755
--- a/README.md
+++ b/README.md
@@ -96,10 +96,32 @@ make dev
浏览器访问 `http://127.0.0.1:8000`。设置 `OPENAI_API_KEY` 后，GPT Structured Outputs 会基于各领域的 `config/capabilities.yaml` 和当前会话状态直接生成通用能力请求；后端只做结构校验、领域路由和确定性执行适配，不用本地语义规则改写 GPT 动作。本地解析器用于未配置 GPT，或 GPT/`auto` 模式因 API Key、模型权限、网络或结构化响应异常而失败时的安全降级；结果会明确标记 `local_rules_fallback`。模型由 `OPENAI_MODEL=gpt-5.6-sol` 指定；兼容 OpenAI Responses API 的服务可通过 `OPENAI_BASE_URL` 指定。无论使用哪种解析器，配置校验、CAN 编译和虚拟执行均为同一确定性路径。
-### 一句话、快捷模式与手势控制
+### 一句话、语音听写、快捷模式与手势控制
首页左侧提供一句话控制，可同时描述座椅、空调、天幕和氛围灯目标；Enter 直接执行，Shift+Enter 换行。观影、午休和零重力是可直接调用的快捷模式，“保存当前”可把当前副驾驶姿态保存为命名模式。复杂跨域自然语言优先使用 GPT 解析，本地规则用于已明确覆盖的离线指令。
+输入框右侧的麦克风按钮提供本地中文语音听写：优先选择 `TAKSTAR V2` 输入设备，`AudioWorklet` 在音频线程采集（旧浏览器才降级到 `ScriptProcessor`），浏览器端静音检测负责自动停录，16 kHz PCM WAV 发送到同源 `/api/speech/transcribe`。服务端再使用 Silero VAD 做权威分段并保留句首/句尾保护区，SenseVoiceSmall 只负责声学识别；统一 `TranscriptAssembler` 负责 final commit、顺序、重复回调和保守的相邻重叠消除。
+
+声学 `raw_text` 永远保留。独立的 `UnderstandingTranscriptProcessor` 根据工程能力目录生成 `clean_text`、结构化 `commands`、`cancelled_fragments` 和 `uncertain`，区分“然后”的正常多指令与“不对、改成、算了”等修正/撤销。无关聊天与否定不会激活控制；超出工程参数范围或表达残缺的结果会显示但禁止直接提交，用户修改确认后才能执行。听写本身不自动执行控制命令，也不保存录音。首次部署：
+
+```bash
+python -m pip install -e ".[speech]"
+python tools/install_sensevoice_model.py
+```
+
+安装脚本同时校验并部署 `artifacts/models/silero-vad/silero_vad.onnx`。默认使用完整精度 SenseVoice、`language=zh`、ITN 和最多 8 个 CPU 线程；VAD 针对短车控语句使用阈值 `0.45`、最短语音 `0.16s`、结束静音 `0.65s`、最长分段 `12s`、前置 `0.25s`、后置 `0.35s`。可通过 `SPEECH_MODEL_*`、`SPEECH_LANGUAGE`、`SPEECH_USE_ITN`、`SPEECH_NUM_THREADS`、`SPEECH_PROVIDER` 与 `SPEECH_VAD_*` 覆盖。
+
+当前 sherpa-onnx 版本支持 SenseVoice 的 `hr_lexicon`/`hr_rule_fsts`，只有同时提供经过验证的文件时才会启用：`SPEECH_HR_LEXICON` 和 `SPEECH_HR_RULE_FSTS`。未对非 transducer 的 SenseVoice 使用 hotword decoding。可插拔 `RecoveryRecognizer` 默认关闭，只为将来让 uncertain case 使用二次识别，不会下载大型模型。
+
+诊断时设置 `SPEECH_DEBUG=1` 输出按 `utteranceId`、segment 边界、raw final、拼接前后、理解结果、Intent 与阶段耗时组织的 JSON 日志；只有额外设置 `SPEECH_DEBUG_RESPONSE=1` 才把 trace 返回浏览器。前端访问参数 `?speechDebug=1`（或 localStorage 同名值）启用控制台日志，生产默认不打印。回归与指标基准：
+
+```bash
+pytest tests/unit/test_speech_transcription.py tests/unit/test_transcript_assembler.py
+python tools/benchmark_speech_pipeline.py --audio-dir artifacts/tmp/speech-regression-audio
+```
+
+麦克风仅可用于安全来源：本机使用 `http://127.0.0.1:8000`，远程使用 HTTPS。
+
顶部“手势控制”打开同页抽屉。浏览器在本机完成 MediaPipe 推理，识别结果仍通过同一自然语言编排、安全校验与执行链路处理。运行必需资源已经纳入本工程：
```text
diff --git a/apps/web-console/README.md b/apps/web-console/README.md
index c016749..f98faac 100755
--- a/apps/web-console/README.md
+++ b/apps/web-console/README.md
@@ -6,6 +6,7 @@ Web 控制台是车机端的远程自编排与同步显示界面。网页保留
- 自编排和自扩展合并为一个实时界面，登录身份固定为 `user`，不再提供模块切换。
- 网页端可输入一句话指令或点击预设模式，车机端同样可发起任务；两端任务与结果统一同步。
+- 一句话输入框支持麦克风听写，优先采集 `TAKSTAR V2`；AudioWorklet 采集、浏览器端静音停录、服务端 Silero VAD、SenseVoiceSmall、幂等 TranscriptAssembler 和结构化理解层组成完整链路。只有生成 commands 的结果才回填；无关内容不覆盖输入框，uncertain 结果需人工修改后才能执行。
- 整车数字孪生位于工作区中央，点击或悬停热点可查看车窗、遮阳帘、主/副驾座椅、阅读灯、氛围灯、前大灯和空调的完整状态。
- 车端设备状态与外设能力板块均可独立隐藏或恢复，面板状态在当前浏览器会话中保留。
- USB 外设接入后同步显示 GSCU 链路、外设名称、类型、已学习能力和最近一次跨设备联动。
@@ -19,6 +20,15 @@ UI 只展示可审计的“决策摘要”和“处理轨迹”，不展示模
make dev
```
+语音听写首次运行前安装运行时、官方完整精度模型和 Silero VAD：
+
+```bash
+python -m pip install -e ".[speech]"
+python tools/install_sensevoice_model.py
+```
+
+录音只用于当前请求，不落盘；`raw_text`、`clean_text` 与结构化 Intent 分字段返回。远程浏览器必须通过 HTTPS 才能访问麦克风。诊断可设置后端 `SPEECH_DEBUG=1` 并用 `?speechDebug=1` 打开前端日志，默认关闭。
+
Android PAD/模拟器联调使用：
```powershell
diff --git a/apps/web-console/app.js b/apps/web-console/app.js
index 2767b36..eb98b52 100755
--- a/apps/web-console/app.js
+++ b/apps/web-console/app.js
@@ -7453,7 +7453,14 @@ async function executeCanCommands(canCommands, { deviceName = "车辆设备" } =
async function submitRequest(event) {
event.preventDefault();
- const utterance = $("#utterance").value.trim();
+ const utteranceInput = $("#utterance");
+ const utterance = utteranceInput.value.trim();
+ const speechSubmission = (
+ utteranceInput.dataset.speechExecutionEligible === "true"
+ && utteranceInput.dataset.speechValue === utterance
+ && Boolean(utteranceInput.dataset.speechUtteranceId)
+ );
+ const requestProvider = speechSubmission ? "local" : state.provider;
const submittedModeProfileId = state.pendingModeProfileId;
state.pendingModeProfileId = null;
if (!utterance) {
@@ -7461,6 +7468,20 @@ async function submitRequest(event) {
showToast("请输入车舱控制请求");
return;
}
+ if (
+ utteranceInput.dataset.speechExecutionEligible === "false"
+ && utteranceInput.dataset.speechValue === utterance
+ ) {
+ if (submittedModeProfileId) setModeExecutionState(submittedModeProfileId, "failed");
+ const speechStatus = utteranceInput.dataset.speechSemanticStatus;
+ const message = speechStatus === "recognized_unsupported"
+ ? "语音意图已识别，但当前车辆能力不支持执行"
+ : speechStatus === "incomplete"
+ ? "语音控制参数不完整，请补充后再执行"
+ : "语音结果存在歧义，请修改确认后再执行";
+ showToast(message);
+ return;
+ }
const driverOnlyCh1 = isDriverOnlyCh1Runtime();
if (state.executionMode === "hardware"
&& state.controlTransport !== "wireless_mqtt"
@@ -7491,12 +7512,15 @@ async function submitRequest(event) {
const requestBody = {
utterance,
client_type: "web_console",
- provider: state.provider,
+ provider: requestProvider,
conversation_id: state.conversationId,
vehicle_id: state.vehicleId || state.runtime?.default_vehicle_id,
execution_mode: state.executionMode,
control_transport: state.controlTransport,
- ...(state.provider === "openai" ? { reasoning_effort: state.reasoningEffort } : {}),
+ ...(requestProvider === "openai" ? { reasoning_effort: state.reasoningEffort } : {}),
+ ...(speechSubmission
+ ? { speech_utterance_id: utteranceInput.dataset.speechUtteranceId }
+ : {}),
...(state.executionMode === "hardware"
&& state.controlTransport !== "wireless_mqtt"
&& !driverOnlyCh1
@@ -7515,7 +7539,7 @@ async function submitRequest(event) {
{ detail: phaseEvent.payload },
));
};
- if (state.provider === "openai") {
+ if (requestProvider === "openai") {
const preview = await postControlPhase(
"/api/requests/preview",
requestBody,
@@ -7578,6 +7602,13 @@ async function submitRequest(event) {
}
renderResult(payload);
completedResult = payload;
+ if (speechSubmission) {
+ delete utteranceInput.dataset.speechUtteranceId;
+ delete utteranceInput.dataset.speechExecutionEligible;
+ delete utteranceInput.dataset.speechSemanticStatus;
+ delete utteranceInput.dataset.speechValue;
+ delete utteranceInput.dataset.speechUncertain;
+ }
const peripheralRoute = String(payload.orchestration_route || "")
.startsWith("nfc_peripheral");
if (state.executionMode === "hardware" && !peripheralRoute) {
diff --git a/apps/web-console/index.html b/apps/web-console/index.html
index 4014b07..bf34391 100755
--- a/apps/web-console/index.html
+++ b/apps/web-console/index.html
@@ -11,13 +11,15 @@
511工程-智能座舱
+
-
+
+
@@ -73,6 +75,12 @@
+
+
Enter 执行 · Shift + Enter 换行
diff --git a/apps/web-console/service-worker.js b/apps/web-console/service-worker.js
index e3a0c14..1f29abb 100644
--- a/apps/web-console/service-worker.js
+++ b/apps/web-console/service-worker.js
@@ -1,4 +1,4 @@
-const CACHE_VERSION = "cognitive-agent-pad-v210";
+const CACHE_VERSION = "cognitive-agent-pad-v216";
const SHELL_CACHE = `${CACHE_VERSION}-shell`;
const RUNTIME_CACHE = `${CACHE_VERSION}-runtime`;
const APP_SHELL = [
@@ -6,11 +6,14 @@ const APP_SHELL = [
"/index.html",
"/styles.css?v=dual-seat-16",
"/network.js?v=android-pad-2",
- "/app.js?v=dual-seat-103",
+ "/app.js?v=dual-seat-104",
"/gesture-control.js?v=android-pad-9",
"/pwa.js?v=android-pad-3",
"/console-v2.css?v=sync-layout-41",
+ "/speech-input.css?v=sensevoice-4",
"/console-v2.js?v=sync-layout-43",
+ "/speech-input.js?v=sensevoice-6",
+ "/speech-audio-worklet.js?v=sensevoice-4",
"/self-extension.css?v=nfc-toy-33",
"/self-extension.js?v=nfc-toy-33",
"/assets/pets/self-extension-cat.png?v=3",
diff --git a/apps/web-console/speech-audio-worklet.js b/apps/web-console/speech-audio-worklet.js
new file mode 100644
index 0000000..b106709
--- /dev/null
+++ b/apps/web-console/speech-audio-worklet.js
@@ -0,0 +1,29 @@
+class SpeechCaptureProcessor extends AudioWorkletProcessor {
+ constructor() {
+ super();
+ this.frameSize = 2048;
+ this.frame = new Float32Array(this.frameSize);
+ this.offset = 0;
+ }
+
+ process(inputs) {
+ const input = inputs[0]?.[0];
+ if (!input) return true;
+ let inputOffset = 0;
+ while (inputOffset < input.length) {
+ const available = Math.min(this.frameSize - this.offset, input.length - inputOffset);
+ this.frame.set(input.subarray(inputOffset, inputOffset + available), this.offset);
+ this.offset += available;
+ inputOffset += available;
+ if (this.offset === this.frameSize) {
+ const ready = this.frame;
+ this.port.postMessage(ready, [ready.buffer]);
+ this.frame = new Float32Array(this.frameSize);
+ this.offset = 0;
+ }
+ }
+ return true;
+ }
+}
+
+registerProcessor("speech-capture-processor", SpeechCaptureProcessor);
diff --git a/apps/web-console/speech-input.css b/apps/web-console/speech-input.css
new file mode 100644
index 0000000..7d72d54
--- /dev/null
+++ b/apps/web-console/speech-input.css
@@ -0,0 +1,257 @@
+.speech-visually-hidden {
+ width: 1px;
+ height: 1px;
+ position: absolute;
+ overflow: hidden;
+ clip: rect(0 0 0 0);
+ clip-path: inset(50%);
+ white-space: nowrap;
+}
+
+/* Keep all composer actions on one calm, optically aligned rail. */
+.console-v2 .v2-request-form .speech-dictation {
+ width: 36px;
+ height: 36px;
+ min-height: 36px;
+ position: absolute;
+ top: 50%;
+ right: 56px;
+ bottom: auto;
+ z-index: 13;
+ display: grid;
+ place-items: center;
+ margin: 0;
+ padding: 0;
+ color: #284647;
+ background: transparent;
+ border: 0;
+ border-radius: 50%;
+ box-shadow: none;
+ cursor: pointer;
+ transform: translateY(-50%);
+ transition: color 160ms ease, background 160ms ease, box-shadow 160ms ease,
+ transform 160ms ease;
+}
+
+.console-v2 .v2-request-form .speech-dictation:hover:not(:disabled),
+.console-v2 .v2-request-form .speech-dictation:focus-visible {
+ color: #08766e;
+ background: #eaf5f3;
+ box-shadow: 0 0 0 3px rgba(8, 118, 110, 0.08);
+ transform: translateY(-50%) scale(1.05);
+}
+
+.console-v2 .v2-request-form .speech-dictation:focus-visible {
+ outline: 2px solid rgba(8, 118, 110, 0.48);
+ outline-offset: 2px;
+}
+
+.console-v2 .v2-request-form .speech-dictation:disabled {
+ color: #9aacaa;
+ cursor: not-allowed;
+ opacity: 0.56;
+}
+
+.console-v2 .v2-request-form .speech-dictation .speech-mic-icon {
+ width: 18px;
+ height: 18px;
+ stroke-width: 2;
+}
+
+.console-v2 .v2-request-form .speech-waveform {
+ height: 16px;
+ display: none;
+ align-items: center;
+ justify-content: center;
+ gap: 2px;
+}
+
+.console-v2 .v2-request-form .speech-waveform i {
+ width: 2px;
+ height: 5px;
+ display: block;
+ background: currentColor;
+ border-radius: 999px;
+}
+
+.console-v2 .v2-request-form .speech-waveform i:nth-child(2),
+.console-v2 .v2-request-form .speech-waveform i:nth-child(4) {
+ height: 10px;
+}
+
+.console-v2 .v2-request-form .speech-waveform i:nth-child(3) {
+ height: 15px;
+}
+
+.console-v2 .v2-request-form .speech-dictation.is-listening,
+.console-v2 .v2-request-form .speech-dictation.is-listening:hover:not(:disabled),
+.console-v2 .v2-request-form .speech-dictation.is-listening:focus-visible {
+ color: #fff;
+ background: #132c2c;
+ box-shadow: 0 0 0 5px rgba(19, 44, 44, 0.09), 0 7px 18px rgba(17, 47, 46, 0.2);
+}
+
+.console-v2 .v2-request-form .speech-dictation.is-listening .speech-mic-icon,
+.console-v2 .v2-request-form .speech-dictation.is-processing .speech-mic-icon {
+ display: none;
+}
+
+.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform,
+.console-v2 .v2-request-form .speech-dictation.is-processing .speech-waveform {
+ display: flex;
+}
+
+.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform i {
+ animation: speech-wave 760ms ease-in-out infinite alternate;
+}
+
+.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform i:nth-child(2) {
+ animation-delay: -420ms;
+}
+
+.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform i:nth-child(3) {
+ animation-delay: -180ms;
+}
+
+.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform i:nth-child(4) {
+ animation-delay: -560ms;
+}
+
+.console-v2 .v2-request-form .speech-dictation.is-processing {
+ color: #08766e;
+ background: #e5f3f1;
+}
+
+.console-v2 .v2-request-form .speech-dictation.is-processing .speech-waveform {
+ animation: speech-processing-breathe 900ms ease-in-out infinite alternate;
+}
+
+.console-v2 .v2-request-form .speech-status {
+ min-height: 16px;
+ position: absolute;
+ right: 140px;
+ bottom: 10px;
+ left: 194px;
+ z-index: 12;
+ overflow: hidden;
+ color: #5f7474;
+ font-size: 10px;
+ font-weight: 650;
+ line-height: 16px;
+ letter-spacing: 0.01em;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ pointer-events: none;
+}
+
+.console-v2 .v2-request-form .command-composer[data-speech-state="listening"] {
+ border-color: rgba(19, 76, 73, 0.36);
+ box-shadow: 0 12px 35px rgba(25, 72, 70, 0.1), inset 0 0 0 1px rgba(19, 76, 73, 0.08);
+}
+
+.console-v2 .v2-request-form .command-composer[data-speech-state="listening"] .speech-status {
+ color: #183e3d;
+}
+
+.console-v2 .v2-request-form .command-composer[data-speech-state="processing"] .speech-status {
+ color: #39706d;
+}
+
+.console-v2 .v2-request-form .command-composer[data-speech-state="success"] .speech-status {
+ color: #08766e;
+}
+
+.console-v2 .v2-request-form #utterance[data-speech-uncertain="true"] {
+ background: linear-gradient(90deg, rgba(201, 132, 31, 0.055), transparent 42%);
+ box-shadow: inset 3px 0 0 rgba(201, 132, 31, 0.58);
+}
+
+.console-v2 .v2-request-form .command-composer[data-speech-state="error"] .speech-status {
+ color: #b54b42;
+}
+
+.console-v2 .v2-request-form .command-composer:not([data-speech-state="idle"]) .composer-hint {
+ opacity: 0;
+}
+
+.console-v2 .v2-request-form #utterance,
+.console-v2.v2-interactive-sync .v2-request-form #utterance,
+.console-v2.v2-passive-display.v2-interactive-sync .v2-request-form #utterance {
+ padding-right: 142px;
+}
+
+.console-v2 .v2-request-form .context-reset,
+.console-v2.v2-interactive-sync .v2-request-form .context-reset,
+.console-v2.v2-passive-display.v2-interactive-sync .v2-request-form .context-reset {
+ width: 30px;
+ height: 30px;
+ top: 50%;
+ right: 98px;
+ bottom: auto;
+ color: #75908e;
+ border-radius: 50%;
+ transform: translateY(-50%);
+}
+
+.console-v2 .v2-request-form .context-reset:hover,
+.console-v2 .v2-request-form .context-reset:focus-visible {
+ color: #08766e;
+ background: #edf6f4;
+}
+
+.console-v2 .v2-request-form .composer-hint,
+.console-v2.v2-interactive-sync .v2-request-form .composer-hint,
+.console-v2.v2-passive-display.v2-interactive-sync .v2-request-form .composer-hint {
+ right: 140px;
+}
+
+@keyframes speech-wave {
+ from { transform: scaleY(0.42); }
+ to { transform: scaleY(1); }
+}
+
+@keyframes speech-processing-breathe {
+ from { opacity: 0.42; transform: scaleX(0.78); }
+ to { opacity: 1; transform: scaleX(1); }
+}
+
+@media (max-width: 720px) {
+ .console-v2 .v2-request-form .speech-status {
+ top: 8px;
+ right: 140px;
+ bottom: auto;
+ left: 14px;
+ font-size: 9px;
+ }
+
+ .console-v2 .v2-request-form .command-composer:not([data-speech-state="idle"]) #utterance {
+ padding-top: 28px;
+ }
+}
+
+@media (max-width: 460px) {
+ .console-v2 .v2-request-form #utterance,
+ .console-v2.v2-interactive-sync .v2-request-form #utterance,
+ .console-v2.v2-passive-display.v2-interactive-sync .v2-request-form #utterance {
+ padding-right: 132px;
+ }
+
+ .console-v2 .v2-request-form .speech-dictation {
+ right: 52px;
+ }
+
+ .console-v2 .v2-request-form .context-reset,
+ .console-v2.v2-interactive-sync .v2-request-form .context-reset,
+ .console-v2.v2-passive-display.v2-interactive-sync .v2-request-form .context-reset {
+ right: 92px;
+ }
+}
+
+@media (prefers-reduced-motion: reduce) {
+ .console-v2 .v2-request-form .speech-dictation,
+ .console-v2 .v2-request-form .speech-waveform,
+ .console-v2 .v2-request-form .speech-waveform i {
+ animation: none !important;
+ transition: none;
+ }
+}
diff --git a/apps/web-console/speech-input.js b/apps/web-console/speech-input.js
new file mode 100644
index 0000000..4557d66
--- /dev/null
+++ b/apps/web-console/speech-input.js
@@ -0,0 +1,527 @@
+(() => {
+ "use strict";
+
+ const TARGET_SAMPLE_RATE = 16000;
+ const MAX_RECORDING_MS = 28000;
+ const NO_SPEECH_TIMEOUT_MS = 8000;
+ const ENDPOINT_SILENCE_MS = 1050;
+ const STATUS_RESET_MS = 5000;
+ const MICROPHONE_CONNECT_TIMEOUT_MS = 12000;
+ const PREFERRED_MICROPHONE = /TAKSTAR\s*V2/i;
+ const DEBUG = new URLSearchParams(window.location.search).get("speechDebug") === "1"
+ || window.localStorage.getItem("speechDebug") === "1";
+ const speech = {
+ phase: "idle",
+ backendReady: false,
+ stream: null,
+ context: null,
+ contextPromise: null,
+ workletReady: false,
+ workletFailed: false,
+ source: null,
+ captureNode: null,
+ silentGain: null,
+ chunks: [],
+ totalSamples: 0,
+ sampleRate: 0,
+ startedAt: 0,
+ speechStartSample: null,
+ lastVoiceSample: null,
+ consecutiveVoiceSamples: 0,
+ noiseFloor: 0.003,
+ statusTimer: null,
+ generation: 0,
+ utteranceId: "",
+ finishPromise: null,
+ applyingResult: false,
+ lastResult: null,
+ };
+
+ const elements = () => ({
+ button: document.querySelector("#speechDictationButton"),
+ composer: document.querySelector("#speechDictationButton")?.closest(".command-composer"),
+ input: document.querySelector("#utterance"),
+ status: document.querySelector("#speechDictationStatus"),
+ });
+
+ function debug(stage, details = {}) {
+ if (!DEBUG) return;
+ console.debug("[speech]", { utteranceId: speech.utteranceId, stage, ...details });
+ }
+
+ function announce(message) {
+ const notify = window.CognitiveAgentConsole?.notify;
+ if (typeof notify === "function") notify(message);
+ }
+
+ function setPhase(phase, message = "") {
+ speech.phase = phase;
+ const { button, composer, status } = elements();
+ if (!button || !composer || !status) return;
+ window.clearTimeout(speech.statusTimer);
+ composer.dataset.speechState = phase;
+ status.textContent = message;
+ button.classList.toggle("is-listening", phase === "listening");
+ button.classList.toggle("is-processing", phase === "processing");
+ button.disabled = phase === "processing" || (!speech.backendReady && phase !== "listening");
+ button.setAttribute("aria-pressed", String(phase === "listening"));
+ button.setAttribute(
+ "aria-label",
+ phase === "listening" ? "结束语音听写" : phase === "processing" ? "正在转写语音" : "开始语音听写",
+ );
+ button.title = phase === "listening"
+ ? "点击结束听写"
+ : speech.backendReady
+ ? "语音听写（自动识别说话结束）"
+ : "语音识别模型未就绪";
+ if (["success", "error"].includes(phase)) {
+ speech.statusTimer = window.setTimeout(() => setPhase("idle"), STATUS_RESET_MS);
+ }
+ }
+
+ function microphoneErrorMessage(error) {
+ switch (error?.name) {
+ case "NotAllowedError":
+ return "未获得麦克风权限，请在浏览器地址栏允许访问麦克风。";
+ case "NotFoundError":
+ return "没有找到可用麦克风，请检查 TAKSTAR V2 接收器。";
+ case "NotReadableError":
+ return "麦克风正被其他程序占用，请关闭占用程序后重试。";
+ case "OverconstrainedError":
+ return "TAKSTAR V2 当前不可用，请重新连接接收器。";
+ case "SecurityError":
+ return "当前页面不允许访问麦克风，请使用 HTTPS 或本机地址。";
+ case "TimeoutError":
+ return "等待麦克风授权超时，请允许访问后重试。";
+ default:
+ return "无法启动麦克风，请检查设备连接和浏览器权限。";
+ }
+ }
+
+ async function getUserMediaWithTimeout(constraints) {
+ let expired = false;
+ let timer = 0;
+ const request = navigator.mediaDevices.getUserMedia(constraints);
+ const timeout = new Promise((_, reject) => {
+ timer = window.setTimeout(() => {
+ expired = true;
+ reject(new DOMException("microphone permission timed out", "TimeoutError"));
+ }, MICROPHONE_CONNECT_TIMEOUT_MS);
+ });
+ request.then((lateStream) => {
+ if (expired) lateStream.getTracks().forEach((track) => track.stop());
+ }).catch(() => {});
+ try {
+ return await Promise.race([request, timeout]);
+ } finally {
+ window.clearTimeout(timer);
+ }
+ }
+
+ async function preferredMicrophoneStream() {
+ const constraints = {
+ channelCount: 1,
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: true,
+ };
+ let stream = await getUserMediaWithTimeout({ audio: constraints, video: false });
+ const currentTrack = stream.getAudioTracks()[0];
+ const devices = await navigator.mediaDevices.enumerateDevices();
+ const preferred = devices.find(
+ (device) => device.kind === "audioinput" && PREFERRED_MICROPHONE.test(device.label),
+ );
+ if (preferred && preferred.deviceId !== currentTrack?.getSettings?.().deviceId) {
+ stream.getTracks().forEach((track) => track.stop());
+ stream = await getUserMediaWithTimeout({
+ audio: { ...constraints, deviceId: { exact: preferred.deviceId } },
+ video: false,
+ });
+ }
+ return stream;
+ }
+
+ async function ensureAudioContext() {
+ if (speech.context && speech.context.state !== "closed") {
+ await speech.context.resume();
+ return speech.context;
+ }
+ if (speech.contextPromise) return speech.contextPromise;
+ speech.contextPromise = (async () => {
+ const AudioContext = window.AudioContext || window.webkitAudioContext;
+ const context = new AudioContext({ latencyHint: "interactive" });
+ await context.resume();
+ if (context.audioWorklet && window.AudioWorkletNode) {
+ try {
+ await context.audioWorklet.addModule("/speech-audio-worklet.js?v=sensevoice-4");
+ speech.workletReady = true;
+ } catch (error) {
+ speech.workletFailed = true;
+ debug("audio_worklet_fallback", { message: error?.message || String(error) });
+ }
+ } else {
+ speech.workletFailed = true;
+ }
+ speech.context = context;
+ return context;
+ })();
+ try {
+ return await speech.contextPromise;
+ } finally {
+ speech.contextPromise = null;
+ }
+ }
+
+ function audioRms(samples) {
+ let sum = 0;
+ for (let index = 0; index < samples.length; index += 1) sum += samples[index] * samples[index];
+ return Math.sqrt(sum / Math.max(1, samples.length));
+ }
+
+ function handleAudio(chunk, generation) {
+ if (speech.phase !== "listening" || generation !== speech.generation) return;
+ speech.chunks.push(chunk);
+ speech.totalSamples += chunk.length;
+ const elapsed = performance.now() - speech.startedAt;
+ const rms = audioRms(chunk);
+ if (elapsed < 700) speech.noiseFloor = (speech.noiseFloor * 0.82) + (rms * 0.18);
+ const voiceThreshold = Math.max(0.009, Math.min(0.035, speech.noiseFloor * 2.8));
+ if (rms >= voiceThreshold) {
+ speech.consecutiveVoiceSamples += chunk.length;
+ if (speech.consecutiveVoiceSamples >= speech.sampleRate * 0.09) {
+ if (speech.speechStartSample === null) {
+ speech.speechStartSample = Math.max(0, speech.totalSamples - (speech.sampleRate * 0.35));
+ setPhase("listening", "听到声音了，请继续说…");
+ debug("speech_started", { sample: speech.speechStartSample, voiceThreshold });
+ }
+ speech.lastVoiceSample = speech.totalSamples;
+ }
+ } else {
+ speech.consecutiveVoiceSamples = 0;
+ }
+ if (
+ speech.lastVoiceSample !== null
+ && speech.totalSamples - speech.lastVoiceSample >= speech.sampleRate * (ENDPOINT_SILENCE_MS / 1000)
+ ) {
+ void finishDictation();
+ } else if (speech.speechStartSample === null && elapsed >= NO_SPEECH_TIMEOUT_MS) {
+ void cancelDictation("未检测到说话声，请靠近麦克风重试。");
+ } else if (elapsed >= MAX_RECORDING_MS) {
+ void finishDictation();
+ }
+ }
+
+ function connectCapture(context, generation) {
+ speech.source = context.createMediaStreamSource(speech.stream);
+ speech.silentGain = context.createGain();
+ speech.silentGain.gain.value = 0;
+ if (speech.workletReady && !speech.workletFailed) {
+ speech.captureNode = new AudioWorkletNode(context, "speech-capture-processor", {
+ numberOfInputs: 1,
+ numberOfOutputs: 1,
+ channelCount: 1,
+ });
+ speech.captureNode.port.onmessage = (event) => {
+ handleAudio(new Float32Array(event.data), generation);
+ };
+ debug("capture_connected", { implementation: "AudioWorklet" });
+ } else {
+ speech.captureNode = context.createScriptProcessor(2048, 1, 1);
+ speech.captureNode.onaudioprocess = (event) => {
+ handleAudio(new Float32Array(event.inputBuffer.getChannelData(0)), generation);
+ };
+ debug("capture_connected", { implementation: "ScriptProcessor-fallback" });
+ }
+ speech.source.connect(speech.captureNode);
+ speech.captureNode.connect(speech.silentGain);
+ speech.silentGain.connect(context.destination);
+ }
+
+ async function startDictation() {
+ if (speech.phase !== "idle" && !["success", "error"].includes(speech.phase)) return;
+ if (!speech.backendReady) {
+ setPhase("error", "语音识别模型未就绪");
+ return;
+ }
+ if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
+ setPhase("error", "请使用 HTTPS 或 127.0.0.1 开启语音听写");
+ announce("当前页面不支持安全的麦克风访问");
+ return;
+ }
+ const generation = speech.generation + 1;
+ speech.generation = generation;
+ speech.utteranceId = `web-${crypto.randomUUID?.() || `${Date.now()}-${generation}`}`;
+ try {
+ setPhase("processing", "正在连接麦克风…");
+ speech.stream = await preferredMicrophoneStream();
+ if (generation !== speech.generation) return;
+ const context = await ensureAudioContext();
+ speech.sampleRate = context.sampleRate;
+ speech.chunks = [];
+ speech.totalSamples = 0;
+ speech.startedAt = performance.now();
+ speech.speechStartSample = null;
+ speech.lastVoiceSample = null;
+ speech.consecutiveVoiceSamples = 0;
+ speech.noiseFloor = 0.003;
+ connectCapture(context, generation);
+ const trackLabel = speech.stream.getAudioTracks()[0]?.label || "默认麦克风";
+ setPhase("listening", `正在听写 · ${trackLabel}`);
+ debug("utterance_started", { sampleRate: speech.sampleRate, trackLabel });
+ } catch (error) {
+ await releaseMicrophone();
+ const message = microphoneErrorMessage(error);
+ setPhase("error", message);
+ announce(message);
+ }
+ }
+
+ async function releaseMicrophone({ closeContext = false } = {}) {
+ if (speech.captureNode?.port) speech.captureNode.port.onmessage = null;
+ if (speech.captureNode && "onaudioprocess" in speech.captureNode) {
+ speech.captureNode.onaudioprocess = null;
+ }
+ for (const node of [speech.source, speech.captureNode, speech.silentGain]) {
+ try { node?.disconnect(); } catch (_error) { /* already disconnected */ }
+ }
+ speech.stream?.getTracks().forEach((track) => track.stop());
+ speech.stream = null;
+ speech.source = null;
+ speech.captureNode = null;
+ speech.silentGain = null;
+ if (closeContext && speech.context && speech.context.state !== "closed") {
+ try { await speech.context.close(); } catch (_error) { /* page is closing */ }
+ speech.context = null;
+ speech.workletReady = false;
+ }
+ }
+
+ function concatenateChunks() {
+ const combined = new Float32Array(speech.totalSamples);
+ let offset = 0;
+ for (const chunk of speech.chunks) {
+ combined.set(chunk, offset);
+ offset += chunk.length;
+ }
+ const start = Math.max(0, Math.floor(speech.speechStartSample || 0));
+ const end = Math.min(
+ combined.length,
+ Math.ceil((speech.lastVoiceSample || combined.length) + (speech.sampleRate * 0.45)),
+ );
+ return combined.slice(start, end);
+ }
+
+ function downsample(samples, inputRate) {
+ if (inputRate === TARGET_SAMPLE_RATE) return samples;
+ const ratio = inputRate / TARGET_SAMPLE_RATE;
+ const output = new Float32Array(Math.max(1, Math.floor(samples.length / ratio)));
+ for (let outputIndex = 0; outputIndex < output.length; outputIndex += 1) {
+ const start = Math.floor(outputIndex * ratio);
+ const end = Math.max(start + 1, Math.min(samples.length, Math.floor((outputIndex + 1) * ratio)));
+ let sum = 0;
+ for (let inputIndex = start; inputIndex < end; inputIndex += 1) sum += samples[inputIndex];
+ output[outputIndex] = sum / (end - start);
+ }
+ return output;
+ }
+
+ function encodePcm16Wav(samples) {
+ const buffer = new ArrayBuffer(44 + (samples.length * 2));
+ const view = new DataView(buffer);
+ const writeAscii = (offset, value) => {
+ for (let index = 0; index < value.length; index += 1) view.setUint8(offset + index, value.charCodeAt(index));
+ };
+ writeAscii(0, "RIFF");
+ view.setUint32(4, 36 + (samples.length * 2), true);
+ writeAscii(8, "WAVE");
+ writeAscii(12, "fmt ");
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, 1, true);
+ view.setUint32(24, TARGET_SAMPLE_RATE, true);
+ view.setUint32(28, TARGET_SAMPLE_RATE * 2, true);
+ view.setUint16(32, 2, true);
+ view.setUint16(34, 16, true);
+ writeAscii(36, "data");
+ view.setUint32(40, samples.length * 2, true);
+ for (let index = 0; index < samples.length; index += 1) {
+ const sample = Math.max(-1, Math.min(1, samples[index]));
+ view.setInt16(44 + (index * 2), sample < 0 ? sample * 32768 : sample * 32767, true);
+ }
+ return buffer;
+ }
+
+ async function transcribe(wav, utteranceId) {
+ const controller = new AbortController();
+ const timeout = window.setTimeout(() => controller.abort(), 45000);
+ try {
+ const response = await fetch("/api/speech/transcribe", {
+ method: "POST",
+ credentials: "same-origin",
+ cache: "no-store",
+ headers: {
+ Accept: "application/json",
+ "Content-Type": "audio/wav",
+ "X-Speech-Utterance-Id": utteranceId,
+ },
+ body: wav,
+ signal: controller.signal,
+ });
+ const payload = await response.json().catch(() => ({}));
+ if (!response.ok) throw new Error(payload.message || `语音转写失败（HTTP ${response.status}）`);
+ debug("server_result", payload);
+ return payload;
+ } finally {
+ window.clearTimeout(timeout);
+ }
+ }
+
+ function applyResult(result) {
+ const { input } = elements();
+ if (!input) throw new Error("没有找到一句话编排输入框");
+ const cleanText = String(result.clean_text || result.text || "").trim();
+ const commands = Array.isArray(result.commands) ? result.commands : [];
+ if (!cleanText || commands.length === 0) {
+ window.dispatchEvent(new CustomEvent("cognitive-agent:speech-result", { detail: result }));
+ return false;
+ }
+ speech.applyingResult = true;
+ input.value = cleanText.slice(0, input.maxLength || 1000);
+ input.dataset.speechUncertain = String(result.uncertain === true);
+ input.dataset.speechExecutionEligible = String(result.execution_eligible === true);
+ input.dataset.speechSemanticStatus = String(result.semantic_status || "");
+ input.dataset.speechValue = input.value.trim();
+ input.dataset.speechUtteranceId = String(result.utterance_id || "");
+ input.dispatchEvent(new Event("input", { bubbles: true }));
+ speech.applyingResult = false;
+ input.focus({ preventScroll: true });
+ window.dispatchEvent(new CustomEvent("cognitive-agent:speech-result", { detail: result }));
+ return true;
+ }
+
+ async function finishDictation() {
+ if (speech.finishPromise) return speech.finishPromise;
+ if (speech.phase !== "listening") return undefined;
+ speech.finishPromise = (async () => {
+ setPhase("processing", "说话结束，正在识别并理解指令…");
+ const hadSpeech = speech.speechStartSample !== null;
+ const samples = hadSpeech ? concatenateChunks() : null;
+ const inputRate = speech.sampleRate;
+ const utteranceId = speech.utteranceId;
+ await releaseMicrophone();
+ speech.chunks = [];
+ if (!hadSpeech || !samples?.length) {
+ setPhase("error", "未检测到说话声，请重试");
+ return;
+ }
+ try {
+ const wav = encodePcm16Wav(downsample(samples, inputRate));
+ const result = await transcribe(wav, utteranceId);
+ speech.lastResult = result;
+ const applied = applyResult(result);
+ const latency = Math.round(result.timings?.total_ms ?? result.inference_ms ?? 0);
+ if (!applied) {
+ setPhase("success", `未形成可执行控制指令 · ${latency} ms`);
+ announce("已保留原始转录，但没有激活控制指令");
+ } else if (result.semantic_status === "recognized_unsupported") {
+ setPhase("success", `已识别，但当前车辆能力不支持 · ${latency} ms`);
+ announce("已完整保留用户语义，但当前车辆能力不支持执行");
+ } else if (result.semantic_status === "incomplete") {
+ setPhase("success", `已识别，但控制参数不完整 · ${latency} ms`);
+ announce("语音中的关键控制参数不完整，当前不会执行");
+ } else if (result.uncertain === true) {
+ setPhase("success", `已转写，但存在歧义，请修改确认 · ${latency} ms`);
+ announce("语音结果需要人工确认，当前不会直接执行");
+ } else {
+ setPhase("success", `已理解为 ${result.commands.length} 条指令 · ${latency} ms`);
+ announce("语音已转换为理解式控制指令");
+ }
+ } catch (error) {
+ const message = error?.name === "AbortError" ? "语音转写超时，请重试。" : error.message;
+ setPhase("error", message || "语音转写失败，请重试");
+ announce(message || "语音转写失败");
+ }
+ })();
+ try {
+ return await speech.finishPromise;
+ } finally {
+ speech.finishPromise = null;
+ }
+ }
+
+ async function cancelDictation(message = "已取消语音听写") {
+ if (!speech.stream && speech.phase === "idle") return;
+ speech.phase = "processing";
+ speech.generation += 1;
+ await releaseMicrophone();
+ speech.chunks = [];
+ setPhase("error", message);
+ }
+
+ async function loadSpeechStatus() {
+ const { button } = elements();
+ if (!button) return;
+ if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
+ button.disabled = true;
+ setPhase("error", "语音听写需要 HTTPS 或本机地址");
+ return;
+ }
+ try {
+ const response = await fetch("/api/speech/status", {
+ credentials: "same-origin",
+ cache: "no-store",
+ headers: { Accept: "application/json" },
+ });
+ const status = await response.json();
+ speech.backendReady = response.ok && status.enabled === true;
+ debug("backend_status", status);
+ setPhase("idle");
+ if (!speech.backendReady) {
+ button.disabled = true;
+ button.title = "SenseVoiceSmall 模型未部署";
+ }
+ } catch (_error) {
+ speech.backendReady = false;
+ button.disabled = true;
+ button.title = "无法连接语音识别服务";
+ }
+ }
+
+ function clearSpeechGuardOnManualEdit(event) {
+ if (speech.applyingResult || !event.isTrusted) return;
+ const { input } = elements();
+ if (!input) return;
+ delete input.dataset.speechUncertain;
+ delete input.dataset.speechExecutionEligible;
+ delete input.dataset.speechSemanticStatus;
+ delete input.dataset.speechValue;
+ delete input.dataset.speechUtteranceId;
+ }
+
+ function initialize() {
+ const { button, input } = elements();
+ if (!button) return;
+ button.addEventListener("click", () => {
+ if (speech.phase === "listening") void finishDictation();
+ else void startDictation();
+ });
+ input?.addEventListener("input", clearSpeechGuardOnManualEdit);
+ document.addEventListener("keydown", (event) => {
+ if (event.key === "Escape" && speech.phase === "listening") {
+ event.preventDefault();
+ void cancelDictation();
+ }
+ });
+ document.addEventListener("visibilitychange", () => {
+ if (document.hidden && speech.phase === "listening") void cancelDictation();
+ });
+ window.addEventListener("pagehide", () => { void releaseMicrophone({ closeContext: true }); });
+ void loadSpeechStatus();
+ }
+
+ window.CognitiveAgentSpeech = {
+ get lastResult() { return speech.lastResult; },
+ get debugEnabled() { return DEBUG; },
+ };
+ document.addEventListener("DOMContentLoaded", initialize);
+})();
diff --git a/config/native_can_capabilities/driver_seat.yaml b/config/native_can_capabilities/driver_seat.yaml
index 452189c..01052a7 100644
--- a/config/native_can_capabilities/driver_seat.yaml
+++ b/config/native_can_capabilities/driver_seat.yaml
@@ -18,7 +18,7 @@ spec:
- name: state
type: string
required: true
- allowed_values: [on, off]
+ allowed_values: ["on", "off"]
description: Absolute ventilation state.
- name: level
type: integer
@@ -41,7 +41,7 @@ spec:
- name: state
type: string
required: true
- allowed_values: [on, off]
+ allowed_values: ["on", "off"]
description: Absolute heating state.
- name: level
type: integer
@@ -64,7 +64,7 @@ spec:
- name: action
type: string
required: true
- allowed_values: [off, strength, mode]
+ allowed_values: ["off", strength, mode]
description: Explicit massage operation; a generic on request is not accepted.
- name: strength
type: integer
diff --git a/contracts/schemas/function-config.schema.json b/contracts/schemas/function-config.schema.json
index 1774de8..3da1347 100755
--- a/contracts/schemas/function-config.schema.json
+++ b/contracts/schemas/function-config.schema.json
@@ -14,7 +14,7 @@
"parameters": {"type": "object"},
"parse_source": {
"type": "string",
- "enum": ["local_rule", "cloud_model", "cloud_supplement", "direct_control"]
+ "enum": ["local_rule", "cloud_model", "cloud_supplement", "direct_control", "speech_semantic"]
},
"motion": {
"type": "object",
diff --git a/docs/speech-benchmark.md b/docs/speech-benchmark.md
new file mode 100644
index 0000000..d9c6117
--- /dev/null
+++ b/docs/speech-benchmark.md
@@ -0,0 +1,67 @@
+# Speech pipeline benchmark — 2026-08-28
+
+## Environment
+
+- Windows / Python 3.13
+- `sherpa-onnx==1.13.6`, CPU provider, 8 ASR threads
+- SenseVoiceSmall full precision, `language=zh`, `use_itn=true`
+- Silero VAD: threshold 0.45, min speech 0.16 s, min silence 0.65 s
+- 48 text regressions（including reported field failures plus ratio/coordination matrices）
+- 16 capability-alignment cases replayed 100 times（1600 samples）
+- 43 catalog capability definitions × 5 resolver projections（215 subcases）
+- 10 local 16 kHz mono Windows SAPI WAVs covering filler, target refinement,
+ cross-domain correction, normal multi-command, number/direction correction,
+ negation, unrelated speech, fast speech and slow speech
+
+The WAV set is synthetic and deliberately kept under ignored `artifacts/tmp`; it is useful for
+repeatability but is not a substitute for TAKSTAR V2 recordings in the real cabin.
+
+## Before / after
+
+| Metric | Before | After |
+|---|---:|---:|
+| Raw exact transcription (10 WAV) | 8/10 | 8/10 |
+| Expected understanding text (same 10 cases) | 2/10 | 10/10 |
+| Intent exact match | no structured speech Intent | 10/10 |
+| Device / action / parameter exact | no structured speech metrics | 10/10 / 10/10 / 10/10 |
+| Correction success | unsafe text-only heuristics | 4/4 |
+| False command activation | 1 observed wrong-direction activation | 0/2 no-command cases |
+| Severe wrong device / wrong number | not reported | 0 / 0 |
+| Duplicate transcript rate | not observable (single callback) | 0% |
+| ASR inference P50 / P95 | 80 / 110 ms | 79.7 / 110.5 ms |
+| Semantic P50 / P95 | 0.03 ms (old regex only) | 0.33 / 0.56 ms（1600 capability samples） |
+| Command resolution P50 / P95 | not available | 0.03 / 0.06 ms |
+| Steady end-to-end P50 / P95 | not instrumented | 95.3 / 134.1 ms |
+| Cold first request | not instrumented end-to-end | 1400.9 ms |
+
+Raw exact stays at 80% because two files differ only in acoustic homophone/punctuation output;
+`raw_text` is intentionally not rewritten. The semantic and business metrics operate on separate
+fields.
+
+## Full precision vs int8 probe
+
+On sherpa-onnx's official Chinese test WAV, both files produced the same transcript in this one
+probe:
+
+| Model | First call including load | Warm P50 | Warm P95 |
+|---|---:|---:|---:|
+| `model.onnx` | 1514.9 ms | 90.9 ms | 92.7 ms |
+| `model.int8.onnx` | 889.6 ms | 77.4 ms | 81.8 ms |
+
+This is insufficient acoustic coverage to prove equal accuracy, so the accuracy-first default
+remains full precision. The model file stays configurable through `SPEECH_MODEL_FILE`.
+
+## VAD tuning probe
+
+Thresholds 0.35 / 0.45 / 0.55 and end silence 0.45 / 0.65 / 0.85 s were replayed against the
+available Chinese test WAV. All combinations found one segment; threshold 0.45 preserved the
+complete utterance boundary. The selected 0.65 s server endpoint, 0.25 s pre-roll and 0.35 s
+post-roll are conservative for short commands. Real cabin noise and long-pause recordings remain
+required before claiming production calibration.
+
+Reproduce current metrics with:
+
+```bash
+.venv/Scripts/python tools/benchmark_speech_pipeline.py \
+ --audio-dir artifacts/tmp/speech-regression-audio
+```
diff --git a/docs/speech-recognition.md b/docs/speech-recognition.md
new file mode 100644
index 0000000..f82dc66
--- /dev/null
+++ b/docs/speech-recognition.md
@@ -0,0 +1,116 @@
+# 中文语音控制链路
+
+## 整改前的实际链路
+
+```text
+getUserMedia
+ -> ScriptProcessor(4096)
+ -> 浏览器 RMS 阈值与前后裁剪
+ -> 单个 16 kHz PCM WAV
+ -> POST /api/speech/transcribe
+ -> 单个 sherpa-onnx OfflineStream
+ -> SenseVoiceSmall 单次 final decode
+ -> regex 文本清理
+ -> 直接覆盖输入框
+ -> 用户提交后进入既有 IntentIR / capability / CAN 链路
+```
+
+旧实现没有 AudioWorklet、ASR Web Worker、WASM、streaming partial、VAD segment、
+chunk overlap 或 segment assembler。因此观测到的“空调打开打……靠背靠座椅靠背”
+不是 partial/final 被前端重复 append，而是 SenseVoice 单次 decode 已返回该 raw 文本；
+旧正则层只是尝试掩盖 acoustic false start，且没有结构化 cancelled intent。
+
+## 整改后的链路
+
+```text
+Microphone (优先 TAKSTAR V2)
+ -> AudioContext（跨 utterance 复用）
+ -> AudioWorklet（旧环境 ScriptProcessor fallback）
+ -> 自适应 RMS endpoint + 350 ms 前置 / 450 ms 后置采集
+ -> 16 kHz PCM WAV + utteranceId
+ -> Silero VAD（0.25 s pre-roll / 0.35 s post-roll，重叠先合并）
+ -> SenseVoiceSmall zh + ITN（每个 VAD segment 一个 OfflineStream）
+ -> final ASR callback
+ -> TranscriptAssembler（replace / commit / order / idempotency / overlap）
+ -> rawText
+ -> UnderstandingTranscriptProcessor
+ -> cleanText + active commands + cancelledFragments + uncertain
+ -> 页面展示与提交安全门
+ -> 用户确认后进入既有 IntentIR / capability / CAN 链路
+```
+
+模型、tokens、VAD native session 和浏览器 AudioContext 都只初始化一次。服务端请求以
+`utteranceId + SHA-256(audio)` 幂等：相同音频返回缓存，不同音频复用同一 ID 返回 409。
+
+## 数据边界
+
+- `raw_text`：SenseVoice 原始证据，只去模型标签和多余空白，永不被语义文本覆盖。
+- `clean_text`：用于输入框显示的理解式文本。
+- `commands`：与工程 capability catalog 对齐的 active Intent 预览。
+- `cancelled_fragments`：否定、自我修正、false start 或显式撤销的证据及 cancelled Intent。
+- `uncertain`：参数越界、缺少方向/单位/设备、拼接冲突等不能安全执行的情况。
+- `execution_eligible`：存在 active commands、无 uncertain、无校验错误时才为 true。
+
+语音 API 本身不执行任何能力。页面会拒绝直接提交仍带有原始 uncertain 标记的语音结果；
+用户编辑后才解除该标记。
+
+## Capability 驱动的语义链路
+
+语音层不再维护可执行设备表。能力事实按以下优先级读取：
+
+1. `domains/*/config/capabilities.yaml` 与
+ `config/native_can_capabilities/*.yaml`；
+2. `CapabilityRegistry` 统一加载 catalog；
+3. `NativeCanRuntime.supported_capabilities`、domain runtime、seat compiler 和
+ mode profile registry 确认可执行性与 executor binding；
+4. `CapabilityResolver` 只把中文槽位或既有一句话解析 action 适配到上述事实。
+
+`UnderstandingTranscriptProcessor` 会优先复用
+`application/native_can_intent.py` 已有的确定性解析结果，所以驾驶台默认目标、CH1
+主驾能力和网页一句话编排保持一致。`vocabulary.py` 只包含 alias、同义词、颜色与
+scope 词法映射，不声明车辆支持哪些能力。
+
+每个 command 同时保留 `target_scope`、`property`、`operation`、`value`、`unit`、
+`qualifiers`、IntentIR action、executor binding 和 executor payload。`clean_text` 从
+这些结构化 command 渲染；unsupported、ambiguous、incomplete 不会产生假的
+executor payload，且 UI 会阻止整条原始语音直接提交。
+
+当前代码证据中，前大灯只有反馈和只读 UI，没有控制 capability；因此“打开前大灯”
+为 `recognized_unsupported`。零重力是 `config/modes/vehicles/denza_n7/zero_gravity.yaml`
+定义的副驾四轴 mode profile，不是主驾单一 capability；明确要求主驾时返回
+`recognized_unsupported`。
+
+能力矩阵始终从 catalog 动态生成：
+
+```bash
+python tools/report_capability_matrix.py --format markdown
+```
+
+## Debug trace
+
+生产默认关闭。设置 `SPEECH_DEBUG=1` 后，每条 JSON 日志包含：
+
+- `utteranceId`；
+- `segmentId`、sequence、开始/结束时间、音频 overlap；
+- raw ASR final、进入拼接器前文本、拼接后文本；
+- clean text、active/cancelled Intent、uncertain reason；
+- audio、VAD、每段 ASR、assembly、semantic、recovery、total 耗时。
+
+`SPEECH_DEBUG_RESPONSE=1` 才把 trace 放进 API 响应。浏览器使用 `?speechDebug=1`
+查看采集实现、设备、阈值、utteranceId 和服务器结果。
+
+## 默认 VAD 与识别配置
+
+| 配置 | 默认值 |
+|---|---:|
+| language | `zh` |
+| use_itn | `true` |
+| Silero threshold | `0.45` |
+| min speech | `0.16 s` |
+| min silence | `0.65 s` |
+| max speech | `12.0 s` |
+| pre-roll | `0.25 s` |
+| post-roll | `0.35 s` |
+
+参数可通过 `SPEECH_VAD_*` 调整。真实 TAKSTAR V2 车内噪声录音仍需要继续扩展 audio
+regression suite；没有真实证据时不自动切换到 int8，也不自动下载 Recovery 模型。
diff --git a/pyproject.toml b/pyproject.toml
index c3b2985..1798c86 100755
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,6 +19,7 @@ api = ["fastapi>=0.110", "uvicorn>=0.27"]
can = ["cantools>=39", "python-can>=4.3"]
llm = ["openai>=1.30"]
ir = ["pyserial>=3.5,<4"]
+speech = ["sherpa-onnx==1.13.6", "sherpa-onnx-bin==1.13.6"]
test = ["pytest>=8", "pytest-cov>=5"]
dev = ["mypy>=1.9", "ruff>=0.4"]
diff --git a/src/cognitive_agent/application/capability_registry.py b/src/cognitive_agent/application/capability_registry.py
index 3320f45..7128eb1 100755
--- a/src/cognitive_agent/application/capability_registry.py
+++ b/src/cognitive_agent/application/capability_registry.py
@@ -2,23 +2,66 @@
from __future__ import annotations
+from dataclasses import dataclass
from pathlib import Path
from typing import Any
import yaml
+@dataclass(frozen=True)
+class CapabilityDefinition:
+ """One capability together with its authoritative catalog provenance."""
+
+ name: str
+ domain: str
+ availability: str
+ catalog_id: str
+ catalog_path: Path
+ executor_binding: str
+ spec: dict[str, Any]
+
+ @property
+ def target_devices(self) -> tuple[str, ...]:
+ return tuple(str(value) for value in self.spec.get("target_devices", ()))
+
+ @property
+ def target_scopes(self) -> tuple[str, ...]:
+ return tuple(str(value) for value in self.spec.get("target_scopes", ()))
+
+ @property
+ def parameters(self) -> tuple[dict[str, Any], ...]:
+ return tuple(dict(value) for value in self.spec.get("parameters", ()))
+
+
class CapabilityRegistry:
"""Expose domain catalogs without embedding domain semantics in prompts or code."""
def __init__(self, repository_root: Path, executable_domains: set[str]) -> None:
self.root = repository_root
self.executable_domains = set(executable_domains)
+ self._definitions: list[CapabilityDefinition] = []
self._snapshot = self._load()
def snapshot(self) -> dict[str, Any]:
return self._snapshot
+ def definitions(self, *, domain: str | None = None) -> tuple[CapabilityDefinition, ...]:
+ """Return catalog entries without inventing aliases or execution support."""
+
+ return tuple(
+ definition
+ for definition in self._definitions
+ if domain is None or definition.domain == domain
+ )
+
+ def find(self, name: str) -> tuple[CapabilityDefinition, ...]:
+ """Return every authoritative definition for a canonical capability id."""
+
+ return tuple(
+ definition for definition in self._definitions if definition.name == name
+ )
+
def _load(self) -> dict[str, Any]:
domains = []
catalog_paths = [
@@ -31,16 +74,37 @@ class CapabilityRegistry:
catalog = yaml.safe_load(catalog_path.read_text(encoding="utf-8"))
domain = catalog["spec"]["domain"]
native_can_catalog = catalog_path.parent.name == "native_can_capabilities"
+ availability = (
+ "native_can"
+ if native_can_catalog
+ else "executable"
+ if domain in self.executable_domains
+ else "catalog_only"
+ )
+ executor_binding = (
+ "NativeCanRuntime/NativeCanExecutor"
+ if native_can_catalog
+ else "SeatCompiler/SeatExecutor"
+ if domain == "seat"
+ else f"ConfiguredDomainRuntime[{domain}]"
+ )
+ catalog_id = str(catalog.get("metadata", {}).get("id", catalog_path.stem))
+ self._definitions.extend(
+ CapabilityDefinition(
+ name=str(capability["name"]),
+ domain=str(domain),
+ availability=availability,
+ catalog_id=catalog_id,
+ catalog_path=catalog_path,
+ executor_binding=executor_binding,
+ spec=dict(capability),
+ )
+ for capability in catalog["spec"]["capabilities"]
+ )
domains.append(
{
"domain": domain,
- "availability": (
- "native_can"
- if native_can_catalog
- else "executable"
- if domain in self.executable_domains
- else "catalog_only"
- ),
+ "availability": availability,
"capabilities": catalog["spec"]["capabilities"],
}
)
diff --git a/src/cognitive_agent/application/chinese_values.py b/src/cognitive_agent/application/chinese_values.py
new file mode 100644
index 0000000..e16a2e7
--- /dev/null
+++ b/src/cognitive_agent/application/chinese_values.py
@@ -0,0 +1,128 @@
+"""Shared deterministic parsing for Chinese scalar control values.
+
+This module only normalizes values that the user actually spoke. Capability
+validation remains responsible for deciding whether a target accepts the
+result and whether its type/range is executable.
+"""
+
+from __future__ import annotations
+
+import re
+
+_ARABIC_PERCENT = re.compile(r"(-?\d+(?:\.\d+)?)\s*[%％]")
+_FRACTION = re.compile(
+ r"([零〇一二两三四五六七八九十百\d]+)分之"
+ r"([零〇一二两三四五六七八九十百点\d.]+)"
+)
+_SLASH_FRACTION = re.compile(r"(? int | None:
+ if not token:
+ return 0
+ if token.isdigit():
+ return int(token)
+ if "百" in token:
+ left, right = token.split("百", 1)
+ hundreds = _DIGITS.get(left[-1], 1) if left else 1
+ tail = _chinese_integer(right)
+ return None if tail is None else hundreds * 100 + tail
+ if "十" in token:
+ left, right = token.split("十", 1)
+ tens = _DIGITS.get(left[-1], 1) if left else 1
+ ones = _DIGITS.get(right[-1], 0) if right else 0
+ return tens * 10 + ones
+ if all(char in _DIGITS for char in token):
+ return int("".join(str(_DIGITS[char]) for char in token))
+ return None
+
+
+def _number(token: str) -> int | float | None:
+ try:
+ value = float(token)
+ except ValueError:
+ if "点" in token:
+ whole, fraction = token.split("点", 1)
+ whole_value = _chinese_integer(whole)
+ if (
+ whole_value is None
+ or not fraction
+ or any(char not in _DIGITS for char in fraction)
+ ):
+ return None
+ value = float(
+ f"{whole_value}."
+ + "".join(str(_DIGITS[char]) for char in fraction)
+ )
+ else:
+ integer = _chinese_integer(token)
+ if integer is None:
+ return None
+ value = float(integer)
+ return int(value) if value.is_integer() else value
+
+
+def parse_percentage_value(text: str) -> int | float | None:
+ """Return an explicitly spoken percentage/ratio/extreme position.
+
+ Bare numbers intentionally do not become percentages. Fractions are
+ rounded to the nearest whole percentage because every current continuous
+ vehicle position capability is integer-valued; the resolver still checks
+ the final range and type.
+ """
+
+ match = _ARABIC_PERCENT.search(text)
+ if match:
+ value = float(match.group(1))
+ return int(value) if value.is_integer() else value
+
+ match = _FRACTION.search(text)
+ if match:
+ denominator = _number(match.group(1))
+ numerator = _number(match.group(2))
+ if denominator not in {None, 0} and numerator is not None:
+ return round(float(numerator) * 100 / float(denominator))
+
+ match = _SLASH_FRACTION.search(text)
+ if match and int(match.group(2)) != 0:
+ return round(int(match.group(1)) * 100 / int(match.group(2)))
+
+ if re.search(r"一半|半开|开一半", text):
+ return 50
+
+ extrema = (
+ (
+ (
+ r"最大(?:位置|开度)|最高(?:位置)?|"
+ r"(?:调|调整|升|抬|开)(?:到|至)(?:最大|最高)|到顶|"
+ r"全开|完全打开|打开到底"
+ ),
+ 100,
+ ),
+ (
+ (
+ r"最小(?:位置|开度)|最低(?:位置)?|"
+ r"(?:调|调整|降|收|关)(?:到|至)(?:最小|最低)|"
+ r"全关|完全关闭|关闭到底"
+ ),
+ 0,
+ ),
+ )
+ for pattern, value in extrema:
+ if re.search(pattern, text):
+ return value
+ return None
diff --git a/src/cognitive_agent/application/native_can_intent.py b/src/cognitive_agent/application/native_can_intent.py
index e097271..1d8ae72 100644
--- a/src/cognitive_agent/application/native_can_intent.py
+++ b/src/cognitive_agent/application/native_can_intent.py
@@ -9,6 +9,7 @@ from __future__ import annotations
import re
from typing import Any
+from cognitive_agent.application.chinese_values import parse_percentage_value
WINDOW_TARGETS = {
"全部": "all",
@@ -595,8 +596,20 @@ def _parse_chinese_driver_seat_commands(compact: str) -> list[dict[str, Any]]:
("加热", "seat.set_driver_heating_native"),
)
present_thermal = [item for item in thermal_features if item[0] in compact]
- if len(present_thermal) == 1:
- label, capability = present_thermal[0]
+ coordinated_thermal = re.search(
+ r"(?:座椅)?(?:通风|加热)(?:和|与|以及|及|、)"
+ r"(?:打开|开启|启动|关闭|关掉|停止)?"
+ r"(?:座椅)?(?:通风|加热)",
+ compact,
+ )
+ shared_thermal_state: str | None = None
+ if coordinated_thermal is not None:
+ shared_prefix = compact[: coordinated_thermal.start()]
+ if re.search(r"关闭|关掉|停止|取消|不要|别开", shared_prefix):
+ shared_thermal_state = "off"
+ elif re.search(r"打开|开启|启动|启用", shared_prefix):
+ shared_thermal_state = "on"
+ for label, capability in present_thermal:
thermal_text = _feature_context(compact, label)
thermal_grade = _grade_alias(thermal_text, include_numeric=False)
parameters: dict[str, Any] | None = None
@@ -608,9 +621,11 @@ def _parse_chinese_driver_seat_commands(compact: str) -> list[dict[str, Any]]:
parameters = {"state": "on", "level": 1}
elif thermal_grade == "low" or re.search(
r"低速|(?:2|二)(?:档|级)", thermal_text
- ):
+ ) or re.search(r"打开|开启|启动|启用", thermal_text):
parameters = {"state": "on", "level": 2}
- elif re.search(r"打开|开启|启动|启用", thermal_text):
+ elif shared_thermal_state == "off":
+ parameters = {"state": "off"}
+ elif shared_thermal_state == "on":
parameters = {"state": "on", "level": 2}
if parameters is not None:
actions.append(_action(
@@ -850,24 +865,11 @@ def _parse_chinese_window_commands(compact: str) -> list[dict[str, Any]]:
window_text = f"{window_text}，{clauses[index + 1]}"
targets, explicit_window_target = _window_targets(window_text)
- position = re.search(
- r"(?:位置|开度|调整到|调到|设为|设置为|开启到|打开到|开到)"
- r"(\d{1,3})%",
- window_text,
- )
- half_position = bool(
- re.search(
- r"(?:开启|打开|开到|开度)?(?:一半|半开)",
- window_text,
- )
- )
- position_pct = (
- int(position[1]) if position else 50 if half_position else None
- )
+ position_pct = parse_percentage_value(window_text)
if (
position_pct is not None
and all(target not in WINDOW_GROUP_MEMBERS for target in targets)
- and position_pct <= 100
+ and 0 <= position_pct <= 100
):
reason = (
"用户指定了单个车窗的绝对开度。"
diff --git a/src/cognitive_agent/application/seat_service.py b/src/cognitive_agent/application/seat_service.py
index 3605ef2..8466681 100755
--- a/src/cognitive_agent/application/seat_service.py
+++ b/src/cognitive_agent/application/seat_service.py
@@ -1445,6 +1445,139 @@ class SeatDemoService:
_direct_can_batch=direct_can_batch,
)
+ def _adapt_speech_understanding(
+ self,
+ understanding: dict[str, Any],
+ context: dict[str, Any],
+ vehicle_profile: Any,
+ ) -> dict[str, Any]:
+ """Revalidate a server-owned speech result into the normal control IR.
+
+ Speech is allowed to skip a second natural-language parse, but it never
+ skips the authoritative capability catalog or ModelPlanAdapter. Native
+ CAN actions and configured-domain/seat actions are routed through the
+ same structures used by the rest of one-sentence orchestration.
+ """
+
+ if understanding.get("execution_eligible") is not True:
+ raise ValueError("speech understanding is not execution eligible")
+ commands = understanding.get("commands")
+ if not isinstance(commands, list) or not commands:
+ raise ValueError("speech understanding has no executable commands")
+
+ native_actions: list[dict[str, Any]] = []
+ model_requests: list[dict[str, Any]] = []
+ detected_domains: list[str] = []
+
+ def add_payload(payload: object, source_text: str) -> None:
+ if not isinstance(payload, dict):
+ raise ValueError("speech command has no validated executor payload")
+ domain = str(payload.get("domain", ""))
+ capability = str(payload.get("capability", ""))
+ target = str(payload.get("target", ""))
+ operation = str(payload.get("operation", "set"))
+ parameters = payload.get("parameters")
+ if not domain or not capability or not target or not isinstance(parameters, dict):
+ raise ValueError("speech executor payload is incomplete")
+
+ definitions = tuple(
+ definition
+ for definition in self.capability_registry.find(capability)
+ if definition.domain == domain
+ )
+ if len(definitions) != 1:
+ raise ValueError(
+ f"speech capability is not uniquely registered: {domain}.{capability}"
+ )
+ if domain not in detected_domains:
+ detected_domains.append(domain)
+ # Catalog availability describes where a capability is declared,
+ # not necessarily which executor owns it. Roof, climate and
+ # ambient capabilities are declared in domain catalogs but some
+ # are byte-exact NativeCanRuntime operations. Keep this routing in
+ # lockstep with CapabilityResolver._executor_binding().
+ if capability in self.native_can_runtime.supported_capabilities:
+ native_actions.append(
+ {
+ "domain": domain,
+ "capability": capability,
+ "target": target,
+ "operation": operation,
+ "parameters": deepcopy(parameters),
+ "parse_source": "speech_semantic",
+ "source_span": source_text,
+ "reason": "语音语义层已按真实能力目录解析并由服务端重新校验。",
+ }
+ )
+ return
+ model_requests.append(
+ {
+ "domain": domain,
+ "capability": capability,
+ "target_scope": target,
+ "operation": operation,
+ "parameters": [
+ {"name": name, "value": value}
+ for name, value in parameters.items()
+ ],
+ "reason": "语音语义层已按真实能力目录解析并由服务端重新校验。",
+ }
+ )
+
+ for command in commands:
+ if not isinstance(command, dict):
+ raise ValueError("speech command must be an object")
+ if (
+ command.get("status") != "active"
+ or command.get("negated") is True
+ or command.get("resolution_status") != "executable"
+ or command.get("validation_issues")
+ ):
+ raise ValueError("speech command failed execution invariants")
+ source_text = str(command.get("source_text") or understanding.get("raw_text") or "")
+ payload = command.get("executor_payload")
+ if command.get("mode_profile_id"):
+ if not isinstance(payload, dict) or not isinstance(payload.get("operations"), list):
+ raise ValueError("speech mode command has no validated operations")
+ for operation_payload in payload["operations"]:
+ add_payload(operation_payload, source_text)
+ else:
+ add_payload(payload, source_text)
+
+ plan = {
+ "detected_domains": detected_domains,
+ "requests": model_requests,
+ "ambiguities": [],
+ "decision_summary": [
+ "复用服务端语音理解结果；未对 cleanText 进行第二次自然语言解析。"
+ ],
+ "confidence": min(
+ float(command.get("confidence", 0.0))
+ for command in commands
+ if isinstance(command, dict)
+ ),
+ }
+ parsed = self.model_plan_adapter.adapt(
+ plan,
+ context=context,
+ vehicle_profile=vehicle_profile,
+ utterance=str(understanding.get("raw_text") or ""),
+ )
+ parsed["native_can_actions"] = native_actions
+ parsed.setdefault("semantic_constraints", [])
+ parsed.setdefault("unresolved_spans", [])
+ if parsed.get("ambiguities") or parsed.get("deferred_requests"):
+ raise ValueError("speech command no longer passes capability resolution")
+ mapped_count = (
+ len(parsed.get("actions", ()))
+ + len(parsed.get("domain_actions", ()))
+ + len(parsed.get("native_can_actions", ()))
+ )
+ expected_count = len(model_requests) + len(native_actions)
+ if mapped_count != expected_count:
+ raise ValueError("speech command mapping lost a validated capability mutation")
+ return parsed
+
def process(
self,
utterance: str,
@@ -1465,6 +1598,7 @@ class SeatDemoService:
_allow_empty_control_result: bool = False,
_on_commands_validated: Callable[[dict[str, Any]], None] | None = None,
_direct_can_batch: tuple[list[str], dict[str, Any]] | None = None,
+ _speech_understanding: dict[str, Any] | None = None,
) -> dict[str, Any]:
control_transport = self._validate_control_transport(control_transport)
# Creating a reusable scene is a configuration workflow. Synthesize
@@ -1490,6 +1624,7 @@ class SeatDemoService:
_allow_empty_control_result,
_on_commands_validated,
_direct_can_batch,
+ _speech_understanding,
)
if control_transport == WIRELESS_MQTT_TRANSPORT:
# The head unit owns both physical CAN channels in wireless mode.
@@ -1511,6 +1646,7 @@ class SeatDemoService:
_allow_empty_control_result,
_on_commands_validated,
_direct_can_batch,
+ _speech_understanding,
)
if hardware_access_bypass or self._driver_only_vehicle_execution_active(
control_transport
@@ -1532,6 +1668,7 @@ class SeatDemoService:
_allow_empty_control_result,
_on_commands_validated,
_direct_can_batch,
+ _speech_understanding,
)
authorized_client_id = self.hardware_access.ensure_authorized(hardware_client_id)
with self.hardware_execution.reserve():
@@ -1552,6 +1689,7 @@ class SeatDemoService:
_allow_empty_control_result,
_on_commands_validated,
_direct_can_batch,
+ _speech_understanding,
)
def _process_once(
@@ -1571,6 +1709,7 @@ class SeatDemoService:
allow_empty_control_result: bool = False,
on_commands_validated: Callable[[dict[str, Any]], None] | None = None,
direct_can_batch: tuple[list[str], dict[str, Any]] | None = None,
+ speech_understanding: dict[str, Any] | None = None,
) -> dict[str, Any]:
control_transport = self._validate_control_transport(control_transport)
utterance = utterance.strip()
@@ -1641,7 +1780,7 @@ class SeatDemoService:
context["mode_profiles"] = self.mode_profiles.public_modes(vehicle_profile.id)
matched_modes = (
[]
- if direct_can_batch is not None
+ if direct_can_batch is not None or speech_understanding is not None
else self.mode_profiles.match_text(
utterance,
vehicle_id=vehicle_profile.id,
@@ -1691,6 +1830,16 @@ class SeatDemoService:
"can_commands": (
list(direct_can_batch[0]) if direct_can_batch is not None else []
),
+ "speech_understanding": (
+ {
+ "utterance_id": speech_understanding.get("utterance_id"),
+ "raw_text": speech_understanding.get("raw_text"),
+ "clean_text": speech_understanding.get("clean_text"),
+ "semantic_status": speech_understanding.get("semantic_status"),
+ }
+ if speech_understanding is not None
+ else None
+ ),
}
started = time.perf_counter()
@@ -1733,6 +1882,19 @@ class SeatDemoService:
parsed = deepcopy(direct_can_batch[1])
provider_used = "direct_can_control"
provider_notice = None
+ elif speech_understanding is not None:
+ parsed = self._adapt_speech_understanding(
+ speech_understanding,
+ context,
+ vehicle_profile,
+ )
+ parsed = self._mark_control_parse_source(
+ parsed,
+ "speech_semantic",
+ overwrite=True,
+ )
+ provider_used = "speech_semantic"
+ provider_notice = "使用服务端缓存并重新校验的语音结构化意图。"
else:
parsed, provider_used, provider_notice = self._interpret(
utterance,
diff --git a/src/cognitive_agent/infrastructure/speech/__init__.py b/src/cognitive_agent/infrastructure/speech/__init__.py
new file mode 100644
index 0000000..e60a9c4
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/__init__.py
@@ -0,0 +1,29 @@
+"""Local speech-recognition infrastructure."""
+
+from cognitive_agent.infrastructure.speech.assembler import TranscriptAssembler
+from cognitive_agent.infrastructure.speech.contracts import (
+ SpeechCommandIntent,
+ UnderstandingTranscript,
+)
+from cognitive_agent.infrastructure.speech.semantic import (
+ SpeechIntentNormalization,
+ normalize_speech_intent,
+)
+from cognitive_agent.infrastructure.speech.sensevoice import (
+ SpeechRecognitionError,
+ SpeechTranscriber,
+)
+from cognitive_agent.infrastructure.speech.understanding import (
+ UnderstandingTranscriptProcessor,
+)
+
+__all__ = [
+ "SpeechCommandIntent",
+ "SpeechIntentNormalization",
+ "SpeechRecognitionError",
+ "SpeechTranscriber",
+ "TranscriptAssembler",
+ "UnderstandingTranscript",
+ "UnderstandingTranscriptProcessor",
+ "normalize_speech_intent",
+]
diff --git a/src/cognitive_agent/infrastructure/speech/assembler.py b/src/cognitive_agent/infrastructure/speech/assembler.py
new file mode 100644
index 0000000..a8e5577
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/assembler.py
@@ -0,0 +1,156 @@
+"""Idempotent assembly of partial and final ASR segment callbacks."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+
+from cognitive_agent.infrastructure.speech.contracts import (
+ AssembledTranscript,
+ SpeechSegmentResult,
+)
+
+_PUNCTUATION = re.compile(r"^[，,。！？!?；;\s]+|[，,。！？!?；;\s]+$")
+
+
+class TranscriptAssembler:
+ """Maintain ordered segment state with replace/commit and overlap semantics."""
+
+ def __init__(self, utterance_id: str) -> None:
+ self.utterance_id = utterance_id
+ self._segments: dict[str, SpeechSegmentResult] = {}
+ self._duplicate_callbacks = 0
+ self._out_of_order_callbacks = 0
+ self._conflicts: list[str] = []
+ self._received = 0
+ self._last_sequence = -1
+ self._complete = False
+
+ def update(self, segment: SpeechSegmentResult) -> AssembledTranscript:
+ if segment.utterance_id != self.utterance_id:
+ raise ValueError("segment utterance_id does not match assembler")
+ if self._complete:
+ self._duplicate_callbacks += 1
+ return self.snapshot()
+
+ self._received += 1
+ segment = replace(segment, received_index=self._received)
+ if segment.sequence < self._last_sequence:
+ self._out_of_order_callbacks += 1
+ self._last_sequence = max(self._last_sequence, segment.sequence)
+
+ previous = self._segments.get(segment.segment_id)
+ if previous is not None:
+ if previous.is_final:
+ if segment.is_final and self._same_commit(previous, segment):
+ self._duplicate_callbacks += 1
+ else:
+ self._conflicts.append(
+ f"segment {segment.segment_id} changed after final commit"
+ )
+ return self.snapshot()
+ if self._same_callback(previous, segment):
+ self._duplicate_callbacks += 1
+ return self.snapshot()
+
+ self._segments[segment.segment_id] = segment
+ return self.snapshot()
+
+ def finish(self) -> AssembledTranscript:
+ self._complete = True
+ return self.snapshot()
+
+ def reset(self, utterance_id: str) -> None:
+ self.__init__(utterance_id)
+
+ def snapshot(self) -> AssembledTranscript:
+ ordered = sorted(
+ self._segments.values(),
+ key=lambda item: (item.sequence, item.start_ms, item.received_index),
+ )
+ finals = tuple(item for item in ordered if item.is_final)
+ partials = tuple(item for item in ordered if not item.is_final)
+ text, overlaps = self._assemble_text(finals, partials)
+ return AssembledTranscript(
+ utterance_id=self.utterance_id,
+ text=text,
+ final_segments=finals,
+ partial_segments=partials,
+ duplicate_callbacks=self._duplicate_callbacks,
+ out_of_order_callbacks=self._out_of_order_callbacks,
+ conflicts=tuple(dict.fromkeys(self._conflicts)),
+ overlaps_removed=tuple(overlaps),
+ complete=self._complete,
+ )
+
+ @staticmethod
+ def _same_commit(first: SpeechSegmentResult, second: SpeechSegmentResult) -> bool:
+ return (
+ first.sequence == second.sequence
+ and first.start_ms == second.start_ms
+ and first.end_ms == second.end_ms
+ and first.text == second.text
+ )
+
+ @classmethod
+ def _same_callback(cls, first: SpeechSegmentResult, second: SpeechSegmentResult) -> bool:
+ return first.is_final == second.is_final and cls._same_commit(first, second)
+
+ @classmethod
+ def _assemble_text(
+ cls,
+ finals: tuple[SpeechSegmentResult, ...],
+ partials: tuple[SpeechSegmentResult, ...],
+ ) -> tuple[str, list[str]]:
+ display = [*finals]
+ final_ids = {item.segment_id for item in finals}
+ display.extend(item for item in partials if item.segment_id not in final_ids)
+ display.sort(key=lambda item: (item.sequence, item.start_ms, item.received_index))
+ if not display:
+ return "", []
+
+ assembled = display[0].text.strip()
+ previous = display[0]
+ overlaps: list[str] = []
+ for current in display[1:]:
+ addition = current.text.strip()
+ overlap = cls._safe_overlap(assembled, addition, previous, current)
+ if overlap:
+ overlaps.append(overlap)
+ addition = addition[len(overlap) :]
+ assembled = cls._join(assembled, addition)
+ previous = current
+ return assembled.strip(), overlaps
+
+ @staticmethod
+ def _safe_overlap(
+ left: str,
+ right: str,
+ previous: SpeechSegmentResult,
+ current: SpeechSegmentResult,
+ ) -> str:
+ """Remove only boundary overlap supported by audio overlap or a long phrase."""
+
+ left_body = _PUNCTUATION.sub("", left)
+ right_body = _PUNCTUATION.sub("", right)
+ maximum = min(len(left_body), len(right_body), 24)
+ for length in range(maximum, 1, -1):
+ candidate = left_body[-length:]
+ if candidate != right_body[:length]:
+ continue
+ audio_evidence = current.audio_overlap_ms > 0 or (
+ current.start_ms < previous.end_ms
+ )
+ if length >= 4 or (audio_evidence and length >= 2):
+ return candidate
+ return ""
+
+ @staticmethod
+ def _join(left: str, right: str) -> str:
+ if not right:
+ return left
+ if not left:
+ return right
+ if left[-1] in "，,。！？!?；;" or right[0] in "，,。！？!?；;":
+ return f"{left}{right}"
+ return f"{left}，{right}"
diff --git a/src/cognitive_agent/infrastructure/speech/capability.py b/src/cognitive_agent/infrastructure/speech/capability.py
new file mode 100644
index 0000000..5aea60a
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/capability.py
@@ -0,0 +1,396 @@
+"""Resolve speech slots against the application's authoritative capability catalogs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from cognitive_agent.application.capability_registry import (
+ CapabilityDefinition,
+ CapabilityRegistry,
+)
+from cognitive_agent.application.mode_profile import ModeProfile, ModeProfileRegistry
+from cognitive_agent.application.native_can_runtime import NativeCanRuntime
+
+ResolutionStatus = Literal[
+ "executable",
+ "unsupported",
+ "ambiguous",
+ "incomplete",
+]
+
+
+@dataclass(frozen=True)
+class CapabilityResolution:
+ """Catalog-backed outcome for one semantic mutation."""
+
+ status: ResolutionStatus
+ definition: CapabilityDefinition | None
+ issues: tuple[str, ...] = ()
+ executor_binding: str | None = None
+ executor_payload: dict[str, Any] | None = None
+ intent_ir_action: dict[str, Any] | None = None
+
+
+@dataclass(frozen=True)
+class ModeResolution:
+ """A repository mode profile and whether it applies to the requested target."""
+
+ status: ResolutionStatus
+ profile: ModeProfile | None
+ issues: tuple[str, ...] = ()
+ executor_payloads: tuple[dict[str, Any], ...] = ()
+
+
+# These are lexical-to-schema selectors, not a second capability registry. A
+# selector can only resolve a capability that is present in CapabilityRegistry.
+_PROPERTY_SELECTORS: dict[str, tuple[str, ...]] = {
+ "hvac.power": ("set_power",),
+ "hvac.temperature": ("temperature",),
+ "hvac.fan_level": ("fan_level",),
+ "seat.ventilation": ("ventilation",),
+ "seat.heating": ("heating",),
+ "seat.longitudinal_position": ("longitudinal_position",),
+ "seat.backrest_recline": ("backrest",),
+ "seat.cushion_tilt": ("cushion_tilt",),
+ "seat.legrest_height": ("legrest_height",),
+ "seat.legrest_extension": ("legrest_extension",),
+ "window.motion": ("set_motion",),
+ "window.position": ("set_position",),
+ "roof.shade_motion": ("shade_motion",),
+ "roof.shade_position": ("shade_position",),
+ "roof.transmittance": ("transmittance",),
+ "ambient.power": ("set_power",),
+ "ambient.color": ("illumination", "color_index"),
+ "ambient.brightness": ("illumination", "brightness_level"),
+ "reading_light.power": ("reading_power",),
+ "indoor_light.power": ("indoor_power",),
+ "front_light.power": ("front_light", "headlight", "head_lamp"),
+}
+
+
+class CapabilityResolver:
+ """Adapt natural-language slots to the capability/executor facts in code."""
+
+ def __init__(self, repository_root: Path) -> None:
+ self.root = Path(repository_root)
+ self.registry = CapabilityRegistry(
+ self.root,
+ {"seat", "climate", "roof", "ambient_light"},
+ )
+ self.native_runtime = NativeCanRuntime(self.root)
+ self.native_supported = self.native_runtime.supported_capabilities
+ self.mode_profiles = ModeProfileRegistry(self.root / "config" / "modes")
+
+ def definitions(self) -> tuple[CapabilityDefinition, ...]:
+ return self.registry.definitions()
+
+ def executor_binding_for(self, definition: CapabilityDefinition) -> str:
+ """Return the runtime that actually owns an executable definition."""
+
+ return self._executor_binding(definition)
+
+ @staticmethod
+ def semantic_property_for(capability: str, parameters: dict[str, Any] | None = None) -> str:
+ """Project an authoritative capability into the speech display vocabulary.
+
+ This is deliberately a projection, not an availability table: whether the
+ capability exists and can execute still comes exclusively from the catalog
+ and executor runtime.
+ """
+
+ values = parameters or {}
+ if capability == "seat.set_driver_axis_native":
+ return {
+ "horizontal": "seat.longitudinal_position",
+ "backrest": "seat.backrest_recline",
+ "basin": "seat.cushion_tilt",
+ "height": "seat.height",
+ }.get(str(values.get("axis")), capability)
+ projections = (
+ ("set_driver_ventilation", "seat.ventilation"),
+ ("seat.set_ventilation", "seat.ventilation"),
+ ("set_driver_heating", "seat.heating"),
+ ("seat.set_heating", "seat.heating"),
+ ("set_backrest_recline", "seat.backrest_recline"),
+ ("set_longitudinal_position", "seat.longitudinal_position"),
+ ("set_cushion_tilt", "seat.cushion_tilt"),
+ ("set_legrest_height", "seat.legrest_height"),
+ ("set_legrest_extension", "seat.legrest_extension"),
+ ("window.set_motion", "window.motion"),
+ ("window.set_position", "window.position"),
+ ("roof.set_shade_motion", "roof.shade_motion"),
+ ("roof.set_shade_position", "roof.shade_position"),
+ ("roof.set_transmittance", "roof.transmittance"),
+ ("ambient_light.set_power", "ambient.power"),
+ ("ambient_light.set_illumination", "ambient.color"),
+ ("ambient_light.set_color_index", "ambient.color_index"),
+ ("ambient_light.set_brightness_level", "ambient.brightness"),
+ ("climate.set_power", "hvac.power"),
+ ("climate.set_temperature", "hvac.temperature"),
+ ("climate.set_fan_level", "hvac.fan_level"),
+ ("interior_light.set_reading_power", "reading_light.power"),
+ ("interior_light.set_indoor_power", "indoor_light.power"),
+ )
+ return next(
+ (property_name for token, property_name in projections if token in capability),
+ capability,
+ )
+
+ def resolve_action(self, action: dict[str, Any], *, source_text: str) -> CapabilityResolution:
+ """Validate an action produced by the application's existing intent parser."""
+
+ capability = str(action.get("capability", ""))
+ return self.resolve(
+ domain=str(action.get("domain", capability.partition(".")[0])),
+ semantic_property=self.semantic_property_for(
+ capability, dict(action.get("parameters", {}))
+ ),
+ target_scope=str(action.get("target", action.get("target_scope", "vehicle"))),
+ target_device=None,
+ operation=str(action.get("operation", "set")),
+ parameters=dict(action.get("parameters", {})),
+ source_text=source_text,
+ preferred_capability=capability,
+ )
+
+ def match_mode(self, text: str) -> ModeProfile | None:
+ matches = self.mode_profiles.match_text(text, vehicle_id="denza_n7")
+ return matches[0] if len(matches) == 1 else None
+
+ def resolve_mode(
+ self,
+ text: str,
+ *,
+ target_scope: str | None,
+ target_explicit: bool,
+ ) -> ModeResolution:
+ matches = self.mode_profiles.match_text(text, vehicle_id="denza_n7")
+ if not matches:
+ return ModeResolution("unsupported", None, ("mode_profile_not_registered",))
+ if len(matches) > 1:
+ return ModeResolution("ambiguous", None, ("multiple_mode_profiles_matched",))
+ profile = matches[0]
+ operation_scopes = {operation.target_scope for operation in profile.operations}
+ if target_explicit and target_scope not in operation_scopes:
+ scopes = ",".join(sorted(operation_scopes))
+ return ModeResolution(
+ "unsupported",
+ profile,
+ (f"mode_profile.{profile.id}仅支持范围{scopes}",),
+ )
+
+ payloads: list[dict[str, Any]] = []
+ issues: list[str] = []
+ for operation in profile.operations:
+ resolution = self.resolve(
+ domain=operation.domain,
+ semantic_property="",
+ target_scope=operation.target_scope,
+ target_device=None,
+ operation=operation.operation,
+ parameters=operation.parameters,
+ source_text=text,
+ preferred_capability=operation.capability,
+ )
+ if resolution.status != "executable" or resolution.executor_payload is None:
+ issues.extend(resolution.issues or ("mode_operation_not_executable",))
+ continue
+ payloads.append(resolution.executor_payload)
+ if issues or len(payloads) != len(profile.operations):
+ return ModeResolution("unsupported", profile, tuple(dict.fromkeys(issues)))
+ return ModeResolution("executable", profile, executor_payloads=tuple(payloads))
+
+ def resolve(
+ self,
+ *,
+ domain: str,
+ semantic_property: str,
+ target_scope: str,
+ target_device: str | None,
+ operation: str,
+ parameters: dict[str, Any],
+ source_text: str,
+ preferred_capability: str | None = None,
+ ) -> CapabilityResolution:
+ candidates = list(self.registry.definitions(domain=domain))
+ if preferred_capability:
+ candidates = [item for item in candidates if item.name == preferred_capability]
+ else:
+ selectors = _PROPERTY_SELECTORS.get(semantic_property, ())
+ candidates = [
+ item for item in candidates if any(selector in item.name for selector in selectors)
+ ]
+ if not candidates:
+ return CapabilityResolution(
+ "unsupported",
+ None,
+ (f"capability_not_registered:{domain}.{semantic_property}",),
+ )
+
+ parameter_names = set(parameters)
+ ranked = sorted(
+ candidates,
+ key=lambda item: self._score(
+ item,
+ target_scope=target_scope,
+ target_device=target_device,
+ parameter_names=parameter_names,
+ ),
+ reverse=True,
+ )
+ definition = ranked[0]
+ top_score = self._score(
+ definition,
+ target_scope=target_scope,
+ target_device=target_device,
+ parameter_names=parameter_names,
+ )
+ equally_ranked = [
+ item
+ for item in ranked
+ if self._score(
+ item,
+ target_scope=target_scope,
+ target_device=target_device,
+ parameter_names=parameter_names,
+ )
+ == top_score
+ ]
+ if len(equally_ranked) > 1 and not preferred_capability:
+ return CapabilityResolution(
+ "ambiguous",
+ None,
+ ("multiple_capabilities_match:" + ",".join(item.name for item in equally_ranked),),
+ )
+
+ issues = self._validate(definition, target_scope, target_device, parameters)
+ missing = [issue for issue in issues if issue.startswith("required_parameter_missing:")]
+ if missing:
+ status: ResolutionStatus = "incomplete"
+ elif issues:
+ status = "unsupported"
+ elif definition.availability == "catalog_only":
+ status = "unsupported"
+ issues.append(f"capability_catalog_only:{definition.name}")
+ elif (
+ definition.availability == "native_can" and definition.name not in self.native_supported
+ ):
+ status = "unsupported"
+ issues.append(f"native_executor_not_implemented:{definition.name}")
+ else:
+ status = "executable"
+
+ binding = self._executor_binding(definition) if status == "executable" else None
+ payload = None
+ intent_action = None
+ if status == "executable":
+ executor_operation = "set" if operation in {"on", "off", "open", "close"} else operation
+ payload = {
+ "domain": domain,
+ "capability": definition.name,
+ "target": target_scope,
+ "operation": executor_operation,
+ "parameters": dict(parameters),
+ "parse_source": "local_rule",
+ }
+ intent_action = {
+ "domain": domain,
+ "capability": definition.name,
+ "target": {
+ "device": definition.target_devices[0] if definition.target_devices else domain,
+ "zone": target_scope,
+ },
+ "operation": executor_operation,
+ "parameters": dict(parameters),
+ "source_span": source_text,
+ "risk_level": str(definition.spec.get("risk_level", "high")),
+ "confirmation_policy": str(
+ definition.spec.get("confirmation_policy", "on_ambiguity")
+ ),
+ }
+ return CapabilityResolution(
+ status,
+ definition,
+ tuple(dict.fromkeys(issues)),
+ binding,
+ payload,
+ intent_action,
+ )
+
+ @staticmethod
+ def _score(
+ definition: CapabilityDefinition,
+ *,
+ target_scope: str,
+ target_device: str | None,
+ parameter_names: set[str],
+ ) -> tuple[int, int, int, int, int]:
+ specs = {str(item["name"]): item for item in definition.parameters}
+ accepted = set(specs)
+ missing_required = sum(
+ 1
+ for name, spec in specs.items()
+ if spec.get("required") and name not in parameter_names
+ )
+ return (
+ int(target_scope in definition.target_scopes),
+ int(not parameter_names - accepted),
+ -missing_required,
+ int(target_device is not None and target_device in definition.target_devices),
+ int(definition.availability == "native_can"),
+ )
+
+ @staticmethod
+ def _validate(
+ definition: CapabilityDefinition,
+ target_scope: str,
+ target_device: str | None,
+ parameters: dict[str, Any],
+ ) -> list[str]:
+ issues: list[str] = []
+ if definition.target_scopes and target_scope not in definition.target_scopes:
+ issues.append(f"{definition.name}不支持范围{target_scope}")
+ if (
+ target_device
+ and definition.target_devices
+ and target_device not in definition.target_devices
+ ):
+ issues.append(f"{definition.name}不支持设备{target_device}")
+ specs = {str(item["name"]): item for item in definition.parameters}
+ for name in parameters.keys() - specs.keys():
+ issues.append(f"unknown_parameter:{definition.name}:{name}")
+ for name, spec in specs.items():
+ if spec.get("required") and name not in parameters:
+ issues.append(f"required_parameter_missing:{definition.name}:{name}")
+ continue
+ if name not in parameters:
+ continue
+ value = parameters[name]
+ expected = str(spec.get("type"))
+ numeric = isinstance(value, (int, float)) and not isinstance(value, bool)
+ valid_type = {
+ "number": numeric,
+ "integer": numeric and float(value).is_integer(),
+ "string": isinstance(value, str),
+ "boolean": isinstance(value, bool),
+ "object": isinstance(value, dict),
+ "array": isinstance(value, list),
+ }.get(expected, False)
+ if not valid_type:
+ issues.append(f"invalid_parameter_type:{definition.name}:{name}:{expected}")
+ continue
+ if numeric and spec.get("minimum") is not None and value < spec["minimum"]:
+ issues.append(f"{definition.name}.{name}低于工程下限{spec['minimum']}")
+ if numeric and spec.get("maximum") is not None and value > spec["maximum"]:
+ issues.append(f"{definition.name}.{name}高于工程上限{spec['maximum']}")
+ allowed = spec.get("allowed_values")
+ if allowed is not None and value not in allowed:
+ issues.append(f"{definition.name}.{name}不在工程允许值中")
+ return issues
+
+ def _executor_binding(self, definition: CapabilityDefinition) -> str:
+ if definition.name in self.native_supported:
+ return "NativeCanRuntime/NativeCanExecutor"
+ return definition.executor_binding
diff --git a/src/cognitive_agent/infrastructure/speech/contracts.py b/src/cognitive_agent/infrastructure/speech/contracts.py
new file mode 100644
index 0000000..6b3bda2
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/contracts.py
@@ -0,0 +1,233 @@
+"""Stable data contracts for the speech-understanding pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+CommandStatus = Literal["active", "cancelled"]
+ResolutionStatus = Literal["executable", "unsupported", "ambiguous", "incomplete"]
+TranscriptStatus = Literal[
+ "recognized_executable",
+ "recognized_unsupported",
+ "ambiguous",
+ "incomplete",
+ "no_command",
+]
+
+
+@dataclass(frozen=True)
+class SpeechCommandIntent:
+ """A conservative speech intent aligned with the project's capability catalog."""
+
+ domain: str
+ capability: str | None
+ target_scope: str
+ target_device: str
+ property_name: str
+ operation: str
+ parameters: dict[str, Any]
+ value: Any
+ unit: str | None
+ qualifiers: dict[str, Any]
+ negated: bool
+ confidence: float
+ source_text: str
+ status: CommandStatus = "active"
+ resolution_status: ResolutionStatus = "executable"
+ validation_issues: tuple[str, ...] = ()
+ executor_binding: str | None = None
+ executor_payload: dict[str, Any] | None = None
+ intent_ir_action: dict[str, Any] | None = None
+ mode_profile_id: str | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "domain": self.domain,
+ "capability": self.capability,
+ "target_scope": self.target_scope,
+ "target_device": self.target_device,
+ "property": self.property_name,
+ "operation": self.operation,
+ "parameters": dict(self.parameters),
+ "value": self.value,
+ "unit": self.unit,
+ "qualifiers": dict(self.qualifiers),
+ "negated": self.negated,
+ "confidence": round(self.confidence, 3),
+ "source_text": self.source_text,
+ "status": self.status,
+ "resolution_status": self.resolution_status,
+ "validation_issues": list(self.validation_issues),
+ "executor_binding": self.executor_binding,
+ "executor_payload": self.executor_payload,
+ "intent_ir_action": self.intent_ir_action,
+ "mode_profile_id": self.mode_profile_id,
+ }
+
+
+@dataclass(frozen=True)
+class CancelledSpeechFragment:
+ """Text that was heard but must never become an active control."""
+
+ text: str
+ reason: str
+ marker: str = ""
+ commands: tuple[SpeechCommandIntent, ...] = ()
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "text": self.text,
+ "reason": self.reason,
+ "marker": self.marker,
+ "commands": [command.to_dict() for command in self.commands],
+ }
+
+
+@dataclass(frozen=True)
+class UnderstandingTranscript:
+ """Raw ASR evidence plus display text and non-executable semantic analysis."""
+
+ raw_text: str
+ clean_text: str
+ commands: tuple[SpeechCommandIntent, ...] = ()
+ cancelled_fragments: tuple[CancelledSpeechFragment, ...] = ()
+ semantic_status: TranscriptStatus = "no_command"
+ uncertain: bool = False
+ reasons: tuple[str, ...] = ()
+ removed_fillers: tuple[str, ...] = ()
+ repair_resolved: bool = False
+ intent_domains: tuple[str, ...] = ()
+ changed: bool = False
+ processing_ms: float = 0.0
+ command_resolution_ms: float = 0.0
+ semantic_trace: tuple[dict[str, Any], ...] = ()
+
+ @property
+ def text(self) -> str:
+ """Backward-compatible display-text alias."""
+
+ return self.clean_text
+
+ @property
+ def execution_eligible(self) -> bool:
+ return bool(self.commands) and not self.uncertain and all(
+ command.status == "active"
+ and not command.negated
+ and command.resolution_status == "executable"
+ and not command.validation_issues
+ for command in self.commands
+ )
+
+ def metadata(self) -> dict[str, object]:
+ return {
+ "changed": self.changed,
+ "removed_fillers": list(self.removed_fillers),
+ "repair_resolved": self.repair_resolved,
+ "intent_domains": list(self.intent_domains),
+ "uncertain": self.uncertain,
+ "semantic_status": self.semantic_status,
+ "reason": list(self.reasons),
+ "execution_eligible": self.execution_eligible,
+ "processing_ms": round(self.processing_ms, 3),
+ "command_resolution_ms": round(self.command_resolution_ms, 3),
+ }
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "raw_text": self.raw_text,
+ "clean_text": self.clean_text,
+ "commands": [command.to_dict() for command in self.commands],
+ "cancelled_fragments": [
+ fragment.to_dict() for fragment in self.cancelled_fragments
+ ],
+ "semantic_status": self.semantic_status,
+ "uncertain": self.uncertain,
+ "reason": list(self.reasons),
+ "execution_eligible": self.execution_eligible,
+ "normalization": self.metadata(),
+ "semantic_trace": [dict(item) for item in self.semantic_trace],
+ }
+
+
+@dataclass(frozen=True)
+class SpeechSegmentResult:
+ """One ordered partial or final ASR callback."""
+
+ utterance_id: str
+ segment_id: str
+ sequence: int
+ start_ms: int
+ end_ms: int
+ text: str
+ is_final: bool
+ audio_overlap_ms: int = 0
+ inference_ms: float = 0.0
+ received_index: int = 0
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "utterance_id": self.utterance_id,
+ "segment_id": self.segment_id,
+ "sequence": self.sequence,
+ "start_ms": self.start_ms,
+ "end_ms": self.end_ms,
+ "raw_text": self.text,
+ "state": "final" if self.is_final else "partial",
+ "audio_overlap_ms": self.audio_overlap_ms,
+ "inference_ms": round(self.inference_ms, 3),
+ }
+
+
+@dataclass(frozen=True)
+class AssembledTranscript:
+ utterance_id: str
+ text: str
+ final_segments: tuple[SpeechSegmentResult, ...]
+ partial_segments: tuple[SpeechSegmentResult, ...]
+ duplicate_callbacks: int = 0
+ out_of_order_callbacks: int = 0
+ conflicts: tuple[str, ...] = ()
+ overlaps_removed: tuple[str, ...] = ()
+ complete: bool = False
+
+ @property
+ def uncertain(self) -> bool:
+ return bool(self.conflicts)
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "utterance_id": self.utterance_id,
+ "text": self.text,
+ "complete": self.complete,
+ "duplicate_callbacks": self.duplicate_callbacks,
+ "out_of_order_callbacks": self.out_of_order_callbacks,
+ "conflicts": list(self.conflicts),
+ "overlaps_removed": list(self.overlaps_removed),
+ "segments": [segment.to_dict() for segment in self.final_segments],
+ }
+
+
+@dataclass
+class SpeechPipelineTimings:
+ audio_read_ms: float = 0.0
+ vad_ms: float = 0.0
+ asr_ms: float = 0.0
+ assembly_ms: float = 0.0
+ semantic_ms: float = 0.0
+ recovery_ms: float = 0.0
+ total_ms: float = 0.0
+ extras: dict[str, float] = field(default_factory=dict)
+
+ def to_dict(self) -> dict[str, float]:
+ result = {
+ "audio_read_ms": round(self.audio_read_ms, 3),
+ "vad_ms": round(self.vad_ms, 3),
+ "asr_ms": round(self.asr_ms, 3),
+ "assembly_ms": round(self.assembly_ms, 3),
+ "semantic_ms": round(self.semantic_ms, 3),
+ "recovery_ms": round(self.recovery_ms, 3),
+ "total_ms": round(self.total_ms, 3),
+ }
+ result.update({key: round(value, 3) for key, value in self.extras.items()})
+ return result
diff --git a/src/cognitive_agent/infrastructure/speech/recovery.py b/src/cognitive_agent/infrastructure/speech/recovery.py
new file mode 100644
index 0000000..e9ceb5b
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/recovery.py
@@ -0,0 +1,46 @@
+"""Optional recovery recognizer contract for uncertain fast-path results."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Protocol
+
+from cognitive_agent.infrastructure.speech.contracts import UnderstandingTranscript
+
+
+@dataclass(frozen=True)
+class RecoveryResult:
+ raw_text: str
+ model: str
+ confidence: float
+ elapsed_ms: float
+
+
+class RecoveryRecognizer(Protocol):
+ @property
+ def available(self) -> bool: ...
+
+ def recognize(
+ self,
+ samples: Sequence[float],
+ sample_rate: int,
+ fast_path: UnderstandingTranscript,
+ ) -> RecoveryResult | None: ...
+
+
+class DisabledRecoveryRecognizer:
+ """Default recovery implementation; never downloads or guesses a model."""
+
+ @property
+ def available(self) -> bool:
+ return False
+
+ def recognize(
+ self,
+ samples: Sequence[float],
+ sample_rate: int,
+ fast_path: UnderstandingTranscript,
+ ) -> RecoveryResult | None:
+ del samples, sample_rate, fast_path
+ return None
diff --git a/src/cognitive_agent/infrastructure/speech/semantic.py b/src/cognitive_agent/infrastructure/speech/semantic.py
new file mode 100644
index 0000000..65e9ceb
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/semantic.py
@@ -0,0 +1,61 @@
+"""Backward-compatible entry point for structured speech understanding."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from cognitive_agent.infrastructure.speech.understanding import (
+ UnderstandingTranscriptProcessor,
+)
+
+
+@dataclass(frozen=True)
+class SpeechIntentNormalization:
+ text: str
+ removed_fillers: tuple[str, ...]
+ repair_resolved: bool
+ intent_domains: tuple[str, ...]
+ changed: bool
+ commands: tuple[object, ...] = ()
+ cancelled_fragments: tuple[object, ...] = ()
+ uncertain: bool = False
+ reasons: tuple[str, ...] = ()
+
+ def metadata(self) -> dict[str, object]:
+ return {
+ "changed": self.changed,
+ "removed_fillers": list(self.removed_fillers),
+ "repair_resolved": self.repair_resolved,
+ "intent_domains": list(self.intent_domains),
+ "uncertain": self.uncertain,
+ "reason": list(self.reasons),
+ }
+
+
+_DEFAULT_PROCESSOR: UnderstandingTranscriptProcessor | None = None
+
+
+def normalize_speech_intent(
+ text: str, repository_root: Path | None = None
+) -> SpeechIntentNormalization:
+ global _DEFAULT_PROCESSOR
+ if repository_root is not None:
+ processor = UnderstandingTranscriptProcessor(repository_root)
+ else:
+ if _DEFAULT_PROCESSOR is None:
+ repository = Path(__file__).resolve().parents[4]
+ _DEFAULT_PROCESSOR = UnderstandingTranscriptProcessor(repository)
+ processor = _DEFAULT_PROCESSOR
+ result = processor.process(text)
+ return SpeechIntentNormalization(
+ text=result.clean_text,
+ removed_fillers=result.removed_fillers,
+ repair_resolved=result.repair_resolved,
+ intent_domains=result.intent_domains,
+ changed=result.changed,
+ commands=result.commands,
+ cancelled_fragments=result.cancelled_fragments,
+ uncertain=result.uncertain,
+ reasons=result.reasons,
+ )
diff --git a/src/cognitive_agent/infrastructure/speech/sensevoice.py b/src/cognitive_agent/infrastructure/speech/sensevoice.py
new file mode 100644
index 0000000..8457aef
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/sensevoice.py
@@ -0,0 +1,493 @@
+"""Low-latency local Chinese speech understanding backed by SenseVoiceSmall."""
+
+from __future__ import annotations
+
+import copy
+import hashlib
+import importlib.metadata
+import importlib.util
+import math
+import os
+import re
+import sys
+import threading
+import time
+import uuid
+import wave
+from array import array
+from collections import OrderedDict
+from collections.abc import Sequence
+from dataclasses import replace
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+
+from cognitive_agent.infrastructure.speech.assembler import TranscriptAssembler
+from cognitive_agent.infrastructure.speech.contracts import (
+ SpeechPipelineTimings,
+ SpeechSegmentResult,
+)
+from cognitive_agent.infrastructure.speech.recovery import DisabledRecoveryRecognizer
+from cognitive_agent.infrastructure.speech.tracing import SpeechTrace, env_flag
+from cognitive_agent.infrastructure.speech.understanding import (
+ UnderstandingTranscriptProcessor,
+)
+from cognitive_agent.infrastructure.speech.vad import SpeechVadSegmenter
+
+SENSEVOICE_MODEL_NAME = "SenseVoiceSmall"
+SENSEVOICE_SAMPLE_RATE = 16_000
+SENSEVOICE_MAX_DURATION_SECONDS = 30.0
+SENSEVOICE_MIN_DURATION_SECONDS = 0.2
+SENSEVOICE_MAX_WAV_BYTES = 2 * 1024 * 1024
+_SENSEVOICE_TAG = re.compile(r"<\|[^|>]+\|>")
+_UTTERANCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,95}$")
+
+
+class SpeechRecognitionError(RuntimeError):
+ """A safe, user-facing speech-recognition failure."""
+
+ def __init__(self, code: str, message: str, *, status: int = 422) -> None:
+ super().__init__(message)
+ self.code = code
+ self.status = status
+
+
+class SpeechTranscriber:
+ """One recognizer, one VAD, and idempotent per-utterance orchestration."""
+
+ def __init__(self, repository_root: Path) -> None:
+ self.repository_root = repository_root
+ default_dir = repository_root / "artifacts" / "models" / "sensevoice-small"
+ self.model_dir = Path(os.getenv("SPEECH_MODEL_DIR", str(default_dir))).expanduser()
+ configured_model = os.getenv("SPEECH_MODEL_FILE", "model.onnx").strip() or "model.onnx"
+ self.model_path = self.model_dir / configured_model
+ self.tokens_path = self.model_dir / "tokens.txt"
+ self.language = os.getenv("SPEECH_LANGUAGE", "zh").strip() or "zh"
+ self.use_itn = env_flag("SPEECH_USE_ITN", True)
+ default_threads = min(8, max(2, (os.cpu_count() or 4) // 2))
+ self.num_threads = max(
+ 1, min(16, int(os.getenv("SPEECH_NUM_THREADS", str(default_threads))))
+ )
+ self.provider = os.getenv("SPEECH_PROVIDER", "cpu").strip().lower() or "cpu"
+ self.hr_lexicon = Path(os.getenv("SPEECH_HR_LEXICON", "")).expanduser()
+ self.hr_rule_fsts = Path(os.getenv("SPEECH_HR_RULE_FSTS", "")).expanduser()
+ self.processor = UnderstandingTranscriptProcessor(repository_root)
+ self.segmenter = SpeechVadSegmenter(repository_root)
+ self.recovery = DisabledRecoveryRecognizer()
+ self._recognizer: Any | None = None
+ self._load_lock = threading.Lock()
+ self._decode_lock = threading.Lock()
+ self._cache_lock = threading.Lock()
+ self._cache: OrderedDict[str, tuple[str, dict[str, object]]] = OrderedDict()
+ self._inflight: dict[str, threading.Event] = {}
+
+ def status(self) -> dict[str, object]:
+ runtime_installed = importlib.util.find_spec("sherpa_onnx") is not None
+ model_installed = self.model_path.is_file() and self.tokens_path.is_file()
+ try:
+ runtime_version = importlib.metadata.version("sherpa-onnx")
+ except importlib.metadata.PackageNotFoundError:
+ runtime_version = ""
+ homophone_enabled = self.hr_lexicon.is_file() and self.hr_rule_fsts.is_file()
+ return {
+ "enabled": runtime_installed and model_installed,
+ "runtime_installed": runtime_installed,
+ "runtime_version": runtime_version,
+ "model_installed": model_installed,
+ "loaded": self._recognizer is not None,
+ "model": SENSEVOICE_MODEL_NAME,
+ "model_file": self.model_path.name,
+ "language": self.language,
+ "use_itn": self.use_itn,
+ "sample_rate": SENSEVOICE_SAMPLE_RATE,
+ "max_duration_seconds": SENSEVOICE_MAX_DURATION_SECONDS,
+ "provider": self.provider,
+ "num_threads": self.num_threads,
+ "decoder": "offline-sensevoice-final-only",
+ "vad": self.segmenter.status(),
+ "homophone_replacer": {
+ "enabled": homophone_enabled,
+ "lexicon": str(self.hr_lexicon) if homophone_enabled else "",
+ "rule_fsts": str(self.hr_rule_fsts) if homophone_enabled else "",
+ },
+ "recovery_recognizer": {"available": self.recovery.available},
+ "debug_logging": env_flag("SPEECH_DEBUG"),
+ }
+
+ def transcribe_wav(
+ self, content: bytes, *, utterance_id: str | None = None
+ ) -> dict[str, object]:
+ resolved_id = self._resolve_utterance_id(utterance_id)
+ digest = hashlib.sha256(content).hexdigest()
+ cached = self._claim_or_wait(resolved_id, digest)
+ if cached is not None:
+ cached["idempotent_replay"] = True
+ return cached
+ try:
+ result = self._transcribe_wav(content, resolved_id)
+ with self._cache_lock:
+ self._cache[resolved_id] = (digest, copy.deepcopy(result))
+ self._cache.move_to_end(resolved_id)
+ while len(self._cache) > 64:
+ self._cache.popitem(last=False)
+ return result
+ finally:
+ with self._cache_lock:
+ event = self._inflight.pop(resolved_id, None)
+ if event is not None:
+ event.set()
+
+ def cached_result(self, utterance_id: str) -> dict[str, object]:
+ """Return a server-owned recognition result for a later control request.
+
+ The browser submits only the opaque utterance id. Executor payloads are
+ recovered from this bounded server cache so client-side JSON can never be
+ promoted into a trusted control plan.
+ """
+
+ resolved_id = self._resolve_utterance_id(utterance_id)
+ with self._cache_lock:
+ cached = self._cache.get(resolved_id)
+ if cached is None:
+ raise SpeechRecognitionError(
+ "speech_result_expired",
+ "语音理解结果已过期，请重新听写后再执行。",
+ status=409,
+ )
+ self._cache.move_to_end(resolved_id)
+ return copy.deepcopy(cached[1])
+
+ def _transcribe_wav(self, content: bytes, utterance_id: str) -> dict[str, object]:
+ trace = SpeechTrace(utterance_id)
+ timings = SpeechPipelineTimings()
+ total_started = time.perf_counter()
+ read_started = time.perf_counter()
+ sample_rate, original_samples = self._read_pcm16_wav(content)
+ samples = self._resample(original_samples, sample_rate, SENSEVOICE_SAMPLE_RATE)
+ timings.audio_read_ms = (time.perf_counter() - read_started) * 1000
+ duration_seconds = len(samples) / SENSEVOICE_SAMPLE_RATE
+ self._validate_duration(duration_seconds)
+ rms = math.sqrt(math.fsum(sample * sample for sample in samples) / len(samples))
+ trace.event(
+ "audio_preprocessed",
+ sampleRate=SENSEVOICE_SAMPLE_RATE,
+ sourceSampleRate=sample_rate,
+ durationMs=round(duration_seconds * 1000),
+ rms=round(rms, 6),
+ )
+ if rms < 0.001:
+ raise SpeechRecognitionError(
+ "no_speech", "未检测到清晰的说话声，请靠近麦克风重试。"
+ )
+
+ segmentation = self.segmenter.segment(samples)
+ timings.vad_ms = segmentation.elapsed_ms
+ trace.event(
+ "vad_complete",
+ usedVad=segmentation.used_vad,
+ fallbackReason=segmentation.fallback_reason,
+ segmentCount=len(segmentation.segments),
+ segments=[
+ {
+ "segmentId": item.segment_id,
+ "startMs": item.start_ms,
+ "endMs": item.end_ms,
+ "audioOverlapMs": item.audio_overlap_ms,
+ }
+ for item in segmentation.segments
+ ],
+ stageMs=round(segmentation.elapsed_ms, 3),
+ )
+ if segmentation.fallback_reason == "vad_found_no_segment":
+ raise SpeechRecognitionError(
+ "no_speech", "VAD 未检测到可靠语音，请靠近麦克风重试。"
+ )
+
+ recognizer = self._get_recognizer()
+ assembler = TranscriptAssembler(utterance_id)
+ labels: dict[str, str] = {}
+ asr_started = time.perf_counter()
+ with self._decode_lock:
+ for segment in segmentation.segments:
+ segment_started = time.perf_counter()
+ try:
+ stream = recognizer.create_stream()
+ stream.accept_waveform(SENSEVOICE_SAMPLE_RATE, segment.samples)
+ recognizer.decode_stream(stream)
+ native_result = stream.result
+ except Exception as exc: # pragma: no cover - native errors vary
+ raise SpeechRecognitionError(
+ "inference_failed",
+ "语音模型暂时无法完成转写，请稍后重试。",
+ status=503,
+ ) from exc
+ inference_ms = (time.perf_counter() - segment_started) * 1000
+ raw_segment = self._clean_text(str(getattr(native_result, "text", "")))
+ labels = {
+ "language": self._clean_label(
+ getattr(native_result, "lang", self.language)
+ ),
+ "emotion": self._clean_label(getattr(native_result, "emotion", "")),
+ "event": self._clean_label(getattr(native_result, "event", "")),
+ }
+ callback = SpeechSegmentResult(
+ utterance_id=utterance_id,
+ segment_id=segment.segment_id,
+ sequence=segment.sequence,
+ start_ms=segment.start_ms,
+ end_ms=segment.end_ms,
+ text=raw_segment,
+ is_final=True,
+ audio_overlap_ms=segment.audio_overlap_ms,
+ inference_ms=inference_ms,
+ )
+ before = assembler.snapshot().text
+ assembled = assembler.update(callback)
+ trace.event(
+ "asr_callback",
+ segmentId=segment.segment_id,
+ sequence=segment.sequence,
+ segmentStartMs=segment.start_ms,
+ segmentEndMs=segment.end_ms,
+ state="final",
+ rawAsrResult=raw_segment,
+ beforeAssembler=before,
+ assembledText=assembled.text,
+ inferenceMs=round(inference_ms, 3),
+ )
+ timings.asr_ms = (time.perf_counter() - asr_started) * 1000
+ assembly_started = time.perf_counter()
+ assembled = assembler.finish()
+ timings.assembly_ms = (time.perf_counter() - assembly_started) * 1000
+ raw_text = assembled.text.strip()
+ if not raw_text:
+ raise SpeechRecognitionError(
+ "no_transcript", "没有识别出有效文字，请重新说一次。"
+ )
+
+ semantic_started = time.perf_counter()
+ understanding = self.processor.process(raw_text)
+ timings.semantic_ms = (time.perf_counter() - semantic_started) * 1000
+ timings.extras["command_resolution_ms"] = understanding.command_resolution_ms
+ if assembled.uncertain:
+ understanding = replace(
+ understanding,
+ uncertain=True,
+ reasons=tuple(
+ dict.fromkeys((*understanding.reasons, "segment_assembly_conflict"))
+ ),
+ )
+ trace.event(
+ "understanding_complete",
+ rawText=raw_text,
+ cleanText=understanding.clean_text,
+ commands=[command.to_dict() for command in understanding.commands],
+ cancelledFragments=[
+ fragment.to_dict() for fragment in understanding.cancelled_fragments
+ ],
+ semanticStatus=understanding.semantic_status,
+ semanticTrace=list(understanding.semantic_trace),
+ uncertain=understanding.uncertain,
+ reason=list(understanding.reasons),
+ stageMs=round(timings.semantic_ms, 3),
+ )
+
+ if understanding.uncertain and self.recovery.available:
+ recovery_started = time.perf_counter()
+ recovered = self.recovery.recognize(
+ samples, SENSEVOICE_SAMPLE_RATE, understanding
+ )
+ timings.recovery_ms = (time.perf_counter() - recovery_started) * 1000
+ trace.event(
+ "recovery_complete",
+ recovered=bool(recovered),
+ stageMs=round(timings.recovery_ms, 3),
+ )
+
+ timings.total_ms = (time.perf_counter() - total_started) * 1000
+ payload: dict[str, object] = {
+ "utterance_id": utterance_id,
+ "text": understanding.clean_text,
+ "clean_text": understanding.clean_text,
+ "raw_text": raw_text,
+ "commands": [command.to_dict() for command in understanding.commands],
+ "cancelled_fragments": [
+ fragment.to_dict() for fragment in understanding.cancelled_fragments
+ ],
+ "semantic_status": understanding.semantic_status,
+ "semantic_trace": list(understanding.semantic_trace),
+ "uncertain": understanding.uncertain,
+ "reason": list(understanding.reasons),
+ "execution_eligible": understanding.execution_eligible,
+ "semantic_cleaned": understanding.changed,
+ "normalization": understanding.metadata(),
+ "assembly": assembled.to_dict(),
+ "segments": [item.to_dict() for item in assembled.final_segments],
+ **labels,
+ "duration_ms": round(duration_seconds * 1000),
+ "inference_ms": round(timings.asr_ms),
+ "timings": timings.to_dict(),
+ "model": SENSEVOICE_MODEL_NAME,
+ "provider": self.provider,
+ "idempotent_replay": False,
+ }
+ debug_events = trace.response_events()
+ if debug_events:
+ payload["debug_trace"] = debug_events
+ return payload
+
+ def _claim_or_wait(
+ self, utterance_id: str, digest: str
+ ) -> dict[str, object] | None:
+ while True:
+ with self._cache_lock:
+ cached = self._cache.get(utterance_id)
+ if cached is not None:
+ cached_digest, payload = cached
+ if cached_digest != digest:
+ raise SpeechRecognitionError(
+ "utterance_id_conflict",
+ "同一 utteranceId 收到了不同录音，已拒绝重复提交。",
+ status=409,
+ )
+ self._cache.move_to_end(utterance_id)
+ return copy.deepcopy(payload)
+ event = self._inflight.get(utterance_id)
+ if event is None:
+ self._inflight[utterance_id] = threading.Event()
+ return None
+ if not event.wait(timeout=50):
+ raise SpeechRecognitionError(
+ "duplicate_request_timeout", "重复语音请求等待超时。", status=503
+ )
+
+ def _get_recognizer(self) -> Any:
+ if self._recognizer is not None:
+ return self._recognizer
+ with self._load_lock:
+ if self._recognizer is not None:
+ return self._recognizer
+ if importlib.util.find_spec("sherpa_onnx") is None:
+ raise SpeechRecognitionError(
+ "runtime_unavailable",
+ "语音识别运行时未安装，请安装 speech 可选依赖。",
+ status=503,
+ )
+ if not self.model_path.is_file() or not self.tokens_path.is_file():
+ raise SpeechRecognitionError(
+ "model_unavailable", "SenseVoiceSmall 模型尚未部署。", status=503
+ )
+ try:
+ import sherpa_onnx
+
+ kwargs: dict[str, object] = {
+ "model": str(self.model_path),
+ "tokens": str(self.tokens_path),
+ "num_threads": self.num_threads,
+ "sample_rate": SENSEVOICE_SAMPLE_RATE,
+ "provider": self.provider,
+ "language": self.language,
+ "use_itn": self.use_itn,
+ "debug": env_flag("SPEECH_ASR_DEBUG"),
+ }
+ if self.hr_lexicon.is_file() and self.hr_rule_fsts.is_file():
+ kwargs["hr_lexicon"] = str(self.hr_lexicon)
+ kwargs["hr_rule_fsts"] = str(self.hr_rule_fsts)
+ self._recognizer = sherpa_onnx.OfflineRecognizer.from_sense_voice(
+ **kwargs
+ )
+ except Exception as exc: # pragma: no cover - native failures vary
+ raise SpeechRecognitionError(
+ "model_load_failed",
+ "SenseVoiceSmall 模型加载失败，请检查模型文件与运行时。",
+ status=503,
+ ) from exc
+ return self._recognizer
+
+ @staticmethod
+ def _resolve_utterance_id(value: str | None) -> str:
+ if value is None or not value.strip():
+ return f"server-{uuid.uuid4()}"
+ candidate = value.strip()
+ if not _UTTERANCE_ID.fullmatch(candidate):
+ raise SpeechRecognitionError(
+ "invalid_utterance_id", "utteranceId 格式无效。", status=400
+ )
+ return candidate
+
+ @staticmethod
+ def _validate_duration(duration_seconds: float) -> None:
+ if duration_seconds < SENSEVOICE_MIN_DURATION_SECONDS:
+ raise SpeechRecognitionError("audio_too_short", "录音时间太短，请重新说一次。")
+ if duration_seconds > SENSEVOICE_MAX_DURATION_SECONDS + 0.5:
+ raise SpeechRecognitionError(
+ "audio_too_long",
+ f"单次听写不能超过 {int(SENSEVOICE_MAX_DURATION_SECONDS)} 秒。",
+ status=413,
+ )
+
+ @staticmethod
+ def _clean_text(text: str) -> str:
+ return re.sub(r"\s+", " ", _SENSEVOICE_TAG.sub("", text)).strip()
+
+ @staticmethod
+ def _clean_label(value: object) -> str:
+ return str(value or "").removeprefix("<|").removesuffix("|>").strip()
+
+ @staticmethod
+ def _resample(
+ samples: Sequence[float], input_rate: int, output_rate: int
+ ) -> list[float]:
+ if input_rate == output_rate:
+ return list(samples)
+ output_length = max(1, round(len(samples) * output_rate / input_rate))
+ if output_length == 1:
+ return [float(samples[0])]
+ ratio = input_rate / output_rate
+ result: list[float] = []
+ for output_index in range(output_length):
+ position = min(len(samples) - 1, output_index * ratio)
+ left = int(position)
+ right = min(len(samples) - 1, left + 1)
+ fraction = position - left
+ result.append(
+ float(samples[left] * (1.0 - fraction) + samples[right] * fraction)
+ )
+ return result
+
+ @staticmethod
+ def _read_pcm16_wav(content: bytes) -> tuple[int, list[float]]:
+ if not content or len(content) > SENSEVOICE_MAX_WAV_BYTES:
+ raise SpeechRecognitionError(
+ "invalid_audio_size", "录音数据为空或超过大小限制。", status=413
+ )
+ try:
+ with wave.open(BytesIO(content), "rb") as source:
+ channels = source.getnchannels()
+ sample_width = source.getsampwidth()
+ sample_rate = source.getframerate()
+ frame_count = source.getnframes()
+ compression = source.getcomptype()
+ frames = source.readframes(frame_count)
+ except (EOFError, wave.Error) as exc:
+ raise SpeechRecognitionError("invalid_audio", "录音格式无效，请重新录制。") from exc
+ if compression != "NONE" or sample_width != 2:
+ raise SpeechRecognitionError("unsupported_audio", "仅支持 16 位 PCM WAV 录音。")
+ if channels < 1 or channels > 2 or sample_rate < 8_000 or sample_rate > 96_000:
+ raise SpeechRecognitionError(
+ "unsupported_audio", "录音声道数或采样率不受支持。"
+ )
+ pcm = array("h")
+ pcm.frombytes(frames)
+ if sys.byteorder != "little":
+ pcm.byteswap()
+ if channels == 2:
+ mono = [
+ (pcm[index] + pcm[index + 1]) / (2 * 32768.0)
+ for index in range(0, len(pcm) - 1, 2)
+ ]
+ else:
+ mono = [sample / 32768.0 for sample in pcm]
+ if not mono:
+ raise SpeechRecognitionError("invalid_audio", "录音中没有音频采样。")
+ return sample_rate, mono
diff --git a/src/cognitive_agent/infrastructure/speech/tracing.py b/src/cognitive_agent/infrastructure/speech/tracing.py
new file mode 100644
index 0000000..6cce299
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/tracing.py
@@ -0,0 +1,59 @@
+"""Opt-in structured diagnostics for one speech utterance."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+LOGGER = logging.getLogger("cognitive_agent.speech")
+_HANDLER_LOCK = threading.Lock()
+
+
+def env_flag(name: str, default: bool = False) -> bool:
+ value = os.getenv(name)
+ if value is None:
+ return default
+ return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+@dataclass
+class SpeechTrace:
+ utterance_id: str
+ enabled: bool = field(default_factory=lambda: env_flag("SPEECH_DEBUG"))
+ include_response: bool = field(
+ default_factory=lambda: env_flag("SPEECH_DEBUG_RESPONSE")
+ )
+ started_at: float = field(default_factory=time.perf_counter)
+ events: list[dict[str, Any]] = field(default_factory=list)
+
+ def __post_init__(self) -> None:
+ if not self.enabled or LOGGER.handlers:
+ return
+ with _HANDLER_LOCK:
+ if LOGGER.handlers:
+ return
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter("%(message)s"))
+ LOGGER.addHandler(handler)
+ LOGGER.setLevel(logging.INFO)
+ LOGGER.propagate = False
+
+ def event(self, stage: str, **details: Any) -> None:
+ if not self.enabled:
+ return
+ payload = {
+ "utteranceId": self.utterance_id,
+ "stage": stage,
+ "elapsedMs": round((time.perf_counter() - self.started_at) * 1000, 3),
+ **details,
+ }
+ self.events.append(payload)
+ LOGGER.info("speech_trace %s", json.dumps(payload, ensure_ascii=False))
+
+ def response_events(self) -> list[dict[str, Any]]:
+ return list(self.events) if self.enabled and self.include_response else []
diff --git a/src/cognitive_agent/infrastructure/speech/understanding.py b/src/cognitive_agent/infrastructure/speech/understanding.py
new file mode 100644
index 0000000..122872d
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/understanding.py
@@ -0,0 +1,1625 @@
+"""Capability-driven Chinese understanding for short vehicle-control speech."""
+
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Iterable
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any
+
+from cognitive_agent.application.chinese_values import parse_percentage_value
+from cognitive_agent.application.native_can_intent import merge_native_can_intent
+from cognitive_agent.infrastructure.speech.capability import CapabilityResolver
+from cognitive_agent.infrastructure.speech.contracts import (
+ CancelledSpeechFragment,
+ ResolutionStatus,
+ SpeechCommandIntent,
+ TranscriptStatus,
+ UnderstandingTranscript,
+)
+from cognitive_agent.infrastructure.speech.vocabulary import (
+ ACTION_ALIASES,
+ CANCELLATION_MARKERS,
+ CORRECTION_MARKERS,
+ DEVICE_TERMS,
+ NEGATION_PREFIXES,
+ POSITIVE_CONNECTORS,
+ SCOPE_ALIASES,
+ DeviceTerm,
+ action_terms_in,
+ color_in,
+ device_terms_in,
+ scope_in,
+)
+
+_CLAUSE_SPLIT = re.compile(r"[\s，,；;。！？!?…]+")
+_POSITIVE_SPLIT = re.compile(
+ rf"[\s，,；;]*({'|'.join(map(re.escape, POSITIVE_CONNECTORS))})[\s，,；;]*"
+)
+_COORDINATION_SPLIT = re.compile(r"\s*(?:以及|和|与|及|、)\s*")
+_CORRECTION_PATTERN = re.compile(
+ rf"[\s，,；;]*({'|'.join(map(re.escape, sorted(CORRECTION_MARKERS, key=len, reverse=True)))}|不[，,]+是)[\s，,；;]*"
+)
+_CANCELLATION_PATTERN = re.compile(
+ rf"[\s，,；;]*({'|'.join(map(re.escape, CANCELLATION_MARKERS))})[\s，,；;]*"
+)
+_VOCAL_FILLER = re.compile(r"(?那个|这个)(?:[\s，,]*(?P=word)){1,}")
+_LEADING_FILLER = re.compile(
+ r"^(?:(?:呃|嗯|额|啊|呐)+[\s，,]*)?(?:那个|这个|就是)[\s，,]*"
+ r"(?=(?:把|将|请|打开|开启|关闭|关掉|调整|调节|设置))"
+)
+_POLITE_PREFIX = re.compile(r"^(?:(?:请|麻烦|拜托)(?:帮我|给我)?|你帮我|帮我|给我)[\s，,]*")
+_POLITE_SUFFIX = re.compile(r"[\s，,]*(?:可以吗|行吗|好吗|好不好|谢谢)[。！？!?]*$")
+_NUMBER = re.compile(r"(? None:
+ repository = repository_root or Path(__file__).resolve().parents[4]
+ if not (repository / "domains").is_dir():
+ repository = Path(__file__).resolve().parents[4]
+ self.resolver = CapabilityResolver(repository)
+
+ def process(
+ self,
+ raw_text: str,
+ *,
+ timestamps: Iterable[object] | None = None,
+ previous_context: UnderstandingTranscript | None = None,
+ ) -> UnderstandingTranscript:
+ del timestamps, previous_context
+ started = time.perf_counter()
+ original = self._tidy(raw_text)
+ prepared, removed_fillers = self._prepare(original)
+ active: list[SpeechCommandIntent] = []
+ cancelled: list[CancelledSpeechFragment] = []
+ reasons: list[str] = []
+ repair_resolved = False
+ resolution_ms = 0.0
+ clauses: list[str] = []
+ shared_target: SpeechCommandIntent | None = None
+
+ if not prepared:
+ reasons.append("filler_only")
+ else:
+ blocks = [part for part in _POSITIVE_SPLIT.split(prepared) if part]
+ blocks = [part for part in blocks if part not in POSITIVE_CONNECTORS]
+ for raw_block in blocks:
+ block = raw_block
+ if (
+ shared_target is not None
+ and not device_terms_in(block)
+ and not _CORRECTION_PATTERN.search(block)
+ and not _CANCELLATION_PATTERN.search(block)
+ ):
+ block = self._inherit_shared_target(block, shared_target)
+ clauses.append(block)
+ (
+ block_active,
+ block_cancelled,
+ block_reasons,
+ repaired,
+ block_resolution_ms,
+ ) = self._process_block(block)
+ active.extend(block_active)
+ cancelled.extend(block_cancelled)
+ reasons.extend(block_reasons)
+ repair_resolved = repair_resolved or repaired
+ resolution_ms += block_resolution_ms
+ if block_active:
+ shared_target = block_active[-1]
+
+ active, obsolete_issues, atomic_issues, atomic_resolution_ms = (
+ self._coalesce_atomic_commands(active)
+ )
+ if obsolete_issues:
+ reasons = [reason for reason in reasons if reason not in obsolete_issues]
+ reasons.extend(atomic_issues)
+ resolution_ms += atomic_resolution_ms
+ reasons.extend(issue for command in active for issue in command.validation_issues)
+ if len(_CORRECTION_PATTERN.findall(prepared)) > 2:
+ reasons.append("correction_chain_too_long")
+ clean_text = self._render(active)
+ if not active:
+ if self._contains_negation(prepared):
+ clean_text = self._sentence(prepared)
+ elif not cancelled and prepared and not self._looks_like_control(prepared):
+ clean_text = ""
+
+ unique_reasons = tuple(dict.fromkeys(reason for reason in reasons if reason))
+ semantic_status = self._semantic_status(active, unique_reasons)
+ benign = {
+ "non_control_speech",
+ "filler_only",
+ "cancelled_by_user",
+ "negated_command",
+ }
+ uncertain = semantic_status in {
+ "recognized_unsupported",
+ "ambiguous",
+ "incomplete",
+ } or any(reason not in benign for reason in unique_reasons)
+ domains = tuple(dict.fromkeys(command.domain for command in active))
+ processing_ms = (time.perf_counter() - started) * 1000
+ semantic_trace = self._semantic_trace(
+ original,
+ prepared,
+ clauses,
+ active,
+ clean_text,
+ semantic_status,
+ )
+ return UnderstandingTranscript(
+ raw_text=original,
+ clean_text=clean_text,
+ commands=tuple(active),
+ cancelled_fragments=tuple(cancelled),
+ semantic_status=semantic_status,
+ uncertain=uncertain,
+ reasons=unique_reasons,
+ removed_fillers=tuple(dict.fromkeys(removed_fillers)),
+ repair_resolved=repair_resolved,
+ intent_domains=domains,
+ changed=clean_text.rstrip("。") != original.rstrip("。"),
+ processing_ms=processing_ms,
+ command_resolution_ms=resolution_ms,
+ semantic_trace=semantic_trace,
+ )
+
+ def _process_block(
+ self, block: str
+ ) -> tuple[
+ list[SpeechCommandIntent],
+ list[CancelledSpeechFragment],
+ list[str],
+ bool,
+ float,
+ ]:
+ block = self._tidy(block)
+ active: list[SpeechCommandIntent] = []
+ cancelled: list[CancelledSpeechFragment] = []
+ reasons: list[str] = []
+ resolution_ms = 0.0
+ cancellation = _CANCELLATION_PATTERN.search(block)
+ if cancellation:
+ prefix = block[: cancellation.start()].strip("，,；;。！？!? ")
+ if prefix:
+ analysis = self._analyse_fragment(prefix)
+ resolution_ms += analysis.resolution_ms
+ cancelled.append(
+ CancelledSpeechFragment(
+ prefix,
+ "cancelled_by_user",
+ cancellation.group(1),
+ self._cancel_commands(analysis.commands),
+ )
+ )
+ reasons.append("cancelled_by_user")
+ suffix = block[cancellation.end() :].strip("，,；;。！？!? ")
+ if suffix:
+ follow = self._analyse_fragment(suffix)
+ resolution_ms += follow.resolution_ms
+ active.extend(follow.commands)
+ reasons.extend(follow.uncertain_reasons)
+ return active, cancelled, reasons, True, resolution_ms
+
+ correction = _CORRECTION_PATTERN.search(block)
+ if correction:
+ prefix = block[: correction.start()].strip("，,；;。！？!? ")
+ suffix = block[correction.end() :].strip("，,；;。！？!? ")
+ analyses = [self._analyse_fragment(item) for item in self._plain_fragments(prefix)]
+ resolution_ms += sum(item.resolution_ms for item in analyses)
+ if analyses:
+ active.extend(command for item in analyses[:-1] for command in item.commands)
+ old = analyses[-1]
+ cancelled.append(
+ CancelledSpeechFragment(
+ old.text,
+ "self_correction",
+ correction.group(1),
+ self._cancel_commands(old.commands),
+ )
+ )
+ else:
+ old = _FragmentAnalysis(prefix, ())
+ if not suffix:
+ reasons.append("incomplete_correction")
+ return active, cancelled, reasons, False, resolution_ms
+ inherited = self._inherit_context(suffix, old)
+ if _CORRECTION_PATTERN.search(inherited) or _CANCELLATION_PATTERN.search(inherited):
+ (
+ new_active,
+ new_cancelled,
+ new_reasons,
+ repaired,
+ nested_resolution_ms,
+ ) = self._process_block(inherited)
+ active.extend(new_active)
+ cancelled.extend(new_cancelled)
+ reasons.extend(new_reasons)
+ return active, cancelled, reasons, repaired, resolution_ms + nested_resolution_ms
+ new = self._analyse_fragment(inherited)
+ resolution_ms += new.resolution_ms
+ active.extend(new.commands)
+ reasons.extend(new.uncertain_reasons)
+ if not new.commands:
+ reasons.append("correction_not_resolved")
+ return active, cancelled, reasons, bool(new.commands), resolution_ms
+
+ analyses = [self._analyse_fragment(item) for item in self._plain_fragments(block)]
+ resolution_ms += sum(item.resolution_ms for item in analyses)
+ for index, analysis in enumerate(analyses):
+ following = analyses[index + 1] if index + 1 < len(analyses) else None
+ if self._is_abandoned_false_start(analysis, following):
+ cancelled.append(
+ CancelledSpeechFragment(
+ analysis.text,
+ "incomplete_false_start",
+ commands=self._cancel_commands(analysis.commands),
+ )
+ )
+ continue
+ if self._is_target_refinement(analysis, following):
+ cancelled.append(
+ CancelledSpeechFragment(
+ analysis.text,
+ "target_refinement",
+ commands=self._cancel_commands(analysis.commands),
+ )
+ )
+ continue
+ if self._contains_negation(analysis.text):
+ cancelled.append(
+ CancelledSpeechFragment(
+ analysis.text,
+ "negated_command",
+ commands=self._cancel_commands(analysis.commands, negated=True),
+ )
+ )
+ reasons.append("negated_command")
+ continue
+ active.extend(analysis.commands)
+ reasons.extend(analysis.uncertain_reasons)
+ if analysis.non_control:
+ reasons.append("non_control_speech")
+ return active, cancelled, reasons, False, resolution_ms
+
+ @staticmethod
+ def _plain_fragments(text: str) -> list[str]:
+ return [part for part in _CLAUSE_SPLIT.split(text) if part]
+
+ def _analyse_fragment(
+ self, text: str, *, allow_coordination: bool = True
+ ) -> _FragmentAnalysis:
+ text = self._tidy(text)
+ if allow_coordination:
+ coordinated_analysis = self._analyse_coordinated_seat_properties(text)
+ if coordinated_analysis is not None:
+ return coordinated_analysis
+ coordinated_analysis = self._analyse_coordinated_controls(text)
+ if coordinated_analysis is not None:
+ return coordinated_analysis
+ application_analysis = self._analyse_application_intent(text)
+ if application_analysis.commands and self._application_slots_preserved(
+ text, application_analysis
+ ):
+ return application_analysis
+ devices = device_terms_in(text)
+ actions = action_terms_in(text)
+ mode = self.resolver.match_mode(text)
+ if mode is not None and (devices or actions):
+ term = (
+ self._most_specific_device(devices)
+ if devices
+ else next(item for item in DEVICE_TERMS if item.canonical == "座椅")
+ )
+ commands, issues, resolution_ms = self._commands_for(
+ text,
+ term,
+ self._select_action(actions) if actions else "set",
+ self._contains_negation(text),
+ )
+ return _FragmentAnalysis(
+ text,
+ tuple(commands),
+ tuple(issues),
+ incomplete=not commands,
+ resolution_ms=resolution_ms,
+ )
+ if not devices or not actions:
+ looks_control = bool(devices or actions or _CONTROL_SCAFFOLD.search(text))
+ return _FragmentAnalysis(
+ text,
+ (),
+ ("incomplete_control_fragment",) if looks_control else (),
+ non_control=not looks_control,
+ incomplete=looks_control,
+ )
+ distinct = list(dict.fromkeys(item[2].canonical for item in devices))
+ if len(distinct) > 1 and not self._same_target_family(devices):
+ return _FragmentAnalysis(
+ text, (), ("multiple_targets_without_connector",), incomplete=True
+ )
+ term = self._most_specific_device(devices)
+ action = self._select_action(actions)
+ commands, issues, resolution_ms = self._commands_for(
+ text, term, action, self._contains_negation(text)
+ )
+ return _FragmentAnalysis(
+ text,
+ tuple(commands),
+ tuple(issues),
+ incomplete=not commands,
+ resolution_ms=resolution_ms,
+ )
+
+ def _analyse_coordinated_seat_properties(
+ self, text: str
+ ) -> _FragmentAnalysis | None:
+ """Expand coordinated properties while retaining one shared seat target.
+
+ Chinese commonly omits the second ``座椅`` in expressions such as
+ ``座椅通风和加热``. This is a grammar relation between two properties,
+ not a new capability table: each expanded term still resolves through
+ the authoritative catalog.
+ """
+
+ match = re.search(
+ r"(?:座椅)?(?P通风|加热)(?:和|与|以及|及|、)"
+ r"(?P打开|开启|启动|关闭|关掉|停止)?"
+ r"(?:座椅)?(?P通风|加热)",
+ text,
+ )
+ if match is None or match.group("first") == match.group("second"):
+ return None
+ actions = action_terms_in(text)
+ prefix_actions = [item for item in actions if item[0] < match.start()]
+ if not prefix_actions:
+ return None
+ shared_action = self._select_action(prefix_actions)
+ semantic_keys = {
+ "通风": "seat.ventilation",
+ "加热": "seat.heating",
+ }
+ terms_by_key = {item.semantic_key: item for item in DEVICE_TERMS}
+ commands: list[SpeechCommandIntent] = []
+ issues: list[str] = []
+ resolution_ms = 0.0
+ features = (match.group("first"), match.group("second"))
+ for index, feature in enumerate(features):
+ term = terms_by_key[semantic_keys[feature]]
+ action = shared_action
+ if index == 1 and match.group("second_action"):
+ local_actions = action_terms_in(match.group("second_action"))
+ if local_actions:
+ action = self._select_action(local_actions)
+ resolved, local_issues, local_ms = self._commands_for(
+ text, term, action, self._contains_negation(text)
+ )
+ commands.extend(resolved)
+ issues.extend(local_issues)
+ resolution_ms += local_ms
+ return _FragmentAnalysis(
+ text,
+ tuple(commands),
+ tuple(dict.fromkeys(issues)),
+ incomplete=not commands,
+ resolution_ms=resolution_ms,
+ )
+
+ def _analyse_coordinated_controls(
+ self, text: str
+ ) -> _FragmentAnalysis | None:
+ """Resolve explicit target lists without leaking actions or scopes.
+
+ Each coordinated part is parsed independently. A missing ON/OFF verb
+ can inherit the shared verb, while target qualifiers remain local to
+ their own part. Other operations are left to the conservative parser
+ because propagating a value/unit across targets can be unsafe.
+ """
+
+ parts = [part for part in _COORDINATION_SPLIT.split(text) if part]
+ if len(parts) < 2:
+ return None
+ devices_by_part = [device_terms_in(part) for part in parts]
+ explicit_device_parts = [index for index, items in enumerate(devices_by_part) if items]
+ if not explicit_device_parts:
+ return None
+
+ actions_by_part = [action_terms_in(part) for part in parts]
+ explicit_actions = [
+ item[2]
+ for actions in actions_by_part
+ for item in actions
+ if item[2] in {"on", "off"}
+ ]
+ if not explicit_actions:
+ return None
+ shared_action = explicit_actions[0]
+
+ common_term: DeviceTerm | None = None
+ if len(explicit_device_parts) == 1:
+ common_term = self._most_specific_device(
+ devices_by_part[explicit_device_parts[0]]
+ )
+ if any(
+ not devices
+ and not any(alias in part for aliases in SCOPE_ALIASES.values() for alias in aliases)
+ for part, devices in zip(parts, devices_by_part, strict=True)
+ ):
+ return None
+ elif not all(devices_by_part):
+ return None
+
+ expanded: list[str] = []
+ for part, devices, actions in zip(
+ parts, devices_by_part, actions_by_part, strict=True
+ ):
+ local_action = next(
+ (item[2] for item in actions if item[2] in {"on", "off"}),
+ shared_action,
+ )
+ action_text = "打开" if local_action == "on" else "关闭"
+ candidate = part
+ if not devices and common_term is not None:
+ candidate = f"{candidate}{common_term.canonical}"
+ if not actions:
+ candidate = f"{action_text}{candidate}"
+ expanded.append(candidate)
+
+ analyses = [
+ self._analyse_fragment(part, allow_coordination=False) for part in expanded
+ ]
+ if any(not analysis.commands for analysis in analyses):
+ return None
+ return _FragmentAnalysis(
+ text,
+ tuple(command for analysis in analyses for command in analysis.commands),
+ tuple(
+ dict.fromkeys(
+ issue
+ for analysis in analyses
+ for issue in analysis.uncertain_reasons
+ )
+ ),
+ incomplete=any(analysis.incomplete for analysis in analyses),
+ resolution_ms=sum(analysis.resolution_ms for analysis in analyses),
+ )
+
+ def _application_slots_preserved(
+ self, text: str, analysis: _FragmentAnalysis
+ ) -> bool:
+ """Reject a legacy parse that silently downgraded an explicit value."""
+
+ expected_pct = self._percentage_value(text)
+ if expected_pct is None:
+ return True
+ parameters = [command.parameters for command in analysis.commands]
+ if any(
+ value == expected_pct
+ for item in parameters
+ for key, value in item.items()
+ if key.endswith("_pct")
+ ):
+ return True
+ # The roof protocol has a native half-open motion exactly equivalent
+ # to 50%; retain that authoritative representation.
+ return expected_pct == 50 and any(
+ item.get("motion") == "half" for item in parameters
+ )
+
+ def _coalesce_atomic_commands(
+ self, commands: list[SpeechCommandIntent]
+ ) -> tuple[list[SpeechCommandIntent], set[str], list[str], float]:
+ """Combine adjacent slots when one catalog capability requires them atomically."""
+
+ output: list[SpeechCommandIntent] = []
+ obsolete_issues: set[str] = set()
+ issues: list[str] = []
+ resolution_ms = 0.0
+ index = 0
+ while index < len(commands):
+ current = commands[index]
+ following = commands[index + 1] if index + 1 < len(commands) else None
+ if (
+ following is not None
+ and current.property_name == following.property_name == "ambient.color"
+ and current.target_scope == following.target_scope
+ and "color_hex" in current.parameters
+ and "brightness_pct" in following.parameters
+ ):
+ parameters = {
+ "color_hex": current.parameters["color_hex"],
+ "brightness_pct": following.parameters["brightness_pct"],
+ }
+ started = time.perf_counter()
+ resolution = self.resolver.resolve(
+ domain="ambient_light",
+ semantic_property="ambient.color",
+ target_scope=current.target_scope,
+ target_device=None,
+ operation="set",
+ parameters=parameters,
+ source_text=f"{current.source_text}，{following.source_text}",
+ preferred_capability="ambient_light.set_illumination",
+ )
+ resolution_ms += (time.perf_counter() - started) * 1000
+ obsolete_issues.update(current.validation_issues)
+ obsolete_issues.update(following.validation_issues)
+ merged_issues = tuple(resolution.issues)
+ output.append(
+ replace(
+ current,
+ capability=(
+ resolution.definition.name
+ if resolution.definition is not None
+ else None
+ ),
+ parameters=parameters,
+ source_text=f"{current.source_text}，{following.source_text}",
+ resolution_status=resolution.status,
+ validation_issues=merged_issues,
+ confidence=0.96 if resolution.status == "executable" else 0.55,
+ executor_binding=resolution.executor_binding,
+ executor_payload=resolution.executor_payload,
+ intent_ir_action=resolution.intent_ir_action,
+ )
+ )
+ issues.extend(merged_issues)
+ index += 2
+ continue
+ output.append(current)
+ index += 1
+ return output, obsolete_issues, issues, resolution_ms
+
+ def _analyse_application_intent(self, text: str) -> _FragmentAnalysis:
+ """Reuse deterministic actions already used by one-sentence orchestration."""
+
+ parsed = merge_native_can_intent({}, text)
+ actions = list(parsed.get("native_can_actions", ()))
+ if not actions:
+ return _FragmentAnalysis(text, ())
+
+ commands: list[SpeechCommandIntent] = []
+ issues: list[str] = []
+ resolution_ms = 0.0
+ for action in actions:
+ started = time.perf_counter()
+ resolution = self.resolver.resolve_action(action, source_text=text)
+ resolution_ms += (time.perf_counter() - started) * 1000
+ definition = resolution.definition
+ parameters = dict(action.get("parameters", {}))
+ scope = str(action.get("target", action.get("target_scope", "vehicle")))
+ explicit_scope, spoken_scope = self._scope_qualifier(text, scope)
+ property_name = self.resolver.semantic_property_for(
+ str(action.get("capability", "")), parameters
+ )
+ operation = self._semantic_operation(parameters, str(action.get("operation", "set")))
+ implicit_parameters = self._implicit_application_parameters(text, parameters)
+ value, unit = self._semantic_value(definition, parameters, implicit_parameters)
+ command_issues = tuple(resolution.issues)
+ command_qualifiers: dict[str, Any] = {
+ "target_explicit": explicit_scope,
+ "spoken_scope": spoken_scope,
+ "parse_source": "application_intent",
+ "implicit_parameters": list(implicit_parameters),
+ }
+ if "level" in parameters and "level" not in implicit_parameters:
+ command_qualifiers["display_level"] = self._display_level(text)
+ commands.append(
+ SpeechCommandIntent(
+ domain=str(action.get("domain", "")),
+ capability=definition.name if definition is not None else None,
+ target_scope=scope,
+ target_device=(
+ definition.target_devices[0]
+ if definition is not None and definition.target_devices
+ else str(action.get("domain", "device"))
+ ),
+ property_name=property_name,
+ operation=operation,
+ parameters=parameters,
+ value=value,
+ unit=unit,
+ qualifiers=command_qualifiers,
+ negated=self._contains_negation(text),
+ confidence=(0.98 if resolution.status == "executable" else 0.55),
+ source_text=text,
+ resolution_status=resolution.status,
+ validation_issues=command_issues,
+ executor_binding=resolution.executor_binding,
+ executor_payload=resolution.executor_payload,
+ intent_ir_action=resolution.intent_ir_action,
+ )
+ )
+ issues.extend(command_issues)
+ return _FragmentAnalysis(
+ text,
+ tuple(commands),
+ tuple(dict.fromkeys(issues)),
+ resolution_ms=resolution_ms,
+ )
+
+ @staticmethod
+ def _semantic_operation(parameters: dict[str, Any], fallback: str) -> str:
+ state = parameters.get("state")
+ if state in {"on", "off"}:
+ return str(state)
+ motion = parameters.get("motion")
+ if motion in {"open", "close", "stop"}:
+ return str(motion)
+ direction = parameters.get("direction")
+ if direction in {"forward", "backward", "up", "down", "stop"}:
+ return str(direction)
+ return fallback
+
+ @staticmethod
+ def _semantic_value(
+ definition: Any,
+ parameters: dict[str, Any],
+ implicit_parameters: tuple[str, ...] = (),
+ ) -> tuple[Any, str | None]:
+ preferred = (
+ "temperature_c",
+ "position_pct",
+ "angle_deg",
+ "angle_delta_deg",
+ "level",
+ "brightness_pct",
+ "brightness_level",
+ "color_hex",
+ "color_index",
+ "state",
+ "motion",
+ "direction",
+ "mode",
+ "slot",
+ )
+ name = next(
+ (item for item in preferred if item in parameters and item not in implicit_parameters),
+ None,
+ )
+ if name is None:
+ name = next((item for item in preferred if item in parameters), None)
+ if name is None:
+ return (next(iter(parameters.values()), None), None)
+ units = {
+ str(item.get("name")): item.get("unit")
+ for item in (definition.parameters if definition is not None else ())
+ }
+ return parameters[name], str(units[name]) if units.get(name) else None
+
+ @staticmethod
+ def _implicit_application_parameters(text: str, parameters: dict[str, Any]) -> tuple[str, ...]:
+ implicit: list[str] = []
+ if "duration_ms" in parameters and not re.search(r"\d+\s*毫秒", text):
+ implicit.append("duration_ms")
+ if "level" in parameters and not re.search(
+ r"(?:\d+|[一二两三四五])\s*(?:档|级)|最低|低档|中档|最高|高档|最弱|最强",
+ text,
+ ):
+ implicit.append("level")
+ return tuple(implicit)
+
+ @classmethod
+ def _display_level(cls, text: str) -> str:
+ if re.search(r"最高|高档|最强", text):
+ return "高档"
+ if re.search(r"最低|低档|最弱", text):
+ return "低档"
+ if re.search(r"中档|中等|适中", text):
+ return "中档"
+ value = cls._numeric_value(text)
+ return f"{cls._format_number(value)}档" if value is not None else ""
+
+ def _commands_for(
+ self, text: str, term: DeviceTerm, action: str, negated: bool
+ ) -> tuple[list[SpeechCommandIntent], list[str], float]:
+ default_scope = (
+ "passenger"
+ if term.domain == "seat"
+ else "front_left"
+ if term.domain == "window"
+ else "all"
+ if term.semantic_key == "reading_light.power"
+ else "vehicle"
+ )
+ scope = scope_in(text, default=default_scope, domain=term.domain)
+ explicit_scope, spoken_scope = self._scope_qualifier(text, scope)
+ qualifiers: dict[str, Any] = {
+ "target_explicit": explicit_scope,
+ "spoken_scope": spoken_scope,
+ }
+ spoken_directions = [
+ action
+ for _, _, action, _ in action_terms_in(text)
+ if action in {"forward", "backward", "up", "down"}
+ ]
+ if spoken_directions:
+ qualifiers["spoken_direction"] = spoken_directions[-1]
+ value = self._numeric_value(text)
+ percent_value = self._percentage_value(text)
+ has_degree = "度" in text
+ issues: list[str] = []
+ commands: list[SpeechCommandIntent] = []
+ resolution_ms = 0.0
+
+ def make(
+ semantic_property: str,
+ operation: str,
+ parameters: dict[str, Any],
+ *,
+ semantic_value: Any = None,
+ unit: str | None = None,
+ confidence: float = 0.94,
+ forced_status: ResolutionStatus | None = None,
+ extra_issues: tuple[str, ...] = (),
+ display_value: str | None = None,
+ ) -> SpeechCommandIntent:
+ nonlocal resolution_ms
+ resolution_started = time.perf_counter()
+ resolution = self.resolver.resolve(
+ domain=term.domain,
+ semantic_property=semantic_property,
+ target_scope=scope,
+ target_device=None,
+ operation=operation,
+ parameters=parameters,
+ source_text=text,
+ )
+ resolution_ms += (time.perf_counter() - resolution_started) * 1000
+ command_issues = tuple(dict.fromkeys((*resolution.issues, *extra_issues)))
+ resolved_status = forced_status or resolution.status
+ if extra_issues and forced_status is None and resolved_status == "executable":
+ resolved_status = "incomplete"
+ definition = resolution.definition
+ command_qualifiers = dict(qualifiers)
+ if display_value:
+ command_qualifiers["display_value"] = display_value
+ return SpeechCommandIntent(
+ domain=term.domain,
+ capability=definition.name if definition is not None else None,
+ target_scope=scope,
+ target_device=(
+ definition.target_devices[0]
+ if definition is not None and definition.target_devices
+ else term.target_device
+ ),
+ property_name=semantic_property,
+ operation=operation,
+ parameters=dict(parameters),
+ value=semantic_value,
+ unit=unit,
+ qualifiers=command_qualifiers,
+ negated=negated,
+ confidence=confidence
+ if resolved_status == "executable" and not command_issues
+ else min(confidence, 0.55),
+ source_text=text,
+ resolution_status=resolved_status,
+ validation_issues=command_issues,
+ executor_binding=resolution.executor_binding
+ if resolved_status == "executable"
+ else None,
+ executor_payload=resolution.executor_payload
+ if resolved_status == "executable"
+ else None,
+ intent_ir_action=resolution.intent_ir_action
+ if resolved_status == "executable"
+ else None,
+ )
+
+ mode_profile = self.resolver.match_mode(text)
+ if mode_profile is not None:
+ mode_started = time.perf_counter()
+ mode_resolution = self.resolver.resolve_mode(
+ text,
+ target_scope=scope,
+ target_explicit=explicit_scope,
+ )
+ resolution_ms += (time.perf_counter() - mode_started) * 1000
+ mode_issues = tuple(mode_resolution.issues)
+ command_scope = scope if explicit_scope else mode_profile.operations[0].target_scope
+ command = SpeechCommandIntent(
+ domain="seat",
+ capability=None,
+ target_scope=command_scope,
+ target_device="seat",
+ property_name="seat.mode",
+ operation="apply",
+ parameters={"mode_profile_id": mode_profile.id},
+ value=mode_profile.name,
+ unit=None,
+ qualifiers=dict(qualifiers),
+ negated=negated,
+ confidence=0.96 if mode_resolution.status == "executable" else 0.55,
+ source_text=text,
+ resolution_status=mode_resolution.status,
+ validation_issues=mode_issues,
+ executor_binding="ModeProfileRegistry/ModelPlanAdapter"
+ if mode_resolution.status == "executable"
+ else None,
+ executor_payload=(
+ {
+ "mode_profile_id": mode_profile.id,
+ "operations": list(mode_resolution.executor_payloads),
+ }
+ if mode_resolution.status == "executable"
+ else None
+ ),
+ mode_profile_id=mode_profile.id,
+ )
+ commands.append(command)
+ issues.extend(mode_issues)
+ return commands, issues, resolution_ms
+
+ if term.domain == "climate":
+ if term.semantic_key == "hvac.fan_level":
+ local = () if value is not None else ("fan_level_incomplete",)
+ commands.append(
+ make(
+ "hvac.fan_level",
+ "set",
+ {"level": int(value)} if value is not None else {},
+ semantic_value=value,
+ unit="level",
+ forced_status="incomplete" if local else None,
+ extra_issues=local,
+ )
+ )
+ elif term.semantic_key == "hvac.temperature":
+ local = () if value is not None else ("temperature_incomplete",)
+ commands.append(
+ make(
+ "hvac.temperature",
+ "set",
+ {"temperature_c": value} if value is not None else {},
+ semantic_value=value,
+ unit="celsius",
+ forced_status="incomplete" if local else None,
+ extra_issues=local,
+ )
+ )
+ else:
+ if action in {"on", "off"}:
+ commands.append(
+ make("hvac.power", action, {"state": action}, semantic_value=action)
+ )
+ if value is not None and (action in {"set", "adjust", "on"} or "温度" in text):
+ commands.append(
+ make(
+ "hvac.temperature",
+ "set",
+ {"temperature_c": value},
+ semantic_value=value,
+ unit="celsius",
+ )
+ )
+ if not commands:
+ issues.append("climate_action_or_value_incomplete")
+ issues.extend(issue for command in commands for issue in command.validation_issues)
+ return commands, issues, resolution_ms
+
+ if term.semantic_key in {"seat.ventilation", "seat.heating"}:
+ local: tuple[str, ...] = ()
+ parameters: dict[str, Any] = {}
+ if action in {"on", "off"} or (action in {"set", "adjust"} and value is not None):
+ parameters["state"] = action if action in {"on", "off"} else "on"
+ if value is not None:
+ parameters["level"] = int(value)
+ else:
+ local = ("seat_thermal_action_incomplete",)
+ commands.append(
+ make(
+ term.semantic_key,
+ action,
+ parameters,
+ semantic_value=parameters.get("state"),
+ unit="level" if "level" in parameters else None,
+ forced_status="incomplete" if local else None,
+ extra_issues=local,
+ )
+ )
+ issues.extend(issue for command in commands for issue in command.validation_issues)
+ return commands, issues, resolution_ms
+
+ if term.domain == "seat":
+ parameters: dict[str, Any] = {}
+ local: list[str] = []
+ semantic_unit: str | None = None
+ if value is not None or percent_value is not None:
+ if term.semantic_key == "seat.backrest_recline":
+ if action in {"forward", "backward"}:
+ if value is not None:
+ parameters["angle_delta_deg"] = abs(value)
+ semantic_unit = "degree"
+ elif percent_value is not None:
+ parameters["position_pct"] = percent_value
+ semantic_unit = "percent"
+ elif percent_value is not None:
+ parameters["position_pct"] = percent_value
+ semantic_unit = "percent"
+ elif has_degree:
+ parameters["angle_deg"] = value
+ semantic_unit = "degree"
+ else:
+ local.append("seat_value_unit_ambiguous")
+ elif percent_value is not None:
+ parameters["position_pct"] = percent_value
+ semantic_unit = "percent"
+ else:
+ local.append("seat_value_unit_ambiguous")
+ if not parameters:
+ local.append("seat_parameter_incomplete")
+ commands.append(
+ make(
+ term.semantic_key,
+ action,
+ parameters,
+ semantic_value=next(iter(parameters.values())) if parameters else value,
+ unit=semantic_unit,
+ confidence=0.9,
+ forced_status="incomplete" if local else None,
+ extra_issues=tuple(local),
+ )
+ )
+ issues.extend(issue for command in commands for issue in command.validation_issues)
+ return commands, issues, resolution_ms
+
+ if term.domain == "window":
+ if percent_value is not None:
+ commands.append(
+ make(
+ "window.position",
+ "set",
+ {"position_pct": percent_value},
+ semantic_value=percent_value,
+ unit="percent",
+ )
+ )
+ elif action in {"on", "forward", "up"}:
+ commands.append(
+ make("window.motion", "open", {"motion": "open"}, semantic_value="open")
+ )
+ elif action in {"off", "backward", "down"}:
+ commands.append(
+ make("window.motion", "close", {"motion": "close"}, semantic_value="close")
+ )
+ else:
+ issues.append("window_action_incomplete")
+ if value is not None and percent_value is None:
+ issues.append("window_numeric_value_without_percent")
+ commands = [
+ replace(
+ command,
+ resolution_status="ambiguous",
+ validation_issues=tuple(
+ dict.fromkeys(
+ (*command.validation_issues, "window_numeric_value_without_percent")
+ )
+ ),
+ executor_binding=None,
+ executor_payload=None,
+ intent_ir_action=None,
+ )
+ for command in commands
+ ]
+ issues.extend(issue for command in commands for issue in command.validation_issues)
+ return commands, issues, resolution_ms
+
+ if term.semantic_key == "roof.shade_motion":
+ half = bool(re.search(r"一半|半开|开一半", text))
+ if half:
+ commands.append(
+ make(
+ "roof.shade_motion",
+ "set",
+ {"motion": "half"},
+ semantic_value=50,
+ unit="percent",
+ )
+ )
+ elif percent_value is not None:
+ commands.append(
+ make(
+ "roof.shade_position",
+ "set",
+ {"position_pct": percent_value},
+ semantic_value=percent_value,
+ unit="percent",
+ )
+ )
+ else:
+ motion = "open" if action == "on" else "close" if action == "off" else ""
+ if motion:
+ commands.append(
+ make("roof.shade_motion", motion, {"motion": motion}, semantic_value=motion)
+ )
+ else:
+ issues.append("roof_shade_action_incomplete")
+ issues.extend(issue for command in commands for issue in command.validation_issues)
+ return commands, issues, resolution_ms
+
+ if term.semantic_key == "roof.transmittance":
+ local = () if percent_value is not None else ("roof_transmittance_incomplete",)
+ commands.append(
+ make(
+ "roof.transmittance",
+ "set",
+ {"transmittance_pct": percent_value} if percent_value is not None else {},
+ semantic_value=percent_value,
+ unit="percent",
+ forced_status="incomplete" if local else None,
+ extra_issues=local,
+ )
+ )
+ issues.extend(issue for command in commands for issue in command.validation_issues)
+ return commands, issues, resolution_ms
+
+ if term.semantic_key == "ambient.power":
+ spoken_color = color_in(text)
+ if action in {"on", "off"}:
+ commands.append(
+ make("ambient.power", action, {"state": action}, semantic_value=action)
+ )
+ if spoken_color is not None:
+ color_hex, color_alias = spoken_color
+ color_parameters: dict[str, Any] = {"color_hex": color_hex}
+ if percent_value is not None and "亮度" in text:
+ color_parameters["brightness_pct"] = percent_value
+ commands.append(
+ make(
+ "ambient.color",
+ "set",
+ color_parameters,
+ semantic_value=color_hex,
+ unit="rgb",
+ display_value=color_alias,
+ )
+ )
+ elif percent_value is not None and "亮度" in text:
+ commands.append(
+ make(
+ "ambient.color",
+ "set",
+ {"brightness_pct": percent_value},
+ semantic_value=percent_value,
+ unit="percent",
+ display_value="brightness_only",
+ )
+ )
+ elif value is not None and "亮度" in text:
+ commands.append(
+ make(
+ "ambient.brightness",
+ "set",
+ {"brightness_level": int(value)},
+ semantic_value=int(value),
+ unit="level",
+ )
+ )
+ if not commands:
+ issues.append("ambient_control_incomplete")
+ issues.extend(issue for command in commands for issue in command.validation_issues)
+ return commands, issues, resolution_ms
+
+ if term.semantic_key in {"reading_light.power", "indoor_light.power", "front_light.power"}:
+ if action in {"on", "off"}:
+ commands.append(
+ make(term.semantic_key, action, {"state": action}, semantic_value=action)
+ )
+ else:
+ issues.append("light_action_incomplete")
+ issues.extend(issue for command in commands for issue in command.validation_issues)
+ return commands, issues, resolution_ms
+
+ issues.append("unsupported_control_expression")
+ return commands, issues, resolution_ms
+
+ def _inherit_context(self, correction: str, old: _FragmentAnalysis) -> str:
+ correction = re.sub(r"^是[\s，,]*", "", correction)
+ if device_terms_in(correction):
+ return correction
+ if not old.commands:
+ devices = device_terms_in(old.text)
+ return (
+ f"{self._most_specific_device(devices).canonical}{correction}"
+ if devices
+ else correction
+ )
+ previous = old.commands[-1]
+ target = self._display_target(previous)
+ if action_terms_in(correction):
+ return f"{target}{correction}"
+ if self._numeric_value(correction) is not None:
+ return f"{target}调整到{correction}"
+ return f"{target}{correction}"
+
+ def _inherit_shared_target(self, fragment: str, command: SpeechCommandIntent) -> str:
+ target = self._display_target(command)
+ if (
+ command.domain == "seat"
+ and not command.qualifiers.get("target_explicit")
+ and command.target_scope in {"driver", "front_left", "passenger", "front_right"}
+ ):
+ scope = "主驾" if command.target_scope in {"driver", "front_left"} else "副驾"
+ target = f"{scope}{target}"
+ return f"{target}{fragment}"
+
+ @staticmethod
+ def _cancel_commands(
+ commands: Iterable[SpeechCommandIntent], *, negated: bool = False
+ ) -> tuple[SpeechCommandIntent, ...]:
+ return tuple(replace(command, status="cancelled", negated=negated) for command in commands)
+
+ @staticmethod
+ def _is_abandoned_false_start(
+ analysis: _FragmentAnalysis, following: _FragmentAnalysis | None
+ ) -> bool:
+ if following is None or not following.commands:
+ return False
+ for _, end, _, alias in action_terms_in(analysis.text):
+ tail = analysis.text[end:].strip()
+ if tail and len(tail) <= 2 and all(char == alias[0] for char in tail):
+ return True
+ return analysis.incomplete and not analysis.commands
+
+ @staticmethod
+ def _is_target_refinement(
+ analysis: _FragmentAnalysis, following: _FragmentAnalysis | None
+ ) -> bool:
+ if following is None or not following.commands or not analysis.commands:
+ return False
+ old, new = analysis.commands[-1], following.commands[0]
+ return (
+ old.domain == new.domain == "seat"
+ and old.property_name == "seat.longitudinal_position"
+ and new.property_name != old.property_name
+ and "度" not in analysis.text
+ and "%" not in analysis.text
+ )
+
+ @staticmethod
+ def _same_target_family(devices: list[tuple[int, int, DeviceTerm, str]]) -> bool:
+ return {item[2].domain for item in devices} == {"seat"}
+
+ @staticmethod
+ def _select_action(actions: list[tuple[int, int, str, str]]) -> str:
+ if not actions:
+ return "set"
+ if actions[-1][2] == "adjust":
+ for _, _, action, _ in reversed(actions[:-1]):
+ if action in {"forward", "backward", "up", "down"}:
+ return action
+ return actions[-1][2]
+
+ @staticmethod
+ def _most_specific_device(devices: list[tuple[int, int, DeviceTerm, str]]) -> DeviceTerm:
+ return max(devices, key=lambda item: (len(item[3]), item[2].canonical != "座椅"))[2]
+
+ @staticmethod
+ def _contains_negation(text: str) -> bool:
+ return any(marker in text for marker in NEGATION_PREFIXES) or bool(
+ re.search(r"(?:不能|不可|不可以)[^，,。！？!?]{0,8}(?:打开|开启|调整|调节|移动)", text)
+ )
+
+ @staticmethod
+ def _looks_like_control(text: str) -> bool:
+ return bool(device_terms_in(text) and action_terms_in(text))
+
+ @staticmethod
+ def _scope_qualifier(text: str, resolved_scope: str) -> tuple[bool, str]:
+ matches = [
+ (match.start(), scope, alias)
+ for scope, aliases in SCOPE_ALIASES.items()
+ for alias in aliases
+ for match in re.finditer(re.escape(alias), text)
+ ]
+ if not matches:
+ return False, ""
+ _, lexical_scope, alias = min(matches)
+ if resolved_scope == "front_left" and lexical_scope == "driver":
+ return True, "主驾"
+ if resolved_scope == "front_right" and lexical_scope == "passenger":
+ return True, "副驾"
+ labels = {
+ "driver": "主驾",
+ "passenger": "副驾",
+ "front_row": "前排",
+ "rear_row": "后排",
+ "rear_left": "左后",
+ "rear_right": "右后",
+ "all": "全部",
+ "vehicle": "全车",
+ }
+ return True, labels.get(lexical_scope, alias)
+
+ @staticmethod
+ def _numeric_value(text: str) -> int | float | None:
+ match = _NUMBER.search(text)
+ if match:
+ value = float(match.group(1))
+ return int(value) if value.is_integer() else value
+ chinese = re.search(r"[零〇一二两三四五六七八九十百点]+", text)
+ if not chinese:
+ return None
+ token = chinese.group(0)
+ digits = {
+ "零": 0,
+ "〇": 0,
+ "一": 1,
+ "二": 2,
+ "两": 2,
+ "三": 3,
+ "四": 4,
+ "五": 5,
+ "六": 6,
+ "七": 7,
+ "八": 8,
+ "九": 9,
+ }
+ if "点" in token:
+ whole, fraction = token.split("点", 1)
+ whole_value = UnderstandingTranscriptProcessor._chinese_integer(whole, digits)
+ if whole_value is None or not fraction or any(char not in digits for char in fraction):
+ return None
+ decimal = "".join(str(digits[char]) for char in fraction)
+ return float(f"{whole_value}.{decimal}")
+ return UnderstandingTranscriptProcessor._chinese_integer(token, digits)
+
+ @classmethod
+ def _percentage_value(cls, text: str) -> int | float | None:
+ return parse_percentage_value(text)
+
+ @staticmethod
+ def _chinese_integer(token: str, digits: dict[str, int]) -> int | None:
+ if not token:
+ return 0
+ if "百" in token:
+ left, right = token.split("百", 1)
+ hundreds = digits.get(left[-1], 1) if left else 1
+ tail = UnderstandingTranscriptProcessor._chinese_integer(right, digits)
+ return None if tail is None else hundreds * 100 + tail
+ if "十" in token:
+ left, right = token.split("十", 1)
+ tens = digits.get(left[-1], 1) if left else 1
+ ones = digits.get(right[-1], 0) if right else 0
+ return tens * 10 + ones
+ if all(char in digits for char in token):
+ return int("".join(str(digits[char]) for char in token))
+ return None
+
+ def _render(self, commands: list[SpeechCommandIntent]) -> str:
+ rendered: list[str] = []
+ index = 0
+ while index < len(commands):
+ command = commands[index]
+ if (
+ command.property_name == "hvac.power"
+ and command.operation == "on"
+ and index + 1 < len(commands)
+ and commands[index + 1].property_name == "hvac.temperature"
+ and commands[index + 1].source_text == command.source_text
+ ):
+ value = commands[index + 1].parameters.get("temperature_c")
+ rendered.append(f"打开空调并设置为{self._format_number(value)}度")
+ index += 2
+ continue
+ rendered.append(self._render_command(command))
+ index += 1
+ return "，然后".join(part for part in rendered if part) + ("。" if rendered else "")
+
+ def _render_command(self, command: SpeechCommandIntent) -> str:
+ parameters = command.parameters
+ prop = command.property_name
+ if command.resolution_status != "executable" and not parameters:
+ # A non-executable command must never look successful after its
+ # critical value/unit was discarded. Preserve the understood
+ # source meaning so the UI can explain or request clarification.
+ return command.source_text.strip("，,；;。！？!? ")
+ if prop == "hvac.power":
+ return f"{'打开' if command.operation == 'on' else '关闭'}{self._scoped_target(command, '空调')}"
+ if prop == "hvac.temperature":
+ return f"将{self._scoped_target(command, '空调')}设置为{self._format_number(parameters.get('temperature_c'))}度"
+ if prop == "hvac.fan_level":
+ return f"将空调风量设置为{self._format_number(parameters.get('level'))}档"
+ if prop in {"seat.ventilation", "seat.heating"}:
+ feature = "座椅通风" if prop.endswith("ventilation") else "座椅加热"
+ implicit = set(command.qualifiers.get("implicit_parameters", ()))
+ if "level" in parameters and "level" not in implicit:
+ level = command.qualifiers.get("display_level") or (
+ f"{self._format_number(parameters['level'])}档"
+ )
+ return f"将{self._scoped_target(command, feature)}设置为{level}"
+ return f"{'打开' if command.operation == 'on' else '关闭'}{self._scoped_target(command, feature)}"
+ if prop == "seat.mode":
+ return f"将{self._scoped_target(command, '座椅')}调整为{command.value}"
+ if prop == "seat.backrest_recline":
+ direction = (
+ "向后"
+ if command.operation == "backward"
+ else "向前"
+ if command.operation == "forward"
+ else ""
+ )
+ if "angle_delta_deg" in parameters:
+ return f"将{self._display_target(command)}{direction}调整{self._format_number(parameters['angle_delta_deg'])}度"
+ if "angle_deg" in parameters:
+ return f"将{self._display_target(command)}调整到{self._format_number(parameters['angle_deg'])}度"
+ if "position_pct" in parameters:
+ return f"将{self._display_target(command)}调整到{self._format_number(parameters['position_pct'])}%"
+ suffix = "一点" if "一点" in command.source_text else ""
+ return f"将{self._display_target(command)}{direction}调整{suffix}"
+ if prop.startswith("seat."):
+ direction = (
+ "向后"
+ if command.operation == "backward"
+ else "向前"
+ if command.operation == "forward"
+ else {
+ "up": "向上",
+ "down": "向下",
+ "forward": "向前",
+ "backward": "向后",
+ }.get(str(command.qualifiers.get("spoken_direction", "")), "")
+ )
+ if "position_pct" in parameters:
+ return f"将{self._display_target(command)}{direction}调整到{self._format_number(parameters['position_pct'])}%"
+ suffix = "一点" if "一点" in command.source_text else ""
+ return f"将{self._display_target(command)}{direction}调整{suffix}"
+ if prop == "window.motion":
+ return f"{'打开' if command.operation == 'open' else '关闭'}{self._display_target(command)}"
+ if prop == "window.position":
+ return f"将{self._display_target(command)}打开到{self._format_number(parameters.get('position_pct'))}%"
+ if prop == "roof.shade_motion":
+ motion = parameters.get("motion")
+ return (
+ "将遮阳帘打开一半"
+ if motion == "half"
+ else f"{'打开' if motion == 'open' else '关闭'}遮阳帘"
+ )
+ if prop == "roof.shade_position":
+ return f"将遮阳帘打开到{self._format_number(parameters.get('position_pct'))}%"
+ if prop == "roof.transmittance":
+ return f"将天幕透光率设置为{self._format_number(parameters.get('transmittance_pct'))}%"
+ if prop == "ambient.color":
+ display = command.qualifiers.get("display_value") or command.value
+ brightness = parameters.get("brightness_pct")
+ if display == "brightness_only":
+ return f"将氛围灯亮度设置为{self._format_number(brightness)}%"
+ return (
+ f"将氛围灯颜色调整为{display}并将亮度设置为{self._format_number(brightness)}%"
+ if brightness is not None
+ else f"将氛围灯颜色调整为{display}"
+ )
+ if prop == "ambient.brightness":
+ return f"将氛围灯亮度设置为{self._format_number(parameters.get('brightness_level'))}档"
+ if prop == "ambient.power":
+ return f"{'打开' if command.operation == 'on' else '关闭'}氛围灯"
+ if prop in {"reading_light.power", "indoor_light.power", "front_light.power"}:
+ return (
+ f"{'打开' if command.operation == 'on' else '关闭'}{self._display_target(command)}"
+ )
+ return command.source_text
+
+ @staticmethod
+ def _display_target(command: SpeechCommandIntent) -> str:
+ names = {
+ "hvac.power": "空调",
+ "hvac.temperature": "空调",
+ "hvac.fan_level": "空调风量",
+ "seat.backrest_recline": "座椅靠背",
+ "seat.longitudinal_position": "座椅",
+ "seat.cushion_tilt": "座椅坐垫",
+ "seat.legrest_height": "座椅腿托",
+ "seat.legrest_extension": "座椅腿托",
+ "seat.ventilation": "座椅通风",
+ "seat.heating": "座椅加热",
+ "window.motion": "车窗",
+ "window.position": "车窗",
+ "roof.shade_motion": "遮阳帘",
+ "roof.shade_position": "遮阳帘",
+ "roof.transmittance": "天幕",
+ "ambient.power": "氛围灯",
+ "ambient.color": "氛围灯",
+ "reading_light.power": "阅读灯",
+ "indoor_light.power": "车内灯",
+ "front_light.power": "前大灯",
+ }
+ scope = (
+ str(command.qualifiers.get("spoken_scope", ""))
+ if command.qualifiers.get("target_explicit")
+ else ""
+ )
+ return f"{scope}{names.get(command.property_name, command.target_device)}"
+
+ @classmethod
+ def _scoped_target(cls, command: SpeechCommandIntent, target: str) -> str:
+ scope = (
+ str(command.qualifiers.get("spoken_scope", ""))
+ if command.qualifiers.get("target_explicit")
+ else ""
+ )
+ return f"{scope}{target}"
+
+ @staticmethod
+ def _format_number(value: object) -> str:
+ if isinstance(value, float) and value.is_integer():
+ return str(int(value))
+ return str(value if value is not None else "")
+
+ @staticmethod
+ def _semantic_status(
+ commands: list[SpeechCommandIntent], reasons: tuple[str, ...]
+ ) -> TranscriptStatus:
+ statuses = {command.resolution_status for command in commands}
+ if "incomplete" in statuses:
+ return "incomplete"
+ if "ambiguous" in statuses:
+ return "ambiguous"
+ if "unsupported" in statuses:
+ return "recognized_unsupported"
+ if commands:
+ return "recognized_executable"
+ if any("incomplete" in reason or "not_resolved" in reason for reason in reasons):
+ return "incomplete"
+ return "no_command"
+
+ @staticmethod
+ def _semantic_trace(
+ raw_text: str,
+ normalized_text: str,
+ clauses: list[str],
+ commands: list[SpeechCommandIntent],
+ clean_text: str,
+ semantic_status: TranscriptStatus,
+ ) -> tuple[dict[str, Any], ...]:
+ clause_slots = []
+ for clause in clauses:
+ clause_slots.append(
+ {
+ "text": clause,
+ "entities": [
+ {
+ "canonical": term.canonical,
+ "domain": term.domain,
+ "semantic_key": term.semantic_key,
+ "source": alias,
+ }
+ for _, _, term, alias in device_terms_in(clause)
+ ],
+ "operations": [action for _, _, action, _ in action_terms_in(clause)],
+ "qualifiers": [
+ alias
+ for aliases in SCOPE_ALIASES.values()
+ for alias in aliases
+ if alias in clause
+ ],
+ "color": color_in(clause),
+ }
+ )
+ return (
+ {"stage": "raw_text", "text": raw_text},
+ {"stage": "normalized_text", "text": normalized_text},
+ {"stage": "clause_segmentation", "clauses": clause_slots},
+ {"stage": "semantic_commands", "commands": [command.to_dict() for command in commands]},
+ {
+ "stage": "intent_ir",
+ "actions": [
+ command.intent_ir_action
+ for command in commands
+ if command.intent_ir_action is not None
+ ],
+ },
+ {
+ "stage": "capability_resolution",
+ "results": [
+ {
+ "property": command.property_name,
+ "capability": command.capability,
+ "status": command.resolution_status,
+ "issues": list(command.validation_issues),
+ "executor_binding": command.executor_binding,
+ }
+ for command in commands
+ ],
+ },
+ {
+ "stage": "executor_payload",
+ "payloads": [
+ command.executor_payload
+ for command in commands
+ if command.executor_payload is not None
+ ],
+ },
+ {"stage": "clean_text", "text": clean_text, "semantic_status": semantic_status},
+ )
+
+ def _prepare(self, text: str) -> tuple[str, list[str]]:
+ removed: list[str] = []
+ text = _POLITE_PREFIX.sub("", text)
+ text = _POLITE_SUFFIX.sub("", text)
+
+ def remove(match: re.Match[str]) -> str:
+ removed.append(match.group(0))
+ return ""
+
+ text = _VOCAL_FILLER.sub(remove, text)
+ text = _ASR_FILLER_BEFORE_VALUE.sub(remove, text)
+
+ def separate_after_value(match: re.Match[str]) -> str:
+ removed.append(match.group(0))
+ return "，"
+
+ text = _ASR_FILLER_AFTER_VALUE.sub(separate_after_value, text)
+ text = _REPEATED_DEMONSTRATIVE.sub(remove, text)
+ previous = None
+ while previous != text:
+ previous = text
+ text = _LEADING_FILLER.sub(remove, text)
+ text = self._collapse_lexical_stutters(text)
+ text = self._collapse_target_echoes(text)
+ return self._tidy(text), removed
+
+ @staticmethod
+ def _collapse_lexical_stutters(text: str) -> str:
+ aliases = sorted(
+ {alias for values in ACTION_ALIASES.values() for alias in values if len(alias) >= 2},
+ key=len,
+ reverse=True,
+ )
+ for alias in aliases:
+ if alias.startswith("调"):
+ continue
+ text = re.sub(rf"{re.escape(alias[0])}{{1,3}}(?={re.escape(alias)})", "", text)
+ return text
+
+ @staticmethod
+ def _collapse_target_echoes(text: str) -> str:
+ for term in DEVICE_TERMS:
+ aliases = sorted(term.aliases, key=len, reverse=True)
+ group = "|".join(map(re.escape, aliases))
+ text = re.sub(
+ rf"(?P{group})(?:的)?(?:靠)?(?P{group})(?=(?:向|往|朝|调|调整|设置|打开|关闭))",
+ term.canonical,
+ text,
+ )
+ for alias in aliases:
+ text = re.sub(rf"(?:{re.escape(alias)}){{2,}}", term.canonical, text)
+ return text
+
+ @staticmethod
+ def _tidy(text: str) -> str:
+ text = re.sub(r"\s+", " ", str(text or "")).strip()
+ text = re.sub(r"\s*([，。！？；,!?;])\s*", r"\1", text)
+ text = re.sub(r"^[，,；;。\s]+", "", text)
+ text = re.sub(r"[，,；;\s]+([。！？!?])", r"\1", text)
+ text = re.sub(r"[，,]{2,}", "，", text)
+ return text.strip()
+
+ @staticmethod
+ def _sentence(text: str) -> str:
+ return text if _TERMINAL.search(text) else f"{text}。"
diff --git a/src/cognitive_agent/infrastructure/speech/vad.py b/src/cognitive_agent/infrastructure/speech/vad.py
new file mode 100644
index 0000000..db02eea
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/vad.py
@@ -0,0 +1,173 @@
+"""Sherpa-ONNX Silero VAD with conservative boundary padding."""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import threading
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from cognitive_agent.infrastructure.speech.tracing import env_flag
+
+
+@dataclass(frozen=True)
+class VadAudioSegment:
+ segment_id: str
+ sequence: int
+ start_sample: int
+ end_sample: int
+ samples: tuple[float, ...]
+ audio_overlap_ms: int = 0
+
+ @property
+ def start_ms(self) -> int:
+ return round(self.start_sample / 16)
+
+ @property
+ def end_ms(self) -> int:
+ return round(self.end_sample / 16)
+
+
+@dataclass(frozen=True)
+class VadSegmentation:
+ segments: tuple[VadAudioSegment, ...]
+ elapsed_ms: float
+ used_vad: bool
+ fallback_reason: str = ""
+
+
+class SpeechVadSegmenter:
+ """Reuse one native VAD instance and reset its state per utterance."""
+
+ sample_rate = 16_000
+
+ def __init__(self, repository_root: Path) -> None:
+ default_model = repository_root / "artifacts" / "models" / "silero-vad" / "silero_vad.onnx"
+ self.model_path = Path(os.getenv("SPEECH_VAD_MODEL", str(default_model))).expanduser()
+ self.enabled = env_flag("SPEECH_VAD_ENABLED", True)
+ self.threshold = float(os.getenv("SPEECH_VAD_THRESHOLD", "0.45"))
+ self.min_speech_seconds = float(os.getenv("SPEECH_VAD_MIN_SPEECH", "0.16"))
+ self.min_silence_seconds = float(os.getenv("SPEECH_VAD_MIN_SILENCE", "0.65"))
+ self.max_speech_seconds = float(os.getenv("SPEECH_VAD_MAX_SPEECH", "12.0"))
+ self.pre_roll_seconds = float(os.getenv("SPEECH_VAD_PRE_ROLL", "0.25"))
+ self.post_roll_seconds = float(os.getenv("SPEECH_VAD_POST_ROLL", "0.35"))
+ self.buffer_seconds = float(os.getenv("SPEECH_VAD_BUFFER", "32"))
+ self.num_threads = max(1, min(4, int(os.getenv("SPEECH_VAD_THREADS", "1"))))
+ self._vad: object | None = None
+ self._lock = threading.Lock()
+
+ @property
+ def available(self) -> bool:
+ return (
+ self.enabled
+ and self.model_path.is_file()
+ and importlib.util.find_spec("sherpa_onnx") is not None
+ )
+
+ def status(self) -> dict[str, object]:
+ return {
+ "enabled": self.enabled,
+ "available": self.available,
+ "loaded": self._vad is not None,
+ "model_path": str(self.model_path),
+ "threshold": self.threshold,
+ "min_speech_seconds": self.min_speech_seconds,
+ "min_silence_seconds": self.min_silence_seconds,
+ "max_speech_seconds": self.max_speech_seconds,
+ "pre_roll_seconds": self.pre_roll_seconds,
+ "post_roll_seconds": self.post_roll_seconds,
+ }
+
+ def segment(self, samples: Sequence[float]) -> VadSegmentation:
+ started = time.perf_counter()
+ if not self.available:
+ return self._fallback(samples, started, "vad_unavailable")
+ with self._lock:
+ vad = self._get_vad()
+ try:
+ vad.reset()
+ chunk_size = 512
+ for offset in range(0, len(samples), chunk_size):
+ vad.accept_waveform(samples[offset : offset + chunk_size])
+ vad.flush()
+ raw_boundaries: list[tuple[int, int]] = []
+ while not vad.empty():
+ segment = vad.front
+ start = max(0, int(segment.start))
+ end = min(len(samples), start + len(segment.samples))
+ if end > start:
+ raw_boundaries.append((start, end))
+ vad.pop()
+ finally:
+ vad.reset()
+ if not raw_boundaries:
+ return self._fallback(samples, started, "vad_found_no_segment")
+
+ pre_roll = round(self.pre_roll_seconds * self.sample_rate)
+ post_roll = round(self.post_roll_seconds * self.sample_rate)
+ padded = [
+ (max(0, start - pre_roll), min(len(samples), end + post_roll))
+ for start, end in raw_boundaries
+ ]
+ # Padding must not create overlapped audio submitted to two ASR streams.
+ merged: list[tuple[int, int]] = []
+ for start, end in padded:
+ if merged and start <= merged[-1][1]:
+ merged[-1] = (merged[-1][0], max(merged[-1][1], end))
+ else:
+ merged.append((start, end))
+ segments = tuple(
+ VadAudioSegment(
+ segment_id=f"segment-{sequence}",
+ sequence=sequence,
+ start_sample=start,
+ end_sample=end,
+ samples=tuple(samples[start:end]),
+ )
+ for sequence, (start, end) in enumerate(merged)
+ )
+ return VadSegmentation(
+ segments,
+ (time.perf_counter() - started) * 1000,
+ used_vad=True,
+ )
+
+ def _get_vad(self):
+ if self._vad is not None:
+ return self._vad
+ import sherpa_onnx
+
+ silero = sherpa_onnx.SileroVadModelConfig(
+ model=str(self.model_path),
+ threshold=self.threshold,
+ min_silence_duration=self.min_silence_seconds,
+ min_speech_duration=self.min_speech_seconds,
+ window_size=512,
+ max_speech_duration=self.max_speech_seconds,
+ )
+ config = sherpa_onnx.VadModelConfig(
+ silero_vad=silero,
+ sample_rate=self.sample_rate,
+ num_threads=self.num_threads,
+ provider="cpu",
+ debug=env_flag("SPEECH_VAD_DEBUG"),
+ )
+ self._vad = sherpa_onnx.VoiceActivityDetector(config, self.buffer_seconds)
+ return self._vad
+
+ @staticmethod
+ def _fallback(
+ samples: Sequence[float], started: float, reason: str
+ ) -> VadSegmentation:
+ segment = VadAudioSegment(
+ "segment-0", 0, 0, len(samples), tuple(samples)
+ )
+ return VadSegmentation(
+ (segment,),
+ (time.perf_counter() - started) * 1000,
+ used_vad=False,
+ fallback_reason=reason,
+ )
diff --git a/src/cognitive_agent/infrastructure/speech/vocabulary.py b/src/cognitive_agent/infrastructure/speech/vocabulary.py
new file mode 100644
index 0000000..744bcaa
--- /dev/null
+++ b/src/cognitive_agent/infrastructure/speech/vocabulary.py
@@ -0,0 +1,283 @@
+"""Chinese lexical aliases used before authoritative capability resolution."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class DeviceTerm:
+ canonical: str
+ domain: str
+ semantic_key: str
+ target_device: str
+ aliases: tuple[str, ...]
+
+
+DEVICE_TERMS: tuple[DeviceTerm, ...] = (
+ DeviceTerm("空调", "climate", "hvac.power", "hvac", ("空调", "空调系统")),
+ DeviceTerm(
+ "空调温度",
+ "climate",
+ "hvac.temperature",
+ "hvac",
+ ("空调温度", "温度"),
+ ),
+ DeviceTerm(
+ "空调风量",
+ "climate",
+ "hvac.fan_level",
+ "hvac",
+ ("空调风量", "风量", "风速"),
+ ),
+ DeviceTerm(
+ "座椅靠背",
+ "seat",
+ "seat.backrest_recline",
+ "seat_backrest_motor",
+ ("座椅靠背", "座椅的靠背", "靠背", "椅背"),
+ ),
+ DeviceTerm(
+ "座椅",
+ "seat",
+ "seat.longitudinal_position",
+ "seat_longitudinal_motor",
+ ("座椅", "坐椅", "座位"),
+ ),
+ DeviceTerm(
+ "座椅坐垫",
+ "seat",
+ "seat.cushion_tilt",
+ "seat_cushion_tilt_motor",
+ ("座椅坐垫", "坐垫", "坐盆"),
+ ),
+ DeviceTerm(
+ "座椅腿托",
+ "seat",
+ "seat.legrest_height",
+ "seat_legrest_lift_motor",
+ ("座椅腿托", "腿托", "脚托", "小腿托"),
+ ),
+ DeviceTerm(
+ "座椅通风",
+ "seat",
+ "seat.ventilation",
+ "seat",
+ ("座椅通风", "通风座椅"),
+ ),
+ DeviceTerm(
+ "座椅加热",
+ "seat",
+ "seat.heating",
+ "seat",
+ ("座椅加热", "加热座椅"),
+ ),
+ DeviceTerm(
+ "车窗",
+ "window",
+ "window.motion",
+ "window",
+ ("车窗", "窗户", "玻璃窗"),
+ ),
+ DeviceTerm(
+ "遮阳帘",
+ "roof",
+ "roof.shade_motion",
+ "roof_shade",
+ ("遮阳帘", "天幕帘", "天窗遮阳帘"),
+ ),
+ DeviceTerm(
+ "天幕",
+ "roof",
+ "roof.transmittance",
+ "roof_glass",
+ ("天幕", "天窗玻璃"),
+ ),
+ DeviceTerm(
+ "氛围灯",
+ "ambient_light",
+ "ambient.power",
+ "ambient_light",
+ ("氛围灯", "环境灯"),
+ ),
+ DeviceTerm(
+ "阅读灯",
+ "interior_light",
+ "reading_light.power",
+ "reading_light",
+ ("阅读灯",),
+ ),
+ DeviceTerm(
+ "车内灯",
+ "interior_light",
+ "indoor_light.power",
+ "indoor_light",
+ ("车内灯", "室内灯", "顶灯"),
+ ),
+ DeviceTerm(
+ "前大灯",
+ "front_light",
+ "front_light.power",
+ "front_light",
+ ("前大灯", "大灯", "前照灯"),
+ ),
+)
+
+SCOPE_ALIASES: dict[str, tuple[str, ...]] = {
+ "driver": ("主驾", "主驾驶", "驾驶位", "司机位", "左前"),
+ "passenger": ("副驾", "副驾驶", "乘客位", "右前"),
+ "front_row": ("前排",),
+ "rear_row": ("后排",),
+ "rear_left": ("左后", "后排左侧"),
+ "rear_right": ("右后", "后排右侧"),
+ "vehicle": ("全车", "整车", "车内"),
+ "all": ("全部", "所有", "四个"),
+}
+
+ACTION_ALIASES: dict[str, tuple[str, ...]] = {
+ "on": ("打开", "开启", "启动", "开"),
+ "off": ("关闭", "关掉", "关上", "停止", "关"),
+ "set": (
+ "调整到",
+ "调整为",
+ "调节到",
+ "调节为",
+ "设置到",
+ "设置为",
+ "调到",
+ "调成",
+ "变成",
+ "设为",
+ "开到",
+ ),
+ "forward": ("向前", "往前", "朝前", "前移", "前倾"),
+ "backward": ("向后", "往后", "朝后", "后移", "后仰"),
+ "up": ("向上", "往上", "升高", "抬高", "抬起"),
+ "down": ("向下", "往下", "降低", "降下"),
+ "adjust": ("调整", "调节", "移动", "调", "移"),
+}
+
+CORRECTION_MARKERS: tuple[str, ...] = (
+ "重新说",
+ "说错了",
+ "准确地说",
+ "应该是",
+ "我是说",
+ "更正为",
+ "更正成",
+ "不对",
+ "口误",
+ "改成",
+ "改为",
+ "重来",
+)
+
+CANCELLATION_MARKERS: tuple[str, ...] = ("算了", "取消", "作废")
+POSITIVE_CONNECTORS: tuple[str, ...] = (
+ "然后",
+ "同时",
+ "并且",
+ "接着",
+ "随后",
+ "再",
+ "并",
+)
+NEGATION_PREFIXES: tuple[str, ...] = ("不要", "别", "不想", "无需", "不必", "禁止")
+UNITS: tuple[str, ...] = ("摄氏度", "百分比", "毫米", "厘米", "毫秒", "分钟", "度", "%", "档", "秒")
+
+COLOR_ALIASES: dict[str, tuple[str, ...]] = {
+ "#FF0000": ("红色", "红灯", "正红"),
+ "#00FF00": ("绿色", "绿灯"),
+ "#0000FF": ("蓝色", "蓝灯"),
+ "#FFFFFF": ("白色", "白灯"),
+ "#FFFF00": ("黄色", "黄灯"),
+ "#FFA500": ("橙色", "橘色"),
+ "#800080": ("紫色", "紫灯"),
+}
+
+
+def _alias_pattern(values: Iterable[str]) -> re.Pattern[str]:
+ ordered = sorted(set(values), key=len, reverse=True)
+ return re.compile("|".join(re.escape(value) for value in ordered))
+
+
+DEVICE_ALIAS_PATTERN = _alias_pattern(
+ alias for term in DEVICE_TERMS for alias in term.aliases
+)
+ACTION_ALIAS_PATTERN = _alias_pattern(
+ alias for aliases in ACTION_ALIASES.values() for alias in aliases
+)
+SCOPE_ALIAS_PATTERN = _alias_pattern(
+ alias for aliases in SCOPE_ALIASES.values() for alias in aliases
+)
+
+
+def device_terms_in(text: str) -> list[tuple[int, int, DeviceTerm, str]]:
+ """Return longest non-overlapping device terms in source order."""
+
+ alias_map = {
+ alias: term
+ for term in DEVICE_TERMS
+ for alias in term.aliases
+ }
+ return [
+ (match.start(), match.end(), alias_map[match.group(0)], match.group(0))
+ for match in DEVICE_ALIAS_PATTERN.finditer(text)
+ ]
+
+
+def action_terms_in(text: str) -> list[tuple[int, int, str, str]]:
+ alias_map = {
+ alias: action
+ for action, aliases in ACTION_ALIASES.items()
+ for alias in aliases
+ }
+ device_spans = [(start, end) for start, end, _, _ in device_terms_in(text)]
+ matches: list[tuple[int, int, str, str]] = []
+ for match in ACTION_ALIAS_PATTERN.finditer(text):
+ # Single-character actions such as ``调`` are useful in ``往后调``, but
+ # must not turn the ``调`` inside ``空调`` into a later action callback.
+ starts_inside_device = any(
+ start <= match.start() < end for start, end in device_spans
+ )
+ if starts_inside_device:
+ continue
+ matches.append(
+ (match.start(), match.end(), alias_map[match.group(0)], match.group(0))
+ )
+ return matches
+
+
+def scope_in(text: str, *, default: str, domain: str | None = None) -> str:
+ matches = [
+ (match.start(), scope)
+ for scope, aliases in SCOPE_ALIASES.items()
+ for alias in aliases
+ for match in re.finditer(re.escape(alias), text)
+ ]
+ scope = min(matches)[1] if matches else default
+ if domain == "window":
+ return {
+ "driver": "front_left",
+ "passenger": "front_right",
+ "front_row": "front",
+ "rear_row": "rear",
+ }.get(scope, scope)
+ return scope
+
+
+def color_in(text: str) -> tuple[str, str] | None:
+ """Return a canonical RGB value and the exact spoken color alias."""
+
+ matches = [
+ (match.start(), color, alias)
+ for color, aliases in COLOR_ALIASES.items()
+ for alias in aliases
+ for match in re.finditer(re.escape(alias), text)
+ ]
+ if not matches:
+ return None
+ _, color, alias = min(matches)
+ return color, alias
diff --git a/src/cognitive_agent/server.py b/src/cognitive_agent/server.py
index e454157..770c5ab 100755
--- a/src/cognitive_agent/server.py
+++ b/src/cognitive_agent/server.py
@@ -12,28 +12,28 @@ import re
import socket
import threading
import time
-from http.cookies import CookieError, SimpleCookie
from http import HTTPStatus
+from http.cookies import CookieError, SimpleCookie
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import ParseResult, parse_qs, quote, unquote, urlparse
from cognitive_agent.config_loader import load_local_env
-
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
# Repository-local launch configuration is deliberate and must not be shadowed
# by stale variables inherited from an IDE or an earlier model experiment.
load_local_env(REPOSITORY_ROOT / ".env", override=True)
load_local_env(REPOSITORY_ROOT / ".env.auth", override=True)
-from cognitive_agent.auth import AuthConfig, AuthRuntime, verify_password # noqa: E402
-from cognitive_agent.application.seat_service import SeatDemoService # noqa: E402
-from cognitive_agent.application.gesture_bridge import GestureCapabilityBridge # noqa: E402
from cognitive_agent.application.calibration.service import ( # noqa: E402
ApiError as CalibrationApiError,
+)
+from cognitive_agent.application.calibration.service import (
CalibrationService,
)
+from cognitive_agent.application.gesture_bridge import GestureCapabilityBridge # noqa: E402
+from cognitive_agent.application.seat_service import SeatDemoService # noqa: E402
from cognitive_agent.application.self_extension import ( # noqa: E402
ExtensionIntegrationError,
GatewayError,
@@ -42,16 +42,20 @@ from cognitive_agent.application.self_extension import ( # noqa: E402
resolve_active_peripheral_command,
)
from cognitive_agent.application.self_repair import SelfRepairService # noqa: E402
+from cognitive_agent.auth import AuthConfig, AuthRuntime, verify_password # noqa: E402
+from cognitive_agent.infrastructure.can.access import ( # noqa: E402
+ HardwareAccessRequiredError,
+)
from cognitive_agent.infrastructure.can.hardware import ( # noqa: E402
HardwareBusyError,
HardwareSafetyError,
)
-from cognitive_agent.infrastructure.can.access import ( # noqa: E402
- HardwareAccessRequiredError,
-)
from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError # noqa: E402
from cognitive_agent.infrastructure.nfc import NfcReaderService # noqa: E402
-
+from cognitive_agent.infrastructure.speech import ( # noqa: E402
+ SpeechRecognitionError,
+ SpeechTranscriber,
+)
STATIC_ROOT = REPOSITORY_ROOT / "apps" / "web-console"
mimetypes.add_type("application/manifest+json", ".webmanifest")
@@ -176,6 +180,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
repository_root=REPOSITORY_ROOT,
)
repair = SelfRepairService()
+ speech = SpeechTranscriber(REPOSITORY_ROOT)
auth = AuthRuntime(AuthConfig.from_env())
vehicle_display_sync = VehicleDisplaySync()
server_version = "CognitiveAgentCabinDemo/2.0"
@@ -342,6 +347,9 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
return
self._json({"error": "not_found"}, HTTPStatus.NOT_FOUND)
return
+ if path == "/api/speech/status":
+ self._json(self.speech.status())
+ return
if path.startswith("/api/requests/"):
self._serve_request(path.removeprefix("/api/requests/"))
return
@@ -357,6 +365,9 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
return
if not self._authorize(path):
return
+ if path == "/api/speech/transcribe":
+ self._handle_speech_transcription()
+ return
if path in {
"/api/hardware/access/request",
"/api/hardware/access/release",
@@ -482,6 +493,32 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
payload.get("control_transport", "wired_can")
),
}
+ speech_utterance_id = str(
+ payload.get("speech_utterance_id", "")
+ ).strip()
+ if speech_utterance_id:
+ if path != "/api/requests":
+ raise ValueError(
+ "speech semantic execution is only accepted by /api/requests"
+ )
+ speech_understanding = self.speech.cached_result(
+ speech_utterance_id
+ )
+ if str(speech_understanding.get("clean_text", "")).strip() != utterance:
+ raise SpeechRecognitionError(
+ "speech_text_changed",
+ "语音文本已被修改，请确认修改后的内容再执行。",
+ status=409,
+ )
+ if speech_understanding.get("execution_eligible") is not True:
+ raise SpeechRecognitionError(
+ "speech_not_executable",
+ "语音意图尚未通过能力或参数校验，已拒绝执行。",
+ status=422,
+ )
+ provider = "local"
+ common_arguments["provider"] = provider
+ common_arguments["_speech_understanding"] = speech_understanding
if (
path != "/api/can/commands"
and "application/x-ndjson" in self.headers.get("Accept", "")
@@ -596,6 +633,12 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
{"error": "extension_request_ambiguous", "message": str(exc)},
HTTPStatus.UNPROCESSABLE_ENTITY,
)
+ except SpeechRecognitionError as exc:
+ fail_display_sync(exc.code, exc)
+ self._json(
+ {"error": exc.code, "message": str(exc)},
+ HTTPStatus(exc.status),
+ )
except (ValueError, json.JSONDecodeError) as exc:
fail_display_sync("invalid_request", exc)
self._json({"error": "invalid_request", "message": str(exc)}, HTTPStatus.BAD_REQUEST)
@@ -1340,6 +1383,41 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
raise ValueError("request body must be a JSON object")
return payload
+ def _handle_speech_transcription(self) -> None:
+ content_type = self.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
+ if content_type not in {"audio/wav", "audio/wave", "audio/x-wav"}:
+ self._json(
+ {
+ "error": "unsupported_media_type",
+ "message": "语音听写接口仅接受 16 位 PCM WAV 音频。",
+ },
+ HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
+ )
+ return
+ try:
+ content_length = int(self.headers.get("Content-Length", "0"))
+ if content_length <= 0 or content_length > 2 * 1024 * 1024:
+ raise SpeechRecognitionError(
+ "invalid_audio_size",
+ "录音数据为空或超过大小限制。",
+ status=413,
+ )
+ result = self.speech.transcribe_wav(
+ self.rfile.read(content_length),
+ utterance_id=self.headers.get("X-Speech-Utterance-Id"),
+ )
+ self._json(result)
+ except SpeechRecognitionError as exc:
+ self._json(
+ {"error": exc.code, "message": str(exc)},
+ HTTPStatus(exc.status),
+ )
+ except (TypeError, ValueError):
+ self._json(
+ {"error": "invalid_audio", "message": "录音请求格式无效。"},
+ HTTPStatus.BAD_REQUEST,
+ )
+
def _authorize(self, path: str) -> bool:
if not self.auth.config.enabled or path in PUBLIC_AUTH_PATHS:
return True
@@ -1612,7 +1690,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
self.send_header("Referrer-Policy", "no-referrer")
self.send_header(
"Permissions-Policy",
- "camera=(self), microphone=(), geolocation=(), payment=(), usb=()",
+ "camera=(self), microphone=(self), geolocation=(), payment=(), usb=()",
)
self.send_header(
"Content-Security-Policy",
diff --git a/tests/e2e/test_seat_web_api.py b/tests/e2e/test_seat_web_api.py
index bc708ea..6861fb9 100755
--- a/tests/e2e/test_seat_web_api.py
+++ b/tests/e2e/test_seat_web_api.py
@@ -13,15 +13,18 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
-from cognitive_agent.auth import AuthConfig, AuthRuntime, hash_password
-from cognitive_agent.application.seat_service import REPOSITORY_ROOT, SeatDemoService
from cognitive_agent.application.mode_profile import ModeProfileRegistry
+from cognitive_agent.application.seat_service import REPOSITORY_ROOT, SeatDemoService
from cognitive_agent.application.self_extension import (
SelfExtensionService,
SimulationGateway,
)
-from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError
+from cognitive_agent.auth import AuthConfig, AuthRuntime, hash_password
from cognitive_agent.infrastructure.can.access import HardwareAccessController
+from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError
+from cognitive_agent.infrastructure.speech.understanding import (
+ UnderstandingTranscriptProcessor,
+)
from cognitive_agent.server import SeatDemoHandler
@@ -332,7 +335,7 @@ operations:
def test_static_console_and_runtime_endpoint(self) -> None:
with self.opener.open(f"{self.base_url}/", timeout=3) as response:
html = response.read().decode("utf-8")
- with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-103", timeout=3) as response:
+ with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-104", timeout=3) as response:
app_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/gesture-control.js?v=android-pad-9", timeout=3
@@ -346,6 +349,18 @@ operations:
f"{self.base_url}/console-v2.css?v=sync-layout-41", timeout=3
) as response:
console_styles = response.read().decode("utf-8")
+ with self.opener.open(
+ f"{self.base_url}/speech-input.js?v=sensevoice-6", timeout=3
+ ) as response:
+ speech_script = response.read().decode("utf-8")
+ with self.opener.open(
+ f"{self.base_url}/speech-input.css?v=sensevoice-4", timeout=3
+ ) as response:
+ speech_styles = response.read().decode("utf-8")
+ with self.opener.open(
+ f"{self.base_url}/speech-audio-worklet.js?v=sensevoice-4", timeout=3
+ ) as response:
+ speech_worklet = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/self-extension.js?v=nfc-toy-33", timeout=3
) as response:
@@ -368,6 +383,8 @@ operations:
nfc_status = json.loads(response.read().decode("utf-8"))
with self.opener.open(f"{self.base_url}/api/runtime", timeout=3) as response:
runtime = json.loads(response.read().decode("utf-8"))
+ with self.opener.open(f"{self.base_url}/api/speech/status", timeout=3) as response:
+ speech_status = json.loads(response.read().decode("utf-8"))
self.assertIn("511工程-智能座舱", html)
self.assertIn('id="seatTwinCanvas"', html)
@@ -382,12 +399,16 @@ operations:
self.assertIn("CMD 0x431 · FB 0x41A · CH1", html)
self.assertIn("CMD 0x431 · FB 0x388 · CH1", html)
self.assertNotIn("透明车体 · 实体设备", html)
- self.assertIn('src="/app.js?v=dual-seat-103"', html)
+ self.assertIn('src="/app.js?v=dual-seat-104"', html)
self.assertIn('href="/styles.css?v=dual-seat-16"', html)
self.assertIn('src="/gesture-control.js?v=android-pad-9"', html)
self.assertNotIn('class="vehicle-system-hud"', html)
self.assertIn('src="/console-v2.js?v=sync-layout-43"', html)
self.assertIn('href="/console-v2.css?v=sync-layout-41"', html)
+ self.assertIn('src="/speech-input.js?v=sensevoice-6"', html)
+ self.assertIn('href="/speech-input.css?v=sensevoice-4"', html)
+ self.assertIn('id="speechDictationButton"', html)
+ self.assertIn('class="speech-waveform"', html)
self.assertIn('', html)
self.assertIn('id="v2BootScreen"', html)
self.assertIn('正在加载车机控制与实时状态', html)
@@ -622,7 +643,29 @@ operations:
self.assertIn('client_type: "web_console"', app_script)
self.assertIn('canvas.addEventListener("webglcontextlost"', app_script)
self.assertIn('"cognitive-agent:module-change"', app_script)
- self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v210"', service_worker)
+ self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v216"', service_worker)
+ self.assertIn('"/speech-input.js?v=sensevoice-6"', service_worker)
+ self.assertIn('"/speech-input.css?v=sensevoice-4"', service_worker)
+ self.assertIn('"/speech-audio-worklet.js?v=sensevoice-4"', service_worker)
+ self.assertIn('fetch("/api/speech/transcribe"', speech_script)
+ self.assertIn('"X-Speech-Utterance-Id"', speech_script)
+ self.assertIn("result.commands", speech_script)
+ self.assertIn("AudioWorkletNode", speech_script)
+ self.assertIn('registerProcessor("speech-capture-processor"', speech_worklet)
+ self.assertIn("navigator.mediaDevices.getUserMedia", speech_script)
+ self.assertIn("PREFERRED_MICROPHONE", speech_script)
+ self.assertIn("ENDPOINT_SILENCE_MS", speech_script)
+ self.assertIn("speechExecutionEligible", speech_script)
+ self.assertIn("speechSemanticStatus", speech_script)
+ self.assertIn('"recognized_unsupported"', speech_script)
+ self.assertIn('"incomplete"', speech_script)
+ self.assertIn("speechExecutionEligible", app_script)
+ self.assertIn("speech_utterance_id", app_script)
+ self.assertIn("speechUtteranceId", speech_script)
+ self.assertIn("语音意图已识别，但当前车辆能力不支持执行", app_script)
+ self.assertIn("语音控制参数不完整，请补充后再执行", app_script)
+ self.assertIn(".speech-dictation.is-listening", speech_styles)
+ self.assertEqual(speech_status["model"], "SenseVoiceSmall")
self.assertIn('"/assets/pets/self-extension-cat.png?v=3"', service_worker)
self.assertIn(".then(() => self.skipWaiting())", service_worker)
self.assertNotIn("const cached = await caches.match(request)", service_worker)
@@ -649,6 +692,7 @@ operations:
self.assertIn("BENCH BUSY", app_script)
self.assertIn("FBXLoader.js", html)
self.assertIn('data-seat-layout="five-seat"', html)
+
self.assertIn('data-active-seat="front-right"', html)
self.assertNotIn('placeholder="请输入服务端配置令牌"', html)
self.assertIn('data-callout="backrest_recline"', html)
@@ -856,13 +900,164 @@ operations:
)
)
+ def test_speech_transcription_endpoint_forwards_wav_without_persisting_it(self) -> None:
+ class SpeechStub:
+ received = b""
+
+ @classmethod
+ def transcribe_wav(
+ cls, content: bytes, *, utterance_id: str | None = None
+ ) -> dict[str, object]:
+ cls.received = content
+ return {
+ "utterance_id": utterance_id,
+ "text": "打开空调到二十四度。",
+ "language": "zh",
+ "duration_ms": 800,
+ "inference_ms": 92,
+ "model": "SenseVoiceSmall",
+ }
+
+ original_speech = SeatDemoHandler.speech
+ SeatDemoHandler.speech = SpeechStub()
+ try:
+ wav_content = b"RIFF-speech-test"
+ request = urllib.request.Request(
+ f"{self.base_url}/api/speech/transcribe",
+ method="POST",
+ headers={
+ "Content-Type": "audio/wav",
+ "Accept": "application/json",
+ "X-Speech-Utterance-Id": "e2e-speech-1",
+ },
+ data=wav_content,
+ )
+ with self.opener.open(request, timeout=3) as response:
+ payload = json.loads(response.read().decode("utf-8"))
+ permissions_policy = response.headers["Permissions-Policy"]
+ finally:
+ SeatDemoHandler.speech = original_speech
+
+ self.assertEqual(SpeechStub.received, wav_content)
+ self.assertEqual(payload["text"], "打开空调到二十四度。")
+ self.assertEqual(payload["utterance_id"], "e2e-speech-1")
+ self.assertIn("microphone=(self)", permissions_policy)
+
+ def test_speech_utterance_id_preserves_structured_slots_into_intent_ir(self) -> None:
+ understanding = UnderstandingTranscriptProcessor(REPOSITORY_ROOT).process(
+ "把副驾腿托向上调整到最大位置"
+ ).to_dict()
+ understanding["utterance_id"] = "e2e-speech-bridge"
+
+ class SpeechStub:
+ @staticmethod
+ def cached_result(utterance_id: str) -> dict[str, object]:
+ if utterance_id != "e2e-speech-bridge":
+ raise AssertionError("unexpected speech utterance id")
+ return understanding
+
+ original_speech = SeatDemoHandler.speech
+ SeatDemoHandler.speech = SpeechStub()
+ try:
+ request = urllib.request.Request(
+ f"{self.base_url}/api/requests",
+ method="POST",
+ headers={"Content-Type": "application/json", "Accept": "application/json"},
+ data=json.dumps(
+ {
+ "utterance": understanding["clean_text"],
+ "speech_utterance_id": "e2e-speech-bridge",
+ "client_type": "web_console",
+ "provider": "openai",
+ "conversation_id": "conv-e2e-speech-bridge",
+ "execution_mode": "dry_run",
+ },
+ ensure_ascii=False,
+ ).encode("utf-8"),
+ )
+ with self.opener.open(request, timeout=3) as response:
+ payload = json.loads(response.read().decode("utf-8"))
+ finally:
+ SeatDemoHandler.speech = original_speech
+
+ self.assertEqual(payload["status"], "succeeded")
+ self.assertEqual(payload["provider_used"], "speech_semantic")
+ self.assertEqual(
+ payload["speech_understanding"]["raw_text"],
+ "把副驾腿托向上调整到最大位置",
+ )
+ action = payload["intent_ir"]["spec"]["actions"][0]
+ self.assertEqual(action["capability"], "seat.set_legrest_height")
+ self.assertEqual(action["target"]["instance"], "passenger")
+ self.assertEqual(action["parameters"], {"position_pct": 100})
+ self.assertEqual(action["provenance"]["parser"], "speech_semantic")
+
+ def test_roof_shade_speech_crosses_server_bridge_as_native_can(self) -> None:
+ cases = (
+ ("把遮阳帘打开", {"motion": "open"}),
+ ("遮阳帘打开一半", {"motion": "half"}),
+ )
+ for index, (utterance, parameters) in enumerate(cases):
+ with self.subTest(utterance=utterance):
+ utterance_id = f"e2e-roof-shade-{index}"
+ understanding = UnderstandingTranscriptProcessor(
+ REPOSITORY_ROOT
+ ).process(utterance).to_dict()
+ understanding["utterance_id"] = utterance_id
+
+ def cached_result(
+ requested_id: str,
+ *,
+ expected_id: str = utterance_id,
+ result: dict[str, object] = understanding,
+ ) -> dict[str, object]:
+ if requested_id != expected_id:
+ raise AssertionError("unexpected speech utterance id")
+ return result
+
+ original_speech = SeatDemoHandler.speech
+ SeatDemoHandler.speech = SimpleNamespace(cached_result=cached_result)
+ try:
+ request = urllib.request.Request(
+ f"{self.base_url}/api/requests",
+ method="POST",
+ headers={
+ "Content-Type": "application/json",
+ "Accept": "application/json",
+ },
+ data=json.dumps(
+ {
+ "utterance": understanding["clean_text"],
+ "speech_utterance_id": utterance_id,
+ "client_type": "web_console",
+ "provider": "openai",
+ "conversation_id": f"conv-e2e-roof-shade-{index}",
+ "execution_mode": "dry_run",
+ },
+ ensure_ascii=False,
+ ).encode("utf-8"),
+ )
+ with self.opener.open(request, timeout=3) as response:
+ payload = json.loads(response.read().decode("utf-8"))
+ finally:
+ SeatDemoHandler.speech = original_speech
+
+ self.assertEqual(payload["status"], "succeeded")
+ self.assertEqual(payload["provider_used"], "speech_semantic")
+ self.assertEqual(payload["deferred_requests"], [])
+ action = payload["intent_ir"]["spec"]["actions"][0]
+ self.assertEqual(action["capability"], "roof.set_shade_motion")
+ self.assertEqual(action["target"]["zone"], "vehicle")
+ self.assertEqual(action["parameters"], parameters)
+ self.assertEqual(action["provenance"]["parser"], "speech_semantic")
+
def test_self_extension_vehicle_success_contract_is_strict(self) -> None:
with self.opener.open(
f"{self.base_url}/self-extension.js?v=extension-44", timeout=3
) as response:
extension_script = response.read().decode("utf-8")
with self.opener.open(
- f"{self.base_url}/app.js?v=dual-seat-103", timeout=3
+ f"{self.base_url}/app.js?v=dual-seat-104", timeout=3
) as response:
app_script = response.read().decode("utf-8")
diff --git a/tests/fixtures/speech_text_regression.json b/tests/fixtures/speech_text_regression.json
new file mode 100644
index 0000000..d0822a5
--- /dev/null
+++ b/tests/fixtures/speech_text_regression.json
@@ -0,0 +1,50 @@
+[
+ {"name":"single_power_on","raw":"打开空调。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":false},
+ {"name":"single_power_off","raw":"关闭空调。","clean":"关闭空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":false},
+ {"name":"single_temperature","raw":"把空调调到24度。","clean":"将空调设置为24度。","capabilities":["climate.set_temperature"],"cancelled":0,"uncertain":false},
+ {"name":"power_and_temperature","raw":"打开空调到24度。","clean":"打开空调并设置为24度。","capabilities":["climate.set_power","climate.set_temperature"],"cancelled":0,"uncertain":false},
+ {"name":"fan_level","raw":"把空调风量调到2档。","clean":"将空调风量设置为2档。","capabilities":["climate.set_fan_level"],"cancelled":0,"uncertain":false},
+ {"name":"backrest_relative","raw":"把座椅靠背向后调24度。","clean":"将座椅靠背向后调整24度。","capabilities":["seat.set_backrest_recline"],"cancelled":0,"uncertain":false},
+ {"name":"window_open","raw":"打开车窗。","clean":"打开车窗。","capabilities":["window.set_motion"],"cancelled":0,"uncertain":false},
+ {"name":"ambient_on","raw":"打开氛围灯。","clean":"打开氛围灯。","capabilities":["ambient_light.set_power"],"cancelled":0,"uncertain":false},
+ {"name":"seat_ventilation","raw":"打开座椅通风。","clean":"打开座椅通风。","capabilities":["seat.set_driver_ventilation_native"],"cancelled":0,"uncertain":false},
+ {"name":"two_commands","raw":"打开空调，然后打开氛围灯。","clean":"打开空调，然后打开氛围灯。","capabilities":["climate.set_power","ambient_light.set_power"],"cancelled":0,"uncertain":false},
+ {"name":"same_domain_scopes","raw":"打开前排空调，再打开后排空调。","clean":"打开前排空调，然后打开后排空调。","capabilities":["climate.set_power","climate.set_power"],"cancelled":0,"uncertain":false},
+ {"name":"fillers","raw":"呃，那个那个那个打开空调到24度。","clean":"打开空调并设置为24度。","capabilities":["climate.set_power","climate.set_temperature"],"cancelled":0,"uncertain":false},
+ {"name":"referential_demonstrative","raw":"打开那个车窗。","clean":"打开车窗。","capabilities":["window.set_motion"],"cancelled":0,"uncertain":false,"preserved_filler":"那个"},
+ {"name":"action_stutter","raw":"打打打开空调。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":false},
+ {"name":"target_echo","raw":"把座椅的靠背靠座椅靠背向后调24度。","clean":"将座椅靠背向后调整24度。","capabilities":["seat.set_backrest_recline"],"cancelled":0,"uncertain":false},
+ {"name":"target_refinement","raw":"座椅调整到呃20呃靠背调整到24度。","clean":"将座椅靠背调整到24度。","capabilities":["seat.set_backrest_recline"],"cancelled":1,"uncertain":true},
+ {"name":"domain_correction","raw":"打开空调，不对，把座椅靠背向后调24度。","clean":"将座椅靠背向后调整24度。","capabilities":["seat.set_backrest_recline"],"cancelled":1,"uncertain":false},
+ {"name":"number_correction","raw":"空调调到24，不对，25度。","clean":"将空调设置为25度。","capabilities":["climate.set_temperature"],"cancelled":1,"uncertain":false},
+ {"name":"direction_correction","raw":"把座椅往前，不对，往后一点。","clean":"将座椅向后调整一点。","capabilities":["seat.set_driver_axis_native"],"cancelled":1,"uncertain":false},
+ {"name":"target_correction","raw":"把座椅，不对，把靠背往后调一点。","clean":"将座椅靠背向后调整一点。","capabilities":["seat.set_driver_axis_native"],"cancelled":1,"uncertain":false},
+ {"name":"action_correction","raw":"把空调关掉，不，是打开。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":1,"uncertain":false},
+ {"name":"multi_not_correction","raw":"空调调到24度，然后把座椅靠背调到20度。","clean":"将空调设置为24度，然后将座椅靠背调整到20度。","capabilities":["climate.set_temperature","seat.set_backrest_recline"],"cancelled":0,"uncertain":true},
+ {"name":"cancel_incomplete","raw":"把空调调……算了。","clean":"","capabilities":[],"cancelled":1,"uncertain":false},
+ {"name":"negated_open","raw":"不要打开空调。","clean":"不要打开空调。","capabilities":[],"cancelled":1,"uncertain":false},
+ {"name":"negated_close","raw":"不要关闭空调。","clean":"不要关闭空调。","capabilities":[],"cancelled":1,"uncertain":false},
+ {"name":"filler_cancel","raw":"嗯嗯，那个……算了。","clean":"","capabilities":[],"cancelled":1,"uncertain":false},
+ {"name":"unrelated_chat","raw":"今天天气还不错。","clean":"","capabilities":[],"cancelled":0,"uncertain":false},
+ {"name":"unrelated_question","raw":"我们中午吃什么？","clean":"","capabilities":[],"cancelled":0,"uncertain":false},
+ {"name":"unknown_device","raw":"打开雨刷。","clean":"","capabilities":[],"cancelled":0,"uncertain":true},
+ {"name":"incomplete_known_target","raw":"空调挺凉快。","clean":"","capabilities":[],"cancelled":0,"uncertain":true},
+ {"name":"decimal_temperature","raw":"把空调调到24.5度。","clean":"将空调设置为24.5度。","capabilities":["climate.set_temperature"],"cancelled":0,"uncertain":false},
+ {"name":"chinese_number","raw":"把空调调到二十四度。","clean":"将空调设置为24度。","capabilities":["climate.set_temperature"],"cancelled":0,"uncertain":false},
+ {"name":"lost_marker_false_start","raw":"空调打开打，把座椅的靠背靠座椅靠背向后调24度。","clean":"将座椅靠背向后调整24度。","capabilities":["seat.set_backrest_recline"],"cancelled":1,"uncertain":false},
+ {"name":"polite_scaffold","raw":"请帮我打开空调，谢谢。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":false},
+ {"name":"front_light_unsupported","raw":"打开前大灯","clean":"打开前大灯。","capabilities":[null],"cancelled":0,"uncertain":true,"execution_eligible":false},
+ {"name":"reading_ambient_color_compound","raw":"打开阅读灯，并且打开氛围灯，并把氛围灯变成红色","clean":"打开阅读灯，然后打开氛围灯，然后将氛围灯颜色调整为红色。","capabilities":["interior_light.set_reading_power","ambient_light.set_power","ambient_light.set_illumination"],"cancelled":0,"uncertain":true,"execution_eligible":false},
+ {"name":"window_half_open","raw":"打开一半车窗","clean":"将车窗打开到50%。","capabilities":["window.set_position"],"cancelled":0,"uncertain":false,"execution_eligible":true},
+ {"name":"passenger_legrest_maximum","raw":"把副驾腿托向上调整到最大位置","clean":"将副驾座椅腿托向上调整到100%。","capabilities":["seat.set_legrest_height"],"cancelled":0,"uncertain":false,"execution_eligible":true},
+ {"name":"driver_zero_gravity_unsupported","raw":"把主驾座椅调整为零重力模式","clean":"将主驾座椅调整为零重力模式。","capabilities":[null],"cancelled":0,"uncertain":true,"execution_eligible":false},
+ {"name":"shade_then_invalid_seat_degree","raw":"打开遮阳帘，然后把副驾座椅向后调整50度","clean":"打开遮阳帘，然后把副驾座椅向后调整50度。","capabilities":["roof.set_shade_motion","seat.set_longitudinal_position"],"parameter_numbers":[],"cancelled":0,"uncertain":true,"execution_eligible":false},
+ {"name":"window_quarter_open","raw":"打开四分之一车窗","clean":"将车窗打开到25%。","capabilities":["window.set_position"],"cancelled":0,"uncertain":false,"execution_eligible":true},
+ {"name":"passenger_thermal_pair_and_backrest_max","raw":"打开副驾座椅通风和加热，然后将副驾座椅靠背调整到最大位置","clean":"打开副驾座椅通风，然后打开副驾座椅加热，然后将副驾座椅靠背调整到100%。","capabilities":["seat.set_ventilation","seat.set_heating","seat.set_backrest_recline"],"cancelled":0,"uncertain":false,"execution_eligible":true},
+ {"name":"driver_thermal_pair_and_temperature","raw":"打开主驾座椅通风和加热，并且把空调调整为24度","clean":"打开主驾座椅通风，然后打开主驾座椅加热，然后将空调设置为24度。","capabilities":["seat.set_driver_ventilation_native","seat.set_driver_heating_native","climate.set_temperature"],"cancelled":0,"uncertain":false,"execution_eligible":true},
+ {"name":"thermal_reverse_order_connector","raw":"打开副驾座椅加热以及通风","clean":"打开副驾座椅加热，然后打开副驾座椅通风。","capabilities":["seat.set_heating","seat.set_ventilation"],"cancelled":0,"uncertain":false,"execution_eligible":true},
+ {"name":"window_three_quarters","raw":"打开四分之三车窗","clean":"将车窗打开到75%。","capabilities":["window.set_position"],"cancelled":0,"uncertain":false,"execution_eligible":true},
+ {"name":"shade_quarter_position","raw":"把遮阳帘打开到四分之一","clean":"将遮阳帘打开到25%。","capabilities":["roof.set_shade_position"],"cancelled":0,"uncertain":false,"execution_eligible":true},
+ {"name":"shade_open_bridge_regression","raw":"把遮阳帘打开","clean":"打开遮阳帘。","capabilities":["roof.set_shade_motion"],"cancelled":0,"uncertain":false,"execution_eligible":true},
+ {"name":"shade_half_open_bridge_regression","raw":"遮阳帘打开一半","clean":"将遮阳帘打开一半。","capabilities":["roof.set_shade_motion"],"cancelled":0,"uncertain":false,"execution_eligible":true}
+]
diff --git a/tests/unit/test_speech_capability_alignment.py b/tests/unit/test_speech_capability_alignment.py
new file mode 100644
index 0000000..5ffe655
--- /dev/null
+++ b/tests/unit/test_speech_capability_alignment.py
@@ -0,0 +1,525 @@
+from __future__ import annotations
+
+import unittest
+from pathlib import Path
+from typing import Any
+
+from cognitive_agent.application.native_can_intent import merge_native_can_intent
+from cognitive_agent.application.native_can_runtime import NativeCanRuntime
+from cognitive_agent.infrastructure.speech.capability import CapabilityResolver
+from cognitive_agent.infrastructure.speech.understanding import (
+ UnderstandingTranscriptProcessor,
+)
+from cognitive_agent.infrastructure.speech.vocabulary import DeviceTerm
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
+
+
+def _valid_value(spec: dict[str, Any]) -> Any:
+ allowed = spec.get("allowed_values")
+ if allowed:
+ return allowed[0]
+ value_type = str(spec.get("type", "string"))
+ if value_type in {"integer", "number"}:
+ minimum = spec.get("minimum", 0)
+ return int(minimum) if value_type == "integer" else float(minimum)
+ if value_type == "boolean":
+ return True
+ if value_type == "object":
+ return {}
+ if value_type == "array":
+ return []
+ if str(spec.get("name")) == "color_hex":
+ return "#FF0000"
+ if str(spec.get("name")) == "preset_name":
+ return "回归测试姿态"
+ return "test"
+
+
+def _sample_parameters(definition: Any) -> dict[str, Any]:
+ specs = list(definition.parameters)
+ selected = [item for item in specs if item.get("required")]
+ if not selected and specs:
+ selected = [specs[0]]
+ return {str(item["name"]): _valid_value(item) for item in selected}
+
+
+class SpeechCapabilityAlignmentTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
+ cls.resolver = CapabilityResolver(REPOSITORY_ROOT)
+ cls.native_runtime = NativeCanRuntime(REPOSITORY_ROOT)
+
+ def test_eight_reported_cases_preserve_structure_and_resolution(self) -> None:
+ cases = (
+ {
+ "text": "把遮阳帘打开",
+ "status": "recognized_executable",
+ "clean": "打开遮阳帘。",
+ "commands": (
+ (
+ "roof.set_shade_motion",
+ "vehicle",
+ "roof.shade_motion",
+ "open",
+ {"motion": "open"},
+ ),
+ ),
+ },
+ {
+ "text": "打开前大灯",
+ "status": "recognized_unsupported",
+ "clean": "打开前大灯。",
+ "commands": ((None, "vehicle", "front_light.power", "on", {"state": "on"}),),
+ },
+ {
+ "text": "打开主驾座椅通风",
+ "status": "recognized_executable",
+ "clean": "打开主驾座椅通风。",
+ "commands": (
+ (
+ "seat.set_driver_ventilation_native",
+ "driver",
+ "seat.ventilation",
+ "on",
+ {"state": "on", "level": 2},
+ ),
+ ),
+ },
+ {
+ "text": "把主驾座椅调整为零重力模式",
+ "status": "recognized_unsupported",
+ "clean": "将主驾座椅调整为零重力模式。",
+ "commands": (
+ (None, "driver", "seat.mode", "apply", {"mode_profile_id": "zero_gravity"}),
+ ),
+ },
+ {
+ "text": "打开一半车窗",
+ "status": "recognized_executable",
+ "clean": "将车窗打开到50%。",
+ "commands": (
+ (
+ "window.set_position",
+ "front_left",
+ "window.position",
+ "set",
+ {"position_pct": 50},
+ ),
+ ),
+ },
+ {
+ "text": "打开氛围灯并调整为红色",
+ "status": "incomplete",
+ "clean": "打开氛围灯，然后将氛围灯颜色调整为红色。",
+ "commands": (
+ ("ambient_light.set_power", "vehicle", "ambient.power", "on", {"state": "on"}),
+ (
+ "ambient_light.set_illumination",
+ "vehicle",
+ "ambient.color",
+ "set",
+ {"color_hex": "#FF0000"},
+ ),
+ ),
+ },
+ {
+ "text": "打开主驾座椅加热",
+ "status": "recognized_executable",
+ "clean": "打开主驾座椅加热。",
+ "commands": (
+ (
+ "seat.set_driver_heating_native",
+ "driver",
+ "seat.heating",
+ "on",
+ {"state": "on", "level": 2},
+ ),
+ ),
+ },
+ {
+ "text": "打开副驾座椅通风",
+ "status": "recognized_executable",
+ "clean": "打开副驾座椅通风。",
+ "commands": (
+ (
+ "seat.set_ventilation",
+ "passenger",
+ "seat.ventilation",
+ "on",
+ {"state": "on"},
+ ),
+ ),
+ },
+ )
+ for case in cases:
+ with self.subTest(text=case["text"]):
+ result = self.processor.process(case["text"])
+ self.assertEqual(result.raw_text, case["text"])
+ self.assertEqual(result.clean_text, case["clean"])
+ self.assertEqual(result.semantic_status, case["status"])
+ self.assertEqual(len(result.commands), len(case["commands"]))
+ for command, expected in zip(result.commands, case["commands"], strict=True):
+ capability, target, property_name, operation, parameters = expected
+ self.assertEqual(command.capability, capability)
+ self.assertEqual(command.target_scope, target)
+ self.assertEqual(command.property_name, property_name)
+ self.assertEqual(command.operation, operation)
+ self.assertEqual(command.parameters, parameters)
+ if command.resolution_status == "executable":
+ self.assertIsNotNone(command.intent_ir_action)
+ self.assertIsNotNone(command.executor_payload)
+ self.assertEqual(command.intent_ir_action["target"]["zone"], target)
+ self.assertEqual(command.executor_payload["target"], target)
+ self.assertEqual(command.executor_payload["parameters"], parameters)
+ else:
+ self.assertIsNone(command.executor_payload)
+
+ def test_semantic_trace_covers_every_required_boundary(self) -> None:
+ result = self.processor.process("打开主驾座椅通风")
+ self.assertEqual(
+ [item["stage"] for item in result.semantic_trace],
+ [
+ "raw_text",
+ "normalized_text",
+ "clause_segmentation",
+ "semantic_commands",
+ "intent_ir",
+ "capability_resolution",
+ "executor_payload",
+ "clean_text",
+ ],
+ )
+
+ def test_explicit_control_slots_cannot_disappear_from_clean_text(self) -> None:
+ cases = (
+ ("打开主驾座椅加热", ("主驾", "座椅加热")),
+ ("打开副驾座椅通风", ("副驾", "座椅通风")),
+ ("打开右后车窗", ("右后", "车窗")),
+ ("打开一半车窗", ("50%", "车窗")),
+ ("把氛围灯调整为红色", ("氛围灯", "红色")),
+ ("把主驾座椅调整为零重力模式", ("主驾", "零重力模式")),
+ )
+ for raw, preserved in cases:
+ with self.subTest(raw=raw):
+ result = self.processor.process(raw)
+ for token in preserved:
+ self.assertIn(token, result.clean_text)
+
+ def test_compound_shared_target_is_preserved_and_gated_atomically(self) -> None:
+ result = self.processor.process("打开氛围灯并调整为红色")
+ self.assertEqual(
+ [command.property_name for command in result.commands],
+ ["ambient.power", "ambient.color"],
+ )
+ self.assertEqual(result.commands[1].value, "#FF0000")
+ self.assertEqual(result.commands[1].resolution_status, "incomplete")
+ self.assertIn("brightness_pct", "".join(result.commands[1].validation_issues))
+ self.assertFalse(result.execution_eligible)
+
+ complete = self.processor.process("把氛围灯调整为红色并将亮度设置为50%")
+ self.assertEqual(len(complete.commands), 1)
+ self.assertEqual(
+ complete.commands[0].parameters,
+ {"color_hex": "#FF0000", "brightness_pct": 50},
+ )
+ self.assertEqual(complete.semantic_status, "recognized_executable")
+ self.assertIn("红色", complete.clean_text)
+ self.assertIn("50%", complete.clean_text)
+
+ def test_reported_compound_and_extreme_position_phrases_preserve_all_slots(self) -> None:
+ compound = self.processor.process(
+ "打开阅读灯，并且打开氛围灯，并把氛围灯变成红色"
+ )
+ self.assertEqual(
+ [command.property_name for command in compound.commands],
+ ["reading_light.power", "ambient.power", "ambient.color"],
+ )
+ self.assertEqual(compound.commands[-1].parameters, {"color_hex": "#FF0000"})
+ self.assertEqual(compound.semantic_status, "incomplete")
+ self.assertFalse(compound.execution_eligible)
+ for token in ("阅读灯", "氛围灯", "红色"):
+ self.assertIn(token, compound.clean_text)
+
+ legrest = self.processor.process("把副驾腿托向上调整到最大位置")
+ self.assertEqual(legrest.semantic_status, "recognized_executable")
+ self.assertTrue(legrest.execution_eligible)
+ self.assertEqual(legrest.commands[0].target_scope, "passenger")
+ self.assertEqual(legrest.commands[0].capability, "seat.set_legrest_height")
+ self.assertEqual(legrest.commands[0].parameters, {"position_pct": 100})
+ for token in ("副驾", "腿托", "向上", "100%"):
+ self.assertIn(token, legrest.clean_text)
+
+ def test_latest_reported_compounds_preserve_every_mutation(self) -> None:
+ passenger = self.processor.process(
+ "打开副驾座椅通风和加热，，然后将副驾座椅靠背调整到最大位置"
+ )
+ self.assertEqual(
+ [command.property_name for command in passenger.commands],
+ ["seat.ventilation", "seat.heating", "seat.backrest_recline"],
+ )
+ self.assertEqual(
+ [command.target_scope for command in passenger.commands],
+ ["passenger", "passenger", "passenger"],
+ )
+ self.assertEqual(passenger.commands[-1].parameters, {"position_pct": 100})
+ self.assertTrue(passenger.execution_eligible)
+ for token in ("副驾座椅通风", "副驾座椅加热", "副驾座椅靠背", "100%"):
+ self.assertIn(token, passenger.clean_text)
+
+ driver = self.processor.process(
+ "打开主驾座椅通风和加热，并且把空调调整为24度"
+ )
+ self.assertEqual(
+ [command.property_name for command in driver.commands],
+ ["seat.ventilation", "seat.heating", "hvac.temperature"],
+ )
+ self.assertEqual(
+ [command.target_scope for command in driver.commands[:2]],
+ ["driver", "driver"],
+ )
+ self.assertEqual(driver.commands[-1].parameters, {"temperature_c": 24})
+ self.assertTrue(driver.execution_eligible)
+ for token in ("主驾座椅通风", "主驾座椅加热", "24度"):
+ self.assertIn(token, driver.clean_text)
+
+ def test_coordinated_seat_properties_cover_order_connectors_and_actions(self) -> None:
+ cases = (
+ ("打开主驾座椅通风与加热", "driver", ("seat.ventilation", "seat.heating"), ("on", "on")),
+ ("打开副驾座椅加热以及通风", "passenger", ("seat.heating", "seat.ventilation"), ("on", "on")),
+ ("关闭主驾座椅通风、加热", "driver", ("seat.ventilation", "seat.heating"), ("off", "off")),
+ ("打开主驾座椅通风和关闭加热", "driver", ("seat.ventilation", "seat.heating"), ("on", "off")),
+ )
+ for text, scope, properties, operations in cases:
+ with self.subTest(text=text):
+ result = self.processor.process(text)
+ self.assertEqual(
+ tuple(command.property_name for command in result.commands),
+ properties,
+ )
+ self.assertEqual(
+ tuple(command.operation for command in result.commands),
+ operations,
+ )
+ self.assertEqual(
+ {command.target_scope for command in result.commands},
+ {scope},
+ )
+ self.assertTrue(result.execution_eligible)
+ self.assertTrue(all(command.executor_payload for command in result.commands))
+
+ def test_coordinated_targets_keep_local_actions_scopes_and_properties(self) -> None:
+ cases = (
+ (
+ "关闭空调和打开氛围灯",
+ (("hvac.power", "off", "vehicle"), ("ambient.power", "on", "vehicle")),
+ ),
+ (
+ "打开主驾座椅通风和副驾座椅加热",
+ (("seat.ventilation", "on", "driver"), ("seat.heating", "on", "passenger")),
+ ),
+ (
+ "打开前排和后排空调",
+ (("hvac.power", "on", "front_row"), ("hvac.power", "on", "rear_row")),
+ ),
+ (
+ "打开主驾和副驾车窗",
+ (("window.motion", "open", "front_left"), ("window.motion", "open", "front_right")),
+ ),
+ )
+ for text, expected in cases:
+ with self.subTest(text=text):
+ result = self.processor.process(text)
+ self.assertEqual(
+ tuple(
+ (command.property_name, command.operation, command.target_scope)
+ for command in result.commands
+ ),
+ expected,
+ )
+ self.assertTrue(result.execution_eligible)
+ for command in result.commands:
+ self.assertEqual(
+ command.executor_payload["target"], command.target_scope
+ )
+
+ def test_ratio_semantics_are_shared_across_continuous_capabilities(self) -> None:
+ cases = (
+ ("打开四分之一车窗", "window.position", {"position_pct": 25}, "25%"),
+ ("打开四分之三车窗", "window.position", {"position_pct": 75}, "75%"),
+ ("打开百分之三十车窗", "window.position", {"position_pct": 30}, "30%"),
+ ("打开1/4车窗", "window.position", {"position_pct": 25}, "25%"),
+ ("把遮阳帘打开到四分之一", "roof.shade_position", {"position_pct": 25}, "25%"),
+ ("把副驾腿托调整到四分之三", "seat.legrest_height", {"position_pct": 75}, "75%"),
+ ("把副驾座椅向后调整50%", "seat.longitudinal_position", {"position_pct": 50}, "50%"),
+ )
+ for text, property_name, parameters, display in cases:
+ with self.subTest(text=text):
+ result = self.processor.process(text)
+ self.assertEqual(len(result.commands), 1)
+ self.assertEqual(result.commands[0].property_name, property_name)
+ self.assertEqual(result.commands[0].parameters, parameters)
+ self.assertIn(display, result.clean_text)
+ self.assertTrue(result.execution_eligible)
+
+ def test_invalid_unit_keeps_all_spoken_slots_and_fails_closed(self) -> None:
+ raw = "打开遮阳帘，然后把副驾座椅向后调整50度"
+ result = self.processor.process(raw)
+ self.assertEqual(result.semantic_status, "incomplete")
+ self.assertFalse(result.execution_eligible)
+ self.assertEqual(result.commands[1].target_scope, "passenger")
+ self.assertEqual(result.commands[1].value, 50)
+ self.assertEqual(result.commands[1].parameters, {})
+ self.assertIsNone(result.commands[1].executor_payload)
+ for token in ("遮阳帘", "副驾", "座椅", "向后", "50度"):
+ self.assertIn(token, result.clean_text)
+ self.assertNotIn("将副驾座椅向后调整。", result.clean_text)
+
+ def test_shared_value_and_coordination_grammar_also_feed_one_line_parser(self) -> None:
+ window = merge_native_can_intent({}, "打开四分之一车窗")
+ self.assertEqual(
+ [
+ (action["capability"], action["target"], action["parameters"])
+ for action in window["native_can_actions"]
+ ],
+ [("window.set_position", "front_left", {"position_pct": 25})],
+ )
+
+ thermal = merge_native_can_intent({}, "打开主驾座椅通风和加热")
+ self.assertEqual(
+ [action["capability"] for action in thermal["native_can_actions"]],
+ [
+ "seat.set_driver_ventilation_native",
+ "seat.set_driver_heating_native",
+ ],
+ )
+ self.assertEqual(
+ [action["parameters"] for action in thermal["native_can_actions"]],
+ [
+ {"state": "on", "level": 2},
+ {"state": "on", "level": 2},
+ ],
+ )
+
+ def test_shared_target_inherits_resolved_driver_scope_and_level_semantics(self) -> None:
+ result = self.processor.process("打开座椅通风并调到三档")
+ self.assertEqual(
+ [command.capability for command in result.commands],
+ [
+ "seat.set_driver_ventilation_native",
+ "seat.set_driver_ventilation_native",
+ ],
+ )
+ self.assertEqual(
+ [command.target_scope for command in result.commands],
+ ["driver", "driver"],
+ )
+ self.assertIn("3档", result.clean_text)
+ self.assertEqual(result.semantic_status, "recognized_unsupported")
+ self.assertFalse(result.execution_eligible)
+
+ passenger = self.processor.process("打开副驾座椅通风并调到三档")
+ self.assertEqual(
+ [command.target_scope for command in passenger.commands],
+ ["passenger", "passenger"],
+ )
+ self.assertTrue(passenger.execution_eligible)
+
+ graded = self.processor.process("打开主驾座椅加热并调到高档")
+ self.assertEqual(graded.commands[-1].parameters["level"], 1)
+ self.assertIn("高档", graded.clean_text)
+
+ def test_unsupported_semantics_never_create_fake_executor_payloads(self) -> None:
+ for text in ("打开前大灯", "把主驾座椅调整为零重力模式"):
+ with self.subTest(text=text):
+ result = self.processor.process(text)
+ self.assertEqual(result.semantic_status, "recognized_unsupported")
+ self.assertTrue(result.clean_text)
+ self.assertTrue(result.commands)
+ self.assertTrue(
+ all(command.executor_payload is None for command in result.commands)
+ )
+
+ def test_zero_gravity_is_real_passenger_mode_but_not_driver_capability(self) -> None:
+ passenger = self.processor.process("打开零重力模式")
+ self.assertEqual(passenger.semantic_status, "recognized_executable")
+ self.assertEqual(passenger.commands[0].mode_profile_id, "zero_gravity")
+ payload = passenger.commands[0].executor_payload
+ self.assertIsNotNone(payload)
+ self.assertEqual(len(payload["operations"]), 4)
+ self.assertEqual(
+ {operation["target"] for operation in payload["operations"]},
+ {"passenger"},
+ )
+
+ driver = self.processor.process("把主驾座椅调整为零重力模式")
+ self.assertEqual(driver.semantic_status, "recognized_unsupported")
+ self.assertIsNone(driver.commands[0].executor_payload)
+
+ def test_native_executor_can_compile_reported_executable_cases(self) -> None:
+ for text in (
+ "把遮阳帘打开",
+ "打开主驾座椅通风",
+ "打开一半车窗",
+ "打开主驾座椅加热",
+ ):
+ with self.subTest(text=text):
+ result = self.processor.process(text)
+ payloads = [command.executor_payload for command in result.commands]
+ self.assertTrue(all(payload is not None for payload in payloads))
+ config = self.native_runtime.build_function_config(
+ payloads[0]["domain"], payloads, "speech-alignment", "test"
+ )
+ plan = self.native_runtime.compile(config)
+ self.assertTrue(plan["spec"]["commands"])
+
+ def test_catalog_driven_resolution_matrix_covers_every_definition(self) -> None:
+ definitions = self.resolver.definitions()
+ self.assertEqual(len(definitions), 43)
+ projections = (
+ "canonical_action",
+ "target_preservation",
+ "parameter_preservation",
+ "intent_ir_projection",
+ "executor_projection",
+ )
+ for definition in definitions:
+ target = definition.target_scopes[0] if definition.target_scopes else "vehicle"
+ parameters = _sample_parameters(definition)
+ expected = (
+ "unsupported"
+ if definition.availability == "catalog_only"
+ or (
+ definition.availability == "native_can"
+ and definition.name not in self.native_runtime.supported_capabilities
+ )
+ else "executable"
+ )
+ for projection in projections:
+ with self.subTest(capability=definition.name, projection=projection):
+ result = self.resolver.resolve(
+ domain=definition.domain,
+ semantic_property=definition.name,
+ target_scope=target,
+ target_device=None,
+ operation="set",
+ parameters=parameters,
+ source_text=f"{projection}:{definition.name}:{target}",
+ preferred_capability=definition.name,
+ )
+ self.assertEqual(result.definition.name, definition.name)
+ self.assertEqual(result.status, expected)
+ if expected == "executable":
+ self.assertEqual(result.executor_payload["target"], target)
+ self.assertEqual(result.executor_payload["parameters"], parameters)
+ self.assertEqual(result.intent_ir_action["target"]["zone"], target)
+ else:
+ self.assertIsNone(result.executor_payload)
+
+ def test_vocabulary_does_not_define_vehicle_capabilities(self) -> None:
+ self.assertNotIn("capability", DeviceTerm.__dataclass_fields__)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/unit/test_speech_orchestration_bridge.py b/tests/unit/test_speech_orchestration_bridge.py
new file mode 100644
index 0000000..848844c
--- /dev/null
+++ b/tests/unit/test_speech_orchestration_bridge.py
@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from cognitive_agent.application.seat_service import SeatDemoService
+from cognitive_agent.infrastructure.speech.understanding import (
+ UnderstandingTranscriptProcessor,
+)
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
+
+
+@pytest.mark.parametrize(
+ ("utterance", "capability", "target", "parameters"),
+ (
+ (
+ "打开一半车窗",
+ "window.set_position",
+ {"device": "window", "zone": "front_left"},
+ {"position_pct": 50},
+ ),
+ (
+ "把副驾腿托向上调整到最大位置",
+ "seat.set_legrest_height",
+ {"device": "seat", "zone": "front_right", "instance": "passenger"},
+ {"position_pct": 100},
+ ),
+ (
+ "把遮阳帘打开",
+ "roof.set_shade_motion",
+ {"device": "roof", "zone": "vehicle"},
+ {"motion": "open"},
+ ),
+ (
+ "遮阳帘打开一半",
+ "roof.set_shade_motion",
+ {"device": "roof", "zone": "vehicle"},
+ {"motion": "half"},
+ ),
+ (
+ "把遮阳帘打开到四分之一",
+ "roof.set_shade_position",
+ {"device": "roof", "zone": "vehicle"},
+ {"position_pct": 25},
+ ),
+ ),
+)
+def test_server_owned_speech_semantics_bypass_second_language_parse(
+ utterance: str,
+ capability: str,
+ target: dict[str, str],
+ parameters: dict[str, int],
+) -> None:
+ processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
+ understanding = processor.process(utterance).to_dict()
+ understanding["utterance_id"] = "bridge-regression"
+
+ with tempfile.TemporaryDirectory() as directory, patch.dict(
+ os.environ,
+ {
+ "COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
+ "COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
+ "COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
+ },
+ clear=False,
+ ):
+ service = SeatDemoService(REPOSITORY_ROOT)
+ service.artifact_root = Path(directory) / "artifacts"
+ with patch.object(
+ service,
+ "_interpret",
+ side_effect=AssertionError("cleanText must not be parsed a second time"),
+ ):
+ result = service.process(
+ understanding["clean_text"],
+ provider="local",
+ execution_mode="dry_run",
+ _speech_understanding=understanding,
+ )
+
+ assert result["status"] == "succeeded"
+ assert result["provider_used"] == "speech_semantic"
+ assert result["speech_understanding"]["raw_text"] == utterance
+ actions = result["intent_ir"]["spec"]["actions"]
+ assert len(actions) == 1
+ assert actions[0]["capability"] == capability
+ assert actions[0]["target"] == target
+ assert actions[0]["parameters"] == parameters
+ assert actions[0]["provenance"]["parser"] == "speech_semantic"
+
+
+def test_non_executable_speech_cannot_enter_orchestration_bridge() -> None:
+ processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
+ understanding = processor.process("打开前大灯").to_dict()
+ understanding["utterance_id"] = "bridge-unsupported"
+
+ with patch.dict(
+ os.environ,
+ {
+ "COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
+ "COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
+ },
+ clear=False,
+ ):
+ service = SeatDemoService(REPOSITORY_ROOT)
+ context = service._context_for_vehicle(
+ service.conversations.snapshot("conv-speech-bridge"),
+ service.vehicle_profiles.get(None),
+ )
+ with pytest.raises(ValueError, match="not execution eligible"):
+ service._adapt_speech_understanding(
+ understanding,
+ context,
+ service.vehicle_profiles.get(None),
+ )
+
+
+def test_every_registered_native_runtime_capability_keeps_native_bridge_binding() -> None:
+ def valid_value(spec: dict[str, object]) -> object:
+ allowed = spec.get("allowed_values")
+ if isinstance(allowed, list) and allowed:
+ return allowed[0]
+ value_type = str(spec.get("type", "string"))
+ if value_type in {"integer", "number"}:
+ return spec.get("minimum", 0)
+ if value_type == "boolean":
+ return True
+ if value_type == "object":
+ return {}
+ if value_type == "array":
+ return []
+ return "test"
+
+ with patch.dict(
+ os.environ,
+ {
+ "COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
+ "COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
+ },
+ clear=False,
+ ):
+ service = SeatDemoService(REPOSITORY_ROOT)
+ context = service._context_for_vehicle(
+ service.conversations.snapshot("conv-native-speech-matrix"),
+ service.vehicle_profiles.get(None),
+ )
+ registered = {
+ definition.name: definition
+ for definition in service.capability_registry.definitions()
+ if definition.name in service.native_can_runtime.supported_capabilities
+ }
+ assert registered
+ for capability, definition in registered.items():
+ parameters = {
+ str(spec["name"]): valid_value(spec)
+ for spec in definition.parameters
+ if spec.get("required")
+ }
+ target = definition.target_scopes[0]
+ understanding = {
+ "raw_text": f"native bridge matrix: {capability}",
+ "execution_eligible": True,
+ "commands": [
+ {
+ "status": "active",
+ "negated": False,
+ "resolution_status": "executable",
+ "validation_issues": [],
+ "confidence": 1.0,
+ "source_text": capability,
+ "executor_payload": {
+ "domain": definition.domain,
+ "capability": capability,
+ "target": target,
+ "operation": "set",
+ "parameters": parameters,
+ },
+ }
+ ],
+ }
+ parsed = service._adapt_speech_understanding(
+ understanding,
+ context,
+ service.vehicle_profiles.get(None),
+ )
+ assert parsed["deferred_requests"] == [], capability
+ assert parsed["actions"] == [], capability
+ assert parsed["domain_actions"] == [], capability
+ assert [
+ action["capability"] for action in parsed["native_can_actions"]
+ ] == [capability]
diff --git a/tests/unit/test_speech_transcription.py b/tests/unit/test_speech_transcription.py
new file mode 100644
index 0000000..5b28b03
--- /dev/null
+++ b/tests/unit/test_speech_transcription.py
@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+import io
+import json
+import math
+import tempfile
+import unittest
+import wave
+from pathlib import Path
+from types import SimpleNamespace
+
+from cognitive_agent.infrastructure.speech.sensevoice import (
+ SpeechRecognitionError,
+ SpeechTranscriber,
+)
+from cognitive_agent.infrastructure.speech.understanding import (
+ UnderstandingTranscriptProcessor,
+)
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
+
+
+def pcm16_wav(*, frequency: float = 220, seconds: float = 0.8) -> bytes:
+ frames = bytearray()
+ for index in range(round(16_000 * seconds)):
+ sample = round(math.sin(index * 2 * math.pi * frequency / 16_000) * 12_000)
+ frames.extend(int(sample).to_bytes(2, "little", signed=True))
+ target = io.BytesIO()
+ with wave.open(target, "wb") as output:
+ output.setnchannels(1)
+ output.setsampwidth(2)
+ output.setframerate(16_000)
+ output.writeframes(frames)
+ return target.getvalue()
+
+
+class _FakeStream:
+ def __init__(self, text: str) -> None:
+ self.result = SimpleNamespace(
+ text=f"<|zh|><|NEUTRAL|><|Speech|>{text}",
+ lang="<|zh|>",
+ emotion="<|NEUTRAL|>",
+ event="<|Speech|>",
+ )
+
+ def accept_waveform(self, sample_rate: int, samples: object) -> None:
+ self.sample_rate = sample_rate
+ self.samples = samples
+
+
+class _FakeRecognizer:
+ def __init__(self, text: str) -> None:
+ self.text = text
+ self.decode_count = 0
+
+ def create_stream(self) -> _FakeStream:
+ return _FakeStream(self.text)
+
+ def decode_stream(self, stream: _FakeStream) -> None:
+ self.decode_count += 1
+
+
+class UnderstandingRegressionTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
+ fixture = REPOSITORY_ROOT / "tests" / "fixtures" / "speech_text_regression.json"
+ cls.cases = json.loads(fixture.read_text(encoding="utf-8"))
+
+ def test_text_regression_suite(self) -> None:
+ for case in self.cases:
+ with self.subTest(case=case["name"]):
+ result = self.processor.process(case["raw"])
+ self.assertEqual(result.raw_text, case["raw"])
+ self.assertEqual(result.clean_text, case["clean"])
+ self.assertEqual(
+ [command.capability for command in result.commands],
+ case["capabilities"],
+ )
+ self.assertEqual(len(result.cancelled_fragments), case["cancelled"])
+ self.assertEqual(result.uncertain, case["uncertain"])
+ self.assertEqual(
+ result.execution_eligible,
+ case.get(
+ "execution_eligible",
+ bool(result.commands) and not result.uncertain,
+ ),
+ )
+ if preserved := case.get("preserved_filler"):
+ self.assertNotIn(preserved, result.removed_fillers)
+
+ def test_cancelled_commands_never_appear_as_active(self) -> None:
+ result = self.processor.process(
+ "打开空调，不对，把座椅靠背向后调24度。"
+ )
+ self.assertEqual([command.domain for command in result.commands], ["seat"])
+ cancelled = result.cancelled_fragments[0].commands
+ self.assertEqual([command.domain for command in cancelled], ["climate"])
+ self.assertTrue(all(command.status == "cancelled" for command in cancelled))
+
+ def test_invalid_seat_absolute_angle_fails_closed(self) -> None:
+ result = self.processor.process("把座椅靠背调整到24度。")
+ self.assertTrue(result.uncertain)
+ self.assertFalse(result.execution_eligible)
+ self.assertIn("低于工程下限95", "".join(result.reasons))
+
+ def test_regular_multi_command_is_not_treated_as_correction(self) -> None:
+ result = self.processor.process("打开空调，然后打开氛围灯。")
+ self.assertEqual(len(result.commands), 2)
+ self.assertFalse(result.cancelled_fragments)
+
+
+class SpeechTranscriberTests(unittest.TestCase):
+ def make_transcriber(self, directory: str, text: str) -> tuple[SpeechTranscriber, _FakeRecognizer]:
+ transcriber = SpeechTranscriber(Path(directory))
+ recognizer = _FakeRecognizer(text)
+ transcriber._recognizer = recognizer
+ return transcriber, recognizer
+
+ def test_transcriber_preserves_raw_and_returns_structured_result(self) -> None:
+ raw = "打开空调，不对，把座椅靠背向后调24度。"
+ with tempfile.TemporaryDirectory() as directory:
+ transcriber, _ = self.make_transcriber(directory, raw)
+ result = transcriber.transcribe_wav(
+ pcm16_wav(), utterance_id="unit-structured"
+ )
+ self.assertEqual(result["raw_text"], raw)
+ self.assertEqual(result["clean_text"], "将座椅靠背向后调整24度。")
+ self.assertEqual(result["commands"][0]["domain"], "seat")
+ self.assertEqual(result["cancelled_fragments"][0]["commands"][0]["domain"], "climate")
+ self.assertIn("timings", result)
+ self.assertEqual(result["segments"][0]["state"], "final")
+
+ def test_filler_only_is_evidence_but_not_a_control_error(self) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ transcriber, _ = self.make_transcriber(directory, "嗯嗯。")
+ result = transcriber.transcribe_wav(
+ pcm16_wav(), utterance_id="unit-filler"
+ )
+ self.assertEqual(result["raw_text"], "嗯嗯。")
+ self.assertEqual(result["clean_text"], "")
+ self.assertEqual(result["commands"], [])
+ self.assertFalse(result["execution_eligible"])
+
+ def test_same_utterance_and_audio_is_decoded_once(self) -> None:
+ audio = pcm16_wav()
+ with tempfile.TemporaryDirectory() as directory:
+ transcriber, recognizer = self.make_transcriber(directory, "打开空调。")
+ first = transcriber.transcribe_wav(audio, utterance_id="unit-idempotent")
+ second = transcriber.transcribe_wav(audio, utterance_id="unit-idempotent")
+ self.assertEqual(recognizer.decode_count, 1)
+ self.assertFalse(first["idempotent_replay"])
+ self.assertTrue(second["idempotent_replay"])
+
+ def test_cached_result_is_server_owned_and_returned_as_a_copy(self) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ transcriber, _ = self.make_transcriber(directory, "打开空调。")
+ transcriber.transcribe_wav(pcm16_wav(), utterance_id="unit-cached")
+ first = transcriber.cached_result("unit-cached")
+ first["clean_text"] = "tampered"
+ second = transcriber.cached_result("unit-cached")
+ self.assertNotEqual(second["clean_text"], "tampered")
+
+ def test_missing_cached_result_is_rejected(self) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ transcriber, _ = self.make_transcriber(directory, "打开空调。")
+ with self.assertRaises(SpeechRecognitionError) as caught:
+ transcriber.cached_result("unit-expired")
+ self.assertEqual(caught.exception.code, "speech_result_expired")
+
+ def test_same_utterance_with_different_audio_is_rejected(self) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ transcriber, _ = self.make_transcriber(directory, "打开空调。")
+ transcriber.transcribe_wav(pcm16_wav(), utterance_id="unit-conflict")
+ with self.assertRaises(SpeechRecognitionError) as caught:
+ transcriber.transcribe_wav(
+ pcm16_wav(frequency=330), utterance_id="unit-conflict"
+ )
+ self.assertEqual(caught.exception.code, "utterance_id_conflict")
+
+ def test_invalid_utterance_id_is_rejected(self) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ transcriber, _ = self.make_transcriber(directory, "打开空调。")
+ with self.assertRaises(SpeechRecognitionError) as caught:
+ transcriber.transcribe_wav(pcm16_wav(), utterance_id="bad id/路径")
+ self.assertEqual(caught.exception.code, "invalid_utterance_id")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/unit/test_transcript_assembler.py b/tests/unit/test_transcript_assembler.py
new file mode 100644
index 0000000..69e2156
--- /dev/null
+++ b/tests/unit/test_transcript_assembler.py
@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import unittest
+
+from cognitive_agent.infrastructure.speech.assembler import TranscriptAssembler
+from cognitive_agent.infrastructure.speech.contracts import SpeechSegmentResult
+
+
+def segment(
+ segment_id: str,
+ sequence: int,
+ text: str,
+ *,
+ final: bool,
+ start_ms: int | None = None,
+ end_ms: int | None = None,
+ overlap_ms: int = 0,
+) -> SpeechSegmentResult:
+ start = sequence * 1000 if start_ms is None else start_ms
+ end = start + 1200 if end_ms is None else end_ms
+ return SpeechSegmentResult(
+ utterance_id="utterance-1",
+ segment_id=segment_id,
+ sequence=sequence,
+ start_ms=start,
+ end_ms=end,
+ text=text,
+ is_final=final,
+ audio_overlap_ms=overlap_ms,
+ )
+
+
+class TranscriptAssemblerTests(unittest.TestCase):
+ def test_partial_replaces_partial_and_final_commits(self) -> None:
+ assembler = TranscriptAssembler("utterance-1")
+ assembler.update(segment("s0", 0, "打开空", final=False))
+ self.assertEqual(assembler.snapshot().text, "打开空")
+ assembler.update(segment("s0", 0, "打开空调", final=False))
+ self.assertEqual(assembler.snapshot().text, "打开空调")
+ final = assembler.update(segment("s0", 0, "打开空调。", final=True))
+ self.assertEqual(final.text, "打开空调。")
+ self.assertFalse(final.partial_segments)
+
+ def test_duplicate_final_callback_is_idempotent(self) -> None:
+ assembler = TranscriptAssembler("utterance-1")
+ callback = segment("s0", 0, "打开空调。", final=True)
+ assembler.update(callback)
+ result = assembler.update(callback)
+ self.assertEqual(result.text, "打开空调。")
+ self.assertEqual(result.duplicate_callbacks, 1)
+
+ def test_final_cannot_be_mutated(self) -> None:
+ assembler = TranscriptAssembler("utterance-1")
+ assembler.update(segment("s0", 0, "打开空调。", final=True))
+ result = assembler.update(segment("s0", 0, "关闭空调。", final=True))
+ self.assertTrue(result.uncertain)
+ self.assertEqual(result.text, "打开空调。")
+
+ def test_out_of_order_callbacks_are_sorted(self) -> None:
+ assembler = TranscriptAssembler("utterance-1")
+ assembler.update(segment("s1", 1, "打开氛围灯。", final=True))
+ result = assembler.update(segment("s0", 0, "打开空调。", final=True))
+ self.assertEqual(result.text, "打开空调。打开氛围灯。")
+ self.assertEqual(result.out_of_order_callbacks, 1)
+
+ def test_audio_supported_boundary_overlap_is_removed(self) -> None:
+ assembler = TranscriptAssembler("utterance-1")
+ assembler.update(
+ segment("s0", 0, "把座椅靠背", final=True, start_ms=0, end_ms=1500)
+ )
+ result = assembler.update(
+ segment(
+ "s1",
+ 1,
+ "靠背向后调24度",
+ final=True,
+ start_ms=1400,
+ end_ms=2600,
+ overlap_ms=100,
+ )
+ )
+ self.assertEqual(result.text, "把座椅靠背，向后调24度")
+ self.assertEqual(result.overlaps_removed, ("靠背",))
+
+ def test_long_text_overlap_is_removed_without_audio_overlap(self) -> None:
+ assembler = TranscriptAssembler("utterance-1")
+ assembler.update(segment("s0", 0, "请打开空调温度", final=True))
+ result = assembler.update(segment("s1", 1, "空调温度调到24度", final=True))
+ self.assertIn("空调温度", result.overlaps_removed)
+
+ def test_repeated_action_in_distinct_commands_is_preserved(self) -> None:
+ assembler = TranscriptAssembler("utterance-1")
+ assembler.update(segment("s0", 0, "打开前排空调", final=True))
+ result = assembler.update(segment("s1", 1, "打开后排空调", final=True))
+ self.assertEqual(result.text, "打开前排空调，打开后排空调")
+ self.assertFalse(result.overlaps_removed)
+
+ def test_finish_rejects_late_callbacks_and_reset_starts_new_utterance(self) -> None:
+ assembler = TranscriptAssembler("utterance-1")
+ assembler.update(segment("s0", 0, "打开空调", final=True))
+ assembler.finish()
+ late = assembler.update(segment("s1", 1, "关闭空调", final=True))
+ self.assertEqual(late.text, "打开空调")
+ assembler.reset("utterance-2")
+ self.assertEqual(assembler.snapshot().utterance_id, "utterance-2")
+ self.assertEqual(assembler.snapshot().text, "")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tools/benchmark_speech_pipeline.py b/tools/benchmark_speech_pipeline.py
new file mode 100644
index 0000000..77b6d2c
--- /dev/null
+++ b/tools/benchmark_speech_pipeline.py
@@ -0,0 +1,288 @@
+"""Measure text semantics and optional real/synthetic WAV speech regressions."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import statistics
+import time
+from pathlib import Path
+from typing import Any
+
+from cognitive_agent.infrastructure.speech.sensevoice import SpeechTranscriber
+from cognitive_agent.infrastructure.speech.understanding import (
+ UnderstandingTranscriptProcessor,
+)
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+TEXT_FIXTURE = REPOSITORY_ROOT / "tests" / "fixtures" / "speech_text_regression.json"
+DEFAULT_AUDIO_DIR = REPOSITORY_ROOT / "artifacts" / "tmp" / "speech-regression-audio"
+NUMBER = re.compile(r"-?\d+(?:\.\d+)?")
+CAPABILITY_ALIGNMENT_TEXTS = (
+ "把遮阳帘打开",
+ "打开前大灯",
+ "打开主驾座椅通风",
+ "把主驾座椅调整为零重力模式",
+ "打开一半车窗",
+ "打开氛围灯并调整为红色",
+ "打开主驾座椅加热",
+ "打开副驾座椅通风",
+ "打开遮阳帘，然后把副驾座椅向后调整50度",
+ "打开四分之一车窗",
+ "打开副驾座椅通风和加热，然后将副驾座椅靠背调整到最大位置",
+ "打开主驾座椅通风和加热，并且把空调调整为24度",
+ "打开副驾座椅加热以及通风",
+ "关闭空调和打开氛围灯",
+ "打开前排和后排空调",
+ "打开主驾和副驾车窗",
+)
+
+AUDIO_EXPECTATIONS: dict[str, dict[str, Any]] = {
+ "01_filler_hvac.wav": {
+ "raw": "呃，那个那个那个打开空调到24度。",
+ "clean": "打开空调并设置为24度。",
+ "capabilities": ["climate.set_power", "climate.set_temperature"],
+ },
+ "02_seat_refine.wav": {
+ "raw": "座椅调整到呃20呃，靠背调整到24度。",
+ "clean": "将座椅靠背调整到24度。",
+ "capabilities": ["seat.set_backrest_recline"],
+ },
+ "03_domain_repair.wav": {
+ "raw": "打开空调，不对，把座椅靠背向后调24度。",
+ "clean": "将座椅靠背向后调整24度。",
+ "capabilities": ["seat.set_backrest_recline"],
+ },
+ "04_multi.wav": {
+ "raw": "空调调到24度，然后把座椅靠背调到20度。",
+ "clean": "将空调设置为24度，然后将座椅靠背调整到20度。",
+ "capabilities": ["climate.set_temperature", "seat.set_backrest_recline"],
+ },
+ "05_number_repair.wav": {
+ "raw": "空调调到24，不对，25度。",
+ "clean": "将空调设置为25度。",
+ "capabilities": ["climate.set_temperature"],
+ },
+ "06_direction_repair.wav": {
+ "raw": "把座椅往前，不对，往后一点。",
+ "clean": "将座椅向后调整一点。",
+ "capabilities": ["seat.set_driver_axis_native"],
+ },
+ "07_negation.wav": {"raw": "不要打开空调。", "clean": "不要打开空调。", "capabilities": []},
+ "08_chat.wav": {"raw": "今天天气不错。", "clean": "", "capabilities": []},
+ "09_fast.wav": {
+ "raw": "打开空调，然后把座椅靠背向后调24度。",
+ "clean": "打开空调，然后将座椅靠背向后调整24度。",
+ "capabilities": ["climate.set_power", "seat.set_backrest_recline"],
+ },
+ "10_slow.wav": {
+ "raw": "打开空调，然后把座椅靠背向后调24度。",
+ "clean": "打开空调，然后将座椅靠背向后调整24度。",
+ "capabilities": ["climate.set_power", "seat.set_backrest_recline"],
+ },
+}
+
+
+def ratio(passed: int, total: int) -> float:
+ return round(passed / total, 4) if total else 0.0
+
+
+def percentile(values: list[float], quantile: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ index = round((len(ordered) - 1) * quantile)
+ return round(ordered[index], 3)
+
+
+def numeric_parameters(commands: list[dict[str, Any]]) -> list[float]:
+ return sorted(
+ float(value)
+ for command in commands
+ for name, value in command.get("parameters", {}).items()
+ if name not in set(command.get("qualifiers", {}).get("implicit_parameters", ()))
+ if isinstance(value, (int, float)) and not isinstance(value, bool)
+ )
+
+
+def expected_numbers(clean: str) -> list[float]:
+ return sorted(float(value) for value in NUMBER.findall(clean))
+
+
+def score_cases(cases: list[dict[str, Any]]) -> dict[str, Any]:
+ def domains(capabilities: list[str | None]) -> set[str]:
+ return {
+ capability.split(".", 1)[0]
+ for capability in capabilities
+ if capability
+ }
+
+ total = len(cases)
+ def parameter_numbers(item: dict[str, Any]) -> list[float]:
+ return [
+ float(value)
+ for value in item.get("parameter_numbers", expected_numbers(item["clean"]))
+ ]
+
+ semantic = sum(item["actual_clean"] == item["clean"] for item in cases)
+ intent = sum(item["actual_capabilities"] == item["capabilities"] for item in cases)
+ device = sum(
+ domains(item["actual_capabilities"]) == domains(item["capabilities"])
+ for item in cases
+ )
+ parameter = sum(item["actual_numbers"] == parameter_numbers(item) for item in cases)
+ corrections = [item for item in cases if item.get("cancelled", 0)]
+ correction_success = sum(
+ item["actual_capabilities"] == item["capabilities"]
+ and item["actual_cancelled"] >= item.get("cancelled", 0)
+ for item in corrections
+ )
+ no_command = [item for item in cases if not item["capabilities"]]
+ false_activation = sum(bool(item["actual_capabilities"]) for item in no_command)
+ wrong_device = sum(
+ domains(item["actual_capabilities"]) != domains(item["capabilities"])
+ for item in cases
+ )
+ wrong_number = sum(item["actual_numbers"] != parameter_numbers(item) for item in cases)
+ semantic_times = [item["semantic_ms"] for item in cases]
+ resolution_times = [item["command_resolution_ms"] for item in cases]
+ return {
+ "case_count": total,
+ "semantic_clean_accuracy": ratio(semantic, total),
+ "intent_exact_match": ratio(intent, total),
+ "device_accuracy": ratio(device, total),
+ "action_accuracy": ratio(intent, total),
+ "parameter_exact_match": ratio(parameter, total),
+ "correction_success_rate": ratio(correction_success, len(corrections)),
+ "false_command_activation_rate": ratio(false_activation, len(no_command)),
+ "severe_wrong_device_count": wrong_device,
+ "severe_wrong_number_count": wrong_number,
+ "semantic_processing_p50_ms": percentile(semantic_times, 0.50),
+ "semantic_processing_p95_ms": percentile(semantic_times, 0.95),
+ "command_resolution_p50_ms": percentile(resolution_times, 0.50),
+ "command_resolution_p95_ms": percentile(resolution_times, 0.95),
+ }
+
+
+def run_text() -> tuple[dict[str, Any], list[dict[str, Any]]]:
+ processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
+ fixtures = json.loads(TEXT_FIXTURE.read_text(encoding="utf-8"))
+ cases: list[dict[str, Any]] = []
+ for fixture in fixtures:
+ started = time.perf_counter()
+ result = processor.process(fixture["raw"])
+ semantic_ms = (time.perf_counter() - started) * 1000
+ commands = [command.to_dict() for command in result.commands]
+ cases.append(
+ {
+ **fixture,
+ "actual_clean": result.clean_text,
+ "actual_capabilities": [command["capability"] for command in commands],
+ "actual_numbers": numeric_parameters(commands),
+ "actual_cancelled": len(result.cancelled_fragments),
+ "semantic_ms": semantic_ms,
+ "command_resolution_ms": result.command_resolution_ms,
+ }
+ )
+ return score_cases(cases), cases
+
+
+def run_audio(audio_dir: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+ transcriber = SpeechTranscriber(REPOSITORY_ROOT)
+ cases: list[dict[str, Any]] = []
+ for name, expected in AUDIO_EXPECTATIONS.items():
+ path = audio_dir / name
+ if not path.is_file():
+ continue
+ wall_started = time.perf_counter()
+ result = transcriber.transcribe_wav(
+ path.read_bytes(), utterance_id=f"benchmark-{path.stem}"
+ )
+ wall_ms = (time.perf_counter() - wall_started) * 1000
+ commands = list(result["commands"])
+ cases.append(
+ {
+ "name": name,
+ **expected,
+ "cancelled": 1 if "repair" in name or "refine" in name else 0,
+ "actual_raw": result["raw_text"],
+ "actual_clean": result["clean_text"],
+ "actual_capabilities": [command["capability"] for command in commands],
+ "actual_numbers": numeric_parameters(commands),
+ "actual_cancelled": len(result["cancelled_fragments"]),
+ "semantic_ms": float(result["timings"]["semantic_ms"]),
+ "command_resolution_ms": float(result["timings"]["command_resolution_ms"]),
+ "asr_ms": float(result["timings"]["asr_ms"]),
+ "end_to_end_ms": float(result["timings"]["total_ms"]),
+ "wall_ms": wall_ms,
+ "duplicate_callbacks": int(result["assembly"]["duplicate_callbacks"]),
+ }
+ )
+ metrics = score_cases(cases)
+ raw_exact = sum(item["actual_raw"] == item["raw"] for item in cases)
+ metrics.update(
+ {
+ "raw_transcription_accuracy": ratio(raw_exact, len(cases)),
+ "duplicate_transcript_rate": ratio(
+ sum(item["duplicate_callbacks"] > 0 for item in cases), len(cases)
+ ),
+ "asr_inference_p50_ms": percentile([item["asr_ms"] for item in cases], 0.50),
+ "asr_inference_p95_ms": percentile([item["asr_ms"] for item in cases], 0.95),
+ "end_to_end_p50_ms": percentile([item["end_to_end_ms"] for item in cases], 0.50),
+ "end_to_end_p95_ms": percentile([item["end_to_end_ms"] for item in cases], 0.95),
+ "cold_start_end_to_end_ms": round(cases[0]["end_to_end_ms"], 3) if cases else 0.0,
+ "steady_state_end_to_end_p50_ms": percentile(
+ [item["end_to_end_ms"] for item in cases[1:]], 0.50
+ ),
+ "steady_state_end_to_end_p95_ms": percentile(
+ [item["end_to_end_ms"] for item in cases[1:]], 0.95
+ ),
+ "wall_mean_ms": round(statistics.fmean(item["wall_ms"] for item in cases), 3)
+ if cases
+ else 0.0,
+ }
+ )
+ return metrics, cases
+
+
+def run_capability_resolution(*, iterations: int = 100) -> dict[str, float | int]:
+ processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
+ semantic_times: list[float] = []
+ resolution_times: list[float] = []
+ for _ in range(iterations):
+ for text in CAPABILITY_ALIGNMENT_TEXTS:
+ result = processor.process(text)
+ semantic_times.append(result.processing_ms)
+ resolution_times.append(result.command_resolution_ms)
+ return {
+ "sample_count": len(semantic_times),
+ "semantic_p50_ms": percentile(semantic_times, 0.50),
+ "semantic_p95_ms": percentile(semantic_times, 0.95),
+ "command_resolution_p50_ms": percentile(resolution_times, 0.50),
+ "command_resolution_p95_ms": percentile(resolution_times, 0.95),
+ }
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--audio-dir", type=Path, default=DEFAULT_AUDIO_DIR)
+ parser.add_argument("--details", action="store_true")
+ args = parser.parse_args()
+ text_metrics, text_cases = run_text()
+ payload: dict[str, Any] = {
+ "text": text_metrics,
+ "capability_alignment": run_capability_resolution(),
+ }
+ if args.audio_dir.is_dir():
+ audio_metrics, audio_cases = run_audio(args.audio_dir)
+ payload["audio"] = audio_metrics
+ if args.details:
+ payload["audio_cases"] = audio_cases
+ if args.details:
+ payload["text_cases"] = text_cases
+ print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/install_sensevoice_model.py b/tools/install_sensevoice_model.py
new file mode 100644
index 0000000..db2c63b
--- /dev/null
+++ b/tools/install_sensevoice_model.py
@@ -0,0 +1,106 @@
+"""Download official SenseVoiceSmall and Silero VAD sherpa-onnx models."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import shutil
+import tarfile
+import tempfile
+import urllib.request
+from pathlib import Path
+
+MODEL_URL = (
+ "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/"
+ "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2"
+)
+MODEL_SHA256 = "f6b2a72ebcb1ac7a764d4cfccd886e6bcb2a95c4657c2199d0ba95ed4b9ea71a"
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_TARGET = REPOSITORY_ROOT / "artifacts" / "models" / "sensevoice-small"
+VAD_URL = (
+ "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/"
+ "silero_vad.onnx"
+)
+VAD_SHA256 = "9e2449e1087496d8d4caba907f23e0bd3f78d91fa552479bb9c23ac09cbb1fd6"
+DEFAULT_VAD_TARGET = (
+ REPOSITORY_ROOT / "artifacts" / "models" / "silero-vad" / "silero_vad.onnx"
+)
+
+
+def _download(url: str, destination: Path) -> None:
+ with urllib.request.urlopen(url, timeout=60) as response, destination.open("wb") as output:
+ shutil.copyfileobj(response, output, length=1024 * 1024)
+
+
+def _sha256(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as source:
+ for chunk in iter(lambda: source.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def install(target: Path) -> None:
+ target = target.resolve()
+ if (target / "model.onnx").is_file() and (target / "tokens.txt").is_file():
+ print(f"SenseVoiceSmall is already installed at {target}")
+ return
+ target.mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory(prefix="sensevoice-install-") as temporary_directory:
+ temporary_root = Path(temporary_directory)
+ archive = temporary_root / "sensevoice-small.tar.bz2"
+ print(f"Downloading the official SenseVoiceSmall model to {archive} ...", flush=True)
+ _download(MODEL_URL, archive)
+ actual_hash = _sha256(archive)
+ if actual_hash != MODEL_SHA256:
+ raise RuntimeError(
+ f"SenseVoiceSmall archive checksum mismatch: {actual_hash} != {MODEL_SHA256}"
+ )
+ extracted = temporary_root / "extracted"
+ extracted.mkdir()
+ with tarfile.open(archive, "r:bz2") as package:
+ package.extractall(extracted, filter="data")
+ model_file = next(extracted.rglob("model.onnx"), None)
+ if model_file is None or not (model_file.parent / "tokens.txt").is_file():
+ raise RuntimeError("SenseVoiceSmall archive does not contain the expected model files")
+ for item in model_file.parent.iterdir():
+ destination = target / item.name
+ if item.is_dir():
+ shutil.copytree(item, destination, dirs_exist_ok=True)
+ else:
+ shutil.copy2(item, destination)
+ print(f"SenseVoiceSmall installed at {target}")
+
+
+def install_vad(target: Path) -> None:
+ target = target.resolve()
+ if target.is_file() and _sha256(target) == VAD_SHA256:
+ print(f"Silero VAD is already installed at {target}")
+ return
+ target.parent.mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory(prefix="silero-vad-install-") as temporary_directory:
+ downloaded = Path(temporary_directory) / "silero_vad.onnx"
+ print(f"Downloading the official Silero VAD model to {downloaded} ...", flush=True)
+ _download(VAD_URL, downloaded)
+ actual_hash = _sha256(downloaded)
+ if actual_hash != VAD_SHA256:
+ raise RuntimeError(
+ f"Silero VAD checksum mismatch: {actual_hash} != {VAD_SHA256}"
+ )
+ shutil.copy2(downloaded, target)
+ print(f"Silero VAD installed at {target}")
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--target", type=Path, default=DEFAULT_TARGET)
+ parser.add_argument("--vad-target", type=Path, default=DEFAULT_VAD_TARGET)
+ parser.add_argument("--skip-vad", action="store_true")
+ args = parser.parse_args()
+ install(args.target)
+ if not args.skip_vad:
+ install_vad(args.vad_target)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/report_capability_matrix.py b/tools/report_capability_matrix.py
new file mode 100644
index 0000000..32640ad
--- /dev/null
+++ b/tools/report_capability_matrix.py
@@ -0,0 +1,88 @@
+"""Print the capability/executor matrix derived from repository catalogs."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+from cognitive_agent.infrastructure.speech.capability import CapabilityResolver
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
+
+
+def parameter_summary(parameters: tuple[dict[str, Any], ...]) -> str:
+ values: list[str] = []
+ for parameter in parameters:
+ name = str(parameter["name"])
+ value_type = str(parameter.get("type", "unknown"))
+ allowed = parameter.get("allowed_values")
+ if allowed is not None:
+ constraint = "{" + ",".join(map(str, allowed)) + "}"
+ elif parameter.get("minimum") is not None or parameter.get("maximum") is not None:
+ constraint = f"[{parameter.get('minimum', '-∞')},{parameter.get('maximum', '+∞')}]"
+ else:
+ constraint = ""
+ unit = f" {parameter['unit']}" if parameter.get("unit") else ""
+ optional = "" if parameter.get("required") else "?"
+ values.append(f"{name}{optional}:{value_type}{constraint}{unit}")
+ return "; ".join(values) or "—"
+
+
+def matrix() -> list[dict[str, Any]]:
+ resolver = CapabilityResolver(REPOSITORY_ROOT)
+ rows = []
+ for definition in resolver.definitions():
+ executable = not (
+ definition.availability == "catalog_only"
+ or (
+ definition.availability == "native_can"
+ and definition.name not in resolver.native_supported
+ )
+ )
+ rows.append(
+ {
+ "device": ",".join(definition.target_devices) or "—",
+ "target": ",".join(definition.target_scopes) or "—",
+ "capability": definition.name,
+ "operation": "set/defined by capability parameters",
+ "value_schema": parameter_summary(definition.parameters),
+ "availability": definition.availability,
+ "execution_eligible": executable,
+ "executor_binding": (
+ resolver.executor_binding_for(definition) if executable else None
+ ),
+ "catalog": str(definition.catalog_path.relative_to(REPOSITORY_ROOT)),
+ }
+ )
+ return rows
+
+
+def as_markdown(rows: list[dict[str, Any]]) -> str:
+ lines = [
+ "| Device | Target / scope | Capability | Value type / range / unit | Executable | Executor binding |",
+ "|---|---|---|---|---:|---|",
+ ]
+ for row in rows:
+ lines.append(
+ "| {device} | {target} | `{capability}` | {value_schema} | {execution_eligible} | {executor_binding} |".format(
+ **row
+ )
+ )
+ return "\n".join(lines)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--format", choices=("json", "markdown"), default="markdown")
+ args = parser.parse_args()
+ rows = matrix()
+ if args.format == "json":
+ print(json.dumps(rows, ensure_ascii=False, indent=2))
+ else:
+ print(as_markdown(rows))
+
+
+if __name__ == "__main__":
+ main()