vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate Chinese speech understanding and capability routing

Browse Source
This commit is contained in:
vee
2026-08-28 15:19:44 +08:00
parent 2529b4896a
commit aa12135b22
39 changed files with 6864 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/web-console/README.md
+10
View File
@@ -6,6 +6,7 @@ Web 控制台是车机端的远程自编排与同步显示界面。网页保留
- 自编排和自扩展合并为一个实时界面，登录身份固定为 `user`，不再提供模块切换。
- 网页端可输入一句话指令或点击预设模式，车机端同样可发起任务；两端任务与结果统一同步。
- 一句话输入框支持麦克风听写，优先采集 `TAKSTAR V2`；AudioWorklet 采集、浏览器端静音停录、服务端 Silero VAD、SenseVoiceSmall、幂等 TranscriptAssembler 和结构化理解层组成完整链路。只有生成 commands 的结果才回填；无关内容不覆盖输入框，uncertain 结果需人工修改后才能执行。
- 整车数字孪生位于工作区中央，点击或悬停热点可查看车窗、遮阳帘、主/副驾座椅、阅读灯、氛围灯、前大灯和空调的完整状态。
- 车端设备状态与外设能力板块均可独立隐藏或恢复，面板状态在当前浏览器会话中保留。
- USB 外设接入后同步显示 GSCU 链路、外设名称、类型、已学习能力和最近一次跨设备联动。
@@ -19,6 +20,15 @@ UI 只展示可审计的“决策摘要”和“处理轨迹”，不展示模
make dev
```
语音听写首次运行前安装运行时、官方完整精度模型和 Silero VAD：
```bash
python -m pip install -e ".[speech]"
python tools/install_sensevoice_model.py
```
录音只用于当前请求，不落盘；`raw_text`、`clean_text` 与结构化 Intent 分字段返回。远程浏览器必须通过 HTTPS 才能访问麦克风。诊断可设置后端 `SPEECH_DEBUG=1` 并用 `?speechDebug=1` 打开前端日志，默认关闭。
Android PAD/模拟器联调使用：
```powershell
apps/web-console/app.js
+35 -4
View File
@@ -7453,7 +7453,14 @@ async function executeCanCommands(canCommands, { deviceName = "车辆设备" } =
async function submitRequest(event) {
event.preventDefault();
const utterance = $("#utterance").value.trim();
const utteranceInput = $("#utterance");
const utterance = utteranceInput.value.trim();
const speechSubmission = (
utteranceInput.dataset.speechExecutionEligible === "true"
&& utteranceInput.dataset.speechValue === utterance
&& Boolean(utteranceInput.dataset.speechUtteranceId)
);
const requestProvider = speechSubmission ? "local" : state.provider;
const submittedModeProfileId = state.pendingModeProfileId;
state.pendingModeProfileId = null;
if (!utterance) {
@@ -7461,6 +7468,20 @@ async function submitRequest(event) {
showToast("请输入车舱控制请求");
return;
}
if (
utteranceInput.dataset.speechExecutionEligible === "false"
&& utteranceInput.dataset.speechValue === utterance
) {
if (submittedModeProfileId) setModeExecutionState(submittedModeProfileId, "failed");
const speechStatus = utteranceInput.dataset.speechSemanticStatus;
const message = speechStatus === "recognized_unsupported"
? "语音意图已识别，但当前车辆能力不支持执行"
: speechStatus === "incomplete"
? "语音控制参数不完整，请补充后再执行"
: "语音结果存在歧义，请修改确认后再执行";
showToast(message);
return;
}
const driverOnlyCh1 = isDriverOnlyCh1Runtime();
if (state.executionMode === "hardware"
&& state.controlTransport !== "wireless_mqtt"
@@ -7491,12 +7512,15 @@ async function submitRequest(event) {
const requestBody = {
utterance,
client_type: "web_console",
provider: state.provider,
provider: requestProvider,
conversation_id: state.conversationId,
vehicle_id: state.vehicleId || state.runtime?.default_vehicle_id,
execution_mode: state.executionMode,
control_transport: state.controlTransport,
...(state.provider === "openai" ? { reasoning_effort: state.reasoningEffort } : {}),
...(requestProvider === "openai" ? { reasoning_effort: state.reasoningEffort } : {}),
...(speechSubmission
? { speech_utterance_id: utteranceInput.dataset.speechUtteranceId }
: {}),
...(state.executionMode === "hardware"
&& state.controlTransport !== "wireless_mqtt"
&& !driverOnlyCh1
@@ -7515,7 +7539,7 @@ async function submitRequest(event) {
{ detail: phaseEvent.payload },
));
};
if (state.provider === "openai") {
if (requestProvider === "openai") {
const preview = await postControlPhase(
"/api/requests/preview",
requestBody,
@@ -7578,6 +7602,13 @@ async function submitRequest(event) {
}
renderResult(payload);
completedResult = payload;
if (speechSubmission) {
delete utteranceInput.dataset.speechUtteranceId;
delete utteranceInput.dataset.speechExecutionEligible;
delete utteranceInput.dataset.speechSemanticStatus;
delete utteranceInput.dataset.speechValue;
delete utteranceInput.dataset.speechUncertain;
}
const peripheralRoute = String(payload.orchestration_route || "")
.startsWith("nfc_peripheral");
if (state.executionMode === "hardware" && !peripheralRoute) {
apps/web-console/index.html
+9 -1
View File
@@ -11,13 +11,15 @@
<title>511工程-智能座舱</title>
<link rel="stylesheet" href="/styles.css?v=dual-seat-16" />
<link rel="stylesheet" href="/console-v2.css?v=sync-layout-41" />
<link rel="stylesheet" href="/speech-input.css?v=sensevoice-4" />
<link rel="stylesheet" href="/self-extension.css?v=nfc-toy-33" />
<script src="/vendor/lucide/0.468.0/lucide.min.js" defer></script>
<script src="/vendor/three/0.147.0/three.min.js" defer></script>
<script src="/vendor/three/0.147.0/fflate.min.js" defer></script>
<script src="/vendor/three/0.147.0/FBXLoader.js" defer></script>
<script src="/network.js?v=android-pad-2" defer></script>
<script src="/app.js?v=dual-seat-103" defer></script>
<script src="/app.js?v=dual-seat-104" defer></script>
<script src="/speech-input.js?v=sensevoice-6" defer></script>
<script type="module" src="/gesture-control.js?v=android-pad-9"></script>
<script src="/pwa.js?v=android-pad-3" defer></script>
<script src="/console-v2.js?v=sync-layout-43" defer></script>
@@ -73,6 +75,12 @@
</div>
<div class="command-composer">
<textarea id="utterance" name="utterance" maxlength="1000" placeholder="例如：有点热，把副驾驶调舒服些，再把天幕调暗">进入观影状态：空调24度、风量2档，天幕调暗，氛围灯换成暖橙色</textarea>
<button class="speech-dictation" id="speechDictationButton" type="button" aria-label="开始语音听写" aria-pressed="false" title="语音听写（自动识别说话结束）">
<i class="speech-mic-icon" data-lucide="mic" aria-hidden="true"></i>
<span class="speech-waveform" aria-hidden="true"><i></i><i></i><i></i><i></i><i></i></span>
<span class="speech-visually-hidden">开始语音听写</span>
</button>
<span class="speech-status" id="speechDictationStatus" role="status" aria-live="polite"></span>
<span class="composer-hint">Enter 执行 · Shift + Enter 换行</span>
</div>
apps/web-console/service-worker.js
+5 -2
View File
@@ -1,4 +1,4 @@
const CACHE_VERSION = "cognitive-agent-pad-v210";
const CACHE_VERSION = "cognitive-agent-pad-v216";
const SHELL_CACHE = `${CACHE_VERSION}-shell`;
const RUNTIME_CACHE = `${CACHE_VERSION}-runtime`;
const APP_SHELL = [
@@ -6,11 +6,14 @@ const APP_SHELL = [
"/index.html",
"/styles.css?v=dual-seat-16",
"/network.js?v=android-pad-2",
"/app.js?v=dual-seat-103",
"/app.js?v=dual-seat-104",
"/gesture-control.js?v=android-pad-9",
"/pwa.js?v=android-pad-3",
"/console-v2.css?v=sync-layout-41",
"/speech-input.css?v=sensevoice-4",
"/console-v2.js?v=sync-layout-43",
"/speech-input.js?v=sensevoice-6",
"/speech-audio-worklet.js?v=sensevoice-4",
"/self-extension.css?v=nfc-toy-33",
"/self-extension.js?v=nfc-toy-33",
"/assets/pets/self-extension-cat.png?v=3",
apps/web-console/speech-audio-worklet.js
+29
View File
@@ -0,0 +1,29 @@
class SpeechCaptureProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.frameSize = 2048;
this.frame = new Float32Array(this.frameSize);
this.offset = 0;
}
process(inputs) {
const input = inputs[0]?.[0];
if (!input) return true;
let inputOffset = 0;
while (inputOffset < input.length) {
const available = Math.min(this.frameSize - this.offset, input.length - inputOffset);
this.frame.set(input.subarray(inputOffset, inputOffset + available), this.offset);
this.offset += available;
inputOffset += available;
if (this.offset === this.frameSize) {
const ready = this.frame;
this.port.postMessage(ready, [ready.buffer]);
this.frame = new Float32Array(this.frameSize);
this.offset = 0;
}
}
return true;
}
}
registerProcessor("speech-capture-processor", SpeechCaptureProcessor);
apps/web-console/speech-input.css
+257
View File
@@ -0,0 +1,257 @@
.speech-visually-hidden {
width: 1px;
height: 1px;
position: absolute;
overflow: hidden;
clip: rect(0 0 0 0);
clip-path: inset(50%);
white-space: nowrap;
}
/* Keep all composer actions on one calm, optically aligned rail. */
.console-v2 .v2-request-form .speech-dictation {
width: 36px;
height: 36px;
min-height: 36px;
position: absolute;
top: 50%;
right: 56px;
bottom: auto;
z-index: 13;
display: grid;
place-items: center;
margin: 0;
padding: 0;
color: #284647;
background: transparent;
border: 0;
border-radius: 50%;
box-shadow: none;
cursor: pointer;
transform: translateY(-50%);
transition: color 160ms ease, background 160ms ease, box-shadow 160ms ease,
transform 160ms ease;
}
.console-v2 .v2-request-form .speech-dictation:hover:not(:disabled),
.console-v2 .v2-request-form .speech-dictation:focus-visible {
color: #08766e;
background: #eaf5f3;
box-shadow: 0 0 0 3px rgba(8, 118, 110, 0.08);
transform: translateY(-50%) scale(1.05);
}
.console-v2 .v2-request-form .speech-dictation:focus-visible {
outline: 2px solid rgba(8, 118, 110, 0.48);
outline-offset: 2px;
}
.console-v2 .v2-request-form .speech-dictation:disabled {
color: #9aacaa;
cursor: not-allowed;
opacity: 0.56;
}
.console-v2 .v2-request-form .speech-dictation .speech-mic-icon {
width: 18px;
height: 18px;
stroke-width: 2;
}
.console-v2 .v2-request-form .speech-waveform {
height: 16px;
display: none;
align-items: center;
justify-content: center;
gap: 2px;
}
.console-v2 .v2-request-form .speech-waveform i {
width: 2px;
height: 5px;
display: block;
background: currentColor;
border-radius: 999px;
}
.console-v2 .v2-request-form .speech-waveform i:nth-child(2),
.console-v2 .v2-request-form .speech-waveform i:nth-child(4) {
height: 10px;
}
.console-v2 .v2-request-form .speech-waveform i:nth-child(3) {
height: 15px;
}
.console-v2 .v2-request-form .speech-dictation.is-listening,
.console-v2 .v2-request-form .speech-dictation.is-listening:hover:not(:disabled),
.console-v2 .v2-request-form .speech-dictation.is-listening:focus-visible {
color: #fff;
background: #132c2c;
box-shadow: 0 0 0 5px rgba(19, 44, 44, 0.09), 0 7px 18px rgba(17, 47, 46, 0.2);
}
.console-v2 .v2-request-form .speech-dictation.is-listening .speech-mic-icon,
.console-v2 .v2-request-form .speech-dictation.is-processing .speech-mic-icon {
display: none;
}
.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform,
.console-v2 .v2-request-form .speech-dictation.is-processing .speech-waveform {
display: flex;
}
.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform i {
animation: speech-wave 760ms ease-in-out infinite alternate;
}
.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform i:nth-child(2) {
animation-delay: -420ms;
}
.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform i:nth-child(3) {
animation-delay: -180ms;
}
.console-v2 .v2-request-form .speech-dictation.is-listening .speech-waveform i:nth-child(4) {
animation-delay: -560ms;
}
.console-v2 .v2-request-form .speech-dictation.is-processing {
color: #08766e;
background: #e5f3f1;
}
.console-v2 .v2-request-form .speech-dictation.is-processing .speech-waveform {
animation: speech-processing-breathe 900ms ease-in-out infinite alternate;
}
.console-v2 .v2-request-form .speech-status {
min-height: 16px;
position: absolute;
right: 140px;
bottom: 10px;
left: 194px;
z-index: 12;
overflow: hidden;
color: #5f7474;
font-size: 10px;
font-weight: 650;
line-height: 16px;
letter-spacing: 0.01em;
text-overflow: ellipsis;
white-space: nowrap;
pointer-events: none;
}
.console-v2 .v2-request-form .command-composer[data-speech-state="listening"] {
border-color: rgba(19, 76, 73, 0.36);
box-shadow: 0 12px 35px rgba(25, 72, 70, 0.1), inset 0 0 0 1px rgba(19, 76, 73, 0.08);
}
.console-v2 .v2-request-form .command-composer[data-speech-state="listening"] .speech-status {
color: #183e3d;
}
.console-v2 .v2-request-form .command-composer[data-speech-state="processing"] .speech-status {
color: #39706d;
}
.console-v2 .v2-request-form .command-composer[data-speech-state="success"] .speech-status {
color: #08766e;
}
.console-v2 .v2-request-form #utterance[data-speech-uncertain="true"] {
background: linear-gradient(90deg, rgba(201, 132, 31, 0.055), transparent 42%);
box-shadow: inset 3px 0 0 rgba(201, 132, 31, 0.58);
}
.console-v2 .v2-request-form .command-composer[data-speech-state="error"] .speech-status {
color: #b54b42;
}
.console-v2 .v2-request-form .command-composer:not([data-speech-state="idle"]) .composer-hint {
opacity: 0;
}
.console-v2 .v2-request-form #utterance,
.console-v2.v2-interactive-sync .v2-request-form #utterance,
.console-v2.v2-passive-display.v2-interactive-sync .v2-request-form #utterance {
padding-right: 142px;
}
.console-v2 .v2-request-form .context-reset,
.console-v2.v2-interactive-sync .v2-request-form .context-reset,
.console-v2.v2-passive-display.v2-interactive-sync .v2-request-form .context-reset {
width: 30px;
height: 30px;
top: 50%;
right: 98px;
bottom: auto;
color: #75908e;
border-radius: 50%;
transform: translateY(-50%);
}
.console-v2 .v2-request-form .context-reset:hover,
.console-v2 .v2-request-form .context-reset:focus-visible {
color: #08766e;
background: #edf6f4;
}
.console-v2 .v2-request-form .composer-hint,
.console-v2.v2-interactive-sync .v2-request-form .composer-hint,
.console-v2.v2-passive-display.v2-interactive-sync .v2-request-form .composer-hint {
right: 140px;
}
@keyframes speech-wave {
from { transform: scaleY(0.42); }
to { transform: scaleY(1); }
}
@keyframes speech-processing-breathe {
from { opacity: 0.42; transform: scaleX(0.78); }
to { opacity: 1; transform: scaleX(1); }
}
@media (max-width: 720px) {
.console-v2 .v2-request-form .speech-status {
top: 8px;
right: 140px;
bottom: auto;
left: 14px;
font-size: 9px;
}
.console-v2 .v2-request-form .command-composer:not([data-speech-state="idle"]) #utterance {
padding-top: 28px;
}
}
@media (max-width: 460px) {
.console-v2 .v2-request-form #utterance,
.console-v2.v2-interactive-sync .v2-request-form #utterance,
.console-v2.v2-passive-display.v2-interactive-sync .v2-request-form #utterance {
padding-right: 132px;
}
.console-v2 .v2-request-form .speech-dictation {
right: 52px;
}
.console-v2 .v2-request-form .context-reset,
.console-v2.v2-interactive-sync .v2-request-form .context-reset,
.console-v2.v2-passive-display.v2-interactive-sync .v2-request-form .context-reset {
right: 92px;
}
}
@media (prefers-reduced-motion: reduce) {
.console-v2 .v2-request-form .speech-dictation,
.console-v2 .v2-request-form .speech-waveform,
.console-v2 .v2-request-form .speech-waveform i {
animation: none !important;
transition: none;
}
}
apps/web-console/speech-input.js
+527
View File
@@ -0,0 +1,527 @@
(() => {
"use strict";
const TARGET_SAMPLE_RATE = 16000;
const MAX_RECORDING_MS = 28000;
const NO_SPEECH_TIMEOUT_MS = 8000;
const ENDPOINT_SILENCE_MS = 1050;
const STATUS_RESET_MS = 5000;
const MICROPHONE_CONNECT_TIMEOUT_MS = 12000;
const PREFERRED_MICROPHONE = /TAKSTAR\s*V2/i;
const DEBUG = new URLSearchParams(window.location.search).get("speechDebug") === "1"
|| window.localStorage.getItem("speechDebug") === "1";
const speech = {
phase: "idle",
backendReady: false,
stream: null,
context: null,
contextPromise: null,
workletReady: false,
workletFailed: false,
source: null,
captureNode: null,
silentGain: null,
chunks: [],
totalSamples: 0,
sampleRate: 0,
startedAt: 0,
speechStartSample: null,
lastVoiceSample: null,
consecutiveVoiceSamples: 0,
noiseFloor: 0.003,
statusTimer: null,
generation: 0,
utteranceId: "",
finishPromise: null,
applyingResult: false,
lastResult: null,
};
const elements = () => ({
button: document.querySelector("#speechDictationButton"),
composer: document.querySelector("#speechDictationButton")?.closest(".command-composer"),
input: document.querySelector("#utterance"),
status: document.querySelector("#speechDictationStatus"),
});
function debug(stage, details = {}) {
if (!DEBUG) return;
console.debug("[speech]", { utteranceId: speech.utteranceId, stage, ...details });
}
function announce(message) {
const notify = window.CognitiveAgentConsole?.notify;
if (typeof notify === "function") notify(message);
}
function setPhase(phase, message = "") {
speech.phase = phase;
const { button, composer, status } = elements();
if (!button || !composer || !status) return;
window.clearTimeout(speech.statusTimer);
composer.dataset.speechState = phase;
status.textContent = message;
button.classList.toggle("is-listening", phase === "listening");
button.classList.toggle("is-processing", phase === "processing");
button.disabled = phase === "processing" || (!speech.backendReady && phase !== "listening");
button.setAttribute("aria-pressed", String(phase === "listening"));
button.setAttribute(
"aria-label",
phase === "listening" ? "结束语音听写" : phase === "processing" ? "正在转写语音" : "开始语音听写",
);
button.title = phase === "listening"
? "点击结束听写"
: speech.backendReady
? "语音听写（自动识别说话结束）"
: "语音识别模型未就绪";
if (["success", "error"].includes(phase)) {
speech.statusTimer = window.setTimeout(() => setPhase("idle"), STATUS_RESET_MS);
}
}
function microphoneErrorMessage(error) {
switch (error?.name) {
case "NotAllowedError":
return "未获得麦克风权限，请在浏览器地址栏允许访问麦克风。";
case "NotFoundError":
return "没有找到可用麦克风，请检查 TAKSTAR V2 接收器。";
case "NotReadableError":
return "麦克风正被其他程序占用，请关闭占用程序后重试。";
case "OverconstrainedError":
return "TAKSTAR V2 当前不可用，请重新连接接收器。";
case "SecurityError":
return "当前页面不允许访问麦克风，请使用 HTTPS 或本机地址。";
case "TimeoutError":
return "等待麦克风授权超时，请允许访问后重试。";
default:
return "无法启动麦克风，请检查设备连接和浏览器权限。";
}
}
async function getUserMediaWithTimeout(constraints) {
let expired = false;
let timer = 0;
const request = navigator.mediaDevices.getUserMedia(constraints);
const timeout = new Promise((_, reject) => {
timer = window.setTimeout(() => {
expired = true;
reject(new DOMException("microphone permission timed out", "TimeoutError"));
}, MICROPHONE_CONNECT_TIMEOUT_MS);
});
request.then((lateStream) => {
if (expired) lateStream.getTracks().forEach((track) => track.stop());
}).catch(() => {});
try {
return await Promise.race([request, timeout]);
} finally {
window.clearTimeout(timer);
}
}
async function preferredMicrophoneStream() {
const constraints = {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
};
let stream = await getUserMediaWithTimeout({ audio: constraints, video: false });
const currentTrack = stream.getAudioTracks()[0];
const devices = await navigator.mediaDevices.enumerateDevices();
const preferred = devices.find(
(device) => device.kind === "audioinput" && PREFERRED_MICROPHONE.test(device.label),
);
if (preferred && preferred.deviceId !== currentTrack?.getSettings?.().deviceId) {
stream.getTracks().forEach((track) => track.stop());
stream = await getUserMediaWithTimeout({
audio: { ...constraints, deviceId: { exact: preferred.deviceId } },
video: false,
});
}
return stream;
}
async function ensureAudioContext() {
if (speech.context && speech.context.state !== "closed") {
await speech.context.resume();
return speech.context;
}
if (speech.contextPromise) return speech.contextPromise;
speech.contextPromise = (async () => {
const AudioContext = window.AudioContext || window.webkitAudioContext;
const context = new AudioContext({ latencyHint: "interactive" });
await context.resume();
if (context.audioWorklet && window.AudioWorkletNode) {
try {
await context.audioWorklet.addModule("/speech-audio-worklet.js?v=sensevoice-4");
speech.workletReady = true;
} catch (error) {
speech.workletFailed = true;
debug("audio_worklet_fallback", { message: error?.message || String(error) });
}
} else {
speech.workletFailed = true;
}
speech.context = context;
return context;
})();
try {
return await speech.contextPromise;
} finally {
speech.contextPromise = null;
}
}
function audioRms(samples) {
let sum = 0;
for (let index = 0; index < samples.length; index += 1) sum += samples[index] * samples[index];
return Math.sqrt(sum / Math.max(1, samples.length));
}
function handleAudio(chunk, generation) {
if (speech.phase !== "listening" || generation !== speech.generation) return;
speech.chunks.push(chunk);
speech.totalSamples += chunk.length;
const elapsed = performance.now() - speech.startedAt;
const rms = audioRms(chunk);
if (elapsed < 700) speech.noiseFloor = (speech.noiseFloor * 0.82) + (rms * 0.18);
const voiceThreshold = Math.max(0.009, Math.min(0.035, speech.noiseFloor * 2.8));
if (rms >= voiceThreshold) {
speech.consecutiveVoiceSamples += chunk.length;
if (speech.consecutiveVoiceSamples >= speech.sampleRate * 0.09) {
if (speech.speechStartSample === null) {
speech.speechStartSample = Math.max(0, speech.totalSamples - (speech.sampleRate * 0.35));
setPhase("listening", "听到声音了，请继续说…");
debug("speech_started", { sample: speech.speechStartSample, voiceThreshold });
}
speech.lastVoiceSample = speech.totalSamples;
}
} else {
speech.consecutiveVoiceSamples = 0;
}
if (
speech.lastVoiceSample !== null
&& speech.totalSamples - speech.lastVoiceSample >= speech.sampleRate * (ENDPOINT_SILENCE_MS / 1000)
) {
void finishDictation();
} else if (speech.speechStartSample === null && elapsed >= NO_SPEECH_TIMEOUT_MS) {
void cancelDictation("未检测到说话声，请靠近麦克风重试。");
} else if (elapsed >= MAX_RECORDING_MS) {
void finishDictation();
}
}
function connectCapture(context, generation) {
speech.source = context.createMediaStreamSource(speech.stream);
speech.silentGain = context.createGain();
speech.silentGain.gain.value = 0;
if (speech.workletReady && !speech.workletFailed) {
speech.captureNode = new AudioWorkletNode(context, "speech-capture-processor", {
numberOfInputs: 1,
numberOfOutputs: 1,
channelCount: 1,
});
speech.captureNode.port.onmessage = (event) => {
handleAudio(new Float32Array(event.data), generation);
};
debug("capture_connected", { implementation: "AudioWorklet" });
} else {
speech.captureNode = context.createScriptProcessor(2048, 1, 1);
speech.captureNode.onaudioprocess = (event) => {
handleAudio(new Float32Array(event.inputBuffer.getChannelData(0)), generation);
};
debug("capture_connected", { implementation: "ScriptProcessor-fallback" });
}
speech.source.connect(speech.captureNode);
speech.captureNode.connect(speech.silentGain);
speech.silentGain.connect(context.destination);
}
async function startDictation() {
if (speech.phase !== "idle" && !["success", "error"].includes(speech.phase)) return;
if (!speech.backendReady) {
setPhase("error", "语音识别模型未就绪");
return;
}
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
setPhase("error", "请使用 HTTPS 或 127.0.0.1 开启语音听写");
announce("当前页面不支持安全的麦克风访问");
return;
}
const generation = speech.generation + 1;
speech.generation = generation;
speech.utteranceId = `web-${crypto.randomUUID?.() || `${Date.now()}-${generation}`}`;
try {
setPhase("processing", "正在连接麦克风…");
speech.stream = await preferredMicrophoneStream();
if (generation !== speech.generation) return;
const context = await ensureAudioContext();
speech.sampleRate = context.sampleRate;
speech.chunks = [];
speech.totalSamples = 0;
speech.startedAt = performance.now();
speech.speechStartSample = null;
speech.lastVoiceSample = null;
speech.consecutiveVoiceSamples = 0;
speech.noiseFloor = 0.003;
connectCapture(context, generation);
const trackLabel = speech.stream.getAudioTracks()[0]?.label || "默认麦克风";
setPhase("listening", `正在听写 · ${trackLabel}`);
debug("utterance_started", { sampleRate: speech.sampleRate, trackLabel });
} catch (error) {
await releaseMicrophone();
const message = microphoneErrorMessage(error);
setPhase("error", message);
announce(message);
}
}
async function releaseMicrophone({ closeContext = false } = {}) {
if (speech.captureNode?.port) speech.captureNode.port.onmessage = null;
if (speech.captureNode && "onaudioprocess" in speech.captureNode) {
speech.captureNode.onaudioprocess = null;
}
for (const node of [speech.source, speech.captureNode, speech.silentGain]) {
try { node?.disconnect(); } catch (_error) { /* already disconnected */ }
}
speech.stream?.getTracks().forEach((track) => track.stop());
speech.stream = null;
speech.source = null;
speech.captureNode = null;
speech.silentGain = null;
if (closeContext && speech.context && speech.context.state !== "closed") {
try { await speech.context.close(); } catch (_error) { /* page is closing */ }
speech.context = null;
speech.workletReady = false;
}
}
function concatenateChunks() {
const combined = new Float32Array(speech.totalSamples);
let offset = 0;
for (const chunk of speech.chunks) {
combined.set(chunk, offset);
offset += chunk.length;
}
const start = Math.max(0, Math.floor(speech.speechStartSample || 0));
const end = Math.min(
combined.length,
Math.ceil((speech.lastVoiceSample || combined.length) + (speech.sampleRate * 0.45)),
);
return combined.slice(start, end);
}
function downsample(samples, inputRate) {
if (inputRate === TARGET_SAMPLE_RATE) return samples;
const ratio = inputRate / TARGET_SAMPLE_RATE;
const output = new Float32Array(Math.max(1, Math.floor(samples.length / ratio)));
for (let outputIndex = 0; outputIndex < output.length; outputIndex += 1) {
const start = Math.floor(outputIndex * ratio);
const end = Math.max(start + 1, Math.min(samples.length, Math.floor((outputIndex + 1) * ratio)));
let sum = 0;
for (let inputIndex = start; inputIndex < end; inputIndex += 1) sum += samples[inputIndex];
output[outputIndex] = sum / (end - start);
}
return output;
}
function encodePcm16Wav(samples) {
const buffer = new ArrayBuffer(44 + (samples.length * 2));
const view = new DataView(buffer);
const writeAscii = (offset, value) => {
for (let index = 0; index < value.length; index += 1) view.setUint8(offset + index, value.charCodeAt(index));
};
writeAscii(0, "RIFF");
view.setUint32(4, 36 + (samples.length * 2), true);
writeAscii(8, "WAVE");
writeAscii(12, "fmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, 1, true);
view.setUint32(24, TARGET_SAMPLE_RATE, true);
view.setUint32(28, TARGET_SAMPLE_RATE * 2, true);
view.setUint16(32, 2, true);
view.setUint16(34, 16, true);
writeAscii(36, "data");
view.setUint32(40, samples.length * 2, true);
for (let index = 0; index < samples.length; index += 1) {
const sample = Math.max(-1, Math.min(1, samples[index]));
view.setInt16(44 + (index * 2), sample < 0 ? sample * 32768 : sample * 32767, true);
}
return buffer;
}
async function transcribe(wav, utteranceId) {
const controller = new AbortController();
const timeout = window.setTimeout(() => controller.abort(), 45000);
try {
const response = await fetch("/api/speech/transcribe", {
method: "POST",
credentials: "same-origin",
cache: "no-store",
headers: {
Accept: "application/json",
"Content-Type": "audio/wav",
"X-Speech-Utterance-Id": utteranceId,
},
body: wav,
signal: controller.signal,
});
const payload = await response.json().catch(() => ({}));
if (!response.ok) throw new Error(payload.message || `语音转写失败（HTTP ${response.status}）`);
debug("server_result", payload);
return payload;
} finally {
window.clearTimeout(timeout);
}
}
function applyResult(result) {
const { input } = elements();
if (!input) throw new Error("没有找到一句话编排输入框");
const cleanText = String(result.clean_text || result.text || "").trim();
const commands = Array.isArray(result.commands) ? result.commands : [];
if (!cleanText || commands.length === 0) {
window.dispatchEvent(new CustomEvent("cognitive-agent:speech-result", { detail: result }));
return false;
}
speech.applyingResult = true;
input.value = cleanText.slice(0, input.maxLength || 1000);
input.dataset.speechUncertain = String(result.uncertain === true);
input.dataset.speechExecutionEligible = String(result.execution_eligible === true);
input.dataset.speechSemanticStatus = String(result.semantic_status || "");
input.dataset.speechValue = input.value.trim();
input.dataset.speechUtteranceId = String(result.utterance_id || "");
input.dispatchEvent(new Event("input", { bubbles: true }));
speech.applyingResult = false;
input.focus({ preventScroll: true });
window.dispatchEvent(new CustomEvent("cognitive-agent:speech-result", { detail: result }));
return true;
}
async function finishDictation() {
if (speech.finishPromise) return speech.finishPromise;
if (speech.phase !== "listening") return undefined;
speech.finishPromise = (async () => {
setPhase("processing", "说话结束，正在识别并理解指令…");
const hadSpeech = speech.speechStartSample !== null;
const samples = hadSpeech ? concatenateChunks() : null;
const inputRate = speech.sampleRate;
const utteranceId = speech.utteranceId;
await releaseMicrophone();
speech.chunks = [];
if (!hadSpeech || !samples?.length) {
setPhase("error", "未检测到说话声，请重试");
return;
}
try {
const wav = encodePcm16Wav(downsample(samples, inputRate));
const result = await transcribe(wav, utteranceId);
speech.lastResult = result;
const applied = applyResult(result);
const latency = Math.round(result.timings?.total_ms ?? result.inference_ms ?? 0);
if (!applied) {
setPhase("success", `未形成可执行控制指令 · ${latency} ms`);
announce("已保留原始转录，但没有激活控制指令");
} else if (result.semantic_status === "recognized_unsupported") {
setPhase("success", `已识别，但当前车辆能力不支持 · ${latency} ms`);
announce("已完整保留用户语义，但当前车辆能力不支持执行");
} else if (result.semantic_status === "incomplete") {
setPhase("success", `已识别，但控制参数不完整 · ${latency} ms`);
announce("语音中的关键控制参数不完整，当前不会执行");
} else if (result.uncertain === true) {
setPhase("success", `已转写，但存在歧义，请修改确认 · ${latency} ms`);
announce("语音结果需要人工确认，当前不会直接执行");
} else {
setPhase("success", `已理解为 ${result.commands.length} 条指令 · ${latency} ms`);
announce("语音已转换为理解式控制指令");
}
} catch (error) {
const message = error?.name === "AbortError" ? "语音转写超时，请重试。" : error.message;
setPhase("error", message || "语音转写失败，请重试");
announce(message || "语音转写失败");
}
})();
try {
return await speech.finishPromise;
} finally {
speech.finishPromise = null;
}
}
async function cancelDictation(message = "已取消语音听写") {
if (!speech.stream && speech.phase === "idle") return;
speech.phase = "processing";
speech.generation += 1;
await releaseMicrophone();
speech.chunks = [];
setPhase("error", message);
}
async function loadSpeechStatus() {
const { button } = elements();
if (!button) return;
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
button.disabled = true;
setPhase("error", "语音听写需要 HTTPS 或本机地址");
return;
}
try {
const response = await fetch("/api/speech/status", {
credentials: "same-origin",
cache: "no-store",
headers: { Accept: "application/json" },
});
const status = await response.json();
speech.backendReady = response.ok && status.enabled === true;
debug("backend_status", status);
setPhase("idle");
if (!speech.backendReady) {
button.disabled = true;
button.title = "SenseVoiceSmall 模型未部署";
}
} catch (_error) {
speech.backendReady = false;
button.disabled = true;
button.title = "无法连接语音识别服务";
}
}
function clearSpeechGuardOnManualEdit(event) {
if (speech.applyingResult || !event.isTrusted) return;
const { input } = elements();
if (!input) return;
delete input.dataset.speechUncertain;
delete input.dataset.speechExecutionEligible;
delete input.dataset.speechSemanticStatus;
delete input.dataset.speechValue;
delete input.dataset.speechUtteranceId;
}
function initialize() {
const { button, input } = elements();
if (!button) return;
button.addEventListener("click", () => {
if (speech.phase === "listening") void finishDictation();
else void startDictation();
});
input?.addEventListener("input", clearSpeechGuardOnManualEdit);
document.addEventListener("keydown", (event) => {
if (event.key === "Escape" && speech.phase === "listening") {
event.preventDefault();
void cancelDictation();
}
});
document.addEventListener("visibilitychange", () => {
if (document.hidden && speech.phase === "listening") void cancelDictation();
});
window.addEventListener("pagehide", () => { void releaseMicrophone({ closeContext: true }); });
void loadSpeechStatus();
}
window.CognitiveAgentSpeech = {
get lastResult() { return speech.lastResult; },
get debugEnabled() { return DEBUG; },
};
document.addEventListener("DOMContentLoaded", initialize);
})();