minghanchen
538 lines
21 KiB
JavaScript
538 lines
21 KiB
JavaScript
(() => {
|
||
"use strict";
|
||
|
||
const TARGET_SAMPLE_RATE = 16000;
|
||
const MAX_RECORDING_MS = 28000;
|
||
const NO_SPEECH_TIMEOUT_MS = 8000;
|
||
const ENDPOINT_SILENCE_MS = 1050;
|
||
const STATUS_RESET_MS = 5000;
|
||
const MICROPHONE_CONNECT_TIMEOUT_MS = 12000;
|
||
const PREFERRED_MICROPHONE = /TAKSTAR\s*V2/i;
|
||
const DEBUG = new URLSearchParams(window.location.search).get("speechDebug") === "1"
|
||
|| window.localStorage.getItem("speechDebug") === "1";
|
||
const speech = {
|
||
phase: "idle",
|
||
backendReady: false,
|
||
stream: null,
|
||
context: null,
|
||
contextPromise: null,
|
||
workletReady: false,
|
||
workletFailed: false,
|
||
source: null,
|
||
captureNode: null,
|
||
silentGain: null,
|
||
chunks: [],
|
||
totalSamples: 0,
|
||
sampleRate: 0,
|
||
startedAt: 0,
|
||
speechStartSample: null,
|
||
lastVoiceSample: null,
|
||
consecutiveVoiceSamples: 0,
|
||
noiseFloor: 0.003,
|
||
statusTimer: null,
|
||
generation: 0,
|
||
utteranceId: "",
|
||
finishPromise: null,
|
||
applyingResult: false,
|
||
lastResult: null,
|
||
};
|
||
|
||
const elements = () => ({
|
||
button: document.querySelector("#speechDictationButton"),
|
||
composer: document.querySelector("#speechDictationButton")?.closest(".command-composer"),
|
||
input: document.querySelector("#utterance"),
|
||
status: document.querySelector("#speechDictationStatus"),
|
||
});
|
||
|
||
function debug(stage, details = {}) {
|
||
if (!DEBUG) return;
|
||
console.debug("[speech]", { utteranceId: speech.utteranceId, stage, ...details });
|
||
}
|
||
|
||
function announce(message) {
|
||
const notify = window.CognitiveAgentConsole?.notify;
|
||
if (typeof notify === "function") notify(message);
|
||
}
|
||
|
||
function setPhase(phase, message = "") {
|
||
speech.phase = phase;
|
||
const { button, composer, status } = elements();
|
||
if (!button || !composer || !status) return;
|
||
window.clearTimeout(speech.statusTimer);
|
||
composer.dataset.speechState = phase;
|
||
status.textContent = message;
|
||
button.classList.toggle("is-listening", phase === "listening");
|
||
button.classList.toggle("is-processing", phase === "processing");
|
||
button.disabled = phase === "processing" || (!speech.backendReady && phase !== "listening");
|
||
button.setAttribute("aria-pressed", String(phase === "listening"));
|
||
button.setAttribute(
|
||
"aria-label",
|
||
phase === "listening" ? "结束语音听写" : phase === "processing" ? "正在转写语音" : "开始语音听写",
|
||
);
|
||
button.title = phase === "listening"
|
||
? "点击结束听写"
|
||
: speech.backendReady
|
||
? "语音听写（自动识别说话结束）"
|
||
: "语音识别模型未就绪";
|
||
if (["success", "error"].includes(phase)) {
|
||
speech.statusTimer = window.setTimeout(() => setPhase("idle"), STATUS_RESET_MS);
|
||
}
|
||
}
|
||
|
||
function microphoneErrorMessage(error) {
|
||
switch (error?.name) {
|
||
case "NotAllowedError":
|
||
return "未获得麦克风权限，请在浏览器地址栏允许访问麦克风。";
|
||
case "NotFoundError":
|
||
return "没有找到可用麦克风，请检查 TAKSTAR V2 接收器。";
|
||
case "NotReadableError":
|
||
return "麦克风正被其他程序占用，请关闭占用程序后重试。";
|
||
case "OverconstrainedError":
|
||
return "TAKSTAR V2 当前不可用，请重新连接接收器。";
|
||
case "SecurityError":
|
||
return "当前页面不允许访问麦克风，请使用 HTTPS 或本机地址。";
|
||
case "TimeoutError":
|
||
return "等待麦克风授权超时，请允许访问后重试。";
|
||
default:
|
||
return "无法启动麦克风，请检查设备连接和浏览器权限。";
|
||
}
|
||
}
|
||
|
||
async function getUserMediaWithTimeout(constraints) {
|
||
let expired = false;
|
||
let timer = 0;
|
||
const request = navigator.mediaDevices.getUserMedia(constraints);
|
||
const timeout = new Promise((_, reject) => {
|
||
timer = window.setTimeout(() => {
|
||
expired = true;
|
||
reject(new DOMException("microphone permission timed out", "TimeoutError"));
|
||
}, MICROPHONE_CONNECT_TIMEOUT_MS);
|
||
});
|
||
request.then((lateStream) => {
|
||
if (expired) lateStream.getTracks().forEach((track) => track.stop());
|
||
}).catch(() => {});
|
||
try {
|
||
return await Promise.race([request, timeout]);
|
||
} finally {
|
||
window.clearTimeout(timer);
|
||
}
|
||
}
|
||
|
||
async function preferredMicrophoneStream() {
|
||
const constraints = {
|
||
channelCount: 1,
|
||
echoCancellation: true,
|
||
noiseSuppression: true,
|
||
autoGainControl: true,
|
||
};
|
||
let stream = await getUserMediaWithTimeout({ audio: constraints, video: false });
|
||
const currentTrack = stream.getAudioTracks()[0];
|
||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||
const preferred = devices.find(
|
||
(device) => device.kind === "audioinput" && PREFERRED_MICROPHONE.test(device.label),
|
||
);
|
||
if (preferred && preferred.deviceId !== currentTrack?.getSettings?.().deviceId) {
|
||
stream.getTracks().forEach((track) => track.stop());
|
||
stream = await getUserMediaWithTimeout({
|
||
audio: { ...constraints, deviceId: { exact: preferred.deviceId } },
|
||
video: false,
|
||
});
|
||
}
|
||
return stream;
|
||
}
|
||
|
||
async function ensureAudioContext() {
|
||
if (speech.context && speech.context.state !== "closed") {
|
||
await speech.context.resume();
|
||
return speech.context;
|
||
}
|
||
if (speech.contextPromise) return speech.contextPromise;
|
||
speech.contextPromise = (async () => {
|
||
const AudioContext = window.AudioContext || window.webkitAudioContext;
|
||
const context = new AudioContext({ latencyHint: "interactive" });
|
||
await context.resume();
|
||
if (context.audioWorklet && window.AudioWorkletNode) {
|
||
try {
|
||
await context.audioWorklet.addModule("/speech-audio-worklet.js?v=sensevoice-4");
|
||
speech.workletReady = true;
|
||
} catch (error) {
|
||
speech.workletFailed = true;
|
||
debug("audio_worklet_fallback", { message: error?.message || String(error) });
|
||
}
|
||
} else {
|
||
speech.workletFailed = true;
|
||
}
|
||
speech.context = context;
|
||
return context;
|
||
})();
|
||
try {
|
||
return await speech.contextPromise;
|
||
} finally {
|
||
speech.contextPromise = null;
|
||
}
|
||
}
|
||
|
||
function audioRms(samples) {
|
||
let sum = 0;
|
||
for (let index = 0; index < samples.length; index += 1) sum += samples[index] * samples[index];
|
||
return Math.sqrt(sum / Math.max(1, samples.length));
|
||
}
|
||
|
||
function handleAudio(chunk, generation) {
|
||
if (speech.phase !== "listening" || generation !== speech.generation) return;
|
||
speech.chunks.push(chunk);
|
||
speech.totalSamples += chunk.length;
|
||
const elapsed = performance.now() - speech.startedAt;
|
||
const rms = audioRms(chunk);
|
||
if (elapsed < 700) speech.noiseFloor = (speech.noiseFloor * 0.82) + (rms * 0.18);
|
||
const voiceThreshold = Math.max(0.009, Math.min(0.035, speech.noiseFloor * 2.8));
|
||
if (rms >= voiceThreshold) {
|
||
speech.consecutiveVoiceSamples += chunk.length;
|
||
if (speech.consecutiveVoiceSamples >= speech.sampleRate * 0.09) {
|
||
if (speech.speechStartSample === null) {
|
||
speech.speechStartSample = Math.max(0, speech.totalSamples - (speech.sampleRate * 0.35));
|
||
setPhase("listening", "听到声音了，请继续说…");
|
||
debug("speech_started", { sample: speech.speechStartSample, voiceThreshold });
|
||
}
|
||
speech.lastVoiceSample = speech.totalSamples;
|
||
}
|
||
} else {
|
||
speech.consecutiveVoiceSamples = 0;
|
||
}
|
||
if (
|
||
speech.lastVoiceSample !== null
|
||
&& speech.totalSamples - speech.lastVoiceSample >= speech.sampleRate * (ENDPOINT_SILENCE_MS / 1000)
|
||
) {
|
||
void finishDictation();
|
||
} else if (speech.speechStartSample === null && elapsed >= NO_SPEECH_TIMEOUT_MS) {
|
||
void cancelDictation("未检测到说话声，请靠近麦克风重试。");
|
||
} else if (elapsed >= MAX_RECORDING_MS) {
|
||
void finishDictation();
|
||
}
|
||
}
|
||
|
||
function connectCapture(context, generation) {
|
||
speech.source = context.createMediaStreamSource(speech.stream);
|
||
speech.silentGain = context.createGain();
|
||
speech.silentGain.gain.value = 0;
|
||
if (speech.workletReady && !speech.workletFailed) {
|
||
speech.captureNode = new AudioWorkletNode(context, "speech-capture-processor", {
|
||
numberOfInputs: 1,
|
||
numberOfOutputs: 1,
|
||
channelCount: 1,
|
||
});
|
||
speech.captureNode.port.onmessage = (event) => {
|
||
handleAudio(new Float32Array(event.data), generation);
|
||
};
|
||
debug("capture_connected", { implementation: "AudioWorklet" });
|
||
} else {
|
||
speech.captureNode = context.createScriptProcessor(2048, 1, 1);
|
||
speech.captureNode.onaudioprocess = (event) => {
|
||
handleAudio(new Float32Array(event.inputBuffer.getChannelData(0)), generation);
|
||
};
|
||
debug("capture_connected", { implementation: "ScriptProcessor-fallback" });
|
||
}
|
||
speech.source.connect(speech.captureNode);
|
||
speech.captureNode.connect(speech.silentGain);
|
||
speech.silentGain.connect(context.destination);
|
||
}
|
||
|
||
async function startDictation() {
|
||
if (speech.phase !== "idle" && !["success", "error"].includes(speech.phase)) return;
|
||
if (!speech.backendReady) {
|
||
setPhase("error", "语音识别模型未就绪");
|
||
return;
|
||
}
|
||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||
setPhase("error", "请使用 HTTPS 或 127.0.0.1 开启语音听写");
|
||
announce("当前页面不支持安全的麦克风访问");
|
||
return;
|
||
}
|
||
const generation = speech.generation + 1;
|
||
speech.generation = generation;
|
||
speech.utteranceId = `web-${crypto.randomUUID?.() || `${Date.now()}-${generation}`}`;
|
||
try {
|
||
setPhase("processing", "正在连接麦克风…");
|
||
speech.stream = await preferredMicrophoneStream();
|
||
if (generation !== speech.generation) return;
|
||
const context = await ensureAudioContext();
|
||
speech.sampleRate = context.sampleRate;
|
||
speech.chunks = [];
|
||
speech.totalSamples = 0;
|
||
speech.startedAt = performance.now();
|
||
speech.speechStartSample = null;
|
||
speech.lastVoiceSample = null;
|
||
speech.consecutiveVoiceSamples = 0;
|
||
speech.noiseFloor = 0.003;
|
||
connectCapture(context, generation);
|
||
const trackLabel = speech.stream.getAudioTracks()[0]?.label || "默认麦克风";
|
||
setPhase("listening", `正在听写 · ${trackLabel}`);
|
||
debug("utterance_started", { sampleRate: speech.sampleRate, trackLabel });
|
||
} catch (error) {
|
||
await releaseMicrophone();
|
||
const message = microphoneErrorMessage(error);
|
||
setPhase("error", message);
|
||
announce(message);
|
||
}
|
||
}
|
||
|
||
async function releaseMicrophone({ closeContext = false } = {}) {
|
||
if (speech.captureNode?.port) speech.captureNode.port.onmessage = null;
|
||
if (speech.captureNode && "onaudioprocess" in speech.captureNode) {
|
||
speech.captureNode.onaudioprocess = null;
|
||
}
|
||
for (const node of [speech.source, speech.captureNode, speech.silentGain]) {
|
||
try { node?.disconnect(); } catch (_error) { /* already disconnected */ }
|
||
}
|
||
speech.stream?.getTracks().forEach((track) => track.stop());
|
||
speech.stream = null;
|
||
speech.source = null;
|
||
speech.captureNode = null;
|
||
speech.silentGain = null;
|
||
if (closeContext && speech.context && speech.context.state !== "closed") {
|
||
try { await speech.context.close(); } catch (_error) { /* page is closing */ }
|
||
speech.context = null;
|
||
speech.workletReady = false;
|
||
}
|
||
}
|
||
|
||
function concatenateChunks() {
|
||
const combined = new Float32Array(speech.totalSamples);
|
||
let offset = 0;
|
||
for (const chunk of speech.chunks) {
|
||
combined.set(chunk, offset);
|
||
offset += chunk.length;
|
||
}
|
||
const start = Math.max(0, Math.floor(speech.speechStartSample || 0));
|
||
const end = Math.min(
|
||
combined.length,
|
||
Math.ceil((speech.lastVoiceSample || combined.length) + (speech.sampleRate * 0.45)),
|
||
);
|
||
return combined.slice(start, end);
|
||
}
|
||
|
||
function downsample(samples, inputRate) {
|
||
if (inputRate === TARGET_SAMPLE_RATE) return samples;
|
||
const ratio = inputRate / TARGET_SAMPLE_RATE;
|
||
const output = new Float32Array(Math.max(1, Math.floor(samples.length / ratio)));
|
||
for (let outputIndex = 0; outputIndex < output.length; outputIndex += 1) {
|
||
const start = Math.floor(outputIndex * ratio);
|
||
const end = Math.max(start + 1, Math.min(samples.length, Math.floor((outputIndex + 1) * ratio)));
|
||
let sum = 0;
|
||
for (let inputIndex = start; inputIndex < end; inputIndex += 1) sum += samples[inputIndex];
|
||
output[outputIndex] = sum / (end - start);
|
||
}
|
||
return output;
|
||
}
|
||
|
||
function encodePcm16Wav(samples) {
|
||
const buffer = new ArrayBuffer(44 + (samples.length * 2));
|
||
const view = new DataView(buffer);
|
||
const writeAscii = (offset, value) => {
|
||
for (let index = 0; index < value.length; index += 1) view.setUint8(offset + index, value.charCodeAt(index));
|
||
};
|
||
writeAscii(0, "RIFF");
|
||
view.setUint32(4, 36 + (samples.length * 2), true);
|
||
writeAscii(8, "WAVE");
|
||
writeAscii(12, "fmt ");
|
||
view.setUint32(16, 16, true);
|
||
view.setUint16(20, 1, true);
|
||
view.setUint16(22, 1, true);
|
||
view.setUint32(24, TARGET_SAMPLE_RATE, true);
|
||
view.setUint32(28, TARGET_SAMPLE_RATE * 2, true);
|
||
view.setUint16(32, 2, true);
|
||
view.setUint16(34, 16, true);
|
||
writeAscii(36, "data");
|
||
view.setUint32(40, samples.length * 2, true);
|
||
for (let index = 0; index < samples.length; index += 1) {
|
||
const sample = Math.max(-1, Math.min(1, samples[index]));
|
||
view.setInt16(44 + (index * 2), sample < 0 ? sample * 32768 : sample * 32767, true);
|
||
}
|
||
return buffer;
|
||
}
|
||
|
||
async function transcribe(wav, utteranceId) {
|
||
const controller = new AbortController();
|
||
const timeout = window.setTimeout(() => controller.abort(), 45000);
|
||
try {
|
||
const response = await fetch("/api/speech/transcribe", {
|
||
method: "POST",
|
||
credentials: "same-origin",
|
||
cache: "no-store",
|
||
headers: {
|
||
Accept: "application/json",
|
||
"Content-Type": "audio/wav",
|
||
"X-Speech-Utterance-Id": utteranceId,
|
||
},
|
||
body: wav,
|
||
signal: controller.signal,
|
||
});
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok) throw new Error(payload.message || `语音转写失败（HTTP ${response.status}）`);
|
||
debug("server_result", payload);
|
||
return payload;
|
||
} finally {
|
||
window.clearTimeout(timeout);
|
||
}
|
||
}
|
||
|
||
function applyResult(result) {
|
||
const { input } = elements();
|
||
if (!input) throw new Error("没有找到一句话编排输入框");
|
||
const cleanText = String(result.clean_text || result.text || "").trim();
|
||
const peripheralEligible = result.peripheral_routing?.execution_eligible === true;
|
||
const inputText = peripheralEligible
|
||
? String(result.routing_text || result.raw_text || cleanText).trim()
|
||
: cleanText;
|
||
const commands = Array.isArray(result.commands) ? result.commands : [];
|
||
if (!inputText || (commands.length === 0 && !peripheralEligible)) {
|
||
window.dispatchEvent(new CustomEvent("cognitive-agent:speech-result", { detail: result }));
|
||
return false;
|
||
}
|
||
speech.applyingResult = true;
|
||
input.value = inputText.slice(0, input.maxLength || 1000);
|
||
input.dataset.speechUncertain = String(result.uncertain === true);
|
||
input.dataset.speechExecutionEligible = String(
|
||
result.execution_eligible === true || peripheralEligible,
|
||
);
|
||
input.dataset.speechSemanticStatus = String(result.semantic_status || "");
|
||
input.dataset.speechValue = input.value.trim();
|
||
input.dataset.speechUtteranceId = String(result.utterance_id || "");
|
||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||
speech.applyingResult = false;
|
||
input.focus({ preventScroll: true });
|
||
window.dispatchEvent(new CustomEvent("cognitive-agent:speech-result", { detail: result }));
|
||
return true;
|
||
}
|
||
|
||
async function finishDictation() {
|
||
if (speech.finishPromise) return speech.finishPromise;
|
||
if (speech.phase !== "listening") return undefined;
|
||
speech.finishPromise = (async () => {
|
||
setPhase("processing", "说话结束，正在识别并理解指令…");
|
||
const hadSpeech = speech.speechStartSample !== null;
|
||
const samples = hadSpeech ? concatenateChunks() : null;
|
||
const inputRate = speech.sampleRate;
|
||
const utteranceId = speech.utteranceId;
|
||
await releaseMicrophone();
|
||
speech.chunks = [];
|
||
if (!hadSpeech || !samples?.length) {
|
||
setPhase("error", "未检测到说话声，请重试");
|
||
return;
|
||
}
|
||
try {
|
||
const wav = encodePcm16Wav(downsample(samples, inputRate));
|
||
const result = await transcribe(wav, utteranceId);
|
||
speech.lastResult = result;
|
||
const applied = applyResult(result);
|
||
const latency = Math.round(result.timings?.total_ms ?? result.inference_ms ?? 0);
|
||
if (!applied) {
|
||
setPhase("success", `未形成可执行控制指令 · ${latency} ms`);
|
||
announce("已保留原始转录，但没有激活控制指令");
|
||
} else if (result.peripheral_routing?.execution_eligible === true) {
|
||
const count = Number(result.peripheral_routing.action_count || 1);
|
||
setPhase("success", `已识别为 ${count} 条外设控制指令 · ${latency} ms`);
|
||
announce("语音已匹配当前 NFC 外设的已学习控制指令");
|
||
} else if (result.semantic_status === "recognized_unsupported") {
|
||
setPhase("success", `已识别，但当前车辆能力不支持 · ${latency} ms`);
|
||
announce("已完整保留用户语义，但当前车辆能力不支持执行");
|
||
} else if (result.semantic_status === "incomplete") {
|
||
setPhase("success", `已识别，但控制参数不完整 · ${latency} ms`);
|
||
announce("语音中的关键控制参数不完整，当前不会执行");
|
||
} else if (result.uncertain === true) {
|
||
setPhase("success", `已转写，但存在歧义，请修改确认 · ${latency} ms`);
|
||
announce("语音结果需要人工确认，当前不会直接执行");
|
||
} else {
|
||
setPhase("success", `已理解为 ${result.commands.length} 条指令 · ${latency} ms`);
|
||
announce("语音已转换为理解式控制指令");
|
||
}
|
||
} catch (error) {
|
||
const message = error?.name === "AbortError" ? "语音转写超时，请重试。" : error.message;
|
||
setPhase("error", message || "语音转写失败，请重试");
|
||
announce(message || "语音转写失败");
|
||
}
|
||
})();
|
||
try {
|
||
return await speech.finishPromise;
|
||
} finally {
|
||
speech.finishPromise = null;
|
||
}
|
||
}
|
||
|
||
async function cancelDictation(message = "已取消语音听写") {
|
||
if (!speech.stream && speech.phase === "idle") return;
|
||
speech.phase = "processing";
|
||
speech.generation += 1;
|
||
await releaseMicrophone();
|
||
speech.chunks = [];
|
||
setPhase("error", message);
|
||
}
|
||
|
||
async function loadSpeechStatus() {
|
||
const { button } = elements();
|
||
if (!button) return;
|
||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||
button.disabled = true;
|
||
setPhase("error", "语音听写需要 HTTPS 或本机地址");
|
||
return;
|
||
}
|
||
try {
|
||
const response = await fetch("/api/speech/status", {
|
||
credentials: "same-origin",
|
||
cache: "no-store",
|
||
headers: { Accept: "application/json" },
|
||
});
|
||
const status = await response.json();
|
||
speech.backendReady = response.ok && status.enabled === true;
|
||
debug("backend_status", status);
|
||
setPhase("idle");
|
||
if (!speech.backendReady) {
|
||
button.disabled = true;
|
||
button.title = "SenseVoiceSmall 模型未部署";
|
||
}
|
||
} catch (_error) {
|
||
speech.backendReady = false;
|
||
button.disabled = true;
|
||
button.title = "无法连接语音识别服务";
|
||
}
|
||
}
|
||
|
||
function clearSpeechGuardOnManualEdit(event) {
|
||
if (speech.applyingResult || !event.isTrusted) return;
|
||
const { input } = elements();
|
||
if (!input) return;
|
||
delete input.dataset.speechUncertain;
|
||
delete input.dataset.speechExecutionEligible;
|
||
delete input.dataset.speechSemanticStatus;
|
||
delete input.dataset.speechValue;
|
||
delete input.dataset.speechUtteranceId;
|
||
}
|
||
|
||
function initialize() {
|
||
const { button, input } = elements();
|
||
if (!button) return;
|
||
button.addEventListener("click", () => {
|
||
if (speech.phase === "listening") void finishDictation();
|
||
else void startDictation();
|
||
});
|
||
input?.addEventListener("input", clearSpeechGuardOnManualEdit);
|
||
document.addEventListener("keydown", (event) => {
|
||
if (event.key === "Escape" && speech.phase === "listening") {
|
||
event.preventDefault();
|
||
void cancelDictation();
|
||
}
|
||
});
|
||
document.addEventListener("visibilitychange", () => {
|
||
if (document.hidden && speech.phase === "listening") void cancelDictation();
|
||
});
|
||
window.addEventListener("pagehide", () => { void releaseMicrophone({ closeContext: true }); });
|
||
void loadSpeechStatus();
|
||
}
|
||
|
||
window.CognitiveAgentSpeech = {
|
||
get lastResult() { return speech.lastResult; },
|
||
get debugEnabled() { return DEBUG; },
|
||
};
|
||
document.addEventListener("DOMContentLoaded", initialize);
|
||
})();
|