vee
1011 lines
38 KiB
JavaScript
1011 lines
38 KiB
JavaScript
(() => {
|
||
"use strict";
|
||
|
||
const TARGET_SAMPLE_RATE = 16000;
|
||
const MAX_RECORDING_MS = 28000;
|
||
const IDLE_BUFFER_RESET_MS = 8000;
|
||
const ENDPOINT_SILENCE_MS = 1050;
|
||
const WAKE_FOLLOWUP_TIMEOUT_MS = 6500;
|
||
const PREVIEW_INTERVAL_MS = 850;
|
||
const MIN_PREVIEW_AUDIO_MS = 320;
|
||
const STATUS_RESET_MS = 5000;
|
||
const MICROPHONE_CONNECT_TIMEOUT_MS = 12000;
|
||
const DEFAULT_WAKE_WORD = "你好，双十一";
|
||
const PREFERRED_MICROPHONE = /TAKSTAR\s*V2/i;
|
||
const ACTIVE_PHASES = new Set(["arming", "armed", "capturing", "finalizing"]);
|
||
const DEBUG = new URLSearchParams(window.location.search).get("speechDebug") === "1"
|
||
|| window.localStorage.getItem("speechDebug") === "1";
|
||
|
||
const speech = {
|
||
phase: "idle",
|
||
backendReady: false,
|
||
wakeWord: DEFAULT_WAKE_WORD,
|
||
inputMode: "wake",
|
||
assistantOpen: false,
|
||
microphonePermissionGranted: false,
|
||
permissionStatus: null,
|
||
wakeRestartTimer: null,
|
||
stream: null,
|
||
context: null,
|
||
contextPromise: null,
|
||
workletReady: false,
|
||
workletFailed: false,
|
||
source: null,
|
||
captureNode: null,
|
||
silentGain: null,
|
||
chunks: [],
|
||
totalSamples: 0,
|
||
sampleRate: 0,
|
||
startedAt: 0,
|
||
speechStartSample: null,
|
||
lastVoiceSample: null,
|
||
consecutiveVoiceSamples: 0,
|
||
noiseFloor: 0.003,
|
||
statusTimer: null,
|
||
statusPollTimer: null,
|
||
generation: 0,
|
||
utteranceId: "",
|
||
finishPromise: null,
|
||
endpointPromise: null,
|
||
endpointing: false,
|
||
previewSequence: 0,
|
||
lastAppliedPreviewSequence: 0,
|
||
lastPreviewAt: 0,
|
||
previewIntervalMs: PREVIEW_INTERVAL_MS,
|
||
previewInFlight: null,
|
||
previewQueued: false,
|
||
previewController: null,
|
||
finalController: null,
|
||
wakeDetected: false,
|
||
wakeDetectedAt: 0,
|
||
lastLiveText: "",
|
||
applyingResult: false,
|
||
lastResult: null,
|
||
};
|
||
|
||
const elements = () => ({
|
||
button: document.querySelector("#speechDictationButton"),
|
||
composer: document.querySelector("#speechDictationButton")?.closest(".command-composer"),
|
||
input: document.querySelector("#utterance"),
|
||
status: document.querySelector("#speechDictationStatus"),
|
||
hiddenLabel: document.querySelector("#speechDictationButton .speech-visually-hidden"),
|
||
});
|
||
|
||
function debug(stage, details = {}) {
|
||
if (!DEBUG) return;
|
||
console.debug("[speech]", { utteranceId: speech.utteranceId, stage, ...details });
|
||
}
|
||
|
||
function announce(message) {
|
||
const notify = window.CognitiveAgentConsole?.notify;
|
||
if (typeof notify === "function") notify(message);
|
||
}
|
||
|
||
function phaseLabel(phase) {
|
||
if (phase === "warming") return "语音模型正在后台加载";
|
||
if (phase === "arming") return speech.inputMode === "manual"
|
||
? "正在开始语音输入"
|
||
: "正在开启语音唤醒";
|
||
if (phase === "armed") return `开始语音输入，后台唤醒词：${speech.wakeWord}`;
|
||
if (phase === "capturing") return "结束本次语音输入";
|
||
if (phase === "finalizing") return "正在整理语音内容";
|
||
return "开始语音输入";
|
||
}
|
||
|
||
function configuredWakeWord(status) {
|
||
const primary = status?.wake_word && typeof status.wake_word === "object"
|
||
? status.wake_word.primary
|
||
: "";
|
||
return String(primary || DEFAULT_WAKE_WORD).trim() || DEFAULT_WAKE_WORD;
|
||
}
|
||
|
||
function speechStatusReadiness(status, responseOk = true) {
|
||
const loadState = String(status?.load_state || status?.warmup?.state || "idle");
|
||
return {
|
||
loadState,
|
||
ready: responseOk && status?.enabled === true
|
||
&& (status.loaded === true || !["loading", "error"].includes(loadState)),
|
||
};
|
||
}
|
||
|
||
function setPhase(phase, message = "") {
|
||
speech.phase = phase;
|
||
const { button, composer, status, hiddenLabel } = elements();
|
||
if (!button || !composer || !status) return;
|
||
window.clearTimeout(speech.statusTimer);
|
||
composer.dataset.speechState = phase;
|
||
status.textContent = message;
|
||
button.classList.toggle("is-armed", phase === "armed");
|
||
button.classList.toggle("is-listening", phase === "capturing");
|
||
button.classList.toggle("is-processing", ["arming", "finalizing"].includes(phase));
|
||
button.disabled = ["warming", "arming", "finalizing"].includes(phase)
|
||
|| (!speech.backendReady && !["armed", "capturing"].includes(phase));
|
||
button.setAttribute("aria-pressed", String(["armed", "capturing"].includes(phase)));
|
||
const label = phaseLabel(phase);
|
||
button.setAttribute("aria-label", label);
|
||
button.title = phase === "warming"
|
||
? "语音模型正在后台加载"
|
||
: phase === "armed"
|
||
? `正在后台等待“${speech.wakeWord}”，点击立即开始录音`
|
||
: phase === "capturing"
|
||
? "正在实时转录，点击可结束"
|
||
: speech.backendReady
|
||
? "开始语音输入"
|
||
: "语音识别模型未就绪";
|
||
if (hiddenLabel) hiddenLabel.textContent = label;
|
||
window.dispatchEvent(new CustomEvent("cognitive-agent:speech-state", {
|
||
detail: { phase, message, input_mode: speech.inputMode, wake_word: speech.wakeWord },
|
||
}));
|
||
if (["success", "error"].includes(phase)) {
|
||
speech.statusTimer = window.setTimeout(() => setPhase("idle"), STATUS_RESET_MS);
|
||
}
|
||
}
|
||
|
||
function microphoneErrorMessage(error) {
|
||
switch (error?.name) {
|
||
case "NotAllowedError":
|
||
return "未获得麦克风权限，请在浏览器地址栏允许访问麦克风。";
|
||
case "NotFoundError":
|
||
return "没有找到可用麦克风，请检查 TAKSTAR V2 接收器。";
|
||
case "NotReadableError":
|
||
return "麦克风正被其他程序占用，请关闭占用程序后重试。";
|
||
case "OverconstrainedError":
|
||
return "TAKSTAR V2 当前不可用，请重新连接接收器。";
|
||
case "SecurityError":
|
||
return "当前页面不允许访问麦克风，请使用 HTTPS 或本机地址。";
|
||
case "TimeoutError":
|
||
return "等待麦克风授权超时，请允许访问后重试。";
|
||
default:
|
||
return "无法启动麦克风，请检查设备连接和浏览器权限。";
|
||
}
|
||
}
|
||
|
||
function backgroundWakeEligible() {
|
||
return speech.backendReady
|
||
&& speech.assistantOpen
|
||
&& !document.hidden
|
||
&& !ACTIVE_PHASES.has(speech.phase)
|
||
&& !speech.stream;
|
||
}
|
||
|
||
function handleMicrophonePermissionChange() {
|
||
const state = speech.permissionStatus?.state || "unknown";
|
||
speech.microphonePermissionGranted = state === "granted";
|
||
if (state === "denied" && speech.stream) {
|
||
void cancelDictation("麦克风权限已关闭", { silent: speech.inputMode === "wake" });
|
||
return;
|
||
}
|
||
if (state === "granted") scheduleWakeStandby();
|
||
}
|
||
|
||
async function microphonePermissionState() {
|
||
if (speech.permissionStatus) {
|
||
if (speech.permissionStatus.state === "denied") return "denied";
|
||
if (speech.microphonePermissionGranted) return "granted";
|
||
return speech.permissionStatus.state;
|
||
}
|
||
if (!navigator.permissions?.query) {
|
||
return speech.microphonePermissionGranted ? "granted" : "unknown";
|
||
}
|
||
try {
|
||
const status = await navigator.permissions.query({ name: "microphone" });
|
||
speech.permissionStatus = status;
|
||
status.addEventListener?.("change", handleMicrophonePermissionChange);
|
||
return status.state;
|
||
} catch (_error) {
|
||
return speech.microphonePermissionGranted ? "granted" : "unknown";
|
||
}
|
||
}
|
||
|
||
async function maybeStartWakeStandby() {
|
||
window.clearTimeout(speech.wakeRestartTimer);
|
||
speech.wakeRestartTimer = null;
|
||
if (!backgroundWakeEligible()) return false;
|
||
const permissionState = await microphonePermissionState();
|
||
if (permissionState !== "granted" || !backgroundWakeEligible()) {
|
||
debug("wake_autostart_skipped", { permissionState });
|
||
return false;
|
||
}
|
||
return startWakeStandby({ automatic: true });
|
||
}
|
||
|
||
function scheduleWakeStandby(delay = 0) {
|
||
window.clearTimeout(speech.wakeRestartTimer);
|
||
speech.wakeRestartTimer = window.setTimeout(() => {
|
||
void maybeStartWakeStandby();
|
||
}, delay);
|
||
}
|
||
|
||
function setAssistantOpen(open) {
|
||
speech.assistantOpen = Boolean(open);
|
||
window.clearTimeout(speech.wakeRestartTimer);
|
||
speech.wakeRestartTimer = null;
|
||
if (!speech.assistantOpen) {
|
||
const stopping = cancelDictation("AI助手已收起，语音输入已停止", { silent: true });
|
||
void stopping.finally(() => {
|
||
// The drawer can be reopened while AudioContext suspension is still
|
||
// finishing. Re-check the latest UI state so that race does not lose
|
||
// the background wake listener.
|
||
if (speech.assistantOpen) scheduleWakeStandby();
|
||
});
|
||
return stopping;
|
||
}
|
||
scheduleWakeStandby();
|
||
return Promise.resolve();
|
||
}
|
||
|
||
async function getUserMediaWithTimeout(constraints) {
|
||
let expired = false;
|
||
let timer = 0;
|
||
const request = navigator.mediaDevices.getUserMedia(constraints);
|
||
const timeout = new Promise((_, reject) => {
|
||
timer = window.setTimeout(() => {
|
||
expired = true;
|
||
reject(new DOMException("microphone permission timed out", "TimeoutError"));
|
||
}, MICROPHONE_CONNECT_TIMEOUT_MS);
|
||
});
|
||
request.then((lateStream) => {
|
||
if (expired) lateStream.getTracks().forEach((track) => track.stop());
|
||
}).catch(() => {});
|
||
try {
|
||
return await Promise.race([request, timeout]);
|
||
} finally {
|
||
window.clearTimeout(timer);
|
||
}
|
||
}
|
||
|
||
async function preferredMicrophoneStream() {
|
||
const constraints = {
|
||
channelCount: 1,
|
||
echoCancellation: true,
|
||
noiseSuppression: true,
|
||
autoGainControl: true,
|
||
};
|
||
let stream = await getUserMediaWithTimeout({ audio: constraints, video: false });
|
||
const currentTrack = stream.getAudioTracks()[0];
|
||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||
const preferred = devices.find(
|
||
(device) => device.kind === "audioinput" && PREFERRED_MICROPHONE.test(device.label),
|
||
);
|
||
if (preferred && preferred.deviceId !== currentTrack?.getSettings?.().deviceId) {
|
||
stream.getTracks().forEach((track) => track.stop());
|
||
stream = await getUserMediaWithTimeout({
|
||
audio: { ...constraints, deviceId: { exact: preferred.deviceId } },
|
||
video: false,
|
||
});
|
||
}
|
||
return stream;
|
||
}
|
||
|
||
async function ensureAudioContext() {
|
||
if (speech.context && speech.context.state !== "closed") {
|
||
await speech.context.resume();
|
||
return speech.context;
|
||
}
|
||
if (speech.contextPromise) return speech.contextPromise;
|
||
speech.contextPromise = (async () => {
|
||
const AudioContext = window.AudioContext || window.webkitAudioContext;
|
||
const context = new AudioContext({ latencyHint: "interactive" });
|
||
await context.resume();
|
||
if (context.audioWorklet && window.AudioWorkletNode) {
|
||
try {
|
||
await context.audioWorklet.addModule("/speech-audio-worklet.js?v=sensevoice-5");
|
||
speech.workletReady = true;
|
||
} catch (error) {
|
||
speech.workletFailed = true;
|
||
debug("audio_worklet_fallback", { message: error?.message || String(error) });
|
||
}
|
||
} else {
|
||
speech.workletFailed = true;
|
||
}
|
||
speech.context = context;
|
||
return context;
|
||
})();
|
||
try {
|
||
return await speech.contextPromise;
|
||
} finally {
|
||
speech.contextPromise = null;
|
||
}
|
||
}
|
||
|
||
function audioRms(samples) {
|
||
let sum = 0;
|
||
for (let index = 0; index < samples.length; index += 1) sum += samples[index] * samples[index];
|
||
return Math.sqrt(sum / Math.max(1, samples.length));
|
||
}
|
||
|
||
function newUtteranceId(generation = speech.generation) {
|
||
return `web-${crypto.randomUUID?.() || `${Date.now()}-${generation}`}`;
|
||
}
|
||
|
||
function resetAttempt({ rotateUtterance = true } = {}) {
|
||
if (rotateUtterance) speech.utteranceId = newUtteranceId();
|
||
speech.chunks = [];
|
||
speech.totalSamples = 0;
|
||
speech.startedAt = performance.now();
|
||
speech.speechStartSample = null;
|
||
speech.lastVoiceSample = null;
|
||
speech.consecutiveVoiceSamples = 0;
|
||
speech.noiseFloor = 0.003;
|
||
speech.previewSequence = 0;
|
||
speech.lastAppliedPreviewSequence = 0;
|
||
speech.lastPreviewAt = 0;
|
||
speech.previewQueued = false;
|
||
speech.wakeDetected = false;
|
||
speech.wakeDetectedAt = 0;
|
||
speech.lastLiveText = "";
|
||
}
|
||
|
||
function clearLegacySpeechGuard(input) {
|
||
if (!input) return;
|
||
delete input.dataset.speechUncertain;
|
||
delete input.dataset.speechExecutionEligible;
|
||
delete input.dataset.speechSemanticStatus;
|
||
delete input.dataset.speechValue;
|
||
}
|
||
|
||
function replaceInput(value, stage) {
|
||
const { input } = elements();
|
||
if (!input) return;
|
||
speech.applyingResult = true;
|
||
clearLegacySpeechGuard(input);
|
||
input.value = String(value || "").slice(0, input.maxLength || 4000);
|
||
input.dataset.speechStage = stage;
|
||
input.dataset.speechUtteranceId = speech.utteranceId;
|
||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||
speech.applyingResult = false;
|
||
input.focus({ preventScroll: true });
|
||
}
|
||
|
||
function concatenateChunks({ trimEnd = false } = {}) {
|
||
const combined = new Float32Array(speech.totalSamples);
|
||
let offset = 0;
|
||
for (const chunk of speech.chunks) {
|
||
combined.set(chunk, offset);
|
||
offset += chunk.length;
|
||
}
|
||
const start = Math.max(0, Math.floor(speech.speechStartSample || 0));
|
||
const end = trimEnd
|
||
? Math.min(
|
||
combined.length,
|
||
Math.ceil((speech.lastVoiceSample || combined.length) + (speech.sampleRate * 0.45)),
|
||
)
|
||
: combined.length;
|
||
return combined.slice(start, end);
|
||
}
|
||
|
||
function downsample(samples, inputRate) {
|
||
if (inputRate === TARGET_SAMPLE_RATE) return samples;
|
||
const ratio = inputRate / TARGET_SAMPLE_RATE;
|
||
const output = new Float32Array(Math.max(1, Math.floor(samples.length / ratio)));
|
||
for (let outputIndex = 0; outputIndex < output.length; outputIndex += 1) {
|
||
const start = Math.floor(outputIndex * ratio);
|
||
const end = Math.max(start + 1, Math.min(samples.length, Math.floor((outputIndex + 1) * ratio)));
|
||
let sum = 0;
|
||
for (let inputIndex = start; inputIndex < end; inputIndex += 1) sum += samples[inputIndex];
|
||
output[outputIndex] = sum / (end - start);
|
||
}
|
||
return output;
|
||
}
|
||
|
||
function encodePcm16Wav(samples) {
|
||
const buffer = new ArrayBuffer(44 + (samples.length * 2));
|
||
const view = new DataView(buffer);
|
||
const writeAscii = (offset, value) => {
|
||
for (let index = 0; index < value.length; index += 1) view.setUint8(offset + index, value.charCodeAt(index));
|
||
};
|
||
writeAscii(0, "RIFF");
|
||
view.setUint32(4, 36 + (samples.length * 2), true);
|
||
writeAscii(8, "WAVE");
|
||
writeAscii(12, "fmt ");
|
||
view.setUint32(16, 16, true);
|
||
view.setUint16(20, 1, true);
|
||
view.setUint16(22, 1, true);
|
||
view.setUint32(24, TARGET_SAMPLE_RATE, true);
|
||
view.setUint32(28, TARGET_SAMPLE_RATE * 2, true);
|
||
view.setUint16(32, 2, true);
|
||
view.setUint16(34, 16, true);
|
||
writeAscii(36, "data");
|
||
view.setUint32(40, samples.length * 2, true);
|
||
for (let index = 0; index < samples.length; index += 1) {
|
||
const sample = Math.max(-1, Math.min(1, samples[index]));
|
||
view.setInt16(44 + (index * 2), sample < 0 ? sample * 32768 : sample * 32767, true);
|
||
}
|
||
return buffer;
|
||
}
|
||
|
||
function previewWav() {
|
||
if (speech.speechStartSample === null || !speech.sampleRate) return null;
|
||
const samples = concatenateChunks();
|
||
if ((samples.length / speech.sampleRate) * 1000 < MIN_PREVIEW_AUDIO_MS) return null;
|
||
return encodePcm16Wav(downsample(samples, speech.sampleRate));
|
||
}
|
||
|
||
function applyPreview(result, { generation, utteranceId, requestedSequence }) {
|
||
if (generation !== speech.generation || utteranceId !== speech.utteranceId) return false;
|
||
const sequence = Number(result.sequence ?? requestedSequence);
|
||
if (!Number.isFinite(sequence) || result.stale === true
|
||
|| sequence <= speech.lastAppliedPreviewSequence) {
|
||
debug("preview_ignored", { sequence, stale: result.stale === true });
|
||
return false;
|
||
}
|
||
speech.lastAppliedPreviewSequence = sequence;
|
||
const wakeWord = result.wake_word && typeof result.wake_word === "object"
|
||
? result.wake_word
|
||
: {};
|
||
const manualInput = speech.inputMode === "manual";
|
||
const detectedWake = manualInput || wakeWord.detected === true;
|
||
const liveText = String(manualInput
|
||
? (result.live_text || result.raw_text || result.text || "")
|
||
: (wakeWord.remainder ?? result.live_text ?? "")).trim();
|
||
|
||
if (!speech.wakeDetected && detectedWake) {
|
||
speech.wakeDetected = true;
|
||
speech.wakeDetectedAt = performance.now();
|
||
if (wakeWord.configured_word) speech.wakeWord = String(wakeWord.configured_word);
|
||
setPhase("capturing", manualInput
|
||
? "正在实时转录…"
|
||
: liveText
|
||
? "已唤醒，正在实时转录…"
|
||
: "已唤醒，请继续说…");
|
||
replaceInput("", "partial");
|
||
if (!manualInput) window.CognitiveAgentAssistant?.open?.();
|
||
debug(manualInput ? "manual_preview_started" : "wake_detected", {
|
||
wakeWord: speech.wakeWord,
|
||
sequence,
|
||
});
|
||
}
|
||
if (speech.wakeDetected && liveText) {
|
||
speech.lastLiveText = liveText;
|
||
replaceInput(liveText, "partial");
|
||
setPhase("capturing", "正在实时转录…");
|
||
}
|
||
window.dispatchEvent(new CustomEvent("cognitive-agent:speech-preview", { detail: result }));
|
||
return true;
|
||
}
|
||
|
||
async function requestPreview({ force = false } = {}) {
|
||
if (!["armed", "capturing"].includes(speech.phase)) return null;
|
||
if (speech.previewInFlight) {
|
||
speech.previewQueued = true;
|
||
return speech.previewInFlight;
|
||
}
|
||
const wav = previewWav();
|
||
if (!wav) return null;
|
||
const now = performance.now();
|
||
if (!force && now - speech.lastPreviewAt < speech.previewIntervalMs) return null;
|
||
speech.lastPreviewAt = now;
|
||
const generation = speech.generation;
|
||
const utteranceId = speech.utteranceId;
|
||
const requestedSequence = speech.previewSequence + 1;
|
||
speech.previewSequence = requestedSequence;
|
||
const controller = new AbortController();
|
||
speech.previewController = controller;
|
||
const timeout = window.setTimeout(() => controller.abort("preview_timeout"), 12000);
|
||
speech.previewInFlight = (async () => {
|
||
try {
|
||
const response = await fetch("/api/speech/preview", {
|
||
method: "POST",
|
||
credentials: "same-origin",
|
||
cache: "no-store",
|
||
headers: {
|
||
Accept: "application/json",
|
||
"Content-Type": "audio/wav",
|
||
"X-Speech-Utterance-Id": utteranceId,
|
||
"X-Speech-Preview-Sequence": String(requestedSequence),
|
||
"X-Speech-Input-Mode": speech.inputMode,
|
||
},
|
||
body: wav,
|
||
signal: controller.signal,
|
||
});
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok) throw new Error(payload.message || `实时转录失败（HTTP ${response.status}）`);
|
||
applyPreview(payload, { generation, utteranceId, requestedSequence });
|
||
debug("preview_result", { sequence: requestedSequence, payload });
|
||
return payload;
|
||
} catch (error) {
|
||
if (error?.name !== "AbortError" && generation === speech.generation) {
|
||
debug("preview_failed", { sequence: requestedSequence, message: error?.message || String(error) });
|
||
}
|
||
return null;
|
||
} finally {
|
||
window.clearTimeout(timeout);
|
||
}
|
||
})();
|
||
try {
|
||
return await speech.previewInFlight;
|
||
} finally {
|
||
speech.previewInFlight = null;
|
||
if (speech.previewController === controller) speech.previewController = null;
|
||
const queued = speech.previewQueued;
|
||
speech.previewQueued = false;
|
||
// Audio callbacks will schedule the next preview after the cadence has
|
||
// elapsed. Do not immediately replay a queued request: that would turn a
|
||
// warm offline decoder into an unbounded request loop under continuous
|
||
// speech. Endpoint flushing below still forces the newest snapshot.
|
||
if (queued) debug("preview_coalesced", { sequence: requestedSequence });
|
||
}
|
||
}
|
||
|
||
async function flushPreview() {
|
||
if (speech.previewInFlight) await speech.previewInFlight;
|
||
if (!["armed", "capturing"].includes(speech.phase)) return null;
|
||
speech.previewQueued = false;
|
||
return requestPreview({ force: true });
|
||
}
|
||
|
||
async function handleEndpoint(generation) {
|
||
if (speech.endpointPromise || generation !== speech.generation) return;
|
||
speech.endpointPromise = (async () => {
|
||
speech.endpointing = true;
|
||
try {
|
||
if (speech.phase === "armed" && speech.inputMode === "wake") {
|
||
await flushPreview();
|
||
if (generation !== speech.generation) return;
|
||
if (!speech.wakeDetected) {
|
||
resetAttempt();
|
||
setPhase("armed", `未检测到唤醒词，请说“${speech.wakeWord}”`);
|
||
return;
|
||
}
|
||
}
|
||
if (speech.phase !== "capturing") return;
|
||
if (!speech.lastLiveText && speech.inputMode === "wake") {
|
||
speech.lastVoiceSample = null;
|
||
speech.consecutiveVoiceSamples = 0;
|
||
setPhase("capturing", "已唤醒，请继续说…");
|
||
return;
|
||
}
|
||
await finishDictation();
|
||
} finally {
|
||
speech.endpointing = false;
|
||
}
|
||
})();
|
||
try {
|
||
await speech.endpointPromise;
|
||
} finally {
|
||
speech.endpointPromise = null;
|
||
}
|
||
}
|
||
|
||
function handleAudio(chunk, generation) {
|
||
if (!["armed", "capturing"].includes(speech.phase) || generation !== speech.generation) return;
|
||
speech.chunks.push(chunk);
|
||
speech.totalSamples += chunk.length;
|
||
const now = performance.now();
|
||
const elapsed = now - speech.startedAt;
|
||
const rms = audioRms(chunk);
|
||
if (speech.speechStartSample === null) {
|
||
speech.noiseFloor = (speech.noiseFloor * 0.92) + (rms * 0.08);
|
||
}
|
||
const voiceThreshold = Math.max(0.009, Math.min(0.035, speech.noiseFloor * 2.8));
|
||
if (rms >= voiceThreshold) {
|
||
speech.consecutiveVoiceSamples += chunk.length;
|
||
if (speech.consecutiveVoiceSamples >= speech.sampleRate * 0.09) {
|
||
if (speech.speechStartSample === null) {
|
||
speech.speechStartSample = Math.max(0, speech.totalSamples - (speech.sampleRate * 0.35));
|
||
setPhase(speech.wakeDetected ? "capturing" : "armed", speech.wakeDetected
|
||
? "正在实时转录…"
|
||
: `正在确认唤醒词“${speech.wakeWord}”…`);
|
||
debug("speech_started", { sample: speech.speechStartSample, voiceThreshold });
|
||
}
|
||
speech.lastVoiceSample = speech.totalSamples;
|
||
}
|
||
} else {
|
||
speech.consecutiveVoiceSamples = 0;
|
||
}
|
||
|
||
if (speech.speechStartSample !== null) {
|
||
void requestPreview();
|
||
const speechDurationMs = ((speech.totalSamples - speech.speechStartSample) / speech.sampleRate) * 1000;
|
||
if (
|
||
speech.lastVoiceSample !== null
|
||
&& speech.totalSamples - speech.lastVoiceSample >= speech.sampleRate * (ENDPOINT_SILENCE_MS / 1000)
|
||
) {
|
||
void handleEndpoint(generation);
|
||
} else if (speechDurationMs >= MAX_RECORDING_MS) {
|
||
if (speech.inputMode === "manual" || (speech.wakeDetected && speech.lastLiveText)) {
|
||
void finishDictation();
|
||
}
|
||
else void cancelDictation(`未检测到完整问题，请重新说“${speech.wakeWord}”`);
|
||
}
|
||
} else if (elapsed >= IDLE_BUFFER_RESET_MS) {
|
||
if (speech.inputMode === "manual") {
|
||
void cancelDictation("未检测到说话声，请重试")
|
||
.then(() => scheduleWakeStandby(1200));
|
||
} else {
|
||
resetAttempt();
|
||
setPhase("armed", `正在等待“${speech.wakeWord}”`);
|
||
}
|
||
}
|
||
|
||
if (speech.inputMode === "wake" && speech.phase === "capturing"
|
||
&& !speech.lastLiveText && speech.wakeDetectedAt
|
||
&& now - speech.wakeDetectedAt >= WAKE_FOLLOWUP_TIMEOUT_MS) {
|
||
void cancelDictation("唤醒后未检测到问题，请重试");
|
||
}
|
||
}
|
||
|
||
function connectCapture(context, generation) {
|
||
speech.source = context.createMediaStreamSource(speech.stream);
|
||
speech.silentGain = context.createGain();
|
||
speech.silentGain.gain.value = 0;
|
||
if (speech.workletReady && !speech.workletFailed) {
|
||
speech.captureNode = new AudioWorkletNode(context, "speech-capture-processor", {
|
||
numberOfInputs: 1,
|
||
numberOfOutputs: 1,
|
||
channelCount: 1,
|
||
});
|
||
speech.captureNode.port.onmessage = (event) => {
|
||
handleAudio(new Float32Array(event.data), generation);
|
||
};
|
||
debug("capture_connected", { implementation: "AudioWorklet" });
|
||
} else {
|
||
speech.captureNode = context.createScriptProcessor(2048, 1, 1);
|
||
speech.captureNode.onaudioprocess = (event) => {
|
||
handleAudio(new Float32Array(event.inputBuffer.getChannelData(0)), generation);
|
||
};
|
||
debug("capture_connected", { implementation: "ScriptProcessor-fallback" });
|
||
}
|
||
speech.source.connect(speech.captureNode);
|
||
speech.captureNode.connect(speech.silentGain);
|
||
speech.silentGain.connect(context.destination);
|
||
}
|
||
|
||
async function startCapture(inputMode, { automatic = false } = {}) {
|
||
if (!speech.backendReady) {
|
||
if (!automatic) setPhase("error", "语音识别模型未就绪");
|
||
return false;
|
||
}
|
||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||
if (!automatic) {
|
||
setPhase("error", "请使用 HTTPS 或 127.0.0.1 开启语音输入");
|
||
announce("当前页面不支持安全的麦克风访问");
|
||
}
|
||
return false;
|
||
}
|
||
if (ACTIVE_PHASES.has(speech.phase)) return false;
|
||
if (automatic) {
|
||
const permissionState = await microphonePermissionState();
|
||
if (permissionState !== "granted" || !backgroundWakeEligible()) return false;
|
||
}
|
||
const generation = speech.generation + 1;
|
||
speech.generation = generation;
|
||
speech.inputMode = inputMode;
|
||
resetAttempt();
|
||
if (inputMode === "manual") {
|
||
speech.wakeDetected = true;
|
||
speech.wakeDetectedAt = performance.now();
|
||
}
|
||
try {
|
||
setPhase("arming", inputMode === "manual" ? "正在开始录音…" : "正在连接唤醒监听…");
|
||
const stream = await preferredMicrophoneStream();
|
||
speech.microphonePermissionGranted = true;
|
||
if (generation !== speech.generation || document.hidden
|
||
|| (automatic && !speech.assistantOpen)) {
|
||
stream.getTracks().forEach((track) => track.stop());
|
||
return false;
|
||
}
|
||
speech.stream = stream;
|
||
const context = await ensureAudioContext();
|
||
if (generation !== speech.generation) {
|
||
stream.getTracks().forEach((track) => track.stop());
|
||
if (speech.stream === stream) speech.stream = null;
|
||
return false;
|
||
}
|
||
speech.sampleRate = context.sampleRate;
|
||
connectCapture(context, generation);
|
||
const trackLabel = stream.getAudioTracks()[0]?.label || "默认麦克风";
|
||
if (inputMode === "manual") {
|
||
replaceInput("", "partial");
|
||
setPhase("capturing", `正在实时转录 · ${trackLabel}`);
|
||
debug("manual_capture_started", { sampleRate: speech.sampleRate, trackLabel });
|
||
} else {
|
||
setPhase("armed", `后台等待“${speech.wakeWord}” · ${trackLabel}`);
|
||
debug("wake_listener_started", { sampleRate: speech.sampleRate, trackLabel });
|
||
}
|
||
return true;
|
||
} catch (error) {
|
||
if (generation !== speech.generation) return false;
|
||
await releaseMicrophone();
|
||
if (error?.name === "NotAllowedError") speech.microphonePermissionGranted = false;
|
||
const message = microphoneErrorMessage(error);
|
||
if (automatic) {
|
||
setPhase("idle");
|
||
debug("wake_listener_unavailable", { message });
|
||
} else {
|
||
setPhase("error", message);
|
||
announce(message);
|
||
}
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async function startManualDictation() {
|
||
window.clearTimeout(speech.wakeRestartTimer);
|
||
speech.wakeRestartTimer = null;
|
||
if (speech.phase === "capturing") return finishDictation();
|
||
if (speech.phase === "armed" && speech.stream) {
|
||
abortRequests("manual_capture_started");
|
||
speech.inputMode = "manual";
|
||
resetAttempt();
|
||
speech.wakeDetected = true;
|
||
speech.wakeDetectedAt = performance.now();
|
||
replaceInput("", "partial");
|
||
setPhase("capturing", "正在实时转录…");
|
||
debug("wake_standby_promoted_to_manual", { sampleRate: speech.sampleRate });
|
||
return true;
|
||
}
|
||
if (ACTIVE_PHASES.has(speech.phase)) return false;
|
||
return startCapture("manual");
|
||
}
|
||
|
||
async function startWakeStandby({ automatic = true } = {}) {
|
||
if (speech.phase === "armed" && speech.inputMode === "wake" && speech.stream) return true;
|
||
if (ACTIVE_PHASES.has(speech.phase)) return false;
|
||
return startCapture("wake", { automatic });
|
||
}
|
||
|
||
function abortRequests(reason = "speech_cancelled") {
|
||
speech.previewController?.abort(reason);
|
||
speech.finalController?.abort(reason);
|
||
speech.previewController = null;
|
||
speech.finalController = null;
|
||
}
|
||
|
||
async function releaseMicrophone({ closeContext = false } = {}) {
|
||
if (speech.captureNode?.port) speech.captureNode.port.onmessage = null;
|
||
if (speech.captureNode && "onaudioprocess" in speech.captureNode) {
|
||
speech.captureNode.onaudioprocess = null;
|
||
}
|
||
for (const node of [speech.source, speech.captureNode, speech.silentGain]) {
|
||
try { node?.disconnect(); } catch (_error) { /* already disconnected */ }
|
||
}
|
||
speech.stream?.getTracks().forEach((track) => track.stop());
|
||
speech.stream = null;
|
||
speech.source = null;
|
||
speech.captureNode = null;
|
||
speech.silentGain = null;
|
||
if (closeContext && speech.context && speech.context.state !== "closed") {
|
||
try { await speech.context.close(); } catch (_error) { /* page is closing */ }
|
||
speech.context = null;
|
||
speech.workletReady = false;
|
||
speech.workletFailed = false;
|
||
} else if (speech.context?.state === "running") {
|
||
try { await speech.context.suspend(); } catch (_error) { /* already suspended */ }
|
||
}
|
||
}
|
||
|
||
async function transcribe(wav, utteranceId, inputMode = speech.inputMode) {
|
||
const controller = new AbortController();
|
||
speech.finalController = controller;
|
||
const timeout = window.setTimeout(() => controller.abort("final_timeout"), 45000);
|
||
try {
|
||
const response = await fetch("/api/speech/transcribe", {
|
||
method: "POST",
|
||
credentials: "same-origin",
|
||
cache: "no-store",
|
||
headers: {
|
||
Accept: "application/json",
|
||
"Content-Type": "audio/wav",
|
||
"X-Speech-Utterance-Id": utteranceId,
|
||
"X-Speech-Input-Mode": inputMode,
|
||
"X-Speech-Wake-Word-Required": inputMode === "wake" ? "1" : "0",
|
||
},
|
||
body: wav,
|
||
signal: controller.signal,
|
||
});
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok) throw new Error(payload.message || `语音转写失败（HTTP ${response.status}）`);
|
||
debug("server_result", payload);
|
||
return payload;
|
||
} finally {
|
||
window.clearTimeout(timeout);
|
||
if (speech.finalController === controller) speech.finalController = null;
|
||
}
|
||
}
|
||
|
||
function applyResult(result) {
|
||
const { input } = elements();
|
||
if (!input) throw new Error("没有找到 AI助手输入框");
|
||
const assistantText = String(
|
||
result.assistant_text || result.clean_text || result.live_text || result.raw_text || result.text || "",
|
||
).trim();
|
||
if (!assistantText) {
|
||
window.dispatchEvent(new CustomEvent("cognitive-agent:speech-result", { detail: result }));
|
||
return false;
|
||
}
|
||
replaceInput(assistantText, "final");
|
||
input.dataset.speechValue = input.value.trim();
|
||
window.dispatchEvent(new CustomEvent("cognitive-agent:speech-result", { detail: result }));
|
||
return true;
|
||
}
|
||
|
||
async function finishDictation() {
|
||
if (speech.finishPromise) return speech.finishPromise;
|
||
if (speech.phase !== "capturing"
|
||
|| (speech.inputMode === "wake" && !speech.wakeDetected)) return undefined;
|
||
speech.finishPromise = (async () => {
|
||
const generation = speech.generation;
|
||
const inputMode = speech.inputMode;
|
||
const hadSpeech = speech.speechStartSample !== null;
|
||
const samples = hadSpeech ? concatenateChunks({ trimEnd: true }) : null;
|
||
const inputRate = speech.sampleRate;
|
||
const utteranceId = speech.utteranceId;
|
||
setPhase("finalizing", "说话结束，正在整理表达…");
|
||
speech.endpointing = true;
|
||
speech.previewController?.abort("speech_finalizing");
|
||
await releaseMicrophone();
|
||
speech.chunks = [];
|
||
if (!hadSpeech || !samples?.length) {
|
||
setPhase("error", "未检测到有效问题，请重试");
|
||
scheduleWakeStandby(1200);
|
||
return;
|
||
}
|
||
try {
|
||
const wav = encodePcm16Wav(downsample(samples, inputRate));
|
||
const result = await transcribe(wav, utteranceId, inputMode);
|
||
if (generation !== speech.generation) return;
|
||
speech.lastResult = result;
|
||
const applied = applyResult(result);
|
||
const latency = Math.round(result.timings?.total_ms ?? result.inference_ms ?? 0);
|
||
if (!applied) {
|
||
setPhase("error", "没有识别出有效内容，请重试");
|
||
return;
|
||
}
|
||
setPhase("success", latency > 0 ? `语音已整理 · ${latency} ms` : "语音已整理，可以发送");
|
||
announce("语音内容已整理，可以发送给 AI助手");
|
||
} catch (error) {
|
||
if (generation !== speech.generation) return;
|
||
const message = error?.name === "AbortError" ? "语音转写已取消或超时，请重试。" : error.message;
|
||
setPhase("error", message || "语音转写失败，请重试");
|
||
announce(message || "语音转写失败");
|
||
} finally {
|
||
speech.endpointing = false;
|
||
if (generation === speech.generation) scheduleWakeStandby(1200);
|
||
}
|
||
})();
|
||
try {
|
||
return await speech.finishPromise;
|
||
} finally {
|
||
speech.finishPromise = null;
|
||
}
|
||
}
|
||
|
||
async function cancelDictation(message = "已关闭语音唤醒", { silent = false } = {}) {
|
||
window.clearTimeout(speech.wakeRestartTimer);
|
||
speech.wakeRestartTimer = null;
|
||
if (!ACTIVE_PHASES.has(speech.phase) && !speech.stream) return;
|
||
speech.generation += 1;
|
||
speech.endpointing = false;
|
||
speech.previewQueued = false;
|
||
abortRequests();
|
||
await releaseMicrophone();
|
||
speech.chunks = [];
|
||
if (silent) setPhase("idle");
|
||
else setPhase("error", message);
|
||
}
|
||
|
||
async function loadSpeechStatus() {
|
||
const { button } = elements();
|
||
if (!button) return;
|
||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||
button.disabled = true;
|
||
setPhase("error", "语音唤醒需要 HTTPS 或本机地址");
|
||
return;
|
||
}
|
||
try {
|
||
const response = await fetch("/api/speech/status", {
|
||
credentials: "same-origin",
|
||
cache: "no-store",
|
||
headers: { Accept: "application/json" },
|
||
});
|
||
const status = await response.json();
|
||
const { loadState, ready } = speechStatusReadiness(status, response.ok);
|
||
speech.backendReady = ready;
|
||
speech.wakeWord = configuredWakeWord(status);
|
||
const recommendedPreviewInterval = Number(status.live_preview?.recommended_interval_ms);
|
||
if (Number.isFinite(recommendedPreviewInterval)) {
|
||
speech.previewIntervalMs = Math.max(350, Math.min(2000, recommendedPreviewInterval));
|
||
}
|
||
debug("backend_status", status);
|
||
window.clearTimeout(speech.statusPollTimer);
|
||
if (response.ok && status.enabled === true && loadState === "loading") {
|
||
setPhase("warming", "语音模型正在后台加载，完成后即可唤醒…");
|
||
speech.statusPollTimer = window.setTimeout(loadSpeechStatus, 1000);
|
||
} else {
|
||
setPhase("idle");
|
||
if (speech.backendReady) scheduleWakeStandby();
|
||
}
|
||
if (!speech.backendReady && loadState !== "loading") {
|
||
button.disabled = true;
|
||
button.title = loadState === "error"
|
||
? "SenseVoiceSmall 模型加载失败"
|
||
: "SenseVoiceSmall 模型未部署";
|
||
}
|
||
} catch (_error) {
|
||
speech.backendReady = false;
|
||
button.disabled = true;
|
||
button.title = "无法连接语音识别服务";
|
||
}
|
||
}
|
||
|
||
function clearSpeechGuardOnManualEdit(event) {
|
||
if (speech.applyingResult || !event.isTrusted) return;
|
||
const { input } = elements();
|
||
clearLegacySpeechGuard(input);
|
||
if (input) {
|
||
delete input.dataset.speechStage;
|
||
delete input.dataset.speechUtteranceId;
|
||
}
|
||
const formalCaptureActive = speech.phase === "capturing"
|
||
|| speech.phase === "finalizing"
|
||
|| (speech.phase === "arming" && speech.inputMode === "manual");
|
||
if (formalCaptureActive) {
|
||
void cancelDictation("已停止语音输入，保留手工编辑内容", { silent: true });
|
||
}
|
||
}
|
||
|
||
function initialize() {
|
||
const { button, input } = elements();
|
||
if (!button) return;
|
||
button.addEventListener("click", () => {
|
||
if (speech.phase === "capturing") void finishDictation();
|
||
else void startManualDictation();
|
||
});
|
||
input?.addEventListener("input", clearSpeechGuardOnManualEdit);
|
||
document.addEventListener("keydown", (event) => {
|
||
if (event.key === "Escape" && ACTIVE_PHASES.has(speech.phase)) {
|
||
event.preventDefault();
|
||
void cancelDictation();
|
||
}
|
||
});
|
||
document.addEventListener("visibilitychange", () => {
|
||
if (document.hidden) void cancelDictation("页面已隐藏，语音输入已停止", { silent: true });
|
||
else scheduleWakeStandby();
|
||
});
|
||
window.addEventListener("pagehide", () => {
|
||
window.clearTimeout(speech.statusPollTimer);
|
||
window.clearTimeout(speech.wakeRestartTimer);
|
||
speech.permissionStatus?.removeEventListener?.("change", handleMicrophonePermissionChange);
|
||
speech.generation += 1;
|
||
abortRequests("page_hidden");
|
||
void releaseMicrophone({ closeContext: true });
|
||
});
|
||
void loadSpeechStatus();
|
||
}
|
||
|
||
window.CognitiveAgentSpeech = {
|
||
get lastResult() { return speech.lastResult; },
|
||
get phase() { return speech.phase; },
|
||
get wakeWord() { return speech.wakeWord; },
|
||
get inputMode() { return speech.inputMode; },
|
||
get debugEnabled() { return DEBUG; },
|
||
startManualDictation,
|
||
startWakeStandby,
|
||
setAssistantOpen,
|
||
stop(message = "已关闭语音唤醒") {
|
||
return cancelDictation(message, { silent: true });
|
||
},
|
||
};
|
||
if (window.__N7WEB_SPEECH_TEST__ === true) {
|
||
window.CognitiveAgentSpeech.__test = {
|
||
applyPreview,
|
||
applyResult,
|
||
configuredWakeWord,
|
||
speechStatusReadiness,
|
||
startManualDictation,
|
||
startWakeStandby,
|
||
finishDictation,
|
||
handleAudio,
|
||
cancelDictation,
|
||
releaseMicrophone,
|
||
transcribe,
|
||
initialize,
|
||
setAssistantOpen,
|
||
setPhase,
|
||
state: speech,
|
||
};
|
||
}
|
||
document.addEventListener("DOMContentLoaded", initialize);
|
||
})();
|