const assert = require("node:assert/strict"); const fs = require("node:fs"); const path = require("node:path"); const test = require("node:test"); const vm = require("node:vm"); const ROOT = path.resolve(__dirname, "../.."); const SPEECH_SOURCE = fs.readFileSync( path.join(ROOT, "apps/web-console/speech-input.js"), "utf8", ); const ASSISTANT_SOURCE = fs.readFileSync( path.join(ROOT, "apps/web-console/assistant-chat.js"), "utf8", ); class MockClassList { constructor() { this.values = new Set(); } toggle(name, force) { if (force === undefined) force = !this.values.has(name); if (force) this.values.add(name); else this.values.delete(name); return force; } contains(name) { return this.values.has(name); } add(...names) { names.forEach((name) => this.values.add(name)); } remove(...names) { names.forEach((name) => this.values.delete(name)); } } class MockEventTarget { constructor() { this.listeners = new Map(); } addEventListener(type, listener) { const listeners = this.listeners.get(type) || []; listeners.push(listener); this.listeners.set(type, listeners); } removeEventListener(type, listener) { const listeners = this.listeners.get(type) || []; this.listeners.set(type, listeners.filter((candidate) => candidate !== listener)); } dispatchEvent(event) { if (!event || !event.type) throw new TypeError("event.type is required"); if (!event.target) event.target = this; for (const listener of this.listeners.get(event.type) || []) listener.call(this, event); return event.defaultPrevented !== true; } } class MockElement extends MockEventTarget { constructor() { super(); this.attributes = new Map(); this.classList = new MockClassList(); this.dataset = {}; this.disabled = false; this.inert = false; this.maxLength = 4000; this.style = {}; this.textContent = ""; this.title = ""; this.value = ""; } append() {} click() { this.dispatchEvent({ type: "click", target: this }); } focus() {} getAttribute(name) { return this.attributes.get(name) ?? null; } setAttribute(name, value) { this.attributes.set(name, String(value)); } toggleAttribute(name, force) { this[name] = Boolean(force); } } class FakeAudioNode { constructor() { this.disconnectCount = 0; this.connections = []; } connect(node) { this.connections.push(node); return node; } disconnect() { this.disconnectCount += 1; } } function createStream({ label = "测试麦克风", deviceId = "default-mic", trackCount = 1 } = {}) { const tracks = Array.from({ length: trackCount }, () => ({ label, stopCount: 0, getSettings: () => ({ deviceId }), stop() { this.stopCount += 1; }, })); return { tracks, getAudioTracks: () => tracks, getTracks: () => tracks, }; } function response(payload, { ok = true, status = ok ? 200 : 500 } = {}) { return { ok, status, json: async () => payload }; } function createHarness({ fetchImpl, getUserMedia, permissionState = "prompt", now = 1000, } = {}) { const composer = new MockElement(); const button = new MockElement(); const input = new MockElement(); const status = new MockElement(); const hiddenLabel = new MockElement(); const drawer = new MockElement(); const drawerToggle = new MockElement(); button.closest = () => composer; const selectors = new Map([ ["#speechDictationButton", button], ["#speechDictationButton .speech-visually-hidden", hiddenLabel], ["#utterance", input], ["#speechDictationStatus", status], ["#assistantDrawer", drawer], ["#assistantDrawerToggle", drawerToggle], ]); const document = new MockEventTarget(); document.body = new MockElement(); document.hidden = false; document.querySelector = (selector) => selectors.get(selector) || null; const windowEvents = new MockEventTarget(); const defaultStream = createStream(); const mediaRequests = []; const audioNodes = []; class FakeAudioContext { constructor() { this.destination = new FakeAudioNode(); this.sampleRate = 16000; this.state = "running"; audioNodes.push(this.destination); } async resume() { this.state = "running"; } async close() { this.state = "closed"; } createMediaStreamSource() { const node = new FakeAudioNode(); audioNodes.push(node); return node; } createGain() { const node = new FakeAudioNode(); node.gain = { value: 1 }; audioNodes.push(node); return node; } createScriptProcessor() { const node = new FakeAudioNode(); node.onaudioprocess = null; audioNodes.push(node); return node; } } const permissionStatus = new MockEventTarget(); permissionStatus.state = permissionState; const fetchCalls = []; const harnessFetch = async (url, options = {}) => { fetchCalls.push({ url, options }); if (fetchImpl) return fetchImpl(url, options, fetchCalls.length - 1); throw new Error(`unexpected fetch: ${url}`); }; const harnessGetUserMedia = async (constraints) => { mediaRequests.push(constraints); return getUserMedia ? getUserMedia(constraints, mediaRequests.length - 1) : defaultStream; }; const schedule = (callback, delay, ...args) => { const timer = setTimeout(callback, delay, ...args); timer.unref?.(); return timer; }; let opened = 0; const sandbox = { AbortController, AudioContext: FakeAudioContext, ArrayBuffer, CustomEvent: class CustomEvent { constructor(type, options = {}) { this.type = type; this.detail = options.detail; } }, DOMException, DataView, Element: MockElement, Event: class Event { constructor(type, options = {}) { this.type = type; this.bubbles = options.bubbles === true; this.isTrusted = false; } }, Float32Array, HTMLFormElement: class HTMLFormElement extends MockElement {}, Math, Promise, Set, String, TextDecoder, Uint8Array, URLSearchParams, clearTimeout, console, crypto: { randomUUID: () => "runtime-test-id" }, document, fetch: harnessFetch, localStorage: { getItem: () => null, removeItem() {}, setItem() {} }, location: { search: "" }, isSecureContext: true, navigator: { mediaDevices: { enumerateDevices: async () => [], getUserMedia: harnessGetUserMedia, }, permissions: { query: async () => permissionStatus }, }, performance: { now: () => (typeof now === "function" ? now() : now) }, requestAnimationFrame: (callback) => callback(), sessionStorage: { getItem: () => null, removeItem() {}, setItem() {} }, setTimeout: schedule, __N7WEB_SPEECH_TEST__: true, __N7WEB_ASSISTANT_TEST__: true, }; sandbox.window = sandbox; sandbox.CognitiveAgentAssistant = { open: () => { opened += 1; } }; sandbox.addEventListener = windowEvents.addEventListener.bind(windowEvents); sandbox.removeEventListener = windowEvents.removeEventListener.bind(windowEvents); sandbox.dispatchEvent = windowEvents.dispatchEvent.bind(windowEvents); vm.createContext(sandbox); vm.runInContext(SPEECH_SOURCE, sandbox, { filename: "speech-input.js" }); return { sandbox, audioNodes, defaultStream, document, elements: { button, composer, drawer, drawerToggle, input, status }, fetchCalls, mediaRequests, opened: () => opened, permissionStatus, windowEvents, }; } function tick() { return new Promise((resolve) => setImmediate(resolve)); } async function settle(rounds = 8) { for (let index = 0; index < rounds; index += 1) await tick(); } test("wake preview uses the structured remainder and ignores duplicate, stale, and out-of-order revisions", () => { const harness = createHarness(); const api = harness.sandbox.CognitiveAgentSpeech.__test; Object.assign(api.state, { generation: 9, utteranceId: "utterance-9", phase: "armed", wakeWord: "你好,小迪", lastAppliedPreviewSequence: 0, }); const context = { generation: 9, utteranceId: "utterance-9", requestedSequence: 2 }; assert.equal(api.applyPreview({ sequence: 2, stale: false, raw_text: "你好小迪打开空调", live_text: "ignored fallback", wake_word: { configured_word: "你好,小迪", detected: true, remainder: "打开空调", }, }, context), true); assert.equal(harness.elements.input.value, "打开空调"); assert.equal(harness.elements.input.dataset.speechStage, "partial"); assert.equal(api.state.phase, "capturing"); assert.equal(api.state.wakeWord, "你好,小迪"); assert.equal(harness.opened(), 1); assert.equal(api.applyPreview({ sequence: 2, wake_word: { detected: true, remainder: "重复错误文本" }, }, context), false); assert.equal(api.applyPreview({ sequence: 1, wake_word: { detected: true, remainder: "乱序错误文本" }, }, { ...context, requestedSequence: 1 }), false); assert.equal(api.applyPreview({ sequence: 3, stale: true, wake_word: { detected: true, remainder: "过期错误文本" }, }, { ...context, requestedSequence: 3 }), false); assert.equal(harness.elements.input.value, "打开空调"); assert.equal(api.applyPreview({ sequence: 4, wake_word: { detected: true, remainder: "打开空调到24度" }, }, { ...context, requestedSequence: 4 }), true); assert.equal(harness.elements.input.value, "打开空调到24度"); }); test("manual preview streams ordinary speech without requiring or displaying a wake word", () => { const harness = createHarness(); const api = harness.sandbox.CognitiveAgentSpeech.__test; Object.assign(api.state, { generation: 10, utteranceId: "manual-preview-10", inputMode: "manual", phase: "capturing", wakeDetected: true, lastAppliedPreviewSequence: 0, }); const context = { generation: 10, utteranceId: "manual-preview-10", requestedSequence: 1, }; assert.equal(api.applyPreview({ sequence: 1, raw_text: "车上有点热", live_text: "车上有点热", wake_word: { detected: false, remainder: "" }, }, context), true); assert.equal(harness.elements.input.value, "车上有点热"); assert.equal(api.state.phase, "capturing"); assert.equal(harness.opened(), 0); assert.equal(api.applyPreview({ sequence: 1, live_text: "重复结果不应覆盖", wake_word: { detected: false }, }, context), false); assert.equal(api.applyPreview({ sequence: 0, live_text: "乱序结果不应覆盖", wake_word: { detected: false }, }, { ...context, requestedSequence: 0 }), false); assert.equal(harness.elements.input.value, "车上有点热"); }); test("clicking the microphone starts manual capture immediately without a wake word", async () => { const harness = createHarness({ fetchImpl: async (url) => { assert.equal(url, "/api/speech/status"); return response({ enabled: true, loaded: true, load_state: "ready", live_preview: { recommended_interval_ms: 850 }, wake_word: { primary: "你好,小迪" }, }); }, }); const api = harness.sandbox.CognitiveAgentSpeech.__test; api.initialize(); await settle(); harness.elements.button.click(); await settle(); assert.equal(api.state.inputMode, "manual"); assert.equal(api.state.phase, "capturing"); assert.equal(api.state.wakeDetected, true); assert.equal(harness.mediaRequests.length, 1); assert.equal(harness.elements.input.dataset.speechStage, "partial"); assert.match(harness.elements.status.textContent, /实时转录/); await api.cancelDictation("test cleanup", { silent: true }); }); test("background wake standby stays gated until wake detection and then shows only the remainder", async () => { const harness = createHarness({ permissionState: "granted" }); const api = harness.sandbox.CognitiveAgentSpeech.__test; Object.assign(api.state, { backendReady: true, assistantOpen: true }); assert.equal(await api.startWakeStandby({ automatic: true }), true); assert.equal(api.state.inputMode, "wake"); assert.equal(api.state.phase, "armed"); const context = { generation: api.state.generation, utteranceId: api.state.utteranceId, requestedSequence: 1, }; harness.elements.input.value = "原有输入"; assert.equal(api.applyPreview({ sequence: 1, raw_text: "打开空调", live_text: "打开空调", wake_word: { detected: false, remainder: "" }, }, context), true); assert.equal(api.state.phase, "armed"); assert.equal(harness.elements.input.value, "原有输入"); assert.equal(api.applyPreview({ sequence: 2, raw_text: "你好小迪打开空调到24度", live_text: "打开空调到24度", wake_word: { detected: true, configured_word: "你好,小迪", remainder: "打开空调到24度", }, }, { ...context, requestedSequence: 2 }), true); assert.equal(api.state.phase, "capturing"); assert.equal(harness.elements.input.value, "打开空调到24度"); assert.equal(harness.elements.input.value.includes("你好"), false); await api.cancelDictation("test cleanup", { silent: true }); }); test("automatic standby never prompts for microphone permission", async () => { const harness = createHarness({ permissionState: "prompt" }); const api = harness.sandbox.CognitiveAgentSpeech.__test; Object.assign(api.state, { backendReady: true, assistantOpen: true }); assert.equal(await api.startWakeStandby({ automatic: true }), false); assert.equal(api.state.phase, "idle"); assert.equal(harness.mediaRequests.length, 0); }); test("manual capture promotes an active wake stream without reacquiring the microphone", async () => { const harness = createHarness({ permissionState: "granted" }); const api = harness.sandbox.CognitiveAgentSpeech.__test; Object.assign(api.state, { backendReady: true, assistantOpen: true }); await api.startWakeStandby({ automatic: true }); const standbyStream = api.state.stream; assert.equal(await api.startManualDictation(), true); assert.equal(api.state.inputMode, "manual"); assert.equal(api.state.phase, "capturing"); assert.equal(api.state.wakeDetected, true); assert.equal(api.state.stream, standbyStream); assert.equal(harness.mediaRequests.length, 1); await api.cancelDictation("test cleanup", { silent: true }); }); test("manual and wake final requests carry explicit mode and wake contracts", async () => { const harness = createHarness({ fetchImpl: async (url) => { assert.equal(url, "/api/speech/transcribe"); return response({ assistant_text: "测试结果" }); }, }); const api = harness.sandbox.CognitiveAgentSpeech.__test; const wav = new harness.sandbox.ArrayBuffer(48); await api.transcribe(wav, "manual-final", "manual"); await api.transcribe(wav, "wake-final", "wake"); assert.deepEqual( harness.fetchCalls.map(({ options }) => ({ mode: options.headers["X-Speech-Input-Mode"], wakeRequired: options.headers["X-Speech-Wake-Word-Required"], utteranceId: options.headers["X-Speech-Utterance-Id"], })), [ { mode: "manual", wakeRequired: "0", utteranceId: "manual-final" }, { mode: "wake", wakeRequired: "1", utteranceId: "wake-final" }, ], ); }); test("manual endpoint silence finalizes recognized speech exactly once", async () => { let previewSequence = 0; const harness = createHarness({ fetchImpl: async (url, options) => { if (url === "/api/speech/preview") { previewSequence = Number(options.headers["X-Speech-Preview-Sequence"]); return response({ sequence: previewSequence, raw_text: "车上有点热", live_text: "车上有点热", wake_word: { detected: false, remainder: "" }, stale: false, }); } assert.equal(url, "/api/speech/transcribe"); return response({ raw_text: "车上有点热", assistant_text: "车上有点热" }); }, }); const api = harness.sandbox.CognitiveAgentSpeech.__test; Object.assign(api.state, { backendReady: true, generation: 31, utteranceId: "manual-silence-31", inputMode: "manual", phase: "capturing", wakeDetected: true, sampleRate: 16000, startedAt: 0, stream: harness.defaultStream, chunks: [], totalSamples: 0, speechStartSample: null, lastVoiceSample: null, consecutiveVoiceSamples: 0, lastPreviewAt: 0, }); api.handleAudio(new Float32Array(6400).fill(0.12), 31); api.handleAudio(new Float32Array(18000), 31); await settle(20); const finalCalls = harness.fetchCalls.filter(({ url }) => url === "/api/speech/transcribe"); assert.equal(finalCalls.length, 1); assert.equal(finalCalls[0].options.headers["X-Speech-Input-Mode"], "manual"); assert.equal(finalCalls[0].options.headers["X-Speech-Wake-Word-Required"], "0"); assert.equal(harness.elements.input.value, "车上有点热"); assert.equal(api.state.phase, "success"); assert.equal(harness.defaultStream.tracks[0].stopCount, 1); }); test("wake standby discards a spoken segment without the wake word and keeps listening", async () => { const harness = createHarness({ fetchImpl: async (url, options) => { assert.equal(url, "/api/speech/preview"); return response({ sequence: Number(options.headers["X-Speech-Preview-Sequence"]), raw_text: "打开空调", live_text: "打开空调", wake_word: { detected: false, remainder: "" }, stale: false, }); }, }); const api = harness.sandbox.CognitiveAgentSpeech.__test; Object.assign(api.state, { generation: 41, utteranceId: "wake-silence-41", inputMode: "wake", phase: "armed", wakeDetected: false, sampleRate: 16000, startedAt: 0, chunks: [], totalSamples: 0, speechStartSample: null, lastVoiceSample: null, consecutiveVoiceSamples: 0, lastPreviewAt: 0, }); harness.elements.input.value = "不要覆盖"; api.handleAudio(new Float32Array(6400).fill(0.12), 41); api.handleAudio(new Float32Array(18000), 41); await settle(20); assert.equal(api.state.phase, "armed"); assert.equal(api.state.wakeDetected, false); assert.equal(harness.elements.input.value, "不要覆盖"); assert.equal( harness.fetchCalls.some(({ url }) => url === "/api/speech/transcribe"), false, ); assert.ok(harness.fetchCalls.length >= 1); }); test("manual silence without any detected voice exits capture and releases the microphone", async () => { let clock = 9000; const harness = createHarness({ now: () => clock }); const api = harness.sandbox.CognitiveAgentSpeech.__test; Object.assign(api.state, { generation: 51, utteranceId: "manual-no-voice-51", inputMode: "manual", phase: "capturing", wakeDetected: true, sampleRate: 16000, startedAt: 0, stream: harness.defaultStream, chunks: [], totalSamples: 0, speechStartSample: null, lastVoiceSample: null, consecutiveVoiceSamples: 0, }); api.handleAudio(new Float32Array(2048), 51); await settle(); assert.equal(api.state.phase, "error"); assert.match(harness.elements.status.textContent, /未检测到说话声/); assert.equal(api.state.stream, null); assert.equal(harness.defaultStream.tracks[0].stopCount, 1); clock += 1; }); test("permission denial leaves no capture resources and gives an actionable error", async () => { const harness = createHarness({ getUserMedia: async () => { throw new DOMException("denied", "NotAllowedError"); }, }); const api = harness.sandbox.CognitiveAgentSpeech.__test; api.state.backendReady = true; assert.equal(await api.startManualDictation(), false); assert.equal(api.state.phase, "error"); assert.equal(api.state.stream, null); assert.equal(harness.mediaRequests.length, 1); assert.match(harness.elements.status.textContent, /麦克风权限/); }); test("ordinary questions use assistant_text even without executable commands", () => { const harness = createHarness(); const api = harness.sandbox.CognitiveAgentSpeech.__test; api.state.utteranceId = "question-1"; harness.elements.input.dataset.speechExecutionEligible = "false"; assert.equal(api.applyResult({ assistant_text: "我们中午吃什么?", clean_text: "", raw_text: "我们中午吃什么?", commands: [], execution_eligible: false, }), true); assert.equal(harness.elements.input.value, "我们中午吃什么?"); assert.equal(harness.elements.input.dataset.speechStage, "final"); assert.equal("speechExecutionEligible" in harness.elements.input.dataset, false); }); test("status reads the primary wake word instead of stringifying its object", () => { const harness = createHarness(); const api = harness.sandbox.CognitiveAgentSpeech.__test; assert.equal( api.configuredWakeWord({ wake_word: { primary: "你好,小迪" } }), "你好,小迪", ); assert.equal(api.configuredWakeWord({}), "你好,小迪"); assert.deepEqual( { ...api.speechStatusReadiness({ enabled: true, loaded: false, load_state: "loading" }) }, { loadState: "loading", ready: false }, ); assert.deepEqual( { ...api.speechStatusReadiness({ enabled: true, loaded: true, load_state: "ready" }) }, { loadState: "ready", ready: true }, ); assert.equal( api.speechStatusReadiness({ enabled: true, load_state: "error" }).ready, false, ); }); test("collapsing the assistant disables standby and releases active capture", async () => { const harness = createHarness(); const speechApi = harness.sandbox.CognitiveAgentSpeech; const state = speechApi.__test.state; const assistantOpenValues = []; const originalSetAssistantOpen = speechApi.setAssistantOpen; speechApi.setAssistantOpen = (open) => { assistantOpenValues.push(open); return originalSetAssistantOpen(open); }; Object.assign(state, { assistantOpen: true, phase: "armed", inputMode: "wake", stream: harness.defaultStream, }); vm.runInContext(ASSISTANT_SOURCE, harness.sandbox, { filename: "assistant-chat.js" }); harness.sandbox.CognitiveAgentAssistant.__test.setDrawerOpen(false); await settle(); assert.deepEqual(assistantOpenValues, [false]); assert.equal(state.assistantOpen, false); assert.equal(state.phase, "idle"); assert.equal(state.stream, null); assert.equal(harness.defaultStream.tracks[0].stopCount, 1); assert.equal(harness.elements.drawer.inert, true); assert.equal( harness.elements.drawerToggle.attributes.get("aria-expanded"), "false", ); }); test("cancellation aborts pending requests, disconnects nodes, and stops every track", async () => { const harness = createHarness(); const api = harness.sandbox.CognitiveAgentSpeech.__test; const stream = createStream({ trackCount: 2 }); const source = new FakeAudioNode(); const capture = new FakeAudioNode(); capture.onaudioprocess = () => {}; const gain = new FakeAudioNode(); const abortReasons = []; Object.assign(api.state, { phase: "armed", stream, source, captureNode: capture, silentGain: gain, previewController: { abort: (reason) => abortReasons.push(["preview", reason]) }, finalController: { abort: (reason) => abortReasons.push(["final", reason]) }, }); await api.cancelDictation("cleanup", { silent: true }); assert.deepEqual(abortReasons, [ ["preview", "speech_cancelled"], ["final", "speech_cancelled"], ]); assert.deepEqual(stream.tracks.map((track) => track.stopCount), [1, 1]); assert.equal(source.disconnectCount, 1); assert.equal(capture.disconnectCount, 1); assert.equal(gain.disconnectCount, 1); assert.equal(capture.onaudioprocess, null); assert.equal(api.state.phase, "idle"); }); test("speech stop releases every microphone track", async () => { const harness = createHarness(); const state = harness.sandbox.CognitiveAgentSpeech.__test.state; let stopped = 0; state.phase = "armed"; state.stream = { getTracks: () => [{ stop: () => { stopped += 1; } }], }; await harness.sandbox.CognitiveAgentSpeech.stop(); assert.equal(stopped, 1); assert.equal(state.stream, null); assert.equal(state.phase, "idle"); });