From 32d179143fd2adba2db07743351a581b66f7cb3a Mon Sep 17 00:00:00 2001 From: minghanchen <17188325+minghanchen@user.noreply.gitee.com> Date: Thu, 3 Sep 2026 13:36:40 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=AF=AD=E9=9F=B3=E8=BE=93?= =?UTF-8?q?=E5=85=A5/=E6=99=AE=E9=80=9A=E6=96=87=E6=9C=AC=E8=BE=93?= =?UTF-8?q?=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/web-console/console-v2.js | 11 +- apps/web-console/index.html | 4 +- apps/web-console/service-worker.js | 6 +- apps/web-console/speech-input.js | 16 +- .../device-capability-card-demo/README.md | 30 + .../device-capability-card-demo/app.js | 288 +++++++ .../device-capability-card-demo/index.html | 149 ++++ .../device-capability-card-demo/package.json | 8 + .../parser-engine.js | 353 +++++++++ .../parser-engine.test.mjs | 34 + .../device-capability-card-demo/styles.css | 746 ++++++++++++++++++ .../application/chinese_values.py | 24 + .../application/native_can_intent.py | 27 +- .../application/seat_service.py | 65 +- .../infrastructure/speech/capability.py | 36 +- .../infrastructure/speech/understanding.py | 21 +- .../infrastructure/speech/vocabulary.py | 3 + src/cognitive_agent/server.py | 88 ++- tests/e2e/test_seat_web_api.py | 152 +++- tests/fixtures/speech_text_regression.json | 5 +- tests/unit/test_frontend_theme_contract.py | 2 +- tests/unit/test_nfc_frontend_contract.py | 3 +- .../test_requested_frontend_enhancements.py | 67 ++ .../unit/test_speech_capability_alignment.py | 34 +- .../unit/test_speech_orchestration_bridge.py | 192 +++++ tests/unit/test_two_stage_can_parsing.py | 7 +- 26 files changed, 2318 insertions(+), 53 deletions(-) create mode 100644 experiments/device-capability-card-demo/README.md create mode 100644 experiments/device-capability-card-demo/app.js create mode 100644 experiments/device-capability-card-demo/index.html create mode 100644 experiments/device-capability-card-demo/package.json create mode 100644 experiments/device-capability-card-demo/parser-engine.js create mode 100644 experiments/device-capability-card-demo/parser-engine.test.mjs create mode 100644 experiments/device-capability-card-demo/styles.css diff --git a/apps/web-console/console-v2.js b/apps/web-console/console-v2.js index f22e8a6..c256dcc 100644 --- a/apps/web-console/console-v2.js +++ b/apps/web-console/console-v2.js @@ -416,6 +416,15 @@ return [...unique.values()]; } + function parsedCommandSourceLabel(source) { + return { + direct_control: "设备控件", + saved_mode_command_flow: "模式卡片", + local_rule: "本地规则", + speech_semantic: "语音语义", + }[source] || "云端模型"; + } + function renderParsedCanCommands(result, { message = "" } = {}) { const panel = $("#v2CanCommandPanel"); const list = $("#v2CanCommandList"); @@ -429,7 +438,7 @@
  • ${escapeHtml(item.channel)} - ${item.parseSource ? `${item.parseSource === "direct_control" ? "设备控件" : item.parseSource === "saved_mode_command_flow" ? "模式卡片" : item.parseSource === "local_rule" ? "本地规则" : "云端模型"}` : ""} + ${item.parseSource ? `${parsedCommandSourceLabel(item.parseSource)}` : ""} ${escapeHtml(item.deliveryLabel)} ${escapeHtml(item.category)} diff --git a/apps/web-console/index.html b/apps/web-console/index.html index a0259f8..44f8415 100755 --- a/apps/web-console/index.html +++ b/apps/web-console/index.html @@ -19,10 +19,10 @@ - + - + diff --git a/apps/web-console/service-worker.js b/apps/web-console/service-worker.js index 87a56b0..f4aba54 100644 --- a/apps/web-console/service-worker.js +++ b/apps/web-console/service-worker.js @@ -1,4 +1,4 @@ -const CACHE_VERSION = "cognitive-agent-pad-v227"; +const CACHE_VERSION = "cognitive-agent-pad-v228"; const SHELL_CACHE = `${CACHE_VERSION}-shell`; const RUNTIME_CACHE = `${CACHE_VERSION}-runtime`; const APP_SHELL = [ @@ -10,11 +10,11 @@ const APP_SHELL = [ "/gesture-control.js?v=android-pad-9", "/pwa.js?v=android-pad-3", "/console-v2.css?v=delivery-state-50", - "/console-v2.js?v=delivery-state-54", + "/console-v2.js?v=delivery-state-55", "/self-extension.css?v=gscu-tcp-42", "/self-extension.js?v=gscu-tcp-41", "/speech-input.css?v=sensevoice-5", - "/speech-input.js?v=sensevoice-6", + "/speech-input.js?v=sensevoice-7", "/speech-audio-worklet.js?v=sensevoice-4", "/assets/pets/self-extension-cat.png?v=3", "/assets/pets/self-extension-cat-entrance.png?v=3", diff --git a/apps/web-console/speech-input.js b/apps/web-console/speech-input.js index 4557d66..e7673ab 100644 --- a/apps/web-console/speech-input.js +++ b/apps/web-console/speech-input.js @@ -380,15 +380,21 @@ const { input } = elements(); if (!input) throw new Error("没有找到一句话编排输入框"); const cleanText = String(result.clean_text || result.text || "").trim(); + const peripheralEligible = result.peripheral_routing?.execution_eligible === true; + const inputText = peripheralEligible + ? String(result.routing_text || result.raw_text || cleanText).trim() + : cleanText; const commands = Array.isArray(result.commands) ? result.commands : []; - if (!cleanText || commands.length === 0) { + if (!inputText || (commands.length === 0 && !peripheralEligible)) { window.dispatchEvent(new CustomEvent("cognitive-agent:speech-result", { detail: result })); return false; } speech.applyingResult = true; - input.value = cleanText.slice(0, input.maxLength || 1000); + input.value = inputText.slice(0, input.maxLength || 1000); input.dataset.speechUncertain = String(result.uncertain === true); - input.dataset.speechExecutionEligible = String(result.execution_eligible === true); + input.dataset.speechExecutionEligible = String( + result.execution_eligible === true || peripheralEligible, + ); input.dataset.speechSemanticStatus = String(result.semantic_status || ""); input.dataset.speechValue = input.value.trim(); input.dataset.speechUtteranceId = String(result.utterance_id || ""); @@ -423,6 +429,10 @@ if (!applied) { setPhase("success", `未形成可执行控制指令 · ${latency} ms`); announce("已保留原始转录，但没有激活控制指令"); + } else if (result.peripheral_routing?.execution_eligible === true) { + const count = Number(result.peripheral_routing.action_count || 1); + setPhase("success", `已识别为 ${count} 条外设控制指令 · ${latency} ms`); + announce("语音已匹配当前 NFC 外设的已学习控制指令"); } else if (result.semantic_status === "recognized_unsupported") { setPhase("success", `已识别，但当前车辆能力不支持 · ${latency} ms`); announce("已完整保留用户语义，但当前车辆能力不支持执行"); diff --git a/experiments/device-capability-card-demo/README.md b/experiments/device-capability-card-demo/README.md new file mode 100644 index 0000000..5ee6d1d --- /dev/null +++ b/experiments/device-capability-card-demo/README.md @@ -0,0 +1,30 @@ +# 设备能力卡片实验 Demo + +这是一个完全独立的静态实验，不导入 Cognitive Agent 的现有前端或后端模块， +也不会调用 API、执行器或 CAN 通道。 + +## 运行 + +```bash +cd experiments/device-capability-card-demo +python3 -m http.server 8877 --bind 127.0.0.1 +``` + +浏览器访问： + +## 测试 + +```bash +cd experiments/device-capability-card-demo +npm test +``` + +## 演示内容 + +- 实时修改设备名称、ID、领域、别名、目标范围和执行通道； +- 启用或停用能力，修改各项功能的自然语言说法； +- 根据能力定义自动生成设备功能卡片； +- 将自然语言转换为结构化控制命令； +- 展示设备匹配、功能匹配、范围解析、参数校验和执行器绑定轨迹； +- 对数值越界、未命中设备、功能停用等情况执行失败关闭； +- 所有执行载荷均带有 `demo_only: true`。 diff --git a/experiments/device-capability-card-demo/app.js b/experiments/device-capability-card-demo/app.js new file mode 100644 index 0000000..2167549 --- /dev/null +++ b/experiments/device-capability-card-demo/app.js @@ -0,0 +1,288 @@ +import { + DEFAULT_DESCRIPTOR, + buildManualCommand, + cloneDescriptor, + parseInstruction, + splitAliases, +} from "./parser-engine.js"; + +const $ = (selector, root = document) => root.querySelector(selector); +const $$ = (selector, root = document) => [...root.querySelectorAll(selector)]; + +let descriptor = cloneDescriptor(); +let latestPayload = null; + +const categoryLabels = { + vehicle_electrical: "车端电器", + seat: "座椅设备", + peripheral: "扩展外设", +}; + +const transportLabels = { + native_can: "原生 CAN", + domain_runtime: "领域执行器", + gscu_tcp: "红外网关", + mqtt: "MQTT", +}; + +const exampleTemplates = [ + (name) => `开启${name}`, + (name) => `把${name}温度调到4度`, + (name) => `将${name}设为速冷模式`, + (name) => `关闭${name}`, + (name) => `把${name}调到零下10度`, +]; + +function escapeHtml(value) { + return String(value ?? "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function fillIdentityForm() { + $("#deviceName").value = descriptor.device.name; + $("#deviceId").value = descriptor.device.id; + $("#deviceDomain").value = descriptor.device.domain; + $("#deviceCategory").value = descriptor.device.category; + $("#deviceAliases").value = descriptor.device.aliases.join("、"); + $("#deviceScope").value = descriptor.device.defaultScope; + $("#deviceTransport").value = descriptor.device.transport; +} + +function syncIdentityFromForm() { + descriptor.device.name = $("#deviceName").value.trim() || "未命名设备"; + descriptor.device.id = $("#deviceId").value.trim() || "unnamed_device"; + descriptor.device.domain = $("#deviceDomain").value.trim() || "custom"; + descriptor.device.category = $("#deviceCategory").value; + descriptor.device.aliases = splitAliases($("#deviceAliases").value); + descriptor.device.defaultScope = $("#deviceScope").value; + descriptor.device.targetScopes = [descriptor.device.defaultScope]; + descriptor.device.transport = $("#deviceTransport").value; + descriptor.device.targetDevice = descriptor.device.id; + renderAll({ preserveResult: true }); +} + +function renderCapabilityEditor() { + $("#capabilityEditor").innerHTML = descriptor.capabilities.map((capability, capabilityIndex) => ` +
    +
    +
    + ${escapeHtml(capability.label)} + ${escapeHtml(capability.id)} +
    + +
    +
    + ${capability.intents.map((intent, intentIndex) => ` + + `).join("")} +
    +
    + `).join(""); +} + +function renderExamples() { + $("#exampleCommands").innerHTML = exampleTemplates.map((makeExample) => { + const utterance = makeExample(descriptor.device.name); + return ``; + }).join(""); +} + +function controlMarkup(capability) { + const ui = capability.ui; + if (ui.type === "select") { + return ` + + `; + } + return ` + + `; +} + +function renderDeviceCard() { + const enabledCapabilities = descriptor.capabilities.filter((item) => item.enabled !== false); + const category = categoryLabels[descriptor.device.category] || "自定义设备"; + const transport = transportLabels[descriptor.device.transport] || descriptor.device.transport; + const icon = descriptor.device.category === "seat" ? "◫" : descriptor.device.category === "peripheral" ? "⌁" : "❄"; + $("#deviceCardPreview").innerHTML = ` +
    +
    + +
    +

    ${escapeHtml(descriptor.device.name)}

    +

    ${escapeHtml(category)} · ${escapeHtml(transport)} · ${escapeHtml(descriptor.device.defaultScope)}

    +
    +
    + 实验就绪 +
    +
    + ${descriptor.readouts.map((readout) => `
    ${escapeHtml(readout.label)}${escapeHtml(readout.value)}
    `).join("")} +
    +
    + ${enabledCapabilities.length ? enabledCapabilities.map(controlMarkup).join("") : "

    没有已启用功能。

    "} +
    +

    功能控件只生成结构化演示数据；执行映射为虚拟引用，不连接现有服务。

    + `; +} + +function compactDescriptor() { + return { + version: descriptor.version, + device: descriptor.device, + capabilities: descriptor.capabilities.map((capability) => ({ + id: capability.id, + property: capability.property, + target_scopes: descriptor.device.targetScopes, + parameters: capability.parameters, + intents: capability.intents, + risk_level: capability.risk, + confirmation_policy: capability.confirmationPolicy, + executor: capability.executor, + enabled: capability.enabled, + })), + }; +} + +function renderDescriptor() { + $("#descriptorPreview").textContent = JSON.stringify(compactDescriptor(), null, 2); +} + +function renderAll({ preserveResult = false } = {}) { + renderCapabilityEditor(); + renderExamples(); + renderDeviceCard(); + renderDescriptor(); + if (!preserveResult) renderResult(null); +} + +function resultPresentation(result) { + if (!result) return { status: "idle", title: "等待解析", message: "输入一句话，查看设备、功能和参数的匹配过程。" }; + if (result.source === "card_control") { + return { + status: "success", + title: "已生成卡片控制载荷", + message: `已生成 ${result.commands.length} 条结构化演示指令；实验页面不会发送到车辆。`, + }; + } + if (result.semantic_status === "recognized_executable") { + return { status: "success", title: "已匹配可执行功能", message: `${result.clean_text} 当前仅生成演示载荷。` }; + } + if (result.semantic_status === "ambiguous" || result.semantic_status === "incomplete") { + return { status: "warning", title: "需要补充信息", message: result.issues.join("；") }; + } + return { status: "error", title: "未生成控制指令", message: result.issues.join("；") || "没有匹配到设备能力。" }; +} + +function renderResult(result) { + const presentation = resultPresentation(result); + $("#resultBanner").dataset.status = presentation.status; + $("#resultTitle").textContent = presentation.title; + $("#resultMessage").textContent = presentation.message; + $("#traceList").innerHTML = result?.trace?.length + ? result.trace.map((item) => `
  • ${escapeHtml(item.stage)}${escapeHtml(item.detail)}
    • `).join("") + : "
  • 尚无轨迹解析后将在这里显示每一步决策。
    • "; + latestPayload = result; + $("#payloadOutput").textContent = JSON.stringify(result || {}, null, 2); + $("#copyPayload").disabled = !result; +} + +function parseCurrentUtterance() { + renderResult(parseInstruction($("#utterance").value, descriptor)); +} + +function bindEvents() { + ["deviceName", "deviceId", "deviceDomain", "deviceCategory", "deviceAliases", "deviceScope", "deviceTransport"] + .forEach((id) => $("#" + id).addEventListener("input", syncIdentityFromForm)); + + $("#capabilityEditor").addEventListener("input", (event) => { + const section = event.target.closest("[data-capability-index]"); + if (!section) return; + const capability = descriptor.capabilities[Number(section.dataset.capabilityIndex)]; + if (event.target.matches("[data-capability-enabled]")) { + capability.enabled = event.target.checked; + } else if (event.target.matches("[data-intent-index]")) { + capability.intents[Number(event.target.dataset.intentIndex)].phrases = splitAliases(event.target.value); + } + renderDeviceCard(); + renderDescriptor(); + }); + + $("#exampleCommands").addEventListener("click", (event) => { + const button = event.target.closest("[data-example]"); + if (!button) return; + $("#utterance").value = button.dataset.example; + parseCurrentUtterance(); + }); + + $("#parseCommand").addEventListener("click", parseCurrentUtterance); + $("#utterance").addEventListener("keydown", (event) => { + if ((event.ctrlKey || event.metaKey) && event.key === "Enter") parseCurrentUtterance(); + }); + + $("#deviceCardPreview").addEventListener("input", (event) => { + if (!event.target.matches("[data-manual-capability]")) return; + const output = event.target.closest(".card-control")?.querySelector("output"); + if (output) output.textContent = `${event.target.value}${event.target.dataset.unit || ""}`; + const values = Object.fromEntries($$("[data-manual-capability]", $("#deviceCardPreview")) + .map((input) => [input.dataset.manualCapability, input.value])); + const result = { + source: "card_control", + semantic_status: "recognized_executable", + execution_eligible: false, + note: "实验页面不允许真实执行", + commands: buildManualCommand(descriptor, values), + }; + renderResult(result); + }); + + $("#copyPayload").addEventListener("click", async () => { + if (!latestPayload) return; + await navigator.clipboard.writeText(JSON.stringify(latestPayload, null, 2)); + const button = $("#copyPayload"); + button.textContent = "已复制"; + window.setTimeout(() => { button.textContent = "复制"; }, 1200); + }); + + $("#resetDemo").addEventListener("click", () => { + descriptor = cloneDescriptor(DEFAULT_DESCRIPTOR); + fillIdentityForm(); + $("#utterance").value = "把冰箱温度调到4度"; + renderAll(); + parseCurrentUtterance(); + }); +} + +fillIdentityForm(); +renderAll(); +bindEvents(); +parseCurrentUtterance(); diff --git a/experiments/device-capability-card-demo/index.html b/experiments/device-capability-card-demo/index.html new file mode 100644 index 0000000..c9c6abc --- /dev/null +++ b/experiments/device-capability-card-demo/index.html @@ -0,0 +1,149 @@ + + + + + + + 设备能力卡片实验室 + + + +
    +
    +
    +

    独立实验 · 不连接车辆

    +

    设备能力卡片实验室

    +

    修改设备名称、别名和功能说法，观察自然语言如何匹配为结构化控制指令。

    +
    + +
    + +
    + +

    安全边界 本页面只在浏览器内生成演示数据，不导入当前项目模块，也不会调用控制接口或发送 CAN 指令。

    +
    + +
    + + +
    +
    + 设备别名 + 功能说法 + 参数校验 + 执行载荷 +
    + +
    +
    + 03 +

    功能卡片

    +
    +
    +
    + +
    +
    + 04 +

    指令解析

    +
    + +
    + + + +
    + +
    + 等待解析 +

    输入一句话，查看设备、功能和参数的匹配过程。

    +
    +
    + +
    +
    +

    匹配轨迹

    +
      +
      +
      +
      +

      结构化结果

      + +
      + 
      {}
      +
      +
      +
      +
      +
      +
      + + + + diff --git a/experiments/device-capability-card-demo/package.json b/experiments/device-capability-card-demo/package.json new file mode 100644 index 0000000..9db6735 --- /dev/null +++ b/experiments/device-capability-card-demo/package.json @@ -0,0 +1,8 @@ +{ + "name": "device-capability-card-demo", + "private": true, + "type": "module", + "scripts": { + "test": "node parser-engine.test.mjs" + } +} diff --git a/experiments/device-capability-card-demo/parser-engine.js b/experiments/device-capability-card-demo/parser-engine.js new file mode 100644 index 0000000..fab6865 --- /dev/null +++ b/experiments/device-capability-card-demo/parser-engine.js @@ -0,0 +1,353 @@ +export const DEFAULT_DESCRIPTOR = { + version: 1, + device: { + id: "cabin_refrigerator", + name: "车载冰箱", + aliases: ["冰箱", "车载冷藏箱", "冷藏箱"], + domain: "refrigerator", + category: "vehicle_electrical", + targetDevice: "refrigerator", + targetScopes: ["vehicle"], + defaultScope: "vehicle", + transport: "native_can", + channel: "实验通道", + }, + readouts: [ + { key: "power", label: "工作状态", value: "制冷中" }, + { key: "temperature", label: "当前温度", value: "6℃" }, + { key: "mode", label: "当前模式", value: "节能" }, + ], + capabilities: [ + { + id: "refrigerator.set_power", + label: "工作状态", + property: "refrigerator.power", + operation: "set", + enabled: true, + risk: "low", + confirmationPolicy: "on_ambiguity", + parameters: [ + { name: "state", type: "string", required: true, allowedValues: ["on", "off"] }, + ], + intents: [ + { label: "开启", phrases: ["打开", "开启", "启动", "开启制冷"], values: { state: "on" } }, + { label: "关闭", phrases: ["关闭", "关掉", "停止", "停止制冷"], values: { state: "off" } }, + ], + ui: { + type: "select", + value: "on", + options: [["off", "关闭"], ["on", "开启"]], + }, + executor: { binding: "NativeCanRuntime", mappingRef: "demo.refrigerator.power" }, + }, + { + id: "refrigerator.set_temperature", + label: "目标温度", + property: "refrigerator.temperature", + operation: "set", + enabled: true, + risk: "medium", + confirmationPolicy: "on_ambiguity", + parameters: [ + { name: "temperature_c", type: "number", unit: "celsius", required: true, minimum: 0, maximum: 15 }, + ], + intents: [ + { + label: "设置温度", + phrases: ["温度", "调到", "设置为", "制冷到", "冷藏到"], + extract: { parameter: "temperature_c", kind: "number" }, + }, + ], + ui: { type: "range", value: 4, minimum: 0, maximum: 15, step: 1, unit: "℃" }, + executor: { binding: "NativeCanRuntime", mappingRef: "demo.refrigerator.temperature" }, + }, + { + id: "refrigerator.set_mode", + label: "工作模式", + property: "refrigerator.mode", + operation: "set", + enabled: true, + risk: "low", + confirmationPolicy: "on_ambiguity", + parameters: [ + { name: "mode", type: "string", required: true, allowedValues: ["eco", "fast_cool"] }, + ], + intents: [ + { label: "节能", phrases: ["节能模式", "省电模式", "节能"], values: { mode: "eco" } }, + { label: "速冷", phrases: ["速冷模式", "快速制冷", "速冷"], values: { mode: "fast_cool" } }, + ], + ui: { + type: "select", + value: "eco", + options: [["eco", "节能"], ["fast_cool", "速冷"]], + }, + executor: { binding: "NativeCanRuntime", mappingRef: "demo.refrigerator.mode" }, + }, + ], +}; + +const SCOPE_ALIASES = { + driver: ["主驾", "驾驶位", "司机位"], + passenger: ["副驾", "副驾驶", "乘客位"], + rear_row: ["后排", "第二排"], + vehicle: ["全车", "整车", "车内"], +}; + +const POLITE_WORDS = /^(?:请|麻烦|拜托|帮我|请帮我|给我)+/; +const END_POLITE_WORDS = /(?:谢谢|可以吗|好吗|行吗)[。！!？?]*$/; + +export function cloneDescriptor(value = DEFAULT_DESCRIPTOR) { + return JSON.parse(JSON.stringify(value)); +} + +export function splitAliases(value) { + return [...new Set(String(value || "") + .split(/[,，、;；|]/) + .map((item) => item.trim()) + .filter(Boolean))]; +} + +export function normalizeText(value) { + return String(value || "") + .normalize("NFKC") + .trim() + .replace(POLITE_WORDS, "") + .replace(END_POLITE_WORDS, "") + .toLowerCase() + .replace(/[\s，,。；;！!？?、]/g, ""); +} + +function longestContained(normalizedText, phrases) { + return [...new Set(phrases.map(normalizeText).filter(Boolean))] + .filter((phrase) => normalizedText.includes(phrase)) + .sort((left, right) => right.length - left.length)[0] || ""; +} + +function chineseInteger(token) { + if (!token) return null; + if (/^\d+$/.test(token)) return Number(token); + const digits = { 零: 0, 〇: 0, 一: 1, 二: 2, 两: 2, 三: 3, 四: 4, 五: 5, 六: 6, 七: 7, 八: 8, 九: 9 }; + if (token.includes("十")) { + const [left, right] = token.split("十", 2); + const tens = left ? digits[left.at(-1)] : 1; + const ones = right ? digits[right.at(-1)] : 0; + return Number.isFinite(tens) && Number.isFinite(ones) ? tens * 10 + ones : null; + } + if ([...token].every((char) => Object.hasOwn(digits, char))) { + return Number([...token].map((char) => digits[char]).join("")); + } + return null; +} + +function extractNumber(text) { + const belowZero = String(text).match(/零下(\d+(?:\.\d+)?|[零〇一二两三四五六七八九十]+)/); + if (belowZero) { + const value = /^\d/.test(belowZero[1]) ? Number(belowZero[1]) : chineseInteger(belowZero[1]); + return value === null ? null : -value; + } + const numeric = String(text).match(/-?\d+(?:\.\d+)?/); + if (numeric) return Number(numeric[0]); + const chinese = String(text).match(/[零〇一二两三四五六七八九十]+/); + return chinese ? chineseInteger(chinese[0]) : null; +} + +function resolveScope(normalizedText, device) { + const explicit = Object.entries(SCOPE_ALIASES) + .flatMap(([scope, aliases]) => aliases.map((alias) => ({ scope, alias: normalizeText(alias) }))) + .find(({ alias }) => normalizedText.includes(alias)); + if (explicit) { + if (!device.targetScopes.includes(explicit.scope)) { + return { status: "unsupported", scope: explicit.scope, explicit: true, reason: `设备不支持“${explicit.alias}”范围` }; + } + return { status: "resolved", scope: explicit.scope, explicit: true, reason: `显式目标：${explicit.alias}` }; + } + if (device.defaultScope && device.targetScopes.includes(device.defaultScope)) { + return { status: "resolved", scope: device.defaultScope, explicit: false, reason: `采用卡片默认目标：${device.defaultScope}` }; + } + if (device.targetScopes.length === 1) { + return { status: "resolved", scope: device.targetScopes[0], explicit: false, reason: `能力仅支持唯一目标：${device.targetScopes[0]}` }; + } + return { status: "ambiguous", scope: null, explicit: false, reason: "存在多个目标范围，需要用户确认" }; +} + +function validateParameters(capability, parameters) { + const issues = []; + for (const schema of capability.parameters || []) { + const value = parameters[schema.name]; + if ((value === undefined || value === null || value === "") && schema.required) { + issues.push(`缺少必填参数 ${schema.name}`); + continue; + } + if (value === undefined || value === null || value === "") continue; + if (schema.type === "number" && typeof value !== "number") { + issues.push(`${schema.name} 必须是数字`); + } + if (schema.minimum !== undefined && value < schema.minimum) { + issues.push(`${schema.name}=${value} 低于下限 ${schema.minimum}`); + } + if (schema.maximum !== undefined && value > schema.maximum) { + issues.push(`${schema.name}=${value} 超过上限 ${schema.maximum}`); + } + if (schema.allowedValues && !schema.allowedValues.includes(value)) { + issues.push(`${schema.name}=${value} 不在允许值中`); + } + } + return issues; +} + +function renderCleanText(descriptor, capability, parameters) { + const deviceName = descriptor.device.name; + if (capability.property.endsWith(".power")) { + return `${parameters.state === "on" ? "开启" : "关闭"}${deviceName}。`; + } + if (capability.property.endsWith(".temperature")) { + return `将${deviceName}设置为${parameters.temperature_c}度。`; + } + if (capability.property.endsWith(".mode")) { + const modeLabels = { eco: "节能模式", fast_cool: "速冷模式" }; + return `将${deviceName}设置为${modeLabels[parameters.mode] || parameters.mode}。`; + } + return `执行${deviceName}的${capability.label}。`; +} + +function baseResult(rawText, normalizedText, trace) { + return { + raw_text: rawText, + normalized_text: normalizedText, + semantic_status: "no_command", + execution_eligible: false, + clean_text: "", + command: null, + issues: [], + trace, + }; +} + +export function parseInstruction(rawText, descriptorInput = DEFAULT_DESCRIPTOR) { + const descriptor = cloneDescriptor(descriptorInput); + const normalizedText = normalizeText(rawText); + const trace = [{ stage: "文本整理", state: "success", detail: normalizedText || "输入为空" }]; + const result = baseResult(String(rawText || ""), normalizedText, trace); + + if (!normalizedText) { + result.semantic_status = "incomplete"; + result.issues.push("请输入自然语言指令"); + trace.push({ stage: "设备识别", state: "error", detail: "没有可分析的文字" }); + return result; + } + + const devicePhrases = [descriptor.device.name, ...(descriptor.device.aliases || [])]; + const matchedDevice = longestContained(normalizedText, devicePhrases); + if (!matchedDevice) { + result.issues.push("指令没有命中设备名称或别名"); + trace.push({ stage: "设备识别", state: "error", detail: `候选：${devicePhrases.join("、")}` }); + return result; + } + trace.push({ stage: "设备识别", state: "success", detail: `“${matchedDevice}” → ${descriptor.device.id}` }); + + const candidates = []; + for (const capability of descriptor.capabilities.filter((item) => item.enabled !== false)) { + for (const intent of capability.intents || []) { + const phrase = longestContained(normalizedText, intent.phrases || []); + if (!phrase) continue; + const parameters = { ...(intent.values || {}) }; + if (intent.extract?.kind === "number") { + const number = extractNumber(rawText); + if (number !== null) parameters[intent.extract.parameter] = number; + } + candidates.push({ + capability, + intent, + phrase, + parameters, + score: matchedDevice.length * 100 + phrase.length, + }); + } + } + + if (!candidates.length) { + result.semantic_status = "incomplete"; + result.issues.push("设备已识别，但没有命中已启用的功能说法"); + trace.push({ stage: "功能识别", state: "error", detail: "请使用卡片中定义的功能说法" }); + return result; + } + + candidates.sort((left, right) => right.score - left.score); + const best = candidates[0]; + const tied = candidates.filter((item) => item.score === best.score); + if (tied.length > 1) { + result.semantic_status = "ambiguous"; + result.issues.push(`同时命中多个功能：${tied.map((item) => item.intent.label).join("、")}`); + trace.push({ stage: "功能识别", state: "error", detail: "最高匹配分相同，需要补充更明确的动作" }); + return result; + } + trace.push({ stage: "功能识别", state: "success", detail: `“${best.phrase}” → ${best.capability.id}` }); + + const scope = resolveScope(normalizedText, descriptor.device); + trace.push({ stage: "目标解析", state: scope.status === "resolved" ? "success" : "error", detail: scope.reason }); + if (scope.status !== "resolved") { + result.semantic_status = scope.status === "ambiguous" ? "ambiguous" : "recognized_unsupported"; + result.issues.push(scope.reason); + return result; + } + + const issues = validateParameters(best.capability, best.parameters); + trace.push({ + stage: "参数校验", + state: issues.length ? "error" : "success", + detail: issues.length ? issues.join("；") : JSON.stringify(best.parameters), + }); + if (issues.length) { + result.semantic_status = Object.keys(best.parameters).length ? "recognized_unsupported" : "incomplete"; + result.issues.push(...issues); + return result; + } + + result.semantic_status = "recognized_executable"; + result.execution_eligible = true; + result.clean_text = renderCleanText(descriptor, best.capability, best.parameters); + result.command = { + domain: descriptor.device.domain, + capability: best.capability.id, + property: best.capability.property, + target_device: descriptor.device.targetDevice, + target_scope: scope.scope, + operation: best.capability.operation, + parameters: best.parameters, + risk_level: best.capability.risk, + confirmation_policy: best.capability.confirmationPolicy, + executor_binding: best.capability.executor.binding, + executor_payload: { + transport: descriptor.device.transport, + mapping_ref: best.capability.executor.mappingRef, + capability: best.capability.id, + target: scope.scope, + parameters: best.parameters, + demo_only: true, + }, + }; + trace.push({ stage: "执行绑定", state: "success", detail: `${best.capability.executor.binding} · 演示载荷，不发送` }); + return result; +} + +export function buildManualCommand(descriptorInput, values) { + const descriptor = cloneDescriptor(descriptorInput); + const commands = []; + for (const capability of descriptor.capabilities.filter((item) => item.enabled !== false)) { + const value = values[capability.id]; + if (value === undefined) continue; + const schema = capability.parameters?.[0]; + if (!schema) continue; + const parsedValue = schema.type === "number" ? Number(value) : value; + const parameters = { [schema.name]: parsedValue }; + const issues = validateParameters(capability, parameters); + if (issues.length) continue; + commands.push({ + capability: capability.id, + target_scope: descriptor.device.defaultScope, + parameters, + demo_only: true, + }); + } + return commands; +} diff --git a/experiments/device-capability-card-demo/parser-engine.test.mjs b/experiments/device-capability-card-demo/parser-engine.test.mjs new file mode 100644 index 0000000..0538210 --- /dev/null +++ b/experiments/device-capability-card-demo/parser-engine.test.mjs @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { DEFAULT_DESCRIPTOR, cloneDescriptor, parseInstruction } from "./parser-engine.js"; + +const successCases = [ + ["开启车载冰箱", "refrigerator.set_power", { state: "on" }], + ["把冰箱温度调到4度", "refrigerator.set_temperature", { temperature_c: 4 }], + ["将冷藏箱设为速冷模式", "refrigerator.set_mode", { mode: "fast_cool" }], + ["关闭冰箱", "refrigerator.set_power", { state: "off" }], +]; + +for (const [utterance, capability, parameters] of successCases) { + const result = parseInstruction(utterance, DEFAULT_DESCRIPTOR); + assert.equal(result.semantic_status, "recognized_executable", utterance); + assert.equal(result.execution_eligible, true, utterance); + assert.equal(result.command.capability, capability, utterance); + assert.deepEqual(result.command.parameters, parameters, utterance); + assert.equal(result.command.executor_payload.demo_only, true, utterance); +} + +const outOfRange = parseInstruction("把冰箱调到零下10度", DEFAULT_DESCRIPTOR); +assert.equal(outOfRange.semantic_status, "recognized_unsupported"); +assert.equal(outOfRange.execution_eligible, false); +assert.match(outOfRange.issues.join(" "), /低于下限/); + +const noDevice = parseInstruction("把空调调到4度", DEFAULT_DESCRIPTOR); +assert.equal(noDevice.semantic_status, "no_command"); +assert.equal(noDevice.execution_eligible, false); + +const disabled = cloneDescriptor(); +disabled.capabilities.find((item) => item.id === "refrigerator.set_temperature").enabled = false; +const disabledResult = parseInstruction("把冰箱温度调到4度", disabled); +assert.equal(disabledResult.semantic_status, "incomplete"); + +console.log(`device capability card demo: ${successCases.length + 3} parser checks passed`); diff --git a/experiments/device-capability-card-demo/styles.css b/experiments/device-capability-card-demo/styles.css new file mode 100644 index 0000000..dc48d0b --- /dev/null +++ b/experiments/device-capability-card-demo/styles.css @@ -0,0 +1,746 @@ +:root { + color-scheme: light dark; + --page: #f2f5f2; + --surface: #ffffff; + --surface-soft: #f7f9f7; + --surface-strong: #e8eeea; + --ink: #18201c; + --muted: #68736c; + --line: #d8dfda; + --accent: #176b4a; + --accent-soft: #dceee5; + --accent-ink: #0f5137; + --warning: #9a640b; + --warning-soft: #fff3d7; + --danger: #a33a3a; + --danger-soft: #fde9e7; + --shadow: 0 18px 55px rgba(32, 50, 40, 0.08); + font-family: Inter, "PingFang SC", "Microsoft YaHei", system-ui, sans-serif; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + min-width: 320px; + background: + radial-gradient(circle at 8% 3%, rgba(23, 107, 74, 0.1), transparent 25rem), + var(--page); + color: var(--ink); +} + +button, +input, +select, +textarea { + font: inherit; +} + +button { + cursor: pointer; +} + +button:disabled { + cursor: not-allowed; + opacity: 0.48; +} + +.demo-shell { + width: min(1440px, calc(100% - 40px)); + margin: 0 auto; + padding: 38px 0 60px; +} + +.page-header { + display: flex; + align-items: flex-end; + justify-content: space-between; + gap: 24px; + margin-bottom: 22px; +} + +.eyebrow { + margin: 0 0 8px; + color: var(--accent); + font-size: 13px; + font-weight: 700; + letter-spacing: 0.12em; +} + +h1, +h2, +h3, +p { + margin-top: 0; +} + +h1 { + margin-bottom: 10px; + font-size: clamp(30px, 4vw, 48px); + line-height: 1.05; + letter-spacing: -0.045em; +} + +.page-intro { + max-width: 720px; + margin-bottom: 0; + color: var(--muted); + line-height: 1.7; +} + +.secondary-button, +.primary-button { + min-height: 42px; + border-radius: 11px; + padding: 0 17px; + font-weight: 700; +} + +.secondary-button { + border: 1px solid var(--line); + background: var(--surface); + color: var(--ink); +} + +.primary-button { + border: 1px solid var(--accent); + background: var(--accent); + color: #fff; +} + +.safety-note { + display: flex; + align-items: center; + gap: 12px; + margin-bottom: 20px; + border: 1px solid color-mix(in srgb, var(--accent) 20%, var(--line)); + border-radius: 14px; + padding: 12px 16px; + background: color-mix(in srgb, var(--accent-soft) 60%, var(--surface)); + color: var(--accent-ink); +} + +.safety-note > span { + display: grid; + width: 28px; + height: 28px; + place-items: center; + border-radius: 50%; + background: var(--accent); + color: #fff; +} + +.safety-note p { + margin: 0; + line-height: 1.55; +} + +.workspace-grid { + display: grid; + grid-template-columns: minmax(320px, 0.75fr) minmax(560px, 1.55fr); + gap: 20px; + align-items: start; +} + +.configuration-panel, +.preview-section, +.parser-section { + border: 1px solid var(--line); + border-radius: 20px; + background: var(--surface); + box-shadow: var(--shadow); +} + +.configuration-panel { + padding: 22px; +} + +.experiment-panel { + display: grid; + gap: 20px; +} + +.preview-section, +.parser-section { + padding: 22px; +} + +.section-heading { + display: flex; + align-items: center; + gap: 10px; + margin-bottom: 18px; +} + +.section-heading > span { + display: grid; + width: 30px; + height: 30px; + place-items: center; + border-radius: 9px; + background: var(--accent-soft); + color: var(--accent-ink); + font-size: 12px; + font-weight: 800; +} + +.section-heading h2 { + margin: 0; + font-size: 19px; + letter-spacing: -0.02em; +} + +.compact-heading { + margin-top: 28px; +} + +.field-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 14px; +} + +.field { + display: grid; + gap: 7px; +} + +.field-wide { + grid-column: 1 / -1; +} + +.field > span, +.intent-alias-field > span { + color: var(--muted); + font-size: 12px; + font-weight: 700; +} + +.field small { + color: var(--muted); + font-size: 11px; + line-height: 1.5; +} + +input, +select, +textarea { + width: 100%; + border: 1px solid var(--line); + border-radius: 10px; + background: var(--surface-soft); + color: var(--ink); + outline: none; + transition: border-color 150ms ease, box-shadow 150ms ease; +} + +input, +select { + min-height: 40px; + padding: 0 11px; +} + +textarea { + resize: vertical; + padding: 12px; + line-height: 1.6; +} + +input:focus, +select:focus, +textarea:focus { + border-color: var(--accent); + box-shadow: 0 0 0 3px color-mix(in srgb, var(--accent) 14%, transparent); +} + +.capability-editor { + display: grid; + gap: 12px; +} + +.capability-definition { + border-top: 1px solid var(--line); + padding-top: 14px; +} + +.capability-definition:first-child { + border-top: 0; + padding-top: 0; +} + +.capability-title { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + margin-bottom: 10px; +} + +.capability-title strong { + font-size: 14px; +} + +.capability-title code { + color: var(--muted); + font-size: 11px; +} + +.toggle-label { + display: inline-flex; + align-items: center; + gap: 7px; + color: var(--muted); + font-size: 12px; +} + +.toggle-label input { + width: 16px; + min-height: 16px; + accent-color: var(--accent); +} + +.intent-list { + display: grid; + gap: 9px; +} + +.intent-alias-field { + display: grid; + grid-template-columns: 72px 1fr; + align-items: center; + gap: 8px; +} + +.intent-alias-field input { + min-height: 36px; + font-size: 12px; +} + +.schema-details { + margin-top: 20px; + border-top: 1px solid var(--line); + padding-top: 16px; +} + +.schema-details summary { + color: var(--accent); + font-size: 13px; + font-weight: 700; + cursor: pointer; +} + +pre { + overflow: auto; + margin: 12px 0 0; + border-radius: 12px; + padding: 15px; + background: #17231d; + color: #d8e6dd; + font: 12px/1.65 "SFMono-Regular", Consolas, monospace; + white-space: pre-wrap; + word-break: break-word; +} + +.pipeline { + display: flex; + align-items: center; + justify-content: center; + gap: 9px; + min-height: 54px; + padding: 10px 16px; + color: var(--muted); + font-size: 12px; +} + +.pipeline span { + border: 1px solid var(--line); + border-radius: 999px; + padding: 7px 11px; + background: var(--surface); + color: var(--ink); + font-weight: 700; +} + +.pipeline i { + color: var(--accent); + font-style: normal; +} + +.device-card { + position: relative; + overflow: hidden; + border: 1px solid color-mix(in srgb, var(--accent) 16%, var(--line)); + border-radius: 18px; + padding: 20px; + background: + linear-gradient(135deg, color-mix(in srgb, var(--accent-soft) 70%, var(--surface)), var(--surface) 52%); +} + +.device-card::after { + position: absolute; + right: -50px; + top: -70px; + width: 180px; + height: 180px; + border-radius: 50%; + background: color-mix(in srgb, var(--accent) 9%, transparent); + content: ""; +} + +.device-card-header { + position: relative; + z-index: 1; + display: flex; + align-items: center; + justify-content: space-between; + gap: 14px; +} + +.device-identity { + display: flex; + align-items: center; + gap: 12px; +} + +.device-icon { + display: grid; + width: 48px; + height: 48px; + place-items: center; + border-radius: 15px; + background: var(--accent); + color: #fff; + font-size: 22px; +} + +.device-identity h3 { + margin: 0 0 4px; + font-size: 21px; +} + +.device-identity p { + margin: 0; + color: var(--muted); + font-size: 12px; +} + +.connection-state { + display: inline-flex; + align-items: center; + gap: 7px; + font-size: 12px; + font-weight: 700; + color: var(--accent-ink); +} + +.connection-state i, +.result-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--accent); +} + +.readout-grid { + position: relative; + z-index: 1; + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 10px; + margin: 20px 0; +} + +.readout-grid div { + border: 1px solid color-mix(in srgb, var(--line) 78%, transparent); + border-radius: 12px; + padding: 12px; + background: color-mix(in srgb, var(--surface) 84%, transparent); +} + +.readout-grid span, +.card-control span { + display: block; + margin-bottom: 6px; + color: var(--muted); + font-size: 11px; + font-weight: 700; +} + +.readout-grid strong { + font-size: 16px; +} + +.card-controls { + position: relative; + z-index: 1; + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 12px; +} + +.card-control input, +.card-control select { + background: color-mix(in srgb, var(--surface) 88%, transparent); +} + +.card-control output { + float: right; + color: var(--accent); + font-size: 12px; + font-weight: 800; +} + +.card-control input[type="range"] { + min-height: auto; + padding: 0; + accent-color: var(--accent); +} + +.card-footnote { + position: relative; + z-index: 1; + margin: 15px 0 0; + color: var(--muted); + font-size: 11px; +} + +.example-row { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-bottom: 12px; +} + +.example-row button, +.text-button { + border: 0; + background: transparent; + color: var(--accent); + font-size: 12px; + font-weight: 700; +} + +.example-row button { + border-radius: 999px; + padding: 7px 10px; + background: var(--accent-soft); +} + +.command-composer { + display: grid; + grid-template-columns: 1fr auto; + gap: 10px; + align-items: stretch; +} + +.command-composer textarea { + min-height: 64px; +} + +.result-banner { + display: flex; + align-items: center; + gap: 12px; + margin: 14px 0; + border-radius: 13px; + padding: 13px 15px; + background: var(--surface-soft); +} + +.result-banner[data-status="success"] { + background: var(--accent-soft); + color: var(--accent-ink); +} + +.result-banner[data-status="warning"] { + background: var(--warning-soft); + color: var(--warning); +} + +.result-banner[data-status="error"] { + background: var(--danger-soft); + color: var(--danger); +} + +.result-banner[data-status="warning"] .result-dot { + background: var(--warning); +} + +.result-banner[data-status="error"] .result-dot { + background: var(--danger); +} + +.result-banner strong { + display: block; + margin-bottom: 3px; + font-size: 14px; +} + +.result-banner p { + margin: 0; + font-size: 12px; + line-height: 1.55; +} + +.result-grid { + display: grid; + grid-template-columns: minmax(240px, 0.85fr) minmax(280px, 1.15fr); + gap: 14px; +} + +.trace-panel, +.payload-panel { + min-width: 0; + border: 1px solid var(--line); + border-radius: 14px; + padding: 15px; +} + +.trace-panel h3, +.payload-heading h3 { + margin: 0; + font-size: 14px; +} + +.trace-panel ol { + display: grid; + gap: 12px; + margin: 15px 0 0; + padding: 0; + list-style: none; + counter-reset: trace; +} + +.trace-panel li { + position: relative; + min-height: 30px; + padding-left: 34px; + color: var(--muted); + font-size: 12px; + line-height: 1.5; + counter-increment: trace; +} + +.trace-panel li::before { + position: absolute; + left: 0; + top: 0; + display: grid; + width: 24px; + height: 24px; + place-items: center; + border-radius: 8px; + background: var(--surface-strong); + color: var(--ink); + content: counter(trace); + font-size: 11px; + font-weight: 800; +} + +.trace-panel li[data-state="success"]::before { + background: var(--accent-soft); + color: var(--accent-ink); +} + +.trace-panel li[data-state="error"]::before { + background: var(--danger-soft); + color: var(--danger); +} + +.trace-panel li strong { + display: block; + color: var(--ink); +} + +.payload-heading { + display: flex; + align-items: center; + justify-content: space-between; +} + +.payload-panel pre { + min-height: 216px; + max-height: 360px; +} + +.sr-only { + position: absolute; + width: 1px; + height: 1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; +} + +@media (prefers-color-scheme: dark) { + :root { + --page: #111713; + --surface: #19211c; + --surface-soft: #202923; + --surface-strong: #29342d; + --ink: #edf4ef; + --muted: #a4b1a8; + --line: #354139; + --accent: #58b98a; + --accent-soft: #243f32; + --accent-ink: #bdebd3; + --warning: #f2c873; + --warning-soft: #463b23; + --danger: #f0a09a; + --danger-soft: #4b2928; + --shadow: 0 18px 55px rgba(0, 0, 0, 0.22); + } + + .primary-button, + .device-icon, + .safety-note > span { + color: #102019; + } +} + +@media (max-width: 980px) { + .workspace-grid { + grid-template-columns: 1fr; + } +} + +@media (max-width: 640px) { + .demo-shell { + width: min(100% - 24px, 1440px); + padding-top: 24px; + } + + .page-header { + align-items: flex-start; + flex-direction: column; + } + + .pipeline { + justify-content: flex-start; + overflow-x: auto; + } + + .pipeline span { + white-space: nowrap; + } + + .field-grid, + .card-controls, + .result-grid { + grid-template-columns: 1fr; + } + + .field-wide { + grid-column: auto; + } + + .readout-grid { + grid-template-columns: 1fr 1fr; + } + + .command-composer { + grid-template-columns: 1fr; + } +} diff --git a/src/cognitive_agent/application/chinese_values.py b/src/cognitive_agent/application/chinese_values.py index e16a2e7..f9dbf37 100644 --- a/src/cognitive_agent/application/chinese_values.py +++ b/src/cognitive_agent/application/chinese_values.py @@ -15,6 +15,12 @@ _FRACTION = re.compile( r"([零〇一二两三四五六七八九十百点\d.]+)" ) _SLASH_FRACTION = re.compile(r"(?[零〇一二两三四五六七八九十百]+" + r"(?:点[零〇一二两三四五六七八九]+)?)" + r"(?P\s*)" + r"(?P摄氏度|℃|度|档|级)" +) _DIGITS = { "零": 0, "〇": 0, @@ -76,6 +82,24 @@ def _number(token: str) -> int | float | None: return int(value) if value.is_integer() else value +def normalize_chinese_unit_values(text: str) -> str: + """Convert explicitly unit-qualified Chinese numbers to Arabic digits. + + Keeping the unit in the match is intentional: discourse words such as + ``一下`` or ``一点`` must not accidentally become control values. Both + typed text and ASR text can call this helper before their domain grammar, + so value recognition does not depend on which input path was used. + """ + + def replace(match: re.Match[str]) -> str: + value = _number(match.group("value")) + if value is None: + return match.group(0) + return f"{value}{match.group('spacing')}{match.group('unit')}" + + return _CHINESE_UNIT_VALUE.sub(replace, text) + + def parse_percentage_value(text: str) -> int | float | None: """Return an explicitly spoken percentage/ratio/extreme position. diff --git a/src/cognitive_agent/application/native_can_intent.py b/src/cognitive_agent/application/native_can_intent.py index b4645b4..c64ace6 100644 --- a/src/cognitive_agent/application/native_can_intent.py +++ b/src/cognitive_agent/application/native_can_intent.py @@ -9,7 +9,10 @@ from __future__ import annotations import re from typing import Any -from cognitive_agent.application.chinese_values import parse_percentage_value +from cognitive_agent.application.chinese_values import ( + normalize_chinese_unit_values, + parse_percentage_value, +) WINDOW_TARGETS = { "全部": "all", @@ -177,7 +180,8 @@ _EXPLICIT_TARGET_PATTERNS = { re.IGNORECASE, ), "seat": re.compile( - r"主驾|驾驶位|驾驶席|司机位|左前(?:座|座椅|座位)?|" + r"主驾|主驾驶(?:座|座椅|席)?|驾驶位|驾驶席|驾驶座(?:椅)?|" + r"司机位|司机座(?:椅)?|左前(?:座|座椅|座位)?|" r"副驾(?:驶|位)?|乘客(?:位|座|座椅|座位)?|右前(?:座|座椅|座位)?|" r"前排|后排|左后|右后|所有座椅|全部座椅|全车座椅|" r"seat\s+(?:driver|front-left|passenger|front-right)", @@ -561,7 +565,11 @@ def _parse_chinese_driver_seat_commands(compact: str) -> list[dict[str, Any]]: clause = f"{clause}{continuation}" continuation_index += 1 index = continuation_index - 1 - if re.search(r"主驾|驾驶位|司机位|左前座椅", clause): + if re.search( + r"主驾|主驾驶(?:座|座椅|席)?|驾驶位|驾驶席|" + r"驾驶座(?:椅)?|司机位|司机座(?:椅)?|左前座椅", + clause, + ): inherited_driver_scope = True elif re.search(r"副驾|副驾驶|乘客|右前", clause): inherited_driver_scope = False @@ -592,7 +600,13 @@ def _parse_chinese_driver_seat_commands(compact: str) -> list[dict[str, Any]]: re.IGNORECASE, ): return [] - explicit_driver = bool(re.search(r"主驾|驾驶位|司机位|左前座椅", compact)) + explicit_driver = bool( + re.search( + r"主驾|主驾驶(?:座|座椅|席)?|驾驶位|驾驶席|" + r"驾驶座(?:椅)?|司机位|司机座(?:椅)?|左前座椅", + compact, + ) + ) if not explicit_driver and not _looks_like_seat_request(compact): return [] default_reason = "用户未指定座椅目标，按驾驶台 CH1 安全默认选择主驾座椅。" @@ -950,7 +964,7 @@ def _parse_chinese_window_commands(compact: str) -> list[dict[str, Any]]: def _parse_chinese_commands(text: str) -> list[dict[str, Any]]: - compact = re.sub(r"\s+", "", text) + compact = re.sub(r"\s+", "", normalize_chinese_unit_values(text)) seat_text = _scoped_text( compact, r"座椅|座位|靠背|椅背|坐盆|坐垫|腿托|脚托|按摩|记忆|预设|" @@ -981,7 +995,7 @@ def _parse_chinese_commands(text: str) -> list[dict[str, Any]]: temperature = re.search( r"(?:空调(?:温度)?|温度)(?:都)?" - r"(?:调整到|调整为|调到|调为|设为|设置为|调至|到|为)?" + r"(?:调整到|调整为|调到|调成|调为|设到|设为|设置为|调至|到|为)?" r"(1[89]|2\d|3[0-2])(?:摄氏)?(?:℃|度)?(?![\d.])", climate_text, ) @@ -1340,6 +1354,7 @@ def _drop_implicit_passenger_ch0_candidates( ambiguities.append( { "field": "seat.target_scope", + "reason": "implicit_driver_default_unresolved", "question": ( "未指定座椅目标时默认主驾 CH1；当前请求无法安全映射为已验证的主驾能力，" "且不会转入副驾 CH0。" diff --git a/src/cognitive_agent/application/seat_service.py b/src/cognitive_agent/application/seat_service.py index bc04595..46484fa 100755 --- a/src/cognitive_agent/application/seat_service.py +++ b/src/cognitive_agent/application/seat_service.py @@ -133,7 +133,9 @@ CONVERSATION_ID_PATTERN = re.compile(r"^conv-[a-z0-9-]{6,80}$") MODE_EDITOR_REQUEST_ID_PATTERN = re.compile(r"^req-[a-z0-9-]{6,100}$") SEAT_SCOPE_PATTERN = re.compile(r"\bseats?\b|座椅|坐椅", re.IGNORECASE) DRIVER_SEAT_SCOPE_PATTERN = re.compile( - r"\bdriver\b|\bfront[-_ ]?left\b|\bleft[-_ ]?front\b|主驾|驾驶位|司机位|左前", + r"\bdriver\b|\bfront[-_ ]?left\b|\bleft[-_ ]?front\b|" + r"主驾|主驾驶(?:座|座椅|席)?|驾驶位|驾驶席|驾驶座(?:椅)?|" + r"司机位|司机座(?:椅)?|左前", re.IGNORECASE, ) PASSENGER_SEAT_SCOPE_PATTERN = re.compile( @@ -2257,13 +2259,9 @@ class SeatDemoService: local_preview=local_preview, model_parse_source=model_parse_source, ) - if self._is_dynamic_mode_stop_request(utterance): + dynamic_mode_stop = self._is_dynamic_mode_stop_request(utterance) + if dynamic_mode_stop: parsed = self._apply_dynamic_mode_stop_request(parsed, utterance) - else: - parsed = self._apply_mode_profiles( - parsed, matched_modes, context, vehicle_profile - ) - parsed = self._mark_control_parse_source(parsed, "local_rule") relative_resolution = ( self.native_can_relative_resolver.resolve(utterance, context) if self.native_can_relative_resolver.matches(utterance) @@ -2282,6 +2280,19 @@ class SeatDemoService: relative_resolution.claimed_controls if relative_resolution else None ), ) + if not dynamic_mode_stop: + # Apply repository-backed modes after the final native-CAN + # merge. Otherwise that merge can discard a valid passenger + # mode operation while enforcing its generic unscoped-seat + # CH1 default. + parsed = self._apply_mode_profiles( + parsed, + matched_modes, + context, + vehicle_profile, + utterance=utterance, + ) + parsed = self._mark_control_parse_source(parsed, "local_rule") if direct_mode_profile is not None: parsed = self._mark_control_parse_source( parsed, @@ -3327,6 +3338,8 @@ class SeatDemoService: matched_modes: list[ModeProfile], context: dict[str, Any], vehicle_profile: VehicleProfile, + *, + utterance: str, ) -> dict[str, Any]: if not matched_modes: return parsed @@ -3354,6 +3367,12 @@ class SeatDemoService: ] return rejected + parsed = self._apply_implicit_mode_target_policy( + parsed, + matched_modes, + utterance, + ) + safety_ambiguities = [ item for item in parsed.get("ambiguities", []) @@ -3430,6 +3449,38 @@ class SeatDemoService: ) return merged + @staticmethod + def _apply_implicit_mode_target_policy( + parsed: dict[str, Any], + matched_modes: list[ModeProfile], + utterance: str, + ) -> dict[str, Any]: + """Let one matched mode own only an omitted seat target. + + The native CH1 parser can add an implicit-driver ambiguity before or + after mode injection. A unique repository-backed mode already carries + authoritative target scopes on every operation, so that one tagged + default is redundant. Explicit scopes and all other safety questions + remain untouched. + """ + + if len(matched_modes) != 1 or ( + DRIVER_SEAT_SCOPE_PATTERN.search(utterance) + or PASSENGER_SEAT_SCOPE_PATTERN.search(utterance) + ): + return parsed + ambiguities = list(parsed.get("ambiguities", [])) + filtered = [ + item + for item in ambiguities + if item.get("reason") != "implicit_driver_default_unresolved" + ] + if len(filtered) == len(ambiguities): + return parsed + resolved = deepcopy(parsed) + resolved["ambiguities"] = filtered + return resolved + @staticmethod def _is_dynamic_mode_stop_request(utterance: str) -> bool: compact = re.sub(r"\s+", "", utterance) diff --git a/src/cognitive_agent/infrastructure/speech/capability.py b/src/cognitive_agent/infrastructure/speech/capability.py index 5aea60a..e1f8e0a 100644 --- a/src/cognitive_agent/infrastructure/speech/capability.py +++ b/src/cognitive_agent/infrastructure/speech/capability.py @@ -165,6 +165,7 @@ class CapabilityResolver: *, target_scope: str | None, target_explicit: bool, + action: str = "apply", ) -> ModeResolution: matches = self.mode_profiles.match_text(text, vehicle_id="denza_n7") if not matches: @@ -172,7 +173,24 @@ class CapabilityResolver: if len(matches) > 1: return ModeResolution("ambiguous", None, ("multiple_mode_profiles_matched",)) profile = matches[0] - operation_scopes = {operation.target_scope for operation in profile.operations} + selected_operations = profile.operations + if action == "stop": + dynamic_operations = tuple( + operation + for operation in profile.operations + if operation.capability == "seat.start_dynamic_mode" + ) + if not dynamic_operations: + return ModeResolution( + "unsupported", + profile, + (f"mode_profile.{profile.id}不支持关闭指令",), + ) + selected_operations = dynamic_operations + + operation_scopes = { + operation.target_scope for operation in selected_operations + } if target_explicit and target_scope not in operation_scopes: scopes = ",".join(sorted(operation_scopes)) return ModeResolution( @@ -183,22 +201,28 @@ class CapabilityResolver: payloads: list[dict[str, Any]] = [] issues: list[str] = [] - for operation in profile.operations: + for operation in selected_operations: + capability = ( + "seat.stop_dynamic_mode" + if action == "stop" + else operation.capability + ) + parameters = {} if action == "stop" else operation.parameters resolution = self.resolve( domain=operation.domain, semantic_property="", target_scope=operation.target_scope, target_device=None, - operation=operation.operation, - parameters=operation.parameters, + operation="set" if action == "stop" else operation.operation, + parameters=parameters, source_text=text, - preferred_capability=operation.capability, + preferred_capability=capability, ) if resolution.status != "executable" or resolution.executor_payload is None: issues.extend(resolution.issues or ("mode_operation_not_executable",)) continue payloads.append(resolution.executor_payload) - if issues or len(payloads) != len(profile.operations): + if issues or len(payloads) != len(selected_operations): return ModeResolution("unsupported", profile, tuple(dict.fromkeys(issues))) return ModeResolution("executable", profile, executor_payloads=tuple(payloads)) diff --git a/src/cognitive_agent/infrastructure/speech/understanding.py b/src/cognitive_agent/infrastructure/speech/understanding.py index 39bc055..73d18c8 100644 --- a/src/cognitive_agent/infrastructure/speech/understanding.py +++ b/src/cognitive_agent/infrastructure/speech/understanding.py @@ -9,7 +9,10 @@ from dataclasses import dataclass, replace from pathlib import Path from typing import Any -from cognitive_agent.application.chinese_values import parse_percentage_value +from cognitive_agent.application.chinese_values import ( + normalize_chinese_unit_values, + parse_percentage_value, +) from cognitive_agent.application.native_can_intent import merge_native_can_intent from cognitive_agent.infrastructure.speech.capability import CapabilityResolver from cognitive_agent.infrastructure.speech.contracts import ( @@ -858,11 +861,13 @@ class UnderstandingTranscriptProcessor: mode_profile = self.resolver.match_mode(text) if mode_profile is not None: + mode_operation = "stop" if action == "off" else "apply" mode_started = time.perf_counter() mode_resolution = self.resolver.resolve_mode( text, target_scope=scope, target_explicit=explicit_scope, + action=mode_operation, ) resolution_ms += (time.perf_counter() - mode_started) * 1000 mode_issues = tuple(mode_resolution.issues) @@ -873,7 +878,7 @@ class UnderstandingTranscriptProcessor: target_scope=command_scope, target_device="seat", property_name="seat.mode", - operation="apply", + operation=mode_operation, parameters={"mode_profile_id": mode_profile.id}, value=mode_profile.name, unit=None, @@ -954,6 +959,14 @@ class UnderstandingTranscriptProcessor: parameters["state"] = action if action in {"on", "off"} else "on" if value is not None: parameters["level"] = int(value) + elif action == "on": + # The passenger-seat text parser executes a bare ON at + # its lowest supported level; verified CH1 driver control + # uses level 2 for the same language. Preserve the + # target-specific executable default in speech semantics, + # but keep clean_text faithful to what the user said. + parameters["level"] = 2 if scope == "driver" else 1 + qualifiers["implicit_parameters"] = ["level"] else: local = ("seat_thermal_action_incomplete",) commands.append( @@ -1317,7 +1330,7 @@ class UnderstandingTranscriptProcessor: @staticmethod def _numeric_value(text: str) -> int | float | None: - text = _NON_NUMERIC_ONE.sub("", text) + text = normalize_chinese_unit_values(_NON_NUMERIC_ONE.sub("", text)) match = _NUMBER.search(text) if match: value = float(match.group(1)) @@ -1415,6 +1428,8 @@ class UnderstandingTranscriptProcessor: return f"将{self._scoped_target(command, feature)}设置为{level}" return f"{'打开' if command.operation == 'on' else '关闭'}{self._scoped_target(command, feature)}" if prop == "seat.mode": + if command.operation == "stop": + return f"关闭{command.value}" return f"将{self._scoped_target(command, '座椅')}调整为{command.value}" if prop == "seat.backrest_recline": direction = ( diff --git a/src/cognitive_agent/infrastructure/speech/vocabulary.py b/src/cognitive_agent/infrastructure/speech/vocabulary.py index 3ce8275..5477a6d 100644 --- a/src/cognitive_agent/infrastructure/speech/vocabulary.py +++ b/src/cognitive_agent/infrastructure/speech/vocabulary.py @@ -212,6 +212,9 @@ ACTION_ALIASES: dict[str, tuple[str, ...]] = { "调成", "调为", "变成", + "进入", + "切换到", + "切换为", "设为", "设成", "设到", diff --git a/src/cognitive_agent/server.py b/src/cognitive_agent/server.py index 87be433..d6f37b3 100755 --- a/src/cognitive_agent/server.py +++ b/src/cognitive_agent/server.py @@ -532,6 +532,8 @@ class SeatDemoHandler(BaseHTTPRequestHandler): speech_utterance_id = str( payload.get("speech_utterance_id", "") ).strip() + speech_understanding: dict[str, object] | None = None + speech_peripheral_route = False if speech_utterance_id: if path != "/api/requests": raise ValueError( @@ -540,13 +542,28 @@ class SeatDemoHandler(BaseHTTPRequestHandler): speech_understanding = self.speech.cached_result( speech_utterance_id ) - if str(speech_understanding.get("clean_text", "")).strip() != utterance: + clean_text = str( + speech_understanding.get("clean_text", "") + ).strip() + raw_text = str( + speech_understanding.get("raw_text", "") + ).strip() + peripheral_routing = self._speech_peripheral_routing(raw_text) + peripheral_transcript = bool( + peripheral_routing.get("execution_eligible") and raw_text + ) + expected_text = raw_text if peripheral_transcript else clean_text + if utterance != expected_text: raise SpeechRecognitionError( "speech_text_changed", "语音文本已被修改，请确认修改后的内容再执行。", status=409, ) - if speech_understanding.get("execution_eligible") is not True: + speech_peripheral_route = peripheral_transcript + if ( + speech_understanding.get("execution_eligible") is not True + and not speech_peripheral_route + ): raise SpeechRecognitionError( "speech_not_executable", "语音意图尚未通过能力或参数校验，已拒绝执行。", @@ -554,7 +571,8 @@ class SeatDemoHandler(BaseHTTPRequestHandler): ) provider = "local" common_arguments["provider"] = provider - common_arguments["_speech_understanding"] = speech_understanding + if not speech_peripheral_route: + common_arguments["_speech_understanding"] = speech_understanding if ( path != "/api/can/commands" and "application/x-ndjson" in self.headers.get("Accept", "") @@ -620,6 +638,14 @@ class SeatDemoHandler(BaseHTTPRequestHandler): ) else: result = self.service.process(**common_arguments) + if speech_peripheral_route and speech_understanding is not None: + result["speech_understanding"] = { + "utterance_id": speech_utterance_id, + "raw_text": speech_understanding.get("raw_text"), + "clean_text": speech_understanding.get("clean_text"), + "semantic_status": speech_understanding.get("semantic_status"), + "routing": "validated_peripheral_text", + } utterance = str(result.get("utterance") or utterance) self.vehicle_display_sync.update_task( status=str(result.get("status", "succeeded")), @@ -815,6 +841,54 @@ class SeatDemoHandler(BaseHTTPRequestHandler): ) return result + def _speech_peripheral_routing(self, utterance: str) -> dict[str, object]: + """Validate whether an ASR transcript may enter peripheral routing. + + Vehicle speech keeps using its cached structured semantic payload. A + learned peripheral cannot be represented by the static vehicle + capability catalog, so it may instead reuse the exact server-owned ASR + transcript only after the live NFC binding and learned action resolve. + """ + + text = str(utterance or "").strip() + if not text: + return {"execution_eligible": False, "action_count": 0} + try: + catalog = self.extension.list_peripheral_capabilities() + nfc_snapshot = self.nfc.snapshot() + partition = partition_active_peripheral_intent( + text, + catalog, + nfc_snapshot, + ) + entries = list(partition.get("peripheral_actions") or []) + unresolved = list( + partition.get("unresolved_peripheral_clauses") or [] + ) + denied = list(partition.get("denied_peripheral_actions") or []) + plan_catalog = active_peripheral_plan_catalog( + text, + catalog, + nfc_snapshot, + ) + direct = None + if not entries and not plan_catalog and not unresolved and not denied: + direct = resolve_active_peripheral_command( + text, + catalog, + nfc_snapshot, + ) + except (ExtensionIntegrationError, GatewayError, ValueError): + return {"execution_eligible": False, "action_count": 0} + action_count = len(entries) or (1 if direct is not None else 0) + if plan_catalog: + action_count = max(action_count, 1) + return { + "execution_eligible": bool(action_count and not unresolved and not denied), + "action_count": action_count, + "has_ordered_plan": bool(plan_catalog), + } + @staticmethod def _residual_has_vehicle_language(utterance: str) -> bool: """Recognize only obvious vehicle-domain text in a claimed mixed request.""" @@ -2206,6 +2280,14 @@ class SeatDemoHandler(BaseHTTPRequestHandler): self.rfile.read(content_length), utterance_id=self.headers.get("X-Speech-Utterance-Id"), ) + raw_text = str(result.get("raw_text") or "").strip() + peripheral_routing = self._speech_peripheral_routing(raw_text) + if peripheral_routing.get("execution_eligible"): + result = { + **result, + "routing_text": raw_text, + "peripheral_routing": peripheral_routing, + } self._json(result) except SpeechRecognitionError as exc: self._json( diff --git a/tests/e2e/test_seat_web_api.py b/tests/e2e/test_seat_web_api.py index 24aa64a..30fae6f 100755 --- a/tests/e2e/test_seat_web_api.py +++ b/tests/e2e/test_seat_web_api.py @@ -624,7 +624,7 @@ operations: ) as response: gesture_script = response.read().decode("utf-8") with self.opener.open( - f"{self.base_url}/console-v2.js?v=delivery-state-54", timeout=3 + f"{self.base_url}/console-v2.js?v=delivery-state-55", timeout=3 ) as response: console_script = response.read().decode("utf-8") with self.opener.open( @@ -632,7 +632,7 @@ operations: ) as response: console_styles = response.read().decode("utf-8") with self.opener.open( - f"{self.base_url}/speech-input.js?v=sensevoice-6", timeout=3 + f"{self.base_url}/speech-input.js?v=sensevoice-7", timeout=3 ) as response: speech_script = response.read().decode("utf-8") with self.opener.open( @@ -685,9 +685,9 @@ operations: self.assertIn('href="/styles.css?v=dual-seat-21"', html) self.assertIn('src="/gesture-control.js?v=android-pad-9"', html) self.assertNotIn('class="vehicle-system-hud"', html) - self.assertIn('src="/console-v2.js?v=delivery-state-54"', html) + self.assertIn('src="/console-v2.js?v=delivery-state-55"', html) self.assertIn('href="/console-v2.css?v=delivery-state-50"', html) - self.assertIn('src="/speech-input.js?v=sensevoice-6"', html) + self.assertIn('src="/speech-input.js?v=sensevoice-7"', html) self.assertIn('href="/speech-input.css?v=sensevoice-5"', html) self.assertIn('id="speechDictationButton"', html) self.assertIn('class="speech-waveform"', html) @@ -959,9 +959,9 @@ operations: self.assertIn("CAN 指令已提交，等待车辆状态确认", console_script) self.assertIn('canvas.addEventListener("webglcontextlost"', app_script) self.assertIn('"cognitive-agent:module-change"', app_script) - self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v227"', service_worker) + self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v228"', service_worker) self.assertIn('"/app.js?v=dual-seat-114"', service_worker) - self.assertIn('"/speech-input.js?v=sensevoice-6"', service_worker) + self.assertIn('"/speech-input.js?v=sensevoice-7"', service_worker) self.assertIn('"/speech-input.css?v=sensevoice-5"', service_worker) self.assertIn('"/speech-audio-worklet.js?v=sensevoice-4"', service_worker) self.assertIn('fetch("/api/speech/transcribe"', speech_script) @@ -979,6 +979,9 @@ operations: self.assertIn("speechExecutionEligible", app_script) self.assertIn("speech_utterance_id", app_script) self.assertIn("speechUtteranceId", speech_script) + self.assertIn("result.peripheral_routing?.execution_eligible", speech_script) + self.assertIn("result.routing_text", speech_script) + self.assertIn('speech_semantic: "语音语义"', console_script) self.assertIn("语音意图已识别，但当前车辆能力不支持执行", app_script) self.assertIn("语音控制参数不完整，请补充后再执行", app_script) self.assertIn(".speech-dictation.is-listening", speech_styles) @@ -1741,6 +1744,143 @@ operations: self.assertEqual(operation["domain"], "self_extension") self.assertTrue(operation["telemetry"]["gateway_acknowledged"]) + def test_speech_routes_exact_asr_text_to_learned_peripheral_and_ui_config( + self, + ) -> None: + original_extension = SeatDemoHandler.extension + original_nfc = SeatDemoHandler.nfc + original_speech = SeatDemoHandler.speech + extension = SelfExtensionService( + SimulationGateway(), + catalog_db_path=Path( + self.extension_temporary_directory.name, + "speech-nfc-route.sqlite3", + ), + ) + extension.sync_peripheral_catalog( + [ + { + "id": "cap-third-party-light-on", + "status": "learned", + "enabled": True, + "device_id": "third-party-light", + "device_name": "第三方氛围灯", + "object_names": ["第三方氛围灯"], + "card_uid": "1A53DD95", + "action": "开启", + "action_names": ["打开", "开启"], + "slot": 9, + } + ] + ) + SeatDemoHandler.extension = extension + SeatDemoHandler.nfc = SimpleNamespace( + snapshot=lambda: { + "revision": 1, + "reader": {"online": True}, + "card": {"action": "present", "uid": "1A53DD95"}, + } + ) + understanding = UnderstandingTranscriptProcessor(REPOSITORY_ROOT).process( + "打开第三方氛围灯" + ).to_dict() + understanding["utterance_id"] = "e2e-speech-peripheral" + + class SpeechStub: + @staticmethod + def transcribe_wav( + content: bytes, + *, + utterance_id: str | None = None, + ) -> dict[str, object]: + if content != b"RIFF-speech-peripheral": + raise AssertionError("unexpected audio payload") + return {**understanding, "utterance_id": utterance_id} + + @staticmethod + def cached_result(utterance_id: str) -> dict[str, object]: + if utterance_id != "e2e-speech-peripheral": + raise AssertionError("unexpected speech utterance id") + return dict(understanding) + + SeatDemoHandler.speech = SpeechStub() + try: + transcription_request = urllib.request.Request( + f"{self.base_url}/api/speech/transcribe", + method="POST", + headers={ + "Content-Type": "audio/wav", + "X-Speech-Utterance-Id": "e2e-speech-peripheral", + }, + data=b"RIFF-speech-peripheral", + ) + with self.opener.open(transcription_request, timeout=3) as response: + transcription = json.loads(response.read().decode("utf-8")) + + wrong_target_request = urllib.request.Request( + f"{self.base_url}/api/requests", + method="POST", + headers={"Content-Type": "application/json"}, + data=json.dumps( + { + "utterance": understanding["clean_text"], + "speech_utterance_id": "e2e-speech-peripheral", + "provider": "local", + "conversation_id": "conv-e2e-speech-wrong-target", + "execution_mode": "dry_run", + }, + ensure_ascii=False, + ).encode("utf-8"), + ) + with self.assertRaises(urllib.error.HTTPError) as rejected: + self.opener.open(wrong_target_request, timeout=3) + self.assertEqual(rejected.exception.code, 409) + rejected_payload = json.loads( + rejected.exception.read().decode("utf-8") + ) + self.assertEqual(rejected_payload["error"], "speech_text_changed") + + control_request = urllib.request.Request( + f"{self.base_url}/api/requests", + method="POST", + headers={"Content-Type": "application/json"}, + data=json.dumps( + { + "utterance": transcription["routing_text"], + "speech_utterance_id": "e2e-speech-peripheral", + "provider": "openai", + "conversation_id": "conv-e2e-speech-peripheral", + "execution_mode": "dry_run", + "client_type": "web_console", + }, + ensure_ascii=False, + ).encode("utf-8"), + ) + with self.opener.open(control_request, timeout=3) as response: + result = json.loads(response.read().decode("utf-8")) + finally: + SeatDemoHandler.extension = original_extension + SeatDemoHandler.nfc = original_nfc + SeatDemoHandler.speech = original_speech + + self.assertEqual(transcription["routing_text"], "打开第三方氛围灯") + self.assertTrue( + transcription["peripheral_routing"]["execution_eligible"] + ) + self.assertEqual(result["status"], "succeeded") + self.assertEqual(result["orchestration_route"], "nfc_peripheral") + self.assertEqual(set(result["function_configs"]), {"self_extension"}) + operation = result["function_configs"]["self_extension"]["spec"][ + "operations" + ][0] + self.assertEqual(operation["capability"], "infrared.emit") + self.assertEqual(operation["parameters"]["slot"], 9) + self.assertEqual(operation["parse_source"], "local_rule") + self.assertEqual( + result["speech_understanding"]["routing"], + "validated_peripheral_text", + ) + def test_main_request_combines_explicit_ir_clause_with_existing_can_parser(self) -> None: original_extension = SeatDemoHandler.extension original_nfc = SeatDemoHandler.nfc diff --git a/tests/fixtures/speech_text_regression.json b/tests/fixtures/speech_text_regression.json index 4be1548..317f853 100644 --- a/tests/fixtures/speech_text_regression.json +++ b/tests/fixtures/speech_text_regression.json @@ -58,5 +58,8 @@ {"name":"elliptical_temperature","raw":"空调二十四度","clean":"将空调设置为24度。","capabilities":["climate.set_temperature"],"cancelled":0,"uncertain":false,"execution_eligible":true}, {"name":"elliptical_value_continuation","raw":"打开空调，然后二十四度","clean":"打开空调，然后将空调设置为24度。","capabilities":["climate.set_power","climate.set_temperature"],"cancelled":0,"uncertain":false,"execution_eligible":true}, {"name":"punctuated_ambient_slots","raw":"氛围灯打开，调成蓝色，亮度百分之五十","clean":"打开氛围灯，然后将氛围灯颜色调整为蓝色并将亮度设置为50%。","capabilities":["ambient_light.set_power","ambient_light.set_illumination"],"cancelled":0,"uncertain":false,"execution_eligible":true}, - {"name":"non_control_continuation_fails_closed","raw":"打开空调，然后开玩笑。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":true,"execution_eligible":false} + {"name":"non_control_continuation_fails_closed","raw":"打开空调，然后开玩笑。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":true,"execution_eligible":false}, + {"name":"rocking_horse_mode_unscoped","raw":"将座椅调整为木马模式。","clean":"将座椅调整为木马模式。","capabilities":[null],"cancelled":0,"uncertain":false,"execution_eligible":true}, + {"name":"rocking_horse_mode_enter","raw":"进入木马模式","clean":"将座椅调整为木马模式。","capabilities":[null],"cancelled":0,"uncertain":false,"execution_eligible":true}, + {"name":"rocking_horse_mode_stop","raw":"关闭木马模式","clean":"关闭木马模式。","capabilities":[null],"cancelled":0,"uncertain":false,"execution_eligible":true} ] diff --git a/tests/unit/test_frontend_theme_contract.py b/tests/unit/test_frontend_theme_contract.py index e54f89b..7732f31 100644 --- a/tests/unit/test_frontend_theme_contract.py +++ b/tests/unit/test_frontend_theme_contract.py @@ -47,7 +47,7 @@ def test_theme_asset_versions_are_consistent_with_the_pwa_shell() -> None: ): assert asset in html assert f'"{asset}"' in service_worker - assert 'const CACHE_VERSION = "cognitive-agent-pad-v227"' in service_worker + assert 'const CACHE_VERSION = "cognitive-agent-pad-v228"' in service_worker def test_vehicle_twin_icons_and_nested_overlays_use_semantic_tokens() -> None: diff --git a/tests/unit/test_nfc_frontend_contract.py b/tests/unit/test_nfc_frontend_contract.py index 8ddd18b..52c584b 100644 --- a/tests/unit/test_nfc_frontend_contract.py +++ b/tests/unit/test_nfc_frontend_contract.py @@ -139,4 +139,5 @@ def test_main_parsed_control_panel_renders_validated_infrared_operations() -> No assert 'frameLabel: "GSCU1B TCP"' in console assert "已解析控制指令" in console assert 'li[data-can-channel="IR"]' in styles - assert "console-v2.js?v=delivery-state-54" in html + assert "console-v2.js?v=delivery-state-55" in html + assert 'speech_semantic: "语音语义"' in console diff --git a/tests/unit/test_requested_frontend_enhancements.py b/tests/unit/test_requested_frontend_enhancements.py index c37bc30..268d415 100644 --- a/tests/unit/test_requested_frontend_enhancements.py +++ b/tests/unit/test_requested_frontend_enhancements.py @@ -568,6 +568,73 @@ def test_mode_editor_prepares_one_to_many_mqtt_message_sequences() -> None: ) +def test_left_sidebar_extracts_speech_and_peripheral_control_commands() -> None: + node = shutil.which("node") + if node is None: + return + console = source("console-v2.js") + helper_start = console.index(" const normalizedCanTarget") + helper_end = console.index("\n\n function renderParsedCanCommands", helper_start) + helpers = console[helper_start:helper_end] + sample = { + "execution_mode": "dry_run", + "function_configs": { + "native_climate": { + "spec": { + "operations": [ + { + "id": "speech-temperature", + "capability": "climate.set_temperature", + "target": {"device": "hvac", "zone": "driver"}, + "parameters": {"temperature_c": 24}, + "parse_source": "speech_semantic", + } + ] + } + }, + "self_extension": { + "spec": { + "operations": [ + { + "id": "speech-peripheral", + "capability": "infrared.emit", + "target": { + "device": "third-party-light", + "name": "第三方氛围灯", + }, + "parameters": {"action": "开启", "slot": 9}, + "parse_source": "local_rule", + } + ] + } + }, + }, + } + script = ( + f"{helpers}\n" + f"const result = {json.dumps(sample, ensure_ascii=False)};\n" + "process.stdout.write(JSON.stringify({" + "commands:parsedCanCommands(result)," + 'speechLabel:parsedCommandSourceLabel("speech_semantic")' + "}));" + ) + completed = subprocess.run( + [node, "-e", script], + check=True, + capture_output=True, + text=True, + ) + output = json.loads(completed.stdout) + + assert output["speechLabel"] == "语音语义" + assert len(output["commands"]) == 2 + assert {item["channel"] for item in output["commands"]} == {"CAN1", "IR"} + assert {item["command"] for item in output["commands"]} == { + "ac temp driver 24", + "infrared emit 9", + } + + def test_interactive_console_keeps_actions_inside_composer_and_collapsed_rails_aligned() -> None: console = source("console-v2.js") styles = source("console-v2.css") diff --git a/tests/unit/test_speech_capability_alignment.py b/tests/unit/test_speech_capability_alignment.py index 8c5a179..47ca6ed 100644 --- a/tests/unit/test_speech_capability_alignment.py +++ b/tests/unit/test_speech_capability_alignment.py @@ -148,7 +148,7 @@ class SpeechCapabilityAlignmentTests(unittest.TestCase): "passenger", "seat.ventilation", "on", - {"state": "on"}, + {"state": "on", "level": 1}, ), ), }, @@ -432,18 +432,38 @@ class SpeechCapabilityAlignmentTests(unittest.TestCase): def test_colloquial_seat_aliases_do_not_turn_pragmatic_one_into_level(self) -> None: cases = ( - ("把驾驶座通风打开", "driver", "seat.set_driver_ventilation_native"), - ("把乘客座通风打开", "passenger", "seat.set_ventilation"), - ("给副驾驶开一下座椅通风", "passenger", "seat.set_ventilation"), + ( + "把驾驶座通风打开", + "driver", + "seat.set_driver_ventilation_native", + {"state": "on", "level": 2}, + ), + ( + "把乘客座通风打开", + "passenger", + "seat.set_ventilation", + {"state": "on", "level": 1}, + ), + ( + "给副驾驶开一下座椅通风", + "passenger", + "seat.set_ventilation", + {"state": "on", "level": 1}, + ), ) - for text, target, capability in cases: + for text, target, capability, parameters in cases: with self.subTest(text=text): result = self.processor.process(text) self.assertTrue(result.execution_eligible) self.assertEqual(len(result.commands), 1) self.assertEqual(result.commands[0].target_scope, target) self.assertEqual(result.commands[0].capability, capability) - self.assertEqual(result.commands[0].parameters, {"state": "on"}) + self.assertEqual(result.commands[0].parameters, parameters) + self.assertIn( + "level", + result.commands[0].qualifiers.get("implicit_parameters", ()), + ) + self.assertNotIn("设置为", result.clean_text) explicit_level = self.processor.process("主驾座椅通风开到一档") self.assertEqual(explicit_level.commands[0].parameters["level"], 1) @@ -455,7 +475,7 @@ class SpeechCapabilityAlignmentTests(unittest.TestCase): ) expected = ( ("seat.set_driver_ventilation_native", "driver", {"state": "on", "level": 2}), - ("seat.set_ventilation", "passenger", {"state": "on"}), + ("seat.set_ventilation", "passenger", {"state": "on", "level": 1}), ("climate.set_temperature", "driver", {"temperature_c": 24}), ) for text in cases: diff --git a/tests/unit/test_speech_orchestration_bridge.py b/tests/unit/test_speech_orchestration_bridge.py index 848844c..514924c 100644 --- a/tests/unit/test_speech_orchestration_bridge.py +++ b/tests/unit/test_speech_orchestration_bridge.py @@ -15,6 +15,44 @@ from cognitive_agent.infrastructure.speech.understanding import ( REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +def _canonical_actions(result: dict[str, object]) -> list[dict[str, object]]: + intent_ir = result.get("intent_ir") + assert isinstance(intent_ir, dict) + spec = intent_ir.get("spec") + assert isinstance(spec, dict) + actions = spec.get("actions") + assert isinstance(actions, list) + return [ + { + "domain": action.get("domain"), + "capability": action.get("capability"), + "target": action.get("target"), + "operation": action.get("operation"), + "parameters": action.get("parameters"), + } + for action in actions + if isinstance(action, dict) + ] + + +def _sorted_canonical_actions(result: dict[str, object]) -> list[dict[str, object]]: + return sorted( + _canonical_actions(result), + key=lambda action: ( + str(action["domain"]), + str(action["capability"]), + repr(action["target"]), + repr(action["parameters"]), + ), + ) + + +def _mode_service(directory: str) -> SeatDemoService: + service = SeatDemoService(REPOSITORY_ROOT) + service.artifact_root = Path(directory) / "artifacts" + return service + + @pytest.mark.parametrize( ("utterance", "capability", "target", "parameters"), ( @@ -194,3 +232,157 @@ def test_every_registered_native_runtime_capability_keeps_native_bridge_binding( assert [ action["capability"] for action in parsed["native_can_actions"] ] == [capability] + + +def test_text_and_speech_share_implicit_mode_target_and_start_command() -> None: + processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT) + understanding = processor.process("将座椅调整为木马模式。").to_dict() + understanding["utterance_id"] = "mode-start-equivalence" + + with tempfile.TemporaryDirectory() as directory, patch.dict( + os.environ, + { + "COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false", + "COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false", + "COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false", + "COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(Path(directory) / "modes"), + }, + clear=False, + ): + service = _mode_service(directory) + text_result = service.process( + "将座椅调整为木马模式。", + provider="local", + conversation_id="conv-text-mode-start", + ) + speech_result = service.process( + understanding["clean_text"], + provider="local", + conversation_id="conv-speech-mode-start", + _speech_understanding=understanding, + ) + + assert understanding["execution_eligible"] is True + assert text_result["status"] == speech_result["status"] == "succeeded" + assert _canonical_actions(text_result) == _canonical_actions(speech_result) + assert _canonical_actions(text_result) == [ + { + "domain": "seat", + "capability": "seat.start_dynamic_mode", + "target": { + "device": "seat", + "zone": "front_right", + "instance": "passenger", + }, + "operation": "activate", + "parameters": {"mode_id": 1}, + } + ] + + +def test_text_and_speech_share_dynamic_mode_stop_command() -> None: + processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT) + understanding = processor.process("关闭木马模式").to_dict() + understanding["utterance_id"] = "mode-stop-equivalence" + + with tempfile.TemporaryDirectory() as directory, patch.dict( + os.environ, + { + "COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false", + "COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false", + "COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false", + "COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(Path(directory) / "modes"), + }, + clear=False, + ): + service = _mode_service(directory) + text_result = service.process( + "关闭木马模式", + provider="local", + conversation_id="conv-text-mode-stop", + ) + speech_result = service.process( + understanding["clean_text"], + provider="local", + conversation_id="conv-speech-mode-stop", + _speech_understanding=understanding, + ) + + assert understanding["clean_text"] == "关闭木马模式。" + assert understanding["execution_eligible"] is True + assert text_result["status"] == speech_result["status"] == "succeeded" + assert _canonical_actions(text_result) == _canonical_actions(speech_result) + assert _canonical_actions(text_result)[0]["capability"] == "seat.stop_dynamic_mode" + assert _canonical_actions(text_result)[0]["parameters"] == {} + + +def test_explicit_incompatible_mode_target_fails_closed_for_both_inputs() -> None: + processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT) + understanding = processor.process("将主驾座椅调整为木马模式。").to_dict() + + with tempfile.TemporaryDirectory() as directory, patch.dict( + os.environ, + { + "COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false", + "COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false", + "COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false", + "COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(Path(directory) / "modes"), + }, + clear=False, + ): + service = _mode_service(directory) + text_result = service.process( + "将主驾座椅调整为木马模式。", + provider="local", + conversation_id="conv-text-mode-rejected", + ) + + assert understanding["semantic_status"] == "recognized_unsupported" + assert understanding["execution_eligible"] is False + assert text_result["status"] == "needs_confirmation" + assert _canonical_actions(text_result) == [] + + +def test_text_and_speech_share_colloquial_multidomain_commands() -> None: + utterance = "先打开主驾座椅通风，再打开副驾座椅通风，最后把空调调成二十四度" + processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT) + understanding = processor.process(utterance).to_dict() + understanding["utterance_id"] = "multidomain-equivalence" + + with tempfile.TemporaryDirectory() as directory, patch.dict( + os.environ, + { + "COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false", + "COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false", + "COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false", + }, + clear=False, + ): + service = _mode_service(directory) + text_result = service.process( + utterance, + provider="local", + conversation_id="conv-text-multidomain", + ) + speech_result = service.process( + understanding["clean_text"], + provider="local", + conversation_id="conv-speech-multidomain", + _speech_understanding=understanding, + ) + + assert understanding["clean_text"] == ( + "打开主驾座椅通风，然后打开副驾座椅通风，然后将空调设置为24度。" + ) + assert understanding["execution_eligible"] is True + assert text_result["status"] == speech_result["status"] == "succeeded" + assert _sorted_canonical_actions(text_result) == _sorted_canonical_actions( + speech_result + ) + assert { + action["capability"] for action in _canonical_actions(text_result) + } == { + "seat.set_driver_ventilation_native", + "seat.set_ventilation", + "climate.set_temperature", + } diff --git a/tests/unit/test_two_stage_can_parsing.py b/tests/unit/test_two_stage_can_parsing.py index 3c6a675..b79b483 100644 --- a/tests/unit/test_two_stage_can_parsing.py +++ b/tests/unit/test_two_stage_can_parsing.py @@ -186,9 +186,10 @@ def test_frontend_two_stage_contract_and_source_badges() -> None: assert '"cognitive-agent:validated-can-commands"' in app assert "本地规则命令已执行；云端补充失败" in app assert 'class="v2-can-command-source"' in console - assert 'item.parseSource === "direct_control" ? "设备控件"' in console - assert 'item.parseSource === "saved_mode_command_flow" ? "模式卡片"' in console - assert 'item.parseSource === "local_rule" ? "本地规则" : "云端模型"' in console + assert 'direct_control: "设备控件"' in console + assert 'speech_semantic: "语音语义"' in console + assert 'saved_mode_command_flow: "模式卡片"' in console + assert 'local_rule: "本地规则"' in console assert 'item.deliveryLabel' in console assert 'data-delivery-state="${escapeHtml(item.deliveryState)}"' in console assert '"cognitive-agent:validated-can-commands"' in console