From 3fc6f2c2a41bb0f7c944b93fe32ec749814fcb7e Mon Sep 17 00:00:00 2001 From: Miayui <774713819@qq.com> Date: Fri, 11 Sep 2026 10:22:24 +0800 Subject: [PATCH] =?UTF-8?q?mqtt=E5=88=87=E6=8D=A2=E4=B8=BA=E8=BF=9C?= =?UTF-8?q?=E7=A8=8B=E6=9C=8D=E5=8A=A1=E5=99=A8=E9=83=A8=E7=BD=B2=EF=BC=8C?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=AE=80=E5=8D=95=E6=8C=87=E4=BB=A4=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E6=97=B6=E9=95=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/web-console/app.js | 40 - apps/web-console/assistant-chat.js | 22 + apps/web-console/console-v2.css | 179 +++- apps/web-console/console-v2.js | 538 ++++++++++- apps/web-console/index.html | 11 +- apps/web-console/service-worker.js | 10 +- .../n7web_remote_mqtt.example.env | 12 + domains/seat/src/seat_domain/intent.py | 3 +- src/cognitive_agent.egg-info/SOURCES.txt | 2 + .../agent/prompts/codex_cockpit_assistant.md | 1 + .../application/assistant_capabilities.py | 10 +- .../application/assistant_plan_guard.py | 6 +- .../application/codex_assistant.py | 677 +++++++++++++- .../application/native_can_intent.py | 20 +- .../application/native_can_relative.py | 3 + .../application/seat_service.py | 183 +++- .../application/self_extension.py | 243 +++-- .../infrastructure/can/hardware.py | 24 +- .../infrastructure/can/mqtt_transport.py | 16 +- .../infrastructure/llm/openai_responses.py | 5 + src/cognitive_agent/server.py | 860 ++++++++++++++---- tests/e2e/test_assistant_web_api.py | 15 +- tests/e2e/test_seat_web_api.py | 169 +++- tests/unit/assistant_stream_runtime.test.cjs | 53 ++ .../unit/test_assistant_frontend_contract.py | 11 +- ..._assistant_validated_control_projection.py | 410 +++++++++ tests/unit/test_codex_assistant_service.py | 308 ++++++- .../test_energy_monitor_frontend_contract.py | 6 +- tests/unit/test_frontend_theme_contract.py | 4 +- tests/unit/test_mqtt_can_transport.py | 59 ++ tests/unit/test_nfc_frontend_contract.py | 5 +- .../test_requested_frontend_enhancements.py | 381 +++++++- tests/unit/test_scene_composer_fast_path.py | 10 +- tests/unit/test_two_stage_can_parsing.py | 478 +++++++++- 34 files changed, 4290 insertions(+), 484 deletions(-) create mode 100644 config/environments/n7web_remote_mqtt.example.env create mode 100644 tests/unit/test_assistant_validated_control_projection.py diff --git a/apps/web-console/app.js b/apps/web-console/app.js index 68eef7d..ada5115 100755 --- a/apps/web-console/app.js +++ b/apps/web-console/app.js @@ -4488,27 +4488,7 @@ function renderControlTransport() { ? runtime?.state_sync?.last_error || "" : `${id === "wireless_mqtt" ? "无线 MQTT" : "有线 CAN"} 当前未配置（仍可手动选择）`; }); - const selected = selectedVehicleTransport(); - const note = $("#vehicleTransportNote"); renderCabinLiveStatus(); - if (!note) return; - if (selected?.state_configured !== true) { - note.textContent = state.controlTransport === "wireless_mqtt" - ? "已手动选择无线控制 · MQTT 当前未配置或不可用" - : "已手动选择有线控制 · ZLG CAN 当前未配置或不可用"; - return; - } - if (state.controlTransport === "wireless_mqtt") { - const connection = transportConnectionOnline("wireless_mqtt", selected) ? "已连接" : "未连接"; - note.textContent = `MQTT ${connection} · 状态 ${selected?.condition_topic || "n7/n7-headunit/can/state"} · 控制 ${selected?.control_topic || "n7/n7-headunit/can/command"}`; - } else { - const connection = transportConnectionOnline("wired_can", selected) ? "已连接" : "未连接"; - const dualChannel = selected?.target_policy - === "dual_channel_wired_can0_passenger_can1_vehicle"; - note.textContent = dualChannel - ? `ZLG 有线 ${connection} · CAN0 副驾经典 CAN 500k · CAN1 车辆 CAN FD 500k/2M` - : `ZLG 有线 ${connection} · CH${selected?.channel ?? 1} · CAN FD 500k/2M`; - } } function selectControlTransport(transport) { @@ -6507,7 +6487,6 @@ async function loadRuntime() { mqttDualChannel ? vehicleCan.configured : hardware.configured, ); const vehicleHardwareReady = Boolean(vehicleCan.configured); - const linkedHardwareReady = seatHardwareReady && vehicleHardwareReady; const ch1DriverOnly = vehicleHardwareReady && !seatHardwareReady; const hardwareExecutionReady = Boolean(hardware.configured || vehicleCan.configured); const gptButton = $('[data-provider="openai"]'); @@ -6532,8 +6511,6 @@ async function loadRuntime() { ? "真实下发已通过后端执行配置校验" : `可手动选择真实下发；当前不可执行：${hardwareExecutionIssue(hardware, vehicleCan)}`; const simulation = state.runtime.simulated_hardware || {}; - const profileText = (hardware.firmware_profiles || []).join(" / ") || "未选择 profile"; - const flashText = hardware.auto_flash_enabled ? "自动烧录已启用" : "自动烧录未启用"; const concurrencyText = hardware.concurrency_policy === "single_flight_fail_fast" ? " · 动作防冲突" : ""; @@ -6541,23 +6518,6 @@ async function loadRuntime() { && hardware.access_policy === "local_approval_idle_lease" ? ` · 本地审批 · 空闲 ${hardware.access_inactivity_timeout_seconds || 180} 秒自动停止` : ""; - const vehicleRouteText = mqttSelected - ? `MQTT ${vehicleCan.host || "127.0.0.1"}:${vehicleCan.port || 1883} · ${vehicleCan.condition_topic || "n7/n7-headunit/can/state"} / ${vehicleCan.control_topic || "n7/n7-headunit/can/command"}` - : `${vehicleCan.backend || "zlgcan"} · CH${vehicleCan.channel ?? 1} 车身 CAN FD ${Number(vehicleCan.arbitration?.bitrate || 500000) / 1000}k/${Number(vehicleCan.data?.bitrate || 2000000) / 1000000}M`; - $("#hardwareNote").textContent = ch1DriverOnly && vehicleHardwareReady && !seatHardwareReady - ? `${vehicleRouteText} · 仅主驾/车端白名单 · CH0/T2G/IAR 已禁用${accessText}${concurrencyText}` - : linkedHardwareReady - ? `${mqttDualChannel ? "MQTT CAN0 副驾座椅 · MQTT CAN1 其余车辆" : `${hardware.backend} · CH${hardware.channel} 座椅 ${hardware.bitrate / 1000} kbit/s`} · ${vehicleRouteText}${mqttDualChannel ? " · 远端自动烧录禁用" : ` · ${profileText} · ${flashText}`}${accessText}${concurrencyText}` - : seatHardwareReady - ? `${hardware.backend} · CH${hardware.channel} 座椅 ${hardware.bitrate / 1000} kbit/s · ${profileText} · ${flashText} · 车端 CAN 未启用，其他领域保持虚拟执行${accessText}${concurrencyText}` - : `可手动选择真实下发；当前不可执行：${hardwareExecutionIssue(hardware, vehicleCan)}`; - $("#hardwareAliasNote").textContent = ch1DriverOnly && vehicleHardwareReady && !seatHardwareReady - ? `${mqttSelected ? "MQTT" : "CH1"} → 车身 CAN / 主驾座椅 · CH0 与副驾 T2G 路径禁止` - : linkedHardwareReady - ? `${mqttDualChannel ? "MQTT CAN0" : "CH0"} → 副驾座椅控制器 · ${mqttDualChannel ? "MQTT CAN1" : "CH1"} → 其余车辆 CAN` - : seatHardwareReady - ? `CH${hardware.channel} → 副驾座椅控制器 · 车身控制暂用虚拟 CAN` - : "尚未配置可执行的物理 CAN 通道"; $("#hardwareCoverage").textContent = ch1DriverOnly && vehicleHardwareReady && !seatHardwareReady ? `${mqttSelected ? "MQTT" : "CH1"} 主驾 + 车端设备 · CAN 字段回读${accessText}${concurrencyText}` : linkedHardwareReady diff --git a/apps/web-console/assistant-chat.js b/apps/web-console/assistant-chat.js index 26b1d3f..74c67d7 100644 --- a/apps/web-console/assistant-chat.js +++ b/apps/web-console/assistant-chat.js @@ -916,6 +916,13 @@ } function applyAssistantEvent(event, message) { + if (event.type === "validated_control_commands" && event.payload) { + window.dispatchEvent(new CustomEvent( + "cognitive-agent:validated-can-commands", + { detail: event.payload }, + )); + return; + } if (event.type === "answer.delta") { setMessageContent(message, `${message.content}${event.delta || ""}`); return; @@ -1094,6 +1101,21 @@ return; } + window.dispatchEvent(new CustomEvent( + "cognitive-agent:control-request-start", + { + detail: { + utterance: messageText, + speech_utterance_id: suggestedMessage + ? "" + : String(input?.dataset.speechUtteranceId || ""), + speech_value: suggestedMessage + ? "" + : String(input?.dataset.speechValue || ""), + }, + }, + )); + createMessage("user", messageText); if (!suggestedMessage) { input.value = ""; diff --git a/apps/web-console/console-v2.css b/apps/web-console/console-v2.css index b1ec1b9..9b41131 100644 --- a/apps/web-console/console-v2.css +++ b/apps/web-console/console-v2.css @@ -281,19 +281,6 @@ box-shadow: 0 5px 13px color-mix(in srgb, var(--v2-accent) 20%, transparent); } -.console-v2 .v2-transport-field > #vehicleTransportNote { - display: -webkit-box; - margin: 6px 1px 0; - overflow: hidden; - color: #809091; - font-size: 8px; - line-height: 1.35; - overflow-wrap: anywhere; - white-space: normal; - -webkit-box-orient: vertical; - -webkit-line-clamp: 2; -} - .v2-sidebar-footer { min-height: 50px; display: grid; @@ -8233,28 +8220,6 @@ body.device-power-drawer-open { opacity: 0.72; } -.console-v2.v2-passive-display.v2-interactive-sync .v2-sync-execution-field .provider-note { - min-height: 0; - margin: 5px 1px 0; - overflow: hidden; - color: #809091; - font-size: 8px; - line-height: 1.35; - overflow-wrap: anywhere; - white-space: normal; -} - -.console-v2.v2-passive-display.v2-interactive-sync .v2-sync-execution-field #hardwareNote { - display: -webkit-box; - -webkit-box-orient: vertical; - -webkit-line-clamp: 2; -} - -.console-v2.v2-passive-display.v2-interactive-sync .v2-sync-execution-field #hardwareAliasNote { - text-overflow: ellipsis; - white-space: nowrap; -} - .console-v2.v2-passive-display.v2-interactive-sync .v2-sync-simulation-fault-field select { min-height: 34px; padding: 0 8px; @@ -8336,8 +8301,83 @@ body.device-power-drawer-open { transform: rotate(180deg); } +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing { + display: grid; + gap: 6px; + padding: 7px 9px; + color: #557270; + background: rgba(232, 244, 242, 0.7); + border-top: 1px solid rgba(19, 105, 99, 0.1); +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing[hidden] { + display: none; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing > p { + margin: 0; + font-size: 8px; + font-weight: 750; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing > header { + display: flex; + align-items: center; + gap: 6px; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing > header strong { + font-size: 9px; + font-weight: 900; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing > header span { + padding: 1px 5px; + color: #2366a1; + background: #e4f0fb; + border-radius: 999px; + font-size: 7px; + font-weight: 850; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing > header b { + margin-left: auto; + color: #08766f; + font: 850 8px/1.4 "SFMono-Regular", Consolas, monospace; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing ol { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 3px 9px; + margin: 0; + padding: 0; + list-style: none; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing li { + min-width: 0; + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 5px; + font-size: 7px; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing li span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing li b { + flex: 0 0 auto; + color: #315e5b; + font: 800 7px/1.4 "SFMono-Regular", Consolas, monospace; +} + .console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-list { - max-height: min(24dvh, 190px); + max-height: min(34dvh, 280px); padding: 7px; overflow: auto; border-top: 1px solid rgba(19, 105, 99, 0.1); @@ -8472,6 +8512,35 @@ body.device-power-drawer-open { overflow-wrap: anywhere; } +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-layer { + min-width: 0; + display: grid; + gap: 1px; + padding: 4px 5px; + border-radius: 5px; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-layer > small { + color: #758886; + font-size: 7px; + font-weight: 900; + letter-spacing: 0.04em; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-semantic { + background: rgba(35, 102, 161, 0.07); + border-left: 2px solid rgba(35, 102, 161, 0.38); +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-semantic code { + color: #245c8d; +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-actual { + background: rgba(8, 118, 111, 0.07); + border-left: 2px solid rgba(8, 118, 111, 0.42); +} + .console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-frame { display: block; color: #8a6b36; @@ -8479,6 +8548,10 @@ body.device-power-drawer-open { overflow-wrap: anywhere; } +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-frame > b { + font-weight: 900; +} + .console-v2.v2-passive-display.v2-interactive-sync .v2-sync-transport-field > .field-label { color: #466664; font-size: 10px; @@ -9530,6 +9603,40 @@ body.device-power-drawer-open { color: var(--command-frame); } +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing { + color: var(--muted-foreground); + background: color-mix(in oklch, var(--accent) 55%, transparent); + border-top-color: var(--border); +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing > header span { + color: var(--command-source); + background: color-mix(in oklch, var(--command-source) 12%, var(--card)); +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing > header b, +.console-v2.v2-passive-display.v2-interactive-sync .v2-command-timing li b { + color: var(--primary); +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-list li[data-can-channel] .v2-can-command-layer > small { + color: var(--muted-foreground); +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-list li[data-can-channel] .v2-can-command-semantic { + background: color-mix(in oklch, var(--command-source) 8%, var(--card)); + border-left-color: color-mix(in oklch, var(--command-source) 52%, var(--border)); +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-list li[data-can-channel] .v2-can-command-semantic code { + color: var(--command-source); +} + +.console-v2.v2-passive-display.v2-interactive-sync .v2-can-command-list li[data-can-channel] .v2-can-command-actual { + background: color-mix(in oklch, var(--primary) 8%, var(--card)); + border-left-color: color-mix(in oklch, var(--primary) 52%, var(--border)); +} + @media (max-width: 1120px) { .console-v2 .v2-mode-rail > .mode-rail-bar { grid-template-columns: auto minmax(150px, 1fr) auto; diff --git a/apps/web-console/console-v2.js b/apps/web-console/console-v2.js index ce92d6d..f023a70 100644 --- a/apps/web-console/console-v2.js +++ b/apps/web-console/console-v2.js @@ -23,15 +23,37 @@ let reasoningPickerControl = null; let validatedCommandsVisible = false; let validatedCommandContext = { requestId: "", utterance: "" }; + let pendingSpeechTiming = null; + let commandTimingState = { + inputMode: "text", + utterance: "", + requestId: "", + requestStartedAt: null, + validatedAt: null, + displayedAt: null, + renderDurationMs: null, + speechStartedAt: null, + speechResultAt: null, + speechTimings: null, + }; function resetValidatedCommandContext() { validatedCommandsVisible = false; validatedCommandContext = { requestId: "", utterance: "" }; } + function commandCorrelationIds(result = {}) { + return [...new Set([ + result.preview_request_id, + result.supplement_of, + result.request_id, + ].map((value) => String(value || "").trim()).filter(Boolean))]; + } + function rememberValidatedCommandContext(result = {}) { + const ids = commandCorrelationIds(result); validatedCommandContext = { - requestId: String(result.request_id || ""), + requestId: ids[0] || "", utterance: String(result.utterance || "").trim(), }; } @@ -39,10 +61,10 @@ function validatedCommandsBelongToTask(task = {}) { if (!validatedCommandsVisible) return false; const result = task.result && typeof task.result === "object" ? task.result : {}; - const requestId = String(result.request_id || ""); + const requestIds = commandCorrelationIds(result); const utterance = String(task.utterance || result.utterance || "").trim(); - if (validatedCommandContext.requestId && requestId - && validatedCommandContext.requestId !== requestId) return false; + if (validatedCommandContext.requestId && requestIds.length + && !requestIds.includes(validatedCommandContext.requestId)) return false; if (validatedCommandContext.utterance && utterance && validatedCommandContext.utterance !== utterance && !validatedCommandContext.utterance.includes(utterance) @@ -83,6 +105,123 @@ : null; }; + const monotonicNow = () => ( + typeof performance !== "undefined" && typeof performance.now === "function" + ? performance.now() + : Date.now() + ); + + const finiteMilliseconds = (value) => { + const numeric = Number(value); + return Number.isFinite(numeric) && numeric >= 0 ? numeric : null; + }; + + const semanticValue = (value) => { + if (value == null) return "null"; + if (["string", "number", "boolean"].includes(typeof value)) return String(value); + try { + return JSON.stringify(value); + } catch (_error) { + return String(value); + } + }; + + function semanticControlInstruction(operation = {}) { + const capability = String(operation.capability || "unknown"); + const rawTarget = operation.target || {}; + const target = rawTarget.zone || rawTarget.instance || rawTarget.device; + const argumentsList = []; + if (target != null && String(target).trim()) { + argumentsList.push(`target=${String(target).replaceAll("_", "-")}`); + } + Object.entries(operation.parameters || {}) + .sort(([left], [right]) => left.localeCompare(right)) + .forEach(([key, value]) => { + argumentsList.push(`${key}=${semanticValue(value)}`); + }); + return `${capability}(${argumentsList.join(", ")})`; + } + + function commandPlanEntries(result = {}) { + const plans = result.command_plans && typeof result.command_plans === "object" + ? Object.entries(result.command_plans) + : []; + if (!plans.length && result.command_plan) plans.push(["default", result.command_plan]); + return plans; + } + + function compiledCommandsForOperation(result, configKey, operation = {}) { + const operationId = String(operation.id || ""); + if (!operationId) return []; + const plans = commandPlanEntries(result); + const preferred = plans.filter(([key]) => key === configKey); + const operationRefs = (command) => { + const raw = command?.operation_ref; + const refs = Array.isArray(raw) ? raw : String(raw || "").split(","); + return refs.map((ref) => String(ref).trim()).filter(Boolean); + }; + const matches = (entries) => entries.flatMap(([planKey, plan]) => ( + (plan?.spec?.commands || []) + .filter((command) => operationRefs(command).includes(operationId)) + .map((command) => ({ command, plan, planKey })) + )); + if (preferred.length) return matches(preferred); + const configurations = result.function_configs && typeof result.function_configs === "object" + ? Object.entries(result.function_configs) + : result.function_config ? [["default", result.function_config]] : []; + return configurations.length === 1 && plans.length === 1 ? matches(plans) : []; + } + + function compiledCommandPresentation(entries, fallback = {}) { + if (!entries.length) return fallback; + const actualCommands = []; + const frames = []; + let channel = fallback.channel || "CAN1"; + entries.forEach(({ command, plan }) => { + const signals = command?.signals || {}; + const message = String(command?.message || ""); + const planDomain = String(plan?.spec?.domain || ""); + const bus = String(command?.bus || "").toLowerCase(); + const slot = boundedInteger(signals.slot, 1, 248); + if (message === "infrared.emit" || planDomain === "self_extension") { + channel = "IR"; + } else if (bus.includes("seat") || bus.includes("can0")) { + channel = "CAN0"; + } else if (bus.includes("body") || bus.includes("can1")) { + channel = "CAN1"; + } + const actual = String(signals.debug_command || "").trim() + || (message === "infrared.emit" && slot !== null ? `infrared emit ${slot}` : "") + || String(fallback.command || message).trim(); + if (actual && !actualCommands.includes(actual)) actualCommands.push(actual); + const frameId = String(signals.frame_id || "").trim(); + const dataHex = String(signals.data_hex || "").trim(); + if (frameId && dataHex) { + frames.push({ label: `${channel} 帧`, value: `${frameId} · ${dataHex}` }); + } else if (message === "infrared.emit" && slot !== null) { + frames.push({ + label: "GSCU1B TCP", + value: JSON.stringify({ action: "emit", data: { no: slot }, type: "infrared" }), + }); + } + }); + const uniqueFrames = frames.filter((frame, index) => ( + frames.findIndex((candidate) => ( + candidate.label === frame.label && candidate.value === frame.value + )) === index + )); + return { + ...fallback, + channel, + command: actualCommands.join("；") || fallback.command || "", + ...(uniqueFrames.length ? { + frame: uniqueFrames[0].value, + frameLabel: uniqueFrames[0].label, + frames: uniqueFrames, + } : {}), + }; + } + function parsedCanCommandForOperation(operation = {}) { const capability = String(operation.capability || ""); const parameters = operation.parameters || {}; @@ -397,15 +536,68 @@ }; }; const commands = []; - configurationEntries.forEach(([_configKey, configuration]) => { - (configuration?.spec?.operations || []).forEach((operation) => { - const item = parsedCanCommandForOperation(operation); - if (item) commands.push({ ...item, ...deliveryForOperation(operation) }); + const validatedControlCommands = Array.isArray(result.validated_control_commands) + ? result.validated_control_commands + : []; + if (validatedControlCommands.length) { + validatedControlCommands.forEach((presentation) => { + const operation = presentation?.operation || {}; + const fallback = parsedCanCommandForOperation(operation) || { + channel: presentation?.domain === "self_extension" ? "IR" : "CAN1", + category: String(operation.capability || "控制指令"), + command: "", + ...(operation.parse_source ? { parseSource: String(operation.parse_source) } : {}), + }; + const channel = new Set(["CAN0", "CAN1", "IR"]).has(presentation?.channel) + ? presentation.channel + : fallback.channel; + const frames = (Array.isArray(presentation?.frames) ? presentation.frames : []) + .map((frame) => ({ + label: String(frame?.label || "控制帧"), + value: String(frame?.value || ""), + })) + .filter((frame) => frame.value); + const command = String(presentation?.actual_command || fallback.command || "").trim(); + if (!command) return; + commands.push({ + ...fallback, + channel, + command, + ...(frames.length ? { + frame: frames[0].value, + frameLabel: frames[0].label, + frames, + } : {}), + semantic: semanticControlInstruction(operation), + operationRef: String(operation.id || ""), + ...deliveryForOperation(operation), + }); }); - }); + } else { + configurationEntries.forEach(([configKey, configuration]) => { + (configuration?.spec?.operations || []).forEach((operation) => { + const fallback = parsedCanCommandForOperation(operation) || { + channel: configuration?.spec?.domain === "self_extension" ? "IR" : "CAN1", + category: String(operation.capability || "控制指令"), + command: "", + ...(operation.parse_source ? { parseSource: String(operation.parse_source) } : {}), + }; + const compiled = compiledCommandsForOperation(result, configKey, operation); + const item = compiledCommandPresentation(compiled, fallback); + if (item.command) { + commands.push({ + ...item, + semantic: semanticControlInstruction(operation), + operationRef: String(operation.id || ""), + ...deliveryForOperation(operation), + }); + } + }); + }); + } const unique = new Map(); commands.forEach((item) => { - const key = `${item.channel}:${item.command}`; + const key = `${item.operationRef || item.semantic}:${item.channel}:${item.command}`; const previous = unique.get(key); if (!previous || (previous.deliveryState !== "submitted" && item.deliveryState === "submitted") @@ -425,7 +617,192 @@ }[source] || "云端模型"; } - function renderParsedCanCommands(result, { message = "" } = {}) { + const commandTraceStageLabels = { + intent_parse: "意图解析", + context_resolve: "上下文解析", + domain_route: "控制域路由", + function_generate: "语义指令生成", + schema_validate: "Schema 校验", + domain_validate: "领域语义校验", + compile: "实际指令编译", + nfc_peripheral_route: "红外能力路由", + local_peripheral_plan: "红外本地规则编译", + nfc_peripheral_sequence: "红外指令序列生成", + }; + + const speechTimingLabels = { + audio_read_ms: "音频读取与重采样", + vad_ms: "语音活动检测", + asr_ms: "语音识别 ASR", + assembly_ms: "分段文本组装", + semantic_ms: "语音语义处理", + command_resolution_ms: "本地指令消歧", + recovery_ms: "语音识别恢复", + total_ms: "语音服务总耗时", + }; + + function parsedCommandTimingRows(result = {}, timingState = {}) { + const rows = []; + const push = (group, key, label, value) => { + const durationMs = finiteMilliseconds(value); + if (durationMs !== null) rows.push({ group, key, label, durationMs }); + }; + if (timingState.inputMode === "speech") { + Object.entries(speechTimingLabels).forEach(([key, label]) => { + push("speech", key, label, timingState.speechTimings?.[key]); + }); + if (timingState.speechStartedAt != null && timingState.speechResultAt != null) { + push( + "speech", + "speech_capture_to_text", + "语音开始 → 文本就绪", + timingState.speechResultAt - timingState.speechStartedAt, + ); + } + } + const traceEvents = Array.isArray(result.trace_events) ? result.trace_events : []; + const compileIndex = traceEvents.map((event) => event?.stage).lastIndexOf("compile"); + const validatedTraceEvents = compileIndex >= 0 + ? traceEvents.slice(0, compileIndex + 1) + : traceEvents; + const localRuleResult = String(result.provider_used || "").startsWith("local") + || parsedCanCommands(result).some((item) => item.parseSource === "local_rule"); + const localBudgetMs = finiteMilliseconds(result.local_control_budget_ms); + const localWithinBudget = result.local_control_within_budget === true; + push( + "backend", + "local_control_elapsed", + localBudgetMs === null + ? "本地规则解析与下发" + : `本地规则解析与下发（${localWithinBudget ? "1 秒内" : "超出 1 秒"}）`, + result.local_control_elapsed_ms, + ); + validatedTraceEvents.forEach((event, index) => { + const stage = String(event?.stage || ""); + if (!(stage in commandTraceStageLabels)) return; + const label = stage === "intent_parse" && localRuleResult + ? "本地规则意图解析" + : commandTraceStageLabels[stage]; + push("backend", `${stage}-${index}`, label, event.duration_ms); + }); + if (timingState.requestStartedAt != null && timingState.validatedAt != null) { + push( + "frontend", + "request_to_validated", + "请求提交 → 编译结果到达", + timingState.validatedAt - timingState.requestStartedAt, + ); + } + push("frontend", "dom_render", "前端指令渲染", timingState.renderDurationMs); + if (timingState.requestStartedAt != null && timingState.displayedAt != null) { + push( + "total", + "request_to_display", + "文本提交 → 指令显示", + timingState.displayedAt - timingState.requestStartedAt, + ); + } + if (timingState.inputMode === "speech" + && timingState.speechStartedAt != null + && timingState.displayedAt != null) { + push( + "total", + "speech_to_display", + "语音开始 → 指令显示", + timingState.displayedAt - timingState.speechStartedAt, + ); + } + return rows; + } + + const formattedDuration = (value) => { + const duration = finiteMilliseconds(value); + if (duration === null) return "--"; + if (duration > 0 && duration < 0.1) return "<0.1 ms"; + return `${duration < 10 ? duration.toFixed(1) : Math.round(duration)} ms`; + }; + + function renderCommandTimings(result = null, { pending = false } = {}) { + const container = $("#v2CommandTiming"); + if (!container) return; + const rows = result ? parsedCommandTimingRows(result, commandTimingState) : []; + if (!rows.length && !pending) { + container.hidden = true; + container.innerHTML = ""; + return; + } + container.hidden = false; + if (!rows.length) { + container.innerHTML = `

阶段耗时计时中 · ${commandTimingState.inputMode === "speech" ? "语音输入" : "普通文本"}

`; + return; + } + const total = [...rows].reverse().find((row) => row.group === "total"); + container.innerHTML = ` +
+ 阶段耗时 + ${commandTimingState.inputMode === "speech" ? "语音输入" : "普通文本"} + ${total ? `${escapeHtml(formattedDuration(total.durationMs))}` : ""} +
+
    ${rows.map((row) => ` +
  1. + ${escapeHtml(row.label)}${escapeHtml(formattedDuration(row.durationMs))} +
    2. `).join("")}
`; + } + + function beginCommandTiming(request = {}) { + const input = $("#utterance"); + const utterance = String(request.utterance ?? input?.value ?? "").trim(); + const speechUtteranceId = String( + request.speech_utterance_id ?? input?.dataset?.speechUtteranceId ?? "", + ); + const speechValue = String( + request.speech_value ?? input?.dataset?.speechValue ?? "", + ); + const speechMatch = Boolean( + speechUtteranceId + && pendingSpeechTiming?.utteranceId === speechUtteranceId + && speechValue === utterance, + ); + commandTimingState = { + inputMode: speechMatch ? "speech" : "text", + utterance, + requestId: "", + requestStartedAt: monotonicNow(), + validatedAt: null, + displayedAt: null, + renderDurationMs: null, + speechStartedAt: speechMatch ? pendingSpeechTiming.startedAt : null, + speechResultAt: speechMatch ? pendingSpeechTiming.resultAt : null, + speechTimings: speechMatch ? { ...(pendingSpeechTiming.timings || {}) } : null, + }; + } + + function commandResultMatchesTimedRequest(result = {}) { + if (commandTimingState.requestStartedAt == null) return true; + const expectedUtterance = String(commandTimingState.utterance || "") + .replace(/\s+/g, "") + .trim(); + const receivedUtterance = String(result.utterance || "") + .replace(/\s+/g, "") + .trim(); + if (expectedUtterance && receivedUtterance + && expectedUtterance !== receivedUtterance + && !expectedUtterance.includes(receivedUtterance) + && !receivedUtterance.includes(expectedUtterance)) return false; + const requestIds = commandCorrelationIds(result); + if (commandTimingState.requestId && requestIds.length + && !requestIds.includes(commandTimingState.requestId)) return false; + if (!commandTimingState.requestId && requestIds.length) { + [commandTimingState.requestId] = requestIds; + } + return true; + } + + function renderParsedCanCommands( + result, + { message = "", pending = false, recordTiming = false, reveal = false } = {}, + ) { + const renderStartedAt = monotonicNow(); const panel = $("#v2CanCommandPanel"); const list = $("#v2CanCommandList"); const count = $("#v2CanCommandCount"); @@ -442,12 +819,31 @@ ${escapeHtml(item.deliveryLabel)} ${escapeHtml(item.category)} - ${escapeHtml(item.command)} - ${item.frame ? `${escapeHtml(item.frameLabel || "固件原生 CAN0")} · ${escapeHtml(item.frame)}` : ""} +
+ 语义控制指令 + ${escapeHtml(item.semantic)} +
+
+ 实际控制指令 + ${escapeHtml(item.command)} +
+ ${(item.frames || (item.frame ? [{ label: item.frameLabel || "控制帧", value: item.frame }] : [])).map((frame) => `${escapeHtml(frame.label)} · ${escapeHtml(frame.value)}`).join("")} `).join("")}`; } else { list.innerHTML = `

${escapeHtml(message || "本次请求未解析出控制指令")}

`; } + if (reveal && commands.length) { + panel.open = true; + if (ui.sidebarCollapsed) setSidebarCollapsed(false); + } + if (recordTiming && commands.length) { + if (commandTimingState.validatedAt == null) commandTimingState.validatedAt = renderStartedAt; + if (commandTimingState.displayedAt == null) { + commandTimingState.displayedAt = monotonicNow(); + commandTimingState.renderDurationMs = commandTimingState.displayedAt - renderStartedAt; + } + } + renderCommandTimings(result, { pending }); requestAnimationFrame(syncSidebarLogBaseline); } @@ -767,6 +1163,11 @@ logKeys: new Set(), extensionFingerprint: "", }; + const nonDispatchTaskSources = new Set(["mode_editor_compile"]); + + function isDispatchTask(task = {}) { + return !nonDispatchTaskSources.has(String(task.source || "").trim()); + } const readText = (selector, fallback = "--") => $(selector)?.textContent?.trim() || fallback; @@ -1625,26 +2026,28 @@ if (revision !== displaySync.revision) { displaySync.revision = revision; const task = snapshot?.task || {}; - setTaskPresentation(task); - const sourceLabel = task.source === "web_console" ? "网页端" : "车机端"; - if (task.utterance) { - const submittedUnverified = task.result?.execution_outcome === "submitted_unverified" - || task.result?.execution_report?.spec?.verification_status === "submitted_unverified"; - appendSyncLog({ - key: `task:${revision}:${task.status}`, - category: "task", - title: `${sourceLabel}${task.status === "running" ? "下发编排任务" : "任务状态已同步"}`, - detail: `${task.utterance} · ${submittedUnverified ? "指令已提交，等待车辆状态确认" : task.status || "idle"}`, - status: submittedUnverified ? "info" : ["succeeded", "partially_succeeded"].includes(task.status) ? "success" : task.status === "failed" ? "error" : "info", - }); - } - const result = task.result; - const requestId = result?.request_id || null; - if (result && requestId !== displaySync.requestId) { - displaySync.requestId = requestId; - window.CognitiveAgentConsole?.applySyncedResult?.(result, { - utterance: task.utterance || "", - }); + if (isDispatchTask(task)) { + setTaskPresentation(task); + const sourceLabel = task.source === "web_console" ? "网页端" : "车机端"; + if (task.utterance) { + const submittedUnverified = task.result?.execution_outcome === "submitted_unverified" + || task.result?.execution_report?.spec?.verification_status === "submitted_unverified"; + appendSyncLog({ + key: `task:${revision}:${task.status}`, + category: "task", + title: `${sourceLabel}${task.status === "running" ? "下发编排任务" : "任务状态已同步"}`, + detail: `${task.utterance} · ${submittedUnverified ? "指令已提交，等待车辆状态确认" : task.status || "idle"}`, + status: submittedUnverified ? "info" : ["succeeded", "partially_succeeded"].includes(task.status) ? "success" : task.status === "failed" ? "error" : "info", + }); + } + const result = task.result; + const requestId = result?.request_id || null; + if (result && requestId !== displaySync.requestId) { + displaySync.requestId = requestId; + window.CognitiveAgentConsole?.applySyncedResult?.(result, { + utterance: task.utterance || "", + }); + } } } const extensionFingerprint = synchronizedCatalog @@ -1763,12 +2166,13 @@ insertVisibleControl(simulationFaultField, "v2-sync-simulation-fault-field"); const canCommandPanel = make("details", "v2-can-command-panel", ` - 已解析控制指令 + 下发控制指令 0 +
-

等待自然语言解析结果

+

等待准备下发的控制指令

`); canCommandPanel.id = "v2CanCommandPanel"; syncOverview.append(canCommandPanel); @@ -1875,17 +2279,57 @@ if (event.target.matches("select[data-device-control]")) storeDeviceControlDraft(event.target); }); $("#v2LogFilter")?.addEventListener("change", renderSyncLogs); - $("#requestForm")?.addEventListener("submit", () => { + const beginPendingControlRequest = (request = {}) => { + beginCommandTiming(request); resetValidatedCommandContext(); - renderParsedCanCommands(null, { message: "正在解析自然语言并生成控制指令…" }); + renderParsedCanCommands(null, { + message: "正在解析自然语言并生成控制指令…", + pending: true, + }); + }; + $("#requestForm")?.addEventListener("submit", () => { + beginPendingControlRequest(); + }, true); + window.addEventListener("cognitive-agent:control-request-start", (event) => { + beginPendingControlRequest(event.detail || {}); + }); + window.addEventListener("cognitive-agent:speech-state", (event) => { + const phase = event.detail?.phase; + if (phase === "capturing" && !pendingSpeechTiming?.captureActive) { + pendingSpeechTiming = { + utteranceId: "", + startedAt: monotonicNow(), + resultAt: null, + timings: null, + captureActive: true, + }; + } else if (["finalizing", "success", "error"].includes(phase) + && pendingSpeechTiming) { + pendingSpeechTiming.captureActive = false; + } + }); + window.addEventListener("cognitive-agent:speech-result", (event) => { + const result = event.detail || {}; + pendingSpeechTiming = { + ...(pendingSpeechTiming || { startedAt: null }), + utteranceId: String(result.utterance_id || ""), + resultAt: monotonicNow(), + timings: result.timings && typeof result.timings === "object" + ? { ...result.timings } + : null, + captureActive: false, + }; }); window.addEventListener("cognitive-agent:validated-can-commands", (event) => { const result = event.detail || {}; + if (!commandResultMatchesTimedRequest(result)) return; validatedCommandsVisible = parsedCanCommands(result).length > 0; if (validatedCommandsVisible) rememberValidatedCommandContext(result); else resetValidatedCommandContext(); renderParsedCanCommands(result, { message: "控制指令已解析，正在等待传输结果…", + recordTiming: true, + reveal: true, }); }); window.addEventListener("cognitive-agent:result", (event) => { @@ -1893,11 +2337,15 @@ const submittedUnverified = result.execution_outcome === "submitted_unverified" || result.execution_report?.spec?.verification_status === "submitted_unverified"; const commands = parsedCanCommands(result); - if (commands.length) { + const matchesTimedRequest = commandResultMatchesTimedRequest(result); + if (matchesTimedRequest && commands.length) { validatedCommandsVisible = true; rememberValidatedCommandContext(result); - renderParsedCanCommands(result); - } else if (!validatedCommandsBelongToTask({ + renderParsedCanCommands(result, { + recordTiming: commandTimingState.displayedAt == null, + reveal: commandTimingState.requestStartedAt != null, + }); + } else if (matchesTimedRequest && !validatedCommandsBelongToTask({ utterance: result.utterance, result, })) { @@ -1917,7 +2365,15 @@ }); }); window.addEventListener("cognitive-agent:request-complete", (event) => { - if (event.detail?.error && !validatedCommandsVisible) { + const completedUtterance = String(event.detail?.utterance || "") + .replace(/\s+/g, "") + .trim(); + const expectedUtterance = String(commandTimingState.utterance || "") + .replace(/\s+/g, "") + .trim(); + const matchesTimedRequest = !expectedUtterance || !completedUtterance + || expectedUtterance === completedUtterance; + if (matchesTimedRequest && !validatedCommandsVisible) { renderParsedCanCommands(null, { message: "本次请求未生成控制指令" }); } if (event.detail?.error) { diff --git a/apps/web-console/index.html b/apps/web-console/index.html index e4d1c87..b5a8c09 100755 --- a/apps/web-console/index.html +++ b/apps/web-console/index.html @@ -10,7 +10,7 @@ 511工程-智能座舱 - + @@ -20,13 +20,13 @@ - + - + - + @@ -131,8 +131,6 @@ 真实下发 -

正在检查 CAN 适配器与 IAR 固件配置

-

所有座椅能力仅控制副驾实体电机、加热与通风执行器

@@ -145,7 +143,6 @@ 无线 CAN
-

正在检查车辆连接