feat: add seat-scoped static and dynamic assistant modes

This commit is contained in:
vee
2026-09-12 14:56:30 +08:00
parent d09b5967f8
commit d8dad22ec0
60 changed files with 5847 additions and 149 deletions
+256
View File
@@ -0,0 +1,256 @@
"use strict";
const assert = require("node:assert/strict");
const path = require("node:path");
const { chromium } = require("playwright");
const [baseURL, evidenceDir] = process.argv.slice(2);
async function main() {
const browser = await chromium.launch({
headless: true,
...(process.env.N7WEB_BROWSER_CHANNEL ? { channel: process.env.N7WEB_BROWSER_CHANNEL }
: process.platform === "win32" ? { channel: "msedge" } : {}),
});
const context = await browser.newContext({ viewport: { width: 1600, height: 1000 }, serviceWorkers: "block" });
const page = await context.newPage();
const errors = [];
const posted = [];
const responses = [];
let mockedExecution = null;
let statusReads = 0;
page.on("pageerror", (error) => errors.push(error.message));
page.on("console", (message) => {
if (message.type() === "error") errors.push(`${message.text()} ${message.location().url || ""}`);
});
await context.route("**/api/**", async (route) => {
const request = route.request();
const pathname = new URL(request.url()).pathname;
if (["/api/hardware/seat-state", "/api/hardware/vehicle-state"].includes(pathname)) {
await route.fulfill({ json: { status: "unavailable", hardware_execution_allowed: false } });
return;
}
if (request.method() !== "GET") posted.push({ pathname, payload: request.postDataJSON() });
if (mockedExecution && pathname === "/api/assistant/stream") {
await route.fulfill({ contentType: "application/x-ndjson", body: [
{ type: "action.completed", success: true, payload: mockedExecution },
{ type: "answer.completed", answer: "动态时间轴已启动,后续阶段会按计划执行。" },
].map((event) => JSON.stringify(event)).join("\n") + "\n" });
return;
}
if (mockedExecution && pathname === "/api/modes/schedule/status") {
statusReads += 1;
assert.equal(new URL(request.url()).searchParams.get("vehicle_id"), "denza_n7");
await route.fulfill({ json: {
...mockedExecution.schedule_execution, vehicle_id: "denza_n7",
phase_index: 1, repeat_index: 0, completed: false,
status: statusReads === 1 ? "running" : "failed",
error: statusReads === 1 ? null : "QA阶段模拟失败",
} });
return;
}
if (["/api/assistant/stream", "/api/mode-profile-drafts/trial", "/api/mode-profiles"].includes(pathname)) {
// Chromium can discard completed streaming response bodies from CDP.
// Fetch the real isolated endpoint and retain its exact response bytes.
const response = await route.fetch();
const text = await response.text();
responses.push({ pathname, text, status: response.status() });
await route.fulfill({ response, body: text });
return;
}
await route.continue();
});
const waitIdle = () => page.waitForFunction(() => !window.CognitiveAgentAssistant.isPending());
const responseFor = (pathname) => page.waitForResponse((response) => new URL(response.url()).pathname === pathname);
try {
await page.goto(baseURL, { waitUntil: "domcontentloaded" });
await page.waitForFunction(() => document.body.classList.contains("assistant-chat-enabled")
&& document.querySelectorAll(".mode-chips button").length > 3);
assert.equal(await page.title(), "511工程-智能座舱");
assert.equal(new URL(page.url()).origin, baseURL);
assert.match(await page.locator("body").innerText(), /整车孪生界面/);
if (!await page.locator('[data-execution-mode="dry_run"]').isVisible()) {
await page.locator('#v2SidebarToggle').click();
}
await page.locator('[data-execution-mode="dry_run"]').click();
assert.equal(await page.evaluate(() => window.CognitiveAgentConsole.getExecutionMode()), "dry_run");
// Exercise the same real assistant endpoint and rendered cards for both
// implicit passenger selection and an explicit driver request. These
// previews stay unsaved, so the timeline save/trial tests remain isolated.
for (const [index, testCase] of [
{ text: "构建一个阅读模式", seat: "passenger", label: "副驾", excluded: "主驾" },
{ text: "构建一个主驾阅读模式", seat: "driver", label: "主驾", excluded: "副驾" },
{ text: "构建一个主驾座椅动态模式", seat: "driver", label: "主驾", excluded: "副驾" },
].entries()) {
const responsePromise = responseFor("/api/assistant/stream");
await page.locator("#utterance").fill(testCase.text);
await page.locator("#utterance").press("Enter");
const response = await responsePromise;
assert.equal(response.status(), 200);
const events = responses.filter((item) => item.pathname === "/api/assistant/stream").at(-1)
.text.trim().split("\n").map((line) => JSON.parse(line));
await waitIdle();
const payload = events.find((event) => event.payload?.mode_profile_draft)?.payload;
assert.ok(payload, `Missing selected-seat draft for ${testCase.text}: ${JSON.stringify(events)}`);
const draft = payload.mode_profile_draft;
const actions = draft.schedule ? draft.schedule.phases.flatMap((phase) => phase.operations) : payload.actions;
const seats = actions.filter((action) => action.capability?.startsWith("seat."))
.map((action) => action.target_scope || action.target?.instance || action.target?.zone);
assert.ok(seats.length, `Missing selected-seat actions for ${testCase.text}`);
assert.deepEqual([...new Set(seats)], [testCase.seat]);
const card = page.locator(`[data-assistant-mode-draft="${draft.source_request_id}"]`);
await card.waitFor({ state: "visible" });
const actionText = await card.locator(".assistant-mode-draft-row").allTextContents();
assert.ok(actionText.some((text) => text.includes(testCase.label)), actionText.join("\n"));
assert.ok(actionText.every((text) => !text.includes(testCase.excluded)), actionText.join("\n"));
await card.scrollIntoViewIfNeeded();
await page.screenshot({ path: path.join(evidenceDir, `seat-scope-preview-${index + 1}.png`) });
}
const savedModes = [];
for (const [index, text] of ["构建一个座椅的动态模式", "构建一个上车预热的功能模式"].entries()) {
const responsePromise = responseFor("/api/assistant/stream");
await page.locator("#utterance").fill(text);
await page.locator("#utterance").press("Enter");
const response = await responsePromise;
assert.equal(response.status(), 200);
const events = responses.filter((item) => item.pathname === "/api/assistant/stream").at(-1).text.trim().split("\n").map((line) => JSON.parse(line));
await waitIdle();
const draftEvent = events.find((event) => event.payload?.mode_profile_draft);
assert.ok(draftEvent, `Missing schedule draft for ${text}: ${JSON.stringify(events)}`);
const draft = draftEvent.payload.mode_profile_draft;
assert.equal(draft.schedule?.schema, "cognitive-agent-mode-schedule/v1");
assert.ok(draft.schedule.phases.length >= 2);
const card = page.locator(`[data-assistant-mode-draft="${draft.source_request_id}"]`);
assert.equal(await card.locator(".assistant-mode-schedule-phase").count(), draft.schedule.phases.length);
assert.equal(await card.locator(".assistant-mode-draft-row").count(), draft.operation_count);
assert.match(await card.innerText(), /重复 \d+ 次 · 总时长/);
assert.match(await card.innerText(), /停止动态模式/);
if (index === 0) {
assert.match(await card.innerText(), /副驾/);
assert.doesNotMatch(await card.innerText(), /主驾/);
} else {
const temperatures = draft.schedule.phases.flatMap((phase) => phase.operations)
.filter((operation) => operation.capability === "climate.set_temperature")
.map((operation) => operation.parameters.temperature_c);
assert.ok(new Set(temperatures).size >= 2, "preheat must change temperature over time");
for (const value of new Set(temperatures)) assert.ok((await card.innerText()).includes(`${value}℃`));
}
await card.scrollIntoViewIfNeeded();
await page.screenshot({ path: path.join(evidenceDir, `schedule-preview-${index + 1}.png`) });
const trialPromise = responseFor("/api/mode-profile-drafts/trial");
await card.locator(".assistant-mode-draft-trial").click();
const trialResponse = await trialPromise;
const trial = JSON.parse(responses.filter((item) => item.pathname === "/api/mode-profile-drafts/trial").at(-1).text);
assert.equal(trialResponse.status(), 200, JSON.stringify(trial));
assert.equal(trial.status, "succeeded", JSON.stringify(trial));
assert.equal(trial.schedule_execution?.status, "completed");
await page.waitForFunction((requestId) => {
const status = document.querySelector(`[data-assistant-mode-draft="${requestId}"] .assistant-mode-draft-status`);
return status?.classList.contains("success") || status?.classList.contains("failed");
}, draft.source_request_id);
assert.match(await card.locator(".assistant-mode-draft-status").innerText(), /试运行通过/);
const savePromise = responseFor("/api/mode-profiles");
await card.locator(".assistant-mode-draft-save").click();
const saveResponse = await savePromise;
const saved = JSON.parse(responses.filter((item) => item.pathname === "/api/mode-profiles").at(-1).text);
assert.ok([200, 201].includes(saveResponse.status()), JSON.stringify(saved));
assert.ok(saved.mode_profile?.id);
savedModes.push(saved.mode_profile);
await page.locator(".assistant-mode-created").filter({ hasText: saved.mode_profile.name }).waitFor({ state: "visible" });
const button = page.locator(`.mode-chips button[data-mode-profile-id="${saved.mode_profile.id}"]`);
await button.waitFor({ state: "visible" });
// Saving reveals the card with smooth scrolling. A tooltip correctly
// closes while its rail is still moving, so wait for that motion to end.
await page.evaluate(() => new Promise((resolve) => {
const rail = document.querySelector('.mode-chips');
let previous = rail.scrollLeft;
let stable = 0;
const sample = () => {
const current = rail.scrollLeft;
stable = Math.abs(current - previous) < 0.1 ? stable + 1 : 0;
previous = current;
if (stable >= 12) resolve();
else requestAnimationFrame(sample);
};
requestAnimationFrame(sample);
}));
await page.mouse.move(1, 1);
const buttonBox = await button.boundingBox();
const railBox = await page.locator('.mode-chips').boundingBox();
const visibleLeft = Math.max(buttonBox.x, railBox.x);
const visibleRight = Math.min(buttonBox.x + buttonBox.width, railBox.x + railBox.width);
assert.ok(visibleRight > visibleLeft, 'saved card is visible in the mode rail');
// A native mouse move does not request another smooth scroll, unlike
// locator.hover() which scrolls the target to the center first.
await page.mouse.move((visibleLeft + visibleRight) / 2, buttonBox.y + buttonBox.height / 2);
const tooltip = page.locator("#modeChipActionPopover");
await tooltip.waitFor({ state: "visible" });
assert.match(await tooltip.innerText(), /重复 \d+ 次 · 总时长/);
assert.equal(await tooltip.locator(".mode-schedule-popover-phase").count(), draft.operation_count);
for (const phase of draft.schedule.phases) assert.ok((await tooltip.innerText()).includes(phase.name));
await tooltip.screenshot({ path: path.join(evidenceDir, `schedule-tooltip-${index + 1}.png`), timeout: 4000 });
await page.mouse.move(1, 1);
}
await page.locator("#modeGalleryToggle").click();
for (const mode of savedModes) {
const shell = page.locator(`#modeGalleryGroups .mode-gallery-card-shell`).filter({ has: page.locator(`button[data-mode-profile-id="${mode.id}"]`) });
const details = shell.locator(".mode-gallery-schedule-details");
await details.locator("summary").click();
assert.match(await details.innerText(), /开始 · 持续/);
assert.match(await details.innerText(), /不自动回滚/);
}
await page.locator("#modeGalleryGroups").evaluate((element) => { element.scrollTop = 0; });
await page.screenshot({ path: path.join(evidenceDir, "schedule-management-desktop.png") });
await page.locator("#modeGalleryClose").click();
await page.setViewportSize({ width: 720, height: 1000 });
if (await page.locator('#v2SidebarToggle').getAttribute('aria-expanded') === 'true') {
await page.locator('#v2SidebarToggle').click();
}
await page.locator(".assistant-mode-draft-scheduled").last().scrollIntoViewIfNeeded();
await page.screenshot({ path: path.join(evidenceDir, "schedule-preview-mobile.png") });
assert.equal(await page.locator("body").evaluate((body) => body.scrollWidth > window.innerWidth + 1), false);
// The backend sequence above is real and strictly dry-run. Separately
// verify the asynchronous UI contract with a synthetic running/failed
// response; it must never launch an actual timed hardware execution.
await page.setViewportSize({ width: 1600, height: 1000 });
const mode = savedModes[1];
mockedExecution = {
request_id: "browser-running-qa", status: "running", execution_mode: "dry_run",
orchestration_route: "mode_schedule_execute", vehicle_id: "denza_n7", mode_profile: mode,
schedule_execution: { status: "running", run_id: "qa-run", session_id: "qa-session" },
};
const messagesBefore = await page.locator('#assistantMessages [data-message-role]').count();
await page.locator(`.mode-chips button[data-mode-profile-id="${mode.id}"]`).click();
await waitIdle();
await page.waitForFunction((id) => document.querySelector(`.mode-chips button[data-mode-profile-id="${id}"]`)?.dataset.modeState === "started", mode.id);
const actionStatus = page.locator('.assistant-action-status').last();
assert.match(await actionStatus.innerText(), /已启动动态时间轴/);
await page.waitForFunction(() => Array.from(document.querySelectorAll('.assistant-action-status')).at(-1)?.textContent.includes('阶段 2'));
await page.waitForFunction(() => Array.from(document.querySelectorAll('.assistant-action-status')).at(-1)?.textContent.includes('QA阶段模拟失败'));
assert.equal(await actionStatus.getAttribute('class'), 'assistant-action-status failed');
assert.equal(statusReads, 2);
assert.equal(await page.locator('#assistantMessages [data-message-role]').count(), messagesBefore + 2);
await page.screenshot({ path: path.join(evidenceDir, "schedule-background-failure.png") });
assert.deepEqual(errors, []);
assert.ok(posted.filter((request) => request.pathname === "/api/assistant/stream").every((request) => request.payload.execution_mode === "dry_run"));
assert.equal(posted.filter((request) => request.pathname === "/api/mode-profile-drafts/trial").length, 2);
assert.equal(posted.filter((request) => request.pathname === "/api/mode-profiles").length, 2);
console.log(`PASS ${baseURL}: default passenger and explicit driver static/dynamic previews -> both exact user phrases -> phased preview -> virtual trial -> save -> top tooltip -> management; desktop 1600x1000 and 720x1000; no JS errors; no hardware requests forwarded`);
console.log(`Evidence: ${evidenceDir}`);
} catch (error) {
await page.screenshot({ path: path.join(evidenceDir, "schedule-browser-failure.png") });
console.error(`Page text: ${(await page.locator("body").innerText()).slice(-5000)}`);
throw error;
} finally {
await context.close();
await browser.close();
}
}
main().catch((error) => { console.error(error); process.exitCode = 1; });
+109
View File
@@ -0,0 +1,109 @@
"""Full browser -> deterministic assistant -> schedule trial/save, without hardware."""
from __future__ import annotations
import os
import shutil
import subprocess
import threading
from http import HTTPStatus
from http.server import ThreadingHTTPServer
from pathlib import Path
from unittest.mock import MagicMock, patch
from urllib.parse import urlparse
import pytest
from cognitive_agent.application.codex_assistant import CodexAssistantService
from cognitive_agent.application.seat_service import SeatDemoService
from cognitive_agent.auth import AuthConfig, AuthRuntime
from cognitive_agent.server import REPOSITORY_ROOT, SeatDemoHandler
def test_mode_schedule_real_browser_preview_trial_and_save(tmp_path: Path) -> None:
node = shutil.which("node")
if not node:
runtime = Path.home() / ".cache" / "codex-runtimes" / "codex-primary-runtime" / "dependencies" / "node" / "bin"
node = next((str(path) for path in (runtime / "node.exe", runtime / "node") if path.is_file()), None)
if not node:
pytest.skip("Node.js is required for the schedule browser regression")
env = dict(os.environ)
modules = Path(node).resolve().parent.parent / "node_modules"
if modules.is_dir():
env["NODE_PATH"] = os.pathsep.join(value for value in (str(modules), env.get("NODE_PATH")) if value)
probe = subprocess.run([node, "-e", "require.resolve('playwright')"], env=env, capture_output=True, encoding="utf-8", check=False)
if probe.returncode:
pytest.skip("Playwright is required for the schedule browser regression")
with patch.dict(os.environ, {
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_WIRED_PASSENGER_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
"COGNITIVE_AGENT_EXECUTION_MODE": "dry_run",
"COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(tmp_path / "user-modes"),
}):
service = SeatDemoService(REPOSITORY_ROOT)
service.artifact_root = tmp_path / "artifacts"
# A failed deterministic route must fail visibly, never launch a model or
# make network calls to an external assistant during the browser test.
client = MagicMock()
client.available = False
client.status = "test-offline"
assistant = CodexAssistantService(client, fast_implicit_controls=True)
forbidden = []
class BrowserHandler(SeatDemoHandler):
auth = AuthRuntime(AuthConfig(False, "", "", b"", 3600, 5, 300, 120, False))
def do_GET(self) -> None: # noqa: N802
path = urlparse(self.path).path
if path in {"/api/hardware/seat-state", "/api/hardware/vehicle-state"}:
# Both legacy GET paths can touch physical adapters or firmware.
self._json({"status": "unavailable", "hardware_execution_allowed": False})
return
if path == "/api/health":
self._json({"status": "ok", "assistant": {"status": "ready", "available": True}})
return
super().do_GET()
def do_POST(self) -> None: # noqa: N802
path = urlparse(self.path).path
if path == "/api/extension/display-sync":
self._json({"status": "ok"})
return
if path not in {"/api/assistant/stream", "/api/mode-profile-drafts/trial", "/api/mode-profiles"}:
forbidden.append(path)
self._json({"error": "browser_test_write_forbidden"}, HTTPStatus.FORBIDDEN)
return
super().do_POST()
def _read_json_body(self, **kwargs):
payload = super()._read_json_body(**kwargs)
if payload.get("execution_mode", "dry_run") != "dry_run":
forbidden.append(f"execution_mode:{payload.get('execution_mode')}")
raise ValueError("Browser regression permits dry_run only")
return payload
def log_message(self, *_args: object) -> None:
pass
BrowserHandler.service = service
BrowserHandler.assistant = assistant
server = ThreadingHTTPServer(("127.0.0.1", 0), BrowserHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
completed = subprocess.run([
node, str(Path(__file__).with_name("mode_schedule_browser.cjs")),
f"http://127.0.0.1:{server.server_port}", str(tmp_path),
], env=env, capture_output=True, encoding="utf-8", timeout=180, check=False)
assert completed.returncode == 0, completed.stdout + completed.stderr
assert not forbidden, forbidden
assert len(list((tmp_path / "user-modes").rglob("*.yaml"))) == 2
finally:
server.shutdown()
server.server_close()
thread.join(timeout=3)
assistant.close()
service.close()
+10 -10
View File
@@ -621,7 +621,7 @@ operations:
def test_static_console_and_runtime_endpoint(self) -> None:
with self.opener.open(f"{self.base_url}/", timeout=3) as response:
html = response.read().decode("utf-8")
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-131", timeout=3) as response:
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-132", timeout=3) as response:
app_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/gesture-control.js?v=android-pad-9", timeout=3
@@ -656,7 +656,7 @@ operations:
) as response:
extension_styles = response.read().decode("utf-8").replace("\r\n", "\n")
with self.opener.open(
f"{self.base_url}/assistant-chat.js?v=codex-assistant-27", timeout=3
f"{self.base_url}/assistant-chat.js?v=codex-assistant-28", timeout=3
) as response:
assistant_script = response.read().decode("utf-8")
with self.opener.open(
@@ -697,7 +697,7 @@ operations:
self.assertIn("CMD 0x431 · FB 0x41A · CH1", html)
self.assertIn("CMD 0x431 · FB 0x388 · CH1", html)
self.assertNotIn("透明车体 · 实体设备", html)
self.assertIn('src="/app.js?v=dual-seat-131"', html)
self.assertIn('src="/app.js?v=dual-seat-132"', html)
self.assertIn('href="/styles.css?v=dual-seat-23"', html)
self.assertIn('src="/gesture-control.js?v=android-pad-9"', html)
self.assertNotIn('class="vehicle-system-hud"', html)
@@ -718,8 +718,8 @@ operations:
self.assertNotIn("滚动查看更多", html)
self.assertIn('aria-label="发送消息"', html)
self.assertIn('id="assistantTranscript"', html)
self.assertIn('href="/assistant-chat.css?v=codex-assistant-27"', html)
self.assertIn('src="/assistant-chat.js?v=codex-assistant-27"', html)
self.assertIn('href="/assistant-chat.css?v=codex-assistant-28"', html)
self.assertIn('src="/assistant-chat.js?v=codex-assistant-28"', html)
self.assertIn('href="/energy-monitor.css?v=energy-monitor-4"', html)
self.assertIn('src="/energy-monitor.js?v=energy-monitor-4"', html)
self.assertNotIn('"admin"', console_script)
@@ -992,14 +992,14 @@ operations:
self.assertIn("CAN 指令已提交,等待车辆状态确认", console_script)
self.assertIn('canvas.addEventListener("webglcontextlost"', app_script)
self.assertIn('"cognitive-agent:module-change"', app_script)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v278"', service_worker)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v279"', service_worker)
self.assertIn('/console-v2.css?v=delivery-state-66', service_worker)
self.assertIn('/console-v2.js?v=delivery-state-62', service_worker)
self.assertIn('/assistant-chat.css?v=codex-assistant-27', service_worker)
self.assertIn('/assistant-chat.js?v=codex-assistant-27', service_worker)
self.assertIn('/assistant-chat.css?v=codex-assistant-28', service_worker)
self.assertIn('/assistant-chat.js?v=codex-assistant-28', service_worker)
self.assertIn('/energy-monitor.css?v=energy-monitor-4', service_worker)
self.assertIn('/energy-monitor.js?v=energy-monitor-4', service_worker)
self.assertIn('"/app.js?v=dual-seat-131"', service_worker)
self.assertIn('"/app.js?v=dual-seat-132"', service_worker)
self.assertIn('"/speech-input.js?v=sensevoice-13"', service_worker)
self.assertIn('"/speech-input.css?v=sensevoice-7"', service_worker)
self.assertIn('"/speech-audio-worklet.js?v=sensevoice-5"', service_worker)
@@ -1456,7 +1456,7 @@ operations:
) as response:
extension_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/app.js?v=dual-seat-131", timeout=3
f"{self.base_url}/app.js?v=dual-seat-132", timeout=3
) as response:
app_script = response.read().decode("utf-8")