vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: restore mode clicks and compose seat-aware assistant scenes

Browse Source
This commit is contained in:
vee
2026-09-12 13:33:38 +08:00
parent 76057be044
commit d09b5967f8
28 changed files with 1247 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/mode_rail_browser.cjs
+178
View File
@@ -0,0 +1,178 @@
const assert = require("node:assert/strict");
const path = require("node:path");
const { chromium } = require("playwright");
const [baseURL, evidenceDir] = process.argv.slice(2);
const desktop = { width: 1600, height: 1000 };
async function main() {
const browser = await chromium.launch({
headless: true,
...(process.env.N7WEB_BROWSER_CHANNEL
? { channel: process.env.N7WEB_BROWSER_CHANNEL }
: process.platform === "win32" ? { channel: "msedge" } : {}),
});
const context = await browser.newContext({ viewport: desktop, serviceWorkers: "block" });
const page = await context.newPage();
const requests = [];
const errors = [];
const unexpectedWrites = [];
let heldResponse = null;
let nextReply = "success";
const answer = (success) => [
{ type: "action.completed", success, payload: {
status: success ? "succeeded" : "failed", execution_mode: "dry_run",
} },
{ type: "answer.completed", answer: success ? "模式执行测试完成" : "模式执行测试失败" },
].map((item) => JSON.stringify(item)).join("\n") + "\n";
const fulfill = (route, success = true) => route.fulfill({
status: 200, contentType: "application/x-ndjson", body: answer(success),
});
page.on("pageerror", (error) => errors.push(error.message));
page.on("console", (message) => {
if (message.type() === "error") errors.push(`${message.text()} ${message.location().url || ''}`);
});
async function handleApiRoute(route) {
const pathname = new URL(route.request().url()).pathname;
if (pathname === '/api/hardware/seat-state') {
// This legacy GET may attempt automatic firmware recovery. Never touch
// the physical gateway while verifying browser interactions.
await route.fulfill({json:{status:'unavailable',hardware_execution_allowed:false}});
return;
}
// The isolated server deliberately does not launch Codex or publish a
// physical display mirror. Keep these background integration probes local.
if (pathname === '/api/health' || pathname === '/api/extension/display-sync') {
await route.fulfill({json:{status:'ok',assistant:{status:'ready',available:true}}});
return;
}
if (pathname === "/api/assistant/stream") {
requests.push(route.request().postDataJSON());
if (nextReply === "hold") { heldResponse = route; return; }
const success = nextReply !== "fail";
nextReply = "success";
await fulfill(route, success);
return;
}
if (route.request().method() !== "GET") {
unexpectedWrites.push(route.request().url());
await route.fulfill({ status: 403, body: "test forbids vehicle writes" });
return;
}
await route.continue();
}
await context.route("**/api/**", handleApiRoute);
const rail = page.locator(".mode-chips");
const office = rail.locator('button[data-mode-profile-id="office"]');
const waitIdle = () => page.waitForFunction(() => !window.CognitiveAgentAssistant.isPending());
const waitState = (id, state) => page.waitForFunction(
({ id, state }) => document.querySelector(`.mode-chips [data-mode-profile-id="${id}"]`)?.dataset.modeState === state,
{ id, state },
);
async function expectOneCommand(action, command) {
const before = requests.length;
await action();
await page.waitForFunction(() => document.querySelector('#utterance').getAttribute('aria-busy') === 'false');
await waitIdle();
assert.equal(requests.length, before + 1, "gesture must submit exactly once");
assert.equal(requests.at(-1).message, command);
}
try {
await page.goto(baseURL, { waitUntil: "domcontentloaded" });
await page.waitForFunction(() => document.querySelectorAll('.mode-chips button').length > 3);
await page.waitForFunction(() => document.body.classList.contains('assistant-chat-enabled'));
assert.equal(await page.title(), "511工程-智能座舱");
assert.equal(new URL(page.url()).pathname, "/");
assert.ok((await page.locator('body').innerText()).includes("整车孪生界面"));
assert.ok(await rail.evaluate((element) => element.scrollWidth > element.clientWidth));
// This is a real mouse sequence: pointerdown -> pointerup -> click. The
// prior pointer capture bug cannot be detected by button.dispatchEvent().
await page.evaluate(() => window.CognitiveAgentAssistant.close());
await expectOneCommand(() => office.click(), "进入办公模式");
await waitState("office", "succeeded");
assert.equal(await page.locator('#assistantDrawerToggle').getAttribute('aria-expanded'), 'true');
assert.ok((await page.locator('#assistantMessages').innerText()).includes('进入办公模式'));
// A small physical hand movement is still a click.
const box = await office.boundingBox();
await expectOneCommand(async () => {
await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2);
await page.mouse.down();
await page.mouse.move(box.x + box.width / 2 + 2, box.y + box.height / 2);
await page.mouse.up();
}, "进入办公模式");
// Dragging from a card must scroll, without accidentally running it.
const beforeDrag = requests.length;
const startScroll = await rail.evaluate((element) => element.scrollLeft);
await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2);
await page.mouse.down();
await page.mouse.move(box.x + box.width / 2 - 90, box.y + box.height / 2, { steps: 8 });
await page.mouse.up();
assert.ok(await rail.evaluate((element) => element.scrollLeft) > startScroll);
assert.equal(requests.length, beforeDrag);
await rail.evaluate((element) => { element.scrollLeft = 0; });
await expectOneCommand(() => office.click(), "进入办公模式");
// Keyboard activation and management cards share the same submission path.
await office.focus();
await expectOneCommand(() => page.keyboard.press('Enter'), "进入办公模式");
await office.focus();
await expectOneCommand(() => page.keyboard.press('Space'), "进入办公模式");
await page.locator('#modeGalleryToggle').click();
await expectOneCommand(() => page.locator('#modeGalleryGroups button[data-mode-profile-id="nap"]').click(), "进入小憩模式");
await waitState("nap", "succeeded");
// Busy requests are not duplicated, nor does another card cancel them.
nextReply = 'hold';
const beforeBusy = requests.length;
await office.click();
await waitState('office', 'running');
await office.click();
assert.equal(requests.length, beforeBusy + 1);
assert.equal(await page.evaluate(() => window.CognitiveAgentAssistant.isPending()), true);
await fulfill(heldResponse);
nextReply = 'success';
await waitIdle();
await waitState('office', 'succeeded');
nextReply = 'fail';
await expectOneCommand(() => office.click(), "进入办公模式");
await waitState('office', 'failed');
// A catalog with few visible cards must also work without overflow.
await rail.evaluate(element => {
for (const child of element.children) {
if (child.dataset.modeProfileId !== 'office') child.style.display = 'none';
}
});
await page.waitForFunction(() => {const e=document.querySelector('.mode-chips');return e.scrollWidth <= e.clientWidth;});
await expectOneCommand(() => office.click(), "进入办公模式");
await rail.evaluate(element => {
for (const child of element.children) child.style.removeProperty('display');
});
await page.setViewportSize(desktop);
await page.screenshot({path:path.join(evidenceDir,'mode-rail-desktop.png')});
assert.deepEqual(errors, []);
assert.deepEqual(unexpectedWrites, []);
// Touch input uses native scrolling and also submits once when tapped.
const touchContext = await browser.newContext({viewport:{width:720,height:1000},hasTouch:true,serviceWorkers:'block'});
await touchContext.route('**/api/**', handleApiRoute);
const touchPage = await touchContext.newPage();
await touchPage.goto(baseURL, {waitUntil:'domcontentloaded'});
await touchPage.waitForFunction(() => document.querySelectorAll('.mode-chips button').length > 3);
const beforeTouch = requests.length;
await touchPage.locator('.mode-chips button[data-mode-profile-id="office"]').tap();
await touchPage.waitForFunction(() => document.querySelector('.mode-chips [data-mode-profile-id="office"]').dataset.modeState === 'succeeded');
assert.equal(requests.length,beforeTouch+1);
assert.equal(requests.at(-1).message,'进入办公模式');
await touchPage.screenshot({path:path.join(evidenceDir,'mode-rail-touch.png')});
await touchContext.close();
console.log('PASS: click, jitter, drag, Enter, Space, gallery, busy, failure, resize, touch; no runtime errors');
} finally {
await browser.close();
}
}
main().catch(error => { console.error(error); process.exitCode = 1; });
tests/e2e/test_mode_rail_browser.py
+86
View File
@@ -0,0 +1,86 @@
"""Exercise real pointer/click dispatch on the complete console in Chromium."""
from __future__ import annotations
import os
import shutil
import subprocess
import threading
from http import HTTPStatus
from http.server import ThreadingHTTPServer
from pathlib import Path
from unittest.mock import patch
import pytest
from cognitive_agent.application.seat_service import SeatDemoService
from cognitive_agent.auth import AuthConfig, AuthRuntime
from cognitive_agent.server import REPOSITORY_ROOT, SeatDemoHandler
def test_mode_rail_real_browser_interactions(tmp_path: Path) -> None:
node = shutil.which("node")
if not node:
pytest.skip("Node.js is required for the browser interaction regression")
env = dict(os.environ)
bundled_modules = Path(node).resolve().parent.parent / "node_modules"
if bundled_modules.is_dir():
env["NODE_PATH"] = os.pathsep.join(
item for item in (str(bundled_modules), env.get("NODE_PATH")) if item
)
available = subprocess.run(
[node, "-e", "require.resolve('playwright')"],
env=env,
capture_output=True,
encoding="utf-8",
check=False,
)
if available.returncode:
pytest.skip("Playwright is required for the real browser regression")
with patch.dict(
os.environ,
{
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_WIRED_PASSENGER_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
},
):
service = SeatDemoService(REPOSITORY_ROOT)
service.artifact_root = tmp_path / "artifacts"
class BrowserHandler(SeatDemoHandler):
auth = AuthRuntime(AuthConfig(False, "", "", b"", 3600, 5, 300, 120, False))
def do_POST(self) -> None: # noqa: N802
# Only GET reaches this test server. Browser requests to the
# assistant are intercepted; unexpected writes fail closed.
self._json({"error": "browser_test_write_forbidden"}, HTTPStatus.FORBIDDEN)
def log_message(self, *_args: object) -> None:
pass
BrowserHandler.service = service
server = ThreadingHTTPServer(("127.0.0.1", 0), BrowserHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
completed = subprocess.run(
[
node,
str(Path(__file__).with_name("mode_rail_browser.cjs")),
f"http://127.0.0.1:{server.server_port}",
str(tmp_path),
],
env=env,
capture_output=True,
encoding="utf-8",
timeout=120,
check=False,
)
assert completed.returncode == 0, completed.stdout + completed.stderr
finally:
server.shutdown()
server.server_close()
thread.join(timeout=3)
service.close()
tests/e2e/test_seat_web_api.py
+10 -10
View File
@@ -621,7 +621,7 @@ operations:
def test_static_console_and_runtime_endpoint(self) -> None:
with self.opener.open(f"{self.base_url}/", timeout=3) as response:
html = response.read().decode("utf-8")
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-130", timeout=3) as response:
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-131", timeout=3) as response:
app_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/gesture-control.js?v=android-pad-9", timeout=3
@@ -656,7 +656,7 @@ operations:
) as response:
extension_styles = response.read().decode("utf-8").replace("\r\n", "\n")
with self.opener.open(
f"{self.base_url}/assistant-chat.js?v=codex-assistant-26", timeout=3
f"{self.base_url}/assistant-chat.js?v=codex-assistant-27", timeout=3
) as response:
assistant_script = response.read().decode("utf-8")
with self.opener.open(
@@ -697,7 +697,7 @@ operations:
self.assertIn("CMD 0x431 · FB 0x41A · CH1", html)
self.assertIn("CMD 0x431 · FB 0x388 · CH1", html)
self.assertNotIn("透明车体 · 实体设备", html)
self.assertIn('src="/app.js?v=dual-seat-130"', html)
self.assertIn('src="/app.js?v=dual-seat-131"', html)
self.assertIn('href="/styles.css?v=dual-seat-23"', html)
self.assertIn('src="/gesture-control.js?v=android-pad-9"', html)
self.assertNotIn('class="vehicle-system-hud"', html)
@@ -718,8 +718,8 @@ operations:
self.assertNotIn("滚动查看更多", html)
self.assertIn('aria-label="发送消息"', html)
self.assertIn('id="assistantTranscript"', html)
self.assertIn('href="/assistant-chat.css?v=codex-assistant-26"', html)
self.assertIn('src="/assistant-chat.js?v=codex-assistant-26"', html)
self.assertIn('href="/assistant-chat.css?v=codex-assistant-27"', html)
self.assertIn('src="/assistant-chat.js?v=codex-assistant-27"', html)
self.assertIn('href="/energy-monitor.css?v=energy-monitor-4"', html)
self.assertIn('src="/energy-monitor.js?v=energy-monitor-4"', html)
self.assertNotIn('"admin"', console_script)
@@ -992,14 +992,14 @@ operations:
self.assertIn("CAN 指令已提交，等待车辆状态确认", console_script)
self.assertIn('canvas.addEventListener("webglcontextlost"', app_script)
self.assertIn('"cognitive-agent:module-change"', app_script)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v277"', service_worker)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v278"', service_worker)
self.assertIn('/console-v2.css?v=delivery-state-66', service_worker)
self.assertIn('/console-v2.js?v=delivery-state-62', service_worker)
self.assertIn('/assistant-chat.css?v=codex-assistant-26', service_worker)
self.assertIn('/assistant-chat.js?v=codex-assistant-26', service_worker)
self.assertIn('/assistant-chat.css?v=codex-assistant-27', service_worker)
self.assertIn('/assistant-chat.js?v=codex-assistant-27', service_worker)
self.assertIn('/energy-monitor.css?v=energy-monitor-4', service_worker)
self.assertIn('/energy-monitor.js?v=energy-monitor-4', service_worker)
self.assertIn('"/app.js?v=dual-seat-130"', service_worker)
self.assertIn('"/app.js?v=dual-seat-131"', service_worker)
self.assertIn('"/speech-input.js?v=sensevoice-13"', service_worker)
self.assertIn('"/speech-input.css?v=sensevoice-7"', service_worker)
self.assertIn('"/speech-audio-worklet.js?v=sensevoice-5"', service_worker)
@@ -1456,7 +1456,7 @@ operations:
) as response:
extension_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/app.js?v=dual-seat-130", timeout=3
f"{self.base_url}/app.js?v=dual-seat-131", timeout=3
) as response:
app_script = response.read().decode("utf-8")