vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化语音输入/普通文本输入

Browse Source
This commit is contained in:
minghanchen
2026-09-03 13:36:40 +08:00
parent 009d8ca73e
commit 32d179143f
26 changed files with 2318 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/test_seat_web_api.py
+146 -6
View File
@@ -624,7 +624,7 @@ operations:
) as response:
gesture_script = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/console-v2.js?v=delivery-state-54", timeout=3
f"{self.base_url}/console-v2.js?v=delivery-state-55", timeout=3
) as response:
console_script = response.read().decode("utf-8")
with self.opener.open(
@@ -632,7 +632,7 @@ operations:
) as response:
console_styles = response.read().decode("utf-8")
with self.opener.open(
f"{self.base_url}/speech-input.js?v=sensevoice-6", timeout=3
f"{self.base_url}/speech-input.js?v=sensevoice-7", timeout=3
) as response:
speech_script = response.read().decode("utf-8")
with self.opener.open(
@@ -685,9 +685,9 @@ operations:
self.assertIn('href="/styles.css?v=dual-seat-21"', html)
self.assertIn('src="/gesture-control.js?v=android-pad-9"', html)
self.assertNotIn('class="vehicle-system-hud"', html)
self.assertIn('src="/console-v2.js?v=delivery-state-54"', html)
self.assertIn('src="/console-v2.js?v=delivery-state-55"', html)
self.assertIn('href="/console-v2.css?v=delivery-state-50"', html)
self.assertIn('src="/speech-input.js?v=sensevoice-6"', html)
self.assertIn('src="/speech-input.js?v=sensevoice-7"', html)
self.assertIn('href="/speech-input.css?v=sensevoice-5"', html)
self.assertIn('id="speechDictationButton"', html)
self.assertIn('class="speech-waveform"', html)
@@ -959,9 +959,9 @@ operations:
self.assertIn("CAN 指令已提交，等待车辆状态确认", console_script)
self.assertIn('canvas.addEventListener("webglcontextlost"', app_script)
self.assertIn('"cognitive-agent:module-change"', app_script)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v227"', service_worker)
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v228"', service_worker)
self.assertIn('"/app.js?v=dual-seat-114"', service_worker)
self.assertIn('"/speech-input.js?v=sensevoice-6"', service_worker)
self.assertIn('"/speech-input.js?v=sensevoice-7"', service_worker)
self.assertIn('"/speech-input.css?v=sensevoice-5"', service_worker)
self.assertIn('"/speech-audio-worklet.js?v=sensevoice-4"', service_worker)
self.assertIn('fetch("/api/speech/transcribe"', speech_script)
@@ -979,6 +979,9 @@ operations:
self.assertIn("speechExecutionEligible", app_script)
self.assertIn("speech_utterance_id", app_script)
self.assertIn("speechUtteranceId", speech_script)
self.assertIn("result.peripheral_routing?.execution_eligible", speech_script)
self.assertIn("result.routing_text", speech_script)
self.assertIn('speech_semantic: "语音语义"', console_script)
self.assertIn("语音意图已识别，但当前车辆能力不支持执行", app_script)
self.assertIn("语音控制参数不完整，请补充后再执行", app_script)
self.assertIn(".speech-dictation.is-listening", speech_styles)
@@ -1741,6 +1744,143 @@ operations:
self.assertEqual(operation["domain"], "self_extension")
self.assertTrue(operation["telemetry"]["gateway_acknowledged"])
def test_speech_routes_exact_asr_text_to_learned_peripheral_and_ui_config(
self,
) -> None:
original_extension = SeatDemoHandler.extension
original_nfc = SeatDemoHandler.nfc
original_speech = SeatDemoHandler.speech
extension = SelfExtensionService(
SimulationGateway(),
catalog_db_path=Path(
self.extension_temporary_directory.name,
"speech-nfc-route.sqlite3",
),
)
extension.sync_peripheral_catalog(
[
{
"id": "cap-third-party-light-on",
"status": "learned",
"enabled": True,
"device_id": "third-party-light",
"device_name": "第三方氛围灯",
"object_names": ["第三方氛围灯"],
"card_uid": "1A53DD95",
"action": "开启",
"action_names": ["打开", "开启"],
"slot": 9,
}
]
)
SeatDemoHandler.extension = extension
SeatDemoHandler.nfc = SimpleNamespace(
snapshot=lambda: {
"revision": 1,
"reader": {"online": True},
"card": {"action": "present", "uid": "1A53DD95"},
}
)
understanding = UnderstandingTranscriptProcessor(REPOSITORY_ROOT).process(
"打开第三方氛围灯"
).to_dict()
understanding["utterance_id"] = "e2e-speech-peripheral"
class SpeechStub:
@staticmethod
def transcribe_wav(
content: bytes,
*,
utterance_id: str | None = None,
) -> dict[str, object]:
if content != b"RIFF-speech-peripheral":
raise AssertionError("unexpected audio payload")
return {**understanding, "utterance_id": utterance_id}
@staticmethod
def cached_result(utterance_id: str) -> dict[str, object]:
if utterance_id != "e2e-speech-peripheral":
raise AssertionError("unexpected speech utterance id")
return dict(understanding)
SeatDemoHandler.speech = SpeechStub()
try:
transcription_request = urllib.request.Request(
f"{self.base_url}/api/speech/transcribe",
method="POST",
headers={
"Content-Type": "audio/wav",
"X-Speech-Utterance-Id": "e2e-speech-peripheral",
},
data=b"RIFF-speech-peripheral",
)
with self.opener.open(transcription_request, timeout=3) as response:
transcription = json.loads(response.read().decode("utf-8"))
wrong_target_request = urllib.request.Request(
f"{self.base_url}/api/requests",
method="POST",
headers={"Content-Type": "application/json"},
data=json.dumps(
{
"utterance": understanding["clean_text"],
"speech_utterance_id": "e2e-speech-peripheral",
"provider": "local",
"conversation_id": "conv-e2e-speech-wrong-target",
"execution_mode": "dry_run",
},
ensure_ascii=False,
).encode("utf-8"),
)
with self.assertRaises(urllib.error.HTTPError) as rejected:
self.opener.open(wrong_target_request, timeout=3)
self.assertEqual(rejected.exception.code, 409)
rejected_payload = json.loads(
rejected.exception.read().decode("utf-8")
)
self.assertEqual(rejected_payload["error"], "speech_text_changed")
control_request = urllib.request.Request(
f"{self.base_url}/api/requests",
method="POST",
headers={"Content-Type": "application/json"},
data=json.dumps(
{
"utterance": transcription["routing_text"],
"speech_utterance_id": "e2e-speech-peripheral",
"provider": "openai",
"conversation_id": "conv-e2e-speech-peripheral",
"execution_mode": "dry_run",
"client_type": "web_console",
},
ensure_ascii=False,
).encode("utf-8"),
)
with self.opener.open(control_request, timeout=3) as response:
result = json.loads(response.read().decode("utf-8"))
finally:
SeatDemoHandler.extension = original_extension
SeatDemoHandler.nfc = original_nfc
SeatDemoHandler.speech = original_speech
self.assertEqual(transcription["routing_text"], "打开第三方氛围灯")
self.assertTrue(
transcription["peripheral_routing"]["execution_eligible"]
)
self.assertEqual(result["status"], "succeeded")
self.assertEqual(result["orchestration_route"], "nfc_peripheral")
self.assertEqual(set(result["function_configs"]), {"self_extension"})
operation = result["function_configs"]["self_extension"]["spec"][
"operations"
][0]
self.assertEqual(operation["capability"], "infrared.emit")
self.assertEqual(operation["parameters"]["slot"], 9)
self.assertEqual(operation["parse_source"], "local_rule")
self.assertEqual(
result["speech_understanding"]["routing"],
"validated_peripheral_text",
)
def test_main_request_combines_explicit_ir_clause_with_existing_can_parser(self) -> None:
original_extension = SeatDemoHandler.extension
original_nfc = SeatDemoHandler.nfc
tests/fixtures/speech_text_regression.json
Vendored
+4 -1
View File
@@ -58,5 +58,8 @@
{"name":"elliptical_temperature","raw":"空调二十四度","clean":"将空调设置为24度。","capabilities":["climate.set_temperature"],"cancelled":0,"uncertain":false,"execution_eligible":true},
{"name":"elliptical_value_continuation","raw":"打开空调，然后二十四度","clean":"打开空调，然后将空调设置为24度。","capabilities":["climate.set_power","climate.set_temperature"],"cancelled":0,"uncertain":false,"execution_eligible":true},
{"name":"punctuated_ambient_slots","raw":"氛围灯打开，调成蓝色，亮度百分之五十","clean":"打开氛围灯，然后将氛围灯颜色调整为蓝色并将亮度设置为50%。","capabilities":["ambient_light.set_power","ambient_light.set_illumination"],"cancelled":0,"uncertain":false,"execution_eligible":true},
{"name":"non_control_continuation_fails_closed","raw":"打开空调，然后开玩笑。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":true,"execution_eligible":false}
{"name":"non_control_continuation_fails_closed","raw":"打开空调，然后开玩笑。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":true,"execution_eligible":false},
{"name":"rocking_horse_mode_unscoped","raw":"将座椅调整为木马模式。","clean":"将座椅调整为木马模式。","capabilities":[null],"cancelled":0,"uncertain":false,"execution_eligible":true},
{"name":"rocking_horse_mode_enter","raw":"进入木马模式","clean":"将座椅调整为木马模式。","capabilities":[null],"cancelled":0,"uncertain":false,"execution_eligible":true},
{"name":"rocking_horse_mode_stop","raw":"关闭木马模式","clean":"关闭木马模式。","capabilities":[null],"cancelled":0,"uncertain":false,"execution_eligible":true}
]
tests/unit/test_frontend_theme_contract.py
+1 -1
View File
@@ -47,7 +47,7 @@ def test_theme_asset_versions_are_consistent_with_the_pwa_shell() -> None:
):
assert asset in html
assert f'"{asset}"' in service_worker
assert 'const CACHE_VERSION = "cognitive-agent-pad-v227"' in service_worker
assert 'const CACHE_VERSION = "cognitive-agent-pad-v228"' in service_worker
def test_vehicle_twin_icons_and_nested_overlays_use_semantic_tokens() -> None:
tests/unit/test_nfc_frontend_contract.py
+2 -1
View File
@@ -139,4 +139,5 @@ def test_main_parsed_control_panel_renders_validated_infrared_operations() -> No
assert 'frameLabel: "GSCU1B TCP"' in console
assert "已解析控制指令" in console
assert 'li[data-can-channel="IR"]' in styles
assert "console-v2.js?v=delivery-state-54" in html
assert "console-v2.js?v=delivery-state-55" in html
assert 'speech_semantic: "语音语义"' in console
tests/unit/test_requested_frontend_enhancements.py
+67
View File
@@ -568,6 +568,73 @@ def test_mode_editor_prepares_one_to_many_mqtt_message_sequences() -> None:
)
def test_left_sidebar_extracts_speech_and_peripheral_control_commands() -> None:
node = shutil.which("node")
if node is None:
return
console = source("console-v2.js")
helper_start = console.index(" const normalizedCanTarget")
helper_end = console.index("\n\n function renderParsedCanCommands", helper_start)
helpers = console[helper_start:helper_end]
sample = {
"execution_mode": "dry_run",
"function_configs": {
"native_climate": {
"spec": {
"operations": [
{
"id": "speech-temperature",
"capability": "climate.set_temperature",
"target": {"device": "hvac", "zone": "driver"},
"parameters": {"temperature_c": 24},
"parse_source": "speech_semantic",
}
]
}
},
"self_extension": {
"spec": {
"operations": [
{
"id": "speech-peripheral",
"capability": "infrared.emit",
"target": {
"device": "third-party-light",
"name": "第三方氛围灯",
},
"parameters": {"action": "开启", "slot": 9},
"parse_source": "local_rule",
}
]
}
},
},
}
script = (
f"{helpers}\n"
f"const result = {json.dumps(sample, ensure_ascii=False)};\n"
"process.stdout.write(JSON.stringify({"
"commands:parsedCanCommands(result),"
'speechLabel:parsedCommandSourceLabel("speech_semantic")'
"}));"
)
completed = subprocess.run(
[node, "-e", script],
check=True,
capture_output=True,
text=True,
)
output = json.loads(completed.stdout)
assert output["speechLabel"] == "语音语义"
assert len(output["commands"]) == 2
assert {item["channel"] for item in output["commands"]} == {"CAN1", "IR"}
assert {item["command"] for item in output["commands"]} == {
"ac temp driver 24",
"infrared emit 9",
}
def test_interactive_console_keeps_actions_inside_composer_and_collapsed_rails_aligned() -> None:
console = source("console-v2.js")
styles = source("console-v2.css")
tests/unit/test_speech_capability_alignment.py
+27 -7
View File
@@ -148,7 +148,7 @@ class SpeechCapabilityAlignmentTests(unittest.TestCase):
"passenger",
"seat.ventilation",
"on",
{"state": "on"},
{"state": "on", "level": 1},
),
),
},
@@ -432,18 +432,38 @@ class SpeechCapabilityAlignmentTests(unittest.TestCase):
def test_colloquial_seat_aliases_do_not_turn_pragmatic_one_into_level(self) -> None:
cases = (
("把驾驶座通风打开", "driver", "seat.set_driver_ventilation_native"),
("把乘客座通风打开", "passenger", "seat.set_ventilation"),
("给副驾驶开一下座椅通风", "passenger", "seat.set_ventilation"),
(
"把驾驶座通风打开",
"driver",
"seat.set_driver_ventilation_native",
{"state": "on", "level": 2},
),
(
"把乘客座通风打开",
"passenger",
"seat.set_ventilation",
{"state": "on", "level": 1},
),
(
"给副驾驶开一下座椅通风",
"passenger",
"seat.set_ventilation",
{"state": "on", "level": 1},
),
)
for text, target, capability in cases:
for text, target, capability, parameters in cases:
with self.subTest(text=text):
result = self.processor.process(text)
self.assertTrue(result.execution_eligible)
self.assertEqual(len(result.commands), 1)
self.assertEqual(result.commands[0].target_scope, target)
self.assertEqual(result.commands[0].capability, capability)
self.assertEqual(result.commands[0].parameters, {"state": "on"})
self.assertEqual(result.commands[0].parameters, parameters)
self.assertIn(
"level",
result.commands[0].qualifiers.get("implicit_parameters", ()),
)
self.assertNotIn("设置为", result.clean_text)
explicit_level = self.processor.process("主驾座椅通风开到一档")
self.assertEqual(explicit_level.commands[0].parameters["level"], 1)
@@ -455,7 +475,7 @@ class SpeechCapabilityAlignmentTests(unittest.TestCase):
)
expected = (
("seat.set_driver_ventilation_native", "driver", {"state": "on", "level": 2}),
("seat.set_ventilation", "passenger", {"state": "on"}),
("seat.set_ventilation", "passenger", {"state": "on", "level": 1}),
("climate.set_temperature", "driver", {"temperature_c": 24}),
)
for text in cases:
tests/unit/test_speech_orchestration_bridge.py
+192
View File
@@ -15,6 +15,44 @@ from cognitive_agent.infrastructure.speech.understanding import (
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
def _canonical_actions(result: dict[str, object]) -> list[dict[str, object]]:
intent_ir = result.get("intent_ir")
assert isinstance(intent_ir, dict)
spec = intent_ir.get("spec")
assert isinstance(spec, dict)
actions = spec.get("actions")
assert isinstance(actions, list)
return [
{
"domain": action.get("domain"),
"capability": action.get("capability"),
"target": action.get("target"),
"operation": action.get("operation"),
"parameters": action.get("parameters"),
}
for action in actions
if isinstance(action, dict)
]
def _sorted_canonical_actions(result: dict[str, object]) -> list[dict[str, object]]:
return sorted(
_canonical_actions(result),
key=lambda action: (
str(action["domain"]),
str(action["capability"]),
repr(action["target"]),
repr(action["parameters"]),
),
)
def _mode_service(directory: str) -> SeatDemoService:
service = SeatDemoService(REPOSITORY_ROOT)
service.artifact_root = Path(directory) / "artifacts"
return service
@pytest.mark.parametrize(
("utterance", "capability", "target", "parameters"),
(
@@ -194,3 +232,157 @@ def test_every_registered_native_runtime_capability_keeps_native_bridge_binding(
assert [
action["capability"] for action in parsed["native_can_actions"]
] == [capability]
def test_text_and_speech_share_implicit_mode_target_and_start_command() -> None:
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
understanding = processor.process("将座椅调整为木马模式。").to_dict()
understanding["utterance_id"] = "mode-start-equivalence"
with tempfile.TemporaryDirectory() as directory, patch.dict(
os.environ,
{
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
"COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(Path(directory) / "modes"),
},
clear=False,
):
service = _mode_service(directory)
text_result = service.process(
"将座椅调整为木马模式。",
provider="local",
conversation_id="conv-text-mode-start",
)
speech_result = service.process(
understanding["clean_text"],
provider="local",
conversation_id="conv-speech-mode-start",
_speech_understanding=understanding,
)
assert understanding["execution_eligible"] is True
assert text_result["status"] == speech_result["status"] == "succeeded"
assert _canonical_actions(text_result) == _canonical_actions(speech_result)
assert _canonical_actions(text_result) == [
{
"domain": "seat",
"capability": "seat.start_dynamic_mode",
"target": {
"device": "seat",
"zone": "front_right",
"instance": "passenger",
},
"operation": "activate",
"parameters": {"mode_id": 1},
}
]
def test_text_and_speech_share_dynamic_mode_stop_command() -> None:
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
understanding = processor.process("关闭木马模式").to_dict()
understanding["utterance_id"] = "mode-stop-equivalence"
with tempfile.TemporaryDirectory() as directory, patch.dict(
os.environ,
{
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
"COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(Path(directory) / "modes"),
},
clear=False,
):
service = _mode_service(directory)
text_result = service.process(
"关闭木马模式",
provider="local",
conversation_id="conv-text-mode-stop",
)
speech_result = service.process(
understanding["clean_text"],
provider="local",
conversation_id="conv-speech-mode-stop",
_speech_understanding=understanding,
)
assert understanding["clean_text"] == "关闭木马模式。"
assert understanding["execution_eligible"] is True
assert text_result["status"] == speech_result["status"] == "succeeded"
assert _canonical_actions(text_result) == _canonical_actions(speech_result)
assert _canonical_actions(text_result)[0]["capability"] == "seat.stop_dynamic_mode"
assert _canonical_actions(text_result)[0]["parameters"] == {}
def test_explicit_incompatible_mode_target_fails_closed_for_both_inputs() -> None:
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
understanding = processor.process("将主驾座椅调整为木马模式。").to_dict()
with tempfile.TemporaryDirectory() as directory, patch.dict(
os.environ,
{
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
"COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(Path(directory) / "modes"),
},
clear=False,
):
service = _mode_service(directory)
text_result = service.process(
"将主驾座椅调整为木马模式。",
provider="local",
conversation_id="conv-text-mode-rejected",
)
assert understanding["semantic_status"] == "recognized_unsupported"
assert understanding["execution_eligible"] is False
assert text_result["status"] == "needs_confirmation"
assert _canonical_actions(text_result) == []
def test_text_and_speech_share_colloquial_multidomain_commands() -> None:
utterance = "先打开主驾座椅通风，再打开副驾座椅通风，最后把空调调成二十四度"
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
understanding = processor.process(utterance).to_dict()
understanding["utterance_id"] = "multidomain-equivalence"
with tempfile.TemporaryDirectory() as directory, patch.dict(
os.environ,
{
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
},
clear=False,
):
service = _mode_service(directory)
text_result = service.process(
utterance,
provider="local",
conversation_id="conv-text-multidomain",
)
speech_result = service.process(
understanding["clean_text"],
provider="local",
conversation_id="conv-speech-multidomain",
_speech_understanding=understanding,
)
assert understanding["clean_text"] == (
"打开主驾座椅通风，然后打开副驾座椅通风，然后将空调设置为24度。"
)
assert understanding["execution_eligible"] is True
assert text_result["status"] == speech_result["status"] == "succeeded"
assert _sorted_canonical_actions(text_result) == _sorted_canonical_actions(
speech_result
)
assert {
action["capability"] for action in _canonical_actions(text_result)
} == {
"seat.set_driver_ventilation_native",
"seat.set_ventilation",
"climate.set_temperature",
}
tests/unit/test_two_stage_can_parsing.py
+4 -3
View File
@@ -186,9 +186,10 @@ def test_frontend_two_stage_contract_and_source_badges() -> None:
assert '"cognitive-agent:validated-can-commands"' in app
assert "本地规则命令已执行；云端补充失败" in app
assert 'class="v2-can-command-source"' in console
assert 'item.parseSource === "direct_control" ? "设备控件"' in console
assert 'item.parseSource === "saved_mode_command_flow" ? "模式卡片"' in console
assert 'item.parseSource === "local_rule" ? "本地规则" : "云端模型"' in console
assert 'direct_control: "设备控件"' in console
assert 'speech_semantic: "语音语义"' in console
assert 'saved_mode_command_flow: "模式卡片"' in console
assert 'local_rule: "本地规则"' in console
assert 'item.deliveryLabel' in console
assert 'data-delivery-state="${escapeHtml(item.deliveryState)}"' in console
assert '"cognitive-agent:validated-can-commands"' in console