vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化语音输入/普通文本输入

Browse Source
This commit is contained in:
minghanchen
2026-09-03 13:36:40 +08:00
parent 009d8ca73e
commit 32d179143f
26 changed files with 2318 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_frontend_theme_contract.py
+1 -1
View File
@@ -47,7 +47,7 @@ def test_theme_asset_versions_are_consistent_with_the_pwa_shell() -> None:
):
assert asset in html
assert f'"{asset}"' in service_worker
assert 'const CACHE_VERSION = "cognitive-agent-pad-v227"' in service_worker
assert 'const CACHE_VERSION = "cognitive-agent-pad-v228"' in service_worker
def test_vehicle_twin_icons_and_nested_overlays_use_semantic_tokens() -> None:
tests/unit/test_nfc_frontend_contract.py
+2 -1
View File
@@ -139,4 +139,5 @@ def test_main_parsed_control_panel_renders_validated_infrared_operations() -> No
assert 'frameLabel: "GSCU1B TCP"' in console
assert "已解析控制指令" in console
assert 'li[data-can-channel="IR"]' in styles
assert "console-v2.js?v=delivery-state-54" in html
assert "console-v2.js?v=delivery-state-55" in html
assert 'speech_semantic: "语音语义"' in console
tests/unit/test_requested_frontend_enhancements.py
+67
View File
@@ -568,6 +568,73 @@ def test_mode_editor_prepares_one_to_many_mqtt_message_sequences() -> None:
)
def test_left_sidebar_extracts_speech_and_peripheral_control_commands() -> None:
node = shutil.which("node")
if node is None:
return
console = source("console-v2.js")
helper_start = console.index(" const normalizedCanTarget")
helper_end = console.index("\n\n function renderParsedCanCommands", helper_start)
helpers = console[helper_start:helper_end]
sample = {
"execution_mode": "dry_run",
"function_configs": {
"native_climate": {
"spec": {
"operations": [
{
"id": "speech-temperature",
"capability": "climate.set_temperature",
"target": {"device": "hvac", "zone": "driver"},
"parameters": {"temperature_c": 24},
"parse_source": "speech_semantic",
}
]
}
},
"self_extension": {
"spec": {
"operations": [
{
"id": "speech-peripheral",
"capability": "infrared.emit",
"target": {
"device": "third-party-light",
"name": "第三方氛围灯",
},
"parameters": {"action": "开启", "slot": 9},
"parse_source": "local_rule",
}
]
}
},
},
}
script = (
f"{helpers}\n"
f"const result = {json.dumps(sample, ensure_ascii=False)};\n"
"process.stdout.write(JSON.stringify({"
"commands:parsedCanCommands(result),"
'speechLabel:parsedCommandSourceLabel("speech_semantic")'
"}));"
)
completed = subprocess.run(
[node, "-e", script],
check=True,
capture_output=True,
text=True,
)
output = json.loads(completed.stdout)
assert output["speechLabel"] == "语音语义"
assert len(output["commands"]) == 2
assert {item["channel"] for item in output["commands"]} == {"CAN1", "IR"}
assert {item["command"] for item in output["commands"]} == {
"ac temp driver 24",
"infrared emit 9",
}
def test_interactive_console_keeps_actions_inside_composer_and_collapsed_rails_aligned() -> None:
console = source("console-v2.js")
styles = source("console-v2.css")
tests/unit/test_speech_capability_alignment.py
+27 -7
View File
@@ -148,7 +148,7 @@ class SpeechCapabilityAlignmentTests(unittest.TestCase):
"passenger",
"seat.ventilation",
"on",
{"state": "on"},
{"state": "on", "level": 1},
),
),
},
@@ -432,18 +432,38 @@ class SpeechCapabilityAlignmentTests(unittest.TestCase):
def test_colloquial_seat_aliases_do_not_turn_pragmatic_one_into_level(self) -> None:
cases = (
("把驾驶座通风打开", "driver", "seat.set_driver_ventilation_native"),
("把乘客座通风打开", "passenger", "seat.set_ventilation"),
("给副驾驶开一下座椅通风", "passenger", "seat.set_ventilation"),
(
"把驾驶座通风打开",
"driver",
"seat.set_driver_ventilation_native",
{"state": "on", "level": 2},
),
(
"把乘客座通风打开",
"passenger",
"seat.set_ventilation",
{"state": "on", "level": 1},
),
(
"给副驾驶开一下座椅通风",
"passenger",
"seat.set_ventilation",
{"state": "on", "level": 1},
),
)
for text, target, capability in cases:
for text, target, capability, parameters in cases:
with self.subTest(text=text):
result = self.processor.process(text)
self.assertTrue(result.execution_eligible)
self.assertEqual(len(result.commands), 1)
self.assertEqual(result.commands[0].target_scope, target)
self.assertEqual(result.commands[0].capability, capability)
self.assertEqual(result.commands[0].parameters, {"state": "on"})
self.assertEqual(result.commands[0].parameters, parameters)
self.assertIn(
"level",
result.commands[0].qualifiers.get("implicit_parameters", ()),
)
self.assertNotIn("设置为", result.clean_text)
explicit_level = self.processor.process("主驾座椅通风开到一档")
self.assertEqual(explicit_level.commands[0].parameters["level"], 1)
@@ -455,7 +475,7 @@ class SpeechCapabilityAlignmentTests(unittest.TestCase):
)
expected = (
("seat.set_driver_ventilation_native", "driver", {"state": "on", "level": 2}),
("seat.set_ventilation", "passenger", {"state": "on"}),
("seat.set_ventilation", "passenger", {"state": "on", "level": 1}),
("climate.set_temperature", "driver", {"temperature_c": 24}),
)
for text in cases:
tests/unit/test_speech_orchestration_bridge.py
+192
View File
@@ -15,6 +15,44 @@ from cognitive_agent.infrastructure.speech.understanding import (
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
def _canonical_actions(result: dict[str, object]) -> list[dict[str, object]]:
intent_ir = result.get("intent_ir")
assert isinstance(intent_ir, dict)
spec = intent_ir.get("spec")
assert isinstance(spec, dict)
actions = spec.get("actions")
assert isinstance(actions, list)
return [
{
"domain": action.get("domain"),
"capability": action.get("capability"),
"target": action.get("target"),
"operation": action.get("operation"),
"parameters": action.get("parameters"),
}
for action in actions
if isinstance(action, dict)
]
def _sorted_canonical_actions(result: dict[str, object]) -> list[dict[str, object]]:
return sorted(
_canonical_actions(result),
key=lambda action: (
str(action["domain"]),
str(action["capability"]),
repr(action["target"]),
repr(action["parameters"]),
),
)
def _mode_service(directory: str) -> SeatDemoService:
service = SeatDemoService(REPOSITORY_ROOT)
service.artifact_root = Path(directory) / "artifacts"
return service
@pytest.mark.parametrize(
("utterance", "capability", "target", "parameters"),
(
@@ -194,3 +232,157 @@ def test_every_registered_native_runtime_capability_keeps_native_bridge_binding(
assert [
action["capability"] for action in parsed["native_can_actions"]
] == [capability]
def test_text_and_speech_share_implicit_mode_target_and_start_command() -> None:
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
understanding = processor.process("将座椅调整为木马模式。").to_dict()
understanding["utterance_id"] = "mode-start-equivalence"
with tempfile.TemporaryDirectory() as directory, patch.dict(
os.environ,
{
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
"COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(Path(directory) / "modes"),
},
clear=False,
):
service = _mode_service(directory)
text_result = service.process(
"将座椅调整为木马模式。",
provider="local",
conversation_id="conv-text-mode-start",
)
speech_result = service.process(
understanding["clean_text"],
provider="local",
conversation_id="conv-speech-mode-start",
_speech_understanding=understanding,
)
assert understanding["execution_eligible"] is True
assert text_result["status"] == speech_result["status"] == "succeeded"
assert _canonical_actions(text_result) == _canonical_actions(speech_result)
assert _canonical_actions(text_result) == [
{
"domain": "seat",
"capability": "seat.start_dynamic_mode",
"target": {
"device": "seat",
"zone": "front_right",
"instance": "passenger",
},
"operation": "activate",
"parameters": {"mode_id": 1},
}
]
def test_text_and_speech_share_dynamic_mode_stop_command() -> None:
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
understanding = processor.process("关闭木马模式").to_dict()
understanding["utterance_id"] = "mode-stop-equivalence"
with tempfile.TemporaryDirectory() as directory, patch.dict(
os.environ,
{
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
"COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(Path(directory) / "modes"),
},
clear=False,
):
service = _mode_service(directory)
text_result = service.process(
"关闭木马模式",
provider="local",
conversation_id="conv-text-mode-stop",
)
speech_result = service.process(
understanding["clean_text"],
provider="local",
conversation_id="conv-speech-mode-stop",
_speech_understanding=understanding,
)
assert understanding["clean_text"] == "关闭木马模式。"
assert understanding["execution_eligible"] is True
assert text_result["status"] == speech_result["status"] == "succeeded"
assert _canonical_actions(text_result) == _canonical_actions(speech_result)
assert _canonical_actions(text_result)[0]["capability"] == "seat.stop_dynamic_mode"
assert _canonical_actions(text_result)[0]["parameters"] == {}
def test_explicit_incompatible_mode_target_fails_closed_for_both_inputs() -> None:
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
understanding = processor.process("将主驾座椅调整为木马模式。").to_dict()
with tempfile.TemporaryDirectory() as directory, patch.dict(
os.environ,
{
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
"COGNITIVE_AGENT_MODE_PROFILE_ROOT": str(Path(directory) / "modes"),
},
clear=False,
):
service = _mode_service(directory)
text_result = service.process(
"将主驾座椅调整为木马模式。",
provider="local",
conversation_id="conv-text-mode-rejected",
)
assert understanding["semantic_status"] == "recognized_unsupported"
assert understanding["execution_eligible"] is False
assert text_result["status"] == "needs_confirmation"
assert _canonical_actions(text_result) == []
def test_text_and_speech_share_colloquial_multidomain_commands() -> None:
utterance = "先打开主驾座椅通风，再打开副驾座椅通风，最后把空调调成二十四度"
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
understanding = processor.process(utterance).to_dict()
understanding["utterance_id"] = "multidomain-equivalence"
with tempfile.TemporaryDirectory() as directory, patch.dict(
os.environ,
{
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
},
clear=False,
):
service = _mode_service(directory)
text_result = service.process(
utterance,
provider="local",
conversation_id="conv-text-multidomain",
)
speech_result = service.process(
understanding["clean_text"],
provider="local",
conversation_id="conv-speech-multidomain",
_speech_understanding=understanding,
)
assert understanding["clean_text"] == (
"打开主驾座椅通风，然后打开副驾座椅通风，然后将空调设置为24度。"
)
assert understanding["execution_eligible"] is True
assert text_result["status"] == speech_result["status"] == "succeeded"
assert _sorted_canonical_actions(text_result) == _sorted_canonical_actions(
speech_result
)
assert {
action["capability"] for action in _canonical_actions(text_result)
} == {
"seat.set_driver_ventilation_native",
"seat.set_ventilation",
"climate.set_temperature",
}
tests/unit/test_two_stage_can_parsing.py
+4 -3
View File
@@ -186,9 +186,10 @@ def test_frontend_two_stage_contract_and_source_badges() -> None:
assert '"cognitive-agent:validated-can-commands"' in app
assert "本地规则命令已执行；云端补充失败" in app
assert 'class="v2-can-command-source"' in console
assert 'item.parseSource === "direct_control" ? "设备控件"' in console
assert 'item.parseSource === "saved_mode_command_flow" ? "模式卡片"' in console
assert 'item.parseSource === "local_rule" ? "本地规则" : "云端模型"' in console
assert 'direct_control: "设备控件"' in console
assert 'speech_semantic: "语音语义"' in console
assert 'saved_mode_command_flow: "模式卡片"' in console
assert 'local_rule: "本地规则"' in console
assert 'item.deliveryLabel' in console
assert 'data-delivery-state="${escapeHtml(item.deliveryState)}"' in console
assert '"cognitive-agent:validated-can-commands"' in console