feat: integrate Chinese speech understanding and capability routing
This commit is contained in:
vee
@@ -13,15 +13,18 @@ from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from cognitive_agent.auth import AuthConfig, AuthRuntime, hash_password
|
||||
from cognitive_agent.application.seat_service import REPOSITORY_ROOT, SeatDemoService
|
||||
from cognitive_agent.application.mode_profile import ModeProfileRegistry
|
||||
from cognitive_agent.application.seat_service import REPOSITORY_ROOT, SeatDemoService
|
||||
from cognitive_agent.application.self_extension import (
|
||||
SelfExtensionService,
|
||||
SimulationGateway,
|
||||
)
|
||||
from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError
|
||||
from cognitive_agent.auth import AuthConfig, AuthRuntime, hash_password
|
||||
from cognitive_agent.infrastructure.can.access import HardwareAccessController
|
||||
from cognitive_agent.infrastructure.llm.openai_responses import OpenAIIntentError
|
||||
from cognitive_agent.infrastructure.speech.understanding import (
|
||||
UnderstandingTranscriptProcessor,
|
||||
)
|
||||
from cognitive_agent.server import SeatDemoHandler
|
||||
|
||||
|
||||
@@ -332,7 +335,7 @@ operations:
|
||||
def test_static_console_and_runtime_endpoint(self) -> None:
|
||||
with self.opener.open(f"{self.base_url}/", timeout=3) as response:
|
||||
html = response.read().decode("utf-8")
|
||||
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-103", timeout=3) as response:
|
||||
with self.opener.open(f"{self.base_url}/app.js?v=dual-seat-104", timeout=3) as response:
|
||||
app_script = response.read().decode("utf-8")
|
||||
with self.opener.open(
|
||||
f"{self.base_url}/gesture-control.js?v=android-pad-9", timeout=3
|
||||
@@ -346,6 +349,18 @@ operations:
|
||||
f"{self.base_url}/console-v2.css?v=sync-layout-41", timeout=3
|
||||
) as response:
|
||||
console_styles = response.read().decode("utf-8")
|
||||
with self.opener.open(
|
||||
f"{self.base_url}/speech-input.js?v=sensevoice-6", timeout=3
|
||||
) as response:
|
||||
speech_script = response.read().decode("utf-8")
|
||||
with self.opener.open(
|
||||
f"{self.base_url}/speech-input.css?v=sensevoice-4", timeout=3
|
||||
) as response:
|
||||
speech_styles = response.read().decode("utf-8")
|
||||
with self.opener.open(
|
||||
f"{self.base_url}/speech-audio-worklet.js?v=sensevoice-4", timeout=3
|
||||
) as response:
|
||||
speech_worklet = response.read().decode("utf-8")
|
||||
with self.opener.open(
|
||||
f"{self.base_url}/self-extension.js?v=nfc-toy-33", timeout=3
|
||||
) as response:
|
||||
@@ -368,6 +383,8 @@ operations:
|
||||
nfc_status = json.loads(response.read().decode("utf-8"))
|
||||
with self.opener.open(f"{self.base_url}/api/runtime", timeout=3) as response:
|
||||
runtime = json.loads(response.read().decode("utf-8"))
|
||||
with self.opener.open(f"{self.base_url}/api/speech/status", timeout=3) as response:
|
||||
speech_status = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
self.assertIn("511工程-智能座舱", html)
|
||||
self.assertIn('id="seatTwinCanvas"', html)
|
||||
@@ -382,12 +399,16 @@ operations:
|
||||
self.assertIn("CMD 0x431 · FB 0x41A · CH1", html)
|
||||
self.assertIn("CMD 0x431 · FB 0x388 · CH1", html)
|
||||
self.assertNotIn("透明车体 · 实体设备", html)
|
||||
self.assertIn('src="/app.js?v=dual-seat-103"', html)
|
||||
self.assertIn('src="/app.js?v=dual-seat-104"', html)
|
||||
self.assertIn('href="/styles.css?v=dual-seat-16"', html)
|
||||
self.assertIn('src="/gesture-control.js?v=android-pad-9"', html)
|
||||
self.assertNotIn('class="vehicle-system-hud"', html)
|
||||
self.assertIn('src="/console-v2.js?v=sync-layout-43"', html)
|
||||
self.assertIn('href="/console-v2.css?v=sync-layout-41"', html)
|
||||
self.assertIn('src="/speech-input.js?v=sensevoice-6"', html)
|
||||
self.assertIn('href="/speech-input.css?v=sensevoice-4"', html)
|
||||
self.assertIn('id="speechDictationButton"', html)
|
||||
self.assertIn('class="speech-waveform"', html)
|
||||
self.assertIn('<body class="console-v2">', html)
|
||||
self.assertIn('id="v2BootScreen"', html)
|
||||
self.assertIn('正在加载车机控制与实时状态', html)
|
||||
@@ -622,7 +643,29 @@ operations:
|
||||
self.assertIn('client_type: "web_console"', app_script)
|
||||
self.assertIn('canvas.addEventListener("webglcontextlost"', app_script)
|
||||
self.assertIn('"cognitive-agent:module-change"', app_script)
|
||||
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v210"', service_worker)
|
||||
self.assertIn('const CACHE_VERSION = "cognitive-agent-pad-v216"', service_worker)
|
||||
self.assertIn('"/speech-input.js?v=sensevoice-6"', service_worker)
|
||||
self.assertIn('"/speech-input.css?v=sensevoice-4"', service_worker)
|
||||
self.assertIn('"/speech-audio-worklet.js?v=sensevoice-4"', service_worker)
|
||||
self.assertIn('fetch("/api/speech/transcribe"', speech_script)
|
||||
self.assertIn('"X-Speech-Utterance-Id"', speech_script)
|
||||
self.assertIn("result.commands", speech_script)
|
||||
self.assertIn("AudioWorkletNode", speech_script)
|
||||
self.assertIn('registerProcessor("speech-capture-processor"', speech_worklet)
|
||||
self.assertIn("navigator.mediaDevices.getUserMedia", speech_script)
|
||||
self.assertIn("PREFERRED_MICROPHONE", speech_script)
|
||||
self.assertIn("ENDPOINT_SILENCE_MS", speech_script)
|
||||
self.assertIn("speechExecutionEligible", speech_script)
|
||||
self.assertIn("speechSemanticStatus", speech_script)
|
||||
self.assertIn('"recognized_unsupported"', speech_script)
|
||||
self.assertIn('"incomplete"', speech_script)
|
||||
self.assertIn("speechExecutionEligible", app_script)
|
||||
self.assertIn("speech_utterance_id", app_script)
|
||||
self.assertIn("speechUtteranceId", speech_script)
|
||||
self.assertIn("语音意图已识别，但当前车辆能力不支持执行", app_script)
|
||||
self.assertIn("语音控制参数不完整，请补充后再执行", app_script)
|
||||
self.assertIn(".speech-dictation.is-listening", speech_styles)
|
||||
self.assertEqual(speech_status["model"], "SenseVoiceSmall")
|
||||
self.assertIn('"/assets/pets/self-extension-cat.png?v=3"', service_worker)
|
||||
self.assertIn(".then(() => self.skipWaiting())", service_worker)
|
||||
self.assertNotIn("const cached = await caches.match(request)", service_worker)
|
||||
@@ -649,6 +692,7 @@ operations:
|
||||
self.assertIn("BENCH BUSY", app_script)
|
||||
self.assertIn("FBXLoader.js", html)
|
||||
self.assertIn('data-seat-layout="five-seat"', html)
|
||||
|
||||
self.assertIn('data-active-seat="front-right"', html)
|
||||
self.assertNotIn('placeholder="请输入服务端配置令牌"', html)
|
||||
self.assertIn('data-callout="backrest_recline"', html)
|
||||
@@ -856,13 +900,164 @@ operations:
|
||||
)
|
||||
)
|
||||
|
||||
def test_speech_transcription_endpoint_forwards_wav_without_persisting_it(self) -> None:
|
||||
class SpeechStub:
|
||||
received = b""
|
||||
|
||||
@classmethod
|
||||
def transcribe_wav(
|
||||
cls, content: bytes, *, utterance_id: str | None = None
|
||||
) -> dict[str, object]:
|
||||
cls.received = content
|
||||
return {
|
||||
"utterance_id": utterance_id,
|
||||
"text": "打开空调到二十四度。",
|
||||
"language": "zh",
|
||||
"duration_ms": 800,
|
||||
"inference_ms": 92,
|
||||
"model": "SenseVoiceSmall",
|
||||
}
|
||||
|
||||
original_speech = SeatDemoHandler.speech
|
||||
SeatDemoHandler.speech = SpeechStub()
|
||||
try:
|
||||
wav_content = b"RIFF-speech-test"
|
||||
request = urllib.request.Request(
|
||||
f"{self.base_url}/api/speech/transcribe",
|
||||
method="POST",
|
||||
headers={
|
||||
"Content-Type": "audio/wav",
|
||||
"Accept": "application/json",
|
||||
"X-Speech-Utterance-Id": "e2e-speech-1",
|
||||
},
|
||||
data=wav_content,
|
||||
)
|
||||
with self.opener.open(request, timeout=3) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
permissions_policy = response.headers["Permissions-Policy"]
|
||||
finally:
|
||||
SeatDemoHandler.speech = original_speech
|
||||
|
||||
self.assertEqual(SpeechStub.received, wav_content)
|
||||
self.assertEqual(payload["text"], "打开空调到二十四度。")
|
||||
self.assertEqual(payload["utterance_id"], "e2e-speech-1")
|
||||
self.assertIn("microphone=(self)", permissions_policy)
|
||||
|
||||
def test_speech_utterance_id_preserves_structured_slots_into_intent_ir(self) -> None:
|
||||
understanding = UnderstandingTranscriptProcessor(REPOSITORY_ROOT).process(
|
||||
"把副驾腿托向上调整到最大位置"
|
||||
).to_dict()
|
||||
understanding["utterance_id"] = "e2e-speech-bridge"
|
||||
|
||||
class SpeechStub:
|
||||
@staticmethod
|
||||
def cached_result(utterance_id: str) -> dict[str, object]:
|
||||
if utterance_id != "e2e-speech-bridge":
|
||||
raise AssertionError("unexpected speech utterance id")
|
||||
return understanding
|
||||
|
||||
original_speech = SeatDemoHandler.speech
|
||||
SeatDemoHandler.speech = SpeechStub()
|
||||
try:
|
||||
request = urllib.request.Request(
|
||||
f"{self.base_url}/api/requests",
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/json", "Accept": "application/json"},
|
||||
data=json.dumps(
|
||||
{
|
||||
"utterance": understanding["clean_text"],
|
||||
"speech_utterance_id": "e2e-speech-bridge",
|
||||
"client_type": "web_console",
|
||||
"provider": "openai",
|
||||
"conversation_id": "conv-e2e-speech-bridge",
|
||||
"execution_mode": "dry_run",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8"),
|
||||
)
|
||||
with self.opener.open(request, timeout=3) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
finally:
|
||||
SeatDemoHandler.speech = original_speech
|
||||
|
||||
self.assertEqual(payload["status"], "succeeded")
|
||||
self.assertEqual(payload["provider_used"], "speech_semantic")
|
||||
self.assertEqual(
|
||||
payload["speech_understanding"]["raw_text"],
|
||||
"把副驾腿托向上调整到最大位置",
|
||||
)
|
||||
action = payload["intent_ir"]["spec"]["actions"][0]
|
||||
self.assertEqual(action["capability"], "seat.set_legrest_height")
|
||||
self.assertEqual(action["target"]["instance"], "passenger")
|
||||
self.assertEqual(action["parameters"], {"position_pct": 100})
|
||||
self.assertEqual(action["provenance"]["parser"], "speech_semantic")
|
||||
|
||||
def test_roof_shade_speech_crosses_server_bridge_as_native_can(self) -> None:
|
||||
cases = (
|
||||
("把遮阳帘打开", {"motion": "open"}),
|
||||
("遮阳帘打开一半", {"motion": "half"}),
|
||||
)
|
||||
for index, (utterance, parameters) in enumerate(cases):
|
||||
with self.subTest(utterance=utterance):
|
||||
utterance_id = f"e2e-roof-shade-{index}"
|
||||
understanding = UnderstandingTranscriptProcessor(
|
||||
REPOSITORY_ROOT
|
||||
).process(utterance).to_dict()
|
||||
understanding["utterance_id"] = utterance_id
|
||||
|
||||
def cached_result(
|
||||
requested_id: str,
|
||||
*,
|
||||
expected_id: str = utterance_id,
|
||||
result: dict[str, object] = understanding,
|
||||
) -> dict[str, object]:
|
||||
if requested_id != expected_id:
|
||||
raise AssertionError("unexpected speech utterance id")
|
||||
return result
|
||||
|
||||
original_speech = SeatDemoHandler.speech
|
||||
SeatDemoHandler.speech = SimpleNamespace(cached_result=cached_result)
|
||||
try:
|
||||
request = urllib.request.Request(
|
||||
f"{self.base_url}/api/requests",
|
||||
method="POST",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
data=json.dumps(
|
||||
{
|
||||
"utterance": understanding["clean_text"],
|
||||
"speech_utterance_id": utterance_id,
|
||||
"client_type": "web_console",
|
||||
"provider": "openai",
|
||||
"conversation_id": f"conv-e2e-roof-shade-{index}",
|
||||
"execution_mode": "dry_run",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8"),
|
||||
)
|
||||
with self.opener.open(request, timeout=3) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
finally:
|
||||
SeatDemoHandler.speech = original_speech
|
||||
|
||||
self.assertEqual(payload["status"], "succeeded")
|
||||
self.assertEqual(payload["provider_used"], "speech_semantic")
|
||||
self.assertEqual(payload["deferred_requests"], [])
|
||||
action = payload["intent_ir"]["spec"]["actions"][0]
|
||||
self.assertEqual(action["capability"], "roof.set_shade_motion")
|
||||
self.assertEqual(action["target"]["zone"], "vehicle")
|
||||
self.assertEqual(action["parameters"], parameters)
|
||||
self.assertEqual(action["provenance"]["parser"], "speech_semantic")
|
||||
|
||||
def test_self_extension_vehicle_success_contract_is_strict(self) -> None:
|
||||
with self.opener.open(
|
||||
f"{self.base_url}/self-extension.js?v=extension-44", timeout=3
|
||||
) as response:
|
||||
extension_script = response.read().decode("utf-8")
|
||||
with self.opener.open(
|
||||
f"{self.base_url}/app.js?v=dual-seat-103", timeout=3
|
||||
f"{self.base_url}/app.js?v=dual-seat-104", timeout=3
|
||||
) as response:
|
||||
app_script = response.read().decode("utf-8")
|
||||
|
||||
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
[
|
||||
{"name":"single_power_on","raw":"打开空调。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":false},
|
||||
{"name":"single_power_off","raw":"关闭空调。","clean":"关闭空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":false},
|
||||
{"name":"single_temperature","raw":"把空调调到24度。","clean":"将空调设置为24度。","capabilities":["climate.set_temperature"],"cancelled":0,"uncertain":false},
|
||||
{"name":"power_and_temperature","raw":"打开空调到24度。","clean":"打开空调并设置为24度。","capabilities":["climate.set_power","climate.set_temperature"],"cancelled":0,"uncertain":false},
|
||||
{"name":"fan_level","raw":"把空调风量调到2档。","clean":"将空调风量设置为2档。","capabilities":["climate.set_fan_level"],"cancelled":0,"uncertain":false},
|
||||
{"name":"backrest_relative","raw":"把座椅靠背向后调24度。","clean":"将座椅靠背向后调整24度。","capabilities":["seat.set_backrest_recline"],"cancelled":0,"uncertain":false},
|
||||
{"name":"window_open","raw":"打开车窗。","clean":"打开车窗。","capabilities":["window.set_motion"],"cancelled":0,"uncertain":false},
|
||||
{"name":"ambient_on","raw":"打开氛围灯。","clean":"打开氛围灯。","capabilities":["ambient_light.set_power"],"cancelled":0,"uncertain":false},
|
||||
{"name":"seat_ventilation","raw":"打开座椅通风。","clean":"打开座椅通风。","capabilities":["seat.set_driver_ventilation_native"],"cancelled":0,"uncertain":false},
|
||||
{"name":"two_commands","raw":"打开空调，然后打开氛围灯。","clean":"打开空调，然后打开氛围灯。","capabilities":["climate.set_power","ambient_light.set_power"],"cancelled":0,"uncertain":false},
|
||||
{"name":"same_domain_scopes","raw":"打开前排空调，再打开后排空调。","clean":"打开前排空调，然后打开后排空调。","capabilities":["climate.set_power","climate.set_power"],"cancelled":0,"uncertain":false},
|
||||
{"name":"fillers","raw":"呃，那个那个那个打开空调到24度。","clean":"打开空调并设置为24度。","capabilities":["climate.set_power","climate.set_temperature"],"cancelled":0,"uncertain":false},
|
||||
{"name":"referential_demonstrative","raw":"打开那个车窗。","clean":"打开车窗。","capabilities":["window.set_motion"],"cancelled":0,"uncertain":false,"preserved_filler":"那个"},
|
||||
{"name":"action_stutter","raw":"打打打开空调。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":false},
|
||||
{"name":"target_echo","raw":"把座椅的靠背靠座椅靠背向后调24度。","clean":"将座椅靠背向后调整24度。","capabilities":["seat.set_backrest_recline"],"cancelled":0,"uncertain":false},
|
||||
{"name":"target_refinement","raw":"座椅调整到呃20呃靠背调整到24度。","clean":"将座椅靠背调整到24度。","capabilities":["seat.set_backrest_recline"],"cancelled":1,"uncertain":true},
|
||||
{"name":"domain_correction","raw":"打开空调，不对，把座椅靠背向后调24度。","clean":"将座椅靠背向后调整24度。","capabilities":["seat.set_backrest_recline"],"cancelled":1,"uncertain":false},
|
||||
{"name":"number_correction","raw":"空调调到24，不对，25度。","clean":"将空调设置为25度。","capabilities":["climate.set_temperature"],"cancelled":1,"uncertain":false},
|
||||
{"name":"direction_correction","raw":"把座椅往前，不对，往后一点。","clean":"将座椅向后调整一点。","capabilities":["seat.set_driver_axis_native"],"cancelled":1,"uncertain":false},
|
||||
{"name":"target_correction","raw":"把座椅，不对，把靠背往后调一点。","clean":"将座椅靠背向后调整一点。","capabilities":["seat.set_driver_axis_native"],"cancelled":1,"uncertain":false},
|
||||
{"name":"action_correction","raw":"把空调关掉，不，是打开。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":1,"uncertain":false},
|
||||
{"name":"multi_not_correction","raw":"空调调到24度，然后把座椅靠背调到20度。","clean":"将空调设置为24度，然后将座椅靠背调整到20度。","capabilities":["climate.set_temperature","seat.set_backrest_recline"],"cancelled":0,"uncertain":true},
|
||||
{"name":"cancel_incomplete","raw":"把空调调……算了。","clean":"","capabilities":[],"cancelled":1,"uncertain":false},
|
||||
{"name":"negated_open","raw":"不要打开空调。","clean":"不要打开空调。","capabilities":[],"cancelled":1,"uncertain":false},
|
||||
{"name":"negated_close","raw":"不要关闭空调。","clean":"不要关闭空调。","capabilities":[],"cancelled":1,"uncertain":false},
|
||||
{"name":"filler_cancel","raw":"嗯嗯，那个……算了。","clean":"","capabilities":[],"cancelled":1,"uncertain":false},
|
||||
{"name":"unrelated_chat","raw":"今天天气还不错。","clean":"","capabilities":[],"cancelled":0,"uncertain":false},
|
||||
{"name":"unrelated_question","raw":"我们中午吃什么？","clean":"","capabilities":[],"cancelled":0,"uncertain":false},
|
||||
{"name":"unknown_device","raw":"打开雨刷。","clean":"","capabilities":[],"cancelled":0,"uncertain":true},
|
||||
{"name":"incomplete_known_target","raw":"空调挺凉快。","clean":"","capabilities":[],"cancelled":0,"uncertain":true},
|
||||
{"name":"decimal_temperature","raw":"把空调调到24.5度。","clean":"将空调设置为24.5度。","capabilities":["climate.set_temperature"],"cancelled":0,"uncertain":false},
|
||||
{"name":"chinese_number","raw":"把空调调到二十四度。","clean":"将空调设置为24度。","capabilities":["climate.set_temperature"],"cancelled":0,"uncertain":false},
|
||||
{"name":"lost_marker_false_start","raw":"空调打开打，把座椅的靠背靠座椅靠背向后调24度。","clean":"将座椅靠背向后调整24度。","capabilities":["seat.set_backrest_recline"],"cancelled":1,"uncertain":false},
|
||||
{"name":"polite_scaffold","raw":"请帮我打开空调，谢谢。","clean":"打开空调。","capabilities":["climate.set_power"],"cancelled":0,"uncertain":false},
|
||||
{"name":"front_light_unsupported","raw":"打开前大灯","clean":"打开前大灯。","capabilities":[null],"cancelled":0,"uncertain":true,"execution_eligible":false},
|
||||
{"name":"reading_ambient_color_compound","raw":"打开阅读灯，并且打开氛围灯，并把氛围灯变成红色","clean":"打开阅读灯，然后打开氛围灯，然后将氛围灯颜色调整为红色。","capabilities":["interior_light.set_reading_power","ambient_light.set_power","ambient_light.set_illumination"],"cancelled":0,"uncertain":true,"execution_eligible":false},
|
||||
{"name":"window_half_open","raw":"打开一半车窗","clean":"将车窗打开到50%。","capabilities":["window.set_position"],"cancelled":0,"uncertain":false,"execution_eligible":true},
|
||||
{"name":"passenger_legrest_maximum","raw":"把副驾腿托向上调整到最大位置","clean":"将副驾座椅腿托向上调整到100%。","capabilities":["seat.set_legrest_height"],"cancelled":0,"uncertain":false,"execution_eligible":true},
|
||||
{"name":"driver_zero_gravity_unsupported","raw":"把主驾座椅调整为零重力模式","clean":"将主驾座椅调整为零重力模式。","capabilities":[null],"cancelled":0,"uncertain":true,"execution_eligible":false},
|
||||
{"name":"shade_then_invalid_seat_degree","raw":"打开遮阳帘，然后把副驾座椅向后调整50度","clean":"打开遮阳帘，然后把副驾座椅向后调整50度。","capabilities":["roof.set_shade_motion","seat.set_longitudinal_position"],"parameter_numbers":[],"cancelled":0,"uncertain":true,"execution_eligible":false},
|
||||
{"name":"window_quarter_open","raw":"打开四分之一车窗","clean":"将车窗打开到25%。","capabilities":["window.set_position"],"cancelled":0,"uncertain":false,"execution_eligible":true},
|
||||
{"name":"passenger_thermal_pair_and_backrest_max","raw":"打开副驾座椅通风和加热，然后将副驾座椅靠背调整到最大位置","clean":"打开副驾座椅通风，然后打开副驾座椅加热，然后将副驾座椅靠背调整到100%。","capabilities":["seat.set_ventilation","seat.set_heating","seat.set_backrest_recline"],"cancelled":0,"uncertain":false,"execution_eligible":true},
|
||||
{"name":"driver_thermal_pair_and_temperature","raw":"打开主驾座椅通风和加热，并且把空调调整为24度","clean":"打开主驾座椅通风，然后打开主驾座椅加热，然后将空调设置为24度。","capabilities":["seat.set_driver_ventilation_native","seat.set_driver_heating_native","climate.set_temperature"],"cancelled":0,"uncertain":false,"execution_eligible":true},
|
||||
{"name":"thermal_reverse_order_connector","raw":"打开副驾座椅加热以及通风","clean":"打开副驾座椅加热，然后打开副驾座椅通风。","capabilities":["seat.set_heating","seat.set_ventilation"],"cancelled":0,"uncertain":false,"execution_eligible":true},
|
||||
{"name":"window_three_quarters","raw":"打开四分之三车窗","clean":"将车窗打开到75%。","capabilities":["window.set_position"],"cancelled":0,"uncertain":false,"execution_eligible":true},
|
||||
{"name":"shade_quarter_position","raw":"把遮阳帘打开到四分之一","clean":"将遮阳帘打开到25%。","capabilities":["roof.set_shade_position"],"cancelled":0,"uncertain":false,"execution_eligible":true},
|
||||
{"name":"shade_open_bridge_regression","raw":"把遮阳帘打开","clean":"打开遮阳帘。","capabilities":["roof.set_shade_motion"],"cancelled":0,"uncertain":false,"execution_eligible":true},
|
||||
{"name":"shade_half_open_bridge_regression","raw":"遮阳帘打开一半","clean":"将遮阳帘打开一半。","capabilities":["roof.set_shade_motion"],"cancelled":0,"uncertain":false,"execution_eligible":true}
|
||||
]
|
||||
@@ -0,0 +1,525 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from cognitive_agent.application.native_can_intent import merge_native_can_intent
|
||||
from cognitive_agent.application.native_can_runtime import NativeCanRuntime
|
||||
from cognitive_agent.infrastructure.speech.capability import CapabilityResolver
|
||||
from cognitive_agent.infrastructure.speech.understanding import (
|
||||
UnderstandingTranscriptProcessor,
|
||||
)
|
||||
from cognitive_agent.infrastructure.speech.vocabulary import DeviceTerm
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _valid_value(spec: dict[str, Any]) -> Any:
|
||||
allowed = spec.get("allowed_values")
|
||||
if allowed:
|
||||
return allowed[0]
|
||||
value_type = str(spec.get("type", "string"))
|
||||
if value_type in {"integer", "number"}:
|
||||
minimum = spec.get("minimum", 0)
|
||||
return int(minimum) if value_type == "integer" else float(minimum)
|
||||
if value_type == "boolean":
|
||||
return True
|
||||
if value_type == "object":
|
||||
return {}
|
||||
if value_type == "array":
|
||||
return []
|
||||
if str(spec.get("name")) == "color_hex":
|
||||
return "#FF0000"
|
||||
if str(spec.get("name")) == "preset_name":
|
||||
return "回归测试姿态"
|
||||
return "test"
|
||||
|
||||
|
||||
def _sample_parameters(definition: Any) -> dict[str, Any]:
|
||||
specs = list(definition.parameters)
|
||||
selected = [item for item in specs if item.get("required")]
|
||||
if not selected and specs:
|
||||
selected = [specs[0]]
|
||||
return {str(item["name"]): _valid_value(item) for item in selected}
|
||||
|
||||
|
||||
class SpeechCapabilityAlignmentTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
|
||||
cls.resolver = CapabilityResolver(REPOSITORY_ROOT)
|
||||
cls.native_runtime = NativeCanRuntime(REPOSITORY_ROOT)
|
||||
|
||||
def test_eight_reported_cases_preserve_structure_and_resolution(self) -> None:
|
||||
cases = (
|
||||
{
|
||||
"text": "把遮阳帘打开",
|
||||
"status": "recognized_executable",
|
||||
"clean": "打开遮阳帘。",
|
||||
"commands": (
|
||||
(
|
||||
"roof.set_shade_motion",
|
||||
"vehicle",
|
||||
"roof.shade_motion",
|
||||
"open",
|
||||
{"motion": "open"},
|
||||
),
|
||||
),
|
||||
},
|
||||
{
|
||||
"text": "打开前大灯",
|
||||
"status": "recognized_unsupported",
|
||||
"clean": "打开前大灯。",
|
||||
"commands": ((None, "vehicle", "front_light.power", "on", {"state": "on"}),),
|
||||
},
|
||||
{
|
||||
"text": "打开主驾座椅通风",
|
||||
"status": "recognized_executable",
|
||||
"clean": "打开主驾座椅通风。",
|
||||
"commands": (
|
||||
(
|
||||
"seat.set_driver_ventilation_native",
|
||||
"driver",
|
||||
"seat.ventilation",
|
||||
"on",
|
||||
{"state": "on", "level": 2},
|
||||
),
|
||||
),
|
||||
},
|
||||
{
|
||||
"text": "把主驾座椅调整为零重力模式",
|
||||
"status": "recognized_unsupported",
|
||||
"clean": "将主驾座椅调整为零重力模式。",
|
||||
"commands": (
|
||||
(None, "driver", "seat.mode", "apply", {"mode_profile_id": "zero_gravity"}),
|
||||
),
|
||||
},
|
||||
{
|
||||
"text": "打开一半车窗",
|
||||
"status": "recognized_executable",
|
||||
"clean": "将车窗打开到50%。",
|
||||
"commands": (
|
||||
(
|
||||
"window.set_position",
|
||||
"front_left",
|
||||
"window.position",
|
||||
"set",
|
||||
{"position_pct": 50},
|
||||
),
|
||||
),
|
||||
},
|
||||
{
|
||||
"text": "打开氛围灯并调整为红色",
|
||||
"status": "incomplete",
|
||||
"clean": "打开氛围灯，然后将氛围灯颜色调整为红色。",
|
||||
"commands": (
|
||||
("ambient_light.set_power", "vehicle", "ambient.power", "on", {"state": "on"}),
|
||||
(
|
||||
"ambient_light.set_illumination",
|
||||
"vehicle",
|
||||
"ambient.color",
|
||||
"set",
|
||||
{"color_hex": "#FF0000"},
|
||||
),
|
||||
),
|
||||
},
|
||||
{
|
||||
"text": "打开主驾座椅加热",
|
||||
"status": "recognized_executable",
|
||||
"clean": "打开主驾座椅加热。",
|
||||
"commands": (
|
||||
(
|
||||
"seat.set_driver_heating_native",
|
||||
"driver",
|
||||
"seat.heating",
|
||||
"on",
|
||||
{"state": "on", "level": 2},
|
||||
),
|
||||
),
|
||||
},
|
||||
{
|
||||
"text": "打开副驾座椅通风",
|
||||
"status": "recognized_executable",
|
||||
"clean": "打开副驾座椅通风。",
|
||||
"commands": (
|
||||
(
|
||||
"seat.set_ventilation",
|
||||
"passenger",
|
||||
"seat.ventilation",
|
||||
"on",
|
||||
{"state": "on"},
|
||||
),
|
||||
),
|
||||
},
|
||||
)
|
||||
for case in cases:
|
||||
with self.subTest(text=case["text"]):
|
||||
result = self.processor.process(case["text"])
|
||||
self.assertEqual(result.raw_text, case["text"])
|
||||
self.assertEqual(result.clean_text, case["clean"])
|
||||
self.assertEqual(result.semantic_status, case["status"])
|
||||
self.assertEqual(len(result.commands), len(case["commands"]))
|
||||
for command, expected in zip(result.commands, case["commands"], strict=True):
|
||||
capability, target, property_name, operation, parameters = expected
|
||||
self.assertEqual(command.capability, capability)
|
||||
self.assertEqual(command.target_scope, target)
|
||||
self.assertEqual(command.property_name, property_name)
|
||||
self.assertEqual(command.operation, operation)
|
||||
self.assertEqual(command.parameters, parameters)
|
||||
if command.resolution_status == "executable":
|
||||
self.assertIsNotNone(command.intent_ir_action)
|
||||
self.assertIsNotNone(command.executor_payload)
|
||||
self.assertEqual(command.intent_ir_action["target"]["zone"], target)
|
||||
self.assertEqual(command.executor_payload["target"], target)
|
||||
self.assertEqual(command.executor_payload["parameters"], parameters)
|
||||
else:
|
||||
self.assertIsNone(command.executor_payload)
|
||||
|
||||
def test_semantic_trace_covers_every_required_boundary(self) -> None:
|
||||
result = self.processor.process("打开主驾座椅通风")
|
||||
self.assertEqual(
|
||||
[item["stage"] for item in result.semantic_trace],
|
||||
[
|
||||
"raw_text",
|
||||
"normalized_text",
|
||||
"clause_segmentation",
|
||||
"semantic_commands",
|
||||
"intent_ir",
|
||||
"capability_resolution",
|
||||
"executor_payload",
|
||||
"clean_text",
|
||||
],
|
||||
)
|
||||
|
||||
def test_explicit_control_slots_cannot_disappear_from_clean_text(self) -> None:
|
||||
cases = (
|
||||
("打开主驾座椅加热", ("主驾", "座椅加热")),
|
||||
("打开副驾座椅通风", ("副驾", "座椅通风")),
|
||||
("打开右后车窗", ("右后", "车窗")),
|
||||
("打开一半车窗", ("50%", "车窗")),
|
||||
("把氛围灯调整为红色", ("氛围灯", "红色")),
|
||||
("把主驾座椅调整为零重力模式", ("主驾", "零重力模式")),
|
||||
)
|
||||
for raw, preserved in cases:
|
||||
with self.subTest(raw=raw):
|
||||
result = self.processor.process(raw)
|
||||
for token in preserved:
|
||||
self.assertIn(token, result.clean_text)
|
||||
|
||||
def test_compound_shared_target_is_preserved_and_gated_atomically(self) -> None:
|
||||
result = self.processor.process("打开氛围灯并调整为红色")
|
||||
self.assertEqual(
|
||||
[command.property_name for command in result.commands],
|
||||
["ambient.power", "ambient.color"],
|
||||
)
|
||||
self.assertEqual(result.commands[1].value, "#FF0000")
|
||||
self.assertEqual(result.commands[1].resolution_status, "incomplete")
|
||||
self.assertIn("brightness_pct", "".join(result.commands[1].validation_issues))
|
||||
self.assertFalse(result.execution_eligible)
|
||||
|
||||
complete = self.processor.process("把氛围灯调整为红色并将亮度设置为50%")
|
||||
self.assertEqual(len(complete.commands), 1)
|
||||
self.assertEqual(
|
||||
complete.commands[0].parameters,
|
||||
{"color_hex": "#FF0000", "brightness_pct": 50},
|
||||
)
|
||||
self.assertEqual(complete.semantic_status, "recognized_executable")
|
||||
self.assertIn("红色", complete.clean_text)
|
||||
self.assertIn("50%", complete.clean_text)
|
||||
|
||||
def test_reported_compound_and_extreme_position_phrases_preserve_all_slots(self) -> None:
|
||||
compound = self.processor.process(
|
||||
"打开阅读灯，并且打开氛围灯，并把氛围灯变成红色"
|
||||
)
|
||||
self.assertEqual(
|
||||
[command.property_name for command in compound.commands],
|
||||
["reading_light.power", "ambient.power", "ambient.color"],
|
||||
)
|
||||
self.assertEqual(compound.commands[-1].parameters, {"color_hex": "#FF0000"})
|
||||
self.assertEqual(compound.semantic_status, "incomplete")
|
||||
self.assertFalse(compound.execution_eligible)
|
||||
for token in ("阅读灯", "氛围灯", "红色"):
|
||||
self.assertIn(token, compound.clean_text)
|
||||
|
||||
legrest = self.processor.process("把副驾腿托向上调整到最大位置")
|
||||
self.assertEqual(legrest.semantic_status, "recognized_executable")
|
||||
self.assertTrue(legrest.execution_eligible)
|
||||
self.assertEqual(legrest.commands[0].target_scope, "passenger")
|
||||
self.assertEqual(legrest.commands[0].capability, "seat.set_legrest_height")
|
||||
self.assertEqual(legrest.commands[0].parameters, {"position_pct": 100})
|
||||
for token in ("副驾", "腿托", "向上", "100%"):
|
||||
self.assertIn(token, legrest.clean_text)
|
||||
|
||||
def test_latest_reported_compounds_preserve_every_mutation(self) -> None:
|
||||
passenger = self.processor.process(
|
||||
"打开副驾座椅通风和加热，，然后将副驾座椅靠背调整到最大位置"
|
||||
)
|
||||
self.assertEqual(
|
||||
[command.property_name for command in passenger.commands],
|
||||
["seat.ventilation", "seat.heating", "seat.backrest_recline"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[command.target_scope for command in passenger.commands],
|
||||
["passenger", "passenger", "passenger"],
|
||||
)
|
||||
self.assertEqual(passenger.commands[-1].parameters, {"position_pct": 100})
|
||||
self.assertTrue(passenger.execution_eligible)
|
||||
for token in ("副驾座椅通风", "副驾座椅加热", "副驾座椅靠背", "100%"):
|
||||
self.assertIn(token, passenger.clean_text)
|
||||
|
||||
driver = self.processor.process(
|
||||
"打开主驾座椅通风和加热，并且把空调调整为24度"
|
||||
)
|
||||
self.assertEqual(
|
||||
[command.property_name for command in driver.commands],
|
||||
["seat.ventilation", "seat.heating", "hvac.temperature"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[command.target_scope for command in driver.commands[:2]],
|
||||
["driver", "driver"],
|
||||
)
|
||||
self.assertEqual(driver.commands[-1].parameters, {"temperature_c": 24})
|
||||
self.assertTrue(driver.execution_eligible)
|
||||
for token in ("主驾座椅通风", "主驾座椅加热", "24度"):
|
||||
self.assertIn(token, driver.clean_text)
|
||||
|
||||
def test_coordinated_seat_properties_cover_order_connectors_and_actions(self) -> None:
|
||||
cases = (
|
||||
("打开主驾座椅通风与加热", "driver", ("seat.ventilation", "seat.heating"), ("on", "on")),
|
||||
("打开副驾座椅加热以及通风", "passenger", ("seat.heating", "seat.ventilation"), ("on", "on")),
|
||||
("关闭主驾座椅通风、加热", "driver", ("seat.ventilation", "seat.heating"), ("off", "off")),
|
||||
("打开主驾座椅通风和关闭加热", "driver", ("seat.ventilation", "seat.heating"), ("on", "off")),
|
||||
)
|
||||
for text, scope, properties, operations in cases:
|
||||
with self.subTest(text=text):
|
||||
result = self.processor.process(text)
|
||||
self.assertEqual(
|
||||
tuple(command.property_name for command in result.commands),
|
||||
properties,
|
||||
)
|
||||
self.assertEqual(
|
||||
tuple(command.operation for command in result.commands),
|
||||
operations,
|
||||
)
|
||||
self.assertEqual(
|
||||
{command.target_scope for command in result.commands},
|
||||
{scope},
|
||||
)
|
||||
self.assertTrue(result.execution_eligible)
|
||||
self.assertTrue(all(command.executor_payload for command in result.commands))
|
||||
|
||||
def test_coordinated_targets_keep_local_actions_scopes_and_properties(self) -> None:
|
||||
cases = (
|
||||
(
|
||||
"关闭空调和打开氛围灯",
|
||||
(("hvac.power", "off", "vehicle"), ("ambient.power", "on", "vehicle")),
|
||||
),
|
||||
(
|
||||
"打开主驾座椅通风和副驾座椅加热",
|
||||
(("seat.ventilation", "on", "driver"), ("seat.heating", "on", "passenger")),
|
||||
),
|
||||
(
|
||||
"打开前排和后排空调",
|
||||
(("hvac.power", "on", "front_row"), ("hvac.power", "on", "rear_row")),
|
||||
),
|
||||
(
|
||||
"打开主驾和副驾车窗",
|
||||
(("window.motion", "open", "front_left"), ("window.motion", "open", "front_right")),
|
||||
),
|
||||
)
|
||||
for text, expected in cases:
|
||||
with self.subTest(text=text):
|
||||
result = self.processor.process(text)
|
||||
self.assertEqual(
|
||||
tuple(
|
||||
(command.property_name, command.operation, command.target_scope)
|
||||
for command in result.commands
|
||||
),
|
||||
expected,
|
||||
)
|
||||
self.assertTrue(result.execution_eligible)
|
||||
for command in result.commands:
|
||||
self.assertEqual(
|
||||
command.executor_payload["target"], command.target_scope
|
||||
)
|
||||
|
||||
def test_ratio_semantics_are_shared_across_continuous_capabilities(self) -> None:
|
||||
cases = (
|
||||
("打开四分之一车窗", "window.position", {"position_pct": 25}, "25%"),
|
||||
("打开四分之三车窗", "window.position", {"position_pct": 75}, "75%"),
|
||||
("打开百分之三十车窗", "window.position", {"position_pct": 30}, "30%"),
|
||||
("打开1/4车窗", "window.position", {"position_pct": 25}, "25%"),
|
||||
("把遮阳帘打开到四分之一", "roof.shade_position", {"position_pct": 25}, "25%"),
|
||||
("把副驾腿托调整到四分之三", "seat.legrest_height", {"position_pct": 75}, "75%"),
|
||||
("把副驾座椅向后调整50%", "seat.longitudinal_position", {"position_pct": 50}, "50%"),
|
||||
)
|
||||
for text, property_name, parameters, display in cases:
|
||||
with self.subTest(text=text):
|
||||
result = self.processor.process(text)
|
||||
self.assertEqual(len(result.commands), 1)
|
||||
self.assertEqual(result.commands[0].property_name, property_name)
|
||||
self.assertEqual(result.commands[0].parameters, parameters)
|
||||
self.assertIn(display, result.clean_text)
|
||||
self.assertTrue(result.execution_eligible)
|
||||
|
||||
def test_invalid_unit_keeps_all_spoken_slots_and_fails_closed(self) -> None:
|
||||
raw = "打开遮阳帘，然后把副驾座椅向后调整50度"
|
||||
result = self.processor.process(raw)
|
||||
self.assertEqual(result.semantic_status, "incomplete")
|
||||
self.assertFalse(result.execution_eligible)
|
||||
self.assertEqual(result.commands[1].target_scope, "passenger")
|
||||
self.assertEqual(result.commands[1].value, 50)
|
||||
self.assertEqual(result.commands[1].parameters, {})
|
||||
self.assertIsNone(result.commands[1].executor_payload)
|
||||
for token in ("遮阳帘", "副驾", "座椅", "向后", "50度"):
|
||||
self.assertIn(token, result.clean_text)
|
||||
self.assertNotIn("将副驾座椅向后调整。", result.clean_text)
|
||||
|
||||
def test_shared_value_and_coordination_grammar_also_feed_one_line_parser(self) -> None:
|
||||
window = merge_native_can_intent({}, "打开四分之一车窗")
|
||||
self.assertEqual(
|
||||
[
|
||||
(action["capability"], action["target"], action["parameters"])
|
||||
for action in window["native_can_actions"]
|
||||
],
|
||||
[("window.set_position", "front_left", {"position_pct": 25})],
|
||||
)
|
||||
|
||||
thermal = merge_native_can_intent({}, "打开主驾座椅通风和加热")
|
||||
self.assertEqual(
|
||||
[action["capability"] for action in thermal["native_can_actions"]],
|
||||
[
|
||||
"seat.set_driver_ventilation_native",
|
||||
"seat.set_driver_heating_native",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
[action["parameters"] for action in thermal["native_can_actions"]],
|
||||
[
|
||||
{"state": "on", "level": 2},
|
||||
{"state": "on", "level": 2},
|
||||
],
|
||||
)
|
||||
|
||||
def test_shared_target_inherits_resolved_driver_scope_and_level_semantics(self) -> None:
|
||||
result = self.processor.process("打开座椅通风并调到三档")
|
||||
self.assertEqual(
|
||||
[command.capability for command in result.commands],
|
||||
[
|
||||
"seat.set_driver_ventilation_native",
|
||||
"seat.set_driver_ventilation_native",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
[command.target_scope for command in result.commands],
|
||||
["driver", "driver"],
|
||||
)
|
||||
self.assertIn("3档", result.clean_text)
|
||||
self.assertEqual(result.semantic_status, "recognized_unsupported")
|
||||
self.assertFalse(result.execution_eligible)
|
||||
|
||||
passenger = self.processor.process("打开副驾座椅通风并调到三档")
|
||||
self.assertEqual(
|
||||
[command.target_scope for command in passenger.commands],
|
||||
["passenger", "passenger"],
|
||||
)
|
||||
self.assertTrue(passenger.execution_eligible)
|
||||
|
||||
graded = self.processor.process("打开主驾座椅加热并调到高档")
|
||||
self.assertEqual(graded.commands[-1].parameters["level"], 1)
|
||||
self.assertIn("高档", graded.clean_text)
|
||||
|
||||
def test_unsupported_semantics_never_create_fake_executor_payloads(self) -> None:
|
||||
for text in ("打开前大灯", "把主驾座椅调整为零重力模式"):
|
||||
with self.subTest(text=text):
|
||||
result = self.processor.process(text)
|
||||
self.assertEqual(result.semantic_status, "recognized_unsupported")
|
||||
self.assertTrue(result.clean_text)
|
||||
self.assertTrue(result.commands)
|
||||
self.assertTrue(
|
||||
all(command.executor_payload is None for command in result.commands)
|
||||
)
|
||||
|
||||
def test_zero_gravity_is_real_passenger_mode_but_not_driver_capability(self) -> None:
|
||||
passenger = self.processor.process("打开零重力模式")
|
||||
self.assertEqual(passenger.semantic_status, "recognized_executable")
|
||||
self.assertEqual(passenger.commands[0].mode_profile_id, "zero_gravity")
|
||||
payload = passenger.commands[0].executor_payload
|
||||
self.assertIsNotNone(payload)
|
||||
self.assertEqual(len(payload["operations"]), 4)
|
||||
self.assertEqual(
|
||||
{operation["target"] for operation in payload["operations"]},
|
||||
{"passenger"},
|
||||
)
|
||||
|
||||
driver = self.processor.process("把主驾座椅调整为零重力模式")
|
||||
self.assertEqual(driver.semantic_status, "recognized_unsupported")
|
||||
self.assertIsNone(driver.commands[0].executor_payload)
|
||||
|
||||
def test_native_executor_can_compile_reported_executable_cases(self) -> None:
|
||||
for text in (
|
||||
"把遮阳帘打开",
|
||||
"打开主驾座椅通风",
|
||||
"打开一半车窗",
|
||||
"打开主驾座椅加热",
|
||||
):
|
||||
with self.subTest(text=text):
|
||||
result = self.processor.process(text)
|
||||
payloads = [command.executor_payload for command in result.commands]
|
||||
self.assertTrue(all(payload is not None for payload in payloads))
|
||||
config = self.native_runtime.build_function_config(
|
||||
payloads[0]["domain"], payloads, "speech-alignment", "test"
|
||||
)
|
||||
plan = self.native_runtime.compile(config)
|
||||
self.assertTrue(plan["spec"]["commands"])
|
||||
|
||||
def test_catalog_driven_resolution_matrix_covers_every_definition(self) -> None:
|
||||
definitions = self.resolver.definitions()
|
||||
self.assertEqual(len(definitions), 43)
|
||||
projections = (
|
||||
"canonical_action",
|
||||
"target_preservation",
|
||||
"parameter_preservation",
|
||||
"intent_ir_projection",
|
||||
"executor_projection",
|
||||
)
|
||||
for definition in definitions:
|
||||
target = definition.target_scopes[0] if definition.target_scopes else "vehicle"
|
||||
parameters = _sample_parameters(definition)
|
||||
expected = (
|
||||
"unsupported"
|
||||
if definition.availability == "catalog_only"
|
||||
or (
|
||||
definition.availability == "native_can"
|
||||
and definition.name not in self.native_runtime.supported_capabilities
|
||||
)
|
||||
else "executable"
|
||||
)
|
||||
for projection in projections:
|
||||
with self.subTest(capability=definition.name, projection=projection):
|
||||
result = self.resolver.resolve(
|
||||
domain=definition.domain,
|
||||
semantic_property=definition.name,
|
||||
target_scope=target,
|
||||
target_device=None,
|
||||
operation="set",
|
||||
parameters=parameters,
|
||||
source_text=f"{projection}:{definition.name}:{target}",
|
||||
preferred_capability=definition.name,
|
||||
)
|
||||
self.assertEqual(result.definition.name, definition.name)
|
||||
self.assertEqual(result.status, expected)
|
||||
if expected == "executable":
|
||||
self.assertEqual(result.executor_payload["target"], target)
|
||||
self.assertEqual(result.executor_payload["parameters"], parameters)
|
||||
self.assertEqual(result.intent_ir_action["target"]["zone"], target)
|
||||
else:
|
||||
self.assertIsNone(result.executor_payload)
|
||||
|
||||
def test_vocabulary_does_not_define_vehicle_capabilities(self) -> None:
|
||||
self.assertNotIn("capability", DeviceTerm.__dataclass_fields__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,196 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from cognitive_agent.application.seat_service import SeatDemoService
|
||||
from cognitive_agent.infrastructure.speech.understanding import (
|
||||
UnderstandingTranscriptProcessor,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("utterance", "capability", "target", "parameters"),
|
||||
(
|
||||
(
|
||||
"打开一半车窗",
|
||||
"window.set_position",
|
||||
{"device": "window", "zone": "front_left"},
|
||||
{"position_pct": 50},
|
||||
),
|
||||
(
|
||||
"把副驾腿托向上调整到最大位置",
|
||||
"seat.set_legrest_height",
|
||||
{"device": "seat", "zone": "front_right", "instance": "passenger"},
|
||||
{"position_pct": 100},
|
||||
),
|
||||
(
|
||||
"把遮阳帘打开",
|
||||
"roof.set_shade_motion",
|
||||
{"device": "roof", "zone": "vehicle"},
|
||||
{"motion": "open"},
|
||||
),
|
||||
(
|
||||
"遮阳帘打开一半",
|
||||
"roof.set_shade_motion",
|
||||
{"device": "roof", "zone": "vehicle"},
|
||||
{"motion": "half"},
|
||||
),
|
||||
(
|
||||
"把遮阳帘打开到四分之一",
|
||||
"roof.set_shade_position",
|
||||
{"device": "roof", "zone": "vehicle"},
|
||||
{"position_pct": 25},
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_server_owned_speech_semantics_bypass_second_language_parse(
|
||||
utterance: str,
|
||||
capability: str,
|
||||
target: dict[str, str],
|
||||
parameters: dict[str, int],
|
||||
) -> None:
|
||||
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
|
||||
understanding = processor.process(utterance).to_dict()
|
||||
understanding["utterance_id"] = "bridge-regression"
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory, patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
|
||||
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
|
||||
"COGNITIVE_AGENT_VEHICLE_MQTT_ENABLED": "false",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
service = SeatDemoService(REPOSITORY_ROOT)
|
||||
service.artifact_root = Path(directory) / "artifacts"
|
||||
with patch.object(
|
||||
service,
|
||||
"_interpret",
|
||||
side_effect=AssertionError("cleanText must not be parsed a second time"),
|
||||
):
|
||||
result = service.process(
|
||||
understanding["clean_text"],
|
||||
provider="local",
|
||||
execution_mode="dry_run",
|
||||
_speech_understanding=understanding,
|
||||
)
|
||||
|
||||
assert result["status"] == "succeeded"
|
||||
assert result["provider_used"] == "speech_semantic"
|
||||
assert result["speech_understanding"]["raw_text"] == utterance
|
||||
actions = result["intent_ir"]["spec"]["actions"]
|
||||
assert len(actions) == 1
|
||||
assert actions[0]["capability"] == capability
|
||||
assert actions[0]["target"] == target
|
||||
assert actions[0]["parameters"] == parameters
|
||||
assert actions[0]["provenance"]["parser"] == "speech_semantic"
|
||||
|
||||
|
||||
def test_non_executable_speech_cannot_enter_orchestration_bridge() -> None:
|
||||
processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
|
||||
understanding = processor.process("打开前大灯").to_dict()
|
||||
understanding["utterance_id"] = "bridge-unsupported"
|
||||
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
|
||||
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
service = SeatDemoService(REPOSITORY_ROOT)
|
||||
context = service._context_for_vehicle(
|
||||
service.conversations.snapshot("conv-speech-bridge"),
|
||||
service.vehicle_profiles.get(None),
|
||||
)
|
||||
with pytest.raises(ValueError, match="not execution eligible"):
|
||||
service._adapt_speech_understanding(
|
||||
understanding,
|
||||
context,
|
||||
service.vehicle_profiles.get(None),
|
||||
)
|
||||
|
||||
|
||||
def test_every_registered_native_runtime_capability_keeps_native_bridge_binding() -> None:
|
||||
def valid_value(spec: dict[str, object]) -> object:
|
||||
allowed = spec.get("allowed_values")
|
||||
if isinstance(allowed, list) and allowed:
|
||||
return allowed[0]
|
||||
value_type = str(spec.get("type", "string"))
|
||||
if value_type in {"integer", "number"}:
|
||||
return spec.get("minimum", 0)
|
||||
if value_type == "boolean":
|
||||
return True
|
||||
if value_type == "object":
|
||||
return {}
|
||||
if value_type == "array":
|
||||
return []
|
||||
return "test"
|
||||
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"COGNITIVE_AGENT_HARDWARE_EXECUTION_ENABLED": "false",
|
||||
"COGNITIVE_AGENT_VEHICLE_CAN_ENABLED": "false",
|
||||
},
|
||||
clear=False,
|
||||
):
|
||||
service = SeatDemoService(REPOSITORY_ROOT)
|
||||
context = service._context_for_vehicle(
|
||||
service.conversations.snapshot("conv-native-speech-matrix"),
|
||||
service.vehicle_profiles.get(None),
|
||||
)
|
||||
registered = {
|
||||
definition.name: definition
|
||||
for definition in service.capability_registry.definitions()
|
||||
if definition.name in service.native_can_runtime.supported_capabilities
|
||||
}
|
||||
assert registered
|
||||
for capability, definition in registered.items():
|
||||
parameters = {
|
||||
str(spec["name"]): valid_value(spec)
|
||||
for spec in definition.parameters
|
||||
if spec.get("required")
|
||||
}
|
||||
target = definition.target_scopes[0]
|
||||
understanding = {
|
||||
"raw_text": f"native bridge matrix: {capability}",
|
||||
"execution_eligible": True,
|
||||
"commands": [
|
||||
{
|
||||
"status": "active",
|
||||
"negated": False,
|
||||
"resolution_status": "executable",
|
||||
"validation_issues": [],
|
||||
"confidence": 1.0,
|
||||
"source_text": capability,
|
||||
"executor_payload": {
|
||||
"domain": definition.domain,
|
||||
"capability": capability,
|
||||
"target": target,
|
||||
"operation": "set",
|
||||
"parameters": parameters,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
parsed = service._adapt_speech_understanding(
|
||||
understanding,
|
||||
context,
|
||||
service.vehicle_profiles.get(None),
|
||||
)
|
||||
assert parsed["deferred_requests"] == [], capability
|
||||
assert parsed["actions"] == [], capability
|
||||
assert parsed["domain_actions"] == [], capability
|
||||
assert [
|
||||
action["capability"] for action in parsed["native_can_actions"]
|
||||
] == [capability]
|
||||
@@ -0,0 +1,190 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import tempfile
|
||||
import unittest
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from cognitive_agent.infrastructure.speech.sensevoice import (
|
||||
SpeechRecognitionError,
|
||||
SpeechTranscriber,
|
||||
)
|
||||
from cognitive_agent.infrastructure.speech.understanding import (
|
||||
UnderstandingTranscriptProcessor,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def pcm16_wav(*, frequency: float = 220, seconds: float = 0.8) -> bytes:
|
||||
frames = bytearray()
|
||||
for index in range(round(16_000 * seconds)):
|
||||
sample = round(math.sin(index * 2 * math.pi * frequency / 16_000) * 12_000)
|
||||
frames.extend(int(sample).to_bytes(2, "little", signed=True))
|
||||
target = io.BytesIO()
|
||||
with wave.open(target, "wb") as output:
|
||||
output.setnchannels(1)
|
||||
output.setsampwidth(2)
|
||||
output.setframerate(16_000)
|
||||
output.writeframes(frames)
|
||||
return target.getvalue()
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.result = SimpleNamespace(
|
||||
text=f"<|zh|><|NEUTRAL|><|Speech|>{text}",
|
||||
lang="<|zh|>",
|
||||
emotion="<|NEUTRAL|>",
|
||||
event="<|Speech|>",
|
||||
)
|
||||
|
||||
def accept_waveform(self, sample_rate: int, samples: object) -> None:
|
||||
self.sample_rate = sample_rate
|
||||
self.samples = samples
|
||||
|
||||
|
||||
class _FakeRecognizer:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
self.decode_count = 0
|
||||
|
||||
def create_stream(self) -> _FakeStream:
|
||||
return _FakeStream(self.text)
|
||||
|
||||
def decode_stream(self, stream: _FakeStream) -> None:
|
||||
self.decode_count += 1
|
||||
|
||||
|
||||
class UnderstandingRegressionTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.processor = UnderstandingTranscriptProcessor(REPOSITORY_ROOT)
|
||||
fixture = REPOSITORY_ROOT / "tests" / "fixtures" / "speech_text_regression.json"
|
||||
cls.cases = json.loads(fixture.read_text(encoding="utf-8"))
|
||||
|
||||
def test_text_regression_suite(self) -> None:
|
||||
for case in self.cases:
|
||||
with self.subTest(case=case["name"]):
|
||||
result = self.processor.process(case["raw"])
|
||||
self.assertEqual(result.raw_text, case["raw"])
|
||||
self.assertEqual(result.clean_text, case["clean"])
|
||||
self.assertEqual(
|
||||
[command.capability for command in result.commands],
|
||||
case["capabilities"],
|
||||
)
|
||||
self.assertEqual(len(result.cancelled_fragments), case["cancelled"])
|
||||
self.assertEqual(result.uncertain, case["uncertain"])
|
||||
self.assertEqual(
|
||||
result.execution_eligible,
|
||||
case.get(
|
||||
"execution_eligible",
|
||||
bool(result.commands) and not result.uncertain,
|
||||
),
|
||||
)
|
||||
if preserved := case.get("preserved_filler"):
|
||||
self.assertNotIn(preserved, result.removed_fillers)
|
||||
|
||||
def test_cancelled_commands_never_appear_as_active(self) -> None:
|
||||
result = self.processor.process(
|
||||
"打开空调，不对，把座椅靠背向后调24度。"
|
||||
)
|
||||
self.assertEqual([command.domain for command in result.commands], ["seat"])
|
||||
cancelled = result.cancelled_fragments[0].commands
|
||||
self.assertEqual([command.domain for command in cancelled], ["climate"])
|
||||
self.assertTrue(all(command.status == "cancelled" for command in cancelled))
|
||||
|
||||
def test_invalid_seat_absolute_angle_fails_closed(self) -> None:
|
||||
result = self.processor.process("把座椅靠背调整到24度。")
|
||||
self.assertTrue(result.uncertain)
|
||||
self.assertFalse(result.execution_eligible)
|
||||
self.assertIn("低于工程下限95", "".join(result.reasons))
|
||||
|
||||
def test_regular_multi_command_is_not_treated_as_correction(self) -> None:
|
||||
result = self.processor.process("打开空调，然后打开氛围灯。")
|
||||
self.assertEqual(len(result.commands), 2)
|
||||
self.assertFalse(result.cancelled_fragments)
|
||||
|
||||
|
||||
class SpeechTranscriberTests(unittest.TestCase):
|
||||
def make_transcriber(self, directory: str, text: str) -> tuple[SpeechTranscriber, _FakeRecognizer]:
|
||||
transcriber = SpeechTranscriber(Path(directory))
|
||||
recognizer = _FakeRecognizer(text)
|
||||
transcriber._recognizer = recognizer
|
||||
return transcriber, recognizer
|
||||
|
||||
def test_transcriber_preserves_raw_and_returns_structured_result(self) -> None:
|
||||
raw = "打开空调，不对，把座椅靠背向后调24度。"
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
transcriber, _ = self.make_transcriber(directory, raw)
|
||||
result = transcriber.transcribe_wav(
|
||||
pcm16_wav(), utterance_id="unit-structured"
|
||||
)
|
||||
self.assertEqual(result["raw_text"], raw)
|
||||
self.assertEqual(result["clean_text"], "将座椅靠背向后调整24度。")
|
||||
self.assertEqual(result["commands"][0]["domain"], "seat")
|
||||
self.assertEqual(result["cancelled_fragments"][0]["commands"][0]["domain"], "climate")
|
||||
self.assertIn("timings", result)
|
||||
self.assertEqual(result["segments"][0]["state"], "final")
|
||||
|
||||
def test_filler_only_is_evidence_but_not_a_control_error(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
transcriber, _ = self.make_transcriber(directory, "嗯嗯。")
|
||||
result = transcriber.transcribe_wav(
|
||||
pcm16_wav(), utterance_id="unit-filler"
|
||||
)
|
||||
self.assertEqual(result["raw_text"], "嗯嗯。")
|
||||
self.assertEqual(result["clean_text"], "")
|
||||
self.assertEqual(result["commands"], [])
|
||||
self.assertFalse(result["execution_eligible"])
|
||||
|
||||
def test_same_utterance_and_audio_is_decoded_once(self) -> None:
|
||||
audio = pcm16_wav()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
transcriber, recognizer = self.make_transcriber(directory, "打开空调。")
|
||||
first = transcriber.transcribe_wav(audio, utterance_id="unit-idempotent")
|
||||
second = transcriber.transcribe_wav(audio, utterance_id="unit-idempotent")
|
||||
self.assertEqual(recognizer.decode_count, 1)
|
||||
self.assertFalse(first["idempotent_replay"])
|
||||
self.assertTrue(second["idempotent_replay"])
|
||||
|
||||
def test_cached_result_is_server_owned_and_returned_as_a_copy(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
transcriber, _ = self.make_transcriber(directory, "打开空调。")
|
||||
transcriber.transcribe_wav(pcm16_wav(), utterance_id="unit-cached")
|
||||
first = transcriber.cached_result("unit-cached")
|
||||
first["clean_text"] = "tampered"
|
||||
second = transcriber.cached_result("unit-cached")
|
||||
self.assertNotEqual(second["clean_text"], "tampered")
|
||||
|
||||
def test_missing_cached_result_is_rejected(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
transcriber, _ = self.make_transcriber(directory, "打开空调。")
|
||||
with self.assertRaises(SpeechRecognitionError) as caught:
|
||||
transcriber.cached_result("unit-expired")
|
||||
self.assertEqual(caught.exception.code, "speech_result_expired")
|
||||
|
||||
def test_same_utterance_with_different_audio_is_rejected(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
transcriber, _ = self.make_transcriber(directory, "打开空调。")
|
||||
transcriber.transcribe_wav(pcm16_wav(), utterance_id="unit-conflict")
|
||||
with self.assertRaises(SpeechRecognitionError) as caught:
|
||||
transcriber.transcribe_wav(
|
||||
pcm16_wav(frequency=330), utterance_id="unit-conflict"
|
||||
)
|
||||
self.assertEqual(caught.exception.code, "utterance_id_conflict")
|
||||
|
||||
def test_invalid_utterance_id_is_rejected(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
transcriber, _ = self.make_transcriber(directory, "打开空调。")
|
||||
with self.assertRaises(SpeechRecognitionError) as caught:
|
||||
transcriber.transcribe_wav(pcm16_wav(), utterance_id="bad id/路径")
|
||||
self.assertEqual(caught.exception.code, "invalid_utterance_id")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,110 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from cognitive_agent.infrastructure.speech.assembler import TranscriptAssembler
|
||||
from cognitive_agent.infrastructure.speech.contracts import SpeechSegmentResult
|
||||
|
||||
|
||||
def segment(
|
||||
segment_id: str,
|
||||
sequence: int,
|
||||
text: str,
|
||||
*,
|
||||
final: bool,
|
||||
start_ms: int | None = None,
|
||||
end_ms: int | None = None,
|
||||
overlap_ms: int = 0,
|
||||
) -> SpeechSegmentResult:
|
||||
start = sequence * 1000 if start_ms is None else start_ms
|
||||
end = start + 1200 if end_ms is None else end_ms
|
||||
return SpeechSegmentResult(
|
||||
utterance_id="utterance-1",
|
||||
segment_id=segment_id,
|
||||
sequence=sequence,
|
||||
start_ms=start,
|
||||
end_ms=end,
|
||||
text=text,
|
||||
is_final=final,
|
||||
audio_overlap_ms=overlap_ms,
|
||||
)
|
||||
|
||||
|
||||
class TranscriptAssemblerTests(unittest.TestCase):
|
||||
def test_partial_replaces_partial_and_final_commits(self) -> None:
|
||||
assembler = TranscriptAssembler("utterance-1")
|
||||
assembler.update(segment("s0", 0, "打开空", final=False))
|
||||
self.assertEqual(assembler.snapshot().text, "打开空")
|
||||
assembler.update(segment("s0", 0, "打开空调", final=False))
|
||||
self.assertEqual(assembler.snapshot().text, "打开空调")
|
||||
final = assembler.update(segment("s0", 0, "打开空调。", final=True))
|
||||
self.assertEqual(final.text, "打开空调。")
|
||||
self.assertFalse(final.partial_segments)
|
||||
|
||||
def test_duplicate_final_callback_is_idempotent(self) -> None:
|
||||
assembler = TranscriptAssembler("utterance-1")
|
||||
callback = segment("s0", 0, "打开空调。", final=True)
|
||||
assembler.update(callback)
|
||||
result = assembler.update(callback)
|
||||
self.assertEqual(result.text, "打开空调。")
|
||||
self.assertEqual(result.duplicate_callbacks, 1)
|
||||
|
||||
def test_final_cannot_be_mutated(self) -> None:
|
||||
assembler = TranscriptAssembler("utterance-1")
|
||||
assembler.update(segment("s0", 0, "打开空调。", final=True))
|
||||
result = assembler.update(segment("s0", 0, "关闭空调。", final=True))
|
||||
self.assertTrue(result.uncertain)
|
||||
self.assertEqual(result.text, "打开空调。")
|
||||
|
||||
def test_out_of_order_callbacks_are_sorted(self) -> None:
|
||||
assembler = TranscriptAssembler("utterance-1")
|
||||
assembler.update(segment("s1", 1, "打开氛围灯。", final=True))
|
||||
result = assembler.update(segment("s0", 0, "打开空调。", final=True))
|
||||
self.assertEqual(result.text, "打开空调。打开氛围灯。")
|
||||
self.assertEqual(result.out_of_order_callbacks, 1)
|
||||
|
||||
def test_audio_supported_boundary_overlap_is_removed(self) -> None:
|
||||
assembler = TranscriptAssembler("utterance-1")
|
||||
assembler.update(
|
||||
segment("s0", 0, "把座椅靠背", final=True, start_ms=0, end_ms=1500)
|
||||
)
|
||||
result = assembler.update(
|
||||
segment(
|
||||
"s1",
|
||||
1,
|
||||
"靠背向后调24度",
|
||||
final=True,
|
||||
start_ms=1400,
|
||||
end_ms=2600,
|
||||
overlap_ms=100,
|
||||
)
|
||||
)
|
||||
self.assertEqual(result.text, "把座椅靠背，向后调24度")
|
||||
self.assertEqual(result.overlaps_removed, ("靠背",))
|
||||
|
||||
def test_long_text_overlap_is_removed_without_audio_overlap(self) -> None:
|
||||
assembler = TranscriptAssembler("utterance-1")
|
||||
assembler.update(segment("s0", 0, "请打开空调温度", final=True))
|
||||
result = assembler.update(segment("s1", 1, "空调温度调到24度", final=True))
|
||||
self.assertIn("空调温度", result.overlaps_removed)
|
||||
|
||||
def test_repeated_action_in_distinct_commands_is_preserved(self) -> None:
|
||||
assembler = TranscriptAssembler("utterance-1")
|
||||
assembler.update(segment("s0", 0, "打开前排空调", final=True))
|
||||
result = assembler.update(segment("s1", 1, "打开后排空调", final=True))
|
||||
self.assertEqual(result.text, "打开前排空调，打开后排空调")
|
||||
self.assertFalse(result.overlaps_removed)
|
||||
|
||||
def test_finish_rejects_late_callbacks_and_reset_starts_new_utterance(self) -> None:
|
||||
assembler = TranscriptAssembler("utterance-1")
|
||||
assembler.update(segment("s0", 0, "打开空调", final=True))
|
||||
assembler.finish()
|
||||
late = assembler.update(segment("s1", 1, "关闭空调", final=True))
|
||||
self.assertEqual(late.text, "打开空调")
|
||||
assembler.reset("utterance-2")
|
||||
self.assertEqual(assembler.snapshot().utterance_id, "utterance-2")
|
||||
self.assertEqual(assembler.snapshot().text, "")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user