vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(n7web): refine spatial UI and NFC-gated extension routing

Browse Source
This commit is contained in:
Miayui
2026-09-16 16:14:10 +08:00
parent 0e4328f548
commit 75ce391312
16 changed files with 925 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cognitive_agent/application/self_extension.py
+180 -59
View File
@@ -380,6 +380,86 @@ class InfraredSlotCatalog:
capabilities.append(sanitized)
return capabilities
def list_routing_capabilities(self) -> list[dict[str, Any]]:
"""Return every named IR slot, annotated with execution authorization.
The slot catalog is the source of truth for natural-language discovery,
while ``infrared_capabilities`` remains the source of truth for physical
execution authorization. Keeping those concerns separate lets a newly
learned device become parseable immediately without allowing an
unbound NFC device to emit infrared commands.
"""
slots = self.list_slots()
authorized = self.list_capabilities()
authorized_by_identity: dict[tuple[str, int], dict[str, Any]] = {}
for item in authorized:
try:
identity = (
str(item.get("device_id") or "").strip(),
validate_slot(item.get("slot")),
)
except GatewayError:
continue
if identity[0]:
authorized_by_identity.setdefault(identity, item)
routing: list[dict[str, Any]] = []
for slot_record in slots:
device_id = str(slot_record.get("device_id") or "").strip()
object_names = _safe_peripheral_identity_names(
slot_record.get("object_names")
)
action_names = [
str(value or "").strip()
for value in slot_record.get("action_names", [])
if str(value or "").strip()
][:24]
if (
not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,63}", device_id)
or not object_names
or not action_names
):
continue
slot = validate_slot(slot_record.get("slot"))
persisted = authorized_by_identity.get((device_id, slot))
item = dict(persisted or {})
card_uid = _normalized_card_uid(item.get("card_uid"))
enabled = item.get("enabled", True) is not False
execution_authorized = bool(card_uid and enabled)
item.update(
{
"id": str(item.get("id") or f"ir-slot:{device_id}:{slot}"),
"status": "learned",
"enabled": enabled if card_uid else False,
"parseable": True,
"execution_authorized": execution_authorized,
"authorization_state": (
"authorized"
if execution_authorized
else "disabled"
if card_uid
else "nfc_binding_required"
),
"device_id": device_id,
"device_name": object_names[0],
"device_aliases": object_names,
"object_names": object_names,
"device_type": str(item.get("device_type") or "other")[:48],
"card_uid": card_uid or None,
"action": action_names[0],
"aliases": action_names,
"action_names": action_names,
"risk": str(item.get("risk") or "low"),
"slot": slot,
"transport": str(item.get("transport") or "gscu_tcp")[:32],
"routing_source": "infrared_slot_catalog",
}
)
routing.append(item)
return routing
def sync_capabilities(
self,
catalog: object,
@@ -1244,6 +1324,28 @@ def resolve_active_peripheral_command(
catalog,
nfc_snapshot,
require_enabled=True,
require_binding=True,
)
def resolve_parseable_peripheral_command(
utterance: str,
catalog: list[dict[str, Any]],
nfc_snapshot: dict[str, Any],
) -> dict[str, Any] | None:
"""Resolve a named learned IR action without granting execution rights.
This is intentionally a parsing-only operation. Callers must inspect the
returned capability's persisted ``card_uid`` and ``enabled`` state before
allowing any physical infrared emit.
"""
return _resolve_bound_peripheral_command(
utterance,
catalog,
nfc_snapshot,
require_enabled=False,
require_binding=False,
)
@@ -1253,6 +1355,7 @@ def _resolve_bound_peripheral_command(
nfc_snapshot: dict[str, Any],
*,
require_enabled: bool,
require_binding: bool = True,
) -> dict[str, Any] | None:
"""Resolve a persistently card-bound command, including disabled entries.
@@ -1279,7 +1382,7 @@ def _resolve_bound_peripheral_command(
):
continue
bound_uid = _normalized_card_uid(raw_item.get("card_uid"))
if not bound_uid:
if require_binding and not bound_uid:
continue
device_score = _device_match_score(compact_utterance, raw_item)
if not device_score:
@@ -1310,6 +1413,17 @@ def _resolve_bound_peripheral_command(
resolved = dict(matches[0][2])
resolved["slot"] = validate_slot(resolved.get("slot"))
resolved["card_uid"] = _normalized_card_uid(resolved.get("card_uid"))
resolved["parseable"] = True
resolved["execution_authorized"] = bool(
resolved["card_uid"] and resolved.get("enabled", True) is not False
)
resolved["authorization_state"] = (
"authorized"
if resolved["execution_authorized"]
else "disabled"
if resolved["card_uid"]
else "nfc_binding_required"
)
return resolved
@@ -1326,10 +1440,12 @@ def partition_active_peripheral_intent(
) -> dict[str, Any]:
"""Claim only explicit learned-peripheral clauses and preserve CAN text.
The function is deliberately conservative: a clause must contain both a
persistently NFC-bound device identity and one learned action. When no peripheral
clause is claimed, ``residual_utterance`` is byte-for-byte the original
trimmed input so the existing CAN parser observes exactly its old input.
The function is deliberately conservative: a clause must contain both an
explicit learned device identity and one learned action. A matching but
unbound device is returned as an authorization denial, never as residual
vehicle text. When no peripheral clause is claimed, ``residual_utterance``
is byte-for-byte the original trimmed input so the existing CAN parser
observes exactly its old input.
"""
original = str(utterance or "").strip()
@@ -1342,12 +1458,16 @@ def partition_active_peripheral_intent(
"denied_peripheral_actions": [],
}
bound_catalog = [
routing_catalog = [
item
for item in catalog[:MAX_CATALOG_ITEMS]
if isinstance(item, dict)
and item.get("status") == "learned"
and bool(_normalized_card_uid(item.get("card_uid")))
]
bound_catalog = [
item
for item in routing_catalog
if bool(_normalized_card_uid(item.get("card_uid")))
]
active_catalog = [
item
@@ -1355,8 +1475,8 @@ def partition_active_peripheral_intent(
if item.get("enabled", True) is not False
]
compact_original = _compact_match_text(original)
if not bound_catalog or not any(
_device_match_score(compact_original, item) for item in bound_catalog
if not routing_catalog or not any(
_device_match_score(compact_original, item) for item in routing_catalog
):
return {
"peripheral_actions": [],
@@ -1380,36 +1500,30 @@ def partition_active_peripheral_intent(
contextual_device_name = ""
for index, clause in enumerate(clauses):
compact_clause = _compact_match_text(clause)
explicit_bound_candidates = [
explicit_routing_candidates = [
item
for item in bound_catalog
for item in routing_catalog
if _device_match_score(compact_clause, item)
]
explicit_candidates = [
item
for item in explicit_bound_candidates
if item.get("enabled", True) is not False
]
if explicit_bound_candidates and _has_unshadowed_vehicle_device_phrase(
if explicit_routing_candidates and _has_unshadowed_vehicle_device_phrase(
compact_clause,
explicit_bound_candidates,
explicit_routing_candidates,
):
residual.append(clause)
contextual_device_id = None
contextual_device_name = ""
continue
clause_candidates = explicit_candidates
bound_clause_candidates = explicit_bound_candidates
clause_candidates = explicit_routing_candidates
resolution_text = clause
if explicit_bound_candidates:
if explicit_routing_candidates:
candidate_device_ids = {
str(item.get("device_id") or "")
for item in explicit_bound_candidates
for item in explicit_routing_candidates
}
if len(candidate_device_ids) == 1:
contextual_device_id = next(iter(candidate_device_ids))
identity_names: list[str] = []
for item in explicit_bound_candidates:
for item in explicit_routing_candidates:
for field in ("object_names", "device_aliases"):
values = item.get(field, [])
if isinstance(values, str):
@@ -1454,56 +1568,51 @@ def partition_active_peripheral_intent(
):
clause_candidates = [
item
for item in active_catalog
if str(item.get("device_id") or "") == contextual_device_id
]
bound_clause_candidates = [
item
for item in bound_catalog
for item in routing_catalog
if str(item.get("device_id") or "") == contextual_device_id
]
resolution_text = f"{contextual_device_name}，{clause}"
if not clause_candidates and not bound_clause_candidates:
if not clause_candidates:
residual.append(clause)
continue
active_resolution_catalog = (
clause_candidates if not explicit_candidates else active_catalog
routing_resolution_catalog = (
clause_candidates if not explicit_routing_candidates else routing_catalog
)
resolved = resolve_active_peripheral_command(
resolution_text, active_resolution_catalog, nfc_snapshot
resolved = resolve_parseable_peripheral_command(
resolution_text, routing_resolution_catalog, nfc_snapshot
)
if resolved is None:
bound_resolution_catalog = (
bound_clause_candidates
if not explicit_bound_candidates
else bound_catalog
)
bound_match = _resolve_bound_peripheral_command(
resolution_text,
bound_resolution_catalog,
nfc_snapshot,
require_enabled=False,
)
if (
bound_match is not None
and bound_match.get("enabled", True) is False
):
denied.append(
{
"index": index,
"clause": clause,
"capability": bound_match,
"reason": "nfc_command_disabled",
}
)
continue
# Explicit third-party device text must never fall through into
# the vehicle CAN model as if it were a cabin actuator request.
unresolved.append(clause)
authorized_device_ids = {
str(item.get("device_id") or "") for item in clause_candidates
}
unresolved_candidates.update(
(str(item.get("id")), item)
for item in clause_candidates
for item in active_catalog
if item.get("id")
and str(item.get("device_id") or "") in authorized_device_ids
)
continue
if not _normalized_card_uid(resolved.get("card_uid")):
denied.append(
{
"index": index,
"clause": clause,
"capability": resolved,
"reason": "nfc_binding_required",
}
)
continue
if resolved.get("enabled", True) is False:
denied.append(
{
"index": index,
"clause": clause,
"capability": resolved,
"reason": "nfc_command_disabled",
}
)
continue
actions.append(
@@ -2275,6 +2384,11 @@ class SelfExtensionService:
return self.infrared_catalog.list_capabilities()
def list_peripheral_routing_capabilities(self) -> list[dict[str, Any]]:
"""Return parseable IR actions with explicit authorization metadata."""
return self.infrared_catalog.list_routing_capabilities()
def sync_peripheral_catalog(self, catalog: object) -> dict[str, Any]:
"""Persist the complete learned capability catalog from the NFC UI."""
@@ -2441,6 +2555,13 @@ class SelfExtensionService:
) -> dict[str, Any]:
"""Execute a persistently NFC-bound match through the main request API."""
if (
capability.get("enabled", True) is False
or not _normalized_card_uid(capability.get("card_uid"))
):
raise ExtensionIntegrationError(
"红外能力尚未完成持久 NFC 授权，已拒绝物理下发。"
)
if not _peripheral_execution_identity_authorized(utterance, capability):
raise ExtensionIntegrationError(
"外设名称未在原始指令中被明确提及，已拒绝红外执行。"
src/cognitive_agent/server.py
+211 -21
View File
@@ -733,7 +733,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
).strip()
peripheral_routing = self._speech_peripheral_routing(raw_text)
peripheral_transcript = bool(
peripheral_routing.get("execution_eligible") and raw_text
peripheral_routing.get("routing_eligible") and raw_text
)
expected_text = raw_text if peripheral_transcript else clean_text
if utterance != expected_text:
@@ -933,6 +933,18 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
HTTPStatus.INTERNAL_SERVER_ERROR,
)
def _peripheral_routing_catalog(self) -> list[dict[str, object]]:
"""Read the parse catalog while remaining compatible with test doubles."""
provider = getattr(
self.extension,
"list_peripheral_routing_capabilities",
None,
)
if callable(provider):
return list(provider())
return list(self.extension.list_peripheral_capabilities())
def _execute_active_peripheral_request(
self,
utterance: str,
@@ -1044,14 +1056,16 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
Vehicle speech keeps using its cached structured semantic payload. A
learned peripheral cannot be represented by the static vehicle
capability catalog, so it may instead reuse the exact server-owned ASR
transcript only after the durable NFC binding and learned action resolve.
transcript once the learned device and action resolve. Execution
authorization is checked separately so an unbound device can be shown
as parsed-and-blocked without ever reaching the infrared transport.
"""
text = str(utterance or "").strip()
if not text:
return {"execution_eligible": False, "action_count": 0}
try:
catalog = self.extension.list_peripheral_capabilities()
catalog = self._peripheral_routing_catalog()
nfc_snapshot = self.nfc.snapshot()
partition = partition_active_peripheral_intent(
text,
@@ -1080,9 +1094,12 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
action_count = len(entries) or (1 if direct is not None else 0)
if plan_catalog:
action_count = max(action_count, 1)
routing_eligible = bool(action_count or unresolved or denied)
return {
"execution_eligible": bool(action_count and not unresolved and not denied),
"routing_eligible": routing_eligible,
"action_count": action_count,
"denied_action_count": len(denied),
"has_ordered_plan": bool(plan_catalog),
}
@@ -1386,7 +1403,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
result: dict[str, object],
entries: list[dict[str, object]],
) -> dict[str, object]:
"""Expose disabled bound commands without discarding allowed work."""
"""Expose parsed-but-unauthorized IR commands without dispatching them."""
if not entries:
return result
@@ -1410,25 +1427,169 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
result["denied_peripheral_actions"] = denied_actions
result["authorization_filtered"] = True
blocked_operations: list[dict[str, object]] = []
blocked_commands: list[dict[str, object]] = []
blocked_results: list[dict[str, object]] = []
for position, item in enumerate(denied_actions, start=1):
reason = str(item.get("reason") or "nfc_command_disabled")
device_name = str(
item.get("device_name") or item.get("device_id") or "未命名外设"
)
action_name = str(item.get("action") or "未命名动作")
operation_ref = (
f"{item.get('capability_id') or 'infrared'}-authorization-{position}"
)
message = (
f"已识别“{device_name} · {action_name}”，但设备尚未完成 NFC "
"绑定授权，未下发。"
if reason == "nfc_binding_required"
else f"已识别“{device_name} · {action_name}”，但该 NFC 能力已被禁用，未下发。"
)
blocked_operations.append(
{
"id": operation_ref,
"capability": "infrared.emit",
"target": {
"device": item.get("device_id"),
"name": item.get("device_name"),
},
"parameters": {
"action": item.get("action"),
"slot": item.get("slot"),
},
"parse_source": "local_rule",
"authorization": {
"authorized": False,
"reason": reason,
},
}
)
blocked_commands.append(
{
"operation_ref": operation_ref,
"bus": "GSCU_TCP",
"message": "infrared.emit",
"signals": {
"slot": item.get("slot"),
"authorization_status": "blocked",
"authorization_reason": reason,
},
}
)
blocked_results.append(
{
"ref": operation_ref,
"domain": "self_extension",
"status": "blocked",
"message": message,
"telemetry": {
"transport": "gscu_tcp",
"slot": item.get("slot"),
"delivery_status": "not_submitted",
"transport_submitted": False,
"gateway_acknowledged": False,
"physical_effect_confirmed": False,
"verification_status": "failed_or_blocked",
"authorization_reason": reason,
},
}
)
function_configs = deepcopy(result.get("function_configs") or {})
command_plans = deepcopy(result.get("command_plans") or {})
config_key = "self_extension_blocked"
suffix = 2
while config_key in function_configs or config_key in command_plans:
config_key = f"self_extension_blocked_{suffix}"
suffix += 1
blocked_config = {
"apiVersion": "cognitive-agent.io/v1alpha1",
"kind": "FunctionConfig",
"metadata": {
"name": config_key,
"labels": {
"execution_scope": "nfc_authorization_required",
},
},
"spec": {
"domain": "self_extension",
"operations": blocked_operations,
},
}
blocked_plan = {
"apiVersion": "cognitive-agent.io/v1alpha1",
"kind": "CommandPlan",
"metadata": {
"name": f"{config_key}-gscu",
"labels": {"authorization_status": "blocked"},
},
"spec": {
"domain": "self_extension",
"platform": "gscu1b",
"commands": blocked_commands,
},
}
function_configs[config_key] = blocked_config
command_plans[config_key] = blocked_plan
result["function_configs"] = function_configs
result["command_plans"] = command_plans
if not isinstance(result.get("function_config"), dict):
result["function_config"] = blocked_config
if not isinstance(result.get("command_plan"), dict):
result["command_plan"] = blocked_plan
result["operation_results"] = [
*deepcopy(list(result.get("operation_results") or [])),
*blocked_results,
]
result["trace_events"] = [
*deepcopy(list(result.get("trace_events") or [])),
{
"stage": "nfc_peripheral_authorization",
"status": "blocked",
"summary": "红外指令已由本地规则解析，但未通过 NFC 执行授权。",
"duration_ms": 0,
},
]
summaries = list(result.get("decision_summary") or [])
labels = "、".join(
str(item.get("clause") or item.get("action") or "未命名动作")
binding_required = [
item
for item in denied_actions
)
summaries.append(
f"持久绑定中有 {len(denied_actions)} 条红外能力未启用，已过滤：{labels}。"
)
if item.get("reason") == "nfc_binding_required"
]
disabled = [
item
for item in denied_actions
if item.get("reason") != "nfc_binding_required"
]
if binding_required:
labels = "、".join(
str(item.get("clause") or item.get("action") or "未命名动作")
for item in binding_required
)
summaries.append(
f"已解析 {len(binding_required)} 条红外控制指令，但设备尚未完成 NFC "
f"绑定授权，未下发：{labels}。"
)
if disabled:
labels = "、".join(
str(item.get("clause") or item.get("action") or "未命名动作")
for item in disabled
)
summaries.append(
f"持久绑定中有 {len(disabled)} 条红外能力未启用，已过滤：{labels}。"
)
result["decision_summary"] = summaries
prior_status = str(result.get("status") or "failed")
if prior_status in {"succeeded", "partially_succeeded", "no_action"}:
result["status"] = "partially_succeeded"
result["success"] = False
result["outcome"] = "failed_or_blocked"
execution_report = result.get("execution_report")
if isinstance(execution_report, dict):
spec = execution_report.get("spec")
if isinstance(spec, dict):
spec["status"] = "partially_succeeded"
spec["status"] = str(result.get("status") or "blocked")
return result
def _execute_request_with_peripherals(
@@ -1465,7 +1626,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
**mode_arguments,
_on_commands_validated=callback,
)
catalog = self.extension.list_peripheral_capabilities()
catalog = self._peripheral_routing_catalog()
nfc_snapshot = self.nfc.snapshot()
partition = partition_active_peripheral_intent(
utterance,
@@ -1479,7 +1640,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
parsed_peripheral_clauses = list(
dict.fromkeys(
str(entry.get("clause") or "").strip()
for entry in entries
for entry in [*entries, *denied]
if isinstance(entry, dict) and str(entry.get("clause") or "").strip()
)
)
@@ -1829,14 +1990,43 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
"supplement_required": True,
}
else:
labels = "、".join(
str(item.get("clause") or "未命名动作")
for item in denied
if isinstance(item, dict)
)
raise ExtensionIntegrationError(
f"以下持久绑定红外能力未启用，无法执行：{labels}。"
local_elapsed_ms = round(
(time.perf_counter() - local_stage_started) * 1000,
3,
)
request_id = f"req-local-blocked-{os.urandom(8).hex()}"
result = {
"request_id": request_id,
"preview_request_id": request_id,
"utterance": utterance,
"conversation_id": conversation_id,
"status": "blocked",
"success": False,
"outcome": "failed_or_blocked",
"phase": "authorization_blocked",
"orchestration_route": "local_peripheral_authorization",
"provider_requested": provider,
"provider_used": "infrared_local_rules",
"execution_mode": execution_mode,
"control_transport": "gscu_tcp",
"decision_summary": [],
"function_config": None,
"function_configs": {},
"command_plan": None,
"command_plans": {},
"trace_events": [],
"validation": [],
"execution_report": None,
"operation_results": [],
"parsed_control_signatures": [],
"preview_elapsed_ms": local_elapsed_ms,
"local_control_budget_ms": LOCAL_CONTROL_BUDGET_MS,
"local_control_elapsed_ms": local_elapsed_ms,
"local_control_within_budget": (
local_elapsed_ms < LOCAL_CONTROL_BUDGET_MS
),
"supplement_required": False,
}
result["natural_language_routing"] = deepcopy(natural_language_routing)
for key in ("parsed_clauses", "resolved_clauses", "unresolved_clauses"):
@@ -3528,7 +3718,7 @@ class SeatDemoHandler(BaseHTTPRequestHandler):
result = {**result, "input_mode": input_mode}
raw_text = str(result.get("raw_text") or "").strip()
peripheral_routing = self._speech_peripheral_routing(raw_text)
if peripheral_routing.get("execution_eligible"):
if peripheral_routing.get("routing_eligible"):
result = {
**result,
"routing_text": raw_text,