vee/Cognitive_Agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate learned peripherals into mode creation

Browse Source 
Expose safe NFC-bound infrared actions to semantic scene composition, require flashing-light actions for romantic and Mid-Autumn modes, and preserve mixed CAN/IR mode flows.

Fix NFC device reassignment and retry extension/toy MQTT publication when an active card's binding catalog changes.
This commit is contained in:
Miayui
2026-09-20 16:37:19 +08:00
parent 904f5f8a6c
commit bfff1b515c
25 changed files with 3073 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cognitive_agent/application/codex_assistant.py
+369 -14
View File
@@ -14,10 +14,12 @@ import time
import uuid
from collections import OrderedDict
from collections.abc import Callable, Mapping, Sequence
from copy import deepcopy
from dataclasses import dataclass, field
from pathlib import Path
from typing import Protocol
from cognitive_agent.application.assistant_plan_guard import MAX_PLAN_REQUESTS
from cognitive_agent.application.mode_profile_intent import (
is_mode_profile_query,
mode_creation_source_name,
@@ -31,11 +33,14 @@ from cognitive_agent.application.mode_schedule_service import public_schedule_st
from cognitive_agent.application.scene_composer import (
SCENE_COMPOSER_INSTRUCTIONS,
SCENE_PLANNER_VERSION,
SCENE_TEMPLATE_AUGMENTATION_INSTRUCTIONS,
SCHEDULE_COMPOSER_INSTRUCTIONS,
SceneCompositionError,
build_scene_output_schema,
normalize_scene_output,
scene_context_for_mode_creation,
scene_required_peripheral_refs,
scene_semantic_guidance,
)
from cognitive_agent.application.scene_templates import normalize_scene_template_name
from cognitive_agent.application.schedule_intent import is_schedule_stop, needs_mode_schedule
@@ -936,6 +941,54 @@ class CodexAssistantService:
if mode_parts:
scene_context = scene_context_for_mode_creation(scene_context, message)
model_context = scene_context_for_mode_creation(model_context, message)
semantic_guidance = scene_semantic_guidance(mode_parts[0])
live_scene_capabilities = {
str(item.get("id") or "")
for item in scene_context.get("capabilities", [])
if isinstance(item, Mapping)
}
raw_preferred_capabilities = semantic_guidance.get(
"preferred_capability_ids", []
)
semantic_guidance["preferred_capability_ids"] = [
str(capability_id)
for capability_id in (
raw_preferred_capabilities
if isinstance(raw_preferred_capabilities, Sequence)
and not isinstance(
raw_preferred_capabilities,
(str, bytes, bytearray),
)
else []
)
if str(capability_id) in live_scene_capabilities
]
required_peripheral_refs = scene_required_peripheral_refs(
mode_parts[0],
scene_context,
)
raw_peripheral_actions = scene_context.get("peripheral_actions", [])
semantic_guidance["peripheral_policy"] = {
"available_action_count": (
len(raw_peripheral_actions)
if isinstance(raw_peripheral_actions, list)
else 0
),
"required_refs": required_peripheral_refs,
"selection_goal": (
"Include as many semantically relevant scene-safe peripheral "
"devices as are useful; normally choose activation plus at most "
"one stable state action per device."
),
"required_reason": (
"Romantic and Mid-Autumn related scenes must include the "
"authorized flashing-light control."
if required_peripheral_refs
else "No peripheral action is mandatory for this name."
),
}
scene_context["scene_semantics"] = semantic_guidance
model_context["scene_semantics"] = deepcopy(semantic_guidance)
exact_mode_matches = (
_catalog_exact_mode_match_count(catalog, mode_parts[0])
if mode_parts
@@ -947,6 +1000,23 @@ class CodexAssistantService:
else None
)
dynamic_scene_request = bool(mode_parts and needs_mode_schedule(message))
scene_peripheral_actions = scene_context.get("peripheral_actions", [])
template_augmentation_preview = bool(
scene_template_plan is not None
and not dynamic_scene_request
and isinstance(scene_peripheral_actions, list)
and scene_peripheral_actions
and self.client.available
and callable(getattr(self.client, "start_structured_turn", None))
)
direct_scene_template_plan = (
self._with_required_scene_peripherals(
scene_template_plan,
scene_context,
)
if scene_template_plan is not None
else None
)
direct_mode_profile_preview = bool(
mode_parts
and (
@@ -955,7 +1025,10 @@ class CodexAssistantService:
# local builder. Only a known-zero match is safe to hand to
# semantic scene composition.
or exact_mode_matches != 0
or scene_template_plan is not None
or (
scene_template_plan is not None
and not template_augmentation_preview
)
)
)
direct_mode_profile_update = (
@@ -1153,6 +1226,32 @@ class CodexAssistantService:
query_capabilities=catalog_query,
)
if template_augmentation_preview and scene_template_plan is not None:
try:
return self._stream_scene_template_augmentation(
message,
message_id=message_id,
emit=emit,
execute_control=execute_control,
scene_context=scene_context,
template_plan=scene_template_plan,
)
except _ScenePlannerCancelled:
return {"cancelled": True}
except Exception as exc: # noqa: BLE001 - trusted base remains usable
LOGGER.warning(
"Scene template peripheral augmentation failed; using base: %s",
exc,
)
return self._stream_direct_vehicle_control(
authorization,
message_id=message_id,
emit=emit,
execute_control=execute_control,
assistant_plan=direct_scene_template_plan,
response_route="scene_template",
)
if direct_vehicle_control:
return self._stream_direct_vehicle_control(
(
@@ -1166,7 +1265,7 @@ class CodexAssistantService:
message_id=message_id,
emit=emit,
execute_control=execute_control,
assistant_plan=scene_template_plan,
assistant_plan=direct_scene_template_plan,
response_route=(
"scene_template"
if scene_template_plan is not None
@@ -1294,6 +1393,97 @@ class CodexAssistantService:
def _copy_scene_plan(plan: Mapping[str, object]) -> dict[str, object]:
return json.loads(json.dumps(dict(plan), ensure_ascii=False, allow_nan=False))
@staticmethod
def _required_scene_peripheral_refs(
scene_context: Mapping[str, object],
) -> list[str]:
semantics = scene_context.get("scene_semantics")
policy = (
semantics.get("peripheral_policy")
if isinstance(semantics, Mapping)
else None
)
raw_refs = policy.get("required_refs", []) if isinstance(policy, Mapping) else []
available_refs = {
str(item.get("ref") or "").strip()
for item in scene_context.get("peripheral_actions", [])
if isinstance(item, Mapping) and str(item.get("ref") or "").strip()
}
return list(
dict.fromkeys(
str(value).strip()
for value in (
raw_refs
if isinstance(raw_refs, Sequence)
and not isinstance(raw_refs, (str, bytes, bytearray))
else []
)
if str(value).strip() in available_refs
)
)
@classmethod
def _with_required_scene_peripherals(
cls,
plan: Mapping[str, object],
scene_context: Mapping[str, object],
*,
maximum_requests: int = MAX_PLAN_REQUESTS,
) -> dict[str, object]:
"""Add mandatory opaque peripheral refs without exposing binding data."""
normalized = cls._copy_scene_plan(plan)
requests = normalized.get("requests")
if not isinstance(requests, list):
raise SceneCompositionError("scene plan requests must be an array")
existing_refs = {
str(parameter.get("value") or "").strip()
for request in requests
if isinstance(request, Mapping)
and request.get("domain") == "self_extension"
and request.get("capability") == "infrared.emit"
for parameter in (
request.get("parameters", [])
if isinstance(request.get("parameters"), list)
else []
)
if isinstance(parameter, Mapping)
and parameter.get("name") == "capability_ref"
}
required_refs = cls._required_scene_peripheral_refs(scene_context)
missing_refs = [ref for ref in required_refs if ref not in existing_refs]
if len(requests) + len(missing_refs) > maximum_requests:
raise SceneCompositionError(
"required peripheral actions exceed the scene action limit"
)
requests.extend(
{
"domain": "self_extension",
"capability": "infrared.emit",
"target_scope": capability_ref,
"operation": "emit",
"duration_ms": None,
"parameters": [
{"name": "capability_ref", "value": capability_ref}
],
"reason": (
"Mode semantics require this currently authorized learned-device "
"action; runtime binding resolution is still required."
),
}
for capability_ref in missing_refs
)
if missing_refs:
domains = normalized.get("detected_domains")
if isinstance(domains, list) and "self_extension" not in domains:
domains.append("self_extension")
summaries = normalized.get("decision_summary")
if isinstance(summaries, list):
summaries.append(
f"已按模式语义加入 {len(missing_refs)} 项必需的已授权外设动作。"
)
return normalized
def _scene_cache_key(
self,
*,
@@ -1385,6 +1575,110 @@ class CodexAssistantService:
while len(self._scene_cache) > self._scene_cache_limit:
self._scene_cache.popitem(last=False)
def _stream_scene_template_augmentation(
self,
message: str,
*,
message_id: str,
emit: EmitEvent,
execute_control: ControlExecutor,
scene_context: Mapping[str, object],
template_plan: Mapping[str, object],
) -> dict[str, object]:
"""Add only model-selected opaque IR refs to an immutable template base."""
peripheral_actions = scene_context.get("peripheral_actions", [])
if not isinstance(peripheral_actions, list) or not peripheral_actions:
return self._stream_direct_vehicle_control(
_VehicleAuthorization(message, "explicit_mode_profile_create"),
message_id=message_id,
emit=emit,
execute_control=execute_control,
assistant_plan=template_plan,
response_route="scene_template",
)
base_requests = template_plan.get("requests", [])
if not isinstance(base_requests, list) or not base_requests:
raise _ScenePlannerFallback("scene template base is empty")
augmentation_context = {
**dict(scene_context),
"capabilities": [],
"scene_semantics": deepcopy(scene_context.get("scene_semantics")),
"device_inventory": [
deepcopy(item)
for item in scene_context.get("device_inventory", [])
if isinstance(item, Mapping)
and item.get("kind") == "learned_infrared"
],
"template_base": {
"immutable": True,
"actions": [
{
"capability": request.get("capability"),
"target_scope": request.get("target_scope"),
"parameters": request.get("parameters"),
}
for request in base_requests
if isinstance(request, Mapping)
],
},
}
augmentation_context["maximum_generated_action_count"] = max(
0,
MAX_PLAN_REQUESTS
- len(base_requests)
- len(self._required_scene_peripheral_refs(scene_context)),
)
augmentation_semantics = augmentation_context.get("scene_semantics")
if isinstance(augmentation_semantics, dict):
augmentation_semantics["grounded"] = False
augmentation_semantics["minimum_action_count"] = 0
augmentation_semantics["ambiguities_allowed"] = True
if (
emit(
{
"type": "work.progress",
"message_id": message_id,
"display_text": "正在根据已授权外设补充常用模式……",
}
)
is False
):
raise _ScenePlannerCancelled()
deadline = time.monotonic() + self._scene_turn_timeout
augmentation = self._request_structured_scene_plan(
message,
message_id=message_id,
emit=emit,
scene_context=augmentation_context,
deadline=deadline,
developer_instructions=SCENE_TEMPLATE_AUGMENTATION_INSTRUCTIONS,
dynamic_override=False,
)
proposed = augmentation.get("requests", [])
if not isinstance(proposed, list) or any(
not isinstance(request, Mapping)
or request.get("domain") != "self_extension"
or request.get("capability") != "infrared.emit"
for request in proposed
):
raise _ScenePlannerFallback("template augmentation returned a native mutation")
if len(base_requests) + len(proposed) > MAX_PLAN_REQUESTS:
raise _ScenePlannerFallback("template augmentation exceeds the scene action limit")
combined = self._copy_scene_plan(template_plan)
combined["requests"] = [
*deepcopy(base_requests),
*deepcopy(proposed),
]
return self._stream_direct_vehicle_control(
_VehicleAuthorization(message, "explicit_mode_profile_create"),
message_id=message_id,
emit=emit,
execute_control=execute_control,
assistant_plan=combined,
response_route="scene_template",
)
def _stream_scene_composition(
self,
message: str,
@@ -1554,17 +1848,49 @@ class CodexAssistantService:
emit: EmitEvent,
scene_context: Mapping[str, object],
deadline: float,
developer_instructions: str | None = None,
dynamic_override: bool | None = None,
) -> dict[str, object]:
start_structured_turn = getattr(self.client, "start_structured_turn", None)
if not callable(start_structured_turn):
raise _ScenePlannerFallback("structured turns are not supported")
planning_started_at = time.monotonic()
dynamic = needs_mode_schedule(message)
output_schema = build_scene_output_schema(scene_context, dynamic=dynamic)
dynamic = (
needs_mode_schedule(message)
if dynamic_override is None
else dynamic_override
)
planner_context = deepcopy(dict(scene_context))
if dynamic:
# Timed IR playback is not yet phase-aware. Keep learned devices in
# inventory so the model can explain availability, but do not offer
# them as selectable actions in a schedule.
planner_context["peripheral_actions"] = []
dynamic_semantics = planner_context.get("scene_semantics")
dynamic_policy = (
dynamic_semantics.get("peripheral_policy")
if isinstance(dynamic_semantics, dict)
else None
)
if isinstance(dynamic_policy, dict):
dynamic_policy["available_action_count"] = 0
dynamic_policy["required_refs"] = []
dynamic_policy["required_reason"] = (
"Timed learned-infrared actions are not supported."
)
for item in planner_context.get("device_inventory", []):
if (
isinstance(item, dict)
and item.get("kind") == "learned_infrared"
and item.get("availability") == "available"
):
item["availability"] = "unavailable"
item["availability_reason"] = "timed_ir_not_supported"
output_schema = build_scene_output_schema(planner_context, dynamic=dynamic)
context_bytes = len(
json.dumps(
dict(scene_context),
planner_context,
ensure_ascii=False,
separators=(",", ":"),
).encode("utf-8")
@@ -1590,7 +1916,14 @@ class CodexAssistantService:
try:
try:
thread_id = self.client.start_thread(
developer_instructions=SCHEDULE_COMPOSER_INSTRUCTIONS if dynamic else SCENE_COMPOSER_INSTRUCTIONS,
developer_instructions=(
developer_instructions
or (
SCHEDULE_COMPOSER_INSTRUCTIONS
if dynamic
else SCENE_COMPOSER_INSTRUCTIONS
)
),
ephemeral=True,
replace_developer_instructions=True,
timeout=self._remaining_scene_time(deadline),
@@ -1604,7 +1937,7 @@ class CodexAssistantService:
thread_id,
message,
application_context={
**dict(scene_context),
**planner_context,
"natural_language_routing": {
"original_utterance": message,
"parsed_clauses": [],
@@ -1729,7 +2062,12 @@ class CodexAssistantService:
raise SceneCompositionError("scene composer output was not valid JSON") from exc
if not isinstance(raw_output, Mapping):
raise SceneCompositionError("scene composer output must be an object")
normalized = normalize_scene_output(raw_output, scene_context, dynamic=dynamic)
normalized = normalize_scene_output(raw_output, planner_context, dynamic=dynamic)
if not dynamic:
normalized = self._with_required_scene_peripherals(
normalized,
planner_context,
)
finished_at = time.monotonic()
LOGGER.info(
"Scene Composer model completed context_bytes=%d schema_bytes=%d "
@@ -3617,6 +3955,12 @@ class CodexAssistantService:
safe_target = CodexAssistantService._safe_result_value(
operation.get("target") or {}, secret=secret
)
if raw_domain == "self_extension" and isinstance(safe_target, dict):
safe_target = {
key: safe_target[key]
for key in ("device", "name")
if key in safe_target
}
safe_parameters = CodexAssistantService._safe_result_value(
operation.get("parameters") or {}, secret=secret
)
@@ -3732,16 +4076,27 @@ class CodexAssistantService:
for item in spec["actions"][:96 if result.get("schedule") else 12]:
if not isinstance(item, dict):
continue
actions.append(
{
safe_action = {
key: CodexAssistantService._safe_result_value(
item[key],
secret=secret,
)
for key in ("domain", "capability", "target", "parameters")
if key in item
}
if (
item.get("domain") == "self_extension"
and isinstance(item.get("target"), Mapping)
):
safe_action["target"] = {
key: CodexAssistantService._safe_result_value(
item[key],
item["target"][key],
secret=secret,
)
for key in ("domain", "capability", "target", "parameters")
if key in item
for key in ("device", "name")
if key in item["target"]
}
)
actions.append(safe_action)
safe["actions"] = actions
infrared_actions: list[dict[str, object]] = []
raw_infrared = result.get("infrared_actions")