const VISION_VERSION = "0.10.14"; const VISION_MODULE = `/vendor/mediapipe/tasks-vision/${VISION_VERSION}/vision_bundle.mjs`; const VISION_WASM = `/vendor/mediapipe/tasks-vision/${VISION_VERSION}/wasm`; const CONNECTIONS = [ [0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [5, 9], [9, 10], [10, 11], [11, 12], [9, 13], [13, 14], [14, 15], [15, 16], [13, 17], [0, 17], [17, 18], [18, 19], [19, 20], ]; const LONG_FINGERS = [[5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], [17, 18, 19, 20]]; const TOKEN_NAMES = { OTHER: "未识别", OPEN_PALM: "张开手掌", CLOSED_FIST: "握拳", POINT_UP: "食指向上", POINT_DOWN: "食指向下", POINT_RIGHT: "食指向右", POINT_LEFT: "食指向左", THUMB_UP: "拇指向上", THUMB_DOWN: "拇指向下", GUN_UP: "手枪手势向上", GUN_DOWN: "手枪手势向下", VICTORY: "V 手势", VICTORY_RIGHT: "V 手势向右", VICTORY_LEFT: "V 手势向左", I_LOVE_YOU: "ILY 手势", }; const ACTIONS = { SEAT_UP: { label: "坐垫前端升高", short: "坐垫 ↑", utterance: "副驾驶座椅坐垫前端升高一点" }, SEAT_DOWN: { label: "坐垫前端降低", short: "坐垫 ↓", utterance: "副驾驶座椅坐垫前端降低一点" }, SEAT_FORWARD: { label: "座椅向前", short: "座椅 →", utterance: "副驾驶座椅向前移动一点" }, SEAT_BACKWARD: { label: "座椅向后", short: "座椅 ←", utterance: "副驾驶座椅向后移动一点" }, BACKREST_FORWARD: { label: "靠背向前", short: "靠背 ↑", utterance: "副驾驶靠背向前调一点" }, BACKREST_BACKWARD: { label: "靠背向后", short: "靠背 ↓", utterance: "副驾驶靠背向后调一点" }, LEGREST_EXTEND: { label: "腿托伸出", short: "腿托伸", utterance: "副驾驶腿托伸出一点" }, LEGREST_RETRACT: { label: "腿托收回", short: "腿托收", utterance: "副驾驶腿托收回一点" }, }; const GUIDE_TOKENS = [ "POINT_UP", "POINT_DOWN", "POINT_RIGHT", "POINT_LEFT", "THUMB_UP", "THUMB_DOWN", "GUN_UP", "GUN_DOWN", ]; const state = { runtime: null, recognizer: null, stream: null, frameRequest: null, lastVideoTime: -1, lastRecognizedToken: null, customMapping: loadCustomMapping(), customizeOpen: false, engine: null, runtimeReady: false, enabled: true, disabledReason: "", cameraStatus: "checking", cameraMessage: "正在检测摄像头…", cameraCheckSequence: 0, }; const $ = (selector) => document.querySelector(selector); function loadCustomMapping() { try { const value = JSON.parse(localStorage.getItem("cognitive-agent.gesture-mapping") || "{}"); return value && typeof value === "object" ? value : {}; } catch { return {}; } } function notify(message) { if (window.CognitiveAgentConsole?.notify) { window.CognitiveAgentConsole.notify(message); return; } const toast = $("#toast"); if (!toast) return; toast.textContent = message; toast.classList.add("visible"); window.setTimeout(() => toast.classList.remove("visible"), 2600); } function cameraFailureDetails(error) { if (!navigator.mediaDevices?.getUserMedia) { return window.isSecureContext === false ? { status: "unsupported", message: "当前页面不是安全连接,浏览器已阻止摄像头访问" } : { status: "unsupported", message: "当前浏览器不支持摄像头访问" }; } switch (error?.name) { case "NotFoundError": case "DevicesNotFoundError": return { status: "unavailable", message: "未检测到可用摄像头" }; case "NotAllowedError": case "PermissionDeniedError": case "SecurityError": return { status: "blocked", message: "摄像头权限被拒绝,请在浏览器设置中允许访问" }; case "NotReadableError": case "TrackStartError": return { status: "busy", message: "摄像头正被其他程序占用或暂时不可用" }; case "OverconstrainedError": case "ConstraintNotSatisfiedError": return { status: "incompatible", message: "摄像头不支持当前画面参数" }; case "AbortError": return { status: "unknown", message: "摄像头启动被中断,请重试" }; default: return { status: "unknown", message: error?.message || "摄像头启动失败" }; } } function syncCameraButtonAvailability() { const button = $("#gestureStartButton"); if (!button || state.stream) return; button.disabled = !state.enabled || !state.runtimeReady || ["checking", "unavailable", "unsupported"].includes(state.cameraStatus); } function setCameraAvailability(status, message, { announce = true } = {}) { state.cameraStatus = status; state.cameraMessage = message; $("#gestureDrawer")?.setAttribute("data-camera-status", status); syncCameraButtonAvailability(); if (announce && !state.stream) $("#gestureSafetyText").textContent = message; } async function refreshCameraAvailability({ announce = true } = {}) { if (!navigator.mediaDevices?.enumerateDevices || !navigator.mediaDevices?.getUserMedia) { const failure = cameraFailureDetails(); setCameraAvailability(failure.status, failure.message, { announce }); return false; } const checkSequence = ++state.cameraCheckSequence; if (!state.stream) setCameraAvailability("checking", "正在检测摄像头…", { announce }); try { const devices = await navigator.mediaDevices.enumerateDevices(); if (checkSequence !== state.cameraCheckSequence) return state.cameraStatus === "available"; const available = devices.some((device) => device.kind === "videoinput"); setCameraAvailability( available ? "available" : "unavailable", available ? "已检测到摄像头,点击开启开始识别" : "未检测到可用摄像头", { announce }, ); return available; } catch (error) { if (checkSequence !== state.cameraCheckSequence) return state.cameraStatus === "available"; const failure = cameraFailureDetails(error); const message = failure.status === "blocked" ? "尚未获得摄像头权限,可点击开启并在浏览器中授权" : "暂时无法检测摄像头,可点击开启时再次确认"; setCameraAvailability("unknown", message, { announce }); return null; } } async function handleCameraDeviceChange() { const streamWasActive = Boolean(state.stream); const available = await refreshCameraAvailability({ announce: !streamWasActive }); const streamStillLive = state.stream?.getVideoTracks().some((track) => track.readyState === "live"); if (streamWasActive && (available === false || !streamStillLive)) { const message = available === false ? "摄像头连接已断开,未检测到其他可用摄像头" : "当前摄像头连接已断开,请重新开启摄像头"; stopCamera({ message }); notify(message); } } function distance(a, b) { return Math.hypot(a.x - b.x, a.y - b.y); } function angle(a, vertex, c) { const first = [a.x - vertex.x, a.y - vertex.y]; const second = [c.x - vertex.x, c.y - vertex.y]; const denominator = Math.hypot(...first) * Math.hypot(...second); if (denominator <= 1e-9) return 0; const cosine = Math.max(-1, Math.min(1, (first[0] * second[0] + first[1] * second[1]) / denominator)); return Math.acos(cosine) * 180 / Math.PI; } function fingerExtended(points, [mcpIndex, pipIndex, dipIndex, tipIndex]) { const [mcp, pip, dip, tip] = [mcpIndex, pipIndex, dipIndex, tipIndex].map((index) => points[index]); return angle(mcp, pip, dip) >= 150 && angle(pip, dip, tip) >= 145 && distance(points[0], tip) >= distance(points[0], pip) * 1.08; } function fingerStates(points) { return LONG_FINGERS.map((indices) => fingerExtended(points, indices)); } function plausibleHand(points) { if (!Array.isArray(points) || points.length !== 21) return false; if (points.some((point) => !Number.isFinite(point.x) || !Number.isFinite(point.y) || point.x < -0.15 || point.x > 1.15 || point.y < -0.15 || point.y > 1.15)) return false; const xs = points.map((point) => point.x); const ys = points.map((point) => point.y); if (Math.max(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys)) < 0.12) return false; const palmWidth = distance(points[5], points[17]); if (palmWidth < 0.025) return false; const edges = [distance(points[5], points[9]), distance(points[9], points[13]), distance(points[13], points[17])]; if (Math.min(...edges) < 0.006 || Math.max(...edges) / Math.min(...edges) > 3.5) return false; return CONNECTIONS.filter(([first, second]) => distance(points[first], points[second]) > 0.004).length >= 16; } function pinchRatio(points) { return distance(points[4], points[8]) / Math.max(distance(points[5], points[17]), 1e-6); } function pointingDirection(points) { const [index, middle, ring, pinky] = fingerStates(points); if (!index || middle || ring || pinky) return null; const origin = points[5]; const tip = points[8]; // The camera preview is mirrored, so horizontal intent follows display space. const dx = -(tip.x - origin.x); const dy = tip.y - origin.y; if (Math.hypot(dx, dy) < distance(points[5], points[17]) * 0.65) return null; if (Math.abs(dx) >= Math.abs(dy)) return dx > 0 ? "RIGHT" : "LEFT"; return dy > 0 ? "DOWN" : "UP"; } function thumbIsExtended(points) { if (!Array.isArray(points) || points.length !== 21) return false; const palmWidth = Math.max(distance(points[5], points[17]), 1e-6); const thumbLength = distance(points[2], points[4]); return angle(points[1], points[2], points[3]) >= 145 && angle(points[2], points[3], points[4]) >= 145 && thumbLength >= palmWidth * 0.65; } function thumbVerticalDirection(points) { if (fingerStates(points).some(Boolean)) return null; const dx = points[4].x - points[2].x; const dy = points[4].y - points[2].y; const length = Math.hypot(dx, dy); const palmWidth = Math.max(distance(points[5], points[17]), 1e-6); if (length < palmWidth * 0.65 || Math.abs(dy) < Math.abs(dx) * 1.6 || !thumbIsExtended(points)) return null; return dy > 0 ? "DOWN" : "UP"; } function gunDirection(points) { const direction = pointingDirection(points); if (!direction || !["UP", "DOWN"].includes(direction)) return null; return thumbIsExtended(points) ? direction : null; } function victoryDirection(points) { const states = fingerStates(points); if (states.join(",") !== "true,true,false,false") return null; const originX = (points[5].x + points[9].x) / 2; const originY = (points[5].y + points[9].y) / 2; const tipX = (points[8].x + points[12].x) / 2; const tipY = (points[8].y + points[12].y) / 2; const dx = -(tipX - originX); const dy = tipY - originY; if (Math.hypot(dx, dy) < distance(points[5], points[17]) * 0.65 || Math.abs(dx) < Math.abs(dy) * 1.25) return null; return dx > 0 ? "RIGHT" : "LEFT"; } function geometryGesture(points) { if (!plausibleHand(points)) return null; const states = fingerStates(points); const pinch = pinchRatio(points); const extendedThumb = thumbIsExtended(points); const gun = gunDirection(points); if (gun) return `GUN_${gun}`; const point = pointingDirection(points); if (point && !extendedThumb && pinch >= 0.75) return `POINT_${point}`; if (states.join(",") === "true,true,false,false" && pinch >= 1.1) { const direction = victoryDirection(points); return direction ? `VICTORY_${direction}` : "VICTORY"; } if (states.join(",") === "true,false,false,true" && pinch >= 1.1) return "I_LOVE_YOU"; if (states.every(Boolean) && pinch >= 1.0) return "OPEN_PALM"; if (states.every((value) => !value)) { const thumb = thumbVerticalDirection(points); if (thumb && pinch >= 0.72) return `THUMB_${thumb}`; if (!extendedThumb && pinch <= 0.62) return "CLOSED_FIST"; } return null; } function normalizedCategory(value) { return String(value || "None").trim().toLowerCase().replaceAll("-", "_").replaceAll(" ", "_"); } function classifyHand(points, categories = []) { const geometry = geometryGesture(points); const top = categories[0] || {}; const runnerUp = categories[1] || {}; const name = normalizedCategory(top.categoryName || top.displayName); const score = Number(top.score || 0); const margin = Math.max(0, score - Number(runnerUp.score || 0)); if (!geometry) return { token: "OTHER", score, reason: "手型未通过严格几何校验" }; if (geometry.startsWith("GUN_")) { return ["pointing_up", "thumb_up", "thumb_down", "none", ""].includes(name) ? { token: geometry, score: Math.max(score, 0.78), reason: "模型与手枪几何一致" } : { token: "OTHER", score, reason: "模型类别与手枪几何冲突" }; } if (geometry.startsWith("POINT_")) { return ["pointing_up", "none", ""].includes(name) ? { token: geometry, score: Math.max(score, 0.75), reason: "严格指向几何" } : { token: "OTHER", score, reason: "模型类别与指向几何冲突" }; } if (["VICTORY_LEFT", "VICTORY_RIGHT"].includes(geometry)) { return ["victory", "none", ""].includes(name) ? { token: geometry, score: Math.max(score, 0.75), reason: "严格 V 字方向几何" } : { token: "OTHER", score, reason: "模型类别与 V 字方向冲突" }; } if (["THUMB_UP", "THUMB_DOWN"].includes(geometry)) { const expected = geometry.toLowerCase(); return (["none", "", expected].includes(name) && (name === "none" || name === "" || score >= 0.6)) ? { token: geometry, score: Math.max(score, 0.6), reason: "模型与拇指几何一致" } : { token: "OTHER", score, reason: "模型类别与拇指几何冲突" }; } const expectedNames = { OPEN_PALM: "open_palm", CLOSED_FIST: "closed_fist", VICTORY: "victory", I_LOVE_YOU: "iloveyou", }; const threshold = geometry === "OPEN_PALM" ? 0.62 : 0.7; if (name !== expectedNames[geometry] || score < threshold || (Number(runnerUp.score || 0) > 0 && margin < 0.12)) { return { token: "OTHER", score, reason: "模型类别、分数或几何不一致" }; } return { token: geometry, score, reason: "模型与几何一致" }; } function mappingForToken(token) { const defaultAction = state.runtime?.actions?.[token]?.action; return state.customMapping[token] || defaultAction || null; } class GestureEngine { constructor(config, onAction) { this.config = config; this.onAction = onAction; this.armed = false; this.currentAction = null; this.candidate = null; this.candidateSince = 0; this.candidateFrames = 0; this.lastActionAt = 0; this.motionStartedAt = 0; this.armSince = 0; this.armFrames = 0; this.neutralSince = 0; this.lockedAction = null; this.fistLatched = false; } resetCandidate() { this.candidate = null; this.candidateSince = 0; this.candidateFrames = 0; } setArmed(value, now = performance.now()) { this.armed = value; this.currentAction = null; this.motionStartedAt = 0; this.lockedAction = null; this.resetCandidate(); this.lastActionAt = now; updateEngineUI(this, value ? "已激活,可直接切换动作" : "已锁止,张开手掌激活"); } stopMotion(message) { this.currentAction = null; this.motionStartedAt = 0; this.resetCandidate(); updateEngineUI(this, message); } update(poses, now) { const fist = poses.some((pose) => pose.token === "CLOSED_FIST"); if (fist) { if (!this.fistLatched || this.currentAction) this.stopMotion("握拳已停止,控制保持激活"); this.fistLatched = true; if (this.armed && now - this.lastActionAt >= this.config.armed_idle_timeout_ms) this.setArmed(false, now); return; } this.fistLatched = false; const primary = poses.length === 1 ? poses[0] : null; if (!this.armed) { if (primary?.token === "OPEN_PALM") { if (!this.armSince) { this.armSince = now; this.armFrames = 1; } else { this.armFrames += 1; } const progress = Math.min(1, (now - this.armSince) / this.config.arm_hold_ms); setGestureProgress(progress); if (progress >= 1 && this.armFrames >= this.config.min_action_frames) { this.armSince = 0; this.armFrames = 0; this.setArmed(true, now); notify("手势控制已激活"); } } else { this.armSince = 0; this.armFrames = 0; setGestureProgress(0); } return; } const action = primary ? mappingForToken(primary.token) : null; if (!action) { if (this.currentAction) this.stopMotion("动作已停止,控制保持激活"); this.resetCandidate(); if (this.lockedAction) { if (!this.neutralSince) this.neutralSince = now; if (now - this.neutralSince >= this.config.neutral_release_ms) this.lockedAction = null; } if (now - this.lastActionAt >= this.config.armed_idle_timeout_ms) { this.setArmed(false, now); notify("连续 3 秒无有效动作,手势控制已安全锁止"); } setGestureProgress(0); return; } this.lastActionAt = now; this.neutralSince = 0; if (this.lockedAction && action !== this.lockedAction) this.lockedAction = null; if (this.lockedAction === action) { updateEngineUI(this, "该动作已达到单次时限,请先释放手势"); setGestureProgress(0); return; } if (this.currentAction && action !== this.currentAction) this.stopMotion("正在确认新动作…"); if (this.currentAction === action) { setGestureProgress(1); if (now - this.motionStartedAt >= this.config.max_continuous_motion_ms) { this.lockedAction = action; this.stopMotion("动作达到 1.5 秒上限,请释放或更换动作"); } return; } if (this.candidate !== action) { this.candidate = action; this.candidateSince = now; this.candidateFrames = 1; } else { this.candidateFrames += 1; } const progress = Math.min(1, (now - this.candidateSince) / this.config.action_hold_ms); setGestureProgress(progress); updateEngineUI(this, `保持手势 · ${Math.round(progress * 100)}%`); if (progress >= 1 && this.candidateFrames >= this.config.min_action_frames) { this.currentAction = action; this.motionStartedAt = now; this.resetCandidate(); updateEngineUI(this, `${ACTIONS[action]?.label || action} · 已执行`); this.onAction(action); } } } function setGestureProgress(value) { const bar = $("#gestureProgressBar"); if (bar) bar.style.transform = `scaleX(${Math.max(0, Math.min(1, value))})`; } function updateEngineUI(engine, message) { const pill = $("#gestureStatePill"); if (!pill) return; pill.className = `gesture-state-pill ${engine.armed ? (engine.currentAction ? "moving" : "armed") : "idle"}`; pill.querySelector("b").textContent = engine.armed ? (engine.currentAction ? "执行中" : "已激活") : "已锁止"; $("#gestureSafetyText").textContent = message; } function executeGestureAction(action) { const definition = ACTIONS[action]; if (!definition) return; const executionMode = window.CognitiveAgentConsole?.getExecutionMode?.() || "dry_run"; const autoSubmit = executionMode !== "hardware"; const accepted = window.CognitiveAgentConsole?.executeUtterance?.(definition.utterance, { autoSubmit }); if (accepted === false) { notify("上一条指令仍在处理中,本次手势未重复提交"); } else if (!autoSubmit) { notify("真实台架模式需要人工授权,指令已填入输入框"); } else { notify(`手势指令:${definition.label}`); } } function drawLandmarks(result) { const canvas = $("#gestureCanvas"); const video = $("#gestureVideo"); if (!canvas || !video.videoWidth) return; if (canvas.width !== video.videoWidth || canvas.height !== video.videoHeight) { canvas.width = video.videoWidth; canvas.height = video.videoHeight; } const context = canvas.getContext("2d"); context.clearRect(0, 0, canvas.width, canvas.height); for (const points of result.landmarks || []) { context.lineWidth = Math.max(2, canvas.width / 360); context.strokeStyle = "rgba(114, 255, 196, .9)"; for (const [first, second] of CONNECTIONS) { context.beginPath(); context.moveTo(points[first].x * canvas.width, points[first].y * canvas.height); context.lineTo(points[second].x * canvas.width, points[second].y * canvas.height); context.stroke(); } context.fillStyle = "#f8ff73"; for (const point of points) { context.beginPath(); context.arc(point.x * canvas.width, point.y * canvas.height, Math.max(3, canvas.width / 230), 0, Math.PI * 2); context.fill(); } } } function renderRecognition(poses) { const detected = $("#gestureDetected"); const expected = $("#gestureExpected"); if (!poses.length) { detected.textContent = "未检测到手"; expected.textContent = state.engine?.armed ? "保持激活" : "张掌激活"; return; } detected.textContent = poses.map((pose) => TOKEN_NAMES[pose.token] || pose.token).join(" + "); const action = poses.length === 1 ? mappingForToken(poses[0].token) : null; expected.textContent = action ? ACTIONS[action]?.label || action : poses.length > 1 ? "多手画面已锁止" : poses[0].reason; const usable = poses.find((pose) => !["OTHER", "OPEN_PALM", "CLOSED_FIST"].includes(pose.token)); if (usable) state.lastRecognizedToken = usable.token; if (state.customizeOpen) updateCustomizeToken(); } async function recognizeLoop() { if (!state.stream || !state.recognizer) return; const video = $("#gestureVideo"); const now = performance.now(); if (video.readyState >= 2 && video.currentTime !== state.lastVideoTime) { state.lastVideoTime = video.currentTime; const result = state.recognizer.recognizeForVideo(video, now); drawLandmarks(result); const poses = (result.landmarks || []).map((points, index) => classifyHand(points, result.gestures?.[index] || [])); renderRecognition(poses); state.engine.update(poses, now); } state.frameRequest = requestAnimationFrame(recognizeLoop); } async function loadGestureRuntime() { const response = await window.CognitiveAgentNetwork.request("/api/gesture/runtime", { cache: "no-store" }); if (!response.ok) throw new Error("无法读取手势能力配置"); state.runtime = await response.json(); state.engine = new GestureEngine(state.runtime.safety, executeGestureAction); renderMappingGrid(); if (!state.runtime.model_available) throw new Error("内置手势模型不可用,请检查工程资源"); } async function loadRecognizer() { if (state.recognizer) return; $("#gestureSafetyText").textContent = "正在加载手势模型…"; const { FilesetResolver, GestureRecognizer } = await import(VISION_MODULE); const fileset = await FilesetResolver.forVisionTasks(VISION_WASM); const modelResponse = await window.CognitiveAgentNetwork.request("/api/gesture/model", { cache: "force-cache", timeoutMs: 60000, }); if (!modelResponse.ok) throw new Error("内置手势模型加载失败"); const modelAssetBuffer = new Uint8Array(await modelResponse.arrayBuffer()); state.recognizer = await GestureRecognizer.createFromOptions(fileset, { baseOptions: { modelAssetBuffer, delegate: "CPU" }, runningMode: "VIDEO", numHands: 2, minHandDetectionConfidence: 0.65, minHandPresenceConfidence: 0.65, minTrackingConfidence: 0.60, }); } async function startCamera() { const button = $("#gestureStartButton"); button.disabled = true; button.querySelector("span").textContent = "正在启动…"; button.setAttribute("aria-label", "正在启动摄像头"); button.setAttribute("title", "正在启动摄像头"); try { if (!state.runtime) await loadGestureRuntime(); await loadRecognizer(); state.stream = await navigator.mediaDevices.getUserMedia({ video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: "user" }, audio: false, }); setCameraAvailability("available", "摄像头可用,点击开启开始识别", { announce: false }); const video = $("#gestureVideo"); video.srcObject = state.stream; await video.play(); $("#gestureCameraEmpty").hidden = true; $("#gestureLiveBadge").hidden = false; $("#gestureStopButton").disabled = false; button.classList.add("active"); button.querySelector("span").textContent = "关闭摄像头"; button.setAttribute("aria-label", "关闭摄像头"); button.setAttribute("title", "关闭摄像头"); $("#gestureSafetyText").textContent = "张开手掌并保持 1 秒以激活"; state.frameRequest = requestAnimationFrame(recognizeLoop); } catch (error) { const failure = cameraFailureDetails(error); setCameraAvailability(failure.status, failure.message, { announce: false }); stopCamera({ message: failure.message }); notify(failure.message); } finally { syncCameraButtonAvailability(); } } function stopCamera({ message = null } = {}) { if (state.frameRequest) cancelAnimationFrame(state.frameRequest); state.frameRequest = null; state.stream?.getTracks().forEach((track) => track.stop()); state.stream = null; state.lastVideoTime = -1; const video = $("#gestureVideo"); if (video) video.srcObject = null; const canvas = $("#gestureCanvas"); canvas?.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height); $("#gestureCameraEmpty").hidden = false; $("#gestureLiveBadge").hidden = true; $("#gestureStopButton").disabled = true; const button = $("#gestureStartButton"); button.classList.remove("active"); button.querySelector("span").textContent = "开启摄像头"; button.setAttribute("aria-label", "开启摄像头"); button.setAttribute("title", "开启摄像头"); state.engine?.setArmed(false); $("#gestureDetected").textContent = "—"; $("#gestureExpected").textContent = "等待启动"; $("#gestureSafetyText").textContent = message || state.cameraMessage || "摄像头尚未开启"; syncCameraButtonAvailability(); } function toggleCamera() { if (state.stream) stopCamera(); else startCamera(); } function renderMappingGrid() { const grid = $("#gestureMapGrid"); if (!grid || !state.runtime) return; grid.innerHTML = GUIDE_TOKENS.map((token) => { const action = mappingForToken(token); return `
${TOKEN_NAMES[token]}→${ACTIONS[action]?.short || "未绑定"}
`; }).join(""); const select = $("#gestureActionSelect"); select.innerHTML = Object.entries(ACTIONS).map(([value, item]) => ``).join(""); } function updateCustomizeToken() { const token = state.lastRecognizedToken; $("#gestureCustomizeToken").textContent = token ? TOKEN_NAMES[token] || token : "请先摆出手势"; if (token) { const action = mappingForToken(token); if (action) $("#gestureActionSelect").value = action; } } function toggleCustomize() { state.customizeOpen = !state.customizeOpen; $("#gestureCustomizePanel").hidden = !state.customizeOpen; $("#gestureCustomizeButton").classList.toggle("active", state.customizeOpen); updateCustomizeToken(); } function bindCurrentGesture() { const token = state.lastRecognizedToken; if (!token || ["OTHER", "OPEN_PALM", "CLOSED_FIST"].includes(token)) { notify("请先保持一个可自定义的有效手势"); return; } state.customMapping[token] = $("#gestureActionSelect").value; localStorage.setItem("cognitive-agent.gesture-mapping", JSON.stringify(state.customMapping)); renderMappingGrid(); notify(`${TOKEN_NAMES[token] || token} 的功能已保存`); } function resetMapping() { state.customMapping = {}; localStorage.removeItem("cognitive-agent.gesture-mapping"); renderMappingGrid(); notify("已恢复默认手势映射"); } function openDrawer() { if (!state.enabled) { notify(state.disabledReason || "当前运行模式未启用手势控制"); return false; } const drawer = $("#gestureDrawer"); if (!drawer) return false; drawer.classList.add("open"); drawer.setAttribute("aria-hidden", "false"); document.body.classList.add("gesture-open"); return true; } function closeDrawer() { stopCamera(); const drawer = $("#gestureDrawer"); drawer?.classList.remove("open"); drawer?.setAttribute("aria-hidden", "true"); document.body.classList.remove("gesture-open"); } function setEnabled(enabled, { reason = "" } = {}) { state.enabled = Boolean(enabled); state.disabledReason = state.enabled ? "" : String(reason || "当前运行模式未启用手势控制"); if (!state.enabled) closeDrawer(); syncCameraButtonAvailability(); const drawer = $("#gestureDrawer"); if (drawer) { drawer.dataset.enabled = String(state.enabled); drawer.title = state.disabledReason; } } window.CognitiveAgentGesture = { open: openDrawer, close: closeDrawer, isOpen: () => $("#gestureDrawer")?.classList.contains("open") === true, setEnabled, }; document.addEventListener("DOMContentLoaded", async () => { $("#gestureStartButton").disabled = true; $("#gestureCloseButton")?.addEventListener("click", closeDrawer); $("#gestureStartButton")?.addEventListener("click", toggleCamera); $("#gestureStopButton")?.addEventListener("click", () => state.engine?.setArmed(false)); $("#gestureCustomizeButton")?.addEventListener("click", toggleCustomize); $("#gestureBindButton")?.addEventListener("click", bindCurrentGesture); $("#gestureResetMapping")?.addEventListener("click", resetMapping); document.addEventListener("keydown", (event) => { if (event.key !== "Escape") return; if ($("#gestureDrawer")?.classList.contains("open")) closeDrawer(); }); document.addEventListener("visibilitychange", () => { if (document.hidden && state.stream) stopCamera(); }); window.addEventListener("beforeunload", stopCamera); navigator.mediaDevices?.addEventListener?.("devicechange", handleCameraDeviceChange); let runtimeError = null; try { await loadGestureRuntime(); state.runtimeReady = Boolean(state.runtime.available && state.runtime.model_available); if (!state.runtimeReady) runtimeError = new Error("内置手势模型不可用"); } catch (error) { runtimeError = error; state.runtimeReady = false; } await refreshCameraAvailability({ announce: !runtimeError }); if (runtimeError) $("#gestureSafetyText").textContent = runtimeError.message; syncCameraButtonAvailability(); });