vee
661 lines
26 KiB
JavaScript
Executable File
661 lines
26 KiB
JavaScript
Executable File
const VISION_VERSION = "0.10.14";
|
|
const VISION_MODULE = `/vendor/mediapipe/tasks-vision/${VISION_VERSION}/vision_bundle.mjs`;
|
|
const VISION_WASM = `/vendor/mediapipe/tasks-vision/${VISION_VERSION}/wasm`;
|
|
|
|
const CONNECTIONS = [
|
|
[0, 1], [1, 2], [2, 3], [3, 4],
|
|
[0, 5], [5, 6], [6, 7], [7, 8],
|
|
[5, 9], [9, 10], [10, 11], [11, 12],
|
|
[9, 13], [13, 14], [14, 15], [15, 16],
|
|
[13, 17], [0, 17], [17, 18], [18, 19], [19, 20],
|
|
];
|
|
|
|
const LONG_FINGERS = [[5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], [17, 18, 19, 20]];
|
|
const TOKEN_NAMES = {
|
|
OTHER: "未识别",
|
|
OPEN_PALM: "张开手掌",
|
|
CLOSED_FIST: "握拳",
|
|
POINT_UP: "食指向上",
|
|
POINT_DOWN: "食指向下",
|
|
POINT_RIGHT: "食指向右",
|
|
POINT_LEFT: "食指向左",
|
|
THUMB_UP: "拇指向上",
|
|
THUMB_DOWN: "拇指向下",
|
|
GUN_UP: "手枪手势向上",
|
|
GUN_DOWN: "手枪手势向下",
|
|
VICTORY: "V 手势",
|
|
VICTORY_RIGHT: "V 手势向右",
|
|
VICTORY_LEFT: "V 手势向左",
|
|
I_LOVE_YOU: "ILY 手势",
|
|
};
|
|
|
|
const ACTIONS = {
|
|
SEAT_UP: { label: "坐垫前端升高", short: "坐垫 ↑", utterance: "副驾驶座椅坐垫前端升高一点" },
|
|
SEAT_DOWN: { label: "坐垫前端降低", short: "坐垫 ↓", utterance: "副驾驶座椅坐垫前端降低一点" },
|
|
SEAT_FORWARD: { label: "座椅向前", short: "座椅 →", utterance: "副驾驶座椅向前移动一点" },
|
|
SEAT_BACKWARD: { label: "座椅向后", short: "座椅 ←", utterance: "副驾驶座椅向后移动一点" },
|
|
BACKREST_FORWARD: { label: "靠背向前", short: "靠背 ↑", utterance: "副驾驶靠背向前调一点" },
|
|
BACKREST_BACKWARD: { label: "靠背向后", short: "靠背 ↓", utterance: "副驾驶靠背向后调一点" },
|
|
LEGREST_EXTEND: { label: "腿托伸出", short: "腿托伸", utterance: "副驾驶腿托伸出一点" },
|
|
LEGREST_RETRACT: { label: "腿托收回", short: "腿托收", utterance: "副驾驶腿托收回一点" },
|
|
};
|
|
|
|
const GUIDE_TOKENS = [
|
|
"POINT_UP", "POINT_DOWN", "POINT_RIGHT", "POINT_LEFT",
|
|
"THUMB_UP", "THUMB_DOWN", "GUN_UP", "GUN_DOWN",
|
|
];
|
|
|
|
const state = {
|
|
runtime: null,
|
|
recognizer: null,
|
|
stream: null,
|
|
frameRequest: null,
|
|
lastVideoTime: -1,
|
|
lastRecognizedToken: null,
|
|
customMapping: loadCustomMapping(),
|
|
customizeOpen: false,
|
|
engine: null,
|
|
};
|
|
|
|
const $ = (selector) => document.querySelector(selector);
|
|
|
|
function loadCustomMapping() {
|
|
try {
|
|
const value = JSON.parse(localStorage.getItem("cognitive-agent.gesture-mapping") || "{}");
|
|
return value && typeof value === "object" ? value : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
function notify(message) {
|
|
if (window.CognitiveAgentConsole?.notify) {
|
|
window.CognitiveAgentConsole.notify(message);
|
|
return;
|
|
}
|
|
const toast = $("#toast");
|
|
if (!toast) return;
|
|
toast.textContent = message;
|
|
toast.classList.add("visible");
|
|
window.setTimeout(() => toast.classList.remove("visible"), 2600);
|
|
}
|
|
|
|
function distance(a, b) {
|
|
return Math.hypot(a.x - b.x, a.y - b.y);
|
|
}
|
|
|
|
function angle(a, vertex, c) {
|
|
const first = [a.x - vertex.x, a.y - vertex.y];
|
|
const second = [c.x - vertex.x, c.y - vertex.y];
|
|
const denominator = Math.hypot(...first) * Math.hypot(...second);
|
|
if (denominator <= 1e-9) return 0;
|
|
const cosine = Math.max(-1, Math.min(1, (first[0] * second[0] + first[1] * second[1]) / denominator));
|
|
return Math.acos(cosine) * 180 / Math.PI;
|
|
}
|
|
|
|
function fingerExtended(points, [mcpIndex, pipIndex, dipIndex, tipIndex]) {
|
|
const [mcp, pip, dip, tip] = [mcpIndex, pipIndex, dipIndex, tipIndex].map((index) => points[index]);
|
|
return angle(mcp, pip, dip) >= 150
|
|
&& angle(pip, dip, tip) >= 145
|
|
&& distance(points[0], tip) >= distance(points[0], pip) * 1.08;
|
|
}
|
|
|
|
function fingerStates(points) {
|
|
return LONG_FINGERS.map((indices) => fingerExtended(points, indices));
|
|
}
|
|
|
|
function plausibleHand(points) {
|
|
if (!Array.isArray(points) || points.length !== 21) return false;
|
|
if (points.some((point) => !Number.isFinite(point.x) || !Number.isFinite(point.y) || point.x < -0.15 || point.x > 1.15 || point.y < -0.15 || point.y > 1.15)) return false;
|
|
const xs = points.map((point) => point.x);
|
|
const ys = points.map((point) => point.y);
|
|
if (Math.max(Math.max(...xs) - Math.min(...xs), Math.max(...ys) - Math.min(...ys)) < 0.12) return false;
|
|
const palmWidth = distance(points[5], points[17]);
|
|
if (palmWidth < 0.025) return false;
|
|
const edges = [distance(points[5], points[9]), distance(points[9], points[13]), distance(points[13], points[17])];
|
|
if (Math.min(...edges) < 0.006 || Math.max(...edges) / Math.min(...edges) > 3.5) return false;
|
|
return CONNECTIONS.filter(([first, second]) => distance(points[first], points[second]) > 0.004).length >= 16;
|
|
}
|
|
|
|
function pinchRatio(points) {
|
|
return distance(points[4], points[8]) / Math.max(distance(points[5], points[17]), 1e-6);
|
|
}
|
|
|
|
function pointingDirection(points) {
|
|
const [index, middle, ring, pinky] = fingerStates(points);
|
|
if (!index || middle || ring || pinky) return null;
|
|
const origin = points[5];
|
|
const tip = points[8];
|
|
// The camera preview is mirrored, so horizontal intent follows display space.
|
|
const dx = -(tip.x - origin.x);
|
|
const dy = tip.y - origin.y;
|
|
if (Math.hypot(dx, dy) < distance(points[5], points[17]) * 0.65) return null;
|
|
if (Math.abs(dx) >= Math.abs(dy)) return dx > 0 ? "RIGHT" : "LEFT";
|
|
return dy > 0 ? "DOWN" : "UP";
|
|
}
|
|
|
|
function thumbIsExtended(points) {
|
|
if (!Array.isArray(points) || points.length !== 21) return false;
|
|
const palmWidth = Math.max(distance(points[5], points[17]), 1e-6);
|
|
const thumbLength = distance(points[2], points[4]);
|
|
return angle(points[1], points[2], points[3]) >= 145
|
|
&& angle(points[2], points[3], points[4]) >= 145
|
|
&& thumbLength >= palmWidth * 0.65;
|
|
}
|
|
|
|
function thumbVerticalDirection(points) {
|
|
if (fingerStates(points).some(Boolean)) return null;
|
|
const dx = points[4].x - points[2].x;
|
|
const dy = points[4].y - points[2].y;
|
|
const length = Math.hypot(dx, dy);
|
|
const palmWidth = Math.max(distance(points[5], points[17]), 1e-6);
|
|
if (length < palmWidth * 0.65 || Math.abs(dy) < Math.abs(dx) * 1.6 || !thumbIsExtended(points)) return null;
|
|
return dy > 0 ? "DOWN" : "UP";
|
|
}
|
|
|
|
function gunDirection(points) {
|
|
const direction = pointingDirection(points);
|
|
if (!direction || !["UP", "DOWN"].includes(direction)) return null;
|
|
return thumbIsExtended(points) ? direction : null;
|
|
}
|
|
|
|
function victoryDirection(points) {
|
|
const states = fingerStates(points);
|
|
if (states.join(",") !== "true,true,false,false") return null;
|
|
const originX = (points[5].x + points[9].x) / 2;
|
|
const originY = (points[5].y + points[9].y) / 2;
|
|
const tipX = (points[8].x + points[12].x) / 2;
|
|
const tipY = (points[8].y + points[12].y) / 2;
|
|
const dx = -(tipX - originX);
|
|
const dy = tipY - originY;
|
|
if (Math.hypot(dx, dy) < distance(points[5], points[17]) * 0.65 || Math.abs(dx) < Math.abs(dy) * 1.25) return null;
|
|
return dx > 0 ? "RIGHT" : "LEFT";
|
|
}
|
|
|
|
function geometryGesture(points) {
|
|
if (!plausibleHand(points)) return null;
|
|
const states = fingerStates(points);
|
|
const pinch = pinchRatio(points);
|
|
const extendedThumb = thumbIsExtended(points);
|
|
const gun = gunDirection(points);
|
|
if (gun) return `GUN_${gun}`;
|
|
const point = pointingDirection(points);
|
|
if (point && !extendedThumb && pinch >= 0.75) return `POINT_${point}`;
|
|
if (states.join(",") === "true,true,false,false" && pinch >= 1.1) {
|
|
const direction = victoryDirection(points);
|
|
return direction ? `VICTORY_${direction}` : "VICTORY";
|
|
}
|
|
if (states.join(",") === "true,false,false,true" && pinch >= 1.1) return "I_LOVE_YOU";
|
|
if (states.every(Boolean) && pinch >= 1.0) return "OPEN_PALM";
|
|
if (states.every((value) => !value)) {
|
|
const thumb = thumbVerticalDirection(points);
|
|
if (thumb && pinch >= 0.72) return `THUMB_${thumb}`;
|
|
if (!extendedThumb && pinch <= 0.62) return "CLOSED_FIST";
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function normalizedCategory(value) {
|
|
return String(value || "None").trim().toLowerCase().replaceAll("-", "_").replaceAll(" ", "_");
|
|
}
|
|
|
|
function classifyHand(points, categories = []) {
|
|
const geometry = geometryGesture(points);
|
|
const top = categories[0] || {};
|
|
const runnerUp = categories[1] || {};
|
|
const name = normalizedCategory(top.categoryName || top.displayName);
|
|
const score = Number(top.score || 0);
|
|
const margin = Math.max(0, score - Number(runnerUp.score || 0));
|
|
if (!geometry) return { token: "OTHER", score, reason: "手型未通过严格几何校验" };
|
|
|
|
if (geometry.startsWith("GUN_")) {
|
|
return ["pointing_up", "thumb_up", "thumb_down", "none", ""].includes(name)
|
|
? { token: geometry, score: Math.max(score, 0.78), reason: "模型与手枪几何一致" }
|
|
: { token: "OTHER", score, reason: "模型类别与手枪几何冲突" };
|
|
}
|
|
if (geometry.startsWith("POINT_")) {
|
|
return ["pointing_up", "none", ""].includes(name)
|
|
? { token: geometry, score: Math.max(score, 0.75), reason: "严格指向几何" }
|
|
: { token: "OTHER", score, reason: "模型类别与指向几何冲突" };
|
|
}
|
|
if (["VICTORY_LEFT", "VICTORY_RIGHT"].includes(geometry)) {
|
|
return ["victory", "none", ""].includes(name)
|
|
? { token: geometry, score: Math.max(score, 0.75), reason: "严格 V 字方向几何" }
|
|
: { token: "OTHER", score, reason: "模型类别与 V 字方向冲突" };
|
|
}
|
|
if (["THUMB_UP", "THUMB_DOWN"].includes(geometry)) {
|
|
const expected = geometry.toLowerCase();
|
|
return (["none", "", expected].includes(name) && (name === "none" || name === "" || score >= 0.6))
|
|
? { token: geometry, score: Math.max(score, 0.6), reason: "模型与拇指几何一致" }
|
|
: { token: "OTHER", score, reason: "模型类别与拇指几何冲突" };
|
|
}
|
|
const expectedNames = {
|
|
OPEN_PALM: "open_palm", CLOSED_FIST: "closed_fist", VICTORY: "victory", I_LOVE_YOU: "iloveyou",
|
|
};
|
|
const threshold = geometry === "OPEN_PALM" ? 0.62 : 0.7;
|
|
if (name !== expectedNames[geometry] || score < threshold || (Number(runnerUp.score || 0) > 0 && margin < 0.12)) {
|
|
return { token: "OTHER", score, reason: "模型类别、分数或几何不一致" };
|
|
}
|
|
return { token: geometry, score, reason: "模型与几何一致" };
|
|
}
|
|
|
|
function mappingForToken(token) {
|
|
const defaultAction = state.runtime?.actions?.[token]?.action;
|
|
return state.customMapping[token] || defaultAction || null;
|
|
}
|
|
|
|
class GestureEngine {
|
|
constructor(config, onAction) {
|
|
this.config = config;
|
|
this.onAction = onAction;
|
|
this.armed = false;
|
|
this.currentAction = null;
|
|
this.candidate = null;
|
|
this.candidateSince = 0;
|
|
this.candidateFrames = 0;
|
|
this.lastActionAt = 0;
|
|
this.motionStartedAt = 0;
|
|
this.armSince = 0;
|
|
this.armFrames = 0;
|
|
this.neutralSince = 0;
|
|
this.lockedAction = null;
|
|
this.fistLatched = false;
|
|
}
|
|
|
|
resetCandidate() {
|
|
this.candidate = null;
|
|
this.candidateSince = 0;
|
|
this.candidateFrames = 0;
|
|
}
|
|
|
|
setArmed(value, now = performance.now()) {
|
|
this.armed = value;
|
|
this.currentAction = null;
|
|
this.motionStartedAt = 0;
|
|
this.lockedAction = null;
|
|
this.resetCandidate();
|
|
this.lastActionAt = now;
|
|
updateEngineUI(this, value ? "已激活，可直接切换动作" : "已锁止，张开手掌激活");
|
|
}
|
|
|
|
stopMotion(message) {
|
|
this.currentAction = null;
|
|
this.motionStartedAt = 0;
|
|
this.resetCandidate();
|
|
updateEngineUI(this, message);
|
|
}
|
|
|
|
update(poses, now) {
|
|
const fist = poses.some((pose) => pose.token === "CLOSED_FIST");
|
|
if (fist) {
|
|
if (!this.fistLatched || this.currentAction) this.stopMotion("握拳已停止，控制保持激活");
|
|
this.fistLatched = true;
|
|
if (this.armed && now - this.lastActionAt >= this.config.armed_idle_timeout_ms) this.setArmed(false, now);
|
|
return;
|
|
}
|
|
this.fistLatched = false;
|
|
|
|
const primary = poses.length === 1 ? poses[0] : null;
|
|
if (!this.armed) {
|
|
if (primary?.token === "OPEN_PALM") {
|
|
if (!this.armSince) {
|
|
this.armSince = now;
|
|
this.armFrames = 1;
|
|
} else {
|
|
this.armFrames += 1;
|
|
}
|
|
const progress = Math.min(1, (now - this.armSince) / this.config.arm_hold_ms);
|
|
setGestureProgress(progress);
|
|
if (progress >= 1 && this.armFrames >= this.config.min_action_frames) {
|
|
this.armSince = 0;
|
|
this.armFrames = 0;
|
|
this.setArmed(true, now);
|
|
notify("手势控制已激活");
|
|
}
|
|
} else {
|
|
this.armSince = 0;
|
|
this.armFrames = 0;
|
|
setGestureProgress(0);
|
|
}
|
|
return;
|
|
}
|
|
|
|
const action = primary ? mappingForToken(primary.token) : null;
|
|
if (!action) {
|
|
if (this.currentAction) this.stopMotion("动作已停止，控制保持激活");
|
|
this.resetCandidate();
|
|
if (this.lockedAction) {
|
|
if (!this.neutralSince) this.neutralSince = now;
|
|
if (now - this.neutralSince >= this.config.neutral_release_ms) this.lockedAction = null;
|
|
}
|
|
if (now - this.lastActionAt >= this.config.armed_idle_timeout_ms) {
|
|
this.setArmed(false, now);
|
|
notify("连续 3 秒无有效动作，手势控制已安全锁止");
|
|
}
|
|
setGestureProgress(0);
|
|
return;
|
|
}
|
|
|
|
this.lastActionAt = now;
|
|
this.neutralSince = 0;
|
|
if (this.lockedAction && action !== this.lockedAction) this.lockedAction = null;
|
|
if (this.lockedAction === action) {
|
|
updateEngineUI(this, "该动作已达到单次时限，请先释放手势");
|
|
setGestureProgress(0);
|
|
return;
|
|
}
|
|
if (this.currentAction && action !== this.currentAction) this.stopMotion("正在确认新动作…");
|
|
|
|
if (this.currentAction === action) {
|
|
setGestureProgress(1);
|
|
if (now - this.motionStartedAt >= this.config.max_continuous_motion_ms) {
|
|
this.lockedAction = action;
|
|
this.stopMotion("动作达到 1.5 秒上限，请释放或更换动作");
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (this.candidate !== action) {
|
|
this.candidate = action;
|
|
this.candidateSince = now;
|
|
this.candidateFrames = 1;
|
|
} else {
|
|
this.candidateFrames += 1;
|
|
}
|
|
const progress = Math.min(1, (now - this.candidateSince) / this.config.action_hold_ms);
|
|
setGestureProgress(progress);
|
|
updateEngineUI(this, `保持手势 · ${Math.round(progress * 100)}%`);
|
|
if (progress >= 1 && this.candidateFrames >= this.config.min_action_frames) {
|
|
this.currentAction = action;
|
|
this.motionStartedAt = now;
|
|
this.resetCandidate();
|
|
updateEngineUI(this, `${ACTIONS[action]?.label || action} · 已执行`);
|
|
this.onAction(action);
|
|
}
|
|
}
|
|
}
|
|
|
|
function setGestureProgress(value) {
|
|
const bar = $("#gestureProgressBar");
|
|
if (bar) bar.style.transform = `scaleX(${Math.max(0, Math.min(1, value))})`;
|
|
}
|
|
|
|
function updateEngineUI(engine, message) {
|
|
const pill = $("#gestureStatePill");
|
|
if (!pill) return;
|
|
pill.className = `gesture-state-pill ${engine.armed ? (engine.currentAction ? "moving" : "armed") : "idle"}`;
|
|
pill.querySelector("b").textContent = engine.armed ? (engine.currentAction ? "执行中" : "已激活") : "已锁止";
|
|
$("#gestureSafetyText").textContent = message;
|
|
$("#gestureArmButton").textContent = engine.armed ? "已激活" : "激活控制";
|
|
}
|
|
|
|
function executeGestureAction(action) {
|
|
const definition = ACTIONS[action];
|
|
if (!definition) return;
|
|
const executionMode = window.CognitiveAgentConsole?.getExecutionMode?.() || "dry_run";
|
|
const autoSubmit = executionMode !== "hardware";
|
|
const accepted = window.CognitiveAgentConsole?.executeUtterance?.(definition.utterance, { autoSubmit });
|
|
if (accepted === false) {
|
|
notify("上一条指令仍在处理中，本次手势未重复提交");
|
|
} else if (!autoSubmit) {
|
|
notify("真实台架模式需要人工授权，指令已填入输入框");
|
|
} else {
|
|
notify(`手势指令：${definition.label}`);
|
|
}
|
|
}
|
|
|
|
function drawLandmarks(result) {
|
|
const canvas = $("#gestureCanvas");
|
|
const video = $("#gestureVideo");
|
|
if (!canvas || !video.videoWidth) return;
|
|
if (canvas.width !== video.videoWidth || canvas.height !== video.videoHeight) {
|
|
canvas.width = video.videoWidth;
|
|
canvas.height = video.videoHeight;
|
|
}
|
|
const context = canvas.getContext("2d");
|
|
context.clearRect(0, 0, canvas.width, canvas.height);
|
|
for (const points of result.landmarks || []) {
|
|
context.lineWidth = Math.max(2, canvas.width / 360);
|
|
context.strokeStyle = "rgba(114, 255, 196, .9)";
|
|
for (const [first, second] of CONNECTIONS) {
|
|
context.beginPath();
|
|
context.moveTo(points[first].x * canvas.width, points[first].y * canvas.height);
|
|
context.lineTo(points[second].x * canvas.width, points[second].y * canvas.height);
|
|
context.stroke();
|
|
}
|
|
context.fillStyle = "#f8ff73";
|
|
for (const point of points) {
|
|
context.beginPath();
|
|
context.arc(point.x * canvas.width, point.y * canvas.height, Math.max(3, canvas.width / 230), 0, Math.PI * 2);
|
|
context.fill();
|
|
}
|
|
}
|
|
}
|
|
|
|
function renderRecognition(poses) {
|
|
const detected = $("#gestureDetected");
|
|
const expected = $("#gestureExpected");
|
|
if (!poses.length) {
|
|
detected.textContent = "未检测到手";
|
|
expected.textContent = state.engine?.armed ? "保持激活" : "张掌激活";
|
|
return;
|
|
}
|
|
detected.textContent = poses.map((pose) => TOKEN_NAMES[pose.token] || pose.token).join(" + ");
|
|
const action = poses.length === 1 ? mappingForToken(poses[0].token) : null;
|
|
expected.textContent = action ? ACTIONS[action]?.label || action : poses.length > 1 ? "多手画面已锁止" : poses[0].reason;
|
|
const usable = poses.find((pose) => !["OTHER", "OPEN_PALM", "CLOSED_FIST"].includes(pose.token));
|
|
if (usable) state.lastRecognizedToken = usable.token;
|
|
if (state.customizeOpen) updateCustomizeToken();
|
|
}
|
|
|
|
async function recognizeLoop() {
|
|
if (!state.stream || !state.recognizer) return;
|
|
const video = $("#gestureVideo");
|
|
const now = performance.now();
|
|
if (video.readyState >= 2 && video.currentTime !== state.lastVideoTime) {
|
|
state.lastVideoTime = video.currentTime;
|
|
const result = state.recognizer.recognizeForVideo(video, now);
|
|
drawLandmarks(result);
|
|
const poses = (result.landmarks || []).map((points, index) => classifyHand(points, result.gestures?.[index] || []));
|
|
renderRecognition(poses);
|
|
state.engine.update(poses, now);
|
|
}
|
|
state.frameRequest = requestAnimationFrame(recognizeLoop);
|
|
}
|
|
|
|
async function loadGestureRuntime() {
|
|
const response = await window.CognitiveAgentNetwork.request("/api/gesture/runtime", { cache: "no-store" });
|
|
if (!response.ok) throw new Error("无法读取手势能力配置");
|
|
state.runtime = await response.json();
|
|
state.engine = new GestureEngine(state.runtime.safety, executeGestureAction);
|
|
renderMappingGrid();
|
|
if (!state.runtime.model_available) throw new Error("内置手势模型不可用，请检查工程资源");
|
|
}
|
|
|
|
async function loadRecognizer() {
|
|
if (state.recognizer) return;
|
|
$("#gestureSafetyText").textContent = "正在加载手势模型…";
|
|
const { FilesetResolver, GestureRecognizer } = await import(VISION_MODULE);
|
|
const fileset = await FilesetResolver.forVisionTasks(VISION_WASM);
|
|
const modelResponse = await window.CognitiveAgentNetwork.request("/api/gesture/model", {
|
|
cache: "force-cache",
|
|
timeoutMs: 60000,
|
|
});
|
|
if (!modelResponse.ok) throw new Error("内置手势模型加载失败");
|
|
const modelAssetBuffer = new Uint8Array(await modelResponse.arrayBuffer());
|
|
state.recognizer = await GestureRecognizer.createFromOptions(fileset, {
|
|
baseOptions: { modelAssetBuffer, delegate: "CPU" },
|
|
runningMode: "VIDEO",
|
|
numHands: 2,
|
|
minHandDetectionConfidence: 0.65,
|
|
minHandPresenceConfidence: 0.65,
|
|
minTrackingConfidence: 0.60,
|
|
});
|
|
}
|
|
|
|
async function startCamera() {
|
|
const button = $("#gestureStartButton");
|
|
button.disabled = true;
|
|
button.querySelector("span").textContent = "正在启动…";
|
|
try {
|
|
if (!state.runtime) await loadGestureRuntime();
|
|
await loadRecognizer();
|
|
state.stream = await navigator.mediaDevices.getUserMedia({
|
|
video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: "user" },
|
|
audio: false,
|
|
});
|
|
const video = $("#gestureVideo");
|
|
video.srcObject = state.stream;
|
|
await video.play();
|
|
$("#gestureCameraEmpty").hidden = true;
|
|
$("#gestureLiveBadge").hidden = false;
|
|
$("#gestureArmButton").disabled = false;
|
|
$("#gestureStopButton").disabled = false;
|
|
button.classList.add("active");
|
|
button.querySelector("span").textContent = "关闭摄像头";
|
|
$("#gestureSafetyText").textContent = "张开手掌并保持 1 秒以激活";
|
|
state.frameRequest = requestAnimationFrame(recognizeLoop);
|
|
} catch (error) {
|
|
$("#gestureSafetyText").textContent = error.message || "摄像头启动失败";
|
|
notify(error.message || "摄像头启动失败");
|
|
stopCamera();
|
|
} finally {
|
|
button.disabled = false;
|
|
}
|
|
}
|
|
|
|
function stopCamera() {
|
|
if (state.frameRequest) cancelAnimationFrame(state.frameRequest);
|
|
state.frameRequest = null;
|
|
state.stream?.getTracks().forEach((track) => track.stop());
|
|
state.stream = null;
|
|
state.lastVideoTime = -1;
|
|
const video = $("#gestureVideo");
|
|
if (video) video.srcObject = null;
|
|
const canvas = $("#gestureCanvas");
|
|
canvas?.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height);
|
|
$("#gestureCameraEmpty").hidden = false;
|
|
$("#gestureLiveBadge").hidden = true;
|
|
$("#gestureArmButton").disabled = true;
|
|
$("#gestureStopButton").disabled = true;
|
|
const button = $("#gestureStartButton");
|
|
button.classList.remove("active");
|
|
button.querySelector("span").textContent = "开启摄像头";
|
|
state.engine?.setArmed(false);
|
|
$("#gestureDetected").textContent = "—";
|
|
$("#gestureExpected").textContent = "等待启动";
|
|
$("#gestureSafetyText").textContent = "摄像头尚未开启";
|
|
}
|
|
|
|
function toggleCamera() {
|
|
if (state.stream) stopCamera(); else startCamera();
|
|
}
|
|
|
|
function renderMappingGrid() {
|
|
const grid = $("#gestureMapGrid");
|
|
if (!grid || !state.runtime) return;
|
|
grid.innerHTML = GUIDE_TOKENS.map((token) => {
|
|
const action = mappingForToken(token);
|
|
return `<div><span>${TOKEN_NAMES[token]}</span><i>→</i><b>${ACTIONS[action]?.short || "未绑定"}</b></div>`;
|
|
}).join("");
|
|
const select = $("#gestureActionSelect");
|
|
select.innerHTML = Object.entries(ACTIONS).map(([value, item]) => `<option value="${value}">${item.label}</option>`).join("");
|
|
}
|
|
|
|
function updateCustomizeToken() {
|
|
const token = state.lastRecognizedToken;
|
|
$("#gestureCustomizeToken").textContent = token ? TOKEN_NAMES[token] || token : "请先摆出手势";
|
|
if (token) {
|
|
const action = mappingForToken(token);
|
|
if (action) $("#gestureActionSelect").value = action;
|
|
}
|
|
}
|
|
|
|
function toggleCustomize() {
|
|
state.customizeOpen = !state.customizeOpen;
|
|
$("#gestureCustomizePanel").hidden = !state.customizeOpen;
|
|
$("#gestureCustomizeButton").classList.toggle("active", state.customizeOpen);
|
|
updateCustomizeToken();
|
|
}
|
|
|
|
function bindCurrentGesture() {
|
|
const token = state.lastRecognizedToken;
|
|
if (!token || ["OTHER", "OPEN_PALM", "CLOSED_FIST"].includes(token)) {
|
|
notify("请先保持一个可自定义的有效手势");
|
|
return;
|
|
}
|
|
state.customMapping[token] = $("#gestureActionSelect").value;
|
|
localStorage.setItem("cognitive-agent.gesture-mapping", JSON.stringify(state.customMapping));
|
|
renderMappingGrid();
|
|
notify(`${TOKEN_NAMES[token] || token} 的功能已保存`);
|
|
}
|
|
|
|
function resetMapping() {
|
|
state.customMapping = {};
|
|
localStorage.removeItem("cognitive-agent.gesture-mapping");
|
|
renderMappingGrid();
|
|
notify("已恢复默认手势映射");
|
|
}
|
|
|
|
function openDrawer() {
|
|
$("#gestureDrawer").classList.add("open");
|
|
$("#gestureDrawer").setAttribute("aria-hidden", "false");
|
|
$("#gestureBackdrop").hidden = false;
|
|
requestAnimationFrame(() => $("#gestureBackdrop").classList.add("visible"));
|
|
$("#gestureToggleButton").setAttribute("aria-expanded", "true");
|
|
document.body.classList.add("gesture-open");
|
|
}
|
|
|
|
function closeDrawer() {
|
|
stopCamera();
|
|
$("#gestureDrawer").classList.remove("open");
|
|
$("#gestureDrawer").setAttribute("aria-hidden", "true");
|
|
$("#gestureBackdrop").classList.remove("visible");
|
|
window.setTimeout(() => { $("#gestureBackdrop").hidden = true; }, 240);
|
|
$("#gestureToggleButton").setAttribute("aria-expanded", "false");
|
|
document.body.classList.remove("gesture-open");
|
|
}
|
|
|
|
function initModeRail() {
|
|
document.querySelectorAll("[data-mode-command]").forEach((button) => button.addEventListener("click", () => {
|
|
window.CognitiveAgentConsole?.executeUtterance?.(button.dataset.modeCommand, { autoSubmit: true });
|
|
}));
|
|
$("#saveModeToggle")?.addEventListener("click", () => {
|
|
const composer = $("#saveModeComposer");
|
|
composer.hidden = !composer.hidden;
|
|
if (!composer.hidden) $("#saveModeName").focus();
|
|
});
|
|
$("#saveModeConfirm")?.addEventListener("click", () => {
|
|
const name = $("#saveModeName").value.trim();
|
|
if (!name) return notify("请先输入模式名称");
|
|
window.CognitiveAgentConsole?.executeUtterance?.(`将当前副驾驶座椅姿态保存为${name}模式`, { autoSubmit: true });
|
|
$("#saveModeComposer").hidden = true;
|
|
$("#saveModeName").value = "";
|
|
});
|
|
}
|
|
|
|
document.addEventListener("DOMContentLoaded", async () => {
|
|
$("#gestureToggleButton")?.addEventListener("click", openDrawer);
|
|
$("#gestureCloseButton")?.addEventListener("click", closeDrawer);
|
|
$("#gestureBackdrop")?.addEventListener("click", closeDrawer);
|
|
$("#gestureStartButton")?.addEventListener("click", toggleCamera);
|
|
$("#gestureArmButton")?.addEventListener("click", () => state.engine?.setArmed(!state.engine.armed));
|
|
$("#gestureStopButton")?.addEventListener("click", () => state.engine?.setArmed(false));
|
|
$("#gestureCustomizeButton")?.addEventListener("click", toggleCustomize);
|
|
$("#gestureBindButton")?.addEventListener("click", bindCurrentGesture);
|
|
$("#gestureResetMapping")?.addEventListener("click", resetMapping);
|
|
document.addEventListener("keydown", (event) => {
|
|
if (event.key === "Escape" && $("#gestureDrawer")?.classList.contains("open")) closeDrawer();
|
|
});
|
|
document.addEventListener("visibilitychange", () => { if (document.hidden && state.stream) stopCamera(); });
|
|
window.addEventListener("beforeunload", stopCamera);
|
|
initModeRail();
|
|
try {
|
|
await loadGestureRuntime();
|
|
$("#gestureSafetyText").textContent = state.runtime.available ? "能力就绪，开启摄像头开始识别" : "内置手势模型不可用";
|
|
} catch (error) {
|
|
$("#gestureSafetyText").textContent = error.message;
|
|
$("#gestureStartButton").disabled = true;
|
|
}
|
|
});
|