jared
8bcc62b045
HTML demos for face, hand, gesture, and posture tracking using MediaPipe. Includes Python CLI tools for processing video files. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
291 lines
10 KiB
HTML
291 lines
10 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
<title>MediaPipe Hand Gesture Recognizer — Single File Demo</title>
|
|
<!-- Material Components (for button styling) -->
|
|
<link href="https://unpkg.com/material-components-web@latest/dist/material-components-web.min.css" rel="stylesheet" />
|
|
<script src="https://unpkg.com/material-components-web@latest/dist/material-components-web.min.js"></script>
|
|
|
|
<style>
|
|
/* Inlined from the CodePen CSS (Sass directives removed) */
|
|
body {
|
|
font-family: Roboto, system-ui, -apple-system, Segoe UI, Helvetica, Arial, sans-serif;
|
|
margin: 2em;
|
|
color: #3d3d3d;
|
|
--mdc-theme-primary: #007f8b;
|
|
--mdc-theme-on-primary: #f1f3f4;
|
|
}
|
|
|
|
h1 { color: #007f8b; }
|
|
h2 { clear: both; }
|
|
|
|
video {
|
|
clear: both;
|
|
display: block;
|
|
transform: rotateY(180deg);
|
|
-webkit-transform: rotateY(180deg);
|
|
-moz-transform: rotateY(180deg);
|
|
height: 280px;
|
|
}
|
|
|
|
section { opacity: 1; transition: opacity 500ms ease-in-out; }
|
|
.removed { display: none; }
|
|
.invisible { opacity: 0.2; }
|
|
|
|
.detectOnClick {
|
|
position: relative;
|
|
float: left;
|
|
width: 48%;
|
|
margin: 2% 1%;
|
|
cursor: pointer;
|
|
z-index: 0;
|
|
font-size: calc(8px + 1.2vw);
|
|
}
|
|
|
|
.videoView {
|
|
position: absolute;
|
|
float: left;
|
|
width: 48%;
|
|
margin: 2% 1%;
|
|
cursor: pointer;
|
|
min-height: 500px;
|
|
}
|
|
|
|
.videoView p,
|
|
.detectOnClick p {
|
|
padding-top: 5px;
|
|
padding-bottom: 5px;
|
|
background-color: #007f8b;
|
|
color: #fff;
|
|
border: 1px dashed rgba(255, 255, 255, 0.7);
|
|
z-index: 2;
|
|
margin: 0;
|
|
}
|
|
|
|
.highlighter { background: rgba(0, 255, 0, 0.25); border: 1px dashed #fff; z-index: 1; position: absolute; }
|
|
.canvas { z-index: 1; position: absolute; pointer-events: none; }
|
|
|
|
.output_canvas {
|
|
transform: rotateY(180deg);
|
|
-webkit-transform: rotateY(180deg);
|
|
-moz-transform: rotateY(180deg);
|
|
}
|
|
|
|
.detectOnClick img { width: 45vw; }
|
|
|
|
.output { display: none; width: 100%; font-size: calc(8px + 1.2vw); }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<section id="demos" class="invisible">
|
|
<h2><br>Demo: Webcam continuous hand gesture detection</h2>
|
|
<p>Use your hand to make gestures in front of the camera to get gesture classification. <br />Click <b>enable webcam</b> below and grant access to the webcam if prompted.</p>
|
|
<PRE>
|
|
Gesture Label Description
|
|
Closed_Fist Hand fully closed into a fist
|
|
Open_Palm Flat open hand
|
|
Pointing_Up Index finger extended upward, others closed
|
|
Thumb_Down Thumb extended downward
|
|
Thumb_Up Thumb extended upward
|
|
Victory Index and middle finger extended in a “V”
|
|
ILoveYou Thumb, index, and pinky extended (ASL “I love you”)
|
|
None No recognized gesture / below confidence threshold
|
|
</PRE>
|
|
|
|
<div id="liveView" class="videoView">
|
|
<button id="webcamButton" class="mdc-button mdc-button--raised">
|
|
<span class="mdc-button__ripple"></span>
|
|
<span class="mdc-button__label">ENABLE WEBCAM</span>
|
|
</button>
|
|
<div style="position: relative;">
|
|
<video id="webcam" autoplay playsinline></video>
|
|
<canvas class="output_canvas" id="output_canvas" width="1280" height="720" style="position: absolute; left: 0; top: 0;"></canvas>
|
|
<p id="gesture_output" class="output"></p>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<script type="module">
|
|
import { GestureRecognizer, FilesetResolver, DrawingUtils } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3";
|
|
|
|
const demosSection = document.getElementById("demos");
|
|
/** @type {GestureRecognizer} */
|
|
let gestureRecognizer;
|
|
let runningMode = "IMAGE";
|
|
/** @type {HTMLButtonElement} */
|
|
let enableWebcamButton;
|
|
let webcamRunning = false;
|
|
const videoHeight = "360px";
|
|
const videoWidth = "480px";
|
|
|
|
// Load the WASM and model, then reveal the demos section
|
|
const createGestureRecognizer = async () => {
|
|
const vision = await FilesetResolver.forVisionTasks(
|
|
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3/wasm"
|
|
);
|
|
gestureRecognizer = await GestureRecognizer.createFromOptions(vision, {
|
|
baseOptions: {
|
|
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task",
|
|
delegate: "GPU"
|
|
},
|
|
runningMode
|
|
});
|
|
demosSection.classList.remove("invisible");
|
|
};
|
|
createGestureRecognizer();
|
|
|
|
/********************************************************************
|
|
// Demo 1: Detect hand gestures in images
|
|
********************************************************************/
|
|
const imageContainers = document.getElementsByClassName("detectOnClick");
|
|
for (let i = 0; i < imageContainers.length; i++) {
|
|
const img = imageContainers[i].children[0];
|
|
img.addEventListener("click", handleClick);
|
|
}
|
|
|
|
async function handleClick(event) {
|
|
if (!gestureRecognizer) {
|
|
alert("Please wait for gestureRecognizer to load");
|
|
return;
|
|
}
|
|
|
|
if (runningMode === "VIDEO") {
|
|
runningMode = "IMAGE";
|
|
await gestureRecognizer.setOptions({ runningMode: "IMAGE" });
|
|
}
|
|
|
|
const parent = event.target.parentNode;
|
|
|
|
// Remove previous overlays
|
|
const allCanvas = parent.getElementsByClassName("canvas");
|
|
for (let i = allCanvas.length - 1; i >= 0; i--) {
|
|
const n = allCanvas[i];
|
|
n.parentNode.removeChild(n);
|
|
}
|
|
|
|
const results = gestureRecognizer.recognize(event.target);
|
|
console.log(results);
|
|
|
|
if (results.gestures && results.gestures.length > 0) {
|
|
const p = parent.querySelector(".classification");
|
|
p.classList.remove("removed");
|
|
|
|
const categoryName = results.gestures[0][0].categoryName;
|
|
const categoryScore = (results.gestures[0][0].score * 100).toFixed(2);
|
|
const handedness = results.handednesses[0][0].displayName;
|
|
|
|
p.innerText = `GestureRecognizer: ${categoryName}\n Confidence: ${categoryScore}%\n Handedness: ${handedness}`;
|
|
p.style.left = "0px";
|
|
p.style.top = event.target.height + "px";
|
|
p.style.width = event.target.width - 10 + "px";
|
|
|
|
const canvas = document.createElement("canvas");
|
|
canvas.setAttribute("class", "canvas");
|
|
canvas.setAttribute("width", event.target.naturalWidth + "px");
|
|
canvas.setAttribute("height", event.target.naturalHeight + "px");
|
|
canvas.style.left = "0px";
|
|
canvas.style.top = "0px";
|
|
canvas.style.width = event.target.width + "px";
|
|
canvas.style.height = event.target.height + "px";
|
|
|
|
parent.appendChild(canvas);
|
|
const canvasCtx = canvas.getContext("2d");
|
|
const drawingUtils = new DrawingUtils(canvasCtx);
|
|
if (results.landmarks) {
|
|
for (const landmarks of results.landmarks) {
|
|
drawingUtils.drawConnectors(landmarks, GestureRecognizer.HAND_CONNECTIONS, { lineWidth: 5 });
|
|
drawingUtils.drawLandmarks(landmarks, { lineWidth: 1 });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/********************************************************************
|
|
// Demo 2: Continuously grab image from webcam stream and detect it.
|
|
********************************************************************/
|
|
const video = document.getElementById("webcam");
|
|
const canvasElement = document.getElementById("output_canvas");
|
|
const canvasCtx = canvasElement.getContext("2d");
|
|
const gestureOutput = document.getElementById("gesture_output");
|
|
|
|
function hasGetUserMedia() {
|
|
return !!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia);
|
|
}
|
|
|
|
if (hasGetUserMedia()) {
|
|
enableWebcamButton = document.getElementById("webcamButton");
|
|
enableWebcamButton.addEventListener("click", enableCam);
|
|
} else {
|
|
console.warn("getUserMedia() is not supported by your browser");
|
|
}
|
|
|
|
function enableCam() {
|
|
if (!gestureRecognizer) {
|
|
alert("Please wait for gestureRecognizer to load");
|
|
return;
|
|
}
|
|
|
|
webcamRunning = !webcamRunning;
|
|
enableWebcamButton.innerText = webcamRunning ? "DISABLE PREDICTIONS" : "ENABLE PREDICTIONS";
|
|
|
|
const constraints = { video: true };
|
|
navigator.mediaDevices.getUserMedia(constraints).then(function (stream) {
|
|
video.srcObject = stream;
|
|
video.addEventListener("loadeddata", predictWebcam);
|
|
});
|
|
}
|
|
|
|
let lastVideoTime = -1;
|
|
let results;
|
|
async function predictWebcam() {
|
|
const webcamElement = document.getElementById("webcam");
|
|
|
|
if (runningMode === "IMAGE") {
|
|
runningMode = "VIDEO";
|
|
await gestureRecognizer.setOptions({ runningMode: "VIDEO" });
|
|
}
|
|
|
|
const nowInMs = Date.now();
|
|
if (video.currentTime !== lastVideoTime) {
|
|
lastVideoTime = video.currentTime;
|
|
results = gestureRecognizer.recognizeForVideo(video, nowInMs);
|
|
}
|
|
|
|
canvasCtx.save();
|
|
canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
|
|
const drawingUtils = new DrawingUtils(canvasCtx);
|
|
|
|
canvasElement.style.height = videoHeight;
|
|
webcamElement.style.height = videoHeight;
|
|
canvasElement.style.width = videoWidth;
|
|
webcamElement.style.width = videoWidth;
|
|
|
|
if (results && results.landmarks) {
|
|
for (const landmarks of results.landmarks) {
|
|
drawingUtils.drawConnectors(landmarks, GestureRecognizer.HAND_CONNECTIONS, { lineWidth: 5 });
|
|
drawingUtils.drawLandmarks(landmarks, { lineWidth: 2 });
|
|
}
|
|
}
|
|
canvasCtx.restore();
|
|
|
|
if (results && results.gestures && results.gestures.length > 0) {
|
|
gestureOutput.style.display = "block";
|
|
gestureOutput.style.width = videoWidth;
|
|
const categoryName = results.gestures[0][0].categoryName;
|
|
const categoryScore = (results.gestures[0][0].score * 100).toFixed(2);
|
|
const handedness = results.handednesses[0][0].displayName;
|
|
gestureOutput.innerText = `GestureRecognizer: ${categoryName}\n Confidence: ${categoryScore} %\n Handedness: ${handedness}`;
|
|
} else {
|
|
gestureOutput.style.display = "none";
|
|
}
|
|
|
|
if (webcamRunning === true) {
|
|
window.requestAnimationFrame(predictWebcam);
|
|
}
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|