Hi, I have a project running with the face-landmarks-detection-from model in version v0.0.3 and I have already done some of the work to upgrade it to version v1.0.2.
The problem I have is how they calculate and use the iris points in the normalisation and mapping of the cursor coordinates to the screen space.
This is my code
DrawService.ts
import { TRIANGULATION } from '../Utilities/Constans';
const drawPath = (
ctx: CanvasRenderingContext2D,
points: [number, number][],
closePath: boolean
): void => {
const region = new Path2D();
region.moveTo(points[0][0], points[0][1]);
for (let i = 1; i < points.length; i++) {
const point: number[] = points[i];
region.lineTo(point[0], point[1]);
}
if (closePath) {
region.closePath();
}
ctx.strokeStyle = 'grey';
ctx.stroke(region);
};
export const drawBox = (predictions: any[], ctx: CanvasRenderingContext2D): void => {
// console.log(predictions, 'predictions');
if (predictions.length > 0) {
for (let i = 0; i < predictions.length; i++) {
const start: any = predictions[i].topLeft;
const end: any = predictions[i].bottomRight;
const size: [number, number] = [end[0] - start[0], end[1] - start[1]];
ctx.beginPath();
ctx.lineWidth = 6;
ctx.strokeStyle = 'red';
ctx.rect(start[0], start[1], size[0], size[1]);
ctx.stroke();
}
}
};
export const drawMesh = (predictions: any[], ctx: CanvasRenderingContext2D): void => {
if (predictions.length > 0) {
for (let i = 0; i < predictions.length; i++) {
const keypoints = predictions[0].keypoints;
for (let j = 0; j < TRIANGULATION.length / 3; j++) {
const pointIndexes: number[] = [
TRIANGULATION[j * 3],
TRIANGULATION[j * 3 + 1],
TRIANGULATION[j * 3 + 2]
];
// console.log(pointIndexes, 'pointIndexes');
// console.log(keypoints, 'keypoints');
if (pointIndexes.every((index) => index < keypoints.length)) {
const points: [number, number][] = pointIndexes.map((index) => {
const keypoint = keypoints[index];
return [keypoint[0], keypoint[1]];
});
drawPath(ctx, points, true);
} else {
console.error('One or more TRIANGULATION indices are outside the keypoint boundaries');
}
}
}
}
};
DetectorService.ts
import Webcam from 'react-webcam';
import * as tf from '@tensorflow/tfjs';
import * as FaceLandmarksDetector from '@tensorflow-models/face-landmarks-detection';
import {
FACEMESHCONFIG,
REFRESH,
XPOINTERMINVALUE,
XPOINTERMAXVALUE,
SLOPEVALUE,
FACERANGE
} from '../Utilities/Constans';
import { drawMesh } from './DrawService';
import { Events } from '../Models/Events';
import { AnnotatedPrediction, FaceMesh } from '@tensorflow-models/facemesh';
import React from 'react';
export const detectorMainService = async (
webcamRef: React.RefObject<Webcam>,
canvasRef: React.RefObject<HTMLCanvasElement>,
eventHandler: (event: Events, complete: boolean) => void
): Promise<void> => {
await tf.setBackend('webgl');
const detector: any = await FaceLandmarksDetector.createDetector(
FaceLandmarksDetector.SupportedModels.MediaPipeFaceMesh,
FACEMESHCONFIG
);
console.log('llego');
let complete: boolean = false;
if (tf.getBackend() === 'webgl') complete = true;
setInterval(async () => {
await meshRecognition(detector, webcamRef, canvasRef).then((event: Events) => {
eventHandler(event, complete);
});
}, REFRESH);
};
const meshRecognition = async (
detector: FaceMesh,
webcamRef: React.RefObject<Webcam>,
canvasRef: React.RefObject<HTMLCanvasElement>
): Promise<Events> => {
if (webcamRef.current && canvasRef.current && webcamRef.current.video?.readyState === 4) {
const video: HTMLVideoElement = webcamRef.current.video;
const videoWidth: number = video.videoWidth;
const videoHeight: number = video.videoHeight;
const predictions: AnnotatedPrediction[] = await detector.estimateFaces(video, false, false);
video.width = videoWidth;
video.height = videoHeight;
canvasRef.current.width = videoWidth;
canvasRef.current.height = videoHeight;
if (predictions.length > 0) {
const prediction: AnnotatedPrediction = predictions[0];
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
const box = prediction.box;
const topLeft = [box.xMin, box.yMin];
const bottomRight = [box.xMax, box.yMax];
const faceTopRightX: number = bottomRight[0];
const faceBottomLeftX: number = topLeft[0];
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
const keypoints = predictions[0].keypoints;
const positionXLeftIris = keypoints[159].x;
const positionYLeftIris = keypoints[159].y;
const midwayBetweenEyes: number[] = [
(keypoints[234].x + keypoints[454].x) / 2,
(keypoints[234].y + keypoints[454].y) / 2,
(keypoints[234].z + keypoints[454].z) / 2
];
event.isOnfRange = checkIsOnRange(
midwayBetweenEyes[0],
midwayBetweenEyes[1],
midwayBetweenEyes[2]
);
if (faceBottomLeftX > 0 && positionXLeftIris !== undefined) {
const positionLeftIrisX: number = video.width - positionXLeftIris;
const positionLeftIrisY: number = video.height - positionYLeftIris;
event.irishXPoint = irishXPointNormalize(positionLeftIrisX, faceTopRightX, faceBottomLeftX);
event.irishYPoint = irishYPointNormalize(positionLeftIrisY, faceTopRightX, faceBottomLeftX);
}
}
const ctx: CanvasRenderingContext2D | null = canvasRef.current.getContext('2d');
if (ctx) {
requestAnimationFrame(() => {
drawMesh(predictions, ctx);
});
}
}
return event;
};
const checkIsOnRange = (valueX: number, valueY: number, valueZ: number): boolean => {
let checkValueX: boolean = false;
let checkValueY: boolean = false;
//let checkValueZ: boolean = false;
//if (valueZ > FACERANGE.minValueZ && valueZ < FACERANGE.maxValueZ) checkValueZ = true;
if (valueX > FACERANGE.minValueX && valueX < FACERANGE.maxValueX) checkValueX = true;
if (valueY > FACERANGE.minValueY && valueY < FACERANGE.maxValueY) checkValueY = true;
if (checkValueX && checkValueY /*&& checkValueZ*/) {
return true;
} else {
return false;
}
};
const irishXPointNormalize = (val: number, max: number, min: number): number => {
const point: number = Math.max(0, Math.min(1, (val - min) / (max - min)));
let smoothedValue: number = smoothMapping(point);
if (point < XPOINTERMINVALUE) smoothedValue = XPOINTERMINVALUE;
if (point > XPOINTERMAXVALUE) smoothedValue = XPOINTERMAXVALUE;
return Number(smoothedValue.toFixed(4));
};
const irishYPointNormalize = (val: number, max: number, min: number): number => {
const point: number = Math.max(0, Math.min(1, (val - min) / (max - min)));
return Number(point.toFixed(4));
};
const smoothMapping = (value: number): number => {
const smoothed = sigmoide(SLOPEVALUE * (value - XPOINTERMINVALUE));
return XPOINTERMINVALUE + (XPOINTERMAXVALUE - XPOINTERMINVALUE) * smoothed;
};
const sigmoide = (value: number): number => {
return 1 / (1 + Math.exp(-value));
};
const event: Events = {
irishXPoint: 0,
irishYPoint: 0,
isOnfRange: false
};
constants.ts
import { FacemeshConfig, FaceRangeConfig } from '../Models/Constans';
import {
MediaPipeFaceMeshMediaPipeModelConfig,
MediaPipeFaceMeshTfjsModelConfig
} from '@tensorflow-models/face-landmarks-detection';
export const TRIANGULATION: number[] = [....]
export const INITIALLETTERS: string[] = [
'A',
'B',
'C',
'D',
'E',
'F',
'G',
'H',
'I',
'J',
'K',
'L',
'M',
'N',
'O',
'P',
'Q',
'R',
'S',
'T',
'U',
'V',
'W',
'X',
'Y',
'Z'
];
export const COMMANDS: string[] = ['Reiniciar', 'Espacio', 'Enter', 'Borrar', 'Borrar Todo'];
// eslint-disable-next-line @typescript-eslint/no-inferrable-types
export const REFRESH: number = 500;
export const XPOINTERMINVALUE: number = 0.336;
export const XPOINTERMAXVALUE: number = 0.37;
export const SLOPEVALUE: number = 0.5; // define la sensibilidad del cursor entre puntos X
export const YPOINTERMINVALUE: number = 0.2;
export const YPOINTERMAXVALUE: number = 0.3;
export const YPOINTER_REFERENCE: number = 0.04; //0.290; //0.860
export const DELAYANIMATION: number = 50;
export const APPNAME = 'komunicare';
export const FACERANGE: FaceRangeConfig = {
// minValueZ: -16, // desactivado
// maxValueZ: -12, // desactivado
minValueX: 270,
maxValueX: 350,
minValueY: 170,
maxValueY: 240
};
export const FACEMESHCONFIG: MediaPipeFaceMeshTfjsModelConfig = {
maxFaces: 1, // Detecta solo un rostro
runtime: 'tfjs', // Usa el modelo de detección de rostros de MediaPipe
refineLandmarks: false
// shouldLoadIrisModel: true, // No carga el modelo de detección del iris
// flipHorizontal: false, // No refleja horizontalmente la imagen de la cámara
// boundingBoxExpansionFactor: 1, // Expandir la región de la caja delimitadora en un 50% - Ejemplo 1.5
// detectionConfidence: 0.6, // Umbral de confianza para la detección de rostros
// landmarkMode: 'all', // Detectar todos los landmarks disponibles - "lite" trae menos landmark
// performanceMode: 'sustained' // Rendimiento constante sino usar fast para que sea mas rapido esto hay que verlo
//modelUrl: "https://example.com/my_custom_model", // URL personalizada para cargar el modelo
/*
anchorsConfig: {
numKeypoints: 468, // Número de puntos clave utilizados por el modelo
inputSize: [192, 192], // Tamaño de entrada del modelo
scales: [0.3, 0.5], // Escalas utilizadas para la detección
},
*/
};
I have the project running with version 0.0.3 of face-landmarks-detection with these parameters (plus the triangulation that I haven’t put here)
Any suggestions?
Thanks