From 426411ca07db203857617f385b14697c4db2f163 Mon Sep 17 00:00:00 2001
From: IEatChili <nanunguyen99@gmail.com>
Date: Thu, 1 Aug 2024 14:29:21 -0400
Subject: feat: added groundwork for face recognition

---
 src/client/views/search/FaceRecognitionHandler.tsx | 86 ++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 src/client/views/search/FaceRecognitionHandler.tsx

(limited to 'src/client/views/search/FaceRecognitionHandler.tsx')

diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx
new file mode 100644
index 000000000..86619b2d1
--- /dev/null
+++ b/src/client/views/search/FaceRecognitionHandler.tsx
@@ -0,0 +1,86 @@
+import * as faceapi from 'face-api.js';
+import { FaceMatcher, TinyFaceDetectorOptions } from 'face-api.js';
+import { Doc, DocListCast, NumListCast } from '../../../fields/Doc';
+import { DocData } from '../../../fields/DocSymbols';
+import { List } from '../../../fields/List';
+import { ObjectField } from '../../../fields/ObjectField';
+import { StrCast } from '../../../fields/Types';
+
+export class FaceRecognitionHandler {
+    static _instance: FaceRecognitionHandler;
+
+    constructor() {
+        FaceRecognitionHandler._instance = this;
+        this.loadModels();
+        if (!Doc.ActiveDashboard![DocData].faceDocuments) {
+            Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>();
+        }
+    }
+
+    async loadModels() {
+        const MODEL_URL = `/models`;
+        await faceapi.loadTinyFaceDetectorModel(MODEL_URL);
+        await faceapi.loadFaceLandmarkTinyModel(MODEL_URL);
+        await faceapi.loadFaceRecognitionModel(MODEL_URL);
+    }
+
+    public static get Instance() {
+        return FaceRecognitionHandler._instance ?? new FaceRecognitionHandler();
+    }
+
+    public async findMatches(doc: Doc, imageURL: string) {
+        const img = await this.loadImage(imageURL);
+
+        const fullFaceDescriptions = await faceapi.detectAllFaces(img, new TinyFaceDetectorOptions()).withFaceLandmarks(true).withFaceDescriptors();
+
+        fullFaceDescriptions.forEach(fd => {
+            const match = this.findMatch(fd.descriptor);
+            if (match) {
+                match[DocData].associatedDocs = new List<Doc>([...DocListCast(match[DocData].associatedDocs), doc]);
+                match[DocData].faceDescriptors = new List<List<number>>([...(match[DocData].faceDescriptors as List<List<number>>), Array.from(fd.descriptor) as List<number>]);
+            } else {
+                const newFaceDocument = new Doc();
+                const converted_array = Array.from(fd.descriptor);
+                newFaceDocument[DocData].faceDescriptors = new List<List<number>>();
+                (newFaceDocument[DocData].faceDescriptors as List<List<number>>).push(converted_array as List<number>);
+                newFaceDocument[DocData].label = `Person ${DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).length + 1}`;
+                newFaceDocument[DocData].associatedDocs = new List<Doc>([doc]);
+
+                Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>([...DocListCast(Doc.ActiveDashboard![DocData].faceDocuments), newFaceDocument]);
+            }
+        });
+    }
+
+    private findMatch(cur_descriptor: Float32Array) {
+        if (DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).length < 1) {
+            return null;
+        }
+
+        const faceDescriptors: faceapi.LabeledFaceDescriptors[] = DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).map(faceDocument => {
+            const float32Array = (faceDocument[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor)));
+            return new faceapi.LabeledFaceDescriptors(StrCast(faceDocument[DocData].label), float32Array);
+        });
+        const faceMatcher = new FaceMatcher(faceDescriptors, 0.6);
+        const match = faceMatcher.findBestMatch(cur_descriptor);
+
+        if (match.label == 'unknown') {
+            return null;
+        } else {
+            for (const doc of DocListCast(Doc.ActiveDashboard![DocData].faceDocuments)) {
+                if (doc[DocData].label === match.label) {
+                    return doc;
+                }
+            }
+        }
+    }
+
+    private loadImage = (src: string): Promise<HTMLImageElement> => {
+        return new Promise((resolve, reject) => {
+            const img = new Image();
+            img.crossOrigin = 'anonymous';
+            img.onload = () => resolve(img);
+            img.onerror = err => reject(err);
+            img.src = src;
+        });
+    };
+}
-- 
cgit v1.2.3-70-g09d2

