2 files changed, 88 insertions, 0 deletions
diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx
index 8242e7c27..ada934aea 100644
--- a/src/client/views/Main.tsx
+++ b/src/client/views/Main.tsx
@@ -61,6 +61,7 @@ import { ImportElementBox } from './nodes/importBox/ImportElementBox';
 import { PresBox, PresElementBox } from './nodes/trails';
 import { SearchBox } from './search/SearchBox';
 import { ImageLabelBox } from './collections/collectionFreeForm/ImageLabelBox';
+import { FaceRecognitionHandler } from './search/FaceRecognitionHandler';
 
 dotenv.config();
 
@@ -96,6 +97,7 @@ FieldLoader.ServerLoadStatus = { requested: 0, retrieved: 0, message: 'cache' };
         new BranchingTrailManager({});
         new PingManager();
         new KeyManager();
+        new FaceRecognitionHandler();
 
         // initialize plugins and classes that require plugins
         CollectionDockingView.Init(TabDocView);
diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx
new file mode 100644
index 000000000..86619b2d1
--- /dev/null
+++ b/src/client/views/search/FaceRecognitionHandler.tsx
@@ -0,0 +1,86 @@
+import * as faceapi from 'face-api.js';
+import { FaceMatcher, TinyFaceDetectorOptions } from 'face-api.js';
+import { Doc, DocListCast, NumListCast } from '../../../fields/Doc';
+import { DocData } from '../../../fields/DocSymbols';
+import { List } from '../../../fields/List';
+import { ObjectField } from '../../../fields/ObjectField';
+import { StrCast } from '../../../fields/Types';
+
+export class FaceRecognitionHandler {
+    static _instance: FaceRecognitionHandler;
+
+    constructor() {
+        FaceRecognitionHandler._instance = this;
+        this.loadModels();
+        if (!Doc.ActiveDashboard![DocData].faceDocuments) {
+            Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>();
+        }
+    }
+
+    async loadModels() {
+        const MODEL_URL = `/models`;
+        await faceapi.loadTinyFaceDetectorModel(MODEL_URL);
+        await faceapi.loadFaceLandmarkTinyModel(MODEL_URL);
+        await faceapi.loadFaceRecognitionModel(MODEL_URL);
+    }
+
+    public static get Instance() {
+        return FaceRecognitionHandler._instance ?? new FaceRecognitionHandler();
+    }
+
+    public async findMatches(doc: Doc, imageURL: string) {
+        const img = await this.loadImage(imageURL);
+
+        const fullFaceDescriptions = await faceapi.detectAllFaces(img, new TinyFaceDetectorOptions()).withFaceLandmarks(true).withFaceDescriptors();
+
+        fullFaceDescriptions.forEach(fd => {
+            const match = this.findMatch(fd.descriptor);
+            if (match) {
+                match[DocData].associatedDocs = new List<Doc>([...DocListCast(match[DocData].associatedDocs), doc]);
+                match[DocData].faceDescriptors = new List<List<number>>([...(match[DocData].faceDescriptors as List<List<number>>), Array.from(fd.descriptor) as List<number>]);
+            } else {
+                const newFaceDocument = new Doc();
+                const converted_array = Array.from(fd.descriptor);
+                newFaceDocument[DocData].faceDescriptors = new List<List<number>>();
+                (newFaceDocument[DocData].faceDescriptors as List<List<number>>).push(converted_array as List<number>);
+                newFaceDocument[DocData].label = `Person ${DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).length + 1}`;
+                newFaceDocument[DocData].associatedDocs = new List<Doc>([doc]);
+
+                Doc.ActiveDashboard![DocData].faceDocuments = new List<Doc>([...DocListCast(Doc.ActiveDashboard![DocData].faceDocuments), newFaceDocument]);
+            }
+        });
+    }
+
+    private findMatch(cur_descriptor: Float32Array) {
+        if (DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).length < 1) {
+            return null;
+        }
+
+        const faceDescriptors: faceapi.LabeledFaceDescriptors[] = DocListCast(Doc.ActiveDashboard![DocData].faceDocuments).map(faceDocument => {
+            const float32Array = (faceDocument[DocData].faceDescriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor)));
+            return new faceapi.LabeledFaceDescriptors(StrCast(faceDocument[DocData].label), float32Array);
+        });
+        const faceMatcher = new FaceMatcher(faceDescriptors, 0.6);
+        const match = faceMatcher.findBestMatch(cur_descriptor);
+
+        if (match.label == 'unknown') {
+            return null;
+        } else {
+            for (const doc of DocListCast(Doc.ActiveDashboard![DocData].faceDocuments)) {
+                if (doc[DocData].label === match.label) {
+                    return doc;
+                }
+            }
+        }
+    }
+
+    private loadImage = (src: string): Promise<HTMLImageElement> => {
+        return new Promise((resolve, reject) => {
+            const img = new Image();
+            img.crossOrigin = 'anonymous';
+            img.onload = () => resolve(img);
+            img.onerror = err => reject(err);
+            img.src = src;
+        });
+    };
+}