cleanup of face recognition. some lint fixes.

1 files changed, 134 insertions, 56 deletions

diff --git a/src/client/views/search/FaceRecognitionHandler.tsx b/src/client/views/search/FaceRecognitionHandler.tsx
index dc271fe73..3ef6f9674 100644
--- a/src/client/views/search/FaceRecognitionHandler.tsx
+++ b/src/client/views/search/FaceRecognitionHandler.tsx
@@ -10,7 +10,22 @@ import { DocumentType } from '../../documents/DocumentTypes';
 import { DocumentManager } from '../../util/DocumentManager';
 
 /**
- * A class that handles face recognition.
+ * A singleton class that handles face recognition and manages face Doc collections for each face found.
+ * Displaying an image doc anywhere will trigger this class to test if the image contains any faces.
+ * If it does, each recognized face will be compared to a global set of faces (each is a face collection Doc
+ * that have already been found. If the face matches a face collection Doc, then it will be added to that
+ * collection along with the numerical representation of the face, its face descriptor.
+ *
+ * Image Doc's that are added to one or more face collection Docs will be given these metadata fields:
+ *     <image data field>_Face<N> - a nunerical representation of the Nth face found in the image
+ *     <image data field>_Faces  - a list of all the numerical face representations found in the image (why is this needed?)
+ *
+ * Face collection Doc's are created for each person identified and are stored in the Dashboard's faceDocument's list
+ *
+ * Each Face collection Doc represents all the images found for that person.  It has these fields:
+ *     face_label - a string label for the person that was recognized (currently it's just a 'face#')
+ *     face_descriptors - a list of all the face descriptors for different images of the person
+ *     face_docList - a list of all image Docs that contain a face for the person
  */
 export class FaceRecognitionHandler {
     static _instance: FaceRecognitionHandler;
@@ -18,13 +33,90 @@ export class FaceRecognitionHandler {
     private _processingDocs: Set<Doc> = new Set();
     private _pendingLoadDocs: Doc[] = [];
 
-    public static FaceField = (target: Doc, doc: Doc) => `${Doc.LayoutFieldKey(target)}_${doc.face_label}`;
-    public static FacesField = (target: Doc) => `${Doc.LayoutFieldKey(target)}_Faces`;
+    private static imgDocFaceField = (imgDoc: Doc, faceDoc: Doc) => `${Doc.LayoutFieldKey(imgDoc)}_${FaceRecognitionHandler.FaceDocLabel(faceDoc)}`;
+    /**
+     * initializes an image with an empty list of face descriptors
+     * @param imgDoc image to initialize
+     */
+    private static initImageDocFaceDescriptors = (imgDoc: Doc) => {
+        imgDoc[DocData][`${Doc.LayoutFieldKey(imgDoc)}_Faces`] = new List<List<number>>();
+    };
+    /**
+     * returns the face descriptors for each face found on an image Doc
+     * @param imgDoc
+     * @returns list of face descriptors
+     */
+    public static ImageDocFaceDescriptors = (imgDoc: Doc) => imgDoc[DocData][`${Doc.LayoutFieldKey(imgDoc)}_Faces`] as List<List<number>>;
+
+    /**
+     * Adds metadata to an image Doc describing a face found in the image
+     * @param imgDoc image Doc containing faces
+     * @param faceDescriptor descriptor for the face found
+     * @param faceDoc face collection Doc containing the same face
+     */
+    public static ImageDocAddFace = (imgDoc: Doc, faceDescriptor: List<number>, faceDoc: Doc) => {
+        const faceFieldKey = FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc);
+        if (imgDoc[DocData][faceFieldKey]) {
+            Cast(imgDoc[DocData][faceFieldKey], listSpec('number'), null).push(faceDescriptor as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
+        } else {
+            imgDoc[DocData][faceFieldKey] = new List<List<number>>([faceDescriptor]);
+        }
+    };
+
+    /**
+     * returns a list of all face collection Docs on the current dashboard
+     * @returns face collection Doc list
+     */
+    public static FaceDocuments = () => DocListCast(Doc.ActiveDashboard?.[DocData].faceDocuments);
+
+    public static DeleteFaceDoc = (faceDoc: Doc) => Doc.ActiveDashboard && Doc.RemoveDocFromList(Doc.ActiveDashboard[DocData], 'faceDocuments', faceDoc);
+
+    /**
+     * returns the labels associated with a face collection Doc
+     * @param faceDoc the face collection Doc
+     * @returns label string
+     */
+    public static FaceDocLabel = (faceDoc: Doc) => StrCast(faceDoc[DocData].face_label);
+    /**
+     * Returns all the face descriptors associated with a face collection Doc
+     * @param faceDoc a face collection Doc
+     * @returns face descriptors
+     */
+    public static FaceDocDescriptors = (faceDoc: Doc) => faceDoc[DocData].face_descriptors as List<List<number>>;
+
+    /**
+     * Returns a list of all face image Docs associated with the face collection
+     * @param faceDoc a face collection Doc
+     * @returns image Docs
+     */
+    public static FaceDocFaces = (faceDoc: Doc) => DocListCast(faceDoc[DocData].face_docList);
+
+    /**
+     * Adds a face image to the list of faces in a face collection Doc, and updates the face collection's list of image descriptors
+     * @param img - image with faces to add to a face collection Doc
+     * @param faceDescriptor - the face descriptor for the face in the image to add
+     * @param faceDoc - the face collection Doc
+     */
+    public static FaceDocAddImageDocFace = (img: Doc, faceDescriptor: List<number>, faceDoc: Doc) => {
+        Doc.AddDocToList(faceDoc, 'face_docList', img);
+        Cast(faceDoc.face_descriptors, listSpec('number'), null).push(faceDescriptor as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
+    };
+
+    /**
+     * Removes a face from a face Doc collection, and updates the face collection's list of image descriptors
+     * @param imgDoc - image with faces to remove from the face Doc collectoin
+     * @param faceDoc - the face Doc collection
+     */
+    public static FaceDocRemoveImageDocFace = (imgDoc: Doc, faceDoc: Doc) => {
+        imgDoc[DocData][FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc)] = new List<List<number>>();
+        Doc.RemoveDocFromList(faceDoc[DocData], 'face_docList', imgDoc);
+        faceDoc[DocData].face_descriptors = new List<List<number>>(FaceRecognitionHandler.FaceDocDescriptors(faceDoc).filter(fd => !(imgDoc[DocData][FaceRecognitionHandler.imgDocFaceField(imgDoc, faceDoc)] as List<List<number>>).includes(fd)));
+    };
 
     constructor() {
         FaceRecognitionHandler._instance = this;
-        this.loadModels().then(() => this._pendingLoadDocs.forEach(this.findMatches));
-        DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.findMatches(dv.Document));
+        this.loadModels().then(() => this._pendingLoadDocs.forEach(this.classifyFacesInImage));
+        DocumentManager.Instance.AddAnyViewRenderedCB(dv => FaceRecognitionHandler.Instance.classifyFacesInImage(dv.Document));
     }
 
     @computed get examinedFaceDocs() {
@@ -48,48 +140,42 @@ export class FaceRecognitionHandler {
 
     /**
      * When a document is added, look for matching face documents.
-     * @param doc The document being analyzed.
+     * @param imgDoc The document being analyzed.
      */
-    public findMatches = async (doc: Doc) => {
+    public classifyFacesInImage = async (imgDoc: Doc) => {
         if (!this._loadedModels || !Doc.ActiveDashboard) {
-            this._pendingLoadDocs.push(doc);
+            this._pendingLoadDocs.push(imgDoc);
             return;
         }
 
-        if (doc.type === DocumentType.LOADING && !doc.loadingError) {
-            setTimeout(() => this.findMatches(doc), 1000);
+        if (imgDoc.type === DocumentType.LOADING && !imgDoc.loadingError) {
+            setTimeout(() => this.classifyFacesInImage(imgDoc), 1000);
             return;
         }
 
-        const imgUrl = ImageCast(doc[Doc.LayoutFieldKey(doc)]);
+        const imgUrl = ImageCast(imgDoc[Doc.LayoutFieldKey(imgDoc)]);
         // If the doc isn't an image or currently already been examined or is being processed, stop examining the document.
-        if (!imgUrl || this.examinedFaceDocs.includes(doc) || this._processingDocs.has(doc)) {
+        if (!imgUrl || this.examinedFaceDocs.includes(imgDoc) || this._processingDocs.has(imgDoc)) {
             return;
         }
 
         // Mark the document as being processed.
-        this._processingDocs.add(doc);
+        this._processingDocs.add(imgDoc);
+        FaceRecognitionHandler.initImageDocFaceDescriptors(imgDoc);
 
         // Get the image the document contains and analyze for faces.
         const [name, type] = imgUrl.url.href.split('.');
         const imageURL = `${name}_o.${type}`;
-
         const img = await this.loadImage(imageURL);
-
-        const fullFaceDescriptions = await faceapi.detectAllFaces(img).withFaceLandmarks().withFaceDescriptors();
-
-        doc[DocData][FaceRecognitionHandler.FacesField(doc)] = new List<List<number>>();
+        const imgDocFaceDescriptions = await faceapi.detectAllFaces(img).withFaceLandmarks().withFaceDescriptors();
 
         // For each face detected, find a match.
-        for (const fd of fullFaceDescriptions) {
-            let match = this.findMatch(fd.descriptor);
-            const converted_list = new List<number>(Array.from(fd.descriptor));
-
-            if (match) {
-                // If a matching Face Document has been found, add the document to the Face Document's associated docs and append the face
-                // descriptor to the Face Document's descriptor list.
-                Doc.AddDocToList(match, 'face_docList', doc);
-                Cast(match.face_descriptors, listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
+        for (const fd of imgDocFaceDescriptions) {
+            let faceDocMatch = this.findMatchingFaceDoc(fd.descriptor);
+            const faceDescriptor = new List<number>(Array.from(fd.descriptor));
+
+            if (faceDocMatch) {
+                FaceRecognitionHandler.FaceDocAddImageDocFace(imgDoc, faceDescriptor, faceDocMatch);
             } else {
                 // If a matching Face Document has not been found, create a new Face Document.
                 Doc.UserDoc().faceDocNum = NumCast(Doc.UserDoc().faceDocNum) + 1;
@@ -98,53 +184,45 @@ export class FaceRecognitionHandler {
                 newFaceDocument.title = `Face ${Doc.UserDoc().faceDocNum}`;
                 newFaceDocument.face = ''; // just to make prettyprinting look better
                 newFaceDocument.face_label = `Face${Doc.UserDoc().faceDocNum}`;
-                newFaceDocument.face_docList = new List<Doc>([doc]);
-                newFaceDocument.face_descriptors = new List<List<number>>([converted_list]);
+                newFaceDocument.face_docList = new List<Doc>([imgDoc]);
+                newFaceDocument.face_descriptors = new List<List<number>>([faceDescriptor]);
 
                 Doc.AddDocToList(Doc.ActiveDashboard[DocData], 'faceDocuments', newFaceDocument);
-                match = newFaceDocument;
+                faceDocMatch = newFaceDocument;
             }
 
             // Assign a field in the document of the matching Face Document.
-            const faceDescripField = FaceRecognitionHandler.FaceField(doc, match);
-            if (doc[DocData][faceDescripField]) {
-                Cast(doc[DocData][faceDescripField], listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
-            } else {
-                doc[DocData][faceDescripField] = new List<List<number>>([converted_list]);
-            }
-
-            Cast(doc[DocData][FaceRecognitionHandler.FacesField(doc)], listSpec('number'), null).push(converted_list as unknown as number); // items are lists of numbers, not numbers, but type system can't handle that
-
-            Doc.AddDocToList(Doc.UserDoc(), 'examinedFaceDocs', doc);
+            FaceRecognitionHandler.ImageDocAddFace(imgDoc, faceDescriptor, faceDocMatch);
+            Doc.AddDocToList(Doc.UserDoc(), 'examinedFaceDocs', imgDoc);
         }
-        this._processingDocs.delete(doc);
+        this._processingDocs.delete(imgDoc);
     };
 
     /**
-     * Finds a matching Face Document given a descriptor
-     * @param cur_descriptor The current descriptor whose match is being searched for.
-     * @returns The most similar Face Document.
+     * Finds the most similar matching Face Document to a face descriptor
+     * @param faceDescriptor face descriptor number list
+     * @returns face Doc
      */
-    private findMatch(cur_descriptor: Float32Array) {
-        if (!Doc.ActiveDashboard || DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).length < 1) {
-            return null;
+    private findMatchingFaceDoc = (faceDescriptor: Float32Array) => {
+        if (!Doc.ActiveDashboard || FaceRecognitionHandler.FaceDocuments().length < 1) {
+            return undefined;
         }
 
-        const faceDescriptors: faceapi.LabeledFaceDescriptors[] = DocListCast(Doc.ActiveDashboard[DocData].faceDocuments).map(faceDocument => {
-            const float32Array = (faceDocument[DocData].face_descriptors as List<List<number>>).map(faceDescriptor => new Float32Array(Array.from(faceDescriptor)));
-            return new faceapi.LabeledFaceDescriptors(StrCast(faceDocument[DocData].face_label), float32Array);
+        const faceDescriptors = FaceRecognitionHandler.FaceDocuments().map(faceDoc => {
+            const float32Array = FaceRecognitionHandler.FaceDocDescriptors(faceDoc).map(fd => new Float32Array(Array.from(fd)));
+            return new faceapi.LabeledFaceDescriptors(FaceRecognitionHandler.FaceDocLabel(faceDoc), float32Array);
         });
         const faceMatcher = new FaceMatcher(faceDescriptors, 0.6);
-        const match = faceMatcher.findBestMatch(cur_descriptor);
+        const match = faceMatcher.findBestMatch(faceDescriptor);
         if (match.label !== 'unknown') {
-            for (const doc of DocListCast(Doc.ActiveDashboard[DocData].faceDocuments)) {
-                if (doc[DocData].face_label === match.label) {
-                    return doc;
+            for (const faceDoc of FaceRecognitionHandler.FaceDocuments()) {
+                if (FaceRecognitionHandler.FaceDocLabel(faceDoc) === match.label) {
+                    return faceDoc;
                 }
             }
         }
-        return null;
-    }
+        return undefined;
+    };
 
     /**
      * Loads an image