Merge branch 'master' of https://github.com/browngraphicslab/Dash-Web into animationtimeline

12 files changed, 286 insertions, 17 deletions

diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts
new file mode 100644
index 000000000..d4085cf76
--- /dev/null
+++ b/src/client/cognitive_services/CognitiveServices.ts
@@ -0,0 +1,135 @@
+import * as request from "request-promise";
+import { Doc, Field } from "../../new_fields/Doc";
+import { Cast } from "../../new_fields/Types";
+import { ImageField } from "../../new_fields/URLField";
+import { List } from "../../new_fields/List";
+import { Docs } from "../documents/Documents";
+import { RouteStore } from "../../server/RouteStore";
+import { Utils } from "../../Utils";
+import { CompileScript } from "../util/Scripting";
+import { ComputedField } from "../../new_fields/ScriptField";
+
+export enum Services {
+    ComputerVision = "vision",
+    Face = "face"
+}
+
+export enum Confidence {
+    Yikes = 0.0,
+    Unlikely = 0.2,
+    Poor = 0.4,
+    Fair = 0.6,
+    Good = 0.8,
+    Excellent = 0.95
+}
+
+export type Tag = { name: string, confidence: number };
+export type Rectangle = { top: number, left: number, width: number, height: number };
+export type Face = { faceAttributes: any, faceId: string, faceRectangle: Rectangle };
+export type Converter = (results: any) => Field;
+
+/**
+ * A file that handles all interactions with Microsoft Azure's Cognitive
+ * Services APIs. These machine learning endpoints allow basic data analytics for
+ * various media types.
+ */
+export namespace CognitiveServices {
+
+    export namespace Image {
+
+        export const analyze = async (imageUrl: string, service: Services) => {
+            return fetch(Utils.prepend(`${RouteStore.cognitiveServices}/${service}`)).then(async response => {
+                let apiKey = await response.text();
+                if (!apiKey) {
+                    return undefined;
+                }
+                let uriBase;
+                let parameters;
+
+                switch (service) {
+                    case Services.Face:
+                        uriBase = 'face/v1.0/detect';
+                        parameters = {
+                            'returnFaceId': 'true',
+                            'returnFaceLandmarks': 'false',
+                            'returnFaceAttributes': 'age,gender,headPose,smile,facialHair,glasses,' +
+                                'emotion,hair,makeup,occlusion,accessories,blur,exposure,noise'
+                        };
+                        break;
+                    case Services.ComputerVision:
+                        uriBase = 'vision/v2.0/analyze';
+                        parameters = {
+                            'visualFeatures': 'Categories,Description,Color,Objects,Tags,Adult',
+                            'details': 'Celebrities,Landmarks',
+                            'language': 'en',
+                        };
+                        break;
+                }
+
+                const options = {
+                    uri: 'https://eastus.api.cognitive.microsoft.com/' + uriBase,
+                    qs: parameters,
+                    body: `{"url": "${imageUrl}"}`,
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'Ocp-Apim-Subscription-Key': apiKey
+                    }
+                };
+
+                let results: any;
+                try {
+                    results = await request.post(options).then(response => JSON.parse(response));
+                } catch (e) {
+                    results = undefined;
+                }
+                return results;
+            });
+        };
+
+        const analyzeDocument = async (target: Doc, service: Services, converter: Converter, storageKey: string) => {
+            let imageData = Cast(target.data, ImageField);
+            if (!imageData || await Cast(target[storageKey], Doc)) {
+                return;
+            }
+            let toStore: any;
+            let results = await analyze(imageData.url.href, service);
+            if (!results) {
+                toStore = "Cognitive Services could not process the given image URL.";
+            } else {
+                if (!results.length) {
+                    toStore = converter(results);
+                } else {
+                    toStore = results.length > 0 ? converter(results) : "Empty list returned.";
+                }
+            }
+            target[storageKey] = toStore;
+        };
+
+        export const generateMetadata = async (target: Doc, threshold: Confidence = Confidence.Excellent) => {
+            let converter = (results: any) => {
+                let tagDoc = new Doc;
+                results.tags.map((tag: Tag) => {
+                    let sanitized = tag.name.replace(" ", "_");
+                    let script = `return (${tag.confidence} >= this.confidence) ? ${tag.confidence} : "${ComputedField.undefined}"`;
+                    let computed = CompileScript(script, { params: { this: "Doc" } });
+                    computed.compiled && (tagDoc[sanitized] = new ComputedField(computed));
+                });
+                tagDoc.title = "Generated Tags";
+                tagDoc.confidence = threshold;
+                return tagDoc;
+            };
+            analyzeDocument(target, Services.ComputerVision, converter, "generatedTags");
+        };
+
+        export const extractFaces = async (target: Doc) => {
+            let converter = (results: any) => {
+                let faceDocs = new List<Doc>();
+                results.map((face: Face) => faceDocs.push(Docs.Get.DocumentHierarchyFromJson(face, `Face: ${face.faceId}`)!));
+                return faceDocs;
+            };
+            analyzeDocument(target, Services.Face, converter, "faces");
+        };
+
+    }
+
+}
\ No newline at end of file
diff --git a/src/client/views/GlobalKeyHandler.ts b/src/client/views/GlobalKeyHandler.ts
index f378b6c0c..e8a588e58 100644
--- a/src/client/views/GlobalKeyHandler.ts
+++ b/src/client/views/GlobalKeyHandler.ts
@@ -4,6 +4,7 @@ import { CollectionDockingView } from "./collections/CollectionDockingView";
 import { MainView } from "./MainView";
 import { DragManager } from "../util/DragManager";
 import { action } from "mobx";
+import { Doc } from "../../new_fields/Doc";
 
 const modifiers = ["control", "meta", "shift", "alt"];
 type KeyHandler = (keycode: string, e: KeyboardEvent) => KeyControlInfo;
@@ -82,6 +83,9 @@ export default class KeyManager {
                     });
                 }, "delete");
                 break;
+            case "enter":
+                SelectionManager.SelectedDocuments().map(selected => Doc.ToggleDetailLayout(selected.props.Document));
+                break;
         }
 
         return {
diff --git a/src/client/views/collections/CollectionVideoView.tsx b/src/client/views/collections/CollectionVideoView.tsx
index d7d5773ba..31a8a93e0 100644
--- a/src/client/views/collections/CollectionVideoView.tsx
+++ b/src/client/views/collections/CollectionVideoView.tsx
@@ -7,6 +7,8 @@ import { CollectionBaseView, CollectionRenderProps, CollectionViewType } from ".
 import { CollectionFreeFormView } from "./collectionFreeForm/CollectionFreeFormView";
 import "./CollectionVideoView.scss";
 import React = require("react");
+import { InkingControl } from "../InkingControl";
+import { InkTool } from "../../../new_fields/InkField";
 
 
 @observer
@@ -19,18 +21,19 @@ export class CollectionVideoView extends React.Component<FieldViewProps> {
     private get uIButtons() {
         let scaling = Math.min(1.8, this.props.ScreenToLocalTransform().Scale);
         let curTime = NumCast(this.props.Document.curPage);
-        return (VideoBox._showControls ? [] : [
-            <div className="collectionVideoView-time" key="time" onPointerDown={this.onResetDown} style={{ transform: `scale(${scaling}, ${scaling})` }}>
-                <span>{"" + Math.round(curTime)}</span>
-                <span style={{ fontSize: 8 }}>{" " + Math.round((curTime - Math.trunc(curTime)) * 100)}</span>
-            </div>,
+        return ([<div className="collectionVideoView-time" key="time" onPointerDown={this.onResetDown} style={{ transform: `scale(${scaling}, ${scaling})` }}>
+            <span>{"" + Math.round(curTime)}</span>
+            <span style={{ fontSize: 8 }}>{" " + Math.round((curTime - Math.trunc(curTime)) * 100)}</span>
+        </div>,
+        VideoBox._showControls ? (null) : [
             <div className="collectionVideoView-play" key="play" onPointerDown={this.onPlayDown} style={{ transform: `scale(${scaling}, ${scaling})` }}>
                 {this._videoBox && this._videoBox.Playing ? "\"" : ">"}
             </div>,
             <div className="collectionVideoView-full" key="full" onPointerDown={this.onFullDown} style={{ transform: `scale(${scaling}, ${scaling})` }}>
                 F
                 </div>
-        ]);
+
+        ]]);
     }
 
     @action
@@ -53,12 +56,33 @@ export class CollectionVideoView extends React.Component<FieldViewProps> {
         }
     }
 
+    _isclick = 0;
     @action
-    onResetDown = () => {
+    onResetDown = (e: React.PointerEvent) => {
         if (this._videoBox) {
             this._videoBox.Pause();
-            this.props.Document.curPage = 0;
+            e.stopPropagation();
+            this._isclick = 0;
+            document.addEventListener("pointermove", this.onPointerMove, true);
+            document.addEventListener("pointerup", this.onPointerUp, true);
+            InkingControl.Instance.switchTool(InkTool.Eraser);
+        }
+    }
+
+    @action
+    onPointerMove = (e: PointerEvent) => {
+        this._isclick += Math.abs(e.movementX) + Math.abs(e.movementY);
+        if (this._videoBox) {
+            this._videoBox.Seek(Math.max(0, NumCast(this.props.Document.curPage, 0) + Math.sign(e.movementX) * 0.0333));
         }
+        e.stopImmediatePropagation();
+    }
+    @action
+    onPointerUp = (e: PointerEvent) => {
+        document.removeEventListener("pointermove", this.onPointerMove, true);
+        document.removeEventListener("pointerup", this.onPointerUp, true);
+        InkingControl.Instance.switchTool(InkTool.None);
+        this._isclick < 10 && (this.props.Document.curPage = 0);
     }
     setVideoBox = (videoBox: VideoBox) => { this._videoBox = videoBox; };
 
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 5d3363d3a..907ba3713 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -351,7 +351,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                     // @TODO: shouldn't always follow target context
                     let linkedFwdContextDocs = [first.length ? await (first[0].targetContext) as Doc : undefined, undefined];
 
-                    let linkedFwdPage = [first.length ? NumCast(first[0].linkedToPage, undefined) : undefined, undefined];
+                    let linkedFwdPage = [first.length ? NumCast(first[0].anchor2Page, undefined) : undefined, undefined];
 
                     if (!linkedFwdDocs.some(l => l instanceof Promise)) {
                         let maxLocation = StrCast(linkedFwdDocs[0].maximizeLocation, "inTab");
diff --git a/src/client/views/nodes/FaceRectangle.tsx b/src/client/views/nodes/FaceRectangle.tsx
new file mode 100644
index 000000000..887efc0d5
--- /dev/null
+++ b/src/client/views/nodes/FaceRectangle.tsx
@@ -0,0 +1,29 @@
+import React = require("react");
+import { observer } from "mobx-react";
+import { observable, runInAction } from "mobx";
+import { RectangleTemplate } from "./FaceRectangles";
+
+@observer
+export default class FaceRectangle extends React.Component<{ rectangle: RectangleTemplate }> {
+    @observable private opacity = 0;
+
+    componentDidMount() {
+        setTimeout(() => runInAction(() => this.opacity = 1), 500);
+    }
+
+    render() {
+        let rectangle = this.props.rectangle;
+        return (
+            <div
+                style={{
+                    ...rectangle.style,
+                    opacity: this.opacity,
+                    transition: "1s ease opacity",
+                    position: "absolute",
+                    borderRadius: 5
+                }}
+            />
+        );
+    }
+
+}
\ No newline at end of file
diff --git a/src/client/views/nodes/FaceRectangles.tsx b/src/client/views/nodes/FaceRectangles.tsx
new file mode 100644
index 000000000..3570531b2
--- /dev/null
+++ b/src/client/views/nodes/FaceRectangles.tsx
@@ -0,0 +1,46 @@
+import React = require("react");
+import { Doc, DocListCast } from "../../../new_fields/Doc";
+import { Cast, NumCast } from "../../../new_fields/Types";
+import { observer } from "mobx-react";
+import { Id } from "../../../new_fields/FieldSymbols";
+import FaceRectangle from "./FaceRectangle";
+
+interface FaceRectanglesProps {
+    document: Doc;
+    color: string;
+    backgroundColor: string;
+}
+
+export interface RectangleTemplate {
+    id: string;
+    style: Partial<React.CSSProperties>;
+}
+
+@observer
+export default class FaceRectangles extends React.Component<FaceRectanglesProps> {
+
+    render() {
+        let faces = DocListCast(Doc.GetProto(this.props.document).faces);
+        let templates: RectangleTemplate[] = faces.map(faceDoc => {
+            let rectangle = Cast(faceDoc.faceRectangle, Doc) as Doc;
+            let style = {
+                top: NumCast(rectangle.top),
+                left: NumCast(rectangle.left),
+                width: NumCast(rectangle.width),
+                height: NumCast(rectangle.height),
+                backgroundColor: `${this.props.backgroundColor}33`,
+                border: `solid 2px ${this.props.color}`,
+            } as React.CSSProperties;
+            return {
+                id: rectangle[Id],
+                style: style
+            };
+        });
+        return (
+            <div>
+                {templates.map(rectangle => <FaceRectangle key={rectangle.id} rectangle={rectangle} />)}
+            </div>
+        );
+    }
+
+}
\ No newline at end of file
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index c3ee1e823..0f60bd0fb 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -25,6 +25,8 @@ import { Docs } from '../../documents/Documents';
 import { DocServer } from '../../DocServer';
 import { Font } from '@react-pdf/renderer';
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { CognitiveServices } from '../../cognitive_services/CognitiveServices';
+import FaceRectangles from './FaceRectangles';
 var requestImageSize = require('../../util/request-image-size');
 var path = require('path');
 const { Howl, Howler } = require('howler');
@@ -195,10 +197,10 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
         let field = Cast(this.Document[this.props.fieldKey], ImageField);
         if (field) {
             let url = field.url.href;
-            let subitems: ContextMenuProps[] = [];
-            subitems.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" });
-            subitems.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" });
-            subitems.push({
+            let funcs: ContextMenuProps[] = [];
+            funcs.push({ description: "Copy path", event: () => Utils.CopyText(url), icon: "expand-arrows-alt" });
+            funcs.push({ description: "Record 1sec audio", event: this.recordAudioAnnotation, icon: "expand-arrows-alt" });
+            funcs.push({
                 description: "Rotate", event: action(() => {
                     let proto = Doc.GetProto(this.props.Document);
                     let nw = this.props.Document.nativeWidth;
@@ -212,7 +214,14 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
                     this.props.Document.height = w;
                 }), icon: "expand-arrows-alt"
             });
-            ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: subitems });
+
+            let modes: ContextMenuProps[] = [];
+            let dataDoc = Doc.GetProto(this.Document);
+            modes.push({ description: "Generate Tags", event: () => CognitiveServices.Image.generateMetadata(dataDoc), icon: "tag" });
+            modes.push({ description: "Find Faces", event: () => CognitiveServices.Image.extractFaces(dataDoc), icon: "camera" });
+
+            ContextMenu.Instance.addItem({ description: "Image Funcs...", subitems: funcs });
+            ContextMenu.Instance.addItem({ description: "Analyze...", subitems: modes });
         }
     }
 
@@ -371,6 +380,7 @@ export class ImageBox extends DocComponent<FieldViewProps, ImageDocument>(ImageD
                         style={{ color: [DocListCast(this.extensionDoc.audioAnnotations).length ? "blue" : "gray", "green", "red"][this._audioState] }} icon={faFileAudio} size="sm" />
                 </div>
                 {/* {this.lightbox(paths)} */}
+                <FaceRectangles document={this.props.Document} color={"#0000FF"} backgroundColor={"#0000FF"} />
             </div>);
     }
 }
\ No newline at end of file
diff --git a/src/client/views/nodes/KeyValueBox.tsx b/src/client/views/nodes/KeyValueBox.tsx
index c9dd9a64e..9fc0f2080 100644
--- a/src/client/views/nodes/KeyValueBox.tsx
+++ b/src/client/views/nodes/KeyValueBox.tsx
@@ -114,7 +114,7 @@ export class KeyValueBox extends React.Component<FieldViewProps> {
         let protos = Doc.GetAllPrototypes(doc);
         for (const proto of protos) {
             Object.keys(proto).forEach(key => {
-                if (!(key in ids)) {
+                if (!(key in ids) && realDoc[key] !== ComputedField.undefined) {
                     ids[key] = key;
                 }
             });
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index 83ad2a3b3..30ad75000 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -59,19 +59,20 @@ export class VideoBox extends DocComponent<FieldViewProps, VideoDocument>(VideoD
         this.Playing = true;
         update && this.player && this.player.play();
         update && this._youtubePlayer && this._youtubePlayer.playVideo();
-        !this._playTimer && (this._playTimer = setInterval(this.updateTimecode, 5));
+        this._youtubePlayer && !this._playTimer && (this._playTimer = setInterval(this.updateTimecode, 5));
         this.updateTimecode();
     }
 
     @action public Seek(time: number) {
         this._youtubePlayer && this._youtubePlayer.seekTo(Math.round(time), true);
+        this.player && (this.player.currentTime = time);
     }
 
     @action public Pause = (update: boolean = true) => {
         this.Playing = false;
         update && this.player && this.player.pause();
         update && this._youtubePlayer && this._youtubePlayer.pauseVideo();
-        this._playTimer && clearInterval(this._playTimer);
+        this._youtubePlayer && this._playTimer && clearInterval(this._playTimer);
         this._playTimer = undefined;
         this.updateTimecode();
     }
@@ -113,6 +114,7 @@ export class VideoBox extends DocComponent<FieldViewProps, VideoDocument>(VideoD
     setVideoRef = (vref: HTMLVideoElement | null) => {
         this._videoRef = vref;
         if (vref) {
+            this._videoRef!.ontimeupdate = this.updateTimecode;
             vref.onfullscreenchange = action((e) => this._fullScreen = vref.webkitDisplayingFullscreen);
             if (this._reactionDisposer) this._reactionDisposer();
             this._reactionDisposer = reaction(() => this.props.Document.curPage, () =>
diff --git a/src/new_fields/ScriptField.ts b/src/new_fields/ScriptField.ts
index e8a1ea28a..e5ec34f57 100644
--- a/src/new_fields/ScriptField.ts
+++ b/src/new_fields/ScriptField.ts
@@ -107,6 +107,8 @@ export namespace ComputedField {
         useComputed = true;
     }
 
+    export const undefined = "__undefined";
+
     export function WithoutComputed<T>(fn: () => T) {
         DisableComputedFields();
         try {
diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts
index 5c13495ff..e30015e39 100644
--- a/src/server/RouteStore.ts
+++ b/src/server/RouteStore.ts
@@ -29,4 +29,7 @@ export enum RouteStore {
     forgot = "/forgotpassword",
     reset = "/reset/:token",
 
+    // APIS
+    cognitiveServices = "/cognitiveservices"
+
 }
\ No newline at end of file
diff --git a/src/server/index.ts b/src/server/index.ts
index 6b4e59bfc..5b086a2cf 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -284,6 +284,20 @@ addSecureRoute(
     RouteStore.getCurrUser
 );
 
+addSecureRoute(Method.GET, (user, res, req) => {
+    let requested = req.params.requestedservice;
+    switch (requested) {
+        case "face":
+            res.send(process.env.FACE);
+            break;
+        case "vision":
+            res.send(process.env.VISION);
+            break;
+        default:
+            res.send(undefined);
+    }
+}, undefined, `${RouteStore.cognitiveServices}/:requestedservice`);
+
 class NodeCanvasFactory {
     create = (width: number, height: number) => {
         var canvas = createCanvas(width, height);