merge

1 files changed, 204 insertions, 9 deletions

diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts
index 02eff3b25..ce829eb1e 100644
--- a/src/client/cognitive_services/CognitiveServices.ts
+++ b/src/client/cognitive_services/CognitiveServices.ts
@@ -5,12 +5,18 @@ import { Docs } from "../documents/Documents";
 import { Utils } from "../../Utils";
 import { InkData } from "../../new_fields/InkField";
 import { UndoManager } from "../util/UndoManager";
+import requestPromise = require("request-promise");
+import { List } from "../../new_fields/List";
+import { ClientRecommender } from "../ClientRecommender";
+import { ImageBox } from "../views/nodes/ImageBox";
 
 type APIManager<D> = { converter: BodyConverter<D>, requester: RequestExecutor };
 type RequestExecutor = (apiKey: string, body: string, service: Service) => Promise<string>;
 type AnalysisApplier<D> = (target: Doc, relevantKeys: string[], data: D, ...args: any) => any;
 type BodyConverter<D> = (data: D) => string;
 type Converter = (results: any) => Field;
+type TextConverter = (results: any, data: string) => Promise<{ keyterms: Field, external_recommendations: any, kp_string: string[] }>;
+type BingConverter = (results: any) => Promise<{ title_vals: string[], url_vals: string[] }>;
 
 export type Tag = { name: string, confidence: number };
 export type Rectangle = { top: number, left: number, width: number, height: number };
@@ -18,7 +24,9 @@ export type Rectangle = { top: number, left: number, width: number, height: numb
 export enum Service {
     ComputerVision = "vision",
     Face = "face",
-    Handwriting = "handwriting"
+    Handwriting = "handwriting",
+    Text = "text",
+    Bing = "bing"
 }
 
 export enum Confidence {
@@ -47,7 +55,8 @@ export namespace CognitiveServices {
         let results: any;
         try {
             results = await manager.requester(apiKey, manager.converter(data), service).then(json => JSON.parse(json));
-        } catch {
+        } catch (e) {
+            throw e;
             results = undefined;
         }
         return results;
@@ -137,7 +146,7 @@ export namespace CognitiveServices {
                 let id = 0;
                 const strokes: AzureStrokeData[] = inkData.map(points => ({
                     id: id++,
-                    points: points.map(({ x, y }) => `${x},${y}`).join(","),
+                    points: points.map(({ X: x, Y: y }) => `${x},${y}`).join(","),
                     language: "en-US"
                 }));
                 return JSON.stringify({
@@ -153,7 +162,7 @@ export namespace CognitiveServices {
                 const serverAddress = "https://api.cognitive.microsoft.com";
                 const endpoint = serverAddress + "/inkrecognizer/v1.0-preview/recognize";
 
-                const promisified = (resolve: any, reject: any) => {
+                return new Promise<string>((resolve, reject) => {
                     xhttp.onreadystatechange = function () {
                         if (this.readyState === 4) {
                             const result = xhttp.responseText;
@@ -171,11 +180,8 @@ export namespace CognitiveServices {
                     xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey);
                     xhttp.setRequestHeader('Content-Type', 'application/json');
                     xhttp.send(body);
-                };
-
-                return new Promise<any>(promisified);
+                });
             },
-
         };
 
         export namespace Appliers {
@@ -188,13 +194,21 @@ export namespace CognitiveServices {
                     results.recognitionUnits && (results = results.recognitionUnits);
                     target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Ink Analysis");
                     const recognizedText = results.map((item: any) => item.recognizedText);
+                    const recognizedObjects = results.map((item: any) => item.recognizedObject);
                     const individualWords = recognizedText.filter((text: string) => text && text.split(" ").length === 1);
-                    target[keys[1]] = individualWords.join(" ");
+                    target[keys[1]] = individualWords.length ? individualWords.join(" ") : recognizedObjects.join(", ");
                 }
 
                 batch.end();
             };
 
+            export const InterpretStrokes = async (strokes: InkData[]) => {
+                let results = await ExecuteQuery(Service.Handwriting, Manager, strokes);
+                if (results) {
+                    results.recognitionUnits && (results = results.recognitionUnits);
+                }
+                return results;
+            }
         }
 
         export interface AzureStrokeData {
@@ -212,4 +226,185 @@ export namespace CognitiveServices {
 
     }
 
+    export namespace BingSearch {
+        export const Manager: APIManager<string> = {
+            converter: (data: string) => {
+                return data;
+            },
+            requester: async (apiKey: string, query: string) => {
+                let xhttp = new XMLHttpRequest();
+                let serverAddress = "https://api.cognitive.microsoft.com";
+                let endpoint = serverAddress + '/bing/v5.0/search?q=' + encodeURIComponent(query);
+                let promisified = (resolve: any, reject: any) => {
+                    xhttp.onreadystatechange = function () {
+                        if (this.readyState === 4) {
+                            let result = xhttp.responseText;
+                            switch (this.status) {
+                                case 200:
+                                    return resolve(result);
+                                case 400:
+                                default:
+                                    return reject(result);
+                            }
+                        }
+                    };
+
+                    if (apiKey) {
+                        xhttp.open("GET", endpoint, true);
+                        xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey);
+                        xhttp.setRequestHeader('Content-Type', 'application/json');
+                        xhttp.send();
+                    }
+                    else {
+                        console.log("API key for BING unavailable");
+                    }
+                };
+                return new Promise<any>(promisified);
+            }
+
+        };
+
+        export namespace Appliers {
+            export const analyzer = async (query: string, converter: BingConverter) => {
+                let results = await ExecuteQuery(Service.Bing, Manager, query);
+                console.log("Bing results: ", results);
+                const { title_vals, url_vals } = await converter(results);
+                return { title_vals, url_vals };
+            };
+        }
+
+    }
+
+    export namespace HathiTrust {
+        export const Manager: APIManager<string> = {
+            converter: (data: string) => {
+                return data;
+            },
+            requester: async (apiKey: string, query: string) => {
+                let xhttp = new XMLHttpRequest();
+                let serverAddress = "https://babel.hathitrust.org/cgi/htd/​";
+                let endpoint = serverAddress + '/bing/v5.0/search?q=' + encodeURIComponent(query);
+                let promisified = (resolve: any, reject: any) => {
+                    xhttp.onreadystatechange = function () {
+                        if (this.readyState === 4) {
+                            let result = xhttp.responseText;
+                            switch (this.status) {
+                                case 200:
+                                    return resolve(result);
+                                case 400:
+                                default:
+                                    return reject(result);
+                            }
+                        }
+                    };
+
+                    if (apiKey) {
+                        xhttp.open("GET", endpoint, true);
+                        xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey);
+                        xhttp.setRequestHeader('Content-Type', 'application/json');
+                        xhttp.send();
+                    }
+                    else {
+                        console.log("API key for BING unavailable");
+                    }
+                };
+                return new Promise<any>(promisified);
+            }
+
+        };
+
+        export namespace Appliers {
+            export const analyzer = async (query: string, converter: BingConverter) => {
+                let results = await ExecuteQuery(Service.Bing, Manager, query);
+                console.log("Bing results: ", results);
+                const { title_vals, url_vals } = await converter(results);
+                return { title_vals, url_vals };
+            };
+        }
+
+    }
+
+
+    export namespace Text {
+        export const Manager: APIManager<string> = {
+            converter: (data: string) => {
+                return JSON.stringify({
+                    documents: [{
+                        id: 1,
+                        language: "en",
+                        text: data
+                    }]
+                });
+            },
+            requester: async (apiKey: string, body: string, service: Service) => {
+                let serverAddress = "https://eastus.api.cognitive.microsoft.com";
+                let endpoint = serverAddress + "/text/analytics/v2.1/keyPhrases";
+                let sampleBody = {
+                    "documents": [
+                        {
+                            "language": "en",
+                            "id": 1,
+                            "text": "Hello world. This is some input text that I love."
+                        }
+                    ]
+                };
+                let actualBody = body;
+                const options = {
+                    uri: endpoint,
+                    body: actualBody,
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'Ocp-Apim-Subscription-Key': apiKey
+                    }
+
+                };
+                return request.post(options);
+            }
+        };
+
+        export namespace Appliers {
+
+            export async function vectorize(keyterms: any, dataDoc: Doc, mainDoc: boolean = false) {
+                console.log("vectorizing...");
+                //keyterms = ["father", "king"];
+
+                let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true };
+                await requestPromise.post(args).then(async (wordvecs) => {
+                    if (wordvecs) {
+                        let indices = Object.keys(wordvecs);
+                        console.log("successful vectorization!");
+                        var vectorValues = new List<number>();
+                        indices.forEach((ind: any) => {
+                            //console.log(wordvec.word);
+                            vectorValues.push(wordvecs[ind]);
+                        });
+                        ClientRecommender.Instance.processVector(vectorValues, dataDoc, mainDoc);
+                    } // adds document to internal doc set
+                    else {
+                        console.log("unsuccessful :( word(s) not in vocabulary");
+                    }
+                    //console.log(vectorValues.size);
+                }
+                );
+            }
+
+            export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: TextConverter, isMainDoc: boolean = false, isInternal: boolean = true) => {
+                let results = await ExecuteQuery(Service.Text, Manager, data);
+                console.log("Cognitive Services keyphrases: ", results);
+                let { keyterms, external_recommendations, kp_string } = await converter(results, data);
+                target[keys[0]] = keyterms;
+                if (isInternal) {
+                    //await vectorize([data], dataDoc, isMainDoc);
+                    await vectorize(kp_string, dataDoc, isMainDoc);
+                } else {
+                    return { recs: external_recommendations, keyterms: keyterms };
+                }
+            };
+
+            // export async function countFrequencies() 
+        }
+
+    }
+
+
 }
\ No newline at end of file