From 17b27d3575d3f91f461262e5ad72a457238d198a Mon Sep 17 00:00:00 2001 From: ab Date: Wed, 7 Aug 2019 16:28:51 -0400 Subject: correlation matrix completed --- .../collectionFreeForm/CollectionFreeFormView.tsx | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 29f9b1429..9344b43d2 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1,6 +1,6 @@ import { library } from "@fortawesome/fontawesome-svg-core"; import { faEye } from "@fortawesome/free-regular-svg-icons"; -import { faCompass, faCompressArrowsAlt, faExpandArrowsAlt, faPaintBrush, faTable, faUpload } from "@fortawesome/free-solid-svg-icons"; +import { faCompass, faCompressArrowsAlt, faExpandArrowsAlt, faPaintBrush, faTable, faUpload, faBrain } from "@fortawesome/free-solid-svg-icons"; import { action, computed } from "mobx"; import { observer } from "mobx-react"; import { Doc, DocListCastAsync, HeightSym, WidthSym } from "../../../../new_fields/Doc"; @@ -37,8 +37,9 @@ import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); import v5 = require("uuid/v5"); +import { ClientRecommender } from "../../../ClientRecommender"; -library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload); +library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBrain); export const panZoomSchema = createSchema({ panX: "number", @@ -596,6 +597,20 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { input.click(); } }); + ContextMenu.Instance.addItem({ + description: "Recommender System", + event: async () => { + new ClientRecommender(); + let activedocs = this.getActiveDocuments(); + await Promise.all(activedocs.map((doc: Doc) => { + console.log(StrCast(doc.title)); + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + })); + console.log(ClientRecommender.Instance.createDistanceMatrix()); + }, + icon: "brain" + }); } -- cgit v1.2.3-70-g09d2 From 17f28e393e0c24fcace33a3ecd5564bc766fe685 Mon Sep 17 00:00:00 2001 From: ab Date: Thu, 8 Aug 2019 13:00:51 -0400 Subject: fuck you react --- src/client/ClientRecommender.ts | 101 -------------- src/client/ClientRecommender.tsx | 148 +++++++++++++++++++++ .../collectionFreeForm/CollectionFreeFormView.tsx | 4 +- src/server/Recommender.ts | 2 + 4 files changed, 153 insertions(+), 102 deletions(-) delete mode 100644 src/client/ClientRecommender.ts create mode 100644 src/client/ClientRecommender.tsx (limited to 'src/client/views/collections') diff --git a/src/client/ClientRecommender.ts b/src/client/ClientRecommender.ts deleted file mode 100644 index 7ff79ab50..000000000 --- a/src/client/ClientRecommender.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { Doc } from "../new_fields/Doc"; -import { StrCast } from "../new_fields/Types"; -import { List } from "../new_fields/List"; -import { CognitiveServices } from "./cognitive_services/CognitiveServices"; - - -var assert = require('assert'); - -export class ClientRecommender { - - static Instance: ClientRecommender; - private docVectors: Set; - - constructor() { - //console.log("creating client recommender..."); - ClientRecommender.Instance = this; - this.docVectors = new Set(); - } - - - /*** - * Computes the cosine similarity between two vectors in Euclidean space. - */ - - private distance(vector1: number[], vector2: number[]) { - assert(vector1.length === vector2.length, "Vectors are not the same length"); - var dotproduct = 0; - var mA = 0; - var mB = 0; - for (let i = 0; i < vector1.length; i++) { // here you missed the i++ - dotproduct += (vector1[i] * vector2[i]); - mA += (vector1[i] * vector1[i]); - mB += (vector2[i] * vector2[i]); - } - mA = Math.sqrt(mA); - mB = Math.sqrt(mB); - var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets - return similarity; - } - - /*** - * Computes the mean of a set of vectors - */ - - public mean(paragraph: Set) { - const n = 200; - const num_words = paragraph.size; - let meanVector = new Array(n).fill(0); // mean vector - paragraph.forEach((wordvec: number[]) => { - for (let i = 0; i < n; i++) { - meanVector[i] += wordvec[i]; - } - }); - meanVector = meanVector.map(x => x / num_words); - this.addToDocSet(meanVector); - return meanVector; - } - - private addToDocSet(vector: number[]) { - if (this.docVectors) { - this.docVectors.add(vector); - } - } - - /*** - * Uses Cognitive Services to extract keywords from a document - */ - - public async extractText(dataDoc: Doc, extDoc: Doc) { - let data = StrCast(dataDoc.title); - //console.log(data); - let converter = (results: any) => { - let keyterms = new List(); - results.documents.forEach((doc: any) => { - let keyPhrases = doc.keyPhrases; - keyPhrases.map((kp: string) => keyterms.push(kp)); - }); - return keyterms; - }; - await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter); - } - - /*** - * Creates distance matrix for all Documents analyzed - */ - - public createDistanceMatrix(documents: Set = this.docVectors) { - const documents_list = Array.from(documents); - const n = documents_list.length; - var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); - for (let i = 0; i < n; i++) { - var doc1 = documents_list[i]; - for (let j = 0; j < n; j++) { - var doc2 = documents_list[j]; - matrix[i][j] = this.distance(doc1, doc2); - } - } - return matrix; - } - -} \ No newline at end of file diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx new file mode 100644 index 000000000..2344ee490 --- /dev/null +++ b/src/client/ClientRecommender.tsx @@ -0,0 +1,148 @@ +import { Doc } from "../new_fields/Doc"; +import { StrCast } from "../new_fields/Types"; +import { List } from "../new_fields/List"; +import { CognitiveServices } from "./cognitive_services/CognitiveServices"; +import React = require("react"); +import { observer } from "mobx-react"; +import { observable, action, computed, reaction } from "mobx"; +var assert = require('assert'); +import Table from 'react-bootstrap/Table'; + +export interface RecommenderProps { + title: string; +} + +@observer +export class ClientRecommender extends React.Component { + + static Instance: ClientRecommender; + private docVectors: Set; + private corr_matrix = observable([[0, 0], [0, 0]]); + @observable private firstnum = 0; + //@observable private corr_matrix: number[][] = [[0, 0], [0, 0]]; + + constructor(props: RecommenderProps) { + //console.log("creating client recommender..."); + super(props); + ClientRecommender.Instance = this; + this.docVectors = new Set(); + //this.corr_matrix = [[0, 0], [0, 0]]; + } + + + /*** + * Computes the cosine similarity between two vectors in Euclidean space. + */ + + private distance(vector1: number[], vector2: number[]) { + assert(vector1.length === vector2.length, "Vectors are not the same length"); + var dotproduct = 0; + var mA = 0; + var mB = 0; + for (let i = 0; i < vector1.length; i++) { // here you missed the i++ + dotproduct += (vector1[i] * vector2[i]); + mA += (vector1[i] * vector1[i]); + mB += (vector2[i] * vector2[i]); + } + mA = Math.sqrt(mA); + mB = Math.sqrt(mB); + var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets + return similarity; + } + + /*** + * Computes the mean of a set of vectors + */ + + public mean(paragraph: Set) { + const n = 200; + const num_words = paragraph.size; + let meanVector = new Array(n).fill(0); // mean vector + paragraph.forEach((wordvec: number[]) => { + for (let i = 0; i < n; i++) { + meanVector[i] += wordvec[i]; + } + }); + meanVector = meanVector.map(x => x / num_words); + this.addToDocSet(meanVector); + return meanVector; + } + + private addToDocSet(vector: number[]) { + if (this.docVectors) { + this.docVectors.add(vector); + } + } + + /*** + * Uses Cognitive Services to extract keywords from a document + */ + + public async extractText(dataDoc: Doc, extDoc: Doc) { + let data = StrCast(dataDoc.title); + //console.log(data); + let converter = (results: any) => { + let keyterms = new List(); + results.documents.forEach((doc: any) => { + let keyPhrases = doc.keyPhrases; + keyPhrases.map((kp: string) => keyterms.push(kp)); + }); + return keyterms; + }; + await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter); + } + + /*** + * Creates distance matrix for all Documents analyzed + */ + + @action + public createDistanceMatrix(documents: Set = this.docVectors) { + //this.corr_matrix[0][0] = 500; + this.firstnum = 500; + const documents_list = Array.from(documents); + const n = documents_list.length; + var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); + for (let i = 0; i < n; i++) { + var doc1 = documents_list[i]; + for (let j = 0; j < n; j++) { + var doc2 = documents_list[j]; + matrix[i][j] = this.distance(doc1, doc2); + } + } + //this.corr_matrix = matrix; + + return matrix; + } + + @computed get first_num() { + return this.firstnum; + } + + dumb_reaction = reaction( + () => this.first_num, + number => { + console.log("number has changed", number); + this.forceUpdate(); + } + ); + + render() { + return (
+

{this.props.title ? this.props.title : "hello"}

+ + + + + + + + + + + +
{this.first_num}{this.corr_matrix[0][1]}
{this.corr_matrix[1][0]}{this.corr_matrix[1][1]}
+
); + } + +} \ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 9344b43d2..5259b9b49 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -523,6 +523,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { super.setCursorPosition(this.getTransform().transformPoint(e.clientX, e.clientY)); } + @action onContextMenu = (e: React.MouseEvent) => { let layoutItems: ContextMenuProps[] = []; layoutItems.push({ @@ -600,7 +601,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { ContextMenu.Instance.addItem({ description: "Recommender System", event: async () => { - new ClientRecommender(); + // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); let activedocs = this.getActiveDocuments(); await Promise.all(activedocs.map((doc: Doc) => { console.log(StrCast(doc.title)); @@ -715,6 +716,7 @@ class CollectionFreeFormViewPannableContents extends React.Component otherwise, reactions won't fire return
{this.props.children} +
; } } \ No newline at end of file diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index ea59703c3..d175b67c7 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -10,6 +10,7 @@ export class Recommender { private _model: any; static Instance: Recommender; + private dimension: number = 0; constructor() { console.log("creating recommender..."); @@ -25,6 +26,7 @@ export class Recommender { return new Promise(res => { w2v.loadModel("./node_modules/word2vec/vectors.txt", function (err: any, model: any) { self._model = model; + self.dimension = model.size; res(model); }); }); -- cgit v1.2.3-70-g09d2 From 116f218c2eeec377fb465027bcfaa7521d9af492 Mon Sep 17 00:00:00 2001 From: ab Date: Thu, 8 Aug 2019 17:48:58 -0400 Subject: euclidian --- src/client/ClientRecommender.scss | 8 ++ src/client/ClientRecommender.tsx | 100 +++++++++++++-------- .../collectionFreeForm/CollectionFreeFormView.tsx | 3 +- 3 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 src/client/ClientRecommender.scss (limited to 'src/client/views/collections') diff --git a/src/client/ClientRecommender.scss b/src/client/ClientRecommender.scss new file mode 100644 index 000000000..710d80f34 --- /dev/null +++ b/src/client/ClientRecommender.scss @@ -0,0 +1,8 @@ +@import "/views/globalCssVariables.scss"; + +.space{ + border: 1px dashed blue; + border-spacing: 25px; + border-collapse: separate; + align-content: center; +} \ No newline at end of file diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 2344ee490..0793f0887 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -6,7 +6,7 @@ import React = require("react"); import { observer } from "mobx-react"; import { observable, action, computed, reaction } from "mobx"; var assert = require('assert'); -import Table from 'react-bootstrap/Table'; +import "./ClientRecommender.scss"; export interface RecommenderProps { title: string; @@ -17,37 +17,53 @@ export class ClientRecommender extends React.Component { static Instance: ClientRecommender; private docVectors: Set; - private corr_matrix = observable([[0, 0], [0, 0]]); - @observable private firstnum = 0; - //@observable private corr_matrix: number[][] = [[0, 0], [0, 0]]; + @observable private corr_matrix = [[0, 0], [0, 0]]; constructor(props: RecommenderProps) { //console.log("creating client recommender..."); super(props); - ClientRecommender.Instance = this; + if (!ClientRecommender.Instance) ClientRecommender.Instance = this; this.docVectors = new Set(); //this.corr_matrix = [[0, 0], [0, 0]]; } + @action + public reset_docs() { + this.docVectors = new Set(); + this.corr_matrix = [[0, 0], [0, 0]]; + } /*** * Computes the cosine similarity between two vectors in Euclidean space. */ - private distance(vector1: number[], vector2: number[]) { + private distance(vector1: number[], vector2: number[], metric: string = "cosine") { assert(vector1.length === vector2.length, "Vectors are not the same length"); - var dotproduct = 0; - var mA = 0; - var mB = 0; - for (let i = 0; i < vector1.length; i++) { // here you missed the i++ - dotproduct += (vector1[i] * vector2[i]); - mA += (vector1[i] * vector1[i]); - mB += (vector2[i] * vector2[i]); + let similarity: number; + switch (metric) { + case "cosine": + var dotproduct = 0; + var mA = 0; + var mB = 0; + for (let i = 0; i < vector1.length; i++) { // here you missed the i++ + dotproduct += (vector1[i] * vector2[i]); + mA += (vector1[i] * vector1[i]); + mB += (vector2[i] * vector2[i]); + } + mA = Math.sqrt(mA); + mB = Math.sqrt(mB); + similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets + return similarity; + case "euclidian": + var sum = 0; + for (let i = 0; i < vector1.length; i++) { + sum += Math.pow(vector1[i] - vector2[i], 2); + } + similarity = Math.sqrt(sum); + return similarity; + default: + return 0; } - mA = Math.sqrt(mA); - mB = Math.sqrt(mB); - var similarity = (dotproduct) / ((mA) * (mB)); // here you needed extra brackets - return similarity; } /*** @@ -98,8 +114,6 @@ export class ClientRecommender extends React.Component { @action public createDistanceMatrix(documents: Set = this.docVectors) { - //this.corr_matrix[0][0] = 500; - this.firstnum = 500; const documents_list = Array.from(documents); const n = documents_list.length; var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); @@ -107,40 +121,48 @@ export class ClientRecommender extends React.Component { var doc1 = documents_list[i]; for (let j = 0; j < n; j++) { var doc2 = documents_list[j]; - matrix[i][j] = this.distance(doc1, doc2); + matrix[i][j] = this.distance(doc1, doc2, "euclidian"); } } - //this.corr_matrix = matrix; - + this.corr_matrix = matrix; return matrix; } - @computed get first_num() { - return this.firstnum; - } - - dumb_reaction = reaction( - () => this.first_num, - number => { - console.log("number has changed", number); - this.forceUpdate(); + @computed + private get generateRows() { + const n = this.corr_matrix.length; + let rows: React.ReactElement[] = []; + for (let i = 0; i < n; i++) { + let children: React.ReactElement[] = []; + for (let j = 0; j < n; j++) { + let cell = React.createElement("td", this.corr_matrix[i][j]); + children.push(cell); + } + let row = React.createElement("tr", { children: children }); + rows.push(row); } - ); + return rows; + } render() { return (

{this.props.title ? this.props.title : "hello"}

- + {/*
- - - + + + - - - + + + +
{this.first_num}{this.corr_matrix[0][1]}
{this.corr_matrix[0][0].toFixed(4)}{this.corr_matrix[0][1].toFixed(4)}
{this.corr_matrix[1][0]}{this.corr_matrix[1][1]}
{this.corr_matrix[1][0].toFixed(4)}{this.corr_matrix[1][1].toFixed(4)}
*/} + + + {this.generateRows} +
); } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 5259b9b49..2b9f32136 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -603,6 +603,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { event: async () => { // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); let activedocs = this.getActiveDocuments(); + ClientRecommender.Instance.reset_docs(); await Promise.all(activedocs.map((doc: Doc) => { console.log(StrCast(doc.title)); const extdoc = doc.data_ext as Doc; @@ -716,7 +717,7 @@ class CollectionFreeFormViewPannableContents extends React.Component otherwise, reactions won't fire return
{this.props.children} - +
; } } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From b8ab5a823ae22b021c09dfd713b77211a51b3eae Mon Sep 17 00:00:00 2001 From: ab Date: Fri, 9 Aug 2019 17:04:25 -0400 Subject: idk man --- solr-8.1.1/server/solr/dash/conf/schema.xml | 2 +- src/client/ClientRecommender.scss | 4 ++++ src/client/ClientRecommender.tsx | 17 ++++++++++------- src/client/util/SearchUtil.ts | 21 ++++++++++++++++++++- src/client/views/SearchBox.tsx | 2 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 6 +++++- src/server/Search.ts | 2 +- 7 files changed, 42 insertions(+), 12 deletions(-) (limited to 'src/client/views/collections') diff --git a/solr-8.1.1/server/solr/dash/conf/schema.xml b/solr-8.1.1/server/solr/dash/conf/schema.xml index 8610786af..c0a4bab07 100644 --- a/solr-8.1.1/server/solr/dash/conf/schema.xml +++ b/solr-8.1.1/server/solr/dash/conf/schema.xml @@ -52,7 +52,7 @@ - + diff --git a/src/client/ClientRecommender.scss b/src/client/ClientRecommender.scss index 710d80f34..49163cdc8 100644 --- a/src/client/ClientRecommender.scss +++ b/src/client/ClientRecommender.scss @@ -5,4 +5,8 @@ border-spacing: 25px; border-collapse: separate; align-content: center; +} + +.wrapper{ + text-align: -webkit-center; } \ No newline at end of file diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 0793f0887..ddaa8a7fc 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -7,6 +7,7 @@ import { observer } from "mobx-react"; import { observable, action, computed, reaction } from "mobx"; var assert = require('assert'); import "./ClientRecommender.scss"; +import { JSXElement } from "babel-types"; export interface RecommenderProps { title: string; @@ -131,22 +132,24 @@ export class ClientRecommender extends React.Component { @computed private get generateRows() { const n = this.corr_matrix.length; - let rows: React.ReactElement[] = []; + let rows: JSX.Element[] = []; for (let i = 0; i < n; i++) { - let children: React.ReactElement[] = []; + let children: JSX.Element[] = []; for (let j = 0; j < n; j++) { - let cell = React.createElement("td", this.corr_matrix[i][j]); + //let cell = React.createElement("td", this.corr_matrix[i][j]); + let cell = {this.corr_matrix[i][j].toFixed(4)}; children.push(cell); } - let row = React.createElement("tr", { children: children }); + //let row = React.createElement("tr", { children: children, key: i }); + let row = {children}; rows.push(row); } return rows; } render() { - return (
-

{this.props.title ? this.props.title : "hello"}

+ return (
+

{this.props.title ? this.props.title : "hello"}

{/* @@ -159,7 +162,7 @@ export class ClientRecommender extends React.Component {
*/} - +
{this.generateRows} diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts index ee5a83710..3a3ba1803 100644 --- a/src/client/util/SearchUtil.ts +++ b/src/client/util/SearchUtil.ts @@ -77,4 +77,23 @@ export namespace SearchUtil { aliasContexts.forEach(result => contexts.aliasContexts.push(...result.ids)); return contexts; } -} \ No newline at end of file + + export async function GetAllDocs() { + const query = "*"; + let response = await rp.get(Utils.prepend('/search'), { + qs: { + query + } + }); + let res: string[] = JSON.parse(response); + const fields = await DocServer.GetRefFields(res); + const docs: Doc[] = []; + for (const id of res) { + const field = fields[id]; + if (field instanceof Doc) { + docs.push(field); + } + } + return docs; + } +} diff --git a/src/client/views/SearchBox.tsx b/src/client/views/SearchBox.tsx index 33cb63df5..17f99fa05 100644 --- a/src/client/views/SearchBox.tsx +++ b/src/client/views/SearchBox.tsx @@ -47,7 +47,7 @@ export class SearchBox extends React.Component { } @action - getResults = async (query: string) => { + public getResults = async (query: string) => { let response = await rp.get(Utils.prepend('/search'), { qs: { query diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 2b9f32136..0beb0086b 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -38,6 +38,8 @@ import { MarqueeView } from "./MarqueeView"; import React = require("react"); import v5 = require("uuid/v5"); import { ClientRecommender } from "../../../ClientRecommender"; +import { SearchUtil } from "../../../util/SearchUtil"; +import { SearchBox } from "../../SearchBox"; library.add(faEye, faTable, faPaintBrush, faExpandArrowsAlt, faCompressArrowsAlt, faCompass, faUpload, faBrain); @@ -603,9 +605,11 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { event: async () => { // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); let activedocs = this.getActiveDocuments(); + let allDocs = await SearchUtil.GetAllDocs(); + allDocs.forEach(doc => console.log(doc.title)); ClientRecommender.Instance.reset_docs(); await Promise.all(activedocs.map((doc: Doc) => { - console.log(StrCast(doc.title)); + //console.log(StrCast(doc.title)); const extdoc = doc.data_ext as Doc; return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); })); diff --git a/src/server/Search.ts b/src/server/Search.ts index 723dc101b..44c1b1298 100644 --- a/src/server/Search.ts +++ b/src/server/Search.ts @@ -14,7 +14,7 @@ export class Search { }); return res; } catch (e) { - // console.warn("Search error: " + e + document); + console.warn("Search error: " + e + document); } } -- cgit v1.2.3-70-g09d2 From 9dd2a31b72e5e527e2dae3b68f856ab8da879e93 Mon Sep 17 00:00:00 2001 From: ab Date: Mon, 12 Aug 2019 16:41:23 -0400 Subject: documentation --- package.json | 2 +- src/client/ClientRecommender.tsx | 18 +++++----- src/client/cognitive_services/CognitiveServices.ts | 42 ++++++++++++---------- src/client/util/SearchUtil.ts | 13 ++++--- .../collectionFreeForm/CollectionFreeFormView.tsx | 1 + src/server/Recommender.ts | 1 + 6 files changed, 45 insertions(+), 32 deletions(-) (limited to 'src/client/views/collections') diff --git a/package.json b/package.json index 1e2c74411..1c7a10ac8 100644 --- a/package.json +++ b/package.json @@ -225,4 +225,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index ddaa8a7fc..63f85c737 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -75,13 +75,15 @@ export class ClientRecommender extends React.Component { const n = 200; const num_words = paragraph.size; let meanVector = new Array(n).fill(0); // mean vector - paragraph.forEach((wordvec: number[]) => { - for (let i = 0; i < n; i++) { - meanVector[i] += wordvec[i]; - } - }); - meanVector = meanVector.map(x => x / num_words); - this.addToDocSet(meanVector); + if (num_words > 0) { // check to see if paragraph actually was vectorized + paragraph.forEach((wordvec: number[]) => { + for (let i = 0; i < n; i++) { + meanVector[i] += wordvec[i]; + } + }); + meanVector = meanVector.map(x => x / num_words); + this.addToDocSet(meanVector); + } return meanVector; } @@ -106,7 +108,7 @@ export class ClientRecommender extends React.Component { }); return keyterms; }; - await CognitiveServices.Text.Manager.analyzer(extDoc, ["key words"], data, converter); + await CognitiveServices.Text.Appliers.analyzer(extDoc, ["key words"], data, converter); } /*** diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 954a05585..75d0760ed 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -258,32 +258,38 @@ export namespace CognitiveServices { }; console.log("requested!"); return request.post(options); - }, - analyzer: async (target: Doc, keys: string[], data: string, converter: Converter) => { - let results = await ExecuteQuery(Service.Text, Manager, data); + } + }; + + export namespace Appliers { + + export async function vectorize(keyterms: any) { + console.log("vectorizing..."); + //keyterms = ["father", "king"]; + let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; + await requestPromise.post(args).then(async (wordvecs) => { + var vectorValues = new Set(); + wordvecs.forEach((wordvec: any) => { + //console.log(wordvec.word); + vectorValues.add(wordvec.values as number[]); + }); + ClientRecommender.Instance.mean(vectorValues); + //console.log(vectorValues.size); + }); + } + + export const analyzer = async (target: Doc, keys: string[], data: string, converter: Converter) => { + let results = await ExecuteQuery(Service.Text, Manager, data); console.log(results); let keyterms = converter(results); //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); target[keys[0]] = keyterms; console.log("analyzed!"); await vectorize(keyterms); - } - }; - async function vectorize(keyterms: any) { - console.log("vectorizing..."); - //keyterms = ["father", "king"]; - let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; - await requestPromise.post(args).then(async (wordvecs) => { - var vectorValues = new Set(); - wordvecs.forEach((wordvec: any) => { - //console.log(wordvec.word); - vectorValues.add(wordvec.values as number[]); - }); - ClientRecommender.Instance.mean(vectorValues); - //console.log(vectorValues.size); - }); + }; } } + } \ No newline at end of file diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts index 3a3ba1803..1fce995d7 100644 --- a/src/client/util/SearchUtil.ts +++ b/src/client/util/SearchUtil.ts @@ -82,18 +82,21 @@ export namespace SearchUtil { const query = "*"; let response = await rp.get(Utils.prepend('/search'), { qs: { - query + q: query } }); - let res: string[] = JSON.parse(response); - const fields = await DocServer.GetRefFields(res); + let result: IdSearchResult = JSON.parse(response); + const { ids, numFound, highlighting } = result; + const docMap = await DocServer.GetRefFields(ids); const docs: Doc[] = []; - for (const id of res) { - const field = fields[id]; + for (const id of ids) { + const field = docMap[id]; if (field instanceof Doc) { docs.push(field); } } return docs; + // const docs = ids.map((id: string) => docMap[id]).filter((doc: any) => doc instanceof Doc); + // return docs as Doc[]; } } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index e9791df4e..d1e8031fd 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -894,6 +894,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { let activedocs = this.getActiveDocuments(); let allDocs = await SearchUtil.GetAllDocs(); allDocs.forEach(doc => console.log(doc.title)); + // clears internal representation of documents as vectors ClientRecommender.Instance.reset_docs(); await Promise.all(activedocs.map((doc: Doc) => { //console.log(StrCast(doc.title)); diff --git a/src/server/Recommender.ts b/src/server/Recommender.ts index d175b67c7..1c95d7ea4 100644 --- a/src/server/Recommender.ts +++ b/src/server/Recommender.ts @@ -70,6 +70,7 @@ export class Recommender { } if (this._model) { let word_vecs = this._model.getVectors(text); + return word_vecs; } } -- cgit v1.2.3-70-g09d2 From e03a1b2cc90e0fdb7789f4826e482e9040aa7075 Mon Sep 17 00:00:00 2001 From: ab Date: Tue, 13 Aug 2019 17:26:49 -0400 Subject: 80% done, garbage collection is much needed --- src/Recommendations.scss | 21 --- src/Recommendations.tsx | 28 ---- src/client/util/SearchUtil.ts | 7 +- src/client/views/MainView.tsx | 2 + src/client/views/Recommendations.scss | 65 ++++++++ src/client/views/Recommendations.tsx | 169 +++++++++++++++++++++ .../collectionFreeForm/CollectionFreeFormView.tsx | 13 +- src/client/views/nodes/DocumentView.tsx | 34 ++++- src/client/views/nodes/ImageBox.tsx | 21 ++- 9 files changed, 297 insertions(+), 63 deletions(-) delete mode 100644 src/Recommendations.scss delete mode 100644 src/Recommendations.tsx create mode 100644 src/client/views/Recommendations.scss create mode 100644 src/client/views/Recommendations.tsx (limited to 'src/client/views/collections') diff --git a/src/Recommendations.scss b/src/Recommendations.scss deleted file mode 100644 index 5129a59d9..000000000 --- a/src/Recommendations.scss +++ /dev/null @@ -1,21 +0,0 @@ -.recommendation-content *{ - display: inline-block; - margin: auto; - border: 1px dashed grey; - padding: 2px 2px; -} - -.recommendation-content { - float: left; - border: 1px solid green; - width: 200px; - align-content: center; -} - -.rec-scroll { - overflow-y: scroll; - height: 300px; - width: auto; - position: absolute; - background: #cdcdcd; -} \ No newline at end of file diff --git a/src/Recommendations.tsx b/src/Recommendations.tsx deleted file mode 100644 index ca1123ef9..000000000 --- a/src/Recommendations.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import { observer } from "mobx-react"; -import React = require("react"); -import { Doc } from "./new_fields/Doc"; -import { NumCast } from "./new_fields/Types"; - -export interface RecProps { - documents: { preview: string, similarity: number }[], - node: Doc; -} - -@observer -export class Recommendations extends React.Component { - render() { - const transform = "translate(" + (NumCast(this.props.node.x) + 350) + "px, " + NumCast(this.props.node.y) + "px" - return ( -
- {this.props.documents.map(doc => { - return ( -
- -
{doc.similarity}
-
- ) - })} -
- ) - } -} \ No newline at end of file diff --git a/src/client/util/SearchUtil.ts b/src/client/util/SearchUtil.ts index 1fce995d7..85e593529 100644 --- a/src/client/util/SearchUtil.ts +++ b/src/client/util/SearchUtil.ts @@ -81,12 +81,13 @@ export namespace SearchUtil { export async function GetAllDocs() { const query = "*"; let response = await rp.get(Utils.prepend('/search'), { - qs: { - q: query - } + qs: + { start: 0, rows: 10000, q: query }, + }); let result: IdSearchResult = JSON.parse(response); const { ids, numFound, highlighting } = result; + console.log(ids.length); const docMap = await DocServer.GetRefFields(ids); const docs: Doc[] = []; for (const id of ids) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 419b15697..0b6fe3876 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -41,6 +41,7 @@ import { ClientUtils } from '../util/ClientUtils'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; //import { DocumentManager } from '../util/DocumentManager'; import { DictationManager } from '../util/DictationManager'; +import { Recommendations } from './Recommendations'; @observer export class MainView extends React.Component { @@ -581,6 +582,7 @@ export class MainView extends React.Component { {this.mainContent} + {this.nodesMenu()} {this.miscButtons} diff --git a/src/client/views/Recommendations.scss b/src/client/views/Recommendations.scss new file mode 100644 index 000000000..5d8f17e37 --- /dev/null +++ b/src/client/views/Recommendations.scss @@ -0,0 +1,65 @@ +@import "globalCssVariables"; + +.rec-content *{ + display: inline-block; + margin: auto; + width: 50; + height: 30px; + border: 1px dashed grey; + padding: 10px 10px; +} + +.rec-content { + float: left; + width: inherit; + align-content: center; +} + +.rec-scroll { + overflow-y: scroll; + overflow-x: hidden; + position: absolute; + // display: flex; + z-index: 10000; + box-shadow: gray 0.2vw 0.2vw 0.4vw; + // flex-direction: column; + background: whitesmoke; + padding-bottom: 10px; + border-radius: 15px; + border: solid #BBBBBBBB 1px; + width: 200px; + text-align: center; + max-height: 250px; + text-transform: uppercase; + color: grey; + letter-spacing: 2px; +} + +.content { + padding: 10px; + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; +} + +.image-background { + pointer-events: none; + background-color: transparent; + width: 50%; + text-align: center; + height: 35px; + margin-left: 5px; +} + +img{ + width: 100%; + height: 100%; +} + +.score { + // margin-left: 15px; + width: 50%; + height: 100%; + text-align: center; +} diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx new file mode 100644 index 000000000..8569996b3 --- /dev/null +++ b/src/client/views/Recommendations.tsx @@ -0,0 +1,169 @@ +import { observer } from "mobx-react"; +import React = require("react"); +import { observable, action } from "mobx"; +import Measure from "react-measure"; +import "./Recommendations.scss"; +import { Doc, DocListCast, WidthSym, HeightSym } from "../../new_fields/Doc"; +import { DocumentIcon } from "./nodes/DocumentIcon"; +import { StrCast, NumCast } from "../../new_fields/Types"; +import { returnFalse, emptyFunction, returnEmptyString, returnOne } from "../../Utils"; +import { Transform } from "../util/Transform"; +import { ObjectField } from "../../new_fields/ObjectField"; +import { DocumentView } from "./nodes/DocumentView"; +import { DocumentType } from "../documents/Documents"; + + +export interface RecProps { + documents: { preview: Doc, similarity: number }[]; + node: Doc; +} + +@observer +export class Recommendations extends React.Component<{}> { + + static Instance: Recommendations; + @observable private _display: boolean = false; + @observable private _pageX: number = 0; + @observable private _pageY: number = 0; + @observable private _width: number = 0; + @observable private _height: number = 0; + @observable private _documents: { preview: Doc, score: number }[] = []; + + constructor(props: {}) { + super(props); + Recommendations.Instance = this; + } + + private DocumentIcon(doc: Doc) { + let layoutresult = StrCast(doc.type); + let renderDoc = doc; + //let box: number[] = []; + if (layoutresult.indexOf(DocumentType.COL) !== -1) { + renderDoc = Doc.MakeDelegate(renderDoc); + let bounds = DocListCast(renderDoc.data).reduce((bounds, doc) => { + var [sptX, sptY] = [NumCast(doc.x), NumCast(doc.y)]; + let [bptX, bptY] = [sptX + doc[WidthSym](), sptY + doc[HeightSym]()]; + return { + x: Math.min(sptX, bounds.x), y: Math.min(sptY, bounds.y), + r: Math.max(bptX, bounds.r), b: Math.max(bptY, bounds.b) + }; + }, { x: Number.MAX_VALUE, y: Number.MAX_VALUE, r: Number.MIN_VALUE, b: Number.MIN_VALUE }); + } + let returnXDimension = () => 50; + let returnYDimension = () => 50; + let scale = () => returnXDimension() / NumCast(renderDoc.nativeWidth, returnXDimension()); + let newRenderDoc = Doc.MakeDelegate(renderDoc); /// newRenderDoc -> renderDoc -> render"data"Doc -> TextProt + const docview =
+ {/* onPointerDown={action(() => { + this._useIcons = !this._useIcons; + this._displayDim = this._useIcons ? 50 : Number(SEARCH_THUMBNAIL_SIZE); + })} + onPointerEnter={action(() => this._displayDim = this._useIcons ? 50 : Number(SEARCH_THUMBNAIL_SIZE))} + onPointerLeave={action(() => this._displayDim = 50)} > */} + +
; + const data = renderDoc.data; + if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data); + newRenderDoc.preview = true; + return docview; + + } + + @action + closeMenu = () => { + this._display = false; + } + + @action + resetDocuments = () => { + this._documents = []; + } + + @action + addDocuments = (documents: { preview: Doc, score: number }[]) => { + this._documents = documents; + } + + @action + displayRecommendations(x: number, y: number) { + this._pageX = x; + this._pageY = y; + this._display = true; + } + + static readonly buffer = 20; + + get pageX() { + const x = this._pageX; + if (x < 0) { + return 0; + } + const width = this._width; + if (x + width > window.innerWidth - Recommendations.buffer) { + return window.innerWidth - Recommendations.buffer - width; + } + return x; + } + + get pageY() { + const y = this._pageY; + if (y < 0) { + return 0; + } + const height = this._height; + if (y + height > window.innerHeight - Recommendations.buffer) { + return window.innerHeight - Recommendations.buffer - height; + } + return y; + } + + render() { + if (!this._display) { + return null; + } + let style = { left: this.pageX, top: this.pageY }; + //const transform = "translate(" + (NumCast(this.props.node.x) + 350) + "px, " + NumCast(this.props.node.y) + "px" + return ( + { this._width = r.offset.width; this._height = r.offset.height; })}> + {({ measureRef }) => ( +
+

Recommendations

+ {this._documents.map(doc => { + return ( +
+ + {this.DocumentIcon(doc.preview)} + + {doc.score} +
+ ); + })} + +
+ ) + } + + + ); + } +} \ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index d1e8031fd..50f7e2dc8 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -896,10 +896,13 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { allDocs.forEach(doc => console.log(doc.title)); // clears internal representation of documents as vectors ClientRecommender.Instance.reset_docs(); - await Promise.all(activedocs.map((doc: Doc) => { - //console.log(StrCast(doc.title)); - const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + await Promise.all(allDocs.map((doc: Doc) => { + console.log(StrCast(doc.title)); + if (doc.type === DocumentType.IMG) { + console.log(doc.title); + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + } })); console.log(ClientRecommender.Instance.createDistanceMatrix()); }, @@ -1013,7 +1016,7 @@ class CollectionFreeFormViewPannableContents extends React.Component otherwise, reactions won't fire return
{this.props.children} - + {/* */}
; } } \ No newline at end of file diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 24bcc0217..3ce4dbd4f 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -40,9 +40,13 @@ import React = require("react"); import { DictationManager } from '../../util/DictationManager'; import { MainView } from '../MainView'; import requestPromise = require('request-promise'); -import { Recommendations } from '../../../Recommendations'; +import { Recommendations } from '../Recommendations'; +import { SearchUtil } from '../../util/SearchUtil'; +import { ClientRecommender } from '../../ClientRecommender'; +import { DocumentType } from '../../documents/Documents'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? +library.add(fa.faBrain); library.add(fa.faTrash); library.add(fa.faShare); library.add(fa.faDownload); @@ -610,6 +614,33 @@ export class DocumentView extends DocComponent(Docu a.click(); } }); + cm.addItem({ + description: "Recommender System", + event: async () => { + if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); + let documents: Doc[] = []; + let allDocs = await SearchUtil.GetAllDocs(); + allDocs.forEach(doc => console.log(doc.title)); + // clears internal representation of documents as vectors + ClientRecommender.Instance.reset_docs(); + await Promise.all(allDocs.map((doc: Doc) => { + if (doc.type === DocumentType.IMG) { + console.log(StrCast(doc.title)); + documents.push(doc); + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + } + })); + console.log(ClientRecommender.Instance.createDistanceMatrix()); + let recDocs: { preview: Doc, score: number }[] = []; + for (let i = 0; i < documents.length; i++) { + recDocs.push({ preview: documents[i], score: i }); + } + Recommendations.Instance.addDocuments(recDocs); + Recommendations.Instance.displayRecommendations(e.pageX + 100, e.pageY); + }, + icon: "brain" + }); cm.addItem({ description: "Delete", event: this.deleteClicked, icon: "trash" }); type User = { email: string, userDocumentId: string }; let usersMenu: ContextMenuProps[] = []; @@ -758,7 +789,6 @@ export class DocumentView extends DocComponent(Docu } - ); } diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 73b892e26..45d389ba6 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -28,6 +28,9 @@ import FaceRectangles from './FaceRectangles'; import { FieldView, FieldViewProps } from './FieldView'; import "./ImageBox.scss"; import React = require("react"); +import { SearchUtil } from '../../util/SearchUtil'; +import { ClientRecommender } from '../../ClientRecommender'; +import { DocumentType } from '../../documents/Documents'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl } = require('howler'); @@ -240,10 +243,20 @@ export class ImageBox extends DocComponent(ImageD } } - extractText = () => { - //Recommender.Instance.extractText(this.dataDoc, this.extensionDoc); - // request recommender - //fetch(Utils.prepend("/recommender"), { body: body, method: "POST", headers: { "content-type": "application/json" } }).then((value) => console.log(value)); + extractText = async () => { + //let activedocs = this.getActiveDocuments(); + let allDocs = await SearchUtil.GetAllDocs(); + allDocs.forEach(doc => console.log(doc.title)); + // clears internal representation of documents as vectors + ClientRecommender.Instance.reset_docs(); + await Promise.all(allDocs.map((doc: Doc) => { + //console.log(StrCast(doc.title)); + if (doc.type === DocumentType.IMG) { + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + } + })); + console.log(ClientRecommender.Instance.createDistanceMatrix()); } generateMetadata = (threshold: Confidence = Confidence.Excellent) => { -- cgit v1.2.3-70-g09d2 From e0bfe978e029268b3901b5d098f946b1a6fc7d0d Mon Sep 17 00:00:00 2001 From: ab Date: Thu, 29 Aug 2019 18:43:32 -0400 Subject: ui fixes, datadoc resolved --- src/client/ClientRecommender.tsx | 83 +++++++++++++++------- src/client/cognitive_services/CognitiveServices.ts | 24 ++++--- src/client/views/MainView.tsx | 1 + src/client/views/Recommendations.tsx | 21 ++++-- .../collectionFreeForm/CollectionFreeFormView.tsx | 21 ------ src/client/views/nodes/DocumentView.tsx | 26 ++++--- src/client/views/nodes/ImageBox.tsx | 16 ----- 7 files changed, 104 insertions(+), 88 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 63f85c737..a6d1a32b3 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -1,5 +1,5 @@ import { Doc } from "../new_fields/Doc"; -import { StrCast } from "../new_fields/Types"; +import { StrCast, Cast } from "../new_fields/Types"; import { List } from "../new_fields/List"; import { CognitiveServices } from "./cognitive_services/CognitiveServices"; import React = require("react"); @@ -8,30 +8,42 @@ import { observable, action, computed, reaction } from "mobx"; var assert = require('assert'); import "./ClientRecommender.scss"; import { JSXElement } from "babel-types"; +import { ToPlainText, RichTextField } from "../new_fields/RichTextField"; export interface RecommenderProps { title: string; } +export interface RecommenderDocument { + actualDoc: Doc; + vectorDoc: number[]; + score: number; +} + @observer export class ClientRecommender extends React.Component { static Instance: ClientRecommender; - private docVectors: Set; + private mainDoc?: RecommenderDocument; + private docVectors: Set = new Set(); @observable private corr_matrix = [[0, 0], [0, 0]]; constructor(props: RecommenderProps) { //console.log("creating client recommender..."); super(props); if (!ClientRecommender.Instance) ClientRecommender.Instance = this; - this.docVectors = new Set(); - //this.corr_matrix = [[0, 0], [0, 0]]; + ClientRecommender.Instance.docVectors = new Set(); + //ClientRecommender.Instance.corr_matrix = [[0, 0], [0, 0]]; } @action public reset_docs() { - this.docVectors = new Set(); - this.corr_matrix = [[0, 0], [0, 0]]; + ClientRecommender.Instance.docVectors = new Set(); + ClientRecommender.Instance.corr_matrix = [[0, 0], [0, 0]]; + } + + public deleteDocs() { + console.log("deleting previews..."); } /*** @@ -67,11 +79,24 @@ export class ClientRecommender extends React.Component { } } + public computeSimilarities() { + ClientRecommender.Instance.docVectors.forEach((doc: RecommenderDocument) => { + if (ClientRecommender.Instance.mainDoc) { + const distance = ClientRecommender.Instance.distance(ClientRecommender.Instance.mainDoc.vectorDoc, doc.vectorDoc, "euclidian"); + doc.score = distance; + } + } + ); + let doclist = Array.from(ClientRecommender.Instance.docVectors); + doclist.sort((a: RecommenderDocument, b: RecommenderDocument) => a.score - b.score); + return doclist; + } + /*** * Computes the mean of a set of vectors */ - public mean(paragraph: Set) { + public mean(paragraph: Set, dataDoc: Doc, mainDoc: boolean) { const n = 200; const num_words = paragraph.size; let meanVector = new Array(n).fill(0); // mean vector @@ -82,14 +107,16 @@ export class ClientRecommender extends React.Component { } }); meanVector = meanVector.map(x => x / num_words); - this.addToDocSet(meanVector); + const internalDoc: RecommenderDocument = { actualDoc: dataDoc, vectorDoc: meanVector, score: 0 }; + if (mainDoc) ClientRecommender.Instance.mainDoc = internalDoc; + ClientRecommender.Instance.addToDocSet(internalDoc); } return meanVector; } - private addToDocSet(vector: number[]) { - if (this.docVectors) { - this.docVectors.add(vector); + private addToDocSet(internalDoc: RecommenderDocument) { + if (ClientRecommender.Instance.docVectors) { + ClientRecommender.Instance.docVectors.add(internalDoc); } } @@ -97,9 +124,11 @@ export class ClientRecommender extends React.Component { * Uses Cognitive Services to extract keywords from a document */ - public async extractText(dataDoc: Doc, extDoc: Doc) { - let data = StrCast(dataDoc.title); - //console.log(data); + public async extractText(dataDoc: Doc, extDoc: Doc, mainDoc: boolean = false) { + let fielddata = Cast(dataDoc.data, RichTextField); + let data: string; + fielddata ? data = fielddata[ToPlainText]() : data = ""; + console.log(data); let converter = (results: any) => { let keyterms = new List(); results.documents.forEach((doc: any) => { @@ -108,7 +137,7 @@ export class ClientRecommender extends React.Component { }); return keyterms; }; - await CognitiveServices.Text.Appliers.analyzer(extDoc, ["key words"], data, converter); + await CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc); } /*** @@ -116,7 +145,7 @@ export class ClientRecommender extends React.Component { */ @action - public createDistanceMatrix(documents: Set = this.docVectors) { + public createDistanceMatrix(documents: Set = ClientRecommender.Instance.docVectors) { const documents_list = Array.from(documents); const n = documents_list.length; var matrix = new Array(n).fill(0).map(() => new Array(n).fill(0)); @@ -124,22 +153,22 @@ export class ClientRecommender extends React.Component { var doc1 = documents_list[i]; for (let j = 0; j < n; j++) { var doc2 = documents_list[j]; - matrix[i][j] = this.distance(doc1, doc2, "euclidian"); + matrix[i][j] = ClientRecommender.Instance.distance(doc1.vectorDoc, doc2.vectorDoc, "euclidian"); } } - this.corr_matrix = matrix; + ClientRecommender.Instance.corr_matrix = matrix; return matrix; } @computed private get generateRows() { - const n = this.corr_matrix.length; + const n = ClientRecommender.Instance.corr_matrix.length; let rows: JSX.Element[] = []; for (let i = 0; i < n; i++) { let children: JSX.Element[] = []; for (let j = 0; j < n; j++) { - //let cell = React.createElement("td", this.corr_matrix[i][j]); - let cell = ; + //let cell = React.createElement("td", ClientRecommender.Instance.corr_matrix[i][j]); + let cell = ; children.push(cell); } //let row = React.createElement("tr", { children: children, key: i }); @@ -151,22 +180,22 @@ export class ClientRecommender extends React.Component { render() { return (
-

{this.props.title ? this.props.title : "hello"}

+

{ClientRecommender.Instance.props.title ? ClientRecommender.Instance.props.title : "hello"}

{/*
{this.corr_matrix[i][j].toFixed(4)}{ClientRecommender.Instance.corr_matrix[i][j].toFixed(4)}
- - + + - - + +
{this.corr_matrix[0][0].toFixed(4)}{this.corr_matrix[0][1].toFixed(4)}{ClientRecommender.Instance.corr_matrix[0][0].toFixed(4)}{ClientRecommender.Instance.corr_matrix[0][1].toFixed(4)}
{this.corr_matrix[1][0].toFixed(4)}{this.corr_matrix[1][1].toFixed(4)}{ClientRecommender.Instance.corr_matrix[1][0].toFixed(4)}{ClientRecommender.Instance.corr_matrix[1][1].toFixed(4)}
*/} - {this.generateRows} + {ClientRecommender.Instance.generateRows}
); diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 75d0760ed..874ee433d 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -263,29 +263,35 @@ export namespace CognitiveServices { export namespace Appliers { - export async function vectorize(keyterms: any) { + export async function vectorize(keyterms: any, dataDoc: Doc, mainDoc: boolean = false) { console.log("vectorizing..."); //keyterms = ["father", "king"]; let args = { method: 'POST', uri: Utils.prepend("/recommender"), body: { keyphrases: keyterms }, json: true }; await requestPromise.post(args).then(async (wordvecs) => { - var vectorValues = new Set(); - wordvecs.forEach((wordvec: any) => { - //console.log(wordvec.word); - vectorValues.add(wordvec.values as number[]); - }); - ClientRecommender.Instance.mean(vectorValues); + if (wordvecs.length > 0) { + console.log("successful vectorization!"); + var vectorValues = new Set(); + wordvecs.forEach((wordvec: any) => { + //console.log(wordvec.word); + vectorValues.add(wordvec.values as number[]); + }); + ClientRecommender.Instance.mean(vectorValues, dataDoc, mainDoc); + } // adds document to internal doc set + else { + console.log("unsuccessful :( word(s) not in vocabulary"); + } //console.log(vectorValues.size); }); } - export const analyzer = async (target: Doc, keys: string[], data: string, converter: Converter) => { + export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: Converter, mainDoc: boolean = false) => { let results = await ExecuteQuery(Service.Text, Manager, data); console.log(results); let keyterms = converter(results); //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); target[keys[0]] = keyterms; console.log("analyzed!"); - await vectorize(keyterms); + await vectorize(keyterms, dataDoc, mainDoc); }; } diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 57eb30439..3a5795077 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -204,6 +204,7 @@ export class MainView extends React.Component { const targets = document.elementsFromPoint(e.x, e.y); if (targets && targets.length && targets[0].className.toString().indexOf("contextMenu") === -1) { ContextMenu.Instance.closeMenu(); + Recommendations.Instance.closeMenu(); } }); diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx index 8569996b3..cf1974c69 100644 --- a/src/client/views/Recommendations.tsx +++ b/src/client/views/Recommendations.tsx @@ -10,8 +10,10 @@ import { returnFalse, emptyFunction, returnEmptyString, returnOne } from "../../ import { Transform } from "../util/Transform"; import { ObjectField } from "../../new_fields/ObjectField"; import { DocumentView } from "./nodes/DocumentView"; -import { DocumentType } from "../documents/Documents"; - +import { DocumentType } from '../documents/DocumentTypes'; +import { ClientRecommender } from "../ClientRecommender"; +import { DocServer } from "../DocServer"; +import { Id } from "../../new_fields/FieldSymbols"; export interface RecProps { documents: { preview: Doc, similarity: number }[]; @@ -28,6 +30,7 @@ export class Recommendations extends React.Component<{}> { @observable private _width: number = 0; @observable private _height: number = 0; @observable private _documents: { preview: Doc, score: number }[] = []; + private previewDocs: Doc[] = []; constructor(props: {}) { super(props); @@ -52,7 +55,8 @@ export class Recommendations extends React.Component<{}> { let returnXDimension = () => 50; let returnYDimension = () => 50; let scale = () => returnXDimension() / NumCast(renderDoc.nativeWidth, returnXDimension()); - let newRenderDoc = Doc.MakeDelegate(renderDoc); /// newRenderDoc -> renderDoc -> render"data"Doc -> TextProt + //let scale = () => 1; + //let newRenderDoc = Doc.MakeDelegate(renderDoc); /// newRenderDoc -> renderDoc -> render"data"Doc -> TextProt const docview =
{/* onPointerDown={action(() => { this._useIcons = !this._useIcons; @@ -62,7 +66,7 @@ export class Recommendations extends React.Component<{}> { onPointerLeave={action(() => this._displayDim = 50)} > */} { ContentScaling={scale} />
; - const data = renderDoc.data; - if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data); - newRenderDoc.preview = true; + // const data = renderDoc.data; + // if (data instanceof ObjectField) newRenderDoc.data = ObjectField.MakeCopy(data); + // newRenderDoc.preview = true; + // this.previewDocs.push(newRenderDoc); return docview; } @@ -92,6 +97,8 @@ export class Recommendations extends React.Component<{}> { @action closeMenu = () => { this._display = false; + this.previewDocs.forEach(doc => DocServer.DeleteDocument(doc[Id])); + this.previewDocs = []; } @action diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 2d4775070..3cef93383 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -857,27 +857,6 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { input.click(); } }); - ContextMenu.Instance.addItem({ - description: "Recommender System", - event: async () => { - // if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); - let activedocs = this.getActiveDocuments(); - let allDocs = await SearchUtil.GetAllDocs(); - allDocs.forEach(doc => console.log(doc.title)); - // clears internal representation of documents as vectors - ClientRecommender.Instance.reset_docs(); - await Promise.all(allDocs.map((doc: Doc) => { - console.log(StrCast(doc.title)); - if (doc.type === DocumentType.IMG) { - console.log(doc.title); - const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); - } - })); - console.log(ClientRecommender.Instance.createDistanceMatrix()); - }, - icon: "brain" - }); layoutItems.push({ description: `${this.fitToBox ? "Unset" : "Set"} Fit To Container`, event: this.fitToContainer, icon: !this.fitToBox ? "expand-arrows-alt" : "compress-arrows-alt" }); layoutItems.push({ description: "reset view", event: () => { this.props.Document.panX = this.props.Document.panY = 0; this.props.Document.scale = 1; }, icon: "compress-arrows-alt" }); layoutItems.push({ diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 2a6e91272..f708a7a3a 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -648,21 +648,31 @@ export class DocumentView extends DocComponent(Docu if (!ClientRecommender.Instance) new ClientRecommender({ title: "Client Recommender" }); let documents: Doc[] = []; let allDocs = await SearchUtil.GetAllDocs(); - allDocs.forEach(doc => console.log(doc.title)); + //allDocs.forEach(doc => console.log(doc.title)); // clears internal representation of documents as vectors ClientRecommender.Instance.reset_docs(); await Promise.all(allDocs.map((doc: Doc) => { - if (doc.type === DocumentType.IMG) { - console.log(StrCast(doc.title)); - documents.push(doc); - const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); + let mainDoc: boolean = false; + const dataDoc = Doc.GetDataDoc(doc); + if (doc.type === DocumentType.TEXT) { + if (dataDoc === Doc.GetDataDoc(this.props.Document)) { + mainDoc = true; + console.log(StrCast(doc.title)); + } + if (!documents.includes(dataDoc)) { + documents.push(dataDoc); + const extdoc = doc.data_ext as Doc; + return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, mainDoc); + } } })); console.log(ClientRecommender.Instance.createDistanceMatrix()); + const doclist = ClientRecommender.Instance.computeSimilarities(); let recDocs: { preview: Doc, score: number }[] = []; - for (let i = 0; i < documents.length; i++) { - recDocs.push({ preview: documents[i], score: i }); + // tslint:disable-next-line: prefer-for-of + for (let i = 0; i < doclist.length; i++) { + console.log(doclist[i].score); + recDocs.push({ preview: doclist[i].actualDoc, score: doclist[i].score }); } Recommendations.Instance.addDocuments(recDocs); Recommendations.Instance.displayRecommendations(e.pageX + 100, e.pageY); diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index ec35465eb..d94e92847 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -240,22 +240,6 @@ export class ImageBox extends DocComponent(ImageD } } - extractText = async () => { - //let activedocs = this.getActiveDocuments(); - let allDocs = await SearchUtil.GetAllDocs(); - allDocs.forEach(doc => console.log(doc.title)); - // clears internal representation of documents as vectors - ClientRecommender.Instance.reset_docs(); - await Promise.all(allDocs.map((doc: Doc) => { - //console.log(StrCast(doc.title)); - if (doc.type === DocumentType.IMG) { - const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc); - } - })); - console.log(ClientRecommender.Instance.createDistanceMatrix()); - } - generateMetadata = (threshold: Confidence = Confidence.Excellent) => { let converter = (results: any) => { let tagDoc = new Doc; -- cgit v1.2.3-70-g09d2 From 19375927c677ad6c99c77d0c7dac17fe7a2712a9 Mon Sep 17 00:00:00 2001 From: ab Date: Mon, 16 Sep 2019 15:26:36 -0400 Subject: beginning to handle external interactions --- src/client/ClientRecommender.tsx | 26 +++++++++++++--------- src/client/cognitive_services/CognitiveServices.ts | 12 ++++++---- src/client/views/Recommendations.tsx | 6 ++++- .../views/collections/CollectionSchemaCells.tsx | 16 ++++++++++++- src/client/views/nodes/DocumentView.tsx | 7 +++++- 5 files changed, 49 insertions(+), 18 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 217c89297..551047df0 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -139,7 +139,7 @@ export class ClientRecommender extends React.Component { let fielddata = Cast(dataDoc.data, RichTextField); let data: string; fielddata ? data = fielddata[ToPlainText]() : data = ""; - let converter = (results: any, data: string) => { + let converter = async (results: any, data: string) => { let keyterms = new List(); // raw keywords let keyterms_counted = new List(); // keywords, where each keyword is repeated as let highKP: string[] = [""]; // most frequent @@ -167,10 +167,10 @@ export class ClientRecommender extends React.Component { }); this.highKP = highKP; console.log(highKP); - this.sendRequest(highKP); - return { keyterms: keyterms, keyterms_counted: keyterms_counted }; + const values = await this.sendRequest(highKP); + return { keyterms: keyterms, keyterms_counted: keyterms_counted, values }; }; - await CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc, internal); + return CognitiveServices.Text.Appliers.analyzer(dataDoc, extDoc, ["key words"], data, converter, mainDoc, internal); } private countFrequencies(keyphrase: string, paragraph: string) { @@ -198,7 +198,9 @@ export class ClientRecommender extends React.Component { private async sendRequest(keywords: string[]) { let query = ""; keywords.forEach((kp: string) => query += " " + kp); - await this.arxivrequest(query); + return new Promise(resolve => { + this.arxivrequest(query).then(resolve); + }); } /** @@ -207,7 +209,7 @@ export class ClientRecommender extends React.Component { arxivrequest = async (query: string) => { let xhttp = new XMLHttpRequest(); - let serveraddress = "http://export.arxiv.org/api" + let serveraddress = "http://export.arxiv.org/api"; let endpoint = serveraddress + "/query?search_query=all:" + query + "&start=0&max_results=1"; let promisified = (resolve: any, reject: any) => { xhttp.onreadystatechange = function () { @@ -217,20 +219,22 @@ export class ClientRecommender extends React.Component { console.log(xml); switch (this.status) { case 200: + let title: string = "Title"; + let url: string = "Url"; //console.log(result); if (xml) { let titles = xml.getElementsByTagName("title"); if (titles && titles.length > 1) { - let text = titles[1].childNodes[0].nodeValue; - console.log(text); + title = titles[1].childNodes[0].nodeValue!; + console.log(title); } let ids = xml.getElementsByTagName("id"); if (ids && ids.length > 1) { - let text = ids[1].childNodes[0].nodeValue; - console.log(text); + url = ids[1].childNodes[0].nodeValue!; + console.log(url); } } - return resolve(result); + return resolve({ title, url }); case 400: default: return reject(result); diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index 8a58355a8..baafb63a1 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -15,7 +15,7 @@ type RequestExecutor = (apiKey: string, body: string, service: Service) => Promi type AnalysisApplier = (target: Doc, relevantKeys: string[], data: D, ...args: any) => any; type BodyConverter = (data: D) => string; type Converter = (results: any) => Field; -type TextConverter = (results: any, data: string) => { keyterms: Field, keyterms_counted: Field }; +type TextConverter = (results: any, data: string) => Promise<{ keyterms: Field, keyterms_counted: Field, values: any }>; export type Tag = { name: string, confidence: number }; export type Rectangle = { top: number, left: number, width: number, height: number }; @@ -288,11 +288,15 @@ export namespace CognitiveServices { export const analyzer = async (dataDoc: Doc, target: Doc, keys: string[], data: string, converter: TextConverter, mainDoc: boolean = false, internal: boolean = true) => { let results = await ExecuteQuery(Service.Text, Manager, data); console.log(results); - let keyterms = converter(results, data); + let { keyterms, values, keyterms_counted } = await converter(results, data); //target[keys[0]] = Docs.Get.DocumentHierarchyFromJson(results, "Key Word Analysis"); - target[keys[0]] = keyterms.keyterms; + target[keys[0]] = keyterms; console.log("analyzed!"); - if (internal) await vectorize(keyterms.keyterms_counted, dataDoc, mainDoc, data); + if (internal) { + await vectorize(keyterms_counted, dataDoc, mainDoc, data); + } else { + return values; + } }; // export async function countFrequencies() diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx index ff6e66492..b7b1d84d0 100644 --- a/src/client/views/Recommendations.tsx +++ b/src/client/views/Recommendations.tsx @@ -158,11 +158,15 @@ export class RecommendationsBox extends React.Component { // } // let style = { left: this.pageX, top: this.pageY }; //const transform = "translate(" + (NumCast(this.props.node.x) + 350) + "px, " + NumCast(this.props.node.y) + "px" + let title = StrCast((this.props.Document.sourceDoc as Doc).title); + if (title.length > 15) { + title = title.substring(0, 15) + "..."; + } return ( // { this._width = r.offset.width; this._height = r.offset.height; })}> // {({ measureRef }) => (
-

Recommendations

+

Recommendations for "{title}"

{DocListCast(this.props.Document.data).map(doc => { return (
diff --git a/src/client/views/collections/CollectionSchemaCells.tsx b/src/client/views/collections/CollectionSchemaCells.tsx index 9c26a08f0..bf8c4b6f7 100644 --- a/src/client/views/collections/CollectionSchemaCells.tsx +++ b/src/client/views/collections/CollectionSchemaCells.tsx @@ -27,6 +27,7 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import { KeyCodes } from "../../northstar/utils/KeyCodes"; import { undoBatch } from "../../util/UndoManager"; +import { List } from "lodash"; library.add(faExpand); @@ -86,10 +87,23 @@ export class CollectionSchemaCell extends React.Component { } @action - onPointerDown = (e: React.PointerEvent): void => { + onPointerDown = async (e: React.PointerEvent): Promise => { this.props.changeFocusedCellByIndex(this.props.row, this.props.col); this.props.setPreviewDoc(this.props.rowProps.original); + const data = await DocListCastAsync(this.props.Document.data); + if (data) { + let url: string; + if (url = StrCast(data[0].href)) { + try { + new URL(url); + const temp = window.open(url)!; + temp.blur(); + window.focus(); + } catch { } + } + } + // this._isEditing = true; // this.props.setIsEditing(true); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 2ae71f1da..a034bc1f4 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -45,6 +45,7 @@ import { RecommendationsBox } from '../Recommendations'; import { SearchUtil } from '../../util/SearchUtil'; import { ClientRecommender } from '../../ClientRecommender'; import { DocumentType } from '../../documents/DocumentTypes'; +import { SchemaHeaderField } from '../../../new_fields/SchemaHeaderField'; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? library.add(fa.faBrain); @@ -774,7 +775,11 @@ export class DocumentView extends DocComponent(Docu ClientRecommender.Instance.reset_docs(); const doc = Doc.GetDataDoc(this.props.Document); const extdoc = doc.data_ext as Doc; - return ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, false); + const values = await ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, false); + const headers = [new SchemaHeaderField("title"), new SchemaHeaderField("href")]; + const body = Docs.Create.FreeformDocument([], { title: values.title }); + body.href = values.url; + CollectionDockingView.Instance.AddRightSplit(Docs.Create.SchemaDocument(headers, [body], { title: `Showing External Recommendations for "${StrCast(doc.title)}"` }), undefined); } onPointerEnter = (e: React.PointerEvent): void => { Doc.BrushDoc(this.props.Document); }; -- cgit v1.2.3-70-g09d2 From e0b84cf329d7a9c9bda1aadbf25894c4bcc8f13a Mon Sep 17 00:00:00 2001 From: ab Date: Mon, 16 Sep 2019 17:15:37 -0400 Subject: demo touches --- src/client/ClientRecommender.tsx | 26 +++++++++++++++------- .../views/collections/CollectionSchemaCells.tsx | 19 +++++++--------- src/client/views/nodes/DocumentView.tsx | 13 ++++++++--- 3 files changed, 36 insertions(+), 22 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 103993a67..14af0a69b 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -212,7 +212,7 @@ export class ClientRecommender extends React.Component { arxivrequest = async (query: string) => { let xhttp = new XMLHttpRequest(); let serveraddress = "http://export.arxiv.org/api"; - let endpoint = serveraddress + "/query?search_query=all:" + query + "&start=0&max_results=1"; + let endpoint = serveraddress + "/query?search_query=all:" + query + "&start=0&max_results=5"; let promisified = (resolve: any, reject: any) => { xhttp.onreadystatechange = function () { if (this.readyState === 4) { @@ -221,22 +221,32 @@ export class ClientRecommender extends React.Component { console.log(xml); switch (this.status) { case 200: - let title: string = "Title"; - let url: string = "Url"; + let title_vals: string[] = []; + let url_vals: string[] = []; //console.log(result); if (xml) { let titles = xml.getElementsByTagName("title"); + let counter = 1; if (titles && titles.length > 1) { - title = titles[1].childNodes[0].nodeValue!; - console.log(title); + while (counter <= 5) { + const title = titles[counter].childNodes[0].nodeValue!; + console.log(title) + title_vals.push(title); + counter++; + } } let ids = xml.getElementsByTagName("id"); + counter = 1; if (ids && ids.length > 1) { - url = ids[1].childNodes[0].nodeValue!; - console.log(url); + while (counter <= 5) { + const url = ids[counter].childNodes[0].nodeValue!; + console.log(url); + url_vals.push(url); + counter++; + } } } - return resolve({ title, url }); + return resolve({ title_vals, url_vals }); case 400: default: return reject(result); diff --git a/src/client/views/collections/CollectionSchemaCells.tsx b/src/client/views/collections/CollectionSchemaCells.tsx index bf8c4b6f7..511fbc3a0 100644 --- a/src/client/views/collections/CollectionSchemaCells.tsx +++ b/src/client/views/collections/CollectionSchemaCells.tsx @@ -91,17 +91,14 @@ export class CollectionSchemaCell extends React.Component { this.props.changeFocusedCellByIndex(this.props.row, this.props.col); this.props.setPreviewDoc(this.props.rowProps.original); - const data = await DocListCastAsync(this.props.Document.data); - if (data) { - let url: string; - if (url = StrCast(data[0].href)) { - try { - new URL(url); - const temp = window.open(url)!; - temp.blur(); - window.focus(); - } catch { } - } + let url: string; + if (url = StrCast(this.props.rowProps.row.href)) { + try { + new URL(url); + const temp = window.open(url)!; + temp.blur(); + window.focus(); + } catch { } } // this._isEditing = true; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index a034bc1f4..a80eafde2 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -777,9 +777,16 @@ export class DocumentView extends DocComponent(Docu const extdoc = doc.data_ext as Doc; const values = await ClientRecommender.Instance.extractText(doc, extdoc ? extdoc : doc, false); const headers = [new SchemaHeaderField("title"), new SchemaHeaderField("href")]; - const body = Docs.Create.FreeformDocument([], { title: values.title }); - body.href = values.url; - CollectionDockingView.Instance.AddRightSplit(Docs.Create.SchemaDocument(headers, [body], { title: `Showing External Recommendations for "${StrCast(doc.title)}"` }), undefined); + let bodies: Doc[] = []; + const titles = values.title_vals; + const urls = values.url_vals; + for (let i = 0; i < 5; i++) { + const body = Docs.Create.FreeformDocument([], { title: titles[i] }); + body.href = urls[i]; + bodies.push(body); + } + + CollectionDockingView.Instance.AddRightSplit(Docs.Create.SchemaDocument(headers, bodies, { title: `Showing External Recommendations for "${StrCast(doc.title)}"` }), undefined); } onPointerEnter = (e: React.PointerEvent): void => { Doc.BrushDoc(this.props.Document); }; -- cgit v1.2.3-70-g09d2 From c6e2eee8e5b035ed4cab1e7bb1315a36d43255c5 Mon Sep 17 00:00:00 2001 From: ab Date: Tue, 8 Oct 2019 16:02:25 -0400 Subject: switching out --- src/client/views/MainView.tsx | 1 - src/client/views/Recommendations.tsx | 2 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 3 --- src/client/views/nodes/DocumentContentsView.tsx | 6 ------ src/client/views/nodes/DocumentView.tsx | 4 ++-- src/client/views/nodes/ImageBox.tsx | 1 - src/new_fields/RichTextField.ts | 17 ++++++++++++++++- 7 files changed, 19 insertions(+), 15 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 17de708a2..e6f500e75 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -41,7 +41,6 @@ import { FilterBox } from './search/FilterBox'; import { SchemaHeaderField, RandomPastel } from '../../new_fields/SchemaHeaderField'; //import { DocumentManager } from '../util/DocumentManager'; import { RecommendationsBox } from './Recommendations'; -import PresModeMenu from './presentationview/PresentationModeMenu'; import { PresBox } from './nodes/PresBox'; import { OverlayView } from './OverlayView'; diff --git a/src/client/views/Recommendations.tsx b/src/client/views/Recommendations.tsx index b7b1d84d0..c44dfc032 100644 --- a/src/client/views/Recommendations.tsx +++ b/src/client/views/Recommendations.tsx @@ -174,7 +174,7 @@ export class RecommendationsBox extends React.Component { {this.DocumentIcon(doc)} {NumCast(doc.score).toFixed(4)} -
DocumentManager.Instance.jumpToDocument(doc, true, undefined, undefined, undefined, this.props.Document.sourceDocContext as Doc)}> +
DocumentManager.Instance.jumpToDocument(doc, false)}>
DocUtils.MakeLink(this.props.Document.sourceDoc as Doc, doc, undefined, "User Selected Link", "Generated from Recommender", undefined)}> diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 3ee069e4c..d2b8afa02 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -12,7 +12,6 @@ import { BoolCast, Cast, DateCast, NumCast, StrCast } from "../../../../new_fiel import { CurrentUserUtils } from "../../../../server/authentication/models/current_user_utils"; import { aggregateBounds, emptyFunction, intersectRect, returnEmptyString, returnOne, Utils } from "../../../../Utils"; import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; -import { DocServer } from "../../../DocServer"; import { Docs } from "../../../documents/Documents"; import { DocumentType } from "../../../documents/DocumentTypes"; import { DocumentManager } from "../../../util/DocumentManager"; @@ -28,7 +27,6 @@ import { InkingCanvas } from "../../InkingCanvas"; import { CollectionFreeFormDocumentView, positionSchema } from "../../nodes/CollectionFreeFormDocumentView"; import { DocumentContentsView } from "../../nodes/DocumentContentsView"; import { documentSchema, DocumentViewProps } from "../../nodes/DocumentView"; -import { FormattedTextBox } from "../../nodes/FormattedTextBox"; import { pageSchema } from "../../nodes/ImageBox"; import PDFMenu from "../../pdf/PDFMenu"; import { CollectionSubView } from "../CollectionSubView"; @@ -41,7 +39,6 @@ import React = require("react"); import v5 = require("uuid/v5"); import { ClientRecommender } from "../../../ClientRecommender"; import { SearchUtil } from "../../../util/SearchUtil"; -import { SearchBox } from "../../SearchBox"; import { RouteStore } from "../../../../server/RouteStore"; import { string, number, elementType } from "prop-types"; import { DocServer } from "../../../DocServer"; diff --git a/src/client/views/nodes/DocumentContentsView.tsx b/src/client/views/nodes/DocumentContentsView.tsx index e5eb8dbce..919c13055 100644 --- a/src/client/views/nodes/DocumentContentsView.tsx +++ b/src/client/views/nodes/DocumentContentsView.tsx @@ -2,8 +2,6 @@ import { computed } from "mobx"; import { observer } from "mobx-react"; import { Doc } from "../../../new_fields/Doc"; import { ScriptField } from "../../../new_fields/ScriptField"; -import { Cast } from "../../../new_fields/Types"; -import { OmitKeys, Without } from "../../../Utils"; import { HistogramBox } from "../../northstar/dash-nodes/HistogramBox"; import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox"; import { CollectionDockingView } from "../collections/CollectionDockingView"; @@ -30,14 +28,10 @@ import { PresElementBox } from "../presentationview/PresElementBox"; import { VideoBox } from "./VideoBox"; import { WebBox } from "./WebBox"; import React = require("react"); -import { FieldViewProps } from "./FieldView"; import { Without, OmitKeys } from "../../../Utils"; import { Cast, StrCast, NumCast } from "../../../new_fields/Types"; import { List } from "../../../new_fields/List"; -import { Doc } from "../../../new_fields/Doc"; -import DirectoryImportBox from "../../util/Import & Export/DirectoryImportBox"; import { RecommendationsBox } from "../../views/Recommendations"; -import { ScriptField } from "../../../new_fields/ScriptField"; const JsxParser = require('react-jsx-parser').default; //TODO Why does this need to be imported like this? type BindingProps = Without; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 0064b98c3..070b1f426 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -621,7 +621,7 @@ export class DocumentView extends DocComponent(Docu recommendations.documentIconHeight = 150; recommendations.sourceDoc = this.props.Document; recommendations.sourceDocContext = this.props.ContainingCollectionView!.props.Document; - CollectionDockingView.Instance.AddRightSplit(recommendations, undefined); + CollectionDockingView.AddRightSplit(recommendations, undefined); // RecommendationsBox.Instance.displayRecommendations(e.pageX + 100, e.pageY); } @@ -641,7 +641,7 @@ export class DocumentView extends DocComponent(Docu body.href = urls[i]; bodies.push(body); } - CollectionDockingView.Instance.AddRightSplit(Docs.Create.SchemaDocument(headers, bodies, { title: `Showing External Recommendations for "${StrCast(doc.title)}"` }), undefined); + CollectionDockingView.AddRightSplit(Docs.Create.SchemaDocument(headers, bodies, { title: `Showing External Recommendations for "${StrCast(doc.title)}"` }), undefined); } onPointerEnter = (e: React.PointerEvent): void => { Doc.BrushDoc(this.props.Document); }; diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index f36b9895f..7ffe64b9b 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -29,7 +29,6 @@ import "./ImageBox.scss"; import React = require("react"); import { SearchUtil } from '../../util/SearchUtil'; import { ClientRecommender } from '../../ClientRecommender'; -import { DocumentType } from '../../documents/Documents'; var requestImageSize = require('../../util/request-image-size'); var path = require('path'); const { Howl } = require('howler'); diff --git a/src/new_fields/RichTextField.ts b/src/new_fields/RichTextField.ts index 390045ee1..f41ea0350 100644 --- a/src/new_fields/RichTextField.ts +++ b/src/new_fields/RichTextField.ts @@ -4,6 +4,9 @@ import { Deserializable } from "../client/util/SerializationHelper"; import { Copy, ToScriptString, ToPlainText } from "./FieldSymbols"; import { scriptingGlobal } from "../client/util/Scripting"; +const delimiter = "\n"; +const joiner = ""; + @scriptingGlobal @Deserializable("RichTextField") export class RichTextField extends ObjectField { @@ -24,7 +27,19 @@ export class RichTextField extends ObjectField { } [ToPlainText]() { - return this.Data; + // Because we're working with plain text, just concatenate all paragraphs + let content = JSON.parse(this.Data).doc.content; + let paragraphs = content.filter((item: any) => item.type === "paragraph"); + + // Functions to flatten ProseMirror paragraph objects (and their components) to plain text + // While this function already exists in state.doc.textBeteen(), it doesn't account for newlines + let blockText = (block: any) => block.text; + let concatenateParagraph = (p: any) => (p.content ? p.content.map(blockText).join(joiner) : "") + delimiter; + + // Concatentate paragraphs and string the result together + let textParagraphs: string[] = paragraphs.map(concatenateParagraph); + let plainText = textParagraphs.join(joiner); + return plainText.substring(0, plainText.length - 1); } } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From e103be2d58da2a6809dd4ad5f0b5f445d8d6c96b Mon Sep 17 00:00:00 2001 From: ab Date: Sat, 16 Nov 2019 16:45:41 -0500 Subject: ibm integrated, bing search -> server next time --- package.json | 7 ++- src/client/ClientRecommender.tsx | 66 ++++++++++++++++++++-- src/client/apis/IBM_Recommender.ts | 6 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 5 -- src/client/views/nodes/DocumentView.tsx | 2 +- src/server/index.ts | 2 +- 6 files changed, 70 insertions(+), 18 deletions(-) (limited to 'src/client/views/collections') diff --git a/package.json b/package.json index 80b65cc68..d41119a47 100644 --- a/package.json +++ b/package.json @@ -57,9 +57,9 @@ "@hig/theme-context": "^2.1.3", "@hig/theme-data": "^2.3.3", "@tensorflow-models/universal-sentence-encoder": "^1.2.0", - "@tensorflow/tfjs": "^1.3.1", + "@tensorflow/tfjs-converter": "^1.3.2", "@tensorflow/tfjs-core": "^1.3.1", - "@tensorflow/tfjs-node": "^1.3.1", + "@tensorflow/tfjs-node": "1.3.1", "@trendmicro/react-dropdown": "^1.3.0", "@types/adm-zip": "^0.4.32", "@types/animejs": "^2.0.2", @@ -238,8 +238,9 @@ "url-loader": "^1.1.2", "uuid": "^3.3.2", "wikijs": "^6.0.1", + "word2vec": "^1.1.4", "words-to-numbers": "^1.5.1", "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index 364ec0fe0..a37434c0d 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -8,6 +8,7 @@ import { observable, action, computed, reaction } from "mobx"; var assert = require('assert'); var sw = require('stopword'); var FeedParser = require('feedparser'); +var https = require('https'); import "./ClientRecommender.scss"; import { JSXElement } from "babel-types"; import { RichTextField } from "../new_fields/RichTextField"; @@ -111,7 +112,7 @@ export class ClientRecommender extends React.Component { } ); let doclist = Array.from(ClientRecommender.Instance.docVectors); - if (distance_metric == "euclidian") { + if (distance_metric === "euclidian") { doclist.sort((a: RecommenderDocument, b: RecommenderDocument) => a.score - b.score); } else { @@ -245,7 +246,27 @@ export class ClientRecommender extends React.Component { if (kp_string.length > 2) kp_string = kp_string.substring(0, kp_string.length - 2); console.log("kp string: ", kp_string); let values = ""; - if (!internal) values = await this.sendRequest(highKP); + if (!internal) { + const parameters: any = { + 'language': 'en', + 'text': data, + 'features': { + 'keywords': { + 'sentiment': true, + 'emotion': true, + 'limit': 3 + } + } + }; + await Identified.PostToServer("/IBMAnalysis", parameters).then(response => { + const sorted_keywords = response.result.keywords; + if (sorted_keywords.length > 0) { + console.log("IBM keyphrase", sorted_keywords[0]); + highKP = [sorted_keywords[0].text]; + } + }); + values = await this.sendRequest(highKP, "bing"); + } return { keyterms: keyterms, external_recommendations: values, kp_string: [kp_string] }; }; if (data !== "") { @@ -290,11 +311,46 @@ export class ClientRecommender extends React.Component { * API for sending arXiv request. */ - private async sendRequest(keywords: string[]) { + private async sendRequest(keywords: string[], api: string) { let query = ""; keywords.forEach((kp: string) => query += " " + kp); - return new Promise(resolve => { - this.arxivrequest(query).then(resolve); + if (api === "arxiv") { + return new Promise(resolve => { + this.arxivrequest(query).then(resolve); + }); + } + else if (api === "bing") { + await this.bingWebSearch(query); + } + else { + return new Promise(resolve => { + this.arxivrequest(query).then(resolve); + }); + } + + } + + bingWebSearch = async (query: string) => { + https.get({ + hostname: 'api.cognitive.microsoft.com', + path: '/bing/v5.0/search?q=' + encodeURIComponent(query), + headers: { 'Ocp-Apim-Subscription-Key': process.env.BING }, + }, (res: any) => { + let body = ''; + res.on('data', (part: any) => body += part); + res.on('end', () => { + for (var header in res.headers) { + if (header.startsWith("bingapis-") || header.startsWith("x-msedge-")) { + console.log(header + ": " + res.headers[header]) + } + } + console.log('\nJSON Response:\n'); + console.dir(JSON.parse(body), { colors: false, depth: null }); + }) + res.on('error', (e: any) => { + console.log('Error: ' + e.message); + throw e; + }); }); } diff --git a/src/client/apis/IBM_Recommender.ts b/src/client/apis/IBM_Recommender.ts index 5f80f5126..da6257f28 100644 --- a/src/client/apis/IBM_Recommender.ts +++ b/src/client/apis/IBM_Recommender.ts @@ -16,19 +16,19 @@ export namespace IBM_Recommender { }); const analyzeParams = { - 'url': 'www.ibm.com', + 'text': 'this is a test of the keyword extraction feature I am integrating into the program', 'features': { 'keywords': { 'sentiment': true, 'emotion': true, 'limit': 3 - } + }, } }; export const analyze = async (_parameters: any): Promise> => { try { - const response = await naturalLanguageUnderstanding.analyze(analyzeParams); + const response = await naturalLanguageUnderstanding.analyze(_parameters); console.log(response); return (JSON.stringify(response, null, 2)); } catch (err) { diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index bb5d99a28..bfec545c6 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -34,11 +34,6 @@ import { CollectionFreeFormRemoteCursors } from "./CollectionFreeFormRemoteCurso import "./CollectionFreeFormView.scss"; import { MarqueeView } from "./MarqueeView"; import React = require("react"); -import v5 = require("uuid/v5"); -import { ClientRecommender } from "../../../ClientRecommender"; -import { SearchUtil } from "../../../util/SearchUtil"; -import { RouteStore } from "../../../../server/RouteStore"; -import { string, number, elementType } from "prop-types"; import { DocServer } from "../../../DocServer"; import { documentSchema, positionSchema } from "../../../../new_fields/documentSchemas"; import { DocumentViewProps } from "../../nodes/DocumentView"; diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 55063a52c..c6ad2f9d7 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -772,7 +772,7 @@ export class DocumentView extends DocComponent(Docu } render() { if (!this.props.Document) return (null); - trace(); + //trace(); const animDims = this.Document.animateToDimensions ? Array.from(this.Document.animateToDimensions) : undefined; const ruleColor = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleColor_" + this.Document.heading]) : undefined; const ruleRounding = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleRounding_" + this.Document.heading]) : undefined; diff --git a/src/server/index.ts b/src/server/index.ts index 2a669f147..45fc7fc07 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1086,7 +1086,7 @@ addSecureRoute({ const batched = BatchedArray.from(media, { batchSize: 25 }); const newMediaItems = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch, collector) => { + async (batch: any, collector: any): Promise => { for (let index = 0; index < batch.length; index++) { const { url, description } = batch[index]; const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(url); -- cgit v1.2.3-70-g09d2 From a231b597d70f8ee9e435a86b00020c93691dc97d Mon Sep 17 00:00:00 2001 From: ab Date: Sat, 23 Nov 2019 16:06:30 -0500 Subject: keyphrase query ui --- src/client/ClientRecommender.tsx | 2 + src/client/cognitive_services/CognitiveServices.ts | 49 ++++++++++++++++++++++ src/client/views/KeyphraseQueryView.scss | 8 ++++ src/client/views/KeyphraseQueryView.tsx | 30 +++++++++++++ .../views/collections/CollectionViewChromes.scss | 2 +- src/client/views/nodes/DocumentView.tsx | 38 ++++++++++------- 6 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 src/client/views/KeyphraseQueryView.scss create mode 100644 src/client/views/KeyphraseQueryView.tsx (limited to 'src/client/views/collections') diff --git a/src/client/ClientRecommender.tsx b/src/client/ClientRecommender.tsx index d2e2330b5..b384a8ebe 100644 --- a/src/client/ClientRecommender.tsx +++ b/src/client/ClientRecommender.tsx @@ -17,6 +17,7 @@ import { listSpec } from "../new_fields/Schema"; import { Identified } from "./Network"; import { ComputedField } from "../new_fields/ScriptField"; import { ImageField } from "../new_fields/URLField"; +import { KeyphraseQueryView } from "./views/KeyphraseQueryView"; export interface RecommenderProps { title: string; @@ -279,6 +280,7 @@ export class ClientRecommender extends React.Component { highKP = [sorted_keywords[0].text]; } }); + let kpqv = new KeyphraseQueryView({ keyphrases: ["hello"] }); ext_recs = await this.sendRequest(highKP, api); } diff --git a/src/client/cognitive_services/CognitiveServices.ts b/src/client/cognitive_services/CognitiveServices.ts index becd38a99..356cf52ca 100644 --- a/src/client/cognitive_services/CognitiveServices.ts +++ b/src/client/cognitive_services/CognitiveServices.ts @@ -276,6 +276,55 @@ export namespace CognitiveServices { } + export namespace HathiTrust { + export const Manager: APIManager = { + converter: (data: string) => { + return data; + }, + requester: async (apiKey: string, query: string) => { + let xhttp = new XMLHttpRequest(); + let serverAddress = "https://babel.hathitrust.org/cgi/htd/​"; + let endpoint = serverAddress + '/bing/v5.0/search?q=' + encodeURIComponent(query); + let promisified = (resolve: any, reject: any) => { + xhttp.onreadystatechange = function () { + if (this.readyState === 4) { + let result = xhttp.responseText; + switch (this.status) { + case 200: + return resolve(result); + case 400: + default: + return reject(result); + } + } + }; + + if (apiKey) { + xhttp.open("GET", endpoint, true); + xhttp.setRequestHeader('Ocp-Apim-Subscription-Key', apiKey); + xhttp.setRequestHeader('Content-Type', 'application/json'); + xhttp.send(); + } + else { + console.log("API key for BING unavailable"); + } + }; + return new Promise(promisified); + } + + }; + + export namespace Appliers { + export const analyzer = async (query: string, converter: BingConverter) => { + let results = await ExecuteQuery(Service.Bing, Manager, query); + console.log("Bing results: ", results); + const { title_vals, url_vals } = await converter(results); + return { title_vals, url_vals }; + }; + } + + } + export namespace Text { export const Manager: APIManager = { diff --git a/src/client/views/KeyphraseQueryView.scss b/src/client/views/KeyphraseQueryView.scss new file mode 100644 index 000000000..ac715e5e7 --- /dev/null +++ b/src/client/views/KeyphraseQueryView.scss @@ -0,0 +1,8 @@ +.fading { + animation: fanOut 1s +} + +@keyframes fanOut { + from {opacity: 0;} + to {opacity: 1;} +} \ No newline at end of file diff --git a/src/client/views/KeyphraseQueryView.tsx b/src/client/views/KeyphraseQueryView.tsx new file mode 100644 index 000000000..1955399f9 --- /dev/null +++ b/src/client/views/KeyphraseQueryView.tsx @@ -0,0 +1,30 @@ +import { observer } from "mobx-react"; +import React = require("react"); +import "./KeyphraseQueryView.scss"; + +// tslint:disable-next-line: class-name +export interface KP_Props { + keyphrases: string[]; +} + +@observer +export class KeyphraseQueryView extends React.Component{ + constructor(props: KP_Props) { + super(props); + console.log("FIRST KEY PHRASE: ", props.keyphrases[0]); + } + + render() { + return ( +
+

Select queries to send:

+ {this.props.keyphrases.map((kp: string) => { + setTimeout(() => { + return (

{kp}

); + }, 1000); + + })} +
+ ); + } +} \ No newline at end of file diff --git a/src/client/views/collections/CollectionViewChromes.scss b/src/client/views/collections/CollectionViewChromes.scss index 64411b5fe..1889a192c 100644 --- a/src/client/views/collections/CollectionViewChromes.scss +++ b/src/client/views/collections/CollectionViewChromes.scss @@ -247,7 +247,7 @@ display:flex; flex-direction: row; width: 150px; - margin: auto 0 auto auto; + margin: auto auto auto auto; } .react-autosuggest__container { diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 2455c320d..b82486109 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -46,6 +46,7 @@ import { RecommendationsBox } from '../RecommendationsBox'; import { SearchUtil } from '../../util/SearchUtil'; import { ClientRecommender } from '../../ClientRecommender'; import { SchemaHeaderField } from '../../../new_fields/SchemaHeaderField'; +import { KeyphraseQueryView } from '../KeyphraseQueryView'; library.add(fa.faBrain); library.add(fa.faEdit); @@ -117,6 +118,7 @@ export class DocumentView extends DocComponent(Docu private _hitTemplateDrag = false; private _mainCont = React.createRef(); private _dropDisposer?: DragManager.DragDropDisposer; + private _showKPQuery: boolean = false; public get ContentDiv() { return this._mainCont.current; } @computed get active() { return SelectionManager.IsSelected(this) || this.props.parentActive(); } @@ -657,6 +659,7 @@ export class DocumentView extends DocComponent(Docu bodies.push(body); } CollectionDockingView.AddRightSplit(Docs.Create.SchemaDocument(headers, bodies, { title: `Showing External Recommendations for "${StrCast(doc.title)}"` }), undefined); + this._showKPQuery = true; } onPointerEnter = (e: React.PointerEvent): void => { Doc.BrushDoc(this.props.Document); }; @@ -807,22 +810,25 @@ export class DocumentView extends DocComponent(Docu const highlightColors = ["transparent", "maroon", "maroon", "yellow", "magenta", "cyan", "orange"]; const highlightStyles = ["solid", "dashed", "solid", "solid", "solid", "solid", "solid", "solid"]; let highlighting = fullDegree && this.layoutDoc.type !== DocumentType.FONTICON && this.layoutDoc.viewType !== CollectionViewType.Linear; - return
Doc.BrushDoc(this.props.Document)} onPointerLeave={e => Doc.UnBrushDoc(this.props.Document)} - style={{ - transition: this.Document.isAnimating !== undefined ? ".5s linear" : StrCast(this.Document.transition), - pointerEvents: this.Document.isBackground && !this.isSelected() ? "none" : "all", - color: StrCast(this.Document.color), - outline: highlighting && !borderRounding ? `${highlightColors[fullDegree]} ${highlightStyles[fullDegree]} ${localScale}px` : "solid 0px", - border: highlighting && borderRounding ? `${highlightStyles[fullDegree]} ${highlightColors[fullDegree]} ${localScale}px` : undefined, - background: this.layoutDoc.type === DocumentType.FONTICON || this.layoutDoc.viewType === CollectionViewType.Linear ? undefined : backgroundColor, - width: animwidth, - height: animheight, - transform: `scale(${this.layoutDoc.fitWidth ? 1 : this.props.ContentScaling()})`, - opacity: this.Document.opacity - }} > - {this.innards} + return
+
Doc.BrushDoc(this.props.Document)} onPointerLeave={e => Doc.UnBrushDoc(this.props.Document)} + style={{ + transition: this.Document.isAnimating !== undefined ? ".5s linear" : StrCast(this.Document.transition), + pointerEvents: this.Document.isBackground && !this.isSelected() ? "none" : "all", + color: StrCast(this.Document.color), + outline: highlighting && !borderRounding ? `${highlightColors[fullDegree]} ${highlightStyles[fullDegree]} ${localScale}px` : "solid 0px", + border: highlighting && borderRounding ? `${highlightStyles[fullDegree]} ${highlightColors[fullDegree]} ${localScale}px` : undefined, + background: this.layoutDoc.type === DocumentType.FONTICON || this.layoutDoc.viewType === CollectionViewType.Linear ? undefined : backgroundColor, + width: animwidth, + height: animheight, + transform: `scale(${this.layoutDoc.fitWidth ? 1 : this.props.ContentScaling()})`, + opacity: this.Document.opacity + }} > + {this.innards} +
+ {this._showKPQuery ? : undefined}
; } } -- cgit v1.2.3-70-g09d2 From b4c2d2add2b863060ce559e49b3953f24050f162 Mon Sep 17 00:00:00 2001 From: Fawn Date: Tue, 14 Jan 2020 14:47:44 -0500 Subject: can add strokes to mobile ink --- src/client/views/collections/CollectionSubView.tsx | 3 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 1 + src/mobile/MobileInterface.tsx | 108 +++++++++++++++------ .../authentication/models/current_user_utils.ts | 4 + 4 files changed, 87 insertions(+), 29 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index 73dc7edc6..d8b575092 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -53,7 +53,9 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { protected createDropAndGestureTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view this.dropDisposer && this.dropDisposer(); this.gestureDisposer && this.gestureDisposer(); + console.log("create drop", ele); if (ele) { + console.log("create drop 2", ele); this.dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this)); this.gestureDisposer = GestureUtils.MakeGestureTarget(ele, this.onGesture.bind(this)); } @@ -135,7 +137,6 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { @undoBatch protected onGesture(e: Event, ge: GestureUtils.GestureEvent) { - } @undoBatch diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 84945c6e6..c3e131184 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -356,6 +356,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { @undoBatch onGesture = (e: Event, ge: GestureUtils.GestureEvent) => { + console.log("on gesture"); switch (ge.gesture) { case GestureUtils.Gestures.Stroke: const points = ge.points; diff --git a/src/mobile/MobileInterface.tsx b/src/mobile/MobileInterface.tsx index 1e0920686..8d342d749 100644 --- a/src/mobile/MobileInterface.tsx +++ b/src/mobile/MobileInterface.tsx @@ -10,8 +10,10 @@ import { DocumentView } from '../client/views/nodes/DocumentView'; import { emptyPath, emptyFunction, returnFalse, returnOne, returnEmptyString, returnTrue } from '../Utils'; import { Transform } from '../client/util/Transform'; import { library } from '@fortawesome/fontawesome-svg-core'; -import { faPenNib, faHighlighter, faEraser, faMousePointer } from '@fortawesome/free-solid-svg-icons'; +import { faPenNib, faHighlighter, faEraser, faMousePointer, faBreadSlice } from '@fortawesome/free-solid-svg-icons'; import { Scripting } from '../client/util/Scripting'; +import { CollectionFreeFormView } from '../client/views/collections/collectionFreeForm/CollectionFreeFormView'; +import GestureOverlay from '../client/views/GestureOverlay'; @observer export default class MobileInterface extends React.Component { @@ -35,52 +37,102 @@ export default class MobileInterface extends React.Component { } } - @action switchCurrentView = (view: "main" | "ink" | "library") => { this.currentView = view; } + @action + switchCurrentView = (view: "main" | "ink" | "library") => { + this.currentView = view; + + if (this.userDoc) { + switch (view) { + case "main": { + const doc = CurrentUserUtils.setupMobileDoc(this.userDoc); + this.userDoc.activeMobile = doc; + break; + } + case "ink": { + const doc = CurrentUserUtils.setupMobileInkingDoc(this.userDoc); + this.userDoc.activeMobile = doc; + break; + } + } + } + } @computed get mainContent() { if (this.mainContainer) { - switch (this.currentView) { - case "main": - return window.screen.width} + PanelHeight={() => window.screen.height} + renderDepth={0} + focus={emptyFunction} + backgroundColor={returnEmptyString} + parentActive={returnTrue} + whenActiveChanged={emptyFunction} + bringToFront={emptyFunction} + ContainingCollectionView={undefined} + ContainingCollectionDoc={undefined} + zoomToScale={emptyFunction} + getScale={returnOne}> + ; + } + return "hello"; + } + + @computed + get inkContent() { + // return
INK
; + if (this.mainContainer) { + return ( + + window.screen.width} PanelHeight={() => window.screen.height} - renderDepth={0} + PanelWidth={() => window.screen.width} + annotationsKey={""} + isAnnotationOverlay={false} focus={emptyFunction} - backgroundColor={returnEmptyString} - parentActive={returnTrue} - whenActiveChanged={emptyFunction} - bringToFront={emptyFunction} + isSelected={returnTrue} // + select={emptyFunction} + active={returnTrue} // + ContentScaling={returnOne} + whenActiveChanged={returnFalse} + CollectionView={undefined} + ScreenToLocalTransform={Transform.Identity} + ruleProvider={undefined} + renderDepth={0} ContainingCollectionView={undefined} ContainingCollectionDoc={undefined} - zoomToScale={emptyFunction} - getScale={returnOne}> - ; - case "ink": - return
INK
; - case "library": - return
LIBRARY
; - } + chromeCollapsed={true}> + + + ); } - return "hello"; } render() { - console.log("rendering mobile"); + const content = this.currentView === "main" ? this.mainContent : this.currentView === "ink" ? this.inkContent : <>; return (
- {this.mainContent} + {content}
); } diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 71369b625..b40179fbc 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -140,6 +140,10 @@ export class CurrentUserUtils { }); } + static setupMobileInkingDoc(userDoc: Doc) { + return Docs.Create.FreeformDocument([], { x: 0, y: 0, width: 10, height: 20, title: "Mobile Inking", backgroundColor: "red" }); + } + // setup the Creator button which will display the creator panel. This panel will include the drag creators and the color picker. when clicked, this panel will be displayed in the target container (ie, sidebarContainer) static setupToolsPanel(sidebarContainer: Doc, doc: Doc) { // setup a masonry view of all he creators -- cgit v1.2.3-70-g09d2 From 3cca58612cde96a3082ca8e190fe2166d531d556 Mon Sep 17 00:00:00 2001 From: Fawn Date: Tue, 14 Jan 2020 17:40:50 -0500 Subject: drawn strokes get added to mobile ink collection --- src/client/views/GestureOverlay.tsx | 2 +- src/client/views/collections/CollectionSubView.tsx | 2 - .../collectionFreeForm/CollectionFreeFormView.tsx | 8 +-- src/mobile/MobileInterface.scss | 7 +++ src/mobile/MobileInterface.tsx | 62 ++++++++++++++-------- .../authentication/models/current_user_utils.ts | 2 +- 6 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 src/mobile/MobileInterface.scss (limited to 'src/client/views/collections') diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 632f2f0d6..4d487c032 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -231,9 +231,9 @@ export default class GestureOverlay extends Touchable { render() { return (
+ {this.currentStroke} {this.props.children} {this._palette} - {this.currentStroke}
); } } diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index d8b575092..e94f24f2c 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -53,9 +53,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { protected createDropAndGestureTarget = (ele: HTMLDivElement) => { //used for stacking and masonry view this.dropDisposer && this.dropDisposer(); this.gestureDisposer && this.gestureDisposer(); - console.log("create drop", ele); if (ele) { - console.log("create drop 2", ele); this.dropDisposer = DragManager.MakeDropTarget(ele, this.drop.bind(this)); this.gestureDisposer = GestureUtils.MakeGestureTarget(ele, this.onGesture.bind(this)); } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index c3e131184..8d376fb57 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -114,6 +114,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { } private addDocument = (newBox: Doc) => { const added = this.props.addDocument(newBox); + console.log("adding doc from freeform", added); added && this.bringToFront(newBox); added && this.updateCluster(newBox); return added; @@ -356,12 +357,12 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { @undoBatch onGesture = (e: Event, ge: GestureUtils.GestureEvent) => { - console.log("on gesture"); switch (ge.gesture) { case GestureUtils.Gestures.Stroke: const points = ge.points; const B = this.getTransform().transformBounds(ge.bounds.left, ge.bounds.top, ge.bounds.width, ge.bounds.height); const inkDoc = Docs.Create.InkDocument(InkingControl.Instance.selectedColor, InkingControl.Instance.selectedTool, parseInt(InkingControl.Instance.selectedWidth), points, { x: B.x, y: B.y, width: B.width, height: B.height }); + console.log("make stroke", inkDoc); this.addDocument(inkDoc); e.stopPropagation(); break; @@ -403,14 +404,14 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { @action pan = (e: PointerEvent | React.Touch | { clientX: number, clientY: number }): void => { // I think it makes sense for the marquee menu to go away when panned. -syip2 - MarqueeOptionsMenu.Instance.fadeOut(true); + MarqueeOptionsMenu.Instance && MarqueeOptionsMenu.Instance.fadeOut(true); let x = this.Document.panX || 0; let y = this.Document.panY || 0; const docs = this.childLayoutPairs.map(pair => pair.layout); const [dx, dy] = this.getTransform().transformDirection(e.clientX - this._lastX, e.clientY - this._lastY); if (!this.isAnnotationOverlay) { - PDFMenu.Instance.fadeOut(true); + PDFMenu.Instance && PDFMenu.Instance.fadeOut(true); const minx = docs.length ? NumCast(docs[0].x) : 0; const maxx = docs.length ? NumCast(docs[0].width) + minx : minx; const miny = docs.length ? NumCast(docs[0].y) : 0; @@ -990,6 +991,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { // otherwise, they are stored in fieldKey. All annotations to this document are stored in the extension document if (!this.extensionDoc) return (null); // let lodarea = this.Document[WidthSym]() * this.Document[HeightSym]() / this.props.ScreenToLocalTransform().Scale / this.props.ScreenToLocalTransform().Scale; + console.log("height freeform", this.isAnnotationOverlay, this.Document.scrollHeight, this.props.PanelHeight()); return
diff --git a/src/mobile/MobileInterface.scss b/src/mobile/MobileInterface.scss new file mode 100644 index 000000000..e4cc919a5 --- /dev/null +++ b/src/mobile/MobileInterface.scss @@ -0,0 +1,7 @@ +.mobileInterface-topButtons { + position: absolute; + display: flex; + justify-content: space-between; + width: 100%; + z-index: 9999; +} \ No newline at end of file diff --git a/src/mobile/MobileInterface.tsx b/src/mobile/MobileInterface.tsx index 8d342d749..3cb08afa7 100644 --- a/src/mobile/MobileInterface.tsx +++ b/src/mobile/MobileInterface.tsx @@ -10,10 +10,16 @@ import { DocumentView } from '../client/views/nodes/DocumentView'; import { emptyPath, emptyFunction, returnFalse, returnOne, returnEmptyString, returnTrue } from '../Utils'; import { Transform } from '../client/util/Transform'; import { library } from '@fortawesome/fontawesome-svg-core'; -import { faPenNib, faHighlighter, faEraser, faMousePointer, faBreadSlice } from '@fortawesome/free-solid-svg-icons'; +import { faPenNib, faHighlighter, faEraser, faMousePointer, faBreadSlice, faTrash, faCheck } from '@fortawesome/free-solid-svg-icons'; import { Scripting } from '../client/util/Scripting'; import { CollectionFreeFormView } from '../client/views/collections/collectionFreeForm/CollectionFreeFormView'; import GestureOverlay from '../client/views/GestureOverlay'; +import { InkingControl } from '../client/views/InkingControl'; +import { InkTool } from '../new_fields/InkField'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import "./MobileInterface.scss"; + +library.add(faTrash, faCheck); @observer export default class MobileInterface extends React.Component { @@ -22,6 +28,9 @@ export default class MobileInterface extends React.Component { @computed private get mainContainer() { return this.userDoc ? FieldValue(Cast(this.userDoc.activeMobile, Doc)) : CurrentUserUtils.GuestMobile; } @observable private currentView: "main" | "ink" | "library" = "main"; + private mainDoc = CurrentUserUtils.setupMobileDoc(this.userDoc); + private inkDoc?: Doc; + constructor(props: Readonly<{}>) { super(props); MobileInterface.Instance = this; @@ -32,8 +41,8 @@ export default class MobileInterface extends React.Component { library.add(...[faPenNib, faHighlighter, faEraser, faMousePointer]); if (this.userDoc && !this.mainContainer) { - const doc = CurrentUserUtils.setupMobileDoc(this.userDoc); - this.userDoc.activeMobile = doc; + // const doc = CurrentUserUtils.setupMobileDoc(this.userDoc); + this.userDoc.activeMobile = this.mainDoc; } } @@ -44,13 +53,14 @@ export default class MobileInterface extends React.Component { if (this.userDoc) { switch (view) { case "main": { - const doc = CurrentUserUtils.setupMobileDoc(this.userDoc); - this.userDoc.activeMobile = doc; + // const doc = CurrentUserUtils.setupMobileDoc(this.userDoc); + this.userDoc.activeMobile = this.mainDoc; break; } case "ink": { - const doc = CurrentUserUtils.setupMobileInkingDoc(this.userDoc); - this.userDoc.activeMobile = doc; + this.inkDoc = CurrentUserUtils.setupMobileInkingDoc(this.userDoc); + this.userDoc.activeMobile = this.inkDoc; + InkingControl.Instance.switchTool(InkTool.Pen); break; } } @@ -64,7 +74,7 @@ export default class MobileInterface extends React.Component { Document={this.mainContainer} DataDoc={undefined} LibraryPath={emptyPath} - addDocument={undefined} + addDocument={returnFalse} addDocTab={returnFalse} pinToPres={emptyFunction} removeDocument={undefined} @@ -89,40 +99,46 @@ export default class MobileInterface extends React.Component { return "hello"; } + onClick = (e: React.MouseEvent) => { + // insert ink as collection into active displayed doc view + // reset ink collection + this.inkDoc = undefined; + console.log("clicked"); + this.switchCurrentView("main"); + InkingControl.Instance.switchTool(InkTool.None); // TODO: switch to previous tool + } + @computed get inkContent() { - // return
INK
; + // TODO: get props from active collection and pass to the new freeform if (this.mainContainer) { return ( - + + +
+ window.screen.height} - PanelWidth={() => window.screen.width} - annotationsKey={""} - isAnnotationOverlay={false} + PanelHeight={() => window.innerHeight} + PanelWidth={() => window.innerWidth} focus={emptyFunction} - isSelected={returnTrue} // + isSelected={returnTrue} select={emptyFunction} - active={returnTrue} // + active={returnTrue} ContentScaling={returnOne} whenActiveChanged={returnFalse} - CollectionView={undefined} ScreenToLocalTransform={Transform.Identity} ruleProvider={undefined} renderDepth={0} ContainingCollectionView={undefined} - ContainingCollectionDoc={undefined} - chromeCollapsed={true}> - + ContainingCollectionDoc={undefined}> + ); } diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index b40179fbc..98092ddfe 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -141,7 +141,7 @@ export class CurrentUserUtils { } static setupMobileInkingDoc(userDoc: Doc) { - return Docs.Create.FreeformDocument([], { x: 0, y: 0, width: 10, height: 20, title: "Mobile Inking", backgroundColor: "red" }); + return Docs.Create.FreeformDocument([], { title: "Mobile Inking", backgroundColor: "white" }); } // setup the Creator button which will display the creator panel. This panel will include the drag creators and the color picker. when clicked, this panel will be displayed in the target container (ie, sidebarContainer) -- cgit v1.2.3-70-g09d2 From 65e164eaec42d1850de7f5e1eba1d4302c3e8230 Mon Sep 17 00:00:00 2001 From: Fawn Date: Wed, 15 Jan 2020 16:08:48 -0500 Subject: mobile interface emits events when switched to inking view and when strokes are drawn, currently with dummy callbacks --- src/client/DocServer.ts | 22 ++++++++++++++++++- src/client/views/GestureOverlay.tsx | 21 ++++++++++++++++++ src/client/views/MainView.tsx | 13 +++++++++-- .../collectionFreeForm/CollectionFreeFormView.tsx | 3 --- src/mobile/MobileInterface.tsx | 25 +++++++++++++++------- src/server/Message.ts | 13 +++++++++++ src/server/Websocket/Websocket.ts | 12 ++++++++++- 7 files changed, 94 insertions(+), 15 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index ed7fbd7ba..cfd9612ed 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -1,10 +1,11 @@ import * as OpenSocket from 'socket.io-client'; -import { MessageStore, YoutubeQueryTypes } from "./../server/Message"; +import { MessageStore, YoutubeQueryTypes, GestureContent } from "./../server/Message"; import { Opt, Doc } from '../new_fields/Doc'; import { Utils, emptyFunction } from '../Utils'; import { SerializationHelper } from './util/SerializationHelper'; import { RefField } from '../new_fields/RefField'; import { Id, HandleUpdate } from '../new_fields/FieldSymbols'; +import GestureOverlay from './views/GestureOverlay'; /** * This class encapsulates the transfer and cross-client synchronization of @@ -64,6 +65,19 @@ export namespace DocServer { } } + export namespace Mobile { + + export function dispatchGesturePoints(content: GestureContent) { + Utils.Emit(_socket, MessageStore.GesturePoints, content); + } + + export function dispatchBoxTrigger(enableBox: boolean) { + // _socket.emit("dispatchBoxTrigger"); + Utils.Emit(_socket, MessageStore.MobileInkBoxTrigger, enableBox); + } + + } + export function init(protocol: string, hostname: string, port: number, identifier: string) { _cache = {}; GUID = identifier; @@ -85,6 +99,12 @@ export namespace DocServer { Utils.AddServerHandler(_socket, MessageStore.ConnectionTerminated, () => { alert("Your connection to the server has been terminated."); }); + _socket.addEventListener("receiveGesturePoints", (content: GestureContent) => { + GestureOverlay.Instance.manualDispatch(content); + }); + _socket.addEventListener("receiveBoxTrigger", (enableBox: boolean) => { + GestureOverlay.Instance.showBox(enableBox); + }); } function errorFunc(): never { diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 4d487c032..4b8dfa119 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -16,6 +16,10 @@ import { Scripting } from "../util/Scripting"; import { FieldValue, Cast } from "../../new_fields/Types"; import { CurrentUserUtils } from "../../server/authentication/models/current_user_utils"; import Palette from "./Palette"; +import MobileInterface from "../../mobile/MobileInterface"; +import { MainView } from "./MainView"; +import { DocServer } from "../DocServer"; +import { GestureContent } from "../../server/Message"; @observer export default class GestureOverlay extends Touchable { @@ -35,6 +39,14 @@ export default class GestureOverlay extends Touchable { GestureOverlay.Instance = this; } + manualDispatch = (content: GestureContent) => { + console.log(content); + } + + showBox = (enableBox: boolean) => { + console.log("enable box?", enableBox); + } + @action handleHandDown = (e: React.TouchEvent) => { const fingers = InteractionUtils.GetMyTargetTouches(e, this.prevPoints, true); @@ -139,6 +151,15 @@ export default class GestureOverlay extends Touchable { const B = this.svgBounds; const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); + if (MobileInterface.Instance.drawingInk) { + const { selectedColor, selectedWidth } = InkingControl.Instance; + DocServer.Mobile.dispatchGesturePoints({ + points: this._points, + color: selectedColor, + width: selectedWidth + }); + } + const result = GestureUtils.GestureRecognizer.Recognize(new Array(points)); let actionPerformed = false; if (result && result.Score > 0.7) { diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 93fb0a07c..f1f7c37a3 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -49,6 +49,7 @@ export class MainView extends React.Component { private _flyoutSizeOnDown = 0; private _urlState: HistoryUtil.DocUrl; private _docBtnRef = React.createRef(); + private _mainViewRef = React.createRef(); @observable private _panelWidth: number = 0; @observable private _panelHeight: number = 0; @@ -507,8 +508,16 @@ export class MainView extends React.Component { return (null); } + get mainViewElement() { + return document.getElementById("mainView-container"); + } + + get mainViewRef() { + return this._mainViewRef; + } + render() { - return (
+ return (
@@ -518,7 +527,7 @@ export class MainView extends React.Component { - + diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 8d376fb57..bbfd22f50 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -114,7 +114,6 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { } private addDocument = (newBox: Doc) => { const added = this.props.addDocument(newBox); - console.log("adding doc from freeform", added); added && this.bringToFront(newBox); added && this.updateCluster(newBox); return added; @@ -362,7 +361,6 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { const points = ge.points; const B = this.getTransform().transformBounds(ge.bounds.left, ge.bounds.top, ge.bounds.width, ge.bounds.height); const inkDoc = Docs.Create.InkDocument(InkingControl.Instance.selectedColor, InkingControl.Instance.selectedTool, parseInt(InkingControl.Instance.selectedWidth), points, { x: B.x, y: B.y, width: B.width, height: B.height }); - console.log("make stroke", inkDoc); this.addDocument(inkDoc); e.stopPropagation(); break; @@ -991,7 +989,6 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { // otherwise, they are stored in fieldKey. All annotations to this document are stored in the extension document if (!this.extensionDoc) return (null); // let lodarea = this.Document[WidthSym]() * this.Document[HeightSym]() / this.props.ScreenToLocalTransform().Scale / this.props.ScreenToLocalTransform().Scale; - console.log("height freeform", this.isAnnotationOverlay, this.Document.scrollHeight, this.props.PanelHeight()); return
diff --git a/src/mobile/MobileInterface.tsx b/src/mobile/MobileInterface.tsx index 3cb08afa7..1429226b4 100644 --- a/src/mobile/MobileInterface.tsx +++ b/src/mobile/MobileInterface.tsx @@ -2,7 +2,7 @@ import React = require('react'); import { observer } from 'mobx-react'; import { computed, action, observable } from 'mobx'; import { CurrentUserUtils } from '../server/authentication/models/current_user_utils'; -import { FieldValue, Cast } from '../new_fields/Types'; +import { FieldValue, Cast, StrCast } from '../new_fields/Types'; import { Doc } from '../new_fields/Doc'; import { Docs } from '../client/documents/Documents'; import { CollectionView } from '../client/views/collections/CollectionView'; @@ -18,6 +18,10 @@ import { InkingControl } from '../client/views/InkingControl'; import { InkTool } from '../new_fields/InkField'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import "./MobileInterface.scss"; +import { SelectionManager } from '../client/util/SelectionManager'; +import { DateField } from '../new_fields/DateField'; +import { GestureUtils } from '../pen-gestures/GestureUtils'; +import { DocServer } from '../client/DocServer'; library.add(faTrash, faCheck); @@ -30,6 +34,7 @@ export default class MobileInterface extends React.Component { private mainDoc = CurrentUserUtils.setupMobileDoc(this.userDoc); private inkDoc?: Doc; + public drawingInk: boolean = false; constructor(props: Readonly<{}>) { super(props); @@ -61,6 +66,10 @@ export default class MobileInterface extends React.Component { this.inkDoc = CurrentUserUtils.setupMobileInkingDoc(this.userDoc); this.userDoc.activeMobile = this.inkDoc; InkingControl.Instance.switchTool(InkTool.Pen); + this.drawingInk = true; + + DocServer.Mobile.dispatchBoxTrigger(true); + break; } } @@ -100,17 +109,17 @@ export default class MobileInterface extends React.Component { } onClick = (e: React.MouseEvent) => { - // insert ink as collection into active displayed doc view - // reset ink collection - this.inkDoc = undefined; - console.log("clicked"); this.switchCurrentView("main"); InkingControl.Instance.switchTool(InkTool.None); // TODO: switch to previous tool + + DocServer.Mobile.dispatchBoxTrigger(false); + + this.inkDoc = undefined; + this.drawingInk = false; } @computed get inkContent() { - // TODO: get props from active collection and pass to the new freeform if (this.mainContainer) { return ( @@ -128,9 +137,9 @@ export default class MobileInterface extends React.Component { PanelHeight={() => window.innerHeight} PanelWidth={() => window.innerWidth} focus={emptyFunction} - isSelected={returnTrue} + isSelected={returnFalse} select={emptyFunction} - active={returnTrue} + active={returnFalse} ContentScaling={returnOne} whenActiveChanged={returnFalse} ScreenToLocalTransform={Transform.Identity} diff --git a/src/server/Message.ts b/src/server/Message.ts index 621abfd1e..fe74dafa5 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -1,4 +1,5 @@ import { Utils } from "../Utils"; +import { Point } from "../pen-gestures/ndollar"; export class Message { private _name: string; @@ -42,6 +43,16 @@ export interface Diff extends Reference { readonly diff: any; } +export interface GestureContent { + readonly points: Array; + readonly width?: string; + readonly color?: string; +} + +export interface MobileInkBoxTriggerContent { + readonly enableBox: boolean; +} + export namespace MessageStore { export const Foo = new Message("Foo"); export const Bar = new Message("Bar"); @@ -51,6 +62,8 @@ export namespace MessageStore { export const GetDocument = new Message("Get Document"); export const DeleteAll = new Message("Delete All"); export const ConnectionTerminated = new Message("Connection Terminated"); + export const GesturePoints = new Message("Gesture Points"); + export const MobileInkBoxTrigger = new Message("Trigger Mobile Ink Box"); export const GetRefField = new Message("Get Ref Field"); export const GetRefFields = new Message("Get Ref Fields"); diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 578147d60..67777f023 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -1,5 +1,5 @@ import { Utils } from "../../Utils"; -import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes } from "../Message"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes, GestureContent } from "../Message"; import { Client } from "../Client"; import { Socket } from "socket.io"; import { Database } from "../database"; @@ -54,6 +54,8 @@ export namespace WebSocket { Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); + Utils.AddServerHandler(socket, MessageStore.GesturePoints, content => processGesturePoints(socket, content)); + Utils.AddServerHandler(socket, MessageStore.MobileInkBoxTrigger, enableBox => processBoxTrigger(socket, enableBox)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); @@ -68,6 +70,14 @@ export namespace WebSocket { logPort("websocket", socketPort); } + function processGesturePoints(socket: Socket, content: GestureContent) { + socket.broadcast.emit("receiveGesturePoints", content); + } + + function processBoxTrigger(socket: Socket, enableBox: boolean) { + socket.broadcast.emit("receiveBoxTrigger", enableBox); + } + function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { const { ProjectCredentials } = GoogleCredentialsLoader; switch (query.type) { -- cgit v1.2.3-70-g09d2 From d1c1e9f10b8523270a872c6f67df25ebb1d9aa25 Mon Sep 17 00:00:00 2001 From: kimdahey Date: Mon, 27 Jan 2020 20:34:27 -0500 Subject: switching event type, some bugs :( --- src/client/views/Touchable.tsx | 65 +++++++++++++-- .../collectionFreeForm/CollectionFreeFormView.tsx | 1 + src/client/views/nodes/DocumentView.tsx | 94 +++++++++------------- 3 files changed, 100 insertions(+), 60 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/Touchable.tsx b/src/client/views/Touchable.tsx index 7800c4019..eed72e3c5 100644 --- a/src/client/views/Touchable.tsx +++ b/src/client/views/Touchable.tsx @@ -8,9 +8,12 @@ const HOLD_DURATION = 1000; export abstract class Touchable extends React.Component { //private holdTimer: NodeJS.Timeout | undefined; - private holdTimer: NodeJS.Timeout | undefined; + holdTimer: NodeJS.Timeout | undefined; private moveDisposer?: InteractionUtils.MultiTouchEventDisposer; private endDisposer?: InteractionUtils.MultiTouchEventDisposer; + private holdMoveDisposer?: InteractionUtils.MultiTouchEventDisposer; + private holdEndDisposer?: InteractionUtils.MultiTouchEventDisposer; + protected abstract multiTouchDisposer?: InteractionUtils.MultiTouchEventDisposer; protected _touchDrag: boolean = false; @@ -65,8 +68,10 @@ export abstract class Touchable extends React.Component { // clearTimeout(this.holdTimer) // this.holdTimer = undefined; // } - this.holdTimer = setTimeout(() => this.handle1PointerHoldStart(te, me), HOLD_DURATION); - // e.stopPropagation(); + console.log(this.holdTimer); + if (!this.holdTimer) { + this.holdTimer = setTimeout(() => this.handle1PointerHoldStart(e, me), HOLD_DURATION); + } // console.log(this.holdTimer); break; case 2: @@ -128,7 +133,12 @@ export abstract class Touchable extends React.Component { } } if (this.holdTimer) { + console.log(this.holdTimer); clearTimeout(this.holdTimer); + console.log(this.holdTimer); + + this.holdTimer = undefined; + console.log(this.holdTimer); console.log("clear"); } this._touchDrag = false; @@ -174,10 +184,17 @@ export abstract class Touchable extends React.Component { this.addEndListeners(); } - handle1PointerHoldStart = (e: React.TouchEvent, me: InteractionUtils.MultiTouchEvent): any => { + handle1PointerHoldStart = (e: Event, me: InteractionUtils.MultiTouchEvent): any => { e.stopPropagation(); - e.preventDefault(); + me.touchEvent.stopPropagation(); + this.holdTimer = undefined; this.removeMoveListeners(); + this.removeEndListeners(); + this.removeHoldMoveListeners(); + this.removeHoldEndListeners(); + this.addHoldMoveListeners(); + this.addHoldEndListeners(); + } addMoveListeners = () => { @@ -200,6 +217,44 @@ export abstract class Touchable extends React.Component { this.endDisposer && this.endDisposer(); } + addHoldMoveListeners = () => { + const handler = (e: Event) => this.handle1PointerHoldMove(e, (e as CustomEvent>).detail); + document.addEventListener("dashOnTouchHoldMove", handler); + this.holdMoveDisposer = () => document.removeEventListener("dashOnTouchHoldMove", handler); + } + + addHoldEndListeners = () => { + const handler = (e: Event) => this.handle1PointerHoldEnd(e, (e as CustomEvent>).detail); + document.addEventListener("dashOnTouchHoldEnd", handler); + this.holdEndDisposer = () => document.removeEventListener("dashOnTouchHoldEnd", handler); + } + + removeHoldMoveListeners = () => { + this.holdMoveDisposer && this.holdMoveDisposer(); + } + + removeHoldEndListeners = () => { + this.holdEndDisposer && this.holdEndDisposer(); + } + + + handle1PointerHoldMove = (e: Event, me: InteractionUtils.MultiTouchEvent): void => { + e.stopPropagation(); + me.touchEvent.stopPropagation(); + } + + + handle1PointerHoldEnd = (e: Event, me: InteractionUtils.MultiTouchEvent): void => { + e.stopPropagation(); + me.touchEvent.stopPropagation(); + this.removeHoldMoveListeners(); + this.removeHoldEndListeners(); + + me.touchEvent.stopPropagation(); + me.touchEvent.preventDefault(); + } + + handleHandDown = (e: React.TouchEvent) => { // e.stopPropagation(); // e.preventDefault(); diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index aaa585b55..2622d9b28 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -494,6 +494,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { const myTouches = InteractionUtils.GetMyTargetTouches(me, this.prevPoints, true); const pt1 = myTouches[0]; const pt2 = myTouches[1]; + console.log(myTouches); if (this.prevPoints.size === 2) { const oldPoint1 = this.prevPoints.get(pt1.identifier); diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 0b6a284d6..ae2ee9e41 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -119,61 +119,45 @@ export class DocumentView extends DocComponent(Docu private _firstY: number = 0; - // handle1PointerHoldStart = (e: React.TouchEvent): any => { - // this.onRadialMenu(e); - // const pt = InteractionUtils.GetMyTargetTouches(e, this.prevPoints, true)[0]; - // this._firstX = pt.pageX; - // this._firstY = pt.pageY; - // e.stopPropagation(); - // e.preventDefault(); - - // document.removeEventListener("touchmove", this.onTouch); - // document.removeEventListener("touchmove", this.handle1PointerHoldMove); - // document.addEventListener("touchmove", this.handle1PointerHoldMove); - // document.removeEventListener("touchend", this.handle1PointerHoldEnd); - // document.addEventListener("touchend", this.handle1PointerHoldEnd); - // } - - // handle1PointerHoldMove = (e: TouchEvent): void => { - // const pt = InteractionUtils.GetMyTargetTouches(me, this.prevPoints, true)[0]; - // if (Math.abs(pt.pageX - this._firstX) > 150 || Math.abs(pt.pageY - this._firstY) > 150) { - // this.handleRelease(); - // } - // document.removeEventListener("touchmove", this.handle1PointerHoldMove); - // document.addEventListener("touchmove", this.handle1PointerHoldMove); - // document.removeEventListener("touchend", this.handle1PointerHoldEnd); - // document.addEventListener("touchend", this.handle1PointerHoldEnd); - // } - - // handleRelease() { - // RadialMenu.Instance.closeMenu(); - // document.removeEventListener("touchmove", this.handle1PointerHoldMove); - // document.removeEventListener("touchend", this.handle1PointerHoldEnd); - // } - - // handle1PointerHoldEnd = (e: TouchEvent): void => { - // RadialMenu.Instance.closeMenu(); - // document.removeEventListener("touchmove", this.handle1PointerHoldMove); - // document.removeEventListener("touchend", this.handle1PointerHoldEnd); - // } - - // @action - // onRadialMenu = (e: React.TouchEvent): void => { - // const pt = InteractionUtils.GetMyTargetTouches(me, this.prevPoints, true)[0]; - - // RadialMenu.Instance.openMenu(); - - // RadialMenu.Instance.addItem({ description: "Open Fields", event: () => this.props.addDocTab(Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 }), undefined, "onRight"), icon: "layer-group", selected: -1 }); - // RadialMenu.Instance.addItem({ description: "Delete this document", event: () => this.props.ContainingCollectionView?.removeDocument(this.props.Document), icon: "trash", selected: -1 }); - // RadialMenu.Instance.addItem({ description: "Open in a new tab", event: () => this.props.addDocTab(this.props.Document, undefined, "onRight"), icon: "folder", selected: -1 }); - // RadialMenu.Instance.addItem({ description: "Pin to Presentation", event: () => this.props.pinToPres(this.props.Document), icon: "map-pin", selected: -1 }); - - // RadialMenu.Instance.displayMenu(pt.pageX - 15, pt.pageY - 15); - // if (!SelectionManager.IsSelected(this, true)) { - // SelectionManager.SelectDoc(this, false); - // } - // e.stopPropagation(); - // } + handle1PointerHoldStart = (e: Event, me: InteractionUtils.MultiTouchEvent): any => { + console.log("S"); + this.onRadialMenu(e, me); + const pt = InteractionUtils.GetMyTargetTouches(me, this.prevPoints, true)[0]; + this._firstX = pt.pageX; + this._firstY = pt.pageY; + + } + + handle1PointerHoldMove = (e: Event, me: InteractionUtils.MultiTouchEvent): void => { + console.log("K"); + const pt = InteractionUtils.GetMyTargetTouches(me, this.prevPoints, true)[0]; + if (Math.abs(pt.pageX - this._firstX) > 150 || Math.abs(pt.pageY - this._firstY) > 150) { + this.handle1PointerHoldEnd(e, me); + } + } + + handle1PointerHoldEnd = (e: Event, me: InteractionUtils.MultiTouchEvent): void => { + console.log("E"); + RadialMenu.Instance.closeMenu(); + } + + @action + onRadialMenu = (e: Event, me: InteractionUtils.MultiTouchEvent): void => { + const pt = InteractionUtils.GetMyTargetTouches(me, this.prevPoints, true)[0]; + + RadialMenu.Instance.openMenu(); + + RadialMenu.Instance.addItem({ description: "Open Fields", event: () => this.props.addDocTab(Docs.Create.KVPDocument(this.props.Document, { width: 300, height: 300 }), undefined, "onRight"), icon: "layer-group", selected: -1 }); + RadialMenu.Instance.addItem({ description: "Delete this document", event: () => this.props.ContainingCollectionView?.removeDocument(this.props.Document), icon: "trash", selected: -1 }); + RadialMenu.Instance.addItem({ description: "Open in a new tab", event: () => this.props.addDocTab(this.props.Document, undefined, "onRight"), icon: "folder", selected: -1 }); + RadialMenu.Instance.addItem({ description: "Pin to Presentation", event: () => this.props.pinToPres(this.props.Document), icon: "map-pin", selected: -1 }); + + RadialMenu.Instance.displayMenu(pt.pageX - 15, pt.pageY - 15); + if (!SelectionManager.IsSelected(this, true)) { + SelectionManager.SelectDoc(this, false); + } + e.stopPropagation(); + } @action componentDidMount() { -- cgit v1.2.3-70-g09d2 From ecf0f5b8f426db9e66c05e759f61294811b15fca Mon Sep 17 00:00:00 2001 From: vellichora Date: Sat, 1 Feb 2020 14:56:19 -0500 Subject: mobile ink overlay is draggable from desktop --- src/client/DocServer.ts | 19 ++- src/client/util/InteractionUtils.ts | 165 ------------------- src/client/util/InteractionUtils.tsx | 182 +++++++++++++++++++++ src/client/views/GestureOverlay.scss | 4 - src/client/views/GestureOverlay.tsx | 25 +-- src/client/views/InkingStroke.tsx | 17 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 1 - src/client/views/nodes/DocumentView.tsx | 1 + src/mobile/MobileInkOverlay.scss | 38 +++++ src/mobile/MobileInkOverlay.tsx | 91 +++++++++-- src/mobile/MobileInterface.scss | 2 +- src/mobile/MobileInterface.tsx | 90 ++++++---- src/server/Message.ts | 14 +- src/server/Websocket/Websocket.ts | 8 +- 14 files changed, 393 insertions(+), 264 deletions(-) delete mode 100644 src/client/util/InteractionUtils.ts create mode 100644 src/client/util/InteractionUtils.tsx create mode 100644 src/mobile/MobileInkOverlay.scss (limited to 'src/client/views/collections') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index a1cb42df2..b20cd3521 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -1,5 +1,5 @@ import * as OpenSocket from 'socket.io-client'; -import { MessageStore, YoutubeQueryTypes, GestureContent, MobileInkBoxContent } from "./../server/Message"; +import { MessageStore, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent, UpdateMobileInkOverlayPosition } from "./../server/Message"; import { Opt, Doc } from '../new_fields/Doc'; import { Utils, emptyFunction } from '../Utils'; import { SerializationHelper } from './util/SerializationHelper'; @@ -72,9 +72,13 @@ export namespace DocServer { Utils.Emit(_socket, MessageStore.GesturePoints, content); } - export function dispatchBoxTrigger(content: MobileInkBoxContent) { + export function dispatchOverlayTrigger(content: MobileInkOverlayContent) { // _socket.emit("dispatchBoxTrigger"); - Utils.Emit(_socket, MessageStore.MobileInkBoxTrigger, content); + Utils.Emit(_socket, MessageStore.MobileInkOverlayTrigger, content); + } + + export function dispatchOverlayPositionUpdate(content: UpdateMobileInkOverlayPosition) { + Utils.Emit(_socket, MessageStore.UpdateMobileInkOverlayPosition, content); } } @@ -100,13 +104,18 @@ export namespace DocServer { Utils.AddServerHandler(_socket, MessageStore.ConnectionTerminated, () => { alert("Your connection to the server has been terminated."); }); + + // mobile ink overlay socket events to communicate between mobile view and desktop view _socket.addEventListener("receiveGesturePoints", (content: GestureContent) => { MobileInkOverlay.Instance.drawStroke(content); }); - _socket.addEventListener("receiveBoxTrigger", (content: MobileInkBoxContent) => { - GestureOverlay.Instance.enableMobileInkBox(content); + _socket.addEventListener("receiveOverlayTrigger", (content: MobileInkOverlayContent) => { + GestureOverlay.Instance.enableMobileInkOverlay(content); MobileInkOverlay.Instance.initMobileInkOverlay(content); }); + _socket.addEventListener("updateMobileOverlayPosition", (content: UpdateMobileInkOverlayPosition) => { + MobileInkOverlay.Instance.updatePosition(content); + }); } function errorFunc(): never { diff --git a/src/client/util/InteractionUtils.ts b/src/client/util/InteractionUtils.ts deleted file mode 100644 index 76b43da3c..000000000 --- a/src/client/util/InteractionUtils.ts +++ /dev/null @@ -1,165 +0,0 @@ -export namespace InteractionUtils { - export const MOUSETYPE = "mouse"; - export const TOUCHTYPE = "touch"; - export const PENTYPE = "pen"; - export const ERASERTYPE = "eraser"; - - const POINTER_PEN_BUTTON = -1; - const REACT_POINTER_PEN_BUTTON = 0; - const ERASER_BUTTON = 5; - - export function GetMyTargetTouches(e: TouchEvent | React.TouchEvent, prevPoints: Map, ignorePen: boolean): React.Touch[] { - const myTouches = new Array(); - for (let i = 0; i < e.targetTouches.length; i++) { - const pt: any = e.targetTouches.item(i); - if (pt && prevPoints.has(pt.identifier)) { - if (ignorePen || (pt.radiusX > 1 && pt.radiusY > 1)) { - myTouches.push(pt); - } - } - } - return myTouches; - } - - export function IsType(e: PointerEvent | React.PointerEvent, type: string): boolean { - switch (type) { - // pen and eraser are both pointer type 'pen', but pen is button 0 and eraser is button 5. -syip2 - case PENTYPE: - return e.pointerType === PENTYPE && (e.button === -1 || e.button === 0); - case ERASERTYPE: - return e.pointerType === PENTYPE && e.button === (e instanceof PointerEvent ? ERASER_BUTTON : ERASER_BUTTON); - default: - return e.pointerType === type; - } - } - - export function TwoPointEuclidist(pt1: React.Touch, pt2: React.Touch): number { - return Math.sqrt(Math.pow(pt1.clientX - pt2.clientX, 2) + Math.pow(pt1.clientY - pt2.clientY, 2)); - } - - /** - * Returns the centroid of an n-arbitrary long list of points (takes the average the x and y components of each point) - * @param pts - n-arbitrary long list of points - */ - export function CenterPoint(pts: React.Touch[]): { X: number, Y: number } { - const centerX = pts.map(pt => pt.clientX).reduce((a, b) => a + b, 0) / pts.length; - const centerY = pts.map(pt => pt.clientY).reduce((a, b) => a + b, 0) / pts.length; - return { X: centerX, Y: centerY }; - } - - /** - * Returns -1 if pinching out, 0 if not pinching, and 1 if pinching in - * @param pt1 - new point that corresponds to oldPoint1 - * @param pt2 - new point that corresponds to oldPoint2 - * @param oldPoint1 - previous point 1 - * @param oldPoint2 - previous point 2 - */ - export function Pinching(pt1: React.Touch, pt2: React.Touch, oldPoint1: React.Touch, oldPoint2: React.Touch): number { - const threshold = 4; - const oldDist = TwoPointEuclidist(oldPoint1, oldPoint2); - const newDist = TwoPointEuclidist(pt1, pt2); - - /** if they have the same sign, then we are either pinching in or out. - * threshold it by 10 (it has to be pinching by at least threshold to be a valid pinch) - * so that it can still pan without freaking out - */ - if (Math.sign(oldDist) === Math.sign(newDist) && Math.abs(oldDist - newDist) > threshold) { - return Math.sign(oldDist - newDist); - } - return 0; - } - - /** - * Returns -1 if pinning and pinching out, 0 if not pinning, and 1 if pinching in - * @param pt1 - new point that corresponds to oldPoint1 - * @param pt2 - new point that corresponds to oldPoint2 - * @param oldPoint1 - previous point 1 - * @param oldPoint2 - previous point 2 - */ - export function Pinning(pt1: React.Touch, pt2: React.Touch, oldPoint1: React.Touch, oldPoint2: React.Touch): number { - const threshold = 4; - - const pt1Dist = TwoPointEuclidist(oldPoint1, pt1); - const pt2Dist = TwoPointEuclidist(oldPoint2, pt2); - - const pinching = Pinching(pt1, pt2, oldPoint1, oldPoint2); - - if (pinching !== 0) { - if ((pt1Dist < threshold && pt2Dist > threshold) || (pt1Dist > threshold && pt2Dist < threshold)) { - return pinching; - } - } - return 0; - } - - export function IsDragging(oldTouches: Map, newTouches: React.Touch[], leniency: number): boolean { - for (const touch of newTouches) { - if (touch) { - const oldTouch = oldTouches.get(touch.identifier); - if (oldTouch) { - if (TwoPointEuclidist(touch, oldTouch) >= leniency) { - return true; - } - } - } - } - return false; - } - - // These might not be very useful anymore, but I'll leave them here for now -syip2 - { - - - /** - * Returns the type of Touch Interaction from a list of points. - * Also returns any data that is associated with a Touch Interaction - * @param pts - List of points - */ - // export function InterpretPointers(pts: React.Touch[]): { type: Opt, data?: any } { - // const leniency = 200; - // switch (pts.length) { - // case 1: - // return { type: OneFinger }; - // case 2: - // return { type: TwoSeperateFingers }; - // case 3: - // let pt1 = pts[0]; - // let pt2 = pts[1]; - // let pt3 = pts[2]; - // if (pt1 && pt2 && pt3) { - // let dist12 = TwoPointEuclidist(pt1, pt2); - // let dist23 = TwoPointEuclidist(pt2, pt3); - // let dist13 = TwoPointEuclidist(pt1, pt3); - // console.log(`distances: ${dist12}, ${dist23}, ${dist13}`); - // let dist12close = dist12 < leniency; - // let dist23close = dist23 < leniency; - // let dist13close = dist13 < leniency; - // let xor2313 = dist23close ? !dist13close : dist13close; - // let xor = dist12close ? !xor2313 : xor2313; - // // three input xor because javascript doesn't have logical xor's - // if (xor) { - // let points: number[] = []; - // let min = Math.min(dist12, dist23, dist13); - // switch (min) { - // case dist12: - // points = [0, 1, 2]; - // break; - // case dist23: - // points = [1, 2, 0]; - // break; - // case dist13: - // points = [0, 2, 1]; - // break; - // } - // return { type: TwoToOneFingers, data: points }; - // } - // else { - // return { type: ThreeSeperateFingers, data: null }; - // } - // } - // default: - // return { type: undefined }; - // } - // } - } -} \ No newline at end of file diff --git a/src/client/util/InteractionUtils.tsx b/src/client/util/InteractionUtils.tsx new file mode 100644 index 000000000..34c7cfa74 --- /dev/null +++ b/src/client/util/InteractionUtils.tsx @@ -0,0 +1,182 @@ +import React = require("react"); + +export namespace InteractionUtils { + export const MOUSETYPE = "mouse"; + export const TOUCHTYPE = "touch"; + export const PENTYPE = "pen"; + export const ERASERTYPE = "eraser"; + + const POINTER_PEN_BUTTON = -1; + const REACT_POINTER_PEN_BUTTON = 0; + const ERASER_BUTTON = 5; + + export function GetMyTargetTouches(e: TouchEvent | React.TouchEvent, prevPoints: Map, ignorePen: boolean): React.Touch[] { + const myTouches = new Array(); + for (let i = 0; i < e.targetTouches.length; i++) { + const pt: any = e.targetTouches.item(i); + if (pt && prevPoints.has(pt.identifier)) { + if (ignorePen || (pt.radiusX > 1 && pt.radiusY > 1)) { + myTouches.push(pt); + } + } + } + return myTouches; + } + + // TODO: find a way to reference this function from InkingStroke instead of copy pastign here. copied bc of weird error when on mobile view + export function CreatePolyline(points: { X: number, Y: number }[], left: number, top: number, color: string, width: number) { + const pts = points.reduce((acc: string, pt: { X: number, Y: number }) => acc + `${pt.X - left},${pt.Y - top} `, ""); + return ( + + ); + } + + export function IsType(e: PointerEvent | React.PointerEvent, type: string): boolean { + switch (type) { + // pen and eraser are both pointer type 'pen', but pen is button 0 and eraser is button 5. -syip2 + case PENTYPE: + return e.pointerType === PENTYPE && (e.button === -1 || e.button === 0); + case ERASERTYPE: + return e.pointerType === PENTYPE && e.button === (e instanceof PointerEvent ? ERASER_BUTTON : ERASER_BUTTON); + default: + return e.pointerType === type; + } + } + + export function TwoPointEuclidist(pt1: React.Touch, pt2: React.Touch): number { + return Math.sqrt(Math.pow(pt1.clientX - pt2.clientX, 2) + Math.pow(pt1.clientY - pt2.clientY, 2)); + } + + /** + * Returns the centroid of an n-arbitrary long list of points (takes the average the x and y components of each point) + * @param pts - n-arbitrary long list of points + */ + export function CenterPoint(pts: React.Touch[]): { X: number, Y: number } { + const centerX = pts.map(pt => pt.clientX).reduce((a, b) => a + b, 0) / pts.length; + const centerY = pts.map(pt => pt.clientY).reduce((a, b) => a + b, 0) / pts.length; + return { X: centerX, Y: centerY }; + } + + /** + * Returns -1 if pinching out, 0 if not pinching, and 1 if pinching in + * @param pt1 - new point that corresponds to oldPoint1 + * @param pt2 - new point that corresponds to oldPoint2 + * @param oldPoint1 - previous point 1 + * @param oldPoint2 - previous point 2 + */ + export function Pinching(pt1: React.Touch, pt2: React.Touch, oldPoint1: React.Touch, oldPoint2: React.Touch): number { + const threshold = 4; + const oldDist = TwoPointEuclidist(oldPoint1, oldPoint2); + const newDist = TwoPointEuclidist(pt1, pt2); + + /** if they have the same sign, then we are either pinching in or out. + * threshold it by 10 (it has to be pinching by at least threshold to be a valid pinch) + * so that it can still pan without freaking out + */ + if (Math.sign(oldDist) === Math.sign(newDist) && Math.abs(oldDist - newDist) > threshold) { + return Math.sign(oldDist - newDist); + } + return 0; + } + + /** + * Returns -1 if pinning and pinching out, 0 if not pinning, and 1 if pinching in + * @param pt1 - new point that corresponds to oldPoint1 + * @param pt2 - new point that corresponds to oldPoint2 + * @param oldPoint1 - previous point 1 + * @param oldPoint2 - previous point 2 + */ + export function Pinning(pt1: React.Touch, pt2: React.Touch, oldPoint1: React.Touch, oldPoint2: React.Touch): number { + const threshold = 4; + + const pt1Dist = TwoPointEuclidist(oldPoint1, pt1); + const pt2Dist = TwoPointEuclidist(oldPoint2, pt2); + + const pinching = Pinching(pt1, pt2, oldPoint1, oldPoint2); + + if (pinching !== 0) { + if ((pt1Dist < threshold && pt2Dist > threshold) || (pt1Dist > threshold && pt2Dist < threshold)) { + return pinching; + } + } + return 0; + } + + export function IsDragging(oldTouches: Map, newTouches: React.Touch[], leniency: number): boolean { + for (const touch of newTouches) { + if (touch) { + const oldTouch = oldTouches.get(touch.identifier); + if (oldTouch) { + if (TwoPointEuclidist(touch, oldTouch) >= leniency) { + return true; + } + } + } + } + return false; + } + + // These might not be very useful anymore, but I'll leave them here for now -syip2 + { + + + /** + * Returns the type of Touch Interaction from a list of points. + * Also returns any data that is associated with a Touch Interaction + * @param pts - List of points + */ + // export function InterpretPointers(pts: React.Touch[]): { type: Opt, data?: any } { + // const leniency = 200; + // switch (pts.length) { + // case 1: + // return { type: OneFinger }; + // case 2: + // return { type: TwoSeperateFingers }; + // case 3: + // let pt1 = pts[0]; + // let pt2 = pts[1]; + // let pt3 = pts[2]; + // if (pt1 && pt2 && pt3) { + // let dist12 = TwoPointEuclidist(pt1, pt2); + // let dist23 = TwoPointEuclidist(pt2, pt3); + // let dist13 = TwoPointEuclidist(pt1, pt3); + // console.log(`distances: ${dist12}, ${dist23}, ${dist13}`); + // let dist12close = dist12 < leniency; + // let dist23close = dist23 < leniency; + // let dist13close = dist13 < leniency; + // let xor2313 = dist23close ? !dist13close : dist13close; + // let xor = dist12close ? !xor2313 : xor2313; + // // three input xor because javascript doesn't have logical xor's + // if (xor) { + // let points: number[] = []; + // let min = Math.min(dist12, dist23, dist13); + // switch (min) { + // case dist12: + // points = [0, 1, 2]; + // break; + // case dist23: + // points = [1, 2, 0]; + // break; + // case dist13: + // points = [0, 2, 1]; + // break; + // } + // return { type: TwoToOneFingers, data: points }; + // } + // else { + // return { type: ThreeSeperateFingers, data: null }; + // } + // } + // default: + // return { type: undefined }; + // } + // } + } +} \ No newline at end of file diff --git a/src/client/views/GestureOverlay.scss b/src/client/views/GestureOverlay.scss index 2996b7073..31601efd4 100644 --- a/src/client/views/GestureOverlay.scss +++ b/src/client/views/GestureOverlay.scss @@ -5,8 +5,4 @@ top: 0; left: 0; touch-action: none; -} - -.mobileInkOverlay { - border: 5px dashed red; } \ No newline at end of file diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 81284b543..a01a86b53 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -19,7 +19,7 @@ import Palette from "./Palette"; import MobileInterface from "../../mobile/MobileInterface"; import { MainView } from "./MainView"; import { DocServer } from "../DocServer"; -import { GestureContent, MobileInkBoxContent } from "../../server/Message"; +import { GestureContent, MobileInkOverlayContent } from "../../server/Message"; import { Point } from "../northstar/model/idea/idea"; import MobileInkOverlay from "../../mobile/MobileInkOverlay"; @@ -217,21 +217,6 @@ export default class GestureOverlay extends Touchable { return { right: right, left: left, bottom: bottom, top: top, width: right - left, height: bottom - top }; } - // TODO: find a way to reference this function from InkingStroke instead of copy pastign here. copied bc of weird error when on mobile view - CreatePolyline(points: { X: number, Y: number }[], left: number, top: number, color?: string, width?: number) { - const pts = points.reduce((acc: string, pt: { X: number, Y: number }) => acc + `${pt.X - left},${pt.Y - top} `, ""); - return ( - - ); - } - @computed get currentStroke() { if (this._points.length <= 1) { return (null); @@ -241,23 +226,23 @@ export default class GestureOverlay extends Touchable { return ( - {this.CreatePolyline(this._points, B.left, B.top, this.Color, this.Width)} + {InteractionUtils.CreatePolyline(this._points, B.left, B.top, this.Color, this.Width)} ); } @action - enableMobileInkBox = (content: MobileInkBoxContent) => { - this.showMobileInkOverlay = content.enableBox; + enableMobileInkOverlay = (content: MobileInkOverlayContent) => { + this.showMobileInkOverlay = content.enableOverlay; } render() { return (
+ {this.showMobileInkOverlay ? : <>} {this.currentStroke} {this.props.children} {this._palette} - {this.showMobileInkOverlay ? : <>}
); } } diff --git a/src/client/views/InkingStroke.tsx b/src/client/views/InkingStroke.tsx index 8b346d5d9..aca507147 100644 --- a/src/client/views/InkingStroke.tsx +++ b/src/client/views/InkingStroke.tsx @@ -10,24 +10,11 @@ import "./InkingStroke.scss"; import { FieldView, FieldViewProps } from "./nodes/FieldView"; import React = require("react"); import { TraceMobx } from "../../new_fields/util"; +import { InteractionUtils } from "../util/InteractionUtils"; type InkDocument = makeInterface<[typeof documentSchema]>; const InkDocument = makeInterface(documentSchema); -export function CreatePolyline(points: { X: number, Y: number }[], left: number, top: number, color?: string, width?: number) { - const pts = points.reduce((acc: string, pt: { X: number, Y: number }) => acc + `${pt.X - left},${pt.Y - top} `, ""); - return ( - - ); -} - @observer export class InkingStroke extends DocExtendableComponent(InkDocument) { public static LayoutString(fieldStr: string) { return FieldView.LayoutString(InkingStroke, fieldStr); } @@ -44,7 +31,7 @@ export class InkingStroke extends DocExtendableComponent(Docu if (!(InteractionUtils.IsType(e, InteractionUtils.MOUSETYPE) || InkingControl.Instance.selectedTool === InkTool.Highlighter || InkingControl.Instance.selectedTool === InkTool.Pen)) { if (!InteractionUtils.IsType(e, InteractionUtils.PENTYPE)) { e.stopPropagation(); + // TODO: check here for panning/inking } return; } diff --git a/src/mobile/MobileInkOverlay.scss b/src/mobile/MobileInkOverlay.scss new file mode 100644 index 000000000..2e45d0441 --- /dev/null +++ b/src/mobile/MobileInkOverlay.scss @@ -0,0 +1,38 @@ +.mobileInkOverlay { + border: 5px dashed red; +} + +.mobileInkOverlay-border { + background-color: rgba(0, 255, 0, .4); + position: absolute; + pointer-events: auto; + cursor: pointer; + + &.top { + width: calc(100% + 20px); + height: 10px; + top: -10px; + left: -10px; + } + + &.left { + width: 10px; + height: calc(100% + 20px); + top: -10px; + left: -10px; + } + + &.right { + width: 10px; + height: calc(100% + 20px); + top: -10px; + right: -10px; + } + + &.bottom { + width: calc(100% + 20px); + height: 10px; + bottom: -10px; + left: -10px; + } +} \ No newline at end of file diff --git a/src/mobile/MobileInkOverlay.tsx b/src/mobile/MobileInkOverlay.tsx index 71dd20c51..5efc7b83a 100644 --- a/src/mobile/MobileInkOverlay.tsx +++ b/src/mobile/MobileInkOverlay.tsx @@ -1,8 +1,9 @@ import React = require('react'); import { observer } from "mobx-react"; -import { MobileInkBoxContent, GestureContent } from "../server/Message"; +import { MobileInkOverlayContent, GestureContent, UpdateMobileInkOverlayPosition } from "../server/Message"; import { observable, action } from "mobx"; import { GestureUtils } from "../pen-gestures/GestureUtils"; +import "./MobileInkOverlay.scss"; @observer @@ -15,6 +16,11 @@ export default class MobileInkOverlay extends React.Component { @observable private _x: number = -300; @observable private _y: number = -300; + @observable private _offsetX: number = 0; + @observable private _offsetY: number = 0; + @observable private _isDragging: boolean = false; + private _mainCont: React.RefObject = React.createRef(); + constructor(props: Readonly<{}>) { super(props); MobileInkOverlay.Instance = this; @@ -28,18 +34,27 @@ export default class MobileInkOverlay extends React.Component { } @action - initMobileInkOverlay(content: MobileInkBoxContent) { + initMobileInkOverlay(content: MobileInkOverlayContent) { const { width, height } = content; const scaledSize = this.initialSize(width ? width : 0, height ? height : 0); - this._width = scaledSize.width; - this._height = scaledSize.height; - this._scale = scaledSize.scale; + this._width = scaledSize.width * .5; + this._height = scaledSize.height * .5; + this._scale = .5; //scaledSize.scale; this._x = 300; // TODO: center on screen this._y = 25; // TODO: center on screen } + @action + updatePosition(content: UpdateMobileInkOverlayPosition) { + const { dx, dy, dsize } = content; + console.log(dx, dy, dsize); + } + drawStroke = (content: GestureContent) => { + // TODO: figure out why strokes drawn in corner of mobile interface dont get inserted + const { points, bounds } = content; + console.log("received points", points, bounds); const B = { right: (bounds.right * this._scale) + this._x, @@ -65,16 +80,66 @@ export default class MobileInkOverlay extends React.Component { ); } + @action + dragStart = (e: React.PointerEvent) => { + console.log("pointer down"); + document.removeEventListener("pointermove", this.dragging); + document.removeEventListener("pointerup", this.dragEnd); + document.addEventListener("pointermove", this.dragging); + document.addEventListener("pointerup", this.dragEnd); + + this._isDragging = true; + this._offsetX = e.pageX - this._mainCont.current!.getBoundingClientRect().left; + this._offsetY = e.pageY - this._mainCont.current!.getBoundingClientRect().top; + + e.preventDefault(); + e.stopPropagation(); + } + + @action + dragging = (e: PointerEvent) => { + const x = e.pageX - this._offsetX; + const y = e.pageY - this._offsetY; + + // TODO: don't allow drag over library? + this._x = Math.min(Math.max(x, 0), window.innerWidth - this._width); + this._y = Math.min(Math.max(y, 0), window.innerHeight - this._height); + + e.preventDefault(); + e.stopPropagation(); + } + + @action + dragEnd = (e: PointerEvent) => { + document.removeEventListener("pointermove", this.dragging); + document.removeEventListener("pointerup", this.dragEnd); + + this._isDragging = false; + + e.preventDefault(); + e.stopPropagation(); + } + render() { + return ( -
+
+
+
+
+
+
); } } \ No newline at end of file diff --git a/src/mobile/MobileInterface.scss b/src/mobile/MobileInterface.scss index e4cc919a5..8abe5a40d 100644 --- a/src/mobile/MobileInterface.scss +++ b/src/mobile/MobileInterface.scss @@ -1,4 +1,4 @@ -.mobileInterface-topButtons { +.mobileInterface-inkInterfaceButtons { position: absolute; display: flex; justify-content: space-between; diff --git a/src/mobile/MobileInterface.tsx b/src/mobile/MobileInterface.tsx index 4840ea374..b191b3afb 100644 --- a/src/mobile/MobileInterface.tsx +++ b/src/mobile/MobileInterface.tsx @@ -10,7 +10,7 @@ import { DocumentView } from '../client/views/nodes/DocumentView'; import { emptyPath, emptyFunction, returnFalse, returnOne, returnEmptyString, returnTrue } from '../Utils'; import { Transform } from '../client/util/Transform'; import { library } from '@fortawesome/fontawesome-svg-core'; -import { faPenNib, faHighlighter, faEraser, faMousePointer, faBreadSlice, faTrash, faCheck } from '@fortawesome/free-solid-svg-icons'; +import { faPenNib, faHighlighter, faEraser, faMousePointer, faBreadSlice, faTrash, faCheck, faLongArrowAltLeft } from '@fortawesome/free-solid-svg-icons'; import { Scripting } from '../client/util/Scripting'; import { CollectionFreeFormView } from '../client/views/collections/collectionFreeForm/CollectionFreeFormView'; import GestureOverlay from '../client/views/GestureOverlay'; @@ -23,7 +23,7 @@ import { DateField } from '../new_fields/DateField'; import { GestureUtils } from '../pen-gestures/GestureUtils'; import { DocServer } from '../client/DocServer'; -library.add(faTrash, faCheck); +library.add(faLongArrowAltLeft); @observer export default class MobileInterface extends React.Component { @@ -68,8 +68,8 @@ export default class MobileInterface extends React.Component { InkingControl.Instance.switchTool(InkTool.Pen); this.drawingInk = true; - DocServer.Mobile.dispatchBoxTrigger({ - enableBox: true, + DocServer.Mobile.dispatchOverlayTrigger({ + enableOverlay: true, width: window.innerWidth, height: window.innerHeight }); @@ -112,12 +112,12 @@ export default class MobileInterface extends React.Component { return "hello"; } - onClick = (e: React.MouseEvent) => { + onBack = (e: React.MouseEvent) => { this.switchCurrentView("main"); InkingControl.Instance.switchTool(InkTool.None); // TODO: switch to previous tool - DocServer.Mobile.dispatchBoxTrigger({ - enableBox: false, + DocServer.Mobile.dispatchOverlayTrigger({ + enableOverlay: false, width: window.innerWidth, height: window.innerHeight }); @@ -126,37 +126,61 @@ export default class MobileInterface extends React.Component { this.drawingInk = false; } + shiftLeft = (e: React.MouseEvent) => { + DocServer.Mobile.dispatchOverlayPositionUpdate({ + dx: -10 + }); + } + + shiftRight = (e: React.MouseEvent) => { + DocServer.Mobile.dispatchOverlayPositionUpdate({ + dx: 10 + }); + } + @computed get inkContent() { + // TODO: support panning and zooming + // TODO: handle moving of ink strokes if (this.mainContainer) { return ( - -
- - +
+
+
+ +
+
+ +
+
+ + +
- window.innerHeight} - PanelWidth={() => window.innerWidth} - focus={emptyFunction} - isSelected={returnFalse} - select={emptyFunction} - active={returnFalse} - ContentScaling={returnOne} - whenActiveChanged={returnFalse} - ScreenToLocalTransform={Transform.Identity} - ruleProvider={undefined} - renderDepth={0} - ContainingCollectionView={undefined} - ContainingCollectionDoc={undefined}> - - + + window.innerHeight} + PanelWidth={() => window.innerWidth} + focus={emptyFunction} + isSelected={returnFalse} + select={emptyFunction} + active={returnFalse} + ContentScaling={returnOne} + whenActiveChanged={returnFalse} + ScreenToLocalTransform={Transform.Identity} + ruleProvider={undefined} + renderDepth={0} + ContainingCollectionView={undefined} + ContainingCollectionDoc={undefined}> + + +
); } } diff --git a/src/server/Message.ts b/src/server/Message.ts index 1958286df..064a19653 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -50,12 +50,18 @@ export interface GestureContent { readonly color?: string; } -export interface MobileInkBoxContent { - readonly enableBox: boolean; +export interface MobileInkOverlayContent { + readonly enableOverlay: boolean; readonly width?: number; readonly height?: number; } +export interface UpdateMobileInkOverlayPosition { + readonly dx?: number; + readonly dy?: number; + readonly dsize?: number; +} + export namespace MessageStore { export const Foo = new Message("Foo"); export const Bar = new Message("Bar"); @@ -65,8 +71,10 @@ export namespace MessageStore { export const GetDocument = new Message("Get Document"); export const DeleteAll = new Message("Delete All"); export const ConnectionTerminated = new Message("Connection Terminated"); + export const GesturePoints = new Message("Gesture Points"); - export const MobileInkBoxTrigger = new Message("Trigger Mobile Ink Box"); + export const MobileInkOverlayTrigger = new Message("Trigger Mobile Ink Overlay"); + export const UpdateMobileInkOverlayPosition = new Message("Update Mobile Ink Overlay Position"); export const GetRefField = new Message("Get Ref Field"); export const GetRefFields = new Message("Get Ref Fields"); diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 16e34bdfc..fe253400c 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -1,5 +1,5 @@ import { Utils } from "../../Utils"; -import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes, GestureContent, MobileInkBoxContent } from "../Message"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent } from "../Message"; import { Client } from "../Client"; import { Socket } from "socket.io"; import { Database } from "../database"; @@ -55,7 +55,7 @@ export namespace WebSocket { Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); Utils.AddServerHandler(socket, MessageStore.GesturePoints, content => processGesturePoints(socket, content)); - Utils.AddServerHandler(socket, MessageStore.MobileInkBoxTrigger, content => processBoxTrigger(socket, content)); + Utils.AddServerHandler(socket, MessageStore.MobileInkOverlayTrigger, content => processOverlayTrigger(socket, content)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); @@ -74,8 +74,8 @@ export namespace WebSocket { socket.broadcast.emit("receiveGesturePoints", content); } - function processBoxTrigger(socket: Socket, content: MobileInkBoxContent) { - socket.broadcast.emit("receiveBoxTrigger", content); + function processOverlayTrigger(socket: Socket, content: MobileInkOverlayContent) { + socket.broadcast.emit("receiveOverlayTrigger", content); } function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { -- cgit v1.2.3-70-g09d2 From dab8f5893ffb8fab05a46695b9d1a690d1171bca Mon Sep 17 00:00:00 2001 From: Stanley Yip Date: Sun, 2 Feb 2020 19:18:23 -0500 Subject: some more stuff --- src/client/documents/Documents.ts | 1 + src/client/views/GestureOverlay.scss | 19 ++++ src/client/views/GestureOverlay.tsx | 111 ++++++++++++++++++--- src/client/views/Touchable.tsx | 2 - .../views/collections/CollectionLinearView.tsx | 3 +- .../authentication/models/current_user_utils.ts | 4 +- 6 files changed, 122 insertions(+), 18 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index fa5707288..64583ae55 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -137,6 +137,7 @@ export interface DocumentOptions { isExpanded?: boolean; // is linear view expanded textTransform?: string; // is linear view expanded letterSpacing?: string; // is linear view expanded + flexDirection?: "unset" | "row" | "column" | "row-reverse" | "column-reverse"; } class EmptyBox { diff --git a/src/client/views/GestureOverlay.scss b/src/client/views/GestureOverlay.scss index d980b0a91..60b53c528 100644 --- a/src/client/views/GestureOverlay.scss +++ b/src/client/views/GestureOverlay.scss @@ -13,6 +13,25 @@ height: 300px; } +.inkToTextDoc-cont { + position: absolute; + width: 300px; + height: 300px; + overflow: hidden; + + .inkToTextDoc-scroller { + overflow: visible; + + .collectionView { + overflow: visible; + + .collectionLinearView-outer { + overflow: visible; + } + } + } +} + .filter-cont { position: absolute; background-color: transparent; diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 284224fb2..0c2e9e1bc 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -2,19 +2,19 @@ import React = require("react"); import { Touchable } from "./Touchable"; import { observer } from "mobx-react"; import "./GestureOverlay.scss"; -import { computed, observable, action, runInAction, IReactionDisposer, reaction } from "mobx"; +import { computed, observable, action, runInAction, IReactionDisposer, reaction, flow } from "mobx"; import { GestureUtils } from "../../pen-gestures/GestureUtils"; import { InteractionUtils } from "../util/InteractionUtils"; import { InkingControl } from "./InkingControl"; import { InkTool, InkData } from "../../new_fields/InkField"; import { Doc } from "../../new_fields/Doc"; import { LinkManager } from "../util/LinkManager"; -import { DocUtils } from "../documents/Documents"; +import { DocUtils, Docs } from "../documents/Documents"; import { undoBatch } from "../util/UndoManager"; import { Scripting } from "../util/Scripting"; import { FieldValue, Cast, NumCast, BoolCast } from "../../new_fields/Types"; import { CurrentUserUtils } from "../../server/authentication/models/current_user_utils"; -import Palette from "./Palette"; +import HorizontalPalette from "./Palette"; import { Utils, emptyPath, emptyFunction, returnFalse, returnOne, returnEmptyString, returnTrue, numberRange } from "../../Utils"; import { DocumentView } from "./nodes/DocumentView"; import { Transform } from "../util/Transform"; @@ -22,6 +22,9 @@ import { DocumentContentsView } from "./nodes/DocumentContentsView"; import { CognitiveServices } from "../cognitive_services/CognitiveServices"; import { DocServer } from "../DocServer"; import htmlToImage from "html-to-image"; +import { ScriptField } from "../../new_fields/ScriptField"; +import { listSpec } from "../../new_fields/Schema"; +import { List } from "../../new_fields/List"; @observer export default class GestureOverlay extends Touchable { @@ -40,11 +43,13 @@ export default class GestureOverlay extends Touchable { @observable private _strokes: InkData[] = []; @observable private _palette?: JSX.Element; @observable private _clipboardDoc?: JSX.Element; + @observable private _possibilities: Doc[] = []; @computed private get height(): number { return Math.max(this._pointerY && this._thumbY ? this._thumbY - this._pointerY : 300, 300); } @computed private get showBounds() { return this.Tool !== ToolglassTools.None; } private _d1: Doc | undefined; + private _inkToTextDoc: Doc | undefined; private _thumbDoc: Doc | undefined; private thumbIdentifier?: number; private pointerIdentifier?: number; @@ -111,7 +116,6 @@ export default class GestureOverlay extends Touchable { ptsToDelete.forEach(pt => this.prevPoints.delete(pt)); const nts = this.getNewTouches(te); - console.log(nts.nt.length); if (nts.nt.length < 5) { const target = document.elementFromPoint(te.changedTouches.item(0).clientX, te.changedTouches.item(0).clientY); @@ -239,10 +243,11 @@ export default class GestureOverlay extends Touchable { const thumbDoc = await Cast(CurrentUserUtils.setupThumbDoc(CurrentUserUtils.UserDocument), Doc); if (thumbDoc) { runInAction(() => { + this._inkToTextDoc = FieldValue(Cast(thumbDoc.inkToTextDoc, Doc)); this._thumbDoc = thumbDoc; this._thumbX = thumb.clientX; this._thumbY = thumb.clientY; - this._palette = ; + this._palette = ; }); } @@ -280,10 +285,18 @@ export default class GestureOverlay extends Touchable { for (let i = 0; i < e.changedTouches.length; i++) { const pt = e.changedTouches.item(i); - if (pt && pt.identifier === this.thumbIdentifier && this._thumbX && this._thumbDoc) { - if (Math.abs(pt.clientX - this._thumbX) > 20) { - this._thumbDoc.selectedIndex = Math.max(0, NumCast(this._thumbDoc.selectedIndex) - Math.sign(pt.clientX - this._thumbX)); - this._thumbX = pt.clientX; + if (pt && pt.identifier === this.thumbIdentifier && this._thumbY) { + if (this._thumbX && this._thumbDoc) { + if (Math.abs(pt.clientX - this._thumbX) > 20) { + this._thumbDoc.selectedIndex = Math.max(0, NumCast(this._thumbDoc.selectedIndex) - Math.sign(pt.clientX - this._thumbX)); + this._thumbX = pt.clientX; + } + } + if (this._thumbY && this._inkToTextDoc) { + if (Math.abs(pt.clientY - this._thumbY) > 20) { + this._inkToTextDoc.selectedIndex = Math.max(0, NumCast(this._inkToTextDoc.selectedIndex) - Math.sign(pt.clientY - this._thumbY)); + this._thumbY = pt.clientY; + } } } if (pt && pt.identifier === this.pointerIdentifier) { @@ -304,6 +317,11 @@ export default class GestureOverlay extends Touchable { this.dispatchGesture(GestureUtils.Gestures.Stroke, s); }); this._strokes = []; + if (NumCast(this._inkToTextDoc?.selectedIndex) > 0) { + const selectedButton = this._possibilities[NumCast(this._inkToTextDoc?.selectedIndex) - 1]; + Cast(selectedButton?.proto?.onClick, ScriptField)?.script.run({ this: selectedButton.proto }, console.log); + } + this._possibilities = []; document.removeEventListener("touchend", this.handleHandUp); } } @@ -382,9 +400,27 @@ export default class GestureOverlay extends Touchable { this._points = []; const results = await CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes); const wordResults = results.filter((r: any) => r.category === "inkWord"); - const possibilities = [wordResults[0]?.recognizedText]; + const possibilities: string[] = []; + if (wordResults[0]?.recognizedText) { + possibilities.push(wordResults[0]?.recognizedText) + } possibilities.push(...wordResults[0]?.alternates?.map((a: any) => a.recognizedString)); - console.log(possibilities); + const r = Math.max(this.svgBounds.right, ...this._strokes.map(s => this.getBounds(s).right)); + const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left)); + const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top)); + runInAction(() => { + console.log(possibilities); + const buttons = possibilities.map(p => Docs.Create.ButtonDocument({ + _height: r - l, _width: 25, backgroundColor: "lightgrey", color: "rgb(34, 34, 34)", x: l, y: t, + title: p, fontSize: 10, letterSpacing: "0px", textTransform: "unset", boxShadow: ".5px .5px 0px rgb(34, 34, 34)", + onClick: ScriptField.MakeScript('Docs.Create.TextDocument(this.title, {_width: 200, _height: 35, x: this.x, y: this.y})') + })); + if (this._inkToTextDoc) { + this._inkToTextDoc.data = new List(buttons); + this._inkToTextDoc.selectedIndex = 0; + } + this._possibilities = buttons; + }); break; case ToolglassTools.IgnoreGesture: this.dispatchGesture(GestureUtils.Gestures.Stroke); @@ -458,8 +494,7 @@ export default class GestureOverlay extends Touchable { return ( [this._strokes.map(l => { - let b = this.getBounds(l); - console.log(b); + const b = this.getBounds(l); return {InteractionUtils.CreatePolyline(l, b.left, b.top, this.Color, this.Width)} ; @@ -512,6 +547,53 @@ export default class GestureOverlay extends Touchable { this._clipboardDoc = undefined; } + @computed + private get inkToTextSuggestions() { + console.log(this._possibilities.length); + if (this._inkToTextDoc && this._possibilities.length) { + const b = Math.max(this.svgBounds.bottom, ...this._strokes.map(s => this.getBounds(s).bottom)); + const r = Math.max(this.svgBounds.right, ...this._strokes.map(s => this.getBounds(s).right)); + const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left)); + const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top)); + + return ( +
+
+ new Transform(-l, -(b + NumCast(this._inkToTextDoc?.selectedIndex, 0) * 25), 1)} + ContentScaling={returnOne} + PanelWidth={() => 300} + PanelHeight={() => 300} + renderDepth={0} + backgroundColor={returnEmptyString} + focus={emptyFunction} + parentActive={returnTrue} + whenActiveChanged={emptyFunction} + bringToFront={emptyFunction} + ContainingCollectionView={undefined} + ContainingCollectionDoc={undefined} + zoomToScale={emptyFunction} + getScale={returnOne} + />; +
+
+ ); + } + return null; + } + render() { return (
@@ -535,7 +617,8 @@ export default class GestureOverlay extends Touchable { display: this.showBounds ? "unset" : "none", }}>
-
); + {this.inkToTextSuggestions} +
); } } diff --git a/src/client/views/Touchable.tsx b/src/client/views/Touchable.tsx index 7800c4019..ead6e2a00 100644 --- a/src/client/views/Touchable.tsx +++ b/src/client/views/Touchable.tsx @@ -92,7 +92,6 @@ export abstract class Touchable extends React.Component { if (!InteractionUtils.IsDragging(this.prevPoints, myTouches, 5) && !this._touchDrag) return; this._touchDrag = true; if (this.holdTimer) { - console.log("CLEAR"); clearTimeout(this.holdTimer); // this.holdTimer = undefined; } @@ -129,7 +128,6 @@ export abstract class Touchable extends React.Component { } if (this.holdTimer) { clearTimeout(this.holdTimer); - console.log("clear"); } this._touchDrag = false; te.stopPropagation(); diff --git a/src/client/views/collections/CollectionLinearView.tsx b/src/client/views/collections/CollectionLinearView.tsx index 67062ae41..e613bf411 100644 --- a/src/client/views/collections/CollectionLinearView.tsx +++ b/src/client/views/collections/CollectionLinearView.tsx @@ -82,13 +82,14 @@ export class CollectionLinearView extends CollectionSubView(LinearDocument) { render() { const guid = Utils.GenerateGuid(); + const flexDir: any = StrCast(this.Document.flexDirection); return
this.props.Document.isExpanded = this.addMenuToggle.current!.checked)} /> -
+
{this.childLayoutPairs.filter((pair) => this.isCurrent(pair.layout)).map((pair, ind) => { const nested = pair.layout._viewType === CollectionViewType.Linear; const dref = React.createRef(); diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index dbde351b3..6c37380f3 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -131,9 +131,11 @@ export class CurrentUserUtils { static setupThumbDoc(userDoc: Doc) { if (!userDoc.thumbDoc) { - userDoc.thumbDoc = Docs.Create.LinearDocument(CurrentUserUtils.setupThumbButtons(userDoc), { + const thumbDoc = Docs.Create.LinearDocument(CurrentUserUtils.setupThumbButtons(userDoc), { _width: 100, _height: 50, ignoreClick: true, lockedPosition: true, _chromeStatus: "disabled", title: "buttons", _autoHeight: true, _yMargin: 5, isExpanded: true, backgroundColor: "white" }); + thumbDoc.inkToTextDoc = Docs.Create.LinearDocument([], { _width: 300, _height: 25, _autoHeight: true, _chromeStatus: "disabled", isExpanded: true, flexDirection: "column" }); + userDoc.thumbDoc = thumbDoc; } return userDoc.thumbDoc; } -- cgit v1.2.3-70-g09d2 From 1007cfb325f2dcddc4365538e4b354d06eb85f2f Mon Sep 17 00:00:00 2001 From: Stanley Yip Date: Tue, 4 Feb 2020 18:12:47 -0500 Subject: ok so the toolglass is working, but it's super slow... i'll fix that later lol --- src/client/views/GestureOverlay.scss | 13 +- src/client/views/GestureOverlay.tsx | 151 ++++++++++++--------- .../collectionFreeForm/CollectionFreeFormView.tsx | 6 +- src/pen-gestures/GestureUtils.ts | 11 +- .../authentication/models/current_user_utils.ts | 2 +- 5 files changed, 109 insertions(+), 74 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/views/GestureOverlay.scss b/src/client/views/GestureOverlay.scss index 60b53c528..f425c438e 100644 --- a/src/client/views/GestureOverlay.scss +++ b/src/client/views/GestureOverlay.scss @@ -16,11 +16,13 @@ .inkToTextDoc-cont { position: absolute; width: 300px; - height: 300px; overflow: hidden; .inkToTextDoc-scroller { overflow: visible; + position: absolute; + width: 100%; + left: -24px; .collectionView { overflow: visible; @@ -30,6 +32,15 @@ } } } + + .shadow { + width: 100%; + height: calc(100% - 25px); + position: absolute; + top: 25px; + background-color: black; + opacity: 0.2; + } } .filter-cont { diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 0c2e9e1bc..0fd6f3dba 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -2,7 +2,7 @@ import React = require("react"); import { Touchable } from "./Touchable"; import { observer } from "mobx-react"; import "./GestureOverlay.scss"; -import { computed, observable, action, runInAction, IReactionDisposer, reaction, flow } from "mobx"; +import { computed, observable, action, runInAction, IReactionDisposer, reaction, flow, trace } from "mobx"; import { GestureUtils } from "../../pen-gestures/GestureUtils"; import { InteractionUtils } from "../util/InteractionUtils"; import { InkingControl } from "./InkingControl"; @@ -25,6 +25,8 @@ import htmlToImage from "html-to-image"; import { ScriptField } from "../../new_fields/ScriptField"; import { listSpec } from "../../new_fields/Schema"; import { List } from "../../new_fields/List"; +import { CollectionViewType } from "./collections/CollectionView"; +import InkCanvas from "./InkCanvas"; @observer export default class GestureOverlay extends Touchable { @@ -63,6 +65,11 @@ export default class GestureOverlay extends Touchable { GestureOverlay.Instance = this; } + componentDidMount = () => { + this._thumbDoc = FieldValue(Cast(CurrentUserUtils.setupThumbDoc(CurrentUserUtils.UserDocument), Doc)); + this._inkToTextDoc = FieldValue(Cast(this._thumbDoc?.inkToTextDoc, Doc)); + } + getNewTouches(e: React.TouchEvent | TouchEvent) { const ntt: (React.Touch | Touch)[] = Array.from(e.targetTouches); const nct: (React.Touch | Touch)[] = Array.from(e.changedTouches); @@ -313,14 +320,23 @@ export default class GestureOverlay extends Touchable { this._palette = undefined; this.thumbIdentifier = undefined; this._thumbDoc = undefined; - this._strokes.forEach(s => { - this.dispatchGesture(GestureUtils.Gestures.Stroke, s); - }); - this._strokes = []; - if (NumCast(this._inkToTextDoc?.selectedIndex) > 0) { - const selectedButton = this._possibilities[NumCast(this._inkToTextDoc?.selectedIndex) - 1]; - Cast(selectedButton?.proto?.onClick, ScriptField)?.script.run({ this: selectedButton.proto }, console.log); + + let scriptWorked = false; + if (NumCast(this._inkToTextDoc?.selectedIndex) > -1) { + const selectedButton = this._possibilities[NumCast(this._inkToTextDoc?.selectedIndex)]; + if (Cast(selectedButton?.proto?.onClick, ScriptField)?.script.run({ this: selectedButton }, console.log).success) { + scriptWorked = true; + console.log("success"); + }; + console.log(Cast(selectedButton?.proto?.onClick, ScriptField)?.script); + } + + if (!scriptWorked) { + this._strokes.forEach(s => { + this.dispatchGesture(GestureUtils.Gestures.Stroke, s); + }); } + this._strokes = []; this._possibilities = []; document.removeEventListener("touchend", this.handleHandUp); } @@ -409,14 +425,14 @@ export default class GestureOverlay extends Touchable { const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left)); const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top)); runInAction(() => { - console.log(possibilities); const buttons = possibilities.map(p => Docs.Create.ButtonDocument({ - _height: r - l, _width: 25, backgroundColor: "lightgrey", color: "rgb(34, 34, 34)", x: l, y: t, + _height: 25, _width: r - l, backgroundColor: "lightgrey", color: "rgb(34, 34, 34)", x: l, y: t, _viewType: CollectionViewType.Linear, + isExpanded: true, title: p, fontSize: 10, letterSpacing: "0px", textTransform: "unset", boxShadow: ".5px .5px 0px rgb(34, 34, 34)", - onClick: ScriptField.MakeScript('Docs.Create.TextDocument(this.title, {_width: 200, _height: 35, x: this.x, y: this.y})') + onClick: ScriptField.MakeScript('createText(this.proto.title, this.x, this.y)') })); - if (this._inkToTextDoc) { - this._inkToTextDoc.data = new List(buttons); + if (this._inkToTextDoc && this._thumbDoc) { + this._inkToTextDoc = this._thumbDoc.inkToTextDoc = Docs.Create.LinearDocument(buttons, { _width: 300, _height: 25, _autoHeight: true, _chromeStatus: "disabled", isExpanded: true, flexDirection: "column" }); this._inkToTextDoc.selectedIndex = 0; } this._possibilities = buttons; @@ -459,7 +475,7 @@ export default class GestureOverlay extends Touchable { document.removeEventListener("pointerup", this.onPointerUp); } - dispatchGesture = (gesture: GestureUtils.Gestures, stroke?: InkData) => { + dispatchGesture = (gesture: GestureUtils.Gestures, stroke?: InkData, data?: any) => { const target = document.elementFromPoint((stroke ?? this._points)[0].X, (stroke ?? this._points)[0].Y); target?.dispatchEvent( new CustomEvent("dashOnGesture", @@ -468,7 +484,8 @@ export default class GestureOverlay extends Touchable { detail: { points: stroke ?? this._points, gesture: gesture, - bounds: this.getBounds(stroke ?? this._points) + bounds: this.getBounds(stroke ?? this._points), + text: data } } ) @@ -489,27 +506,21 @@ export default class GestureOverlay extends Touchable { return this.getBounds(this._points); } - @computed get currentStrokes() { + @computed get elements() { const B = this.svgBounds; + return [ + this.props.children, + this._palette, - return ( [this._strokes.map(l => { const b = this.getBounds(l); return - {InteractionUtils.CreatePolyline(l, b.left, b.top, this.Color, this.Width)} + {InteractionUtils.CreatePolyline(l, b.left, b.top, GestureOverlay.Instance.Color, GestureOverlay.Instance.Width)} ; }), this._points.length <= 1 ? (null) : - {InteractionUtils.CreatePolyline(this._points, B.left, B.top, this.Color, this.Width)} + {InteractionUtils.CreatePolyline(this._points, B.left, B.top, GestureOverlay.Instance.Color, GestureOverlay.Instance.Width)} ] - ); - } - - @computed get elements() { - return [ - this.props.children, - this._palette, - this.currentStrokes ]; } @@ -548,53 +559,59 @@ export default class GestureOverlay extends Touchable { } @computed + private get suggestionContent() { + const b = Math.max(this.svgBounds.bottom, ...this._strokes.map(s => this.getBounds(s).bottom)); + const r = Math.max(this.svgBounds.right, ...this._strokes.map(s => this.getBounds(s).right)); + const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left)); + const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top)); + return ( +
+
+ new Transform(-l, -(b + NumCast(this._inkToTextDoc?.selectedIndex, 0) * 25), 1)} + ContentScaling={returnOne} + PanelWidth={() => 300} + PanelHeight={() => 300} + renderDepth={0} + backgroundColor={returnEmptyString} + focus={emptyFunction} + parentActive={returnTrue} + whenActiveChanged={emptyFunction} + bringToFront={emptyFunction} + ContainingCollectionView={undefined} + ContainingCollectionDoc={undefined} + zoomToScale={emptyFunction} + getScale={returnOne} + />; +
+
+
+
) + } + private get inkToTextSuggestions() { - console.log(this._possibilities.length); if (this._inkToTextDoc && this._possibilities.length) { - const b = Math.max(this.svgBounds.bottom, ...this._strokes.map(s => this.getBounds(s).bottom)); - const r = Math.max(this.svgBounds.right, ...this._strokes.map(s => this.getBounds(s).right)); - const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left)); - const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top)); - return ( -
-
- new Transform(-l, -(b + NumCast(this._inkToTextDoc?.selectedIndex, 0) * 25), 1)} - ContentScaling={returnOne} - PanelWidth={() => 300} - PanelHeight={() => 300} - renderDepth={0} - backgroundColor={returnEmptyString} - focus={emptyFunction} - parentActive={returnTrue} - whenActiveChanged={emptyFunction} - bringToFront={emptyFunction} - ContainingCollectionView={undefined} - ContainingCollectionDoc={undefined} - zoomToScale={emptyFunction} - getScale={returnOne} - />; -
-
+ this.suggestionContent ); } return null; } render() { + trace(); return (
{this.elements} @@ -647,4 +664,8 @@ Scripting.addGlobal(function resetPen() { GestureOverlay.Instance.Color = GestureOverlay.Instance.SavedColor ?? "rgb(244, 67, 54)"; GestureOverlay.Instance.Width = GestureOverlay.Instance.SavedWidth ?? 5; }); +}); +Scripting.addGlobal(function createText(text: any, x: any, y: any) { + console.log("creating"); + GestureOverlay.Instance.dispatchGesture("text", [{ X: x, Y: y }], text); }); \ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 41ef8c2a6..17139d9d9 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -412,7 +412,11 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { this.addDocument(Docs.Create.FreeformDocument(sel, { title: "nested collection", x: bounds.x, y: bounds.y, _width: bWidth, _height: bHeight, _panX: 0, _panY: 0 })); sel.forEach(d => this.props.removeDocument(d)); break; - + case GestureUtils.Gestures.Text: + if (ge.text) { + const B = this.getTransform().transformPoint(ge.points[0].X, ge.points[0].Y); + this.addDocument(Docs.Create.TextDocument(ge.text, { title: ge.text, x: B[0], y: B[1] })); + } } } diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts index 4b5ad6684..4e3493c1c 100644 --- a/src/pen-gestures/GestureUtils.ts +++ b/src/pen-gestures/GestureUtils.ts @@ -6,18 +6,16 @@ import { Doc, WidthSym, HeightSym } from "../new_fields/Doc"; import { NumCast } from "../new_fields/Types"; import { CollectionFreeFormView } from "../client/views/collections/collectionFreeForm/CollectionFreeFormView"; import { Rect } from "react-measure"; +import { Scripting } from "../client/util/Scripting"; export namespace GestureUtils { - namespace GestureDataTypes { - export type BoxData = Array; - } - export class GestureEvent { constructor( readonly gesture: Gestures, readonly points: PointData[], readonly bounds: Rect, - readonly callbackFn?: Function + readonly callbackFn?: Function, + readonly text?: any ) { } } @@ -38,7 +36,8 @@ export namespace GestureUtils { Box = "box", Line = "line", Stroke = "stroke", - Scribble = "scribble" + Scribble = "scribble", + Text = "text" } export const GestureRecognizer = new NDollarRecognizer(false); diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 6c37380f3..b0ea2f9ad 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -137,7 +137,7 @@ export class CurrentUserUtils { thumbDoc.inkToTextDoc = Docs.Create.LinearDocument([], { _width: 300, _height: 25, _autoHeight: true, _chromeStatus: "disabled", isExpanded: true, flexDirection: "column" }); userDoc.thumbDoc = thumbDoc; } - return userDoc.thumbDoc; + return Cast(userDoc.thumbDoc, Doc); } static setupMobileDoc(userDoc: Doc) { -- cgit v1.2.3-70-g09d2 From 8767ec49fbb927ccde96f9f89562109703535d4e Mon Sep 17 00:00:00 2001 From: Stanley Yip Date: Thu, 6 Feb 2020 12:40:57 -0500 Subject: asdjklf --- src/client/documents/Documents.ts | 1 + src/client/views/GestureOverlay.scss | 13 +-- src/client/views/GestureOverlay.tsx | 127 ++++++--------------- .../collectionFreeForm/CollectionFreeFormView.tsx | 1 + .../collectionFreeForm/MarqueeOptionsMenu.tsx | 8 ++ .../collections/collectionFreeForm/MarqueeView.tsx | 29 ++++- 6 files changed, 78 insertions(+), 101 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 64583ae55..1c13eb079 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -138,6 +138,7 @@ export interface DocumentOptions { textTransform?: string; // is linear view expanded letterSpacing?: string; // is linear view expanded flexDirection?: "unset" | "row" | "column" | "row-reverse" | "column-reverse"; + selectedIndex?: number; } class EmptyBox { diff --git a/src/client/views/GestureOverlay.scss b/src/client/views/GestureOverlay.scss index f425c438e..2fad87ee0 100644 --- a/src/client/views/GestureOverlay.scss +++ b/src/client/views/GestureOverlay.scss @@ -17,19 +17,18 @@ position: absolute; width: 300px; overflow: hidden; + pointer-events: none; .inkToTextDoc-scroller { overflow: visible; position: absolute; width: 100%; - left: -24px; - .collectionView { - overflow: visible; - - .collectionLinearView-outer { - overflow: visible; - } + .menuItem-cont { + width: 100%; + height: 20px; + padding: 2.5px; + border-bottom: .5px solid black; } } diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 0fd6f3dba..e8262af1b 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -26,7 +26,7 @@ import { ScriptField } from "../../new_fields/ScriptField"; import { listSpec } from "../../new_fields/Schema"; import { List } from "../../new_fields/List"; import { CollectionViewType } from "./collections/CollectionView"; -import InkCanvas from "./InkCanvas"; +import TouchScrollableMenu, { TouchScrollableMenuItem } from "./TouchScrollableMenu"; @observer export default class GestureOverlay extends Touchable { @@ -40,12 +40,15 @@ export default class GestureOverlay extends Touchable { @observable private _thumbX?: number; @observable private _thumbY?: number; + @observable private _selectedIndex: number = -1; + @observable private _menuX: number = -300; + @observable private _menuY: number = -300; @observable private _pointerY?: number; @observable private _points: { X: number, Y: number }[] = []; @observable private _strokes: InkData[] = []; @observable private _palette?: JSX.Element; @observable private _clipboardDoc?: JSX.Element; - @observable private _possibilities: Doc[] = []; + @observable private _possibilities: JSX.Element[] = []; @computed private get height(): number { return Math.max(this._pointerY && this._thumbY ? this._thumbY - this._pointerY : 300, 300); } @computed private get showBounds() { return this.Tool !== ToolglassTools.None; } @@ -254,6 +257,8 @@ export default class GestureOverlay extends Touchable { this._thumbDoc = thumbDoc; this._thumbX = thumb.clientX; this._thumbY = thumb.clientY; + this._menuX = thumb.clientX + 50; + this._menuY = thumb.clientY; this._palette = ; }); } @@ -294,15 +299,14 @@ export default class GestureOverlay extends Touchable { const pt = e.changedTouches.item(i); if (pt && pt.identifier === this.thumbIdentifier && this._thumbY) { if (this._thumbX && this._thumbDoc) { - if (Math.abs(pt.clientX - this._thumbX) > 20) { + if (Math.abs(pt.clientX - this._thumbX) > 30) { this._thumbDoc.selectedIndex = Math.max(0, NumCast(this._thumbDoc.selectedIndex) - Math.sign(pt.clientX - this._thumbX)); this._thumbX = pt.clientX; } } if (this._thumbY && this._inkToTextDoc) { if (Math.abs(pt.clientY - this._thumbY) > 20) { - this._inkToTextDoc.selectedIndex = Math.max(0, NumCast(this._inkToTextDoc.selectedIndex) - Math.sign(pt.clientY - this._thumbY)); - this._thumbY = pt.clientY; + this._selectedIndex = Math.max(0, -Math.floor((pt.clientY - this._thumbY) / 20)); } } } @@ -324,11 +328,10 @@ export default class GestureOverlay extends Touchable { let scriptWorked = false; if (NumCast(this._inkToTextDoc?.selectedIndex) > -1) { const selectedButton = this._possibilities[NumCast(this._inkToTextDoc?.selectedIndex)]; - if (Cast(selectedButton?.proto?.onClick, ScriptField)?.script.run({ this: selectedButton }, console.log).success) { + if (selectedButton) { + selectedButton.props.onClick(); scriptWorked = true; - console.log("success"); - }; - console.log(Cast(selectedButton?.proto?.onClick, ScriptField)?.script); + } } if (!scriptWorked) { @@ -337,6 +340,7 @@ export default class GestureOverlay extends Touchable { }); } this._strokes = []; + this._points = []; this._possibilities = []; document.removeEventListener("touchend", this.handleHandUp); } @@ -398,12 +402,12 @@ export default class GestureOverlay extends Touchable { } @action - onPointerUp = async (e: PointerEvent) => { + onPointerUp = (e: PointerEvent) => { if (this._points.length > 1) { const B = this.svgBounds; const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); - const initialPoint = this._points[0]; + const initialPoint = this._points[0.]; const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + this.height; const yInGlass = initialPoint.Y > (this._thumbY ?? Number.MAX_SAFE_INTEGER) - this.height && initialPoint.Y < (this._thumbY ?? Number.MAX_SAFE_INTEGER); @@ -414,28 +418,21 @@ export default class GestureOverlay extends Touchable { document.removeEventListener("pointerup", this.onPointerUp); this._strokes.push(new Array(...this._points)); this._points = []; - const results = await CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes); - const wordResults = results.filter((r: any) => r.category === "inkWord"); - const possibilities: string[] = []; - if (wordResults[0]?.recognizedText) { - possibilities.push(wordResults[0]?.recognizedText) - } - possibilities.push(...wordResults[0]?.alternates?.map((a: any) => a.recognizedString)); - const r = Math.max(this.svgBounds.right, ...this._strokes.map(s => this.getBounds(s).right)); - const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left)); - const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top)); - runInAction(() => { - const buttons = possibilities.map(p => Docs.Create.ButtonDocument({ - _height: 25, _width: r - l, backgroundColor: "lightgrey", color: "rgb(34, 34, 34)", x: l, y: t, _viewType: CollectionViewType.Linear, - isExpanded: true, - title: p, fontSize: 10, letterSpacing: "0px", textTransform: "unset", boxShadow: ".5px .5px 0px rgb(34, 34, 34)", - onClick: ScriptField.MakeScript('createText(this.proto.title, this.x, this.y)') - })); - if (this._inkToTextDoc && this._thumbDoc) { - this._inkToTextDoc = this._thumbDoc.inkToTextDoc = Docs.Create.LinearDocument(buttons, { _width: 300, _height: 25, _autoHeight: true, _chromeStatus: "disabled", isExpanded: true, flexDirection: "column" }); - this._inkToTextDoc.selectedIndex = 0; + CognitiveServices.Inking.Appliers.InterpretStrokes(this._strokes).then((results) => { + const wordResults = results.filter((r: any) => r.category === "inkWord" || r.category === "paragraph"); + const possibilities: string[] = []; + if (wordResults[0]?.recognizedText) { + possibilities.push(wordResults[0]?.recognizedText) } - this._possibilities = buttons; + possibilities.push(...wordResults[0]?.alternates?.map((a: any) => a.recognizedString)); + console.log(possibilities); + const r = Math.max(this.svgBounds.right, ...this._strokes.map(s => this.getBounds(s).right)); + const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left)); + const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top)); + runInAction(() => { + this._possibilities = possibilities.map(p => + GestureOverlay.Instance.dispatchGesture(GestureUtils.Gestures.Text, [{ X: l, Y: t }], p)} />); + }); }); break; case ToolglassTools.IgnoreGesture: @@ -511,10 +508,9 @@ export default class GestureOverlay extends Touchable { return [ this.props.children, this._palette, - [this._strokes.map(l => { const b = this.getBounds(l); - return + return {InteractionUtils.CreatePolyline(l, b.left, b.top, GestureOverlay.Instance.Color, GestureOverlay.Instance.Width)} ; }), @@ -558,58 +554,6 @@ export default class GestureOverlay extends Touchable { this._clipboardDoc = undefined; } - @computed - private get suggestionContent() { - const b = Math.max(this.svgBounds.bottom, ...this._strokes.map(s => this.getBounds(s).bottom)); - const r = Math.max(this.svgBounds.right, ...this._strokes.map(s => this.getBounds(s).right)); - const l = Math.min(this.svgBounds.left, ...this._strokes.map(s => this.getBounds(s).left)); - const t = Math.min(this.svgBounds.top, ...this._strokes.map(s => this.getBounds(s).top)); - return ( -
-
- new Transform(-l, -(b + NumCast(this._inkToTextDoc?.selectedIndex, 0) * 25), 1)} - ContentScaling={returnOne} - PanelWidth={() => 300} - PanelHeight={() => 300} - renderDepth={0} - backgroundColor={returnEmptyString} - focus={emptyFunction} - parentActive={returnTrue} - whenActiveChanged={emptyFunction} - bringToFront={emptyFunction} - ContainingCollectionView={undefined} - ContainingCollectionDoc={undefined} - zoomToScale={emptyFunction} - getScale={returnOne} - />; -
-
-
-
) - } - - private get inkToTextSuggestions() { - if (this._inkToTextDoc && this._possibilities.length) { - return ( - this.suggestionContent - ); - } - return null; - } - render() { trace(); return ( @@ -618,8 +562,8 @@ export default class GestureOverlay extends Touchable {
@@ -627,14 +571,14 @@ export default class GestureOverlay extends Touchable {
- {this.inkToTextSuggestions} +
); } } @@ -666,6 +610,5 @@ Scripting.addGlobal(function resetPen() { }); }); Scripting.addGlobal(function createText(text: any, x: any, y: any) { - console.log("creating"); GestureOverlay.Instance.dispatchGesture("text", [{ X: x, Y: y }], text); }); \ No newline at end of file diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 17139d9d9..2b160f2ff 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -984,6 +984,7 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { ; } + @computed get contentScaling() { if (this.props.annotationsKey) return 0; const hscale = this.nativeHeight ? this.props.PanelHeight() / this.nativeHeight : 1; diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index 71f265484..db4b674b5 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -11,6 +11,7 @@ export default class MarqueeOptionsMenu extends AntimodeMenu { public createCollection: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public delete: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public summarize: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; + public inkToText: (e: KeyboardEvent | React.PointerEvent | undefined) => void = unimplementedFunction; public showMarquee: () => void = unimplementedFunction; public hideMarquee: () => void = unimplementedFunction; @@ -43,6 +44,13 @@ export default class MarqueeOptionsMenu extends AntimodeMenu { onPointerDown={this.delete}> , + , ]; return this.getElement(buttons); } diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index ef2fc2ad1..112df8f05 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -1,12 +1,12 @@ import { action, computed, observable } from "mobx"; import { observer } from "mobx-react"; -import { Doc, DocListCast } from "../../../../new_fields/Doc"; +import { Doc, DocListCast, DataSym, WidthSym, HeightSym } from "../../../../new_fields/Doc"; import { InkField } from "../../../../new_fields/InkField"; import { List } from "../../../../new_fields/List"; import { listSpec } from "../../../../new_fields/Schema"; import { SchemaHeaderField } from "../../../../new_fields/SchemaHeaderField"; import { ComputedField } from "../../../../new_fields/ScriptField"; -import { Cast, NumCast, StrCast } from "../../../../new_fields/Types"; +import { Cast, NumCast, StrCast, FieldValue } from "../../../../new_fields/Types"; import { CurrentUserUtils } from "../../../../server/authentication/models/current_user_utils"; import { Utils } from "../../../../Utils"; import { Docs } from "../../../documents/Documents"; @@ -19,6 +19,7 @@ import "./MarqueeView.scss"; import React = require("react"); import MarqueeOptionsMenu from "./MarqueeOptionsMenu"; import { SubCollectionViewProps } from "../CollectionSubView"; +import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -204,6 +205,7 @@ export class MarqueeView extends React.Component { + const selected = this.marqueeSelect(false); + if (e instanceof KeyboardEvent ? e.key === "i" : true) { + const inks = selected.filter(s => s.proto?.type === "ink"); + const sets = selected.filter(s => s.proto?.type === "text") + const inkFields = inks.map(i => FieldValue(Cast(i.data, InkField))); + CognitiveServices.Inking.Appliers.InterpretStrokes(inkFields.filter(i => i instanceof InkField).map(i => i!.inkData)).then((results) => { + const wordResults = results.filter((r: any) => r.category === "inkWord"); + console.log(wordResults); + for (const word of wordResults) { + const indices: number[] = word.strokeIds; + const r = Math.floor(Math.random() * 256); + const g = Math.floor(Math.random() * 256); + const b = Math.floor(Math.random() * 256); + indices.forEach(i => { + inks[i].color = `rgb(${r}, ${g}, ${b})`; + }) + } + }); + } + } + @action summary = (e: KeyboardEvent | React.PointerEvent | undefined) => { const bounds = this.Bounds; -- cgit v1.2.3-70-g09d2 From 593016303351f365d6ae13413b72d77495ea6a03 Mon Sep 17 00:00:00 2001 From: Stanley Yip Date: Sat, 8 Feb 2020 14:35:20 -0500 Subject: syntax highlighting :) --- src/client/documents/Documents.ts | 1 + src/client/views/GestureOverlay.tsx | 22 +++++-- .../collectionFreeForm/CollectionFreeFormView.tsx | 74 +++++++++++++++++++++- .../collections/collectionFreeForm/MarqueeView.tsx | 24 +++++-- src/pen-gestures/GestureUtils.ts | 2 + src/pen-gestures/ndollar.ts | 11 +++- 6 files changed, 119 insertions(+), 15 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 1c13eb079..81a6ff802 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -139,6 +139,7 @@ export interface DocumentOptions { letterSpacing?: string; // is linear view expanded flexDirection?: "unset" | "row" | "column" | "row-reverse" | "column-reverse"; selectedIndex?: number; + syntaxColor?: string; // can be applied to text for syntax highlighting all matches in the text } class EmptyBox { diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index e8262af1b..0b77b4b86 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -50,7 +50,7 @@ export default class GestureOverlay extends Touchable { @observable private _clipboardDoc?: JSX.Element; @observable private _possibilities: JSX.Element[] = []; - @computed private get height(): number { return Math.max(this._pointerY && this._thumbY ? this._thumbY - this._pointerY : 300, 300); } + @computed private get height(): number { return 2 * Math.max(this._pointerY && this._thumbY ? this._thumbY - this._pointerY : 300, 300); } @computed private get showBounds() { return this.Tool !== ToolglassTools.None; } private _d1: Doc | undefined; @@ -408,8 +408,8 @@ export default class GestureOverlay extends Touchable { const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); const initialPoint = this._points[0.]; - const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + this.height; - const yInGlass = initialPoint.Y > (this._thumbY ?? Number.MAX_SAFE_INTEGER) - this.height && initialPoint.Y < (this._thumbY ?? Number.MAX_SAFE_INTEGER); + const xInGlass = initialPoint.X > (this._thumbX ?? Number.MAX_SAFE_INTEGER) && initialPoint.X < (this._thumbX ?? Number.MAX_SAFE_INTEGER) + (this.height); + const yInGlass = initialPoint.Y > (this._thumbY ?? Number.MAX_SAFE_INTEGER) - (this.height) && initialPoint.Y < (this._thumbY ?? Number.MAX_SAFE_INTEGER); if (this.Tool !== ToolglassTools.None && xInGlass && yInGlass) { switch (this.Tool) { @@ -450,6 +450,14 @@ export default class GestureOverlay extends Touchable { this.dispatchGesture(GestureUtils.Gestures.Box); actionPerformed = true; break; + case GestureUtils.Gestures.StartBracket: + this.dispatchGesture(GestureUtils.Gestures.StartBracket); + actionPerformed = true; + break; + case GestureUtils.Gestures.EndBracket: + this.dispatchGesture(GestureUtils.Gestures.EndBracket); + actionPerformed = true; + break; case GestureUtils.Gestures.Line: actionPerformed = this.handleLineGesture(); break; @@ -562,8 +570,8 @@ export default class GestureOverlay extends Touchable {
@@ -571,8 +579,8 @@ export default class GestureOverlay extends Touchable {
= new Map(); private _clusterDistance: number = 75; private _hitCluster = false; private _layoutComputeReaction: IReactionDisposer | undefined; @@ -411,11 +416,78 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { }); this.addDocument(Docs.Create.FreeformDocument(sel, { title: "nested collection", x: bounds.x, y: bounds.y, _width: bWidth, _height: bHeight, _panX: 0, _panY: 0 })); sel.forEach(d => this.props.removeDocument(d)); + e.stopPropagation(); + break; + case GestureUtils.Gestures.StartBracket: + const start = this.getTransform().transformPoint(Math.min(...ge.points.map(p => p.X)), Math.min(...ge.points.map(p => p.Y))); + this._inkToTextStartX = start[0]; + this._inkToTextStartY = start[1]; + console.log("start"); + break; + case GestureUtils.Gestures.EndBracket: + console.log("end"); + if (this._inkToTextStartX && this._inkToTextStartY) { + const end = this.getTransform().transformPoint(Math.max(...ge.points.map(p => p.X)), Math.max(...ge.points.map(p => p.Y))); + const setDocs = this.getActiveDocuments().filter(s => s.proto?.type === "text" && s.color); + const sets = setDocs.map((sd) => { + return Cast(sd.data, RichTextField)?.Text as string; + }); + if (sets.length && sets[0]) { + this._wordPalette.clear(); + const colors = setDocs.map(sd => FieldValue(sd.color) as string); + sets.forEach((st: string, i: number) => { + const words = st.split(","); + words.forEach(word => { + this._wordPalette.set(word, colors[i]); + }); + }); + } + const inks = this.getActiveDocuments().filter(doc => { + if (doc.type === "ink") { + const l = NumCast(doc.x); + const r = l + doc[WidthSym](); + const t = NumCast(doc.y); + const b = t + doc[HeightSym](); + const pass = !(this._inkToTextStartX! > r || end[0] < l || this._inkToTextStartY! > b || end[1] < t); + return pass; + } + return false; + }); + const inkFields = inks.map(i => Cast(i.data, InkField)); + CognitiveServices.Inking.Appliers.InterpretStrokes(inkFields.filter(i => i instanceof InkField).map(i => i!.inkData)).then((results) => { + const wordResults = results.filter((r: any) => r.category === "inkWord"); + console.log(wordResults); + for (const word of wordResults) { + const indices: number[] = word.strokeIds; + indices.forEach(i => { + const otherInks: Doc[] = []; + indices.forEach(i2 => i2 !== i && otherInks.push(inks[i2])); + inks[i].relatedInks = new List(otherInks); + const uniqueColors: string[] = []; + Array.from(this._wordPalette.values()).forEach(c => uniqueColors.indexOf(c) === -1 && uniqueColors.push(c)); + inks[i].alternativeColors = new List(uniqueColors); + if (this._wordPalette.has(word.recognizedText)) { + inks[i].color = this._wordPalette.get(word.recognizedText); + } + else { + for (const alt of word.alternates) { + if (this._wordPalette.has(alt.recognizedString)) { + inks[i].color = this._wordPalette.get(alt.recognizedString); + break; + } + } + } + }); + } + }); + this._inkToTextStartX = end[0]; + } break; case GestureUtils.Gestures.Text: if (ge.text) { const B = this.getTransform().transformPoint(ge.points[0].X, ge.points[0].Y); this.addDocument(Docs.Create.TextDocument(ge.text, { title: ge.text, x: B[0], y: B[1] })); + e.stopPropagation(); } } } diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index 112df8f05..19a71012a 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -20,6 +20,7 @@ import React = require("react"); import MarqueeOptionsMenu from "./MarqueeOptionsMenu"; import { SubCollectionViewProps } from "../CollectionSubView"; import { CognitiveServices } from "../../../cognitive_services/CognitiveServices"; +import { RichTextField } from "../../../../new_fields/RichTextField"; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -368,18 +369,29 @@ export class MarqueeView extends React.Component s.proto?.type === "ink"); - const sets = selected.filter(s => s.proto?.type === "text") - const inkFields = inks.map(i => FieldValue(Cast(i.data, InkField))); + const setDocs = selected.filter(s => s.proto?.type === "text" && s.color); + const sets = setDocs.map((sd) => { + return Cast(sd.data, RichTextField)?.Text as string; + }); + const colors = setDocs.map(sd => FieldValue(sd.color) as string); + const wordToColor = new Map(); + console.log(sets); + sets.forEach((st: string, i: number) => { + const words = st.split(","); + words.forEach(word => { + wordToColor.set(word, colors[i]); + }); + }); + const inkFields = inks.map(i => Cast(i.data, InkField)); CognitiveServices.Inking.Appliers.InterpretStrokes(inkFields.filter(i => i instanceof InkField).map(i => i!.inkData)).then((results) => { const wordResults = results.filter((r: any) => r.category === "inkWord"); console.log(wordResults); for (const word of wordResults) { const indices: number[] = word.strokeIds; - const r = Math.floor(Math.random() * 256); - const g = Math.floor(Math.random() * 256); - const b = Math.floor(Math.random() * 256); indices.forEach(i => { - inks[i].color = `rgb(${r}, ${g}, ${b})`; + if (wordToColor.has(word.recognizedText)) { + inks[i].color = wordToColor.get(word.recognizedText); + } }) } }); diff --git a/src/pen-gestures/GestureUtils.ts b/src/pen-gestures/GestureUtils.ts index 4e3493c1c..f14c573c3 100644 --- a/src/pen-gestures/GestureUtils.ts +++ b/src/pen-gestures/GestureUtils.ts @@ -35,6 +35,8 @@ export namespace GestureUtils { export enum Gestures { Box = "box", Line = "line", + StartBracket = "startbracket", + EndBracket = "endbracket", Stroke = "stroke", Scribble = "scribble", Text = "text" diff --git a/src/pen-gestures/ndollar.ts b/src/pen-gestures/ndollar.ts index 9e15ada2d..643d58591 100644 --- a/src/pen-gestures/ndollar.ts +++ b/src/pen-gestures/ndollar.ts @@ -142,7 +142,7 @@ export class Result { // // NDollarRecognizer constants // -const NumMultistrokes = 2; +const NumMultistrokes = 4; const NumPoints = 96; const SquareSize = 250.0; const OneDThreshold = 0.25; // customize to desired gesture set (usually 0.20 - 0.35) @@ -178,6 +178,15 @@ export class NDollarRecognizer { this.Multistrokes[1] = new Multistroke(GestureUtils.Gestures.Line, useBoundedRotationInvariance, new Array( new Array(new Point(12, 347), new Point(119, 347)) )); + this.Multistrokes[2] = new Multistroke(GestureUtils.Gestures.StartBracket, useBoundedRotationInvariance, new Array( + // new Array(new Point(145, 20), new Point(30, 21), new Point(34, 150)) + new Array(new Point(31, 25), new Point(145, 20), new Point(31, 25), new Point(34, 150)) + )); + this.Multistrokes[3] = new Multistroke(GestureUtils.Gestures.EndBracket, useBoundedRotationInvariance, new Array( + // new Array(new Point(150, 21), new Point(149, 150), new Point(26, 152)) + // new Array(new Point(150, 150), new Point(150, 0), new Point(150, 150), new Point(0, 150)) + new Array(new Point(10, 100), new Point(50, 12), new Point(100, 103)) + )); // // PREDEFINED STROKES -- cgit v1.2.3-70-g09d2 From 05c429412d36531bed4e8ece889479fcf93faea6 Mon Sep 17 00:00:00 2001 From: kimdahey Date: Sun, 9 Feb 2020 16:43:01 -0500 Subject: functioning --- package-lock.json | 4 +- package.json | 2 +- .../views/collections/CollectionDockingView.tsx | 69 ++++++++++++++++++++++ .../collectionFreeForm/CollectionFreeFormView.tsx | 40 ++++++++++++- 4 files changed, 110 insertions(+), 5 deletions(-) (limited to 'src/client/views/collections') diff --git a/package-lock.json b/package-lock.json index 833710fe7..049c91fcf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16239,7 +16239,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "requires": { "safe-buffer": "~5.1.0" @@ -18560,7 +18560,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "requires": { "string-width": "^1.0.1", diff --git a/package.json b/package.json index 67b9b1630..7e0adfba6 100644 --- a/package.json +++ b/package.json @@ -253,4 +253,4 @@ "xoauth2": "^1.2.0", "youtube": "^0.1.0" } -} \ No newline at end of file +} diff --git a/src/client/views/collections/CollectionDockingView.tsx b/src/client/views/collections/CollectionDockingView.tsx index 82cb3bc88..7a6d54ac2 100644 --- a/src/client/views/collections/CollectionDockingView.tsx +++ b/src/client/views/collections/CollectionDockingView.tsx @@ -249,6 +249,75 @@ export class CollectionDockingView extends React.Component(); + private _pullCoords: number[] = [0, 0]; public get displayName() { return "CollectionFreeFormView(" + this.props.Document.title?.toString() + ")"; } // this makes mobx trace() statements more descriptive @observable.shallow _layoutElements: ViewDefResult[] = []; // shallow because some layout items (eg pivot labels) are just generated 'divs' and can't be frozen as observables @@ -596,7 +598,6 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { const myTouches = InteractionUtils.GetMyTargetTouches(me, this.prevPoints, true); const pt1 = myTouches[0]; const pt2 = myTouches[1]; - console.log(myTouches); if (this.prevPoints.size === 2) { const oldPoint1 = this.prevPoints.get(pt1.identifier); @@ -625,7 +626,11 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { // use the centerx and centery as the "new mouse position" const centerX = Math.min(pt1.clientX, pt2.clientX) + Math.abs(pt2.clientX - pt1.clientX) / 2; const centerY = Math.min(pt1.clientY, pt2.clientY) + Math.abs(pt2.clientY - pt1.clientY) / 2; - this.pan({ clientX: centerX, clientY: centerY }); + + if (!this._pullCoords[0] && !this._pullCoords[1]) { // if we are not bezel movement + this.pan({ clientX: centerX, clientY: centerY }); + } + this._lastX = centerX; this._lastY = centerY; } @@ -650,6 +655,12 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { const centerY = Math.min(pt1.clientY, pt2.clientY) + Math.abs(pt2.clientY - pt1.clientY) / 2; this._lastX = centerX; this._lastY = centerY; + + // determine if we are using a bezel movement + if ((this.props.PanelWidth() - this._lastX) < 100 || this._lastX < 100 || (this.props.PanelHeight() - this._lastY < 100) || this._lastY < 120) { // to account for header + this._pullCoords = [this._lastX, this._lastY]; + } + this.removeMoveListeners(); this.addMoveListeners(); this.removeEndListeners(); @@ -660,12 +671,37 @@ export class CollectionFreeFormView extends CollectionSubView(PanZoomDocument) { } cleanUpInteractions = () => { + + if (this._pullCoords[0] !== 0 && this._pullCoords[1] !== 0) { // if bezel mvmt was activated + const xDiff = this._pullCoords[0] - this._lastX; + const yDiff = this._pullCoords[1] - this._lastY; + + console.log('went thru', this._pullCoords); + if ((this._lastX < this._pullCoords[0]) && (yDiff < xDiff)) { // pull from right + console.log('pulled from right'); + // CollectionDockingView.AddRightSplit(this.Document, undefined); + CollectionDockingView.AddSplit(this.Document, "right", undefined); + } else if ((this._lastY > this._pullCoords[1]) && (yDiff < xDiff)) { // pull from top + console.log('pulled from top'); + CollectionDockingView.AddSplit(this.Document, "top", undefined); + } else if ((this._lastY < this._pullCoords[1]) && (yDiff > xDiff)) { // pull from bottom + console.log('pulled from bottom'); + CollectionDockingView.AddSplit(this.Document, "bottom", undefined); + } else if ((this._lastX > this._pullCoords[0]) && (yDiff > xDiff)) { // pull from left + console.log('pulled from left'); + CollectionDockingView.AddSplit(this.Document, "left", undefined); + } + } + + this._pullCoords = [0, 0]; + document.removeEventListener("pointermove", this.onPointerMove); document.removeEventListener("pointerup", this.onPointerUp); this.removeMoveListeners(); this.removeEndListeners(); } + @action zoom = (pointX: number, pointY: number, deltaY: number): void => { let deltaScale = deltaY > 0 ? (1 / 1.1) : 1.1; -- cgit v1.2.3-70-g09d2 From e8fcbbf57b2a2f443d9c280ce10558cf9d51c632 Mon Sep 17 00:00:00 2001 From: vellichora Date: Sun, 9 Feb 2020 17:11:24 -0500 Subject: can upload collection from mobile to desktop --- src/client/DocServer.ts | 9 +- src/client/util/DragManager.ts | 2 +- .../collections/CollectionMasonryViewFieldRow.tsx | 1 + .../views/collections/CollectionStackingView.tsx | 2 + .../CollectionStackingViewFieldColumn.tsx | 1 + src/client/views/collections/CollectionSubView.tsx | 2 +- src/client/views/collections/CollectionView.tsx | 2 +- .../views/collections/CollectionViewChromes.tsx | 1 + src/mobile/ImageUpload.tsx | 1 + src/mobile/MobileInkOverlay.tsx | 53 +++++++- src/mobile/MobileInterface.scss | 8 ++ src/mobile/MobileInterface.tsx | 150 ++++++++++++--------- src/server/ApiManagers/UploadManager.ts | 1 + src/server/Message.ts | 6 + src/server/Websocket/Websocket.ts | 7 +- .../authentication/models/current_user_utils.ts | 21 +-- src/server/server_Initialization.ts | 24 ++-- 17 files changed, 195 insertions(+), 96 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/DocServer.ts b/src/client/DocServer.ts index c03764471..e09251855 100644 --- a/src/client/DocServer.ts +++ b/src/client/DocServer.ts @@ -1,5 +1,5 @@ import * as OpenSocket from 'socket.io-client'; -import { MessageStore, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent } from "./../server/Message"; +import { MessageStore, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent, MobileDocumentUploadContent } from "./../server/Message"; import { Opt, Doc } from '../new_fields/Doc'; import { Utils, emptyFunction } from '../Utils'; import { SerializationHelper } from './util/SerializationHelper'; @@ -81,6 +81,10 @@ export namespace DocServer { Utils.Emit(_socket, MessageStore.UpdateMobileInkOverlayPosition, content); } + export function dispatchMobileDocumentUpload(content: MobileDocumentUploadContent) { + Utils.Emit(_socket, MessageStore.MobileDocumentUpload, content); + } + } export function init(protocol: string, hostname: string, port: number, identifier: string) { @@ -116,6 +120,9 @@ export namespace DocServer { _socket.addEventListener("receiveUpdateOverlayPosition", (content: UpdateMobileInkOverlayPositionContent) => { MobileInkOverlay.Instance.updatePosition(content); }); + _socket.addEventListener("receiveMobileDocumentUpload", (content: MobileDocumentUploadContent) => { + MobileInkOverlay.Instance.uploadDocument(content); + }); } function errorFunc(): never { diff --git a/src/client/util/DragManager.ts b/src/client/util/DragManager.ts index df2f5fe3c..0bb8b531d 100644 --- a/src/client/util/DragManager.ts +++ b/src/client/util/DragManager.ts @@ -179,7 +179,7 @@ export namespace DragManager { ); } element.dataset.canDrop = "true"; - const handler = (e: Event) => dropFunc(e, (e as CustomEvent).detail); + const handler = (e: Event) => { console.log("drop target reveied docs"); dropFunc(e, (e as CustomEvent).detail); }; element.addEventListener("dashOnDrop", handler); return () => { element.removeEventListener("dashOnDrop", handler); diff --git a/src/client/views/collections/CollectionMasonryViewFieldRow.tsx b/src/client/views/collections/CollectionMasonryViewFieldRow.tsx index 80752303c..26e0cc35a 100644 --- a/src/client/views/collections/CollectionMasonryViewFieldRow.tsx +++ b/src/client/views/collections/CollectionMasonryViewFieldRow.tsx @@ -73,6 +73,7 @@ export class CollectionMasonryViewFieldRow extends React.Component { + console.log("masronry row drop"); this._createAliasSelected = false; if (de.complete.docDragData) { (this.props.parent.Document.dropConverter instanceof ScriptField) && diff --git a/src/client/views/collections/CollectionStackingView.tsx b/src/client/views/collections/CollectionStackingView.tsx index 992820fc7..83c90810e 100644 --- a/src/client/views/collections/CollectionStackingView.tsx +++ b/src/client/views/collections/CollectionStackingView.tsx @@ -240,6 +240,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { @undoBatch @action drop = (e: Event, de: DragManager.DropEvent) => { + console.log("DROP STACKIN G2"); const where = [de.x, de.y]; let targInd = -1; let plusOne = 0; @@ -270,6 +271,7 @@ export class CollectionStackingView extends CollectionSubView(doc => doc) { @undoBatch @action onDrop = async (e: React.DragEvent): Promise => { + console.log("DROP STACKING"); const where = [e.clientX, e.clientY]; let targInd = -1; this._docXfs.map((cd, i) => { diff --git a/src/client/views/collections/CollectionStackingViewFieldColumn.tsx b/src/client/views/collections/CollectionStackingViewFieldColumn.tsx index 39b4e4e1d..65c4b3195 100644 --- a/src/client/views/collections/CollectionStackingViewFieldColumn.tsx +++ b/src/client/views/collections/CollectionStackingViewFieldColumn.tsx @@ -57,6 +57,7 @@ export class CollectionStackingViewFieldColumn extends React.Component { + console.log("column drop stacking"); this._createAliasSelected = false; if (de.complete.docDragData) { const key = StrCast(this.props.parent.props.Document.sectionFilter); diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index e94f24f2c..b35af2314 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -6,7 +6,7 @@ import { Id } from "../../../new_fields/FieldSymbols"; import { List } from "../../../new_fields/List"; import { listSpec } from "../../../new_fields/Schema"; import { ScriptField } from "../../../new_fields/ScriptField"; -import { Cast } from "../../../new_fields/Types"; +import { Cast, StrCast } from "../../../new_fields/Types"; import { CurrentUserUtils } from "../../../server/authentication/models/current_user_utils"; import { Utils } from "../../../Utils"; import { DocServer } from "../../DocServer"; diff --git a/src/client/views/collections/CollectionView.tsx b/src/client/views/collections/CollectionView.tsx index 88023783b..1d399e26f 100644 --- a/src/client/views/collections/CollectionView.tsx +++ b/src/client/views/collections/CollectionView.tsx @@ -138,7 +138,7 @@ export class CollectionView extends Touchable { let index = value.reduce((p, v, i) => (v instanceof Doc && v === doc) ? i : p, -1); index = index !== -1 ? index : value.reduce((p, v, i) => (v instanceof Doc && Doc.AreProtosEqual(v, doc)) ? i : p, -1); - ContextMenu.Instance.clearItems(); + ContextMenu.Instance && ContextMenu.Instance.clearItems(); if (index !== -1) { value.splice(index, 1); return true; diff --git a/src/client/views/collections/CollectionViewChromes.tsx b/src/client/views/collections/CollectionViewChromes.tsx index a870b6043..01dc21f95 100644 --- a/src/client/views/collections/CollectionViewChromes.tsx +++ b/src/client/views/collections/CollectionViewChromes.tsx @@ -231,6 +231,7 @@ export class CollectionViewBaseChrome extends React.Component { + console.log("toggle collapse"); this.props.CollectionView.props.Document.chromeStatus = this.props.CollectionView.props.Document.chromeStatus === "enabled" ? "collapsed" : "enabled"; if (this.props.collapse) { this.props.collapse(this.props.CollectionView.props.Document.chromeStatus !== "enabled"); diff --git a/src/mobile/ImageUpload.tsx b/src/mobile/ImageUpload.tsx index 3304e8e22..10bd78075 100644 --- a/src/mobile/ImageUpload.tsx +++ b/src/mobile/ImageUpload.tsx @@ -47,6 +47,7 @@ class Uploader extends React.Component { const upload = window.location.origin + "/upload"; this.status = "uploading image"; + console.log("uploading image", formData); const res = await fetch(upload, { method: 'POST', body: formData diff --git a/src/mobile/MobileInkOverlay.tsx b/src/mobile/MobileInkOverlay.tsx index 6b65aa436..ed4cca5b9 100644 --- a/src/mobile/MobileInkOverlay.tsx +++ b/src/mobile/MobileInkOverlay.tsx @@ -1,9 +1,13 @@ import React = require('react'); import { observer } from "mobx-react"; -import { MobileInkOverlayContent, GestureContent, UpdateMobileInkOverlayPositionContent } from "../server/Message"; +import { MobileInkOverlayContent, GestureContent, UpdateMobileInkOverlayPositionContent, MobileDocumentUploadContent } from "../server/Message"; import { observable, action } from "mobx"; import { GestureUtils } from "../pen-gestures/GestureUtils"; import "./MobileInkOverlay.scss"; +import { StrCast } from '../new_fields/Types'; +import { DragManager } from "../client/util/DragManager"; +import { DocServer } from '../client/DocServer'; +import { Doc } from '../new_fields/Doc'; @observer @@ -67,7 +71,7 @@ export default class MobileInkOverlay extends React.Component { height: bounds.height * this._scale, }; - const target = document.elementFromPoint(points[0].X, points[0].Y); + const target = document.elementFromPoint(this._x + 10, this._y + 10); target?.dispatchEvent( new CustomEvent("dashOnGesture", { @@ -82,6 +86,43 @@ export default class MobileInkOverlay extends React.Component { ); } + uploadDocument = async (content: MobileDocumentUploadContent) => { + const { docId } = content; + console.log("receive upload document id", docId); + const doc = await DocServer.GetRefField(docId); + + if (doc && doc instanceof Doc) { + console.log("parsed upload document into doc", StrCast(doc.proto!.title)); + + const target = document.elementFromPoint(this._x + 10, this._y + 10); + console.log("the target is", target); + + const dragData = new DragManager.DocumentDragData([doc]); + const complete = new DragManager.DragCompleteEvent(false, dragData); + console.log("the drag data is", dragData); + + if (target) { + target.dispatchEvent( + new CustomEvent("dashOnDrop", + { + bubbles: true, + detail: { + x: this._x, + y: this._y, + complete: complete, + altKey: false, + metaKey: false, + ctrlKey: false + } + } + ) + ); + } else { + alert("TARGET IS UNDEFINED"); + } + } + } + @action dragStart = (e: React.PointerEvent) => { document.removeEventListener("pointermove", this.dragging); @@ -132,15 +173,17 @@ export default class MobileInkOverlay extends React.Component { transform: `translate(${this._x}px, ${this._y}px)`, zIndex: 30000, pointerEvents: "none", - borderStyle: this._isDragging ? "solid" : "dashed" - }} + borderStyle: this._isDragging ? "solid" : "dashed", + backgroundColor: "rgba(255, 0, 0, 0.3)" + } + } ref={this._mainCont} >
-
+
); } } \ No newline at end of file diff --git a/src/mobile/MobileInterface.scss b/src/mobile/MobileInterface.scss index 8abe5a40d..8b0ebcd53 100644 --- a/src/mobile/MobileInterface.scss +++ b/src/mobile/MobileInterface.scss @@ -1,7 +1,15 @@ .mobileInterface-inkInterfaceButtons { position: absolute; + top: -50px; display: flex; justify-content: space-between; width: 100%; z-index: 9999; + height: 50px; +} + +.mobileInterface-container { + height: calc(100% - 50px); + margin-top: 50px; + position: relative; } \ No newline at end of file diff --git a/src/mobile/MobileInterface.tsx b/src/mobile/MobileInterface.tsx index 665d9a168..a1ef0a5d1 100644 --- a/src/mobile/MobileInterface.tsx +++ b/src/mobile/MobileInterface.tsx @@ -22,6 +22,18 @@ import { SelectionManager } from '../client/util/SelectionManager'; import { DateField } from '../new_fields/DateField'; import { GestureUtils } from '../pen-gestures/GestureUtils'; import { DocServer } from '../client/DocServer'; +import { DocumentDecorations } from '../client/views/DocumentDecorations'; +import { OverlayView } from '../client/views/OverlayView'; +import { DictationOverlay } from '../client/views/DictationOverlay'; +import SharingManager from '../client/util/SharingManager'; +import { PreviewCursor } from '../client/views/PreviewCursor'; +import { ContextMenu } from '../client/views/ContextMenu'; +import { RadialMenu } from '../client/views/nodes/RadialMenu'; +import PDFMenu from '../client/views/pdf/PDFMenu'; +import MarqueeOptionsMenu from '../client/views/collections/collectionFreeForm/MarqueeOptionsMenu'; +import GoogleAuthenticationManager from '../client/apis/GoogleAuthenticationManager'; +import { listSpec } from '../new_fields/Schema'; +import { Id } from '../new_fields/FieldSymbols'; library.add(faLongArrowAltLeft); @@ -31,7 +43,7 @@ export default class MobileInterface extends React.Component { @computed private get userDoc() { return CurrentUserUtils.UserDocument; } @computed private get mainContainer() { return this.userDoc ? FieldValue(Cast(this.userDoc.activeMobile, Doc)) : CurrentUserUtils.GuestMobile; } // @observable private currentView: "main" | "ink" | "upload" = "main"; - private mainDoc: Doc = CurrentUserUtils.setupMobileDoc(this.userDoc); + private mainDoc: any = CurrentUserUtils.setupMobileDoc(this.userDoc); @observable private renderView?: () => JSX.Element; // private inkDoc?: Doc; @@ -49,7 +61,6 @@ export default class MobileInterface extends React.Component { library.add(...[faPenNib, faHighlighter, faEraser, faMousePointer]); if (this.userDoc && !this.mainContainer) { - // const doc = CurrentUserUtils.setupMobileDoc(this.userDoc); this.userDoc.activeMobile = this.mainDoc; } } @@ -76,48 +87,22 @@ export default class MobileInterface extends React.Component { }); } - // @action - // switchCurrentView = (view: "main" | "ink" | "upload") => { - // this.currentView = view; - - // if (this.userDoc) { - // switch (view) { - // case "main": { - // // const doc = CurrentUserUtils.setupMobileDoc(this.userDoc); - // this.userDoc.activeMobile = this.mainDoc; - // break; - // } - // case "ink": { - // this.inkDoc = CurrentUserUtils.setupMobileInkingDoc(this.userDoc); - // this.userDoc.activeMobile = this.inkDoc; - // InkingControl.Instance.switchTool(InkTool.Pen); - // this.drawingInk = true; - - // DocServer.Mobile.dispatchOverlayTrigger({ - // enableOverlay: true, - // width: window.innerWidth, - // height: window.innerHeight - // }); - - // break; - // } - // case "upload": { - // this.uploadDoc = CurrentUserUtils.setupMobileUploadDoc(this.userDoc); - // this.userDoc.activeMobile = this.uploadDoc; - - // } - // } - // } - // } + onSwitchUpload = () => { + DocServer.Mobile.dispatchOverlayTrigger({ + enableOverlay: true, + width: 100, + height: 100 + }); + } renderDefaultContent = () => { - console.log("rendering default content"); + console.log("rendering default content", this.mainContainer); if (this.mainContainer) { return { console.log("want to add doc to default content", StrCast(doc.title)); return false; }} addDocTab={returnFalse} pinToPres={emptyFunction} removeDocument={undefined} @@ -192,42 +177,54 @@ export default class MobileInterface extends React.Component {
- - window.innerHeight} - PanelWidth={() => window.innerWidth} - focus={emptyFunction} - isSelected={returnFalse} - select={emptyFunction} - active={returnFalse} - ContentScaling={returnOne} - whenActiveChanged={returnFalse} - ScreenToLocalTransform={Transform.Identity} - ruleProvider={undefined} - renderDepth={0} - ContainingCollectionView={undefined} - ContainingCollectionDoc={undefined}> - - + window.innerHeight} + PanelWidth={() => window.innerWidth} + focus={emptyFunction} + isSelected={returnFalse} + select={emptyFunction} + active={returnFalse} + ContentScaling={returnOne} + whenActiveChanged={returnFalse} + ScreenToLocalTransform={Transform.Identity} + ruleProvider={undefined} + renderDepth={0} + ContainingCollectionView={undefined} + ContainingCollectionDoc={undefined}> +
); } } - upload = () => { + upload = async (e: React.MouseEvent) => { + if (this.mainContainer) { + const data = Cast(this.mainContainer.data, listSpec(Doc)); + if (data) { + const uploadDoc = await data[1]; // TODO: ensure this is the collection to upload + console.log("UPLOADING DOCUMENT FROM MOBILE", uploadDoc[Id], StrCast(uploadDoc.proto!.title)); + if (uploadDoc) { + DocServer.Mobile.dispatchMobileDocumentUpload({ + docId: uploadDoc[Id] + }); + } + } + } + e.stopPropagation(); + e.preventDefault(); } renderUploadContent() { if (this.mainContainer) { return ( -
+
@@ -240,7 +237,7 @@ export default class MobileInterface extends React.Component { Document={this.mainContainer} DataDoc={undefined} LibraryPath={emptyPath} - addDocument={returnFalse} + addDocument={(doc: Doc) => { console.log("want to add doc", StrCast(doc.title)); return false; }} addDocTab={returnFalse} pinToPres={emptyFunction} removeDocument={undefined} @@ -266,13 +263,35 @@ export default class MobileInterface extends React.Component { } } + onDragOver = (e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + } + render() { // const content = this.currentView === "main" ? this.mainContent : // this.currentView === "ink" ? this.inkContent : // this.currentView === "upload" ? this.uploadContent : <>; return ( -
- {this.renderView ? this.renderView() : this.renderDefaultContent()} +
+ {/* + + {this.renderView ? this.renderView() : this.renderDefaultContent()} + */} + + {/* + + */} + + + {this.renderView ? this.renderView() : this.renderDefaultContent()} + + + {/* */} + + {/* + + */}
); } @@ -281,5 +300,6 @@ export default class MobileInterface extends React.Component { Scripting.addGlobal(function switchMobileView(doc: (userDoc: Doc) => Doc, renderView?: () => JSX.Element, onSwitch?: () => void) { return MobileInterface.Instance.switchCurrentView(doc, renderView, onSwitch); }); Scripting.addGlobal(function onSwitchMobileInking() { return MobileInterface.Instance.onSwitchInking(); }); Scripting.addGlobal(function renderMobileInking() { return MobileInterface.Instance.renderInkingContent(); }); +Scripting.addGlobal(function onSwitchMobileUpload() { return MobileInterface.Instance.onSwitchUpload(); }); Scripting.addGlobal(function renderMobileUpload() { return MobileInterface.Instance.renderUploadContent(); }); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 74f45ae62..e76d9b7a2 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -42,6 +42,7 @@ export default class UploadManager extends ApiManager { method: Method.POST, subscription: "/upload", secureHandler: async ({ req, res }) => { + console.log("/upload register"); const form = new formidable.IncomingForm(); form.uploadDir = pathToDirectory(Directory.parsed_files); form.keepExtensions = true; diff --git a/src/server/Message.ts b/src/server/Message.ts index 236df3f3c..c23a2f0a8 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -1,5 +1,6 @@ import { Utils } from "../Utils"; import { Point } from "../pen-gestures/ndollar"; +import { Doc } from "../new_fields/Doc"; export class Message { private _name: string; @@ -62,6 +63,10 @@ export interface UpdateMobileInkOverlayPositionContent { readonly dsize?: number; } +export interface MobileDocumentUploadContent { + readonly docId: string; +} + export namespace MessageStore { export const Foo = new Message("Foo"); export const Bar = new Message("Bar"); @@ -75,6 +80,7 @@ export namespace MessageStore { export const GesturePoints = new Message("Gesture Points"); export const MobileInkOverlayTrigger = new Message("Trigger Mobile Ink Overlay"); export const UpdateMobileInkOverlayPosition = new Message("Update Mobile Ink Overlay Position"); + export const MobileDocumentUpload = new Message("Upload Document From Mobile"); export const GetRefField = new Message("Get Ref Field"); export const GetRefFields = new Message("Get Ref Fields"); diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 77816c897..798bdae67 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -1,5 +1,5 @@ import { Utils } from "../../Utils"; -import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent } from "../Message"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent, MobileDocumentUploadContent } from "../Message"; import { Client } from "../Client"; import { Socket } from "socket.io"; import { Database } from "../database"; @@ -57,6 +57,7 @@ export namespace WebSocket { Utils.AddServerHandler(socket, MessageStore.GesturePoints, content => processGesturePoints(socket, content)); Utils.AddServerHandler(socket, MessageStore.MobileInkOverlayTrigger, content => processOverlayTrigger(socket, content)); Utils.AddServerHandler(socket, MessageStore.UpdateMobileInkOverlayPosition, content => processUpdateOverlayPosition(socket, content)); + Utils.AddServerHandler(socket, MessageStore.MobileDocumentUpload, content => processMobileDocumentUpload(socket, content)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); @@ -83,6 +84,10 @@ export namespace WebSocket { socket.broadcast.emit("receiveUpdateOverlayPosition", content); } + function processMobileDocumentUpload(socket: Socket, content: MobileDocumentUploadContent) { + socket.broadcast.emit("receiveMobileDocumentUpload", content); + } + function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { const { ProjectCredentials } = GoogleCredentialsLoader; switch (query.type) { diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index 817cf40b1..3e5953ac1 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -103,7 +103,7 @@ export class CurrentUserUtils { { title: "use scrubber", icon: "eraser", click: 'activateScrubber(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this);', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "green", activePen: doc }, { title: "use drag", icon: "mouse-pointer", click: 'deactivateInk();this.activePen.pen = this;', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "white", activePen: doc }, { title: "draw", icon: "pen-nib", click: 'switchMobileView(setupMobileInkingDoc, renderMobileInking, onSwitchMobileInking);', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "red", activePen: doc }, - { title: "upload", icon: "upload", click: 'switchMobileView(setupMobileUploadDoc, renderMobileUpload);', backgroundColor: "orange" }, + { title: "upload", icon: "upload", click: 'switchMobileView(setupMobileUploadDoc, renderMobileUpload, onSwitchMobileUpload);', backgroundColor: "orange" }, { title: "upload", icon: "upload", click: 'uploadImageMobile();', backgroundColor: "cyan" }, ]; return docProtoData.filter(d => !buttons || !buttons.includes(d.title)).map(data => Docs.Create.FontIconDocument({ @@ -138,7 +138,9 @@ export class CurrentUserUtils { } static setupMobileDoc(userDoc: Doc) { - return userDoc.activeMoble ?? Docs.Create.MasonryDocument(CurrentUserUtils.setupMobileButtons(userDoc), { + const webDoc = Docs.Create.WebDocument("https://wikipedia.com", { title: "Mobile Upload Web", chromeStatus: "enabled" }); + + return userDoc.activeMoble ?? Docs.Create.MasonryDocument([webDoc, ...CurrentUserUtils.setupMobileButtons(userDoc)], { columnWidth: 100, ignoreClick: true, lockedPosition: true, chromeStatus: "disabled", title: "buttons", autoHeight: true, yMargin: 5 }); } @@ -148,13 +150,14 @@ export class CurrentUserUtils { } static setupMobileUploadDoc(userDoc: Doc) { - console.log("setup mobile upload", window.innerWidth, window.innerHeight); - const webDoc = Docs.Create.WebDocument("https://wikipedia.com", { title: "Mobile Upload Web", chromeStatus: "enabled" }); - const uploadDoc = Docs.Create.StackingDocument([], { title: "Mobile Upload", backgroundColor: "pink" }); - return Docs.Create.StackingDocument([webDoc, uploadDoc], { - title: "Mobile Upload", backgroundColor: "white", - columnWidth: window.innerWidth, ignoreClick: true, lockedPosition: true, chromeStatus: "disabled", autoHeight: true, yMargin: 5, - width: window.innerWidth, height: window.innerHeight + const webDoc = Docs.Create.WebDocument("https://yahoo.com", { title: "Upload Images From the Web", chromeStatus: "enabled" }); + const uploadDoc = Docs.Create.StackingDocument([], { title: "Mobile Upload Collection", backgroundColor: "pink" }); + console.log("window size", window.innerWidth, window.innerHeight); + // return Docs.Create.StackingDocument([webDoc, uploadDoc], { + // columnWidth: window.innerWidth, //ignoreClick: true, lockedPosition: true, chromeStatus: "disabled", title: "Mobile Upload", autoHeight: true, yMargin: 5 + // }); + return Docs.Create.StackingDocument([webDoc, uploadDoc], {//...CurrentUserUtils.setupMobileButtons(userDoc)], { + columnWidth: 100, lockedPosition: true, chromeStatus: "disabled", title: "Upload", autoHeight: true, yMargin: 30 }); } diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index cbe070293..5f1ecc733 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -42,18 +42,18 @@ export default async function InitializeServer(routeSetter: RouteSetter) { } }; app.use(cors(corsOptions)); - app.use("*", ({ user, originalUrl }, res, next) => { - if (user && !originalUrl.includes("Heartbeat")) { - const userEmail = (user as any).email; - if (userEmail) { - timeMap[userEmail] = Date.now(); - } - } - if (!user && originalUrl === "/") { - return res.redirect("/login"); - } - next(); - }); + // app.use("*", ({ user, originalUrl }, res, next) => { + // if (user && !originalUrl.includes("Heartbeat")) { + // const userEmail = (user as any).email; + // if (userEmail) { + // timeMap[userEmail] = Date.now(); + // } + // } + // if (!user && originalUrl === "/") { + // return res.redirect("/login"); + // } + // next(); + // }); app.use(wdm(compiler, { publicPath: config.output.publicPath })); app.use(whm(compiler)); -- cgit v1.2.3-70-g09d2 From dba26063754e6a882c8a3eb238b04a935989f406 Mon Sep 17 00:00:00 2001 From: kimdahey Date: Mon, 10 Feb 2020 04:52:04 -0500 Subject: started long press for iframe to copy image --- src/client/util/DragManager.ts | 6 +- src/client/views/collections/CollectionSubView.tsx | 2 + src/client/views/nodes/WebBox.tsx | 114 ++++++++++++++++++++- 3 files changed, 119 insertions(+), 3 deletions(-) (limited to 'src/client/views/collections') diff --git a/src/client/util/DragManager.ts b/src/client/util/DragManager.ts index 0bb8b531d..745540ff9 100644 --- a/src/client/util/DragManager.ts +++ b/src/client/util/DragManager.ts @@ -179,7 +179,7 @@ export namespace DragManager { ); } element.dataset.canDrop = "true"; - const handler = (e: Event) => { console.log("drop target reveied docs"); dropFunc(e, (e as CustomEvent).detail); }; + const handler = (e: Event) => { dropFunc(e, (e as CustomEvent).detail); }; element.addEventListener("dashOnDrop", handler); return () => { element.removeEventListener("dashOnDrop", handler); @@ -265,6 +265,10 @@ export namespace DragManager { StartDrag([ele], dragData, downX, downY, options); } + export function StartImgDrag(ele: HTMLElement, downX: number, downY: number) { + StartDrag([ele], {}, downX, downY); + } + function StartDrag(eles: HTMLElement[], dragData: { [id: string]: any }, downX: number, downY: number, options?: DragOptions, finishDrag?: (dropData: DragCompleteEvent) => void) { eles = eles.filter(e => e); if (!dragDiv) { diff --git a/src/client/views/collections/CollectionSubView.tsx b/src/client/views/collections/CollectionSubView.tsx index b35af2314..b5067ea9e 100644 --- a/src/client/views/collections/CollectionSubView.tsx +++ b/src/client/views/collections/CollectionSubView.tsx @@ -180,6 +180,7 @@ export function CollectionSubView(schemaCtor: (doc: Doc) => T) { } const html = e.dataTransfer.getData("text/html"); const text = e.dataTransfer.getData("text/plain"); + console.log(html); if (text && text.startsWith("(schemaCtor: (doc: Doc) => T) { const img = tags[0].startsWith("img") ? tags[0] : tags.length > 1 && tags[1].startsWith("img") ? tags[1] : ""; if (img) { const split = img.split("src=\"")[1].split("\"")[0]; + console.log("creating image", split); const doc = Docs.Create.ImageDocument(split, { ...options, width: 300 }); ImageUtils.ExtractExif(doc); this.props.addDocument(doc); diff --git a/src/client/views/nodes/WebBox.tsx b/src/client/views/nodes/WebBox.tsx index 0b23c3bec..c405dd9d4 100644 --- a/src/client/views/nodes/WebBox.tsx +++ b/src/client/views/nodes/WebBox.tsx @@ -21,6 +21,9 @@ import "./WebBox.scss"; import React = require("react"); import { DocAnnotatableComponent } from "../DocComponent"; import { documentSchema } from "../../../new_fields/documentSchemas"; +import { Id } from "../../../new_fields/FieldSymbols"; +import { DragManager } from "../../util/DragManager"; +import { ImageUtils } from "../../util/Import & Export/ImageUtils"; library.add(faStickyNote); @@ -34,6 +37,12 @@ export class WebBox extends DocAnnotatableComponent @observable private collapsed: boolean = true; @observable private url: string = ""; + private _longPressSecondsHack?: NodeJS.Timeout; + private _iframeRef = React.createRef(); + private _iframeDragRef = React.createRef(); + @observable private _pressX: number = 0; + @observable private _pressY: number = 0; + componentWillMount() { const field = Cast(this.props.Document[this.props.fieldKey], WebField); @@ -51,6 +60,49 @@ export class WebBox extends DocAnnotatableComponent this.setURL(); } + componentDidMount() { + document.addEventListener("pointerup", this.onLongPressUp); + document.addEventListener("pointermove", this.onLongPressMove); + // this._iframeRef.current?.contentWindow?.document.addEventListener("mousedown", (event: MouseEvent) => { + // console.log("clicked inside the iframe?"); + // }); + // const iframe = document.getElementById(this.props.Document.proto![Id]); + // if (iframe) { + // iframe.addEventListener('pointerdown', function (event) { + // const B = iframe.getBoundingClientRect(); + // const e = new CustomEvent('pointerdown', { bubbles: true, cancelable: false }); + // // e.clientX = e.clientX + B?.left; + // // e.clientY = e.clientY + B?.top; + // console.log("custom event pointer down"); + // iframe.dispatchEvent(e); + // }) + // } + // if (this._iframeRef.current) { + // console.log("resetting iframes events"); + // const self = this; + // this._iframeRef.current.addEventListener('pointermove', function (event) { + // const B = self._iframeRef.current?.getBoundingClientRect(); + // const e = new CustomEvent('pointermove', { bubbles: true, cancelable: false }); + // // e.clientX = e.clientX + B?.left; + // // e.clientY = e.clientY + B?.top; + // self._iframeRef.current?.dispatchEvent(e); + // }); + // this._iframeRef.current.addEventListener('pointerdown', function (event) { + // const B = self._iframeRef.current?.getBoundingClientRect(); + // const e = new CustomEvent('pointerdown', { bubbles: true, cancelable: false }); + // // e.clientX = e.clientX + B?.left; + // // e.clientY = e.clientY + B?.top; + // console.log("custom event pointer down"); + // self._iframeRef.current?.dispatchEvent(e); + // }) + // } + } + + componentWillUnmount() { + document.removeEventListener("pointerup", this.onLongPressUp); + document.removeEventListener("pointermove", this.onLongPressMove); + } + @action onURLChange = (e: React.ChangeEvent) => { this.url = e.target.value; @@ -165,6 +217,62 @@ export class WebBox extends DocAnnotatableComponent } } + // TODO: make this actually a long press + onLongPressDown = (e: React.PointerEvent) => { + + console.log("press down", e.clientX, e.clientX); + this._pressX = e.clientX; + this._pressY = e.clientY; + this._longPressSecondsHack = setTimeout(() => { + console.log("start the drag!!"); + const B = this._iframeRef.current?.getBoundingClientRect(); + const iframeDoc = this._iframeRef.current?.contentDocument; + if (B && iframeDoc) { + console.log("frame doc", iframeDoc); + console.log("get point", this._pressX, B.left, this._pressY, B.top); + const element = iframeDoc.elementFromPoint(this._pressX - B.left, this._pressY - B.top); + console.log("found element", element); + if (element) { + e.stopPropagation(); + e.preventDefault(); + const clone = element.cloneNode(true) as HTMLElement; + + if (clone.nodeName === "IMG") { + const src = clone.getAttribute("src"); // TODO: may not always work + + if (src) { + const doc = Docs.Create.ImageDocument(src, { width: 300 }); + ImageUtils.ExtractExif(doc); + + console.log("start image drag", this._pressX, this._pressY, doc); + // document.dispatchEvent() + const dragData = new DragManager.DocumentDragData([doc]); + DragManager.StartDocumentDrag([clone], dragData, this._pressX, this._pressY); + } + } + } + } + }, 1500); + // e.stopPropagation(); + // e.preventDefault(); + } + + onLongPressMove = (e: PointerEvent) => { + this._pressX = e.clientX; + this._pressY = e.clientY; + } + + onLongPressUp = (e: PointerEvent) => { + console.log("press up"); + if (this._longPressSecondsHack) { + clearTimeout(this._longPressSecondsHack); + console.log("long press cancelled"); + } + // e.stopPropagation(); + // e.preventDefault(); + } + + @computed get content() { const field = this.dataDoc[this.props.fieldKey]; @@ -172,9 +280,9 @@ export class WebBox extends DocAnnotatableComponent if (field instanceof HtmlField) { view = ; } else if (field instanceof WebField) { - view =