diff options
Diffstat (limited to 'src/client/views/GestureOverlay.tsx')
| -rw-r--r-- | src/client/views/GestureOverlay.tsx | 400 |
1 files changed, 332 insertions, 68 deletions
diff --git a/src/client/views/GestureOverlay.tsx b/src/client/views/GestureOverlay.tsx index 3a2738c3b..935eeb251 100644 --- a/src/client/views/GestureOverlay.tsx +++ b/src/client/views/GestureOverlay.tsx @@ -7,30 +7,35 @@ import { emptyFunction } from '../../Utils'; import { Doc, Opt, returnEmptyDoclist } from '../../fields/Doc'; import { InkData, InkField, InkTool } from '../../fields/InkField'; import { NumCast } from '../../fields/Types'; +import { Gestures } from '../../pen-gestures/GestureTypes'; +import { GestureUtils } from '../../pen-gestures/GestureUtils'; +import { Result } from '../../pen-gestures/ndollar'; +import { DocumentType } from '../documents/DocumentTypes'; +import { Docs } from '../documents/Documents'; +import { InteractionUtils } from '../util/InteractionUtils'; +import { ScriptingGlobals } from '../util/ScriptingGlobals'; +import { Transform } from '../util/Transform'; +import './GestureOverlay.scss'; +import { InkingStroke } from './InkingStroke'; +import { ObservableReactComponent } from './ObservableReactComponent'; +import { returnEmptyDocViewList } from './StyleProvider'; +import { CollectionFreeFormView } from './collections/collectionFreeForm'; import { ActiveArrowEnd, ActiveArrowScale, ActiveArrowStart, ActiveDash, + ActiveFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, + DocumentView, SetActiveArrowStart, SetActiveDash, SetActiveFillColor, SetActiveInkColor, SetActiveInkWidth, } from './nodes/DocumentView'; -import { Gestures } from '../../pen-gestures/GestureTypes'; -import { GestureUtils } from '../../pen-gestures/GestureUtils'; -import { InteractionUtils } from '../util/InteractionUtils'; -import { ScriptingGlobals } from '../util/ScriptingGlobals'; -import { Transform } from '../util/Transform'; -import './GestureOverlay.scss'; -import { ObservableReactComponent } from './ObservableReactComponent'; -import { returnEmptyDocViewList } from './StyleProvider'; -import { ActiveFillColor, DocumentView } from './nodes/DocumentView'; - export enum ToolglassTools { InkToText = 'inktotext', IgnoreGesture = 'ignoregesture', @@ -41,10 +46,12 @@ interface GestureOverlayProps { isActive: boolean; } @observer +/** + * class for gestures. will determine if what the user drew is a gesture, and will transform the ink stroke into the shape the user + * drew or perform the gesture's action + */ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChildren<GestureOverlayProps>> { - // eslint-disable-next-line no-use-before-define static Instance: GestureOverlay; - // eslint-disable-next-line no-use-before-define static Instances: GestureOverlay[] = []; @observable public InkShape: Opt<Gestures> = undefined; @@ -127,70 +134,331 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil // SetActiveArrowEnd('none'); } } + /** + * this method returns if what the user drew is a scribble. if it is, it will determine what documents need + * to be deleted and then it will delete them. + * @returns + */ + isScribble() { + const ffView = DocumentView.allViews().find(view => view.ComponentView instanceof CollectionFreeFormView); + const cuspArray = this.getCusps(); + const cuspBooleanArray: boolean[] = []; + const docsToDelete: Doc[] = []; + const childDocs = (ffView?.ComponentView as CollectionFreeFormView).childDocs; + childDocs.filter(doc => doc.type === 'ink').map(doc => DocumentView.getDocumentView(doc, DocumentView.getDocumentView(doc))); + if ((ffView?.ComponentView as CollectionFreeFormView).childDocs) { + //how many cusps the scribble hsa + if (cuspArray.length > 4) { + for (let i = 0; i < cuspArray.length - 2; i++) { + let hasDocInTriangle = false; + for (const doc of childDocs) { + const point1 = cuspArray[i]; + const point2 = cuspArray[i + 1]; + const point3 = cuspArray[i + 2]; + const triangleObject = { p1: { X: point1.X, Y: point1.Y }, p2: { X: point2.X, Y: point2.Y }, p3: { X: point3.X, Y: point3.Y } }; + const otherInk = DocumentView.getDocumentView(doc)?.ComponentView as InkingStroke; + if (otherInk instanceof InkingStroke) { + const { inkData: otherInkData } = otherInk?.inkScaledData() ?? { inkData: [] }; + const otherScreenPts = otherInkData.map(point => otherInk.ptToScreen(point)); + if (doc.title === 'line') { + if (this.doesLineIntersectTriangle(otherScreenPts, triangleObject)) { + docsToDelete.push(doc); + hasDocInTriangle = true; + cuspBooleanArray.push(true); + } + } else { + if (this.isAnyPointInTriangle(triangleObject, otherScreenPts)) { + docsToDelete.push(doc); + hasDocInTriangle = true; + cuspBooleanArray.push(true); + } + } + } + } + cuspBooleanArray.push(hasDocInTriangle); + } + if (this.determineIfScribble(cuspBooleanArray)) { + docsToDelete.forEach(doc => { + ffView?.ComponentView?.removeDocument?.(doc); + }); + this._points = []; + return true; + } + } + } + return false; + } + /** + * this method determines if what the user drew is a scribble based on certain criteria. + * @param cuspBooleanArray will take in an array of booleans tht represent what triangles in the scribble + * has an object in it. the 1st triangle is the 0th index, 2nd triangle is the 1st index and so on. + * @returns + */ + determineIfScribble(cuspBooleanArray: boolean[]) { + if (!cuspBooleanArray) { + return false; + } + const quarterArrayLength = Math.ceil((cuspBooleanArray.length - 2) * 0.25); + let hasObjectInFirstAndLast25 = true; + for (let i = 0; i < quarterArrayLength; i++) { + if (cuspBooleanArray[i] == false || cuspBooleanArray[cuspBooleanArray.length - 1 - i] == false) { + hasObjectInFirstAndLast25 = false; + } + } + const trueCount = cuspBooleanArray.filter(value => value).length; + const percentageTrues = trueCount / cuspBooleanArray.length; + return percentageTrues > 0.65 || hasObjectInFirstAndLast25; + } + /** + * determines if two rectangles are overlapping each other + * @param rect1 the rectangle object has has a minX,maxX,minY, and maxY + * @param rect2 + * @returns + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + isRectangleOverlap(rect1: any, rect2: any): boolean { + const noOverlap = rect1.maxX < rect2.minX || rect1.minX > rect2.maxX || rect1.maxY < rect2.minY || rect1.minY > rect2.maxY; + + return !noOverlap; + } + /** + * determines if there is a point in a triangle used to determine what triangles in the scribble have an object + * @param pt the point in question + * @param triangle the triangle with 3 points + * @returns true or false if point is in triangle + */ + isPointInTriangle(pt: { X: number; Y: number }, triangle: { p1: { X: number; Y: number }; p2: { X: number; Y: number }; p3: { X: number; Y: number } }): boolean { + const area = (v1: { X: number; Y: number }, v2: { X: number; Y: number }, v3: { X: number; Y: number }) => Math.abs((v1.X * (v2.Y - v3.Y) + v2.X * (v3.Y - v1.Y) + v3.X * (v1.Y - v2.Y)) / 2.0); + + const A = area(triangle.p1, triangle.p2, triangle.p3); + + const A1 = area(pt, triangle.p2, triangle.p3); + const A2 = area(triangle.p1, pt, triangle.p3); + const A3 = area(triangle.p1, triangle.p2, pt); + + return A === A1 + A2 + A3; + } + /** + * determines if any points in an array are in a triangle + * @param triangle the triangle with 3 points + * @param points the point in question + * @returns true or false if point is in triangle + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + isAnyPointInTriangle(triangle: any, points: any[]): boolean { + for (const point of points) { + if (this.isPointInTriangle(point, triangle)) { + return true; + } + } + return false; + } + /** + * determines if a line intersects a triangle. used for scribble gesture since the line doesnt have a lot + * of points across is so isPointInTriangle will not work for it. + * @param line is pointData + * @param triangle triangle with 3 points + * @returns true or false if it intersects + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + doesLineIntersectTriangle(line: any, triangle: any): boolean { + const edges = [ + { start: triangle.p1, end: triangle.p2 }, + { start: triangle.p2, end: triangle.p3 }, + { start: triangle.p3, end: triangle.p1 }, + ]; + + for (const edge of edges) { + if (this.doLinesIntersect(line, edge)) { + return true; + } + } + + return false; + } + /** + * used in doesLineIntersectTriangle, splits up the triangle into 3 lines and runs this method + * individually + * @param line1 point data where 0th index is the coordinate of beginnning of line and last index is the coordinate of end of limne + * @param line2 same thing + * @returns + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + doLinesIntersect(line1: any, line2: any): boolean { + const A = line1[0]; + const B = line1[line1.length - 1]; + const { start: C, end: D } = line2; + const denominator = (B.X - A.X) * (D.Y - C.Y) - (B.Y - A.Y) * (D.X - C.X); + if (denominator === 0) return false; + + const numerator1 = (A.Y - C.Y) * (D.X - C.X) - (A.X - C.X) * (D.Y - C.Y); + const numerator2 = (A.Y - C.Y) * (B.X - A.X) - (A.X - C.X) * (B.Y - A.Y); + + const r = numerator1 / denominator; + const s = numerator2 / denominator; + + return r >= 0 && r <= 1 && s >= 0 && s <= 1; + } + + dryInk = () => { + const newPoints = this._points.reduce((p, pts) => { + p.push([pts.X, pts.Y]); + return p; + }, [] as number[][]); + newPoints.pop(); + const controlPoints: { X: number; Y: number }[] = []; + + const bezierCurves = fitCurve.default(newPoints, 10); + Array.from(bezierCurves).forEach(curve => { + controlPoints.push({ X: curve[0][0], Y: curve[0][1] }); + controlPoints.push({ X: curve[1][0], Y: curve[1][1] }); + controlPoints.push({ X: curve[2][0], Y: curve[2][1] }); + controlPoints.push({ X: curve[3][0], Y: curve[3][1] }); + }); + const dist = Math.sqrt((controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y)); + if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0]; + this._points.length = 0; + this._points.push(...controlPoints); + this.dispatchGesture(Gestures.Stroke); + }; @action onPointerUp = () => { DocumentView.DownDocView = undefined; if (this._points.length > 1) { const B = this.svgBounds; const points = this._points.map(p => ({ X: p.X - B.left, Y: p.Y - B.top })); - + const { Name, Score } = + (this.InkShape + ? new Result(this.InkShape, 1, Date.now) + : Doc.UserDoc().recognizeGestures && points.length > 2 + ? GestureUtils.GestureRecognizer.Recognize([points]) + : undefined) ?? + new Result(Gestures.Stroke, 1, Date.now); // prettier-ignore // if any of the shape is activated in the CollectionFreeFormViewChrome - if (this.InkShape) { - this.makeBezierPolygon(this.InkShape, false); - this.dispatchGesture(this.InkShape); - this.primCreated(); - } - // if we're not drawing in a toolglass try to recognize as gesture - else { - // need to decide when to turn gestures back on - const result = points.length > 2 && GestureUtils.GestureRecognizer.Recognize([points]); - let actionPerformed = false; - if (Doc.UserDoc().recognizeGestures && result && result.Score > 0.7) { - switch (result.Name) { - case Gestures.Line: - case Gestures.Triangle: - case Gestures.Rectangle: - case Gestures.Circle: - this.makeBezierPolygon(result.Name, true); - actionPerformed = this.dispatchGesture(result.Name); - break; - case Gestures.Scribble: - console.log('scribble'); - break; - default: - } - } - - // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document - if (!actionPerformed) { - const newPoints = this._points.reduce((p, pts) => { - p.push([pts.X, pts.Y]); - return p; - }, [] as number[][]); - newPoints.pop(); - const controlPoints: { X: number; Y: number }[] = []; - - const bezierCurves = fitCurve.default(newPoints, 10); - Array.from(bezierCurves).forEach(curve => { - controlPoints.push({ X: curve[0][0], Y: curve[0][1] }); - controlPoints.push({ X: curve[1][0], Y: curve[1][1] }); - controlPoints.push({ X: curve[2][0], Y: curve[2][1] }); - controlPoints.push({ X: curve[3][0], Y: curve[3][1] }); - }); - const dist = Math.sqrt( - (controlPoints[0].X - controlPoints.lastElement().X) * (controlPoints[0].X - controlPoints.lastElement().X) + (controlPoints[0].Y - controlPoints.lastElement().Y) * (controlPoints[0].Y - controlPoints.lastElement().Y) - ); - // eslint-disable-next-line prefer-destructuring - if (controlPoints.length > 4 && dist < 10) controlPoints[controlPoints.length - 1] = controlPoints[0]; - this._points.length = 0; - this._points.push(...controlPoints); - this.dispatchGesture(Gestures.Stroke); + // need to decide when to turn gestures back on + const actionPerformed = ((name: Gestures) => { + switch (name) { + case Gestures.Line: + case Gestures.Triangle: + case Gestures.Rectangle: + case Gestures.Circle: + this.makeBezierPolygon(Name, true); + return this.dispatchGesture(name); + case Gestures.RightAngle: + return this.convertToText().length > 0; + default: + case Gestures.Stroke: + return this.isScribble(); } - } + })(Score < 0.7 ? Gestures.Stroke : (Name as Gestures)); + // if no gesture (or if the gesture was unsuccessful), "dry" the stroke into an ink document + if (!actionPerformed) this.dryInk(); } this._points.length = 0; }; + /** + * used in the rightAngle gesture to convert handwriting into text. will only work on collections + * TODO: make it work on individual ink docs. + */ + convertToText() { + const ffView = DocumentView.allViews().find(view => view.ComponentView instanceof CollectionFreeFormView); + let minX = 999999999; + let maxX = -999999999; + let minY = 999999999; + let maxY = -999999999; + const textDocs: Doc[] = []; + (ffView?.ComponentView as CollectionFreeFormView).childDocs + .filter(doc => doc.type === DocumentType.COL) + .forEach(doc => { + if (typeof doc.width === 'number' && typeof doc.height === 'number' && typeof doc.x === 'number' && typeof doc.y === 'number') { + const bounds = DocumentView.getDocumentView(doc)?.getBounds; + if (bounds) { + const rect1 = { minX: bounds.left, maxX: bounds.right, minY: bounds.top, maxY: bounds.bottom }; + if (this.isRectangleOverlap(rect1, this.getExtremeCoordinates())) { + if (doc.x < minX) { + minX = doc.x; + } + if (doc.x > maxX) { + maxX = doc.x; + } + if (doc.y < minY) { + minY = doc.y; + } + if (doc.y + doc.height > maxY) { + maxY = doc.y + doc.height; + } + const newDoc = Docs.Create.TextDocument(doc.transcription as string, { title: '', x: doc.x as number, y: minY }); + newDoc.height = doc.height; + newDoc.width = doc.width; + if (ffView?.ComponentView?.addDocument && ffView?.ComponentView?.removeDocument) { + ffView.ComponentView.addDocument(newDoc); + ffView.ComponentView.removeDocument(doc); + } + textDocs.push(newDoc); + } + } + } + }); + return textDocs; + } + /** + * used to determine how many cusps and where the cusps are in order + * @returns will return an array containing the coordinates of the sharp cusps + */ + getCusps() { + const points = this._points.map(p => ({ X: p.X, Y: p.Y })); + const arrayOfPoints: { X: number; Y: number }[] = []; + arrayOfPoints.push(points[0]); + for (let i = 0; i < points.length - 2; i++) { + const point1 = points[i]; + const point2 = points[i + 1]; + const point3 = points[i + 2]; + if (this.find_angle(point1, point2, point3) < 90) { + arrayOfPoints.push(point2); + } + } + arrayOfPoints.push(points[points.length - 1]); + return arrayOfPoints; + } + /** + * will look through an array of point data and return the coordinates of the smallest box that can fit all the points + * @returns the minX,maxX,minY,maxY of the box + */ + getExtremeCoordinates() { + const coordinates = this._points; + if (coordinates.length === 0) { + throw new Error('Coordinates array is empty'); + } + let minX = coordinates[0].X; + let maxX = coordinates[0].X; + let minY = coordinates[0].Y; + let maxY = coordinates[0].Y; + + coordinates.forEach(coord => { + if (coord.X < minX) minX = coord.X; + if (coord.X > maxX) maxX = coord.X; + if (coord.Y < minY) minY = coord.Y; + if (coord.Y > maxY) maxY = coord.Y; + }); + return { + minX, + maxX, + minY, + maxY, + }; + } + /** + * takes in three points and then determines the angle of the points. used to determine if the cusp + * is sharp enoug + * @returns + */ + find_angle(A: { X: number; Y: number }, B: { X: number; Y: number }, C: { X: number; Y: number }) { + const AB = Math.sqrt(Math.pow(B.X - A.X, 2) + Math.pow(B.Y - A.Y, 2)); + const BC = Math.sqrt(Math.pow(B.X - C.X, 2) + Math.pow(B.Y - C.Y, 2)); + const AC = Math.sqrt(Math.pow(C.X - A.X, 2) + Math.pow(C.Y - A.Y, 2)); + return Math.acos((BC * BC + AB * AB - AC * AC) / (2 * BC * AB)) * (180 / Math.PI); + } makeBezierPolygon = (shape: string, gesture: boolean) => { const xs = this._points.map(p => p.X); const ys = this._points.map(p => p.Y); @@ -389,7 +657,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil this._strokes.map((l, i) => { const b = { left: -20000, right: 20000, top: -20000, bottom: 20000, width: 40000, height: 40000 }; // this.getBounds(l, true); return ( - // eslint-disable-next-line react/no-array-index-key <svg key={i} width={b.width} height={b.height} style={{ top: 0, left: 0, transform: `translate(${b.left}px, ${b.top}px)`, pointerEvents: 'none', position: 'absolute', zIndex: 30000, overflow: 'visible' }}> {InteractionUtils.CreatePolyline( l, @@ -511,7 +778,6 @@ export class GestureOverlay extends ObservableReactComponent<React.PropsWithChil } ScriptingGlobals.add('GestureOverlay', GestureOverlay); -// eslint-disable-next-line prefer-arrow-callback ScriptingGlobals.add(function setPen(width: string, color: string, fill: string, arrowStart: string, arrowEnd: string, dash: string) { runInAction(() => { GestureOverlay.Instance.SavedColor = ActiveInkColor(); @@ -524,7 +790,6 @@ ScriptingGlobals.add(function setPen(width: string, color: string, fill: string, SetActiveDash(dash); }); }); -// eslint-disable-next-line prefer-arrow-callback ScriptingGlobals.add(function resetPen() { runInAction(() => { SetActiveInkColor(GestureOverlay.Instance.SavedColor ?? 'rgb(0, 0, 0)'); @@ -532,7 +797,6 @@ ScriptingGlobals.add(function resetPen() { }); }, 'resets the pen tool'); ScriptingGlobals.add( - // eslint-disable-next-line prefer-arrow-callback function createText(text: string, X: number, Y: number) { GestureOverlay.Instance.dispatchGesture(Gestures.Text, [{ X, Y }], text); }, |