path: root/src/client/views/InkTranscription.tsx

import * as iink from 'iink-ts';
import { action, observable } from 'mobx';
import * as React from 'react';
import { imageUrlToBase64 } from '../../ClientUtils';
import { aggregateBounds } from '../../Utils';
import { Doc, DocListCast } from '../../fields/Doc';
import { InkData, InkInkTool, InkTool } from '../../fields/InkField';
import { Cast, DateCast, ImageCast, InkCast, NumCast } from '../../fields/Types';
import { ImageField, URLField } from '../../fields/URLField';
import { gptHandwriting } from '../apis/gpt/GPT';
import { DocumentType } from '../documents/DocumentTypes';
import { Docs } from '../documents/Documents';
import './InkTranscription.scss';
import { InkingStroke } from './InkingStroke';
import { CollectionFreeFormView, MarqueeView } from './collections/collectionFreeForm';
import { DocumentView } from './nodes/DocumentView';
/**
 * Class component that handles inking in writing mode
 */
export class InkTranscription extends React.Component {
    // eslint-disable-next-line no-use-before-define
    static Instance: InkTranscription;

    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    @observable _mathRegister: any = undefined;
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    @observable _mathRef: any = undefined;
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    @observable _textRegister: any = undefined;
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    @observable _textRef: any = undefined;
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    @observable _iinkEditor: any = undefined;
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    @observable _iinkMathEditor: any = undefined;
    private currGroup?: Doc;
    private collectionFreeForm?: CollectionFreeFormView;

    constructor(props: Readonly<object>) {
        super(props);

        InkTranscription.Instance = this;
    }
    @action
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    setMathRef = async (r: any) => {
        if (!this._mathRegister && r) {
            const options = {
                configuration: {
                    server: {
                        scheme: 'https' as iink.TScheme,
                        host: 'cloud.myscript.com',
                        applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca',
                        hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f',
                        protocol: 'WEBSOCKET',
                    },
                    recognition: {
                        type: 'MATH',
                        math: {
                            mimeTypes: ['application/x-latex'] as unknown as 'application/vnd.myscript.jiix'[],
                        },
                    },
                },
            };
            this._iinkMathEditor = await iink.Editor.load(r, 'INKV2', options);
            this._mathRegister = r;
            // eslint-disable-next-line @typescript-eslint/no-explicit-any
            r?.addEventListener('exported', (e: any) => this.exportInk(e));

            return (this._textRef = r);
        }
    };
    @action
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    setTextRef = async (r: any) => {
        if (!this._textRegister && r) {
            const options = {
                configuration: {
                    server: {
                        scheme: 'https' as iink.TScheme,
                        host: 'cloud.myscript.com',
                        applicationKey: 'c0901093-5ac5-4454-8e64-0def0f13f2ca',
                        hmacKey: 'f6465cca-1856-4492-a6a4-e2395841be2f',
                        protocol: 'WEBSOCKET',
                    },
                    recognition: {
                        type: 'TEXT',
                        lang: 'en_US',
                        text: {
                            mimeTypes: ['application/vnd.myscript.jiix'] as 'application/vnd.myscript.jiix'[],
                        },
                    },
                },
            };
            this._iinkEditor = await iink.Editor.load(r, 'INKV2', options);
            this._textRegister = r;
            // eslint-disable-next-line @typescript-eslint/no-explicit-any
            r?.addEventListener('exported', (e: any) => this.exportInk(e));

            return (this._textRef = r);
        }
    };

    _ffview: CollectionFreeFormView | undefined;

    /**
     * Handles processing Dash Doc data for ink transcription.
     *
     * @param groupDoc the group which contains the ink strokes we want to transcribe
     * @param inkDocs the ink docs contained within the selected group
     * @param math boolean whether to do math transcription or not
     */
    transcribeInk = (ffview: CollectionFreeFormView, groupDoc: Doc | undefined, inkDocs: Doc[], math: boolean) => {
        if (!groupDoc) return;
        const validInks = inkDocs.filter(s => s.type === DocumentType.INK);

        const strokes: InkData[] = [];

        const times: number[] = [];
        validInks
            .filter(i => InkCast(i[Doc.LayoutDataKey(i)]))
            .forEach(i => {
                const d = InkCast(i[Doc.LayoutDataKey(i)])!;
                const authorTime = DateCast(i.author_date)?.getDate().getTime() ?? 0;
                const inkStroke = DocumentView.getDocumentView(i)?.ComponentView as InkingStroke;
                strokes.push(d.inkData.map(pd => inkStroke.ptToScreen({ X: pd.X, Y: pd.Y })));
                times.push(authorTime);
            });
        this._ffview = ffview;
        this.currGroup = groupDoc;
        const pointerData = strokes.map((stroke, i) => this.inkJSON(stroke, times[i]));
        (math ? this._iinkMathEditor : this._iinkEditor).importPointEvents(pointerData);
    };
    convertPointsToString(points: InkData[]): string {
        return points[0].map(point => `new Point(${point.X}, ${point.Y})`).join(',');
    }
    convertPointsToString2(points: InkData[]): string {
        return points[0].map(point => `(${point.X},${point.Y})`).join(',');
    }

    /**
     * Converts the Dash Ink Data to JSON.
     *
     * @param stroke The dash ink data
     * @param time the time of the stroke
     * @returns json object representation of ink data
     */
    inkJSON = (stroke: InkData, time: number) => {
        interface strokeData {
            x: number;
            y: number;
            t: number;
            p: number;
        }
        const strokeObjects: strokeData[] = [];
        stroke.forEach(point => {
            const tempObject: strokeData = {
                x: point.X,
                y: point.Y,
                t: time,
                p: 1.0,
            };
            strokeObjects.push(tempObject);
        });
        return {
            pointerType: 'PEN',
            pointerId: 1,
            pointers: strokeObjects,
        };
    };

    /**
     * Creates subgroups for each word for the whole text transcription
     * @param wordInkDocMap the mapping of words to ink strokes (Ink Docs)
     */
    subgroupsTranscriptions = (wordInkDocMap: Map<string, Doc[]>) => {
        // iterate through the keys of wordInkDocMap
        wordInkDocMap.forEach(async (inkDocs: Doc[], word: string) => {
            const selected = inkDocs.slice();
            if (!selected) {
                return;
            }
            const ctx = await Cast(selected[0].embedContainer, Doc);
            if (!ctx) {
                return;
            }
            const docView: CollectionFreeFormView = DocumentView.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;
            // DocumentManager.Instance.getDocumentView(ctx)?.ComponentView as CollectionFreeFormView;

            if (!docView) return;
            const marqViewRef = docView._marqueeViewRef.current;
            if (!marqViewRef) return;
            this.groupInkDocs(selected, docView, word);
        });
    };

    /**
     * Event listener function for when the 'exported' event is heard.
     *
     * @param e the event objects
     * @param ref the ref to the editor
     */
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    exportInk = (e: { detail: { [key: string]: string } }) => {
        const exports = e.detail;
        if (exports['application/x-latex']) {
            const latex = exports['application/x-latex'];
            if (this.currGroup) {
                this.currGroup.text = latex;
                this.currGroup.title = latex;
            }

            this._ffview?.addDocument(
                Docs.Create.EquationDocument(latex, {
                    title: '',
                    x: this.currGroup?.x as number,
                    y: (this.currGroup?.y as number) + (this.currGroup?.height as number),
                    nativeHeight: 40,
                    _height: 50,
                    nativeWidth: 40,
                    _width: 50,
                })
            );
            // this.showRecogBox(latex as string);
            this._iinkMathEditor.clear();
        } else if (exports['application/vnd.myscript.jiix']) {
            const lastJiix = exports['application/vnd.myscript.jiix'] as unknown as { words: string[]; label: string };
            // map timestamp to strokes
            const timestampWord = new Map<number, string>();
            // eslint-disable-next-line @typescript-eslint/no-explicit-any
            lastJiix.words.map((word: any) => {
                if (word.items) {
                    word.items.forEach((i: { id: string; timestamp: string; X: Array<number>; Y: Array<number>; F: Array<number> }) => {
                        const ms = Date.parse(i.timestamp);
                        timestampWord.set(ms, word.label);
                    });
                }
            });

            const wordInkDocMap = new Map<string, Doc[]>();
            if (this.currGroup) {
                DocListCast(this.currGroup.data).forEach((inkDoc: Doc) => {
                    // just having the times match up and be a unique value (actual timestamp doesn't matter)
                    const ms = (DateCast(inkDoc.author_date)?.getDate().getTime() ?? 0) + 14400000;
                    const word = timestampWord.get(ms);
                    if (word) {
                        const entry = wordInkDocMap.get(word);
                        if (entry) {
                            entry.push(inkDoc);
                            wordInkDocMap.set(word, entry);
                        } else {
                            const newEntry = [inkDoc];
                            wordInkDocMap.set(word, newEntry);
                        }
                    }
                });
                if (lastJiix.words.length > 1) this.subgroupsTranscriptions(wordInkDocMap);
            }
            this.showRecogBox(lastJiix.label);
            this._iinkEditor.clear();
        }
    };
    private showRecogBox(text: string) {
        if (this.currGroup) {
            let response;
            // DocumentView.getDocumentView(this.currGroup)?.ComponentView?.updateIcon?.();
            // const image = await this.getIcon();
            // const { href } = (image as URLField).url;
            // const hrefParts = href.split('.');
            // const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`;
            // try {
            //     const hrefBase64 = await imageUrlToBase64(hrefComplete);
            //     response = await gptHandwriting(hrefBase64);
            // } catch {
            //     console.error('Error getting image');
            // }
            const textBoxText = 'iink: ' + text + '\n' + '\n' + 'ChatGPT: ' + response;

            this.currGroup.transcription = response;
            this.currGroup.title = response;
            if (!this.currGroup.hasTextBox) {
                const newDoc = Docs.Create.TextDocument(textBoxText, { title: '', x: this.currGroup.x as number, y: (this.currGroup.y as number) + (this.currGroup.height as number) });
                newDoc.height = 200;
                this.collectionFreeForm?.addDocument(newDoc);
                this.currGroup.hasTextBox = true;
            }
        }
    }

    /**
     * gets the icon of the collection that was just made
     * @returns the image of the collection
     */
    async getIcon() {
        const docView = DocumentView.getDocumentView(this.currGroup);
        if (docView) {
            docView.ComponentView?.updateIcon?.();
            return new Promise<ImageField | undefined>(res => setTimeout(() => res(ImageCast(docView.Document.icon)), 1000));
        }
        return undefined;
    }

    /**
     * Creates the ink grouping once the user leaves the writing mode.
     */
    createInkGroup() {
        // TODO nda - if document being added to is a inkGrouping then we can just add to that group
        if (Doc.ActiveTool === InkTool.Ink && [InkInkTool.Write, InkInkTool.Math].includes(Doc.ActiveInk)) {
            CollectionFreeFormView.collectionsWithUnprocessedInk.forEach(ffView => {
                // TODO: nda - will probably want to go through ffView unprocessed docs and then see if any of the inksToGroup docs are in it and only use those
                const selected = ffView.unprocessedDocs;
                const newCollection = this.groupInkDocs(
                    selected.filter(doc => doc.embedContainer),
                    ffView
                );
                ffView.unprocessedDocs = [];

                InkTranscription.Instance.transcribeInk(ffView, newCollection, selected, Doc.ActiveInk === InkInkTool.Math);
            });
        }
        CollectionFreeFormView.collectionsWithUnprocessedInk.clear();
    }

    /**
     * Creates the groupings for a given list of ink docs on a specific doc view
     * @param selected: the list of ink docs to create a grouping of
     * @param docView: the view in which we want the grouping to be created
     * @param word: optional param if the group we are creating is a word (subgrouping individual words)
     * @returns a new collection Doc or undefined if the grouping fails
     */
    groupInkDocs(selected: Doc[], docView: CollectionFreeFormView, word?: string): Doc | undefined {
        this.collectionFreeForm = docView;
        const bounds: { x: number; y: number; width?: number; height?: number }[] = [];

        // calculate the necessary bounds from the selected ink docs
        selected.forEach(
            action(d => {
                const x = NumCast(d.x);
                const y = NumCast(d.y);
                const width = NumCast(d._width);
                const height = NumCast(d._height);
                bounds.push({ x, y, width, height });
            })
        );

        // calculate the aggregated bounds
        const aggregBounds = aggregateBounds(bounds, 0, 0);
        const marqViewRef = docView._marqueeViewRef.current;

        // set the vals for bounds in marqueeView
        if (marqViewRef) {
            marqViewRef._downX = aggregBounds.x;
            marqViewRef._downY = aggregBounds.y;
            marqViewRef._lastX = aggregBounds.r;
            marqViewRef._lastY = aggregBounds.b;
        }

        // map through all the selected ink strokes and create the groupings
        selected.forEach(
            action(d => {
                const dx = NumCast(d.x);
                const dy = NumCast(d.y);
                delete d.x;
                delete d.y;
                delete d.activeFrame;
                delete d._timecodeToShow; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
                delete d._timecodeToHide; // bcz: this should be automatic somehow.. along with any other properties that were logically associated with the original collection
                // calculate pos based on bounds
                if (marqViewRef?.Bounds) {
                    d.x = dx - marqViewRef.Bounds.left - marqViewRef.Bounds.width / 2;
                    d.y = dy - marqViewRef.Bounds.top - marqViewRef.Bounds.height / 2;
                }
                return d;
            })
        );
        docView.props.removeDocument?.(selected);
        // Gets a collection based on the selected nodes using a marquee view ref
        const newCollection = MarqueeView.getCollection(selected, undefined, true, marqViewRef?.Bounds ?? { top: 1, left: 1, width: 1, height: 1 });
        // if the grouping we are creating is an individual word
        if (word) {
            newCollection.title = word;
        }

        // nda - bug: when deleting a stroke before leaving writing mode, delete the stroke from unprocessed ink docs
        docView.props.addDocument?.(newCollection);
        newCollection.hasTextBox = false;
        return newCollection;
    }

    render() {
        return (
            <div className="ink-transcription" style={{ pointerEvents: 'none' }}>
                <div className="math-editor" ref={this.setMathRef}></div>
                <div className="text-editor" ref={this.setTextRef}></div>
            </div>
        );
    }
}