path: root/src/client/views/StyleProviderQuiz.tsx

import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import { Tooltip } from '@mui/material';
import { runInAction } from 'mobx';
import * as React from 'react';
import { returnFalse, setupMoveUpEvents } from '../../ClientUtils';
import { emptyFunction, unimplementedFunction } from '../../Utils';
import { Doc, DocListCast, Opt } from '../../fields/Doc';
import { DocData } from '../../fields/DocSymbols';
import { List } from '../../fields/List';
import { NumCast, StrCast } from '../../fields/Types';
import { Networking } from '../Network';
import { GPTCallType, gptAPICall } from '../apis/gpt/GPT';
import { Docs } from '../documents/Documents';
import { ContextMenu } from './ContextMenu';
import { ContextMenuProps } from './ContextMenuItem';
import { StyleProp } from './StyleProp';
import './StyleProviderQuiz.scss';
import { DocumentViewProps } from './nodes/DocumentView';
import { FieldViewProps } from './nodes/FieldView';
import { ImageBox } from './nodes/ImageBox';
import { ImageUtility } from './nodes/imageEditor/imageEditorUtils/ImageHandler';
import { AnchorMenu } from './pdf/AnchorMenu';

export namespace styleProviderQuiz {
    enum quizMode {
        SMART = 'smart',
        NORMAL = 'normal',
        NONE = 'none',
    }

    async function selectUrlToBase64(blob: Blob): Promise<string> {
        try {
            return new Promise((resolve, reject) => {
                const reader = new FileReader();
                reader.readAsDataURL(blob);
                reader.onloadend = () => resolve(reader.result as string);
                reader.onerror = error => reject(error);
            });
        } catch (error) {
            console.error('Error:', error);
            throw error;
        }
    }
    /**
     * Creates label boxes over text on the image to be filled in.
     * @param boxes
     * @param texts
     */
    async function createBoxes(img: ImageBox, boxes: number[][][], texts: string[]) {
        img.Document.quizBoxes = new List<Doc>([]);
        for (let i = 0; i < boxes.length; i++) {
            const coords = boxes[i] ? boxes[i] : [];
            const width = coords[1][0] - coords[0][0];
            const height = coords[2][1] - coords[0][1];
            const text = texts[i];

            const newCol = Docs.Create.LabelDocument({
                _width: width,
                _height: height,
                _layout_fitWidth: true,
                title: '',
            });
            const scaling = 1 / (img._props.NativeDimScaling?.() || 1);
            newCol.x = coords[0][0] + NumCast(img.marqueeref.current?.left) * scaling;
            newCol.y = coords[0][1] + NumCast(img.marqueeref.current?.top) * scaling;

            newCol.zIndex = 1000;
            newCol.forceActive = true;
            newCol.quiz = text;
            newCol['$' + Doc.LayoutDataKey(newCol) + '_transform'] = 'none';
            Doc.AddDocToList(img.Document, 'quizBoxes', newCol);
            img.addDocument(newCol);
            //   img._loading = false;
        }
    }

    /**
     * Calls backend to find any text on an image. Gets the text and the
     * coordinates of the text and creates label boxes at those locations.
     * @param quiz
     * @param i
     */
    async function pushInfo(imgBox: ImageBox, quiz: quizMode, i?: string) {
        imgBox.Document._quizMode = quiz;
        const quizBoxes = DocListCast(imgBox.Document.quizBoxes);
        if (!quizBoxes.length) {
            runInAction(() => (imgBox.Loading = true));

            const response = (await Networking.PostToServer('/labels', { file: i ? i : imgBox.paths[0], drag: i ? 'drag' : 'full', smart: quiz })) as { result: string };
            const replacedResponse = response.result.replace(/ '/g, '"').replace(/',/g, '",').replace(/\{'/g, '{"').replace(/':/g, '":').replace(/'\]/g, '"]').replace(/\['/g, '["');
            const parsedResponse = JSON.parse(replacedResponse) as { boxes: number[][][]; text: string[] };
            if (parsedResponse.boxes.length != 0) {
                createBoxes(imgBox, parsedResponse.boxes, parsedResponse.text);
            }
            runInAction(() => (imgBox.Loading = false));
        } else quizBoxes.forEach(box => (box.hidden = false));
    }

    async function createCanvas(img: ImageBox) {
        const canvas = document.createElement('canvas');
        const scaling = 1 / (img._props.NativeDimScaling?.() || 1);
        const w = AnchorMenu.Instance.marqueeWidth * scaling;
        const h = AnchorMenu.Instance.marqueeHeight * scaling;
        canvas.width = w;
        canvas.height = h;
        const ctx = canvas.getContext('2d'); // draw image to canvas. scale to target dimensions
        if (ctx) {
            img.imageRef && ctx.drawImage(img.imageRef, NumCast(img.marqueeref.current?.left) * scaling, NumCast(img.marqueeref.current?.top) * scaling, w, h, 0, 0, w, h);
        }
        const blob = await ImageUtility.canvasToBlob(canvas);
        return selectUrlToBase64(blob);
    }

    // /**
    //  * Create flashcards from an image.
    //  */
    // async function makeFlashcardsForImage(img: ImageBox) {
    //     img.Loading = true;
    //     try {
    //         const hrefBase64 = await createCanvas(img);
    //         const response = await gptImageLabel(hrefBase64, 'Make flashcards out of this image with each question and answer labeled as "question" and "answer". Do not label each flashcard and do not include asterisks: ');
    //         AnchorMenu.Instance.transferToFlashcard(response, NumCast(img.layoutDoc.x), NumCast(img.layoutDoc.y));
    //     } catch (error) {
    //         console.log('Error', error);
    //     }
    //     img.Loading = false;
    // }

    /**
     * Calls the createCanvas and pushInfo methods to convert the
     * image to a form that can be passed to GPT and find the locations
     * of the text.
     */
    async function makeLabels(img: ImageBox) {
        try {
            const hrefBase64 = await createCanvas(img);
            pushInfo(img, quizMode.NORMAL, hrefBase64);
        } catch (error) {
            console.log('Error', error);
        }
    }

    /**
     * Determines whether two words should be considered
     * the same, allowing minor typos.
     * @param str1
     * @param str2
     * @returns
     */
    function levenshteinDistance(str1: string, str2: string) {
        const len1 = str1.length;
        const len2 = str2.length;
        const dp = Array.from(Array(len1 + 1), () => Array(len2 + 1).fill(0));

        if (len1 === 0) return len2;
        if (len2 === 0) return len1;

        for (let i = 0; i <= len1; i++) dp[i][0] = i;
        for (let j = 0; j <= len2; j++) dp[0][j] = j;

        for (let i = 1; i <= len1; i++) {
            for (let j = 1; j <= len2; j++) {
                const cost = str1[i - 1] === str2[j - 1] ? 0 : 1;
                dp[i][j] = Math.min(
                    dp[i - 1][j] + 1, // deletion
                    dp[i][j - 1] + 1, // insertion
                    dp[i - 1][j - 1] + cost // substitution
                );
            }
        }

        return dp[len1][len2];
    }

    /**
     * Different algorithm for determining string similarity.
     * @param str1
     * @param str2
     * @returns
     */
    function jaccardSimilarity(str1: string, str2: string) {
        const set1 = new Set(str1.split(' '));
        const set2 = new Set(str2.split(' '));

        const intersection = new Set([...set1].filter(x => set2.has(x)));
        const union = new Set([...set1, ...set2]);

        return intersection.size / union.size;
    }

    /**
     * Averages the jaccardSimilarity and levenshteinDistance scores
     * to determine string similarity for the labelboxes answers and
     * the users response.
     * @param str1
     * @param str2
     * @returns
     */
    function stringSimilarity(str1: string, str2: string) {
        const levenshteinDist = levenshteinDistance(str1, str2);
        const levenshteinScore = 1 - levenshteinDist / Math.max(str1.length, str2.length);

        const jaccardScore = jaccardSimilarity(str1, str2);

        // Combine the scores with a higher weight on Jaccard similarity
        return 0.5 * levenshteinScore + 0.5 * jaccardScore;
    }
    /**
     * Returns whether two strings are similar
     * @param input
     * @param target
     * @returns
     */
    function compareWords(input: string, target: string) {
        const distance = stringSimilarity(input.toLowerCase(), target.toLowerCase());
        return distance >= 0.7;
    }

    /**
     * GPT returns a hex color for what color the label box should be based on
     * the correctness of the users answer.
     * @param inputString
     * @returns
     */
    function extractHexAndSentences(inputString: string) {
        // Regular expression to match a hexadecimal number at the beginning followed by a period and sentences
        const regex = /^#([0-9A-Fa-f]+)\.\s*(.+)$/;
        const match = inputString.replace('\n', ' ').match(regex);

        if (match) {
            const hexNumber = match[1];
            const sentences = match[2].trim();
            return { hexNumber, sentences };
        } else {
            return { error: 'The input string does not match the expected format.' };
        }
    }
    function imgQuizBoxes(img: ImageBox) {
        return DocListCast(img.Document.quizBoxes);
    }
    function imgQuizMode(img: ImageBox) {
        return StrCast(img.Document._quizMode);
    }

    /**
     * Check whether the contents of the label boxes on an image are correct.
     */
    function check(img: ImageBox) {
        //this._loading = true;
        imgQuizBoxes(img).forEach(async doc => {
            const input = StrCast(doc.$title);
            if (imgQuizMode(img) == quizMode.SMART && input) {
                const questionText = 'Question: What was labeled in this image?';
                const rubricText = ' Rubric: ' + StrCast(doc.quiz);
                const queryText =
                    questionText +
                    ' UserAnswer: ' +
                    input +
                    '. ' +
                    rubricText +
                    '. One sentence and evaluate based on meaning, not wording. Provide a hex color at the beginning with a period after it on a scale of green (minor details missed) to red (big error) for how correct the answer is. Example: "#FFFFFF. Pasta is delicious."';
                const response = await gptAPICall(queryText, GPTCallType.QUIZDOC);
                const hexSent = extractHexAndSentences(response);
                doc.quiz = hexSent.sentences?.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric');
                doc.backgroundColor = '#' + hexSent.hexNumber;
            } else {
                const match = compareWords(input, StrCast(doc.quiz).trim());
                if (input) {
                    doc.backgroundColor = match ? '#11c249' : '#eb2d2d';
                }
            }
        });
        //this._loading = false;
    }

    function redo(img: ImageBox) {
        imgQuizBoxes(img).forEach(doc => {
            doc.$title = '';
            doc.$backgroundColor = '#e4e4e4';
        });
    }

    /**
     * Get rid of all the label boxes on the images.
     */
    function exitQuizMode(img: ImageBox) {
        img.Document._quizMode = quizMode.NONE;
        DocListCast(img.Document.quizBoxes).forEach(box => {
            box.hidden = true;
        });
    }

    export function quizStyleProvider(doc: Opt<Doc>, props: Opt<FieldViewProps & DocumentViewProps>, property: string) {
        const editLabelAnswer = (qdoc: Doc) => {
            // when click the pencil, set the text to the quiz content. when click off, set the quiz text to that and set textbox to nothing.
            if (!qdoc._editLabel) {
                qdoc.title = StrCast(qdoc.quiz);
            } else {
                qdoc.quiz = StrCast(qdoc.title);
                qdoc.title = '';
            }
            qdoc._editLabel = !qdoc._editLabel;
        };
        const editAnswer = (qdoc: Opt<Doc>) => {
            return (
                <Tooltip
                    title={
                        <div className="answer-tooltip" style={{ minWidth: '150px' }}>
                            {qdoc?._editLabel ? 'save' : 'edit correct answer'}
                        </div>
                    }>
                    <div className="answer-tool-tip" onPointerDown={e => setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => qdoc && editLabelAnswer(qdoc))}>
                        <FontAwesomeIcon className="edit-icon" color={qdoc?._editLabel ? 'white' : 'black'} icon="pencil" size="sm" />
                    </div>
                </Tooltip>
            );
        };
        const answerIcon = (qdoc: Opt<Doc>) => {
            return (
                <Tooltip
                    title={
                        <div className="answer-tooltip" style={{ minWidth: '150px' }}>
                            {StrCast(qdoc?.quiz ?? '')}
                        </div>
                    }>
                    <div className="answer-tool-tip">
                        <FontAwesomeIcon className="q-icon" icon="circle" color="white" />
                        <FontAwesomeIcon className="answer-icon" icon="question" />
                    </div>
                </Tooltip>
            );
        };
        const checkIcon = (img: ImageBox) => (
            <Tooltip title={<div className="dash-tooltip">Check</div>}>
                <div className="check-icon" onPointerDown={() => check(img)}>
                    <FontAwesomeIcon icon="circle-check" size="lg" />
                </div>
            </Tooltip>
        );
        const redoIcon = (img: ImageBox) => (
            <Tooltip title={<div className="dash-tooltip">Redo</div>}>
                <div className="redo-icon" onPointerDown={() => redo(img)}>
                    <FontAwesomeIcon icon="redo-alt" size="lg" />
                </div>
            </Tooltip>
        );

        const imgBox = props?.DocumentView?.().ComponentView as ImageBox;
        switch (property) {
            case StyleProp.Decorations:
                {
                    if (doc?.quiz) {
                        // this should only be set on Labels that are part of an image quiz
                        return (
                            <>
                                {editAnswer(doc?.[DocData])}
                                {answerIcon(doc)}
                            </>
                        );
                    } else if (imgBox?.Document._quizMode && imgBox.Document._quizMode !== quizMode.NONE) {
                        return (
                            <>
                                {checkIcon(imgBox)}
                                {redoIcon(imgBox)}
                            </>
                        );
                    }
                }
                break;
            case StyleProp.ContextMenuItems:
                if (imgBox) {
                    const quizes: ContextMenuProps[] = [];
                    quizes.push({
                        description: 'Smart Check',
                        event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.SMART) : () => exitQuizMode(imgBox),
                        icon: 'pen-to-square',
                    });
                    quizes.push({
                        description: 'Normal',
                        event: doc?.quizMode == quizMode.NONE ? () => pushInfo(imgBox, quizMode.NORMAL) : () => exitQuizMode(imgBox),
                        icon: 'pencil',
                    });
                    ContextMenu.Instance?.addItem({ description: 'Quiz Mode', subitems: quizes, icon: 'file-pen' });
                }
                break;
            case StyleProp.AnchorMenuItems:
                AnchorMenu.Instance.makeLabels = imgBox ? () => makeLabels(props?.DocumentView?.().ComponentView as ImageBox) : unimplementedFunction;
        }
        return undefined;
    }
}