diff options
Diffstat (limited to 'src/client/views/nodes/ComparisonBox.tsx')
| -rw-r--r-- | src/client/views/nodes/ComparisonBox.tsx | 758 |
1 files changed, 566 insertions, 192 deletions
diff --git a/src/client/views/nodes/ComparisonBox.tsx b/src/client/views/nodes/ComparisonBox.tsx index 3e1c415d6..139978a13 100644 --- a/src/client/views/nodes/ComparisonBox.tsx +++ b/src/client/views/nodes/ComparisonBox.tsx @@ -1,4 +1,5 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { MathJax, MathJaxContext } from 'better-react-mathjax'; import { Tooltip } from '@mui/material'; import { action, computed, makeObservable, observable, reaction } from 'mobx'; import { observer } from 'mobx-react'; @@ -9,7 +10,7 @@ import { Doc, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { RichTextField } from '../../../fields/RichTextField'; import { DocCast, NumCast, RTFCast, StrCast, toList } from '../../../fields/Types'; -import { GPTCallType, gptAPICall } from '../../apis/gpt/GPT'; +import { GPTCallType, gptAPICall, gptImageLabel } from '../../apis/gpt/GPT'; import '../pdf/GPTPopup/GPTPopup.scss'; import { DocUtils } from '../../documents/DocUtils'; import { DocumentType } from '../../documents/DocumentTypes'; @@ -26,37 +27,136 @@ import { FieldView, FieldViewProps } from './FieldView'; import { FormattedTextBox } from './formattedText/FormattedTextBox'; import ReactLoading from 'react-loading'; -enum RevealOp { - Hover = 'hover', - Flip = 'flip', -} -enum UsePath { - Alternate = 'alternate', -} +import { ContextMenu } from '../ContextMenu'; +import { ContextMenuProps } from '../ContextMenuItem'; +import { tickStep } from 'd3'; +import { CollectionCarouselView } from '../collections/CollectionCarouselView'; +import { FollowLinkScript } from '../../documents/DocUtils'; +import { schema } from '../nodes/formattedText/schema_rts'; +import { Id } from '../../../fields/FieldSymbols'; +import axios from 'axios'; +import ReactMarkdown from 'react-markdown'; +import { WebField, nullAudio } from '../../../fields/URLField'; + +const API_URL = 'https://api.unsplash.com/search/photos'; @observer export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ComparisonBox, fieldKey); } private _disposers: (DragManager.DragDropDisposer | undefined)[] = [undefined, undefined]; - private _closeRef = React.createRef<HTMLDivElement>(); - @observable _inputValue = ''; - @observable _outputValue = ''; - @observable _loading = false; - @observable _errorMessage = ''; - @observable _outputMessage = ''; - @observable _animating = ''; - @observable private _isEmpty = false; - @observable _yRelativeToTop: boolean = true; - constructor(props: FieldViewProps) { super(props); makeObservable(this); + this.setListening(); } - @action handleInputChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => { - this._inputValue = e.target.value; - console.log(this._inputValue); + + @observable private _inputValue = ''; + @observable private _outputValue = ''; + @observable private _loading = false; + @observable private _isEmpty = false; + @observable private _audio: Doc = Docs.Create.TextDocument(''); + @observable childActive = false; + @observable _yRelativeToTop: boolean = true; + @observable _animating = ''; + @observable mathJaxConfig = { + loader: { load: ['input/asciimath'] }, }; + private _ref = React.createRef<HTMLDivElement>(); + + get revealOp() { + return this.layoutDoc[`_${this._props.fieldKey}_revealOp`]; + } + get clipHeightKey() { + return '_' + this._props.fieldKey + '_clipHeight'; + } + + get clipWidthKey() { + return '_' + this._props.fieldKey + '_clipWidth'; + } + + @computed get clipWidth() { + return NumCast(this.layoutDoc[this.clipWidthKey], 50); + } + + @computed get clipHeight() { + return NumCast(this.layoutDoc[this.clipHeightKey], 200); + } + + @computed get overlayAlternateIcon() { + const usepath = this.layoutDoc[`_${this._props.fieldKey}_usePath`]; + return ( + <Tooltip title={<div className="dash-tooltip">flip</div>}> + <div + className="formattedTextBox-alternateButton" + onPointerDown={e => + setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => { + if (!this.layoutDoc[`_${this._props.fieldKey}_revealOp`] || this.layoutDoc[`_${this._props.fieldKey}_revealOp`] === 'flip') { + this.flipFlashcard(); + + // console.log('Print Front of cards: ' + (RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text ?? '')); + // console.log('Print Back of cards: ' + (RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text ?? '')); + } + }) + } + style={{ + background: this.revealOp === 'hover' ? 'gray' : usepath === 'alternate' ? 'white' : 'black', + color: this.revealOp === 'hover' ? 'black' : usepath === 'alternate' ? 'black' : 'white', + display: 'inline-block', + }}> + <div key="alternate" className="formattedTextBox-flip"> + <FontAwesomeIcon icon="turn-up" size="1x" /> + </div> + </div> + </Tooltip> + ); + } + + @computed get flashcardMenu() { + return ( + <div> + <Tooltip + title={ + this.layoutDoc[`_${this._props.fieldKey}_usePath`] === 'alternate' ? ( + <div className="dash-tooltip">Flip to front side to use GPT</div> + ) : ( + <div className="dash-tooltip">Ask GPT to create an answer on the back side of the flashcard based on your question on the front</div> + ) + }> + <div style={{ position: 'absolute', bottom: '3px', right: '50px', cursor: 'pointer' }} onPointerDown={e => (!this.layoutDoc[`_${this._props.fieldKey}_usePath`] ? this.findImageTags() : null)}> + <FontAwesomeIcon icon="lightbulb" size="xl" /> + </div> + </Tooltip> + {DocCast(this.Document.embedContainer).type_collection === 'carousel' ? null : ( + <div> + <Tooltip title={<div>Create a flashcard pile</div>}> + <div style={{ position: 'absolute', bottom: '3px', right: '74px', cursor: 'pointer' }} onPointerDown={e => this.createFlashcardPile([this.Document], false)}> + <FontAwesomeIcon icon="folder-plus" size="xl" /> + </div> + </Tooltip> + <Tooltip title={<div className="dash-tooltip">Create new flashcard stack based on text</div>}> + <div style={{ position: 'absolute', bottom: '3px', right: '104px', cursor: 'pointer' }} onClick={e => this.gptFlashcardPile()}> + <FontAwesomeIcon icon="layer-group" size="xl" /> + </div> + </Tooltip> + </div> + )} + <Tooltip title={<div className="dash-tooltip">Hover to reveal</div>}> + <div style={{ position: 'absolute', bottom: '3px', right: '25px', cursor: 'pointer' }} onClick={e => this.handleHover()}> + <FontAwesomeIcon color={this.revealOp === 'hover' ? 'blue' : 'black'} icon="hand-point-up" size="xl" /> + </div> + </Tooltip> + {/* <Tooltip title={<div className="dash-tooltip">Remove this side of the flashcard</div>}> + <div + style={{ position: 'absolute', bottom: '3px', right: '80px', cursor: 'pointer' }} + onPointerDown={e => this.closeDown(e, this.layoutDoc[`_${this._props.fieldKey}_usePath`] === 'alternate' ? this._props.fieldKey + '_1' : this._props.fieldKey + '_0')}> + <FontAwesomeIcon color={this.revealOp === 'hover' ? 'blue' : 'black'} icon="trash-can" size="xl" /> + </div> + </Tooltip> */} + {/* {this.overlayAlternateIcon} */} + </div> + ); + } componentDidMount() { this._props.setContentViewBox?.(this); @@ -65,28 +165,15 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() selected => !selected && (this.childActive = false) // what it should update to ); } - protected createDropTarget = (ele: HTMLDivElement | null, fieldKey: string, disposerId: number) => { this._disposers[disposerId]?.(); + // this.childActive = true; if (ele) { this._disposers[disposerId] = DragManager.MakeDropTarget(ele, (e, dropEvent) => this.internalDrop(e, dropEvent, fieldKey), this.layoutDoc); } }; - @computed get useAlternate() { return this.layoutDoc[`_${this.fieldKey}_usePath`] === UsePath.Alternate; } // prettier-ignore - @computed get revealOp() { return this.layoutDoc[`_${this.fieldKey}_revealOp`] as Opt<RevealOp>; } // prettier-ignore - @computed get clipWidth() { return NumCast(this.layoutDoc[`_${this.fieldKey}_clipWidth`], 50); } // prettier-ignore - set useAlternate(alt: boolean) { this.layoutDoc[`_${this.fieldKey}_usePath`] = alt ? UsePath.Alternate : undefined; } // prettier-ignore - set revealOp(op: Opt<RevealOp>){ this.layoutDoc[`_${this.fieldKey}_revealOp`] = op; } // prettier-ignore - set clipWidth(width: number) { this.layoutDoc[`_${this.fieldKey}_clipWidth`] = width; } // prettier-ignore - - animateClipWidth = action((clipWidth: number, duration = 200 /* ms */) => { - this._animating = `all ${duration}ms`; // turn on clip animation transition, then turn it off at end of animation - setTimeout(action(() => { this._animating = ''; }), duration); // prettier-ignore - this.clipWidth = clipWidth; - }); - - internalDrop = undoable((e: Event, dropEvent: DragManager.DropEvent, fieldKey: string) => { + private internalDrop = undoable((e: Event, dropEvent: DragManager.DropEvent, fieldKey: string) => { if (dropEvent.complete.docDragData) { const { droppedDocuments } = dropEvent.complete.docDragData; const added = dropEvent.complete.docDragData.moveDocument?.(droppedDocuments, this.Document, (doc: Doc | Doc[]) => this.addDoc(toList(doc).lastElement(), fieldKey)); @@ -99,38 +186,162 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() return undefined; }, 'internal drop'); - registerSliding = (e: React.PointerEvent<HTMLDivElement>, targetWidth: number) => { + private registerSliding = (e: React.PointerEvent<HTMLDivElement>, targetWidth: number) => { if (e.button !== 2) { setupMoveUpEvents( this, e, this.onPointerMove, emptyFunction, - action((clickEv, doubleTap) => { + action((moveEv, doubleTap) => { if (doubleTap) { this.childActive = true; if (!this.dataDoc[this.fieldKey + '_1'] && !this.dataDoc[this.fieldKey]) this.dataDoc[this.fieldKey + '_1'] = DocUtils.copyDragFactory(Doc.UserDoc().emptyNote as Doc); if (!this.dataDoc[this.fieldKey + '_2'] && !this.dataDoc[this.fieldKey + '_alternate']) this.dataDoc[this.fieldKey + '_2'] = DocUtils.copyDragFactory(Doc.UserDoc().emptyNote as Doc); - // DocumentView.addViewRenderedCb(DocCast(this.dataDoc[this.fieldKey + '_1']), dv => { - // dv?.select(false); - // }); } }), - true, + false, undefined, - () => !this._isAnyChildContentActive && this.animateClipWidth((targetWidth * 100) / this._props.PanelWidth()) + action(() => { + if (this.childActive) return; + this._animating = 'all 200ms'; + // on click, animate slider movement to the targetWidth + this.layoutDoc[this.clipWidthKey] = (targetWidth * 100) / this._props.PanelWidth(); + // this.layoutDoc[this.clipHeightKey] = (targetWidth * 100) / this._props.PanelHeight(); + + setTimeout( + action(() => { + this._animating = ''; + }), + 200 + ); + }) ); } }; - onPointerMove = ({ movementX }: PointerEvent) => { + // private onClick(e: React.PointerEvent<HTMLDivElement>) { + // setupMoveUpEvents( + // this, e, this.onPointerMOve, emptyFunction(), () => {this._isAnyChildContentActive = true;}, emptyFunction(), emptyFunction() + // ) + // } + + @action + private onPointerMove = ({ movementX }: PointerEvent) => { const width = movementX * this.ScreenToLocalBoxXf().Scale + (this.clipWidth / 100) * this._props.PanelWidth(); - if (width > 5 && width < this._props.PanelWidth()) { - this.clipWidth = (width * 100) / this._props.PanelWidth(); + if (width && width > 5 && width < this._props.PanelWidth()) { + this.layoutDoc[this.clipWidthKey] = (width * 100) / this._props.PanelWidth(); } return false; }; + @action handleInputChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => { + this._inputValue = e.target.value; + console.log(this._inputValue); + }; + + // this.closeDown(e, this.layoutDoc[`_${this._props.fieldKey}_usePath`] === 'alternate' ? this.fieldKey + '_0' : this.fieldKey + '_1')} + @action activateContent = () => { + this.childActive = true; + }; + + @action handleRenderGPTClick = async () => { + // Call the GPT model and get the output + // await this.pushInfo(); + // console.log('PHONETIC TRANSCRIPTION: ' + DocCast(this._audio)[DocData]); + // this.Document.audio = this._audio; + console.log('Phonetic transcription: ' + DocCast(this.Document.audio).phoneticTranscription); + const phonTrans = DocCast(this.Document.audio).phoneticTranscription; + // const phonTrans = 's'; + if (phonTrans) { + // console.log(phonTrans.toString()); + this._inputValue = StrCast(phonTrans); + console.log('INPUT:' + this._inputValue); + this.askGPTPhonemes(this._inputValue); + } else if (this._inputValue) this.askGPT(GPTCallType.QUIZ); + this.layoutDoc[`_${this._props.fieldKey}_usePath`] = 'alternate'; + this._outputValue = ''; + }; + + askGPTPhonemes = async (phonemes: string) => { + const sentence = StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text); + const phon = 'w ʌ ɪ z j ɔː ɹ n e ɪ m '; + const phon2 = 'h ʌ ɛ r j ʌ t ʌ d eɪ'; + const phon3 = 'ʃ eɪ oʊ s i ʃ oʊ z b aɪ ð ə s iː ʃ oʊ'; + const phon4 = 'kamo estas hɔi'; + const phon5 = 'la s e n a l'; + console.log('REG' + this.recognition.lang); + const promptEng = + 'Consider all possible phonetic transcriptions of the intended sentence "' + + sentence + + '" that is standard in American speech without showing the user. Compare each word in the following phonemes with those phonetic transcriptions without displaying anything to the user: "' + + phonemes + + '". Steps to do this: Align the words with each word in the intended sentence by combining the phonemes to get a pronunciation that resembles the word in order. Do not describe phonetic corrections with the phonetic alphabet - describe it by providing other examples of how it should sound. Note if a word or sound missing, including missing vowels and consonants. If there is an additional word that does not match with the provided sentence, say so. For each word, if any letters mismatch and would sound weird in American speech and they are not allophones of the same phoneme and they are far away from each on the ipa vowel chat and that pronunciation is not normal for the meaning of the word, note this difference and explain how it is supposed to sound. Only note the difference if they are not allophones of the same phoneme and if they are far away on the vowel chart. The goal is to be understood, not sound like a native speaker. Just so you know, "i" sounds like "ee" as in "bee", not "ih" as an "lick". Interpret "ɹ" as the same as "r". Interpret "ʌ" as the same as "ə". If "ɚ", "ɔː", and "ɔ" are options for pronunciation, do not choose "ɚ". Ignore differences with colons. Ignore redundant letters and words and sounds and the splitting of words; do not mention this since there could be repeated words in the sentence. Provide a response like this: "Lets work on improving the pronunciation of "coffee." You said "ceeffee," which is close, but we need to adjust the vowel sound. In American English, "coffee" is pronounced /ˈkɔːfi/, with a long "aw" sound. Try saying "kah-fee." Your intonation is good, but try putting a bit more stress on "like" in the sentence "I would like a coffee with milk." This will make your speech sound more natural. Keep practicing, and lets try saying the whole sentence again!"'; + const promptSpa = + 'Consider all possible phonetic transcriptions of the intended sentence "' + + 'como estás hoy' + + '" that is standard in Spanish speech without showing the user. Compare each word in the following phonemes with those phonetic transcriptions without displaying anything to the user: "' + + phon4 + + '". Steps to do this: Align the words with each word in the intended sentence by combining the phonemes to get a pronunciation that resembles the word in order. Do not describe phonetic corrections with the phonetic alphabet - describe it by providing other examples of how it should sound. Note if a word or sound missing, including missing vowels and consonants. If there is an additional word that does not match with the provided sentence, say so. For each word, if any letters mismatch and would sound weird in Spanish speech and they are not allophones of the same phoneme and they are far away from each on the ipa vowel chat and that pronunciation is not normal for the meaning of the word, note this difference and explain how it is supposed to sound. Only note the difference if they are not allophones of the same phoneme and if they are far away on the vowel chart; say good job if it would be understood by a native Spanish speaker. Just so you know, "i" sounds like "ee" as in "bee", not "ih" as an "lick". Interpret "ɹ" as the same as "r". Interpret "ʌ" as the same as "ə". Do not make "θ" and "f" interchangable. Do not make "n" and "ɲ" interchangable. Do not make "e" and "i" interchangable. If "ɚ", "ɔː", and "ɔ" are options for pronunciation, do not choose "ɚ". Ignore differences with colons. Ignore redundant letters and words and sounds and the splitting of words; do not mention this since there could be repeated words in the sentence. Identify "ɔi" sounds like "oy". Ignore accents and do not say anything to the user about this.'; + const promptAll = + 'Consider all possible phonetic transcriptions of the intended sentence "' + + sentence + + '" that is standard in ' + + this.convertAbr() + + ' speech without showing the user. Compare each word in the following phonemes with those phonetic transcriptions without displaying anything to the user: "' + + phonemes + + '". Steps to do this: Align the words with each word in the intended sentence by combining the phonemes to get a pronunciation that resembles the word in order. Do not describe phonetic corrections with the phonetic alphabet - describe it by providing other examples of how it should sound. Note if a word or sound missing, including missing vowels and consonants. If there is an additional word that does not match with the provided sentence, say so. For each word, if any letters mismatch and would sound weird in ' + + this.convertAbr() + + ' speech and they are not allophones of the same phoneme and they are far away from each on the ipa vowel chat and that pronunciation is not normal for the meaning of the word, note this difference and explain how it is supposed to sound. Just so you know, "i" sounds like "ee" as in "bee", not "ih" as an "lick". Interpret "ɹ" as the same as "r". Interpret "ʌ" as the same as "ə". Do not make "θ" and "f" interchangable. Do not make "n" and "ɲ" interchangable. Do not make "e" and "i" interchangable. If "ɚ", "ɔː", and "ɔ" are options for pronunciation, do not choose "ɚ". Ignore differences with colons. Ignore redundant letters and words and sounds and the splitting of words; do not mention this since there could be repeated words in the sentence. Provide a response like this: "Lets work on improving the pronunciation of "coffee." You said "cawffee," which is close, but we need to adjust the vowel sound. In American English, "coffee" is pronounced /ˈkɔːfi/, with a long "aw" sound. Try saying "kah-fee." Your intonation is good, but try putting a bit more stress on "like" in the sentence "I would like a coffee with milk." This will make your speech sound more natural. Keep practicing, and lets try saying the whole sentence again!"'; + + switch (this.recognition.lang) { + case 'en-US': + console.log('English'); + this._outputValue = await gptAPICall(promptEng, GPTCallType.PRONUNCIATION); + break; + case 'es-ES': + console.log('Spanish'); + this._outputValue = await gptAPICall(promptSpa, GPTCallType.PRONUNCIATION); + break; + default: + console.log('All'); + this._outputValue = await gptAPICall(promptAll, GPTCallType.PRONUNCIATION); + break; + } + }; + + pushInfo = async () => { + const formData = new FormData(); + + console.log(DocCast(this._audio).dataDoc); + const audio = { + file: this._audio.url, + }; + const response = await axios.post('http://localhost:105/recognize/', audio, { + headers: { + 'Content-Type': 'application/json', + }, + }); + this.Document.phoneticTranscription = response.data['transcription']; + console.log('RESPONSE: ' + response.data['transcription']); + }; + + @action handleHover = () => { + if (this.revealOp === 'hover') { + this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'flip'; + this.Document.forceActive = false; + } else { + this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'hover'; + this.Document.forceActive = true; + } + //this.revealOp === 'hover' ? (this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'flip') : (this.layoutDoc[`_${this._props.fieldKey}_revealOp`] = 'hover'); + }; + + @action handleRenderClick = () => { + // Call the GPT model and get the output + this.layoutDoc[`_${this._props.fieldKey}_usePath`] = undefined; + }; + getAnchor = (addAsAnnotation: boolean, pinProps?: PinProps) => { const anchor = Docs.Create.ConfigDocument({ title: 'CompareAnchor:' + this.Document.title, @@ -190,10 +401,8 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() ); }; docStyleProvider = (doc: Opt<Doc>, props: Opt<FieldViewProps>, property: string): any => { - switch (property) { - case StyleProp.PointerEvents: return 'none'; - default: return this._props.styleProvider?.(doc, props, property); - } // prettier-ignore + if (property === StyleProp.PointerEvents) return 'none'; + return this._props.styleProvider?.(doc, props, property); }; moveDoc1 = (docs: Doc | Doc[], targetCol: Doc | undefined, addDoc: any) => toList(docs).reduce((res, doc: Doc) => res && this.moveDoc(doc, addDoc, this.fieldKey + '_1'), true); moveDoc2 = (docs: Doc | Doc[], targetCol: Doc | undefined, addDoc: any) => toList(docs).reduce((res, doc: Doc) => res && this.moveDoc(doc, addDoc, this.fieldKey + '_2'), true); @@ -232,6 +441,8 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() return layoutTemplateString; }; + _closeRef = React.createRef<HTMLDivElement>(); + createFlashcardPile(collectionArr: Doc[], gpt: boolean) { const newCol = Docs.Create.CarouselDocument(collectionArr, { _width: NumCast(this.layoutDoc['_' + this._props.fieldKey + '_width'], 250) + 50, @@ -255,13 +466,48 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() } gptFlashcardPile = async () => { - var text = await this.askGPT(GPTCallType.FLASHCARD); + var text = await this.askGPT(GPTCallType.STACK); + console.log(text); var senArr = text?.split('Question: '); var collectionArr: Doc[] = []; for (let i = 1; i < senArr?.length!; i++) { const newDoc = Docs.Create.ComparisonDocument(senArr![i], { _layout_isFlashcard: true, _width: 300, _height: 300 }); - newDoc.text = senArr![i]; + + if (StrCast(senArr![i]).includes('Keyword: ')) { + const question = StrCast(senArr![i]).split('Keyword: '); + const img = await this.fetchImages(question[1]); + // newDoc['image'] = img; + // const newDoc = Docs.Create.TextDocument(dataSplit[1]); + const textSide1 = question[0].includes('Answer: ') ? question[0].split('Answer: ')[0] : question[0]; + const textDoc1 = Docs.Create.TextDocument(question[0]); + const rtfiel = new RichTextField( + JSON.stringify({ + doc: { + type: 'doc', + content: [ + { + type: 'paragraph', + attrs: { align: null, color: null, id: null, indent: null, inset: null, lineSpacing: null, paddingBottom: null, paddingTop: null }, + content: [ + { type: 'text', text: question[0].includes('Answer: ') ? question[0].split('Answer: ')[0] : question[0] }, + { type: 'dashDoc', attrs: { width: '200px', height: '200px', title: 'dashDoc', float: 'unset', hidden: false, docId: img![Id] } }, + ], + }, + ], + }, + selection: { type: 'text', anchor: 2, head: 2 }, + }), + textSide1 + ); + + textDoc1[DocData].text = rtfiel; + DocCast(newDoc)[DocData][this.fieldKey + '_1'] = textDoc1; + + DocCast(newDoc)[DocData][this.fieldKey + '_0'] = Docs.Create.TextDocument(question[0].includes('Answer: ') ? question[0].split('Answer: ')[1] : question[1]); + // Doc.AddToMyOverlay(img!); + } + collectionArr.push(newDoc); } this.createFlashcardPile(collectionArr, true); @@ -271,109 +517,15 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() * Flips a flashcard to the alternate side for the user to view. */ flipFlashcard = () => { - this.useAlternate = !this.useAlternate; + const usePath = this.layoutDoc[`_${this._props.fieldKey}_usePath`]; + this.layoutDoc[`_${this._props.fieldKey}_usePath`] = usePath === undefined ? 'alternate' : undefined; }; /** * Changes the view option to hover for a flashcard. */ - hoverFlip = (alternate: boolean) => { - if (this.revealOp === RevealOp.Hover) this.useAlternate = alternate; - }; - /** - * Creates the button used to flip the flashcards. - */ - @computed get overlayAlternateIcon() { - return ( - <Tooltip title={<div className="dash-tooltip">flip</div>}> - <div - className="formattedTextBox-alternateButton" - onPointerDown={e => - setupMoveUpEvents(e.target, e, returnFalse, emptyFunction, () => { - if (!this.revealOp || this.revealOp === RevealOp.Flip) { - this.flipFlashcard(); - - // console.log('Print Front of cards: ' + (RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text ?? '')); - // console.log('Print Back of cards: ' + (RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text ?? '')); - } - }) - } - style={{ - background: this.useAlternate ? 'white' : 'black', - color: this.useAlternate ? 'black' : 'white', - }}> - <div key="alternate" className="formattedTextBox-flip"> - <FontAwesomeIcon icon="turn-up" size="1x" /> - </div> - </div> - </Tooltip> - ); - } - - @computed get flashcardMenu() { - return ( - <div> - <Tooltip - title={this.useAlternate ? <div className="dash-tooltip">Flip to front side to use GPT</div> : <div className="dash-tooltip">Ask GPT to create an answer on the back side of the flashcard based on your question on the front</div>}> - <div style={{ position: 'absolute', bottom: '3px', right: '50px', cursor: 'pointer' }} onPointerDown={e => (!this.layoutDoc[`_${this._props.fieldKey}_usePath`] ? this.askGPT(GPTCallType.CHATCARD) : null)}> - <FontAwesomeIcon icon="lightbulb" size="xl" /> - </div> - </Tooltip> - {DocCast(this.Document.embedContainer).type_collection === 'carousel' ? null : ( - <div> - <Tooltip title={<div>Create a flashcard pile</div>}> - <div style={{ position: 'absolute', bottom: '3px', right: '74px', cursor: 'pointer' }} onPointerDown={e => this.createFlashcardPile([this.Document], false)}> - <FontAwesomeIcon icon="folder-plus" size="xl" /> - </div> - </Tooltip> - <Tooltip title={<div className="dash-tooltip">Create new flashcard stack based on text</div>}> - <div style={{ position: 'absolute', bottom: '3px', right: '104px', cursor: 'pointer' }} onClick={e => this.gptFlashcardPile()}> - <FontAwesomeIcon icon="layer-group" size="xl" /> - </div> - </Tooltip> - </div> - )} - <Tooltip title={<div className="dash-tooltip">Hover to reveal</div>}> - <div style={{ position: 'absolute', bottom: '3px', right: '25px', cursor: 'pointer' }} onClick={e => this.handleHover()}> - <FontAwesomeIcon color={this.revealOp === RevealOp.Hover ? 'blue' : 'black'} icon="hand-point-up" size="xl" /> - </div> - </Tooltip> - {/* <Tooltip title={<div className="dash-tooltip">Remove this side of the flashcard</div>}> - <div - style={{ position: 'absolute', bottom: '3px', right: '80px', cursor: 'pointer' }} - onPointerDown={e => this.closeDown(e, this.layoutDoc[`_${this._props.fieldKey}_usePath`] === 'alternate' ? this._props.fieldKey + '_1' : this._props.fieldKey + '_0')}> - <FontAwesomeIcon color={this.revealOp === 'hover' ? 'blue' : 'black'} icon="trash-can" size="xl" /> - </div> - </Tooltip> */} - {/* {this.overlayAlternateIcon} */} - </div> - ); - } - // this.closeDown(e, this.layoutDoc[`_${this._props.fieldKey}_usePath`] === 'alternate' ? this.fieldKey + '_0' : this.fieldKey + '_1')} - @action activateContent = () => { - this.childActive = true; - }; - - @action handleRenderGPTClick = () => { - // Call the GPT model and get the output - this.useAlternate = true; - this._outputValue = ''; - if (this._inputValue) this.askGPT(GPTCallType.QUIZ); - }; - - @action handleHover = () => { - if (this.revealOp === RevealOp.Hover) { - this.revealOp = RevealOp.Flip; - this.Document.forceActive = false; - } else { - this.revealOp = RevealOp.Hover; - this.Document.forceActive = true; - } - }; - - @action handleRenderClick = () => { - // Call the GPT model and get the output - this.useAlternate = false; + hoverFlip = (side: string | undefined) => { + if (this.layoutDoc[`_${this._props.fieldKey}_revealOp`] === 'hover') this.layoutDoc[`_${this._props.fieldKey}_usePath`] = side; }; animateRes = (resIndex: number, newText: string, callType: GPTCallType) => { @@ -422,15 +574,20 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() if (callType == GPTCallType.CHATCARD) { DocCast(this.dataDoc[this.props.fieldKey + '_0'])[DocData].text = res; // this.flipFlashcard(); + } else if (callType == GPTCallType.QUIZ) { + console.log(this._inputValue); + this._outputValue = res.replace(/UserAnswer/g, "user's answer").replace(/Rubric/g, 'rubric'); } - if (callType == GPTCallType.QUIZ) this._outputValue = res; // DocCast(this.dataDoc[this.props.fieldKey + '_0'])[DocData].text = res; // this._outputValue = res; else if (callType === GPTCallType.FLASHCARD) { // console.log(res); this._loading = false; return res; + } else if (callType === GPTCallType.STACK) { } + this._loading = false; + return res; // console.log(res); } catch (err) { console.error('GPT call failed'); @@ -444,7 +601,182 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() // const cm = ContextMenu.Instance; // cm.addItem({ description: 'Create an Answer on the Back', event: () => this.askGPT(GPTCallType.CHATCARD), icon: 'pencil' }); // }; - @observable childActive = false; + + findImageTags = async () => { + // const d = DocCast(this.dataDoc[this.props.fieldKey + '_0']); + // const copy = Doc.MakeCopy(this.Document, true); + const c = this.DocumentView?.().ContentDiv!.getElementsByTagName('img'); + // this.ProseRef?.getElementsByTagName('img'); + if (c?.length === 0) await this.askGPT(GPTCallType.CHATCARD); + if (c) { + this._loading = true; + for (let i of c) { + console.log(i); + if (i.className !== 'ProseMirror-separator') await this.getImageDesc(i.src); + } + this._loading = false; + // this.flipFlashcard(); + } + // console.log('HI' + this.ProseRef?.getElementsByTagName('img')); + }; + + static imageUrlToBase64 = async (imageUrl: string): Promise<string> => { + try { + const response = await fetch(imageUrl); + const blob = await response.blob(); + + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => resolve(reader.result as string); + reader.onerror = error => reject(error); + }); + } catch (error) { + console.error('Error:', error); + throw error; + } + }; + + getImageDesc = async (u: string) => { + try { + const hrefBase64 = await ComparisonBox.imageUrlToBase64(u); + const response = await gptImageLabel(hrefBase64, 'Answer the following question as a short flashcard response. Do not include a label.' + (this.dataDoc.text as RichTextField)?.Text); + + DocCast(this.dataDoc[this.props.fieldKey + '_0'])[DocData].text = response; + } catch (error) { + console.log('Error'); + } + }; + + fetchImages = async (selection: string) => { + try { + const { data } = await axios.get(`${API_URL}?query=${selection}&page=1&per_page=${1}&client_id=Q4zruu6k6lum2kExiGhLNBJIgXDxD6NNj0SRHH_XXU0`); + console.log(data.results); + const imageSnapshot = Docs.Create.ImageDocument(data.results[0].urls.small, { + _nativeWidth: Doc.NativeWidth(this.layoutDoc), + _nativeHeight: Doc.NativeHeight(this.layoutDoc), + x: NumCast(this.layoutDoc.x), + y: NumCast(this.layoutDoc.y), + onClick: FollowLinkScript(), + _width: 150, + _height: 150, + title: '--snapshot' + NumCast(this.layoutDoc._layout_currentTimecode) + ' image-', + }); + imageSnapshot['x'] = this.layoutDoc['x']; + imageSnapshot['y'] = this.layoutDoc['y']; + return imageSnapshot; + } catch (error) { + console.log(error); + } + }; + + // handleSelection = async (selection: string, newDoc: Doc) => { + // const images = await this.fetchImages(selection); + // return images; + // // Doc.AddDocToList(Doc.MyRecentlyClosed, 'data', dashDoc, undefined, true, true); + // images!.embedContainer = newDoc; + // Doc.AddEmbedding(newDoc, images!); + // const c = this.DocumentView?.().ContentDiv!.getElementsByClassName('afterBox-cont'); + // for (let i in c) { + // console.log('HERE' + i); + // } + // this.addDoc(images!, this.fieldKey + '_0'); + // Doc.AddEmbedding(newDoc, images!); + // this._props. + // Doc.AddToMyOverlay(images!); + // const node = schema.nodes.dashDoc.create({ + // width: NumCast(images?._width), + // height: NumCast(images?._height), + // title: 'dashDoc', + // docId: images![Id], + // float: 'unset', + // }); + // }; + + @observable private _listening = false; + @observable transcriptElement = ''; + SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + @observable recognition = new this.SpeechRecognition(); + + handleResult = (e: SpeechRecognitionEvent) => { + let interimTranscript = ''; + let finalTranscript = ''; + for (let i = e.resultIndex; i < e.results.length; i++) { + const transcript = e.results[i][0].transcript; + if (e.results[i].isFinal) { + finalTranscript += transcript; + } else { + interimTranscript += transcript; + } + } + console.log(interimTranscript); + this._inputValue += finalTranscript; + }; + + setListening = () => { + const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + if (SpeechRecognition) { + this.recognition.continuous = true; + this.recognition.interimResults = true; + this.recognition.lang = 'en-US'; + this.recognition.onresult = this.handleResult.bind(this); + } + ContextMenu.Instance.setLangIndex(0); + }; + + setLanguage = (e: React.MouseEvent, language: string, ind: number) => { + this.recognition.lang = language; + ContextMenu.Instance.setLangIndex(ind); + }; + + startListening = () => { + this.recognition.start(); + this._listening = true; + }; + + stopListening = () => { + this.recognition.stop(); + this._listening = false; + }; + + convertAbr = () => { + switch (this.recognition.lang) { + case 'en-US': + return 'English'; + case 'es-ES': + return 'Spanish'; + case 'fr-FR': + return 'French'; + case 'it-IT': + return 'Italian'; + case 'zh-CH': + return 'Mandarin Chinese'; + case 'ja': + return 'Japanese'; + default: + return 'Korean'; + } + }; + + openContextMenu = (x: number, y: number, evalu: boolean) => { + ContextMenu.Instance.clearItems(); + ContextMenu.Instance.addItem({ description: 'English', event: e => this.setLanguage(e, 'en-US', 0) }); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'Spanish', event: e => this.setLanguage(e, 'es-ES', 1 )}); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'French', event: e => this.setLanguage(e, 'fr-FR', 2) }); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'Italian', event: e => this.setLanguage(e, 'it-IT', 3) }); //prettier-ignore + if (!evalu) ContextMenu.Instance.addItem({ description: 'Mandarin Chinese', event: e => this.setLanguage(e, 'zh-CH', 4) }); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'Japanese', event: e => this.setLanguage(e, 'ja', 5) }); //prettier-ignore + ContextMenu.Instance.addItem({ description: 'Korean', event: e => this.setLanguage(e, 'ko', 6) }); //prettier-ignore + ContextMenu.Instance.displayMenu(x, y); + }; + + evaluatePronunciation = () => { + const newAudio = Docs.Create.AudioDocument(nullAudio, { _width: 200, _height: 100 }); + this.Document.audio = newAudio[DocData]; + // DocCast(this.Document.embedContainer)()._props.addDocument?.(newAudio); + this._props.DocumentView?.()._props.addDocument?.(newAudio); + // Doc.AddToMyOverlay(newAudio); + }; render() { const clearButton = (which: string) => ( @@ -466,6 +798,8 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() // whichDoc['backgroundColor'] = this.layoutDoc['backgroundColor']; return targetDoc || layoutString ? ( + // <MathJaxContext config={this.mathJaxConfig}> + // <MathJax> <> <DocumentView // eslint-disable-next-line react/jsx-props-no-spreading @@ -487,8 +821,10 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() hideLinkButton pointerEvents={this.childActive ? undefined : returnNone} /> + {/* </MathJax> */} {/* <div style={{ position: 'absolute', top: '-5px', left: '2px' }}>{layoutString ? null : clearButton(whichSlot)}</div> */} - </> // placeholder image if doc is missingleft: `${NumCast(this.layoutDoc.width, 200) - 33}px` + {/* </MathJaxContext> // placeholder image if doc is missingleft: `${NumCast(this.layoutDoc.width, 200) - 33}px` */} + </> ) : ( <div className="placeholder"> <FontAwesomeIcon className="upload-icon" icon="cloud-upload-alt" size="lg" /> @@ -511,37 +847,43 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() ); if (this.Document._layout_isFlashcard) { - const side = this.useAlternate ? 1 : 0; + const side = this.layoutDoc[`_${this._props.fieldKey}_usePath`] === 'alternate' ? 1 : 0; // add text box to each side when comparison box is first created // (!this.dataDoc[this.fieldKey + '_0'] && this.dataDoc[this._props.fieldKey + '_0'] !== 'empty') if (!this.dataDoc[this.fieldKey + '_0'] && !this._isEmpty) { - const dataSplit = StrCast(this.dataDoc.data).split('Answer: '); + const dataSplit = StrCast(this.dataDoc.data).includes('Keyword: ') ? StrCast(this.dataDoc.data).split('Keyword: ') : StrCast(this.dataDoc.data).split('Answer: '); const newDoc = Docs.Create.TextDocument(dataSplit[1]); - // if there is text from the pdf ai cards, put the question on the front side. - // eslint-disable-next-line prefer-destructuring - // newDoc.text = dataSplit[1]; - newDoc['backgroundColor'] = 'lightgray'; this.addDoc(newDoc, this.fieldKey + '_0'); - // DocCast(this.dataDoc[this.fieldKey + '_0'])[DocData].text = dataSplit[1]; - // DocCast(this.dataDoc[this.fieldKey + '_0']).text = dataSplit[1]; - // console.log('HI' + DocCast(this.dataDoc[this.fieldKey + '_0']).text); - //console.log('HEREEE' + StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text)); } if (!this.dataDoc[this.fieldKey + '_1'] && !this._isEmpty) { - const dataSplit = StrCast(this.dataDoc.data).split('Answer: '); + const dataSplit = StrCast(this.dataDoc.data).includes('Keyword: ') ? StrCast(this.dataDoc.data).split('Keyword: ') : StrCast(this.dataDoc.data).split('Answer: '); const newDoc = Docs.Create.TextDocument(dataSplit[0]); this.addDoc(newDoc, this.fieldKey + '_1'); - // if there is text from the pdf ai cards, put the answer on the alternate side. - // eslint-disable-next-line prefer-destructuring - - // newDoc[DocData].text = dataSplit[0]; - // console.log('HEREEE' + StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text)); - // console.log('HI' + DocCast(this.dataDoc[this.fieldKey + '_1']).text); - // DocCast(this.dataDoc[this.props.fieldKey + '_1'])[DocData].text = dataSplit[0]; - // console.log('HEREEE' + StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_0']).text)?.Text)); - // DocCast(this.dataDoc[this.fieldKey + '_1'])[DocData].text = dataSplit[0]; + // if (this.Document.image) { + // console.log('ID: ' + DocCast(this.Document.image)[Id]); + // const rtfiel = new RichTextField( + // JSON.stringify({ + // doc: { + // type: 'doc', + // content: [ + // { + // type: 'paragraph', + // attrs: { align: null, color: null, id: null, indent: null, inset: null, lineSpacing: null, paddingBottom: null, paddingTop: null }, + // content: [ + // { type: 'text', text: dataSplit[0] }, + // { type: 'dashDoc', attrs: { width: '200px', height: '200px', title: 'dashDoc', float: 'unset', hidden: false, docId: DocCast(this.Document.image)[Id] } }, + // ], + // }, + // ], + // }, + // selection: { type: 'text', anchor: 2, head: 2 }, + // }) + // ); + + // newDoc[DocData].text = rtfiel; + // } } // render the QuizCards @@ -554,12 +896,14 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() <p style={{ display: text === '' ? 'flex' : 'none', color: 'white', marginLeft: '10px' }}>Return to all flashcards and add text to both sides. </p> <div className="input-box"> <textarea - value={this.useAlternate ? this._outputValue : this._inputValue} + value={this.layoutDoc[`_${this._props.fieldKey}_usePath`] === 'alternate' ? this._outputValue : this._inputValue} onChange={this.handleInputChange} - onScroll={e => e.stopPropagation()} - placeholder={!this.useAlternate ? 'Enter a response for GPT to evaluate.' : ''} - readOnly={this.useAlternate} - /> + onScroll={e => { + e.stopPropagation(); + e.preventDefault(); + }} + placeholder={!this.layoutDoc[`_${this._props.fieldKey}_usePath`] ? 'Enter a response for GPT to evaluate.' : ''} + readOnly={this.layoutDoc[`_${this._props.fieldKey}_usePath`] === 'alternate'}></textarea> {this._loading ? ( <div className="loading-spinner" style={{ position: 'absolute' }}> @@ -567,15 +911,40 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() </div> ) : null} </div> - <div className="submit-button" style={{ overflow: 'hidden', display: this.useAlternate ? 'none' : 'flex' }}> - <button type="button" onClick={this.handleRenderGPTClick} style={{ borderRadius: '2px', marginBottom: '3px' }}> - Submit - </button> - </div> - <div className="submit-button" style={{ overflow: 'hidden', marginBottom: '2px', display: this.useAlternate ? 'flex' : 'none' }}> - <button type="button" onClick={this.handleRenderClick} style={{ borderRadius: '2px' }}> - Redo the Question - </button> + <div> + <div className="submit-button" style={{ overflow: 'hidden', display: 'flex', width: '100%' }}> + <div + className="submit-buttonschema-header-button" + onPointerDown={e => this.openContextMenu(e.clientX, e.clientY, false)} + style={{ position: 'absolute', top: '5px', left: '11px', zIndex: '100', width: '5px', height: '5px', cursor: 'pointer' }}> + <FontAwesomeIcon color={'white'} icon="caret-down" /> + </div> + <button className="submit-buttonrecord" onClick={this._listening ? this.stopListening : this.startListening} style={{ background: this._listening ? 'lightgray' : '', borderRadius: '2px' }}> + {<FontAwesomeIcon icon="microphone" size="lg" />} + </button> + <div + className="submit-buttonschema-header-button" + onPointerDown={e => this.openContextMenu(e.clientX, e.clientY, true)} + style={{ position: 'absolute', top: '5px', left: '50px', zIndex: '100', width: '5px', height: '5px', cursor: 'pointer' }}> + <FontAwesomeIcon color={'white'} icon="caret-down" /> + </div> + <button + className="submit-buttonpronunciation" + onClick={this.evaluatePronunciation} + style={{ display: 'inline-flex', alignItems: 'center', background: this._listening ? 'lightgray' : '', borderRadius: '2px', width: '100%' }}> + Evaluate Pronunciation + </button> + + {this.layoutDoc[`_${this._props.fieldKey}_usePath`] !== 'alternate' ? ( + <button className="submit-buttonsubmit" type="button" onClick={this.handleRenderGPTClick} style={{ borderRadius: '2px', marginBottom: '3px', width: '100%' }}> + Submit + </button> + ) : ( + <button className="submit-buttonsubmit" type="button" onClick={this.handleRenderClick} style={{ display: 'inline-flex', alignItems: 'center', borderRadius: '2px', marginBottom: '3px', width: '100%' }}> + Redo the Question + </button> + )} + </div> </div> </div> ); @@ -588,9 +957,15 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() className={`comparisonBox${this._props.isContentActive() ? '-interactive' : ''}`} /* change className to easily disable/enable pointer events in CSS */ // onContextMenu={this.specificMenu} style={{ display: 'flex', flexDirection: 'column', overflow: 'hidden' }} - onMouseEnter={() => this.hoverFlip(true)} - onMouseLeave={() => this.hoverFlip(false)}> - {!this.useAlternate && StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text) === '' && !this.childActive ? <p className="comparisonBox-explain">Enter text in the flashcard. </p> : null} + onMouseEnter={() => { + this.hoverFlip('alternate'); + }} + onMouseLeave={() => { + this.hoverFlip(undefined); + }} + // onPointerUp={() => (this._isAnyChildContentActive = true)} + > + {/* {!this.layoutDoc[`_${this._props.fieldKey}_usePath`] && StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text) === '' && !this.childActive ? <p className="explain">Enter text in the flashcard. </p> : null} */} {displayBox(`${this.fieldKey}_${side === 0 ? 1 : 0}`, side, this._props.PanelWidth() - 3)} {this._loading ? ( <div className="loading-spinner" style={{ position: 'absolute' }}> @@ -614,10 +989,9 @@ export class ComparisonBox extends ViewBoxAnnotatableComponent<FieldViewProps>() className="slide-bar" style={{ left: `calc(${this.clipWidth + '%'} - 0.5px)`, - transition: this._animating, cursor: this.clipWidth < 5 ? 'e-resize' : this.clipWidth / 100 > (this._props.PanelWidth() - 5) / this._props.PanelWidth() ? 'w-resize' : undefined, }} - onPointerDown={e => this.registerSliding(e, this._props.PanelWidth() / 2)} /* if clicked, return slide-bar to center */ + onPointerDown={e => !this.childActive && this.registerSliding(e, this._props.PanelWidth() / 2)} /* if clicked, return slide-bar to center */ > <div className="slide-handle" /> </div> |