1 files changed, 79 insertions, 17 deletions
diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx
index ab7605829..faf96d616 100644
--- a/src/client/views/nodes/ImageBox.tsx
+++ b/src/client/views/nodes/ImageBox.tsx
@@ -11,7 +11,6 @@ import { DocData } from '../../../fields/DocSymbols';
 import { Id } from '../../../fields/FieldSymbols';
 import { InkTool } from '../../../fields/InkField';
 import { ObjectField } from '../../../fields/ObjectField';
-import { Cast, ImageCast, NumCast, StrCast } from '../../../fields/Types';
 import { ImageField } from '../../../fields/URLField';
 import { TraceMobx } from '../../../fields/util';
 import { emptyFunction } from '../../../Utils';
@@ -33,6 +32,7 @@ import { StyleProp } from '../StyleProp';
 import { DocumentView } from './DocumentView';
 import { FieldView, FieldViewProps } from './FieldView';
 import { FocusViewOptions } from './FocusViewOptions';
+import { DocCast, NumCast, RTFCast, StrCast, ImageCast, Cast, toList } from '../../../fields/Types';
 import './ImageBox.scss';
 import { OpenWhere } from './OpenWhere';
 import { URLField } from '../../../fields/URLField';
@@ -43,7 +43,9 @@ import { basename } from 'path';
 import { ImageUtility } from './generativeFill/generativeFillUtils/ImageHandler';
 import { dropActionType } from '../../util/DropActionTypes';
 import { canvasSize } from './generativeFill/generativeFillUtils/generativeFillConstants';
+import Tesseract from 'tesseract.js';
 import axios from 'axios';
+import { TupleType } from 'typescript';
 
 export class ImageEditorData {
     // eslint-disable-next-line no-use-before-define
@@ -354,6 +356,49 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         }
     };
 
+    pushInfo = async () => {
+        const formData = new FormData();
+
+        const img = {
+            file: this.paths[0],
+        };
+        const response = await axios.post('http://localhost:105/labels/', img, {
+            headers: {
+                'Content-Type': 'application/json',
+            },
+        });
+
+        console.log('RESPONSE:');
+        console.log(response.data['boxes']);
+        console.log(response.data['text']);
+        this.createBoxes(response.data['boxes'], response.data['text']);
+    };
+
+    createBoxes = (boxes: [[[number, number]]], texts: [string]) => {
+        for (var i = 0; i < boxes.length; i++) {
+            const coords = boxes[i] ? boxes[i] : [];
+            const width = coords[1][0] - coords[0][0];
+            const height = coords[2][1] - coords[0][1];
+            const text = texts[i];
+
+            const newCol = Docs.Create.TextDocument('', {
+                _width: width,
+                //width * NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']),
+                _height: height,
+                //height * NumCast(this.dataDoc[this.fieldKey + '_nativeHeight']),
+                _layout_fitWidth: true,
+                // _layout_autoHeight: true,
+            });
+            newCol.x = coords[0][0];
+            newCol.y = coords[0][1];
+            // newCol.x = x * NumCast(this.dataDoc[this.fieldKey + '_nativeWidth']);
+            // newCol.y = y * NumCast(this.dataDoc[this.fieldKey + '_nativeHeight']);
+            newCol.zIndex = 1000;
+            newCol.forceActive = true;
+            newCol.quiz = text;
+            this.addDocument(newCol);
+        }
+    };
     // static imageUrlToBase64 = async (imageUrl: string): Promise<string> => {
     //     try {
     //         const response = await fetch(imageUrl);
@@ -405,6 +450,36 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         this._loading = false;
     };
 
+    getText = () => {
+        console.log(StrCast(RTFCast(DocCast(this.dataDoc[this.fieldKey + '_1']).text)?.Text));
+    };
+
+    getImageLabels2 = async () => {
+        this._loading = true;
+        try {
+            // const hrefBase64 = await this.createCanvas();
+            // const hw = await gptImageLabel(hrefBase64, 'Find the image dimensions. Return as height and width.');
+            // const response = await gptImageLabel(
+            //     hrefBase64,
+            //     //'What is the height and width of the image'
+            //     'For each group of words in the image, find the x-coordinate and ycoordinate of the top left corner. Find the width and height of the group. Return this information in this format with the correct information replacing the underscores: "observed word: _, x: _, y: _, width: _, height: _," No additional text, asterisks and put it all in one line. Divide the x and width by the width of the image. Divide the y and the height by the height of the image.'
+            // );
+            // console.log(hw);
+            // console.log('RESPONSE: ' + response);
+            // this.createTextboxes(response);
+            Tesseract.recognize(this.paths[0], 'eng', {
+                logger: m => console.log(m),
+            }).then(({ data: { text, words } }) => {
+                console.log('OCR Result:', text);
+                console.log('Words with bounding boxes:', words);
+            });
+            //AnchorMenu.Instance.transferToFlashcard(response, NumCast(this.layoutDoc['x']), NumCast(this.layoutDoc['y']));
+        } catch (error) {
+            console.log('Error');
+        }
+        this._loading = false;
+    };
+
     createTextboxes = (response: string) => {
         const groups = response.replace('*', '').toLowerCase().split('observed word: ');
         groups.shift();
@@ -450,27 +525,14 @@ export class ImageBox extends ViewBoxAnnotatableComponent<FieldViewProps>() {
         this._imageRef = iref;
     };
 
-    pushInfo = async () => {
-        const formData = new FormData();
-
-        const newArticle = {
-            file: '/files/audio/6b412a6222d631a7fff8a8320.mp3',
-        };
-        const response = await axios.post('http://localhost:105/recognize/', newArticle, {
-            headers: {
-                'Content-Type': 'application/json',
-            },
-        });
-        console.log('RESPONSE: ' + response.data['transcription']);
-    };
-
     specificContextMenu = (): void => {
         const field = Cast(this.dataDoc[this.fieldKey], ImageField);
         if (field) {
             const funcs: ContextMenuProps[] = [];
             // funcs.push({ description: 'Create ai flashcards', event: () => this.getImageDesc(), icon: 'id-card' });
-            // funcs.push({ description: 'Push info', event: this.pushInfo, icon: 'redo-alt' });
-            funcs.push({ description: 'Get Labels', event: this.getImageLabels, icon: 'redo-alt' });
+            funcs.push({ description: 'Push info', event: this.pushInfo, icon: 'redo-alt' });
+            // funcs.push({ description: 'Get Labels2', event: this.getImageLabels2, icon: 'redo-alt' });
+            // funcs.push({ description: 'Get Labels', event: this.getImageLabels, icon: 'redo-alt' });
             funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' });
             funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' });
             funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' });