From c358fba1ee2aa54a97373d07e7b218c74dfd9bf0 Mon Sep 17 00:00:00 2001 From: alyssaf16 Date: Tue, 12 Nov 2024 01:01:23 -0500 Subject: flashcards w assistant finally workgit add -A --- src/server/flashcard/labels.py | 285 ++++++++++++++++++++++++++++++++++ src/server/flashcard/requirements.txt | 12 ++ src/server/flashcard/venv/pyvenv.cfg | 3 + 3 files changed, 300 insertions(+) create mode 100644 src/server/flashcard/labels.py create mode 100644 src/server/flashcard/requirements.txt create mode 100644 src/server/flashcard/venv/pyvenv.cfg (limited to 'src/server') diff --git a/src/server/flashcard/labels.py b/src/server/flashcard/labels.py new file mode 100644 index 000000000..546fc4bd3 --- /dev/null +++ b/src/server/flashcard/labels.py @@ -0,0 +1,285 @@ +import base64 +import numpy as np +import base64 +import easyocr +import sys +from PIL import Image +from io import BytesIO +import requests +import json +import numpy as np + +class BoundingBoxUtils: + """Utility class for bounding box operations and OCR result corrections.""" + + @staticmethod + def is_close(box1, box2, x_threshold=20, y_threshold=20): + """ + Determines if two bounding boxes are horizontally and vertically close. + + Parameters: + box1, box2 (list): The bounding boxes to compare. + x_threshold (int): The threshold for horizontal proximity. + y_threshold (int): The threshold for vertical proximity. + + Returns: + bool: True if boxes are close, False otherwise. + """ + horizontally_close = (abs(box1[2] - box2[0]) < x_threshold or # Right edge of box1 and left edge of box2 + abs(box2[2] - box1[0]) < x_threshold or # Right edge of box2 and left edge of box1 + abs(box1[2] - box2[2]) < x_threshold or + abs(box2[0] - box1[0]) < x_threshold) + + vertically_close = (abs(box1[3] - box2[1]) < y_threshold or # Bottom edge of box1 and top edge of box2 + abs(box2[3] - box1[1]) < y_threshold or + box1[1] == box2[1] or box1[3] == box2[3]) + + return horizontally_close and vertically_close + + @staticmethod + def adjust_bounding_box(bbox, original_text, corrected_text): + """ + Adjusts a bounding box based on differences in text length. + + Parameters: + bbox (list): The original bounding box coordinates. + original_text (str): The original text detected by OCR. + corrected_text (str): The corrected text after cleaning. + + Returns: + list: The adjusted bounding box. + """ + if not bbox or len(bbox) != 4: + return bbox + + # Adjust the x-coordinates slightly to account for text correction + x_adjustment = 5 + adjusted_bbox = [ + [bbox[0][0] + x_adjustment, bbox[0][1]], + [bbox[1][0], bbox[1][1]], + [bbox[2][0] + x_adjustment, bbox[2][1]], + [bbox[3][0], bbox[3][1]] + ] + return adjusted_bbox + + @staticmethod + def correct_ocr_results(results): + """ + Corrects common OCR misinterpretations in the detected text and adjusts bounding boxes accordingly. + + Parameters: + results (list): A list of OCR results, each containing bounding box, text, and confidence score. + + Returns: + list: Corrected OCR results with adjusted bounding boxes. + """ + corrections = { + "~": "", # Replace '~' with empty string + "-": "" # Replace '-' with empty string + } + + corrected_results = [] + for (bbox, text, prob) in results: + corrected_text = ''.join(corrections.get(char, char) for char in text) + adjusted_bbox = BoundingBoxUtils.adjust_bounding_box(bbox, text, corrected_text) + corrected_results.append((adjusted_bbox, corrected_text, prob)) + + return corrected_results + + @staticmethod + def convert_to_json_serializable(data): + """ + Converts a list containing various types, including numpy types, to a JSON-serializable format. + + Parameters: + data (list): A list containing numpy or other non-serializable types. + + Returns: + list: A JSON-serializable version of the input list. + """ + def convert_element(element): + if isinstance(element, list): + return [convert_element(e) for e in element] + elif isinstance(element, tuple): + return tuple(convert_element(e) for e in element) + elif isinstance(element, np.integer): + return int(element) + elif isinstance(element, np.floating): + return float(element) + elif isinstance(element, np.ndarray): + return element.tolist() + else: + return element + + return convert_element(data) + +class ImageLabelProcessor: + """Class to process images and perform OCR with EasyOCR.""" + + VERTICAL_THRESHOLD = 20 + HORIZONTAL_THRESHOLD = 8 + + def __init__(self, img_source, source_type, smart_mode): + self.img_source = img_source + self.source_type = source_type + self.smart_mode = smart_mode + self.img_val = self.load_image() + + def load_image(self): + """Load image from either a base64 string or URL.""" + if self.source_type == 'drag': + return self._load_base64_image() + else: + return self._load_url_image() + + def _load_base64_image(self): + """Decode and save the base64 image.""" + base64_string = self.img_source + if base64_string.startswith("data:image"): + base64_string = base64_string.split(",")[1] + + + # Decode the base64 string + image_data = base64.b64decode(base64_string) + image = Image.open(BytesIO(image_data)).convert('RGB') + image.save("temp_image.jpg") + return "temp_image.jpg" + + def _load_url_image(self): + """Download image from URL and return it in byte format.""" + url = self.img_source + response = requests.get(url) + image = Image.open(BytesIO(response.content)).convert('RGB') + + image_bytes = BytesIO() + image.save(image_bytes, format='PNG') + return image_bytes.getvalue() + + def process_image(self): + """Process the image and return the OCR results.""" + if self.smart_mode: + return self._process_smart_mode() + else: + return self._process_standard_mode() + + def _process_smart_mode(self): + """Process the image in smart mode using EasyOCR.""" + reader = easyocr.Reader(['en']) + result = reader.readtext(self.img_val, detail=1, paragraph=True) + + all_boxes = [bbox for bbox, text in result] + all_texts = [text for bbox, text in result] + + response_data = { + 'status': 'success', + 'message': 'Data received', + 'boxes': BoundingBoxUtils.convert_to_json_serializable(all_boxes), + 'text': BoundingBoxUtils.convert_to_json_serializable(all_texts), + } + + return response_data + + def _process_standard_mode(self): + """Process the image in standard mode using EasyOCR.""" + reader = easyocr.Reader(['en']) + results = reader.readtext(self.img_val) + + filtered_results = BoundingBoxUtils.correct_ocr_results([ + (bbox, text, prob) for bbox, text, prob in results if prob >= 0.7 + ]) + + return self._merge_and_prepare_response(filtered_results) + + def are_vertically_close(self, box1, box2): + """Check if two bounding boxes are vertically close.""" + box1_bottom = max(box1[2][1], box1[3][1]) + box2_top = min(box2[0][1], box2[1][1]) + vertical_distance = box2_top - box1_bottom + + box1_left = box1[0][0] + box2_left = box2[0][0] + box1_right = box1[1][0] + box2_right = box2[1][0] + hori_close = abs(box2_left - box1_left) <= self.HORIZONTAL_THRESHOLD or abs(box2_right - box1_right) <= self.HORIZONTAL_THRESHOLD + + return vertical_distance <= self.VERTICAL_THRESHOLD and hori_close + + def merge_boxes(self, boxes, texts): + """Merge multiple bounding boxes and their associated text.""" + x_coords = [] + y_coords = [] + + # Collect all x and y coordinates + for box in boxes: + for point in box: + x_coords.append(point[0]) + y_coords.append(point[1]) + + # Create the merged bounding box + merged_box = [ + [min(x_coords), min(y_coords)], + [max(x_coords), min(y_coords)], + [max(x_coords), max(y_coords)], + [min(x_coords), max(y_coords)] + ] + + # Combine the texts + merged_text = ' '.join(texts) + + return merged_box, merged_text + + def _merge_and_prepare_response(self, filtered_results): + """Merge vertically close boxes and prepare the final response.""" + current_boxes, current_texts = [], [] + all_boxes, all_texts = [], [] + + for ind in range(len(filtered_results) - 1): + if not current_boxes: + current_boxes.append(filtered_results[ind][0]) + current_texts.append(filtered_results[ind][1]) + + if self.are_vertically_close(filtered_results[ind][0], filtered_results[ind + 1][0]): + current_boxes.append(filtered_results[ind + 1][0]) + current_texts.append(filtered_results[ind + 1][1]) + else: + merged = self.merge_boxes(current_boxes, current_texts) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + current_boxes, current_texts = [], [] + + if current_boxes: + merged = self.merge_boxes(current_boxes, current_texts) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + + if not current_boxes and filtered_results: + merged = self.merge_boxes([filtered_results[-1][0]], [filtered_results[-1][1]]) + all_boxes.append(merged[0]) + all_texts.append(merged[1]) + + response = { + 'status': 'success', + 'message': 'Data received', + 'boxes': BoundingBoxUtils.convert_to_json_serializable(all_boxes), + 'text': BoundingBoxUtils.convert_to_json_serializable(all_texts), + } + + return response + +# Main execution function +def labels(): + """Main function to handle image OCR processing based on input arguments.""" + source_type = sys.argv[2] + smart_mode = (sys.argv[3] == 'smart') + with open(sys.argv[1], 'r') as f: + img_source = f.read() + # Create ImageLabelProcessor instance + processor = ImageLabelProcessor(img_source, source_type, smart_mode) + response = processor.process_image() + + # Print and return the response + print(response) + return response + + +labels() diff --git a/src/server/flashcard/requirements.txt b/src/server/flashcard/requirements.txt new file mode 100644 index 000000000..eb92a819b --- /dev/null +++ b/src/server/flashcard/requirements.txt @@ -0,0 +1,12 @@ +easyocr==1.7.1 +requests==2.32.3 +pillow==10.4.0 +numpy==1.26.4 +tqdm==4.66.4 +Werkzeug==3.0.3 +python-dateutil==2.9.0.post0 +six==1.16.0 +certifi==2024.6.2 +charset-normalizer==3.3.2 +idna==3.7 +urllib3==1.26.19 \ No newline at end of file diff --git a/src/server/flashcard/venv/pyvenv.cfg b/src/server/flashcard/venv/pyvenv.cfg new file mode 100644 index 000000000..740014e00 --- /dev/null +++ b/src/server/flashcard/venv/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /Library/Frameworks/Python.framework/Versions/3.10/bin +include-system-site-packages = false +version = 3.10.11 -- cgit v1.2.3-70-g09d2 From 89424e0a8efc6cf3364a2fd1ffc85c9d0d837453 Mon Sep 17 00:00:00 2001 From: bobzel Date: Fri, 22 Nov 2024 10:27:33 -0500 Subject: added initial Firefly endpoint and hanged smartDrawHandler to generate an image and an svg. --- src/client/util/bezierFit.ts | 3 +- src/client/views/MainView.tsx | 32 ++++++++++++-- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 3 +- src/client/views/pdf/AnchorMenu.tsx | 9 ++-- src/client/views/smartdraw/SmartDrawHandler.tsx | 29 ++++++------ src/server/ApiManagers/DataVizManager.ts | 2 +- src/server/ApiManagers/FireflyManager.ts | 51 ++++++++++++++++++++++ src/server/DashUploadUtils.ts | 3 +- src/server/index.ts | 3 +- webpack.config.js | 7 ++- 10 files changed, 113 insertions(+), 29 deletions(-) create mode 100644 src/server/ApiManagers/FireflyManager.ts (limited to 'src/server') diff --git a/src/client/util/bezierFit.ts b/src/client/util/bezierFit.ts index d52460023..84b27e84c 100644 --- a/src/client/util/bezierFit.ts +++ b/src/client/util/bezierFit.ts @@ -703,7 +703,6 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); - coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); lastPt = { X: parseInt(match[3]), Y: parseInt(match[4]) }; } else if (match[0].startsWith('C')) { coordList.push({ X: parseInt(match[5]), Y: parseInt(match[6]) }); @@ -720,7 +719,7 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { } }); const hasZ = attributes.d.match(/Z/); - if (hasZ) { + if (hasZ || attributes.fill) { coordList.push(lastPt); coordList.push(startPt); coordList.push(startPt); diff --git a/src/client/views/MainView.tsx b/src/client/views/MainView.tsx index 7779d339f..0d071fe4f 100644 --- a/src/client/views/MainView.tsx +++ b/src/client/views/MainView.tsx @@ -8,7 +8,7 @@ import { observer } from 'mobx-react'; import * as React from 'react'; import ResizeObserver from 'resize-observer-polyfill'; import '../../../node_modules/browndash-components/dist/styles/global.min.css'; -import { ClientUtils, lightOrDark, returnEmptyFilter, returnFalse, returnTrue, returnZero, setupMoveUpEvents } from '../../ClientUtils'; +import { ClientUtils, returnEmptyFilter, returnFalse, returnTrue, returnZero, setupMoveUpEvents } from '../../ClientUtils'; import { emptyFunction } from '../../Utils'; import { Doc, DocListCast, GetDocFromUrl, Opt, returnEmptyDoclist } from '../../fields/Doc'; import { DocData } from '../../fields/DocSymbols'; @@ -1023,10 +1023,36 @@ export class MainView extends ObservableReactComponent { {[ ...SnappingManager.HorizSnapLines.map(l => ( - + )), ...SnappingManager.VertSnapLines.map(l => ( - + )), ]} diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index a61705250..3ef6bdd8b 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -431,7 +431,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = Docs.Create.FunctionPlotDocument([], options); break; case 'dataviz': - case 'data_viz': + case 'data_viz': { const { fileUrl, id } = await Networking.PostToServer('/createCSV', { filename: (options.title as string).replace(/\s+/g, '') + '.csv', data: data, @@ -439,6 +439,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { doc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data) }); this.addCSVForAnalysis(doc, id); break; + } case 'chat': doc = Docs.Create.ChatDocument(options); break; diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index 5ab9b556c..fe03f32a5 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -131,12 +131,15 @@ export class AnchorMenu extends AntimodeMenu { /** * Creates a GPT drawing based on selected text. */ - gptDraw = async (e: React.PointerEvent) => { + gptDraw = (e: React.PointerEvent) => { try { SmartDrawHandler.Instance.AddDrawing = this.createDrawingAnnotation; runInAction(() => (this._isLoading = true)); - await SmartDrawHandler.Instance.drawWithGPT({ X: e.clientX, Y: e.clientY }, this._selectedText, 5, 100, true); - runInAction(() => (this._isLoading = false)); + SmartDrawHandler.Instance.drawWithGPT({ X: e.clientX, Y: e.clientY }, this._selectedText, 5, 100, true)?.then( + action(() => { + this._isLoading = false; + }) + ); } catch (err) { console.error(err); } diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index d0f6566a5..342b91bd9 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -13,6 +13,7 @@ import { Doc, DocListCast } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { InkData, InkField, InkTool } from '../../../fields/InkField'; import { BoolCast, ImageCast, NumCast, StrCast } from '../../../fields/Types'; +import { Networking } from '../../Network'; import { GPTCallType, gptAPICall, gptDrawingColor } from '../../apis/gpt/GPT'; import { Docs } from '../../documents/Documents'; import { SettingsManager } from '../../util/SettingsManager'; @@ -21,7 +22,8 @@ import { SVGToBezier, SVGType } from '../../util/bezierFit'; import { InkingStroke } from '../InkingStroke'; import { ObservableReactComponent } from '../ObservableReactComponent'; import { MarqueeView } from '../collections/collectionFreeForm'; -import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkDash, ActiveInkFillColor, ActiveInkBezierApprox, ActiveInkColor, ActiveInkWidth, ActiveIsInkMask, DocumentView } from '../nodes/DocumentView'; +import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkBezierApprox, ActiveInkColor, ActiveInkDash, ActiveInkFillColor, ActiveInkWidth, ActiveIsInkMask, DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; +import { OpenWhere } from '../nodes/OpenWhere'; import './SmartDrawHandler.scss'; export interface DrawingOptions { @@ -230,20 +232,21 @@ export class SmartDrawHandler extends ObservableReactComponent { * Calls GPT API to create a drawing based on user input. */ @action - drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { - if (input === '') return; - this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; - const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); - if (!res) { - console.error('GPT call failed'); - return; - } - const strokeData = await this.parseSvg(res, startPt, false, autoColor); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + drawWithGPT = (startPt: { X: number; Y: number }, prompt: string, complexity: number, size: number, autoColor: boolean) => { + if (prompt === '') return; + this._lastInput = { text: prompt, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; + + Networking.PostToServer('/queryFireflyImage', { prompt }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, { title: prompt }), OpenWhere.addRight)); + + const result = gptAPICall(`"${prompt}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true).then(res => + this.parseSvg(res, startPt, false, autoColor).then(strokeData => { + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + }) + ); this._errorOccurredOnce = false; - return strokeData; + return result; }; /** diff --git a/src/server/ApiManagers/DataVizManager.ts b/src/server/ApiManagers/DataVizManager.ts index 88f22992d..d2028f23b 100644 --- a/src/server/ApiManagers/DataVizManager.ts +++ b/src/server/ApiManagers/DataVizManager.ts @@ -9,7 +9,7 @@ export default class DataVizManager extends ApiManager { register({ method: Method.GET, subscription: '/csvData', - secureHandler: async ({ req, res }) => { + secureHandler: ({ req, res }) => { const uri = req.query.uri as string; return new Promise(resolve => { diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts new file mode 100644 index 000000000..04fa8f065 --- /dev/null +++ b/src/server/ApiManagers/FireflyManager.ts @@ -0,0 +1,51 @@ +import { DashUploadUtils } from '../DashUploadUtils'; +import { _invalid, _success, Method } from '../RouteManager'; +import ApiManager, { Registration } from './ApiManager'; + +export default class FireflyManager extends ApiManager { + askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { + const fetched = fetch('https://ims-na1.adobelogin.com/ims/token/v3', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, + }) + .then(response => response.json()) + .then((data: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: `{ "prompt": "${prompt}" }`, + }) + .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .catch(error => { + console.error('Error:', error); + return ''; + }) + ) + .catch(error => { + console.error('Error:', error); + return ''; + }); + return fetched; + }; + protected initialize(register: Registration): void { + register({ + method: Method.POST, + subscription: '/queryFireflyImage', + secureHandler: ({ req, res }) => + this.askFirefly(req.body.prompt).then(fire => + DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info.accessPaths.agnostic.client); + }) + ), + }); + } +} diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 1e55a885a..032d13d43 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -369,7 +369,8 @@ export namespace DashUploadUtils { */ export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename: string, prefix = '', cleanUp = true): Promise => { const { requestable, source, ...remaining } = metadata; - const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split('/')[1].toLowerCase()}`; + const dfltSuffix = remaining.contentType.split('/')[1].toLowerCase(); + const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${dfltSuffix === 'xml' ? 'jpg' : dfltSuffix}`; const { images } = Directory; const information: Upload.ImageInformation = { accessPaths: { diff --git a/src/server/index.ts b/src/server/index.ts index 88dbd232d..1f9af9ee0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,6 +7,7 @@ import AssistantManager from './ApiManagers/AssistantManager'; import DataVizManager from './ApiManagers/DataVizManager'; import DeleteManager from './ApiManagers/DeleteManager'; import DownloadManager from './ApiManagers/DownloadManager'; +import FireflyManager from './ApiManagers/FireflyManager'; import GeneralGoogleManager from './ApiManagers/GeneralGoogleManager'; import SessionManager from './ApiManagers/SessionManager'; import UploadManager from './ApiManagers/UploadManager'; @@ -71,6 +72,7 @@ function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManage new GeneralGoogleManager(), /* new GooglePhotosManager(), */ new DataVizManager(), new AssistantManager(), + new FireflyManager(), ]; // initialize API Managers @@ -112,7 +114,6 @@ function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManage }); const serve: PublicHandler = ({ req, res }) => { - // eslint-disable-next-line new-cap const detector = new mobileDetect(req.headers['user-agent'] || ''); const filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; res.sendFile(path.join(__dirname, '../../deploy/' + filename)); diff --git a/webpack.config.js b/webpack.config.js index e1afc64e5..67417fb02 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,4 +1,4 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ const path = require('path'); const webpack = require('webpack'); const HtmlWebpackPlugin = require('html-webpack-plugin'); @@ -36,7 +36,6 @@ function transferEnvironmentVariables() { } const resolvedClientSide = Object.keys(parsed).reduce((mapping, envKey) => { if (envKey.startsWith(prefix)) { - // eslint-disable-next-line mapping[`process.env.${envKey.replace(prefix, '')}`] = JSON.stringify(parsed[envKey]); } return mapping; @@ -112,7 +111,7 @@ module.exports = { test: /\.scss|css$/, exclude: /\.module\.scss$/i, use: [ - { loader: 'style-loader' }, // eslint-disable-next-line prettier/prettier + { loader: 'style-loader' }, // { loader: 'css-loader' }, { loader: 'sass-loader' }, ], @@ -127,7 +126,7 @@ module.exports = { { test: /\.module\.scss$/i, use: [ - { loader: 'style-loader' }, // eslint-disable-next-line prettier/prettier + { loader: 'style-loader' }, // { loader: 'css-loader', options: { modules: true } }, { loader: 'sass-loader' }, ], -- cgit v1.2.3-70-g09d2 From 50fb7536822269b3735e2fc7e3e70ca816529799 Mon Sep 17 00:00:00 2001 From: bobzel Date: Fri, 22 Nov 2024 10:53:40 -0500 Subject: Update FireflyManager.ts --- src/server/ApiManagers/FireflyManager.ts | 37 +++++++++++++++----------------- 1 file changed, 17 insertions(+), 20 deletions(-) (limited to 'src/server') diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 04fa8f065..a110d112f 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -3,32 +3,29 @@ import { _invalid, _success, Method } from '../RouteManager'; import ApiManager, { Registration } from './ApiManager'; export default class FireflyManager extends ApiManager { - askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { - const fetched = fetch('https://ims-na1.adobelogin.com/ims/token/v3', { + getBearerToken = () => + fetch('https://ims-na1.adobelogin.com/ims/token/v3', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', }, body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, + }).catch(error => { + console.error('Error:', error); + return ''; + }); + askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { + const fetched = fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${process.env._CLIENT_FIREFLY_BEARER_TOKEN}`], + ], + body: `{ "prompt": "${prompt}" }`, }) - .then(response => response.json()) - .then((data: { access_token: string }) => - fetch('https://firefly-api.adobe.io/v3/images/generate', { - method: 'POST', - headers: [ - ['Content-Type', 'application/json'], - ['Accept', 'application/json'], - ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], - ['Authorization', `Bearer ${data.access_token}`], - ], - body: `{ "prompt": "${prompt}" }`, - }) - .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) - .catch(error => { - console.error('Error:', error); - return ''; - }) - ) + .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) .catch(error => { console.error('Error:', error); return ''; -- cgit v1.2.3-70-g09d2 From ad90e036b5ba32cb17314d9629a5738d87717ff0 Mon Sep 17 00:00:00 2001 From: bobzel Date: Fri, 22 Nov 2024 11:56:01 -0500 Subject: fixed resamppling images uploaded from non-local source. --- src/server/DashUploadUtils.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'src/server') diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 032d13d43..351351ca5 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-use-before-define */ import axios from 'axios'; import { exec, spawn } from 'child_process'; import { green, red } from 'colors'; @@ -47,7 +48,8 @@ if (isMainThread) { async function workerResampleImage(message: { imgSourcePath: string; outputPath: string; origSuffix: string; unlinkSource: boolean }) { const { imgSourcePath, outputPath, origSuffix, unlinkSource } = message; - const sizes = !origSuffix ? [{ width: 400, suffix: SizeSuffix.Medium }] : DashUploadUtils.imageResampleSizes(path.extname(imgSourcePath)); + const extension = path.extname(imgSourcePath); + const sizes = !origSuffix ? [{ width: 400, suffix: SizeSuffix.Medium }] : DashUploadUtils.imageResampleSizes(extension === '.xml' ? '.png' : extension); // prettier-ignore Jimp.read(imgSourcePath) .then(img => @@ -60,7 +62,7 @@ if (isMainThread) { } } -// eslint-disable-next-line @typescript-eslint/no-var-requires +// eslint-disable-next-line @typescript-eslint/no-require-imports const requestImageSize = require('../client/util/request-image-size'); export enum SizeSuffix { @@ -401,10 +403,10 @@ export namespace DashUploadUtils { writtenFiles = {}; } } else { - const unlinkSrcWhenFinished = isLocal().test(source) && cleanUp; + const unlinkSrcWhenFinished = cleanUp; // isLocal().test(source) && cleanUp; try { writtenFiles = await outputResizedImages(metadata.source, resolved, unlinkSrcWhenFinished); - } catch (e) { + } catch { // input is a blob or other, try reading it to create a metadata source file. const reqSource = request(metadata.source); const readStream: Stream = reqSource instanceof Promise ? await reqSource : reqSource; @@ -416,7 +418,7 @@ export namespace DashUploadUtils { .on('error', () => rej()); }); writtenFiles = await outputResizedImages(readSource, resolved, unlinkSrcWhenFinished); - fs.unlink(readSource, err => console.log("Couldn't unlink temporary image file:" + readSource, err)); + //fs.unlink(readSource, err => console.log("Couldn't unlink temporary image file:" + readSource, err)); } } Array.from(Object.keys(writtenFiles)).forEach(suffix => { -- cgit v1.2.3-70-g09d2 From 2fee49a7cd342331633bf39d1ac837838d252a18 Mon Sep 17 00:00:00 2001 From: bobzel Date: Fri, 22 Nov 2024 15:00:07 -0500 Subject: switched to providing secret instead of bearer token for adobe api access --- src/server/ApiManagers/FireflyManager.ts | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) (limited to 'src/server') diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index a110d112f..5e3ba1f83 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -15,21 +15,25 @@ export default class FireflyManager extends ApiManager { return ''; }); askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { - const fetched = fetch('https://firefly-api.adobe.io/v3/images/generate', { - method: 'POST', - headers: [ - ['Content-Type', 'application/json'], - ['Accept', 'application/json'], - ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], - ['Authorization', `Bearer ${process.env._CLIENT_FIREFLY_BEARER_TOKEN}`], - ], - body: `{ "prompt": "${prompt}" }`, - }) - .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) - .catch(error => { - console.error('Error:', error); - return ''; - }); + const fetched = this.getBearerToken().then(response => + response.json().then((data: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: `{ "prompt": "${prompt}" }`, + }) + .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .catch(error => { + console.error('Error:', error); + return ''; + }) + ) + ); return fetched; }; protected initialize(register: Registration): void { -- cgit v1.2.3-70-g09d2 From bf46473a8a19a2af6b3f5ef069cfd20871bebbcf Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Wed, 4 Dec 2024 18:25:27 -0500 Subject: added structure and upload image firefly api calls --- src/server/ApiManagers/FireflyManager.ts | 107 +++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 src/server/ApiManagers/FireflyManager.ts (limited to 'src/server') diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts new file mode 100644 index 000000000..29aed2c54 --- /dev/null +++ b/src/server/ApiManagers/FireflyManager.ts @@ -0,0 +1,107 @@ +import { DashUploadUtils } from '../DashUploadUtils'; +import { _invalid, _success, Method } from '../RouteManager'; +import ApiManager, { Registration } from './ApiManager'; + +export default class FireflyManager extends ApiManager { + getBearerToken = () => + fetch('https://ims-na1.adobelogin.com/ims/token/v3', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, + }).catch(error => { + console.error('Error:', error); + return ''; + }); + askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { + const fetched = this.getBearerToken().then(response => + response.json().then((data: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: `{ "prompt": "${prompt}" }`, + }) + .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .catch(error => { + console.error('Error:', error); + return ''; + }) + ) + ); + return fetched; + }; + + askFireflyStructure = (prompt: string = 'a realistic illustration of a cat coding', strength: number, imageReference: ) => { + const fetched = this.getBearerToken().then(response => + response.json().then((data: { access_token: string }) => { + const body = { + prompt: prompt, + structure: { + strength: strength, + imageReference: { + source: { + uploadId: + } + } + } + } + return fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: JSON.stringify(body), + }) + .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .catch(error => { + console.error('Error:', error); + return ''; + }) + }) + ); + return fetched; + }; + + uploadImageToFirefly = (image: File | Blob) => { + const fetched = this.getBearerToken().then(response => + response.json().then((data: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/uploads', { + method: 'POST', + headers: [ + ['Content-Type', image.type], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], // You can replace this with a dynamic token if needed + ], + body: image, + }) + .then(response => response.json()) + .then(data => data.uploadId) // Extract the uploadId from the response + .catch(error => { + console.error('Error uploading image:', error); + return ''; + }); + }; + + protected initialize(register: Registration): void { + register({ + method: Method.POST, + subscription: '/queryFireflyImage', + secureHandler: ({ req, res }) => + this.askFirefly(req.body.prompt).then(fire => + DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info.accessPaths.agnostic.client); + }) + ), + }); + } +} -- cgit v1.2.3-70-g09d2 From 0066635b2a9ae5fde15defd9dd76c6d7dc1d8959 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Thu, 5 Dec 2024 00:22:08 -0500 Subject: added image to blob logic and firefly api call in drawingFillHandler --- src/client/views/smartdraw/DrawingFillHandler.tsx | 48 ++++++++++++ src/server/ApiManagers/FireflyManager.ts | 90 ++++++++++++----------- 2 files changed, 96 insertions(+), 42 deletions(-) (limited to 'src/server') diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index e69de29bb..508ee557d 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -0,0 +1,48 @@ +import { action, makeObservable } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Doc } from '../../../fields/Doc'; +import { ImageCast } from '../../../fields/Types'; +import { ImageField } from '../../../fields/URLField'; +import { Docs } from '../../documents/Documents'; +import { Networking } from '../../Network'; +import { makeUserTemplateButtonOrImage } from '../../util/DropConverter'; +import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; +import { ImageUtility } from '../nodes/imageEditor/imageEditorUtils/ImageHandler'; +import { OpenWhere } from '../nodes/OpenWhere'; +import { ObservableReactComponent } from '../ObservableReactComponent'; + +@observer +export class DrawingFillHandler extends ObservableReactComponent { + static Instance: DrawingFillHandler; + + constructor(props: object) { + super(props); + makeObservable(this); + DrawingFillHandler.Instance = this; + } + + @action + drawingToImage = async (drawing: Doc, prompt: string) => { + const imageField = await DocumentView.GetDocImage(drawing); + if (!imageField) return; + const image = new Image(); + image.src = imageField.url?.href; + await new Promise((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => reject(new Error('Error loading image')); + }); + + const canvas = document.createElement('canvas'); + canvas.width = image.width; + canvas.height = image.height; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + ctx.globalCompositeOperation = 'source-over'; + ctx.clearRect(0, 0, image.width, image.height); + ctx.drawImage(image, 0, 0); + const blob: Blob = await ImageUtility.canvasToBlob(canvas); + const strength: number = 100; + Networking.PostToServer('/queryFireflyImage', { prompt, blob, strength }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, {}), OpenWhere.addRight)); + }; +} diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 29aed2c54..55fa1e461 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -14,43 +14,43 @@ export default class FireflyManager extends ApiManager { console.error('Error:', error); return ''; }); - askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { - const fetched = this.getBearerToken().then(response => - response.json().then((data: { access_token: string }) => - fetch('https://firefly-api.adobe.io/v3/images/generate', { - method: 'POST', - headers: [ - ['Content-Type', 'application/json'], - ['Accept', 'application/json'], - ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], - ['Authorization', `Bearer ${data.access_token}`], - ], - body: `{ "prompt": "${prompt}" }`, - }) - .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) - .catch(error => { - console.error('Error:', error); - return ''; - }) - ) - ); - return fetched; - }; + // askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { + // const fetched = this.getBearerToken().then(response => + // response.json().then((data: { access_token: string }) => + // fetch('https://firefly-api.adobe.io/v3/images/generate', { + // method: 'POST', + // headers: [ + // ['Content-Type', 'application/json'], + // ['Accept', 'application/json'], + // ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + // ['Authorization', `Bearer ${data.access_token}`], + // ], + // body: `{ "prompt": "${prompt}" }`, + // }) + // .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + // .catch(error => { + // console.error('Error:', error); + // return ''; + // }) + // ) + // ); + // return fetched; + // }; - askFireflyStructure = (prompt: string = 'a realistic illustration of a cat coding', strength: number, imageReference: ) => { + askFirefly = (prompt: string = 'a realistic illustration of a cat coding', uploadId: string, strength: number) => { const fetched = this.getBearerToken().then(response => response.json().then((data: { access_token: string }) => { - const body = { + const body: any = { prompt: prompt, structure: { strength: strength, imageReference: { source: { - uploadId: - } - } - } - } + uploadId: uploadId, + }, + }, + }, + }; return fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ @@ -65,13 +65,13 @@ export default class FireflyManager extends ApiManager { .catch(error => { console.error('Error:', error); return ''; - }) - }) + }); + }) ); return fetched; }; - uploadImageToFirefly = (image: File | Blob) => { + uploadImageToFirefly = (image: Blob) => { const fetched = this.getBearerToken().then(response => response.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/uploads', { @@ -79,29 +79,35 @@ export default class FireflyManager extends ApiManager { headers: [ ['Content-Type', image.type], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], - ['Authorization', `Bearer ${data.access_token}`], // You can replace this with a dynamic token if needed + ['Authorization', `Bearer ${data.access_token}`], ], body: image, }) - .then(response => response.json()) - .then(data => data.uploadId) // Extract the uploadId from the response - .catch(error => { - console.error('Error uploading image:', error); - return ''; - }); + .then(response => response.json()) + .then(data => data.uploadId) // Extract the uploadId from the response + .catch(error => { + console.error('Error:', error); + return ''; + }) + ) + ); + return fetched; }; protected initialize(register: Registration): void { register({ method: Method.POST, subscription: '/queryFireflyImage', - secureHandler: ({ req, res }) => - this.askFirefly(req.body.prompt).then(fire => + secureHandler: async ({ req, res }) => { + const { prompt, imageBlob, strength = 0.5 } = req.body; + const uploadId = imageBlob ? await this.uploadImageToFirefly(imageBlob) : null; + this.askFirefly(prompt, uploadId, strength).then(fire => DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info.accessPaths.agnostic.client); }) - ), + ); + }, }); } } -- cgit v1.2.3-70-g09d2 From b9316fd7280bbddbc73966ce432b5054c07c3af0 Mon Sep 17 00:00:00 2001 From: bobzel Date: Thu, 5 Dec 2024 10:48:27 -0500 Subject: playing with ink stroke editing and firefly --- .env2 | 4 +++ src/.DS_Store | Bin 10244 -> 10244 bytes src/client/views/InkStrokeProperties.ts | 28 ++++++++++++++-- src/server/ApiManagers/FireflyManager.ts | 53 +++++++++++++++++++++++++++---- 4 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 .env2 (limited to 'src/server') diff --git a/.env2 b/.env2 new file mode 100644 index 000000000..08ee154e1 --- /dev/null +++ b/.env2 @@ -0,0 +1,4 @@ + +export _CLIENT_FIREFLY_CLIENT_ID='62eb3cc56d8d47c88ff67103ded0003f' +export _CLIENT_FIREFLY_SECRET='p8e-4JxcVXG6izNEnf0ceEPO1zSZ7U13TMUQ' +export _CLIENT_FIREFLY_TOKEN='eyJhbGciOiJSUzI1NiIsIng1dSI6Imltc19uYTEta2V5LWF0LTEuY2VyIiwia2lkIjoiaW1zX25hMS1rZXktYXQtMSIsIml0dCI6ImF0In0.eyJpZCI6IjE3MzMyMzgwMjg3NDVfNDg0ODM3ZTAtMmU3Zi00NWFkLTkzZjEtZjM0NWRkNDM3MzAwX3VlMSIsIm9yZyI6IjVEOEUxREMzNjcyNTQ2MDIwQTQ5NUZGMEBBZG9iZU9yZyIsInR5cGUiOiJhY2Nlc3NfdG9rZW4iLCJjbGllbnRfaWQiOiI2MmViM2NjNTZkOGQ0N2M4OGZmNjcxMDNkZWQwMDAzZiIsInVzZXJfaWQiOiI2MzQ1MUQ5RTY3MjU1MEQ4MEE0OTVFQ0FAdGVjaGFjY3QuYWRvYmUuY29tIiwiYXMiOiJpbXMtbmExIiwiYWFfaWQiOiI2MzQ1MUQ5RTY3MjU1MEQ4MEE0OTVFQ0FAdGVjaGFjY3QuYWRvYmUuY29tIiwiY3RwIjozLCJtb2kiOiI3NDc2MDZiZCIsImV4cGlyZXNfaW4iOiI4NjQwMDAwMCIsInNjb3BlIjoiZmlyZWZseV9hcGksYWRkaXRpb25hbF9pbmZvLG9wZW5pZCxzZXNzaW9uLEFkb2JlSUQsZmZfYXBpcyxyZWFkX29yZ2FuaXphdGlvbnMiLCJjcmVhdGVkX2F0IjoiMTczMzIzODAyODc0NSJ9.O-HGx7ji_-RZJXmCnsPVzdTmJA4s_fA0Gj0s7z1FZ6MPtzdXa9MSYkMYdo4ImfiYZGf73SeJACEpfTdgyd7shG3qXBydeZV2exGQcN6eve88zXMBsc98aCi0mPz2eql4MomJk9xGXuk3ZuZmVAbsAVfwScPU_uyl75DZAq4_4e6lyXakEti9ACYnEl-7TQQeEgK6Mr4R8giUWvYVAdmSK4xOq67yjBToj6S-fv_s2dBRY58x_fMkORCrJnDLEnSdXS2LXHXAyM_Up9EaezIovlVhwz04BuwrH0vVOJN7uHgFOeWbjULaGGYy2Wzb1hPeRVd7VIRiw3sHavthTSJ3KA' diff --git a/src/.DS_Store b/src/.DS_Store index 9b66f8d8e..1ef749033 100644 Binary files a/src/.DS_Store and b/src/.DS_Store differ diff --git a/src/client/views/InkStrokeProperties.ts b/src/client/views/InkStrokeProperties.ts index 358274f0e..8165b7f4e 100644 --- a/src/client/views/InkStrokeProperties.ts +++ b/src/client/views/InkStrokeProperties.ts @@ -74,6 +74,7 @@ export class InkStrokeProperties { doc._height = (newYrange.max - newYrange.min) * ptsYscale + NumCast(doc.stroke_width); doc.x = oldXrange.coord + (newXrange.min - oldXrange.min) * ptsXscale; doc.y = oldYrange.coord + (newYrange.min - oldYrange.min) * ptsYscale; + Doc.SetInPlace(doc, 'stroke', new InkField(newPoints), true); appliedFunc = true; } @@ -248,7 +249,7 @@ export class InkStrokeProperties { /** * Handles the movement/scaling of a control point. */ - moveControlPtHandle = undoable((inkView: DocumentView, deltaX: number, deltaY: number, controlIndex: number, origInk?: InkData) => { + moveControlPtHandle = undoable((inkView: DocumentView, deltaX: number, deltaY: number, controlIndex: number, origInk?: InkData, noPush?: boolean) => { inkView && this.applyFunction(inkView, (view: DocumentView, ink: InkData) => { const order = controlIndex % 4; @@ -288,10 +289,11 @@ export class InkStrokeProperties { const { finalCtrls: rightCtrls /* , error: errorRight */ } = FitOneCurve(samplesRight, { X: startDir.x, Y: startDir.y }, { X: endDir.x, Y: endDir.y }); finalCtrls = finalCtrls.concat(rightCtrls); newink.splice(this._currentPoint - 4, 8, ...finalCtrls); - return newink; } - return ink.map((pt, i) => { + const prevData = (inkView.ComponentView as InkingStroke).screenCtrlPts; + + const newInk = ink.map((pt, i) => { const leftHandlePoint = order === 0 && i === controlIndex + 1; const rightHandlePoint = order === 0 && controlIndex !== 0 && i === controlIndex - 2; if (controlIndex === i || (order === 0 && controlIndex !== 0 && i === controlIndex - 1) || (order === 3 && i === controlIndex - 1)) { @@ -312,6 +314,26 @@ export class InkStrokeProperties { } return pt; }); + + if (inkView.IsSelected && !noPush && !InkingStroke.IsClosed(prevData)) { + const ffview = CollectionFreeFormView.from(inkView); + ffview?.childDocs.forEach(d => { + const oinkDoc = DocumentView.getDocumentView(d); + const oinkView = oinkDoc?.ComponentView as InkingStroke; + if (oinkView instanceof InkingStroke && oinkDoc && oinkDoc.Document !== inkView.Document) { + const { nearestSeg, nearestT } = InkStrokeProperties.nearestPtToStroke(oinkView.screenCtrlPts, prevData[controlIndex]); + const locdelta = inkView.screenToContentsTransform().inverse().transformDirection(deltaX, deltaY); + const index = nearestSeg + Math.round(nearestT) * 4; + const oinkData = oinkView.inkScaledData().inkData; + const closed = InkingStroke.IsClosed(oinkData); + InkStrokeProperties.Instance.moveControlPtHandle(oinkDoc, locdelta[0], locdelta[1], index, undefined, true); + if (index === oinkData.length && closed) { + InkStrokeProperties.Instance.moveControlPtHandle(oinkDoc, locdelta[0], locdelta[1], 0, undefined, true); + } + } + }); + } + return newInk; }); }, 'move ink ctrl pt'); diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 5e3ba1f83..e10e43704 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -16,7 +16,7 @@ export default class FireflyManager extends ApiManager { }); askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { const fetched = this.getBearerToken().then(response => - response.json().then((data: { access_token: string }) => + (response as Response).json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ @@ -36,16 +36,57 @@ export default class FireflyManager extends ApiManager { ); return fetched; }; + askFireflyText = (testshotpng: Blob) => { + const fetched = this.getBearerToken().then(response => + (response as Response).json().then((data: { access_token: string }) => { + return fetch('https://sensei.adobe.io/services/v2/predict', { + method: 'POST', + headers: [ + ['Prefer', 'respond-async, wait=59'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['content-type', 'multipart/form-data'], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: ((form: FormData) => { + form.append('file', testshotpng); + form.append( + 'contentAnalyzerRequests', + JSON.stringify({ + 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + }) + ); + return form; + })(new FormData()), + }).then(response2 => + response2 + .json() + .then(json => { + console.log(json); + return ''; + }) + .catch(error => { + console.error('Error:', error); + return ''; + }) + ); + }) + ); + return fetched; + }; protected initialize(register: Registration): void { register({ method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.askFirefly(req.body.prompt).then(fire => - DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info.accessPaths.agnostic.client); - }) + fetch('http://localhost:1050/files/images/testshot.png').then(json => + json.blob().then(file => + this.askFireflyText(file).then(fire => + DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info.accessPaths.agnostic.client); + }) + ) + ) ), }); } -- cgit v1.2.3-70-g09d2 From 629e2bef5d4dd4599de010b62e5b26304ee620dd Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 9 Dec 2024 12:53:35 -0500 Subject: added firefly text in image request --- package-lock.json | 6 ++ package.json | 1 + src/server/ApiManagers/FireflyManager.ts | 94 ++++++++++++++++++++++++-------- src/server/ApiManagers/UploadManager.ts | 5 -- src/server/DashUploadUtils.ts | 4 +- 5 files changed, 79 insertions(+), 31 deletions(-) (limited to 'src/server') diff --git a/package-lock.json b/package-lock.json index 8c2cf0694..05f6d540f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -171,6 +171,7 @@ "npm": "^10.8.1", "openai": "^4.26.0", "p-limit": "^6.1.0", + "parse-multipart-data": "^1.5.0", "passport": "^0.7.0", "passport-google-oauth20": "^2.0.0", "passport-local": "^1.0.0", @@ -30590,6 +30591,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse-multipart-data": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/parse-multipart-data/-/parse-multipart-data-1.5.0.tgz", + "integrity": "sha512-ck5zaMF0ydjGfejNMnlo5YU2oJ+pT+80Jb1y4ybanT27j+zbVP/jkYmCrUGsEln0Ox/hZmuvgy8Ra7AxbXP2Mw==" + }, "node_modules/parse-passwd": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", diff --git a/package.json b/package.json index 7c2730245..949f76599 100644 --- a/package.json +++ b/package.json @@ -254,6 +254,7 @@ "npm": "^10.8.1", "openai": "^4.26.0", "p-limit": "^6.1.0", + "parse-multipart-data": "^1.5.0", "passport": "^0.7.0", "passport-google-oauth20": "^2.0.0", "passport-local": "^1.0.0", diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index e10e43704..30fbdc577 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -1,6 +1,7 @@ import { DashUploadUtils } from '../DashUploadUtils'; import { _invalid, _success, Method } from '../RouteManager'; import ApiManager, { Registration } from './ApiManager'; +import * as multipart from 'parse-multipart-data'; export default class FireflyManager extends ApiManager { getBearerToken = () => @@ -27,7 +28,7 @@ export default class FireflyManager extends ApiManager { ], body: `{ "prompt": "${prompt}" }`, }) - .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) .catch(error => { console.error('Error:', error); return ''; @@ -36,7 +37,9 @@ export default class FireflyManager extends ApiManager { ); return fetched; }; - askFireflyText = (testshotpng: Blob) => { + getImageText = (testshotpng: Blob) => { + const inputFileVarName = 'infile'; + const outputVarName = 'result'; const fetched = this.getBearerToken().then(response => (response as Response).json().then((data: { access_token: string }) => { return fetch('https://sensei.adobe.io/services/v2/predict', { @@ -44,31 +47,67 @@ export default class FireflyManager extends ApiManager { headers: [ ['Prefer', 'respond-async, wait=59'], ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], - ['content-type', 'multipart/form-data'], + // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string ['Authorization', `Bearer ${data.access_token}`], ], body: ((form: FormData) => { - form.append('file', testshotpng); - form.append( + form.set(inputFileVarName, testshotpng); + form.set( 'contentAnalyzerRequests', JSON.stringify({ 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + 'sensei:invocation_mode': 'asynchronous', + 'sensei:invocation_batch': false, + 'sensei:engines': [ + { + 'sensei:execution_info': { + 'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + }, + 'sensei:inputs': { + documents: [ + { + 'sensei:multipart_field_name': inputFileVarName, + 'dc:format': 'image/png', + }, + ], + }, + 'sensei:params': { + correct_with_dictionary: true, + }, + 'sensei:outputs': { + result: { + 'sensei:multipart_field_name': outputVarName, + 'dc:format': 'application/json', + }, + }, + }, + ], }) ); return form; })(new FormData()), - }).then(response2 => - response2 - .json() - .then(json => { - console.log(json); - return ''; - }) - .catch(error => { - console.error('Error:', error); - return ''; - }) - ); + }).then(response2 => { + const contentType = response2.headers.get('content-type') ?? ''; + if (contentType.includes('application/json')) { + return response2.json().then((json: object) => JSON.stringify(json)); + } + if (contentType.includes('multipart')) { + return response2 + .arrayBuffer() + .then(arrayBuffer => + multipart + .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) + .filter(part => part.name === outputVarName) + .map(part => JSON.parse(part.data.toString()[0])) + .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') + ) + .catch(error => { + console.error('Error:', error); + return ''; + }); + } + return response2.text(); + }); }) ); return fetched; @@ -77,15 +116,24 @@ export default class FireflyManager extends ApiManager { register({ method: Method.POST, subscription: '/queryFireflyImage', + secureHandler: ({ req, res }) => + this.askFirefly(req.body).then(fire => { + DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info.accessPaths.agnostic.client); + }); + }), + }); + + register({ + method: Method.POST, + subscription: '/queryFireflyImageText', secureHandler: ({ req, res }) => fetch('http://localhost:1050/files/images/testshot.png').then(json => json.blob().then(file => - this.askFireflyText(file).then(fire => - DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info.accessPaths.agnostic.client); - }) - ) + this.getImageText(file).then(text => { + _success(res, text); + }) ) ), }); diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 868373474..7bfdd5aec 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -147,13 +147,10 @@ export default class UploadManager extends ApiManager { if (doc.id) { doc.id = getId(doc.id); } - // eslint-disable-next-line no-restricted-syntax for (const key in doc.fields) { - // eslint-disable-next-line no-continue if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue; const field = doc.fields[key]; - // eslint-disable-next-line no-continue if (field === undefined || field === null) continue; if (field.__type === 'Doc') { @@ -182,11 +179,9 @@ export default class UploadManager extends ApiManager { let docids: string[] = []; let linkids: string[] = []; try { - // eslint-disable-next-line no-restricted-syntax for (const name in files) { if (Object.prototype.hasOwnProperty.call(files, name)) { const f = files[name]; - // eslint-disable-next-line no-continue if (!f) continue; const path2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set? const zip = new AdmZip(path2.filepath); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 351351ca5..028116779 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -223,7 +223,6 @@ export namespace DashUploadUtils { const parseExifData = async (source: string) => { const image = await request.get(source, { encoding: null }); const { /* data, */ error } = await new Promise<{ data: ExifData; error: string | undefined }>(resolve => { - // eslint-disable-next-line no-new new ExifImage({ image }, (exifError, data) => { resolve({ data, error: exifError?.message }); }); @@ -302,7 +301,6 @@ export namespace DashUploadUtils { // Bundle up the information into an object return { source, - // eslint-disable-next-line radix contentSize: parseInt(headers[size]), contentType: headers[type], nativeWidth, @@ -370,7 +368,7 @@ export namespace DashUploadUtils { * @returns the accessPaths for the resized files. */ export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename: string, prefix = '', cleanUp = true): Promise => { - const { requestable, source, ...remaining } = metadata; + const { requestable, ...remaining } = metadata; const dfltSuffix = remaining.contentType.split('/')[1].toLowerCase(); const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${dfltSuffix === 'xml' ? 'jpg' : dfltSuffix}`; const { images } = Directory; -- cgit v1.2.3-70-g09d2 From 48c3b802a3c8fd446ecbd33747fe702b11170dfe Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 9 Dec 2024 12:54:49 -0500 Subject: from last --- src/server/ApiManagers/FireflyManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/server') diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 30fbdc577..cc4c218bf 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -117,12 +117,12 @@ export default class FireflyManager extends ApiManager { method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.askFirefly(req.body).then(fire => { + this.askFirefly(req.body.prompt).then(fire => DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info.accessPaths.agnostic.client); - }); - }), + }) + ), }); register({ -- cgit v1.2.3-70-g09d2 From a5a7769e0c400f0a58a9b53ae13b338a26eaa919 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 11 Dec 2024 13:11:43 -0500 Subject: fixes for uploading remote images to sample and save them properly. extnesions fo firefly manager to query text in images, expand images, and tweaks to generate images. --- package-lock.json | 23 +++++-- package.json | 3 +- src/client/views/nodes/ImageBox.tsx | 31 +++++++++ src/server/ApiManagers/FireflyManager.ts | 114 +++++++++++++++++++++++++++---- src/server/ApiManagers/UploadManager.ts | 8 ++- src/server/DashUploadUtils.ts | 37 ++++++---- 6 files changed, 183 insertions(+), 33 deletions(-) (limited to 'src/server') diff --git a/package-lock.json b/package-lock.json index 05f6d540f..46057f615 100644 --- a/package-lock.json +++ b/package-lock.json @@ -100,6 +100,7 @@ "depcheck": "^1.4.7", "dompurify": "^3.1.7", "dotenv": "^16.4.5", + "dropbox": "^10.34.0", "eslint-webpack-plugin": "^4.1.0", "exif": "^0.6.0", "exifr": "^7.1.3", @@ -128,7 +129,7 @@ "google-auth-library": "^9.4.1", "googleapis": "^144.0.0", "googlephotos": "^0.3.5", - "got": "^14.0.0", + "got": "^14.4.5", "howler": "^2.2.4", "html-to-image": "^1.11.11", "html-to-text": "^9.0.5", @@ -19701,6 +19702,20 @@ "resolved": "https://registry.npmjs.org/double-bits/-/double-bits-1.1.1.tgz", "integrity": "sha512-BCLEIBq0O/DWoA7BsCu/R+RP0ZXiowP8BhtJT3qeuuQEBpnS8LK/Wo6UTJQv6v8mK1fj8n90YziHLwGdM5whSg==" }, + "node_modules/dropbox": { + "version": "10.34.0", + "resolved": "https://registry.npmjs.org/dropbox/-/dropbox-10.34.0.tgz", + "integrity": "sha512-5jb5/XzU0fSnq36/hEpwT5/QIep7MgqKuxghEG44xCu7HruOAjPdOb3x0geXv5O/hd0nHpQpWO+r5MjYTpMvJg==", + "dependencies": { + "node-fetch": "^2.6.1" + }, + "engines": { + "node": ">=0.10.3" + }, + "peerDependencies": { + "@types/node-fetch": "^2.5.7" + } + }, "node_modules/dynamic-dedupe": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/dynamic-dedupe/-/dynamic-dedupe-0.3.0.tgz", @@ -22149,9 +22164,9 @@ } }, "node_modules/got": { - "version": "14.4.4", - "resolved": "https://registry.npmjs.org/got/-/got-14.4.4.tgz", - "integrity": "sha512-tqiF7eSgTBwQkxb1LxsEpva8TaMYVisbhplrFVmw9GQE3855Z+MH/mnsXLLOkDxR6hZJRFMj5VTAZ8lmTF8ZOA==", + "version": "14.4.5", + "resolved": "https://registry.npmjs.org/got/-/got-14.4.5.tgz", + "integrity": "sha512-sq+uET8TnNKRNnjEOPJzMcxeI0irT8BBNmf+GtZcJpmhYsQM1DSKmCROUjPWKsXZ5HzwD5Cf5/RV+QD9BSTxJg==", "dependencies": { "@sindresorhus/is": "^7.0.1", "@szmarczak/http-timer": "^5.0.1", diff --git a/package.json b/package.json index 949f76599..7ca0c8b38 100644 --- a/package.json +++ b/package.json @@ -183,6 +183,7 @@ "depcheck": "^1.4.7", "dompurify": "^3.1.7", "dotenv": "^16.4.5", + "dropbox": "^10.34.0", "eslint-webpack-plugin": "^4.1.0", "exif": "^0.6.0", "exifr": "^7.1.3", @@ -211,7 +212,7 @@ "google-auth-library": "^9.4.1", "googleapis": "^144.0.0", "googlephotos": "^0.3.5", - "got": "^14.0.0", + "got": "^14.4.5", "howler": "^2.2.4", "html-to-image": "^1.11.11", "html-to-text": "^9.0.5", diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index ff879a2ab..8c7ec959e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -38,6 +38,8 @@ import { FieldView, FieldViewProps } from './FieldView'; import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; +import { Upload } from '../../../server/SharedMediaTypes'; +import { ImageUtils } from '../../util/Import & Export/ImageUtils'; export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -309,6 +311,35 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { funcs.push({ description: 'Rotate Clockwise 90', event: this.rotate, icon: 'redo-alt' }); funcs.push({ description: `Show ${this.layoutDoc._showFullRes ? 'Dynamic Res' : 'Full Res'}`, event: this.resolution, icon: 'expand' }); funcs.push({ description: 'Set Native Pixel Size', event: this.setNativeSize, icon: 'expand-arrows-alt' }); + funcs.push({ + description: 'GetImageText', + event: () => { + Networking.PostToServer('/queryFireflyImageText', { + file: (file => { + const ext = extname(file); + return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); + })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), + }).then(text => alert(text)); + }, + icon: 'expand-arrows-alt', + }); + funcs.push({ + description: 'Expand Image', + event: () => { + Networking.PostToServer('/expandImage', { + prompt: 'sunny skies', + file: (file => { + const ext = extname(file); + return file.replace(ext, (this._error ? '_o' : this._curSuffix) + ext); + })(ImageCast(this.Document[Doc.LayoutFieldKey(this.Document)])?.url.href), + }).then((info: Upload.ImageInformation) => { + const img = Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { title: 'expand:' + this.Document.title }); + DocUtils.assignImageInfo(info, img); + this._props.addDocTab(img, OpenWhere.addRight); + }); + }, + icon: 'expand-arrows-alt', + }); funcs.push({ description: 'Copy path', event: () => ClientUtils.CopyText(this.choosePath(field.url)), icon: 'copy' }); funcs.push({ description: 'Open Image Editor', diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index cc4c218bf..d757a23fe 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -1,7 +1,11 @@ +import { Dropbox, files } from 'dropbox'; +import * as fs from 'fs'; +import * as multipart from 'parse-multipart-data'; +import * as path from 'path'; import { DashUploadUtils } from '../DashUploadUtils'; -import { _invalid, _success, Method } from '../RouteManager'; +import { _error, _invalid, _success, Method } from '../RouteManager'; +import { Directory, filesDirectory } from '../SocketData'; import ApiManager, { Registration } from './ApiManager'; -import * as multipart from 'parse-multipart-data'; export default class FireflyManager extends ApiManager { getBearerToken = () => @@ -13,11 +17,11 @@ export default class FireflyManager extends ApiManager { body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, }).catch(error => { console.error('Error:', error); - return ''; + return undefined; }); - askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { + generateImage = (prompt: string = 'a realistic illustration of a cat coding') => { const fetched = this.getBearerToken().then(response => - (response as Response).json().then((data: { access_token: string }) => + response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ @@ -28,20 +32,66 @@ export default class FireflyManager extends ApiManager { ], body: `{ "prompt": "${prompt}" }`, }) - .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .then(response2 => response2.json().then(json => (json.outputs?.[0] as { image: { url: string } })?.image.url)) .catch(error => { console.error('Error:', error); - return ''; + return undefined; }) ) ); return fetched; }; - getImageText = (testshotpng: Blob) => { + expandImage = (imgUrl: string, prompt?: string) => { + const dropboxImgUrl = imgUrl; + const fetched = this.getBearerToken().then(response => + response + ?.json() + .then((data: { access_token: string }) => { + return fetch('https://firefly-api.adobe.io/v3/images/expand', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: JSON.stringify({ + image: { + source: { + url: dropboxImgUrl, + }, + }, + numVariations: 1, + seeds: [0], + size: { + width: 3048, + height: 2048, + }, + prompt: prompt ?? 'cloudy skies', + placement: { + inset: { + left: 0, + top: 0, + right: 0, + bottom: 0, + }, + alignment: { + horizontal: 'center', + vertical: 'center', + }, + }, + }), + }); + }) + .then(resp => resp.json()) + ); + return fetched; + }; + getImageText = (imageBlob: Blob) => { const inputFileVarName = 'infile'; const outputVarName = 'result'; const fetched = this.getBearerToken().then(response => - (response as Response).json().then((data: { access_token: string }) => { + response?.json().then((data: { access_token: string }) => { return fetch('https://sensei.adobe.io/services/v2/predict', { method: 'POST', headers: [ @@ -51,7 +101,7 @@ export default class FireflyManager extends ApiManager { ['Authorization', `Bearer ${data.access_token}`], ], body: ((form: FormData) => { - form.set(inputFileVarName, testshotpng); + form.set(inputFileVarName, imageBlob); form.set( 'contentAnalyzerRequests', JSON.stringify({ @@ -98,7 +148,7 @@ export default class FireflyManager extends ApiManager { multipart .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) .filter(part => part.name === outputVarName) - .map(part => JSON.parse(part.data.toString()[0])) + .map(part => JSON.parse(part.data.toString())[0]) .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') ) .catch(error => { @@ -117,10 +167,10 @@ export default class FireflyManager extends ApiManager { method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.askFirefly(req.body.prompt).then(fire => - DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { + this.generateImage(req.body.prompt).then(url => + DashUploadUtils.UploadImage(url ?? '').then(info => { if (info instanceof Error) _invalid(res, info.message); - else _success(res, info.accessPaths.agnostic.client); + else _success(res, info); }) ), }); @@ -128,6 +178,7 @@ export default class FireflyManager extends ApiManager { register({ method: Method.POST, subscription: '/queryFireflyImageText', + // eslint-disable-next-line @typescript-eslint/no-unused-vars secureHandler: ({ req, res }) => fetch('http://localhost:1050/files/images/testshot.png').then(json => json.blob().then(file => @@ -137,5 +188,40 @@ export default class FireflyManager extends ApiManager { ) ), }); + register({ + method: Method.POST, + subscription: '/expandImage', + secureHandler: ({ req, res }) => + new Promise((resolve, reject) => { + const dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN }); + fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(req.body.file)}`), undefined, (err, contents) => { + if (err) { + console.log('Error: ', err); + reject(); + } else { + dbx.filesUpload({ path: `/Apps/browndash/${path.basename(req.body.file)}`, contents }) + .then(response => { + dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => { + console.log(link.result); + this.expandImage(link.result.link, req.body.prompt).then(text => { + if (text.error_code) _error(res, text.message); + else + DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + resolve(); + }); + }); + }); + }) + .catch(uploadErr => { + console.log(uploadErr); + _error(res, 'upload to dropbox failed'); + reject(); + }); + } + }); + }), + }); } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 7bfdd5aec..5a880901b 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -70,10 +70,16 @@ export default class UploadManager extends ApiManager { ]); } else { fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`)); + // original filenames with '.'s, such as a Macbook screenshot, can be a problem - their extension is not kept in formidable's newFilename. + // This makes sure that the extension is preserved in the newFilename. + const fixNewFilename = (f: formidable.File) => { + if (path.extname(f.originalFilename ?? '') !== path.extname(f.newFilename)) f.newFilename = f.newFilename + path.extname(f.originalFilename ?? ''); + return f; + }; const results = ( await Promise.all( Array.from(Object.keys(files)).map( - async key => (!files[key] ? undefined : DashUploadUtils.upload(files[key]![0] /* , key */)) // key is the guid used by the client to track upload progress. + async key => (!files[key] ? undefined : DashUploadUtils.upload(fixNewFilename(files[key][0]) /* , key */)) // key is the guid used by the client to track upload progress. ) ) ).filter(result => result && !(result.result instanceof Error)); diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 028116779..a06fa3b6e 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -22,7 +22,6 @@ import { AzureManager } from './ApiManagers/AzureManager'; import { AcceptableMedia, Upload } from './SharedMediaTypes'; import { Directory, clientPathToFile, filesDirectory, pathToDirectory, publicDirectory, serverPathToFile } from './SocketData'; import { resolvedServerUrl } from './server_Initialization'; - import { Worker, isMainThread, parentPort } from 'worker_threads'; // Create an array to store worker threads @@ -55,7 +54,8 @@ if (isMainThread) { .then(img => sizes.forEach(({ width, suffix }) => img.resize({ w: width || img.bitmap.width }) - .write(InjectSize(outputPath, suffix) as `${string}.${string}`) + .write(InjectSize(outputPath, suffix) as `${string}.${string}`) + .catch(e => console.log("Jimp error:", e)) )) .catch(e => console.log('Error Jimp:', e)) .finally(() => unlinkSource && unlinkSync(imgSourcePath)); @@ -343,15 +343,24 @@ export namespace DashUploadUtils { const outputPath = path.resolve(pathToDirectory(Directory.images), outputFileName); const sizes = imageResampleSizes(path.extname(outputFileName)); - const imgReadStream = new Duplex(); - imgReadStream.push(fs.readFileSync(imgSourcePath)); - imgReadStream.push(null); - await Promise.all( - sizes.map(({ suffix }) => - new Promise(res => - imgReadStream.pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res) - ) - )); // prettier-ignore + if (unlinkSource) { + const imgReadStream = new Duplex(); + imgReadStream.push(fs.readFileSync(imgSourcePath)); + imgReadStream.push(null); + await Promise.all( + sizes.map(({ suffix }) => + new Promise(res => + imgReadStream.pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res) + ) + )); // prettier-ignore + } else { + await Promise.all( + sizes.map(({ suffix }) => + new Promise(res => + request.get(imgSourcePath).pipe(createWriteStream(writtenFiles[suffix] = InjectSize(outputPath, suffix))).on('close', res) + ) + )); // prettier-ignore + } workerResample(imgSourcePath, outputPath, SizeSuffix.Original, unlinkSource); return writtenFiles; @@ -450,7 +459,7 @@ export namespace DashUploadUtils { } const outputFile = filename || result.filename || ''; - return UploadInspectedImage(result, outputFile, prefix); + return UploadInspectedImage(result, outputFile, prefix, isLocal().exec(source) ? true : false); }; type md5 = 'md5'; @@ -568,7 +577,9 @@ export namespace DashUploadUtils { switch (category) { case 'image': if (imageFormats.includes(format)) { - const result = await UploadImage(filepath, basename(filepath)); + const outputName = basename(filepath); + const extname = path.extname(originalFilename ?? ''); + const result = await UploadImage(filepath, outputName.endsWith(extname) ? outputName : outputName + extname, undefined); return { source: file, result }; } fs.unlink(filepath, () => {}); -- cgit v1.2.3-70-g09d2 From 0034a4b7d1bbaf5c6909d76b1a2dac85054fa4ff Mon Sep 17 00:00:00 2001 From: bobzel Date: Sat, 14 Dec 2024 14:27:03 -0500 Subject: preparing to merge --- src/client/views/smartdraw/DrawingFillHandler.tsx | 6 ++--- src/server/ApiManagers/FireflyManager.ts | 28 +++-------------------- 2 files changed, 6 insertions(+), 28 deletions(-) (limited to 'src/server') diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 8f71b150d..6f111e95b 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -23,9 +23,9 @@ // const response = await fetch(hrefComplete); // const blob: Blob = await response.blob(); // const strength: number = 100; -// const img = await Networking.PostToServer('/queryFireflyImage', { prompt, blob, strength }); +// const img = await Networking.PostToServer('/oldQueryFireflyImage', { prompt, blob, strength }); // DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, {}), OpenWhere.addRight); -// // Networking.PostToServer('/queryFireflyImage', { prompt, blob, strength }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, {}), OpenWhere.addRight)); +// // Networking.PostToServer('/oldQueryFireflyImage', { prompt, blob, strength }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, {}), OpenWhere.addRight)); // } catch (error) { // console.error('Error fetching image:', error); // return; @@ -44,7 +44,7 @@ // // ctx.drawImage(image, 0, 0); // // const blob: Blob = await ImageUtility.canvasToBlob(canvas); // // const strength: number = 100; -// // Networking.PostToServer('/queryFireflyImage', { prompt, blob, strength }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, {}), OpenWhere.addRight)); +// // Networking.PostToServer('/oldQueryFireflyImage', { prompt, blob, strength }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, {}), OpenWhere.addRight)); // // }; // }; // } diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 55fa1e461..d9a1ef621 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -14,30 +14,8 @@ export default class FireflyManager extends ApiManager { console.error('Error:', error); return ''; }); - // askFirefly = (prompt: string = 'a realistic illustration of a cat coding') => { - // const fetched = this.getBearerToken().then(response => - // response.json().then((data: { access_token: string }) => - // fetch('https://firefly-api.adobe.io/v3/images/generate', { - // method: 'POST', - // headers: [ - // ['Content-Type', 'application/json'], - // ['Accept', 'application/json'], - // ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], - // ['Authorization', `Bearer ${data.access_token}`], - // ], - // body: `{ "prompt": "${prompt}" }`, - // }) - // .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) - // .catch(error => { - // console.error('Error:', error); - // return ''; - // }) - // ) - // ); - // return fetched; - // }; - askFirefly = (prompt: string = 'a realistic illustration of a cat coding', uploadId: string, strength: number) => { + askFireflyOld = (prompt: string = 'a realistic illustration of a cat coding', uploadId: string, strength: number) => { const fetched = this.getBearerToken().then(response => response.json().then((data: { access_token: string }) => { const body: any = { @@ -97,11 +75,11 @@ export default class FireflyManager extends ApiManager { protected initialize(register: Registration): void { register({ method: Method.POST, - subscription: '/queryFireflyImage', + subscription: '/oldQueryFireflyImage', secureHandler: async ({ req, res }) => { const { prompt, imageBlob, strength = 0.5 } = req.body; const uploadId = imageBlob ? await this.uploadImageToFirefly(imageBlob) : null; - this.askFirefly(prompt, uploadId, strength).then(fire => + this.askFireflyOld(prompt, uploadId, strength).then(fire => DashUploadUtils.UploadImage(JSON.parse(fire).url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info.accessPaths.agnostic.client); -- cgit v1.2.3-70-g09d2 From 38b8c63d374415142acefcf6b6f736646f78de44 Mon Sep 17 00:00:00 2001 From: bobzel Date: Sat, 14 Dec 2024 14:37:50 -0500 Subject: starting merge with master --- src/server/ApiManagers/FireflyManager.ts | 217 ++++++++++++++++++++++++++++++- 1 file changed, 212 insertions(+), 5 deletions(-) (limited to 'src/server') diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index d9a1ef621..4425aff43 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -1,5 +1,10 @@ +import { Dropbox, files } from 'dropbox'; +import * as fs from 'fs'; +import * as multipart from 'parse-multipart-data'; +import * as path from 'path'; import { DashUploadUtils } from '../DashUploadUtils'; -import { _invalid, _success, Method } from '../RouteManager'; +import { _error, _invalid, _success, Method } from '../RouteManager'; +import { Directory, filesDirectory } from '../SocketData'; import ApiManager, { Registration } from './ApiManager'; export default class FireflyManager extends ApiManager { @@ -12,12 +17,12 @@ export default class FireflyManager extends ApiManager { body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, }).catch(error => { console.error('Error:', error); - return ''; + return undefined; }); askFireflyOld = (prompt: string = 'a realistic illustration of a cat coding', uploadId: string, strength: number) => { const fetched = this.getBearerToken().then(response => - response.json().then((data: { access_token: string }) => { + response?.json().then((data: { access_token: string }) => { const body: any = { prompt: prompt, structure: { @@ -51,7 +56,7 @@ export default class FireflyManager extends ApiManager { uploadImageToFirefly = (image: Blob) => { const fetched = this.getBearerToken().then(response => - response.json().then((data: { access_token: string }) => + response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/uploads', { method: 'POST', headers: [ @@ -71,7 +76,149 @@ export default class FireflyManager extends ApiManager { ); return fetched; }; - + generateImage = (prompt: string = 'a realistic illustration of a cat coding') => { + const fetched = this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: `{ "prompt": "${prompt}" }`, + }) + .then(response2 => response2.json().then(json => (json.outputs?.[0] as { image: { url: string } })?.image.url)) + .catch(error => { + console.error('Error:', error); + return undefined; + }) + ) + ); + return fetched; + }; + expandImage = (imgUrl: string, prompt?: string) => { + const dropboxImgUrl = imgUrl; + const fetched = this.getBearerToken().then(response => + response + ?.json() + .then((data: { access_token: string }) => { + return fetch('https://firefly-api.adobe.io/v3/images/expand', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: JSON.stringify({ + image: { + source: { + url: dropboxImgUrl, + }, + }, + numVariations: 1, + seeds: [0], + size: { + width: 3048, + height: 2048, + }, + prompt: prompt ?? 'cloudy skies', + placement: { + inset: { + left: 0, + top: 0, + right: 0, + bottom: 0, + }, + alignment: { + horizontal: 'center', + vertical: 'center', + }, + }, + }), + }); + }) + .then(resp => resp.json()) + ); + return fetched; + }; + getImageText = (imageBlob: Blob) => { + const inputFileVarName = 'infile'; + const outputVarName = 'result'; + const fetched = this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => { + return fetch('https://sensei.adobe.io/services/v2/predict', { + method: 'POST', + headers: [ + ['Prefer', 'respond-async, wait=59'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string + ['Authorization', `Bearer ${data.access_token}`], + ], + body: ((form: FormData) => { + form.set(inputFileVarName, imageBlob); + form.set( + 'contentAnalyzerRequests', + JSON.stringify({ + 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + 'sensei:invocation_mode': 'asynchronous', + 'sensei:invocation_batch': false, + 'sensei:engines': [ + { + 'sensei:execution_info': { + 'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + }, + 'sensei:inputs': { + documents: [ + { + 'sensei:multipart_field_name': inputFileVarName, + 'dc:format': 'image/png', + }, + ], + }, + 'sensei:params': { + correct_with_dictionary: true, + }, + 'sensei:outputs': { + result: { + 'sensei:multipart_field_name': outputVarName, + 'dc:format': 'application/json', + }, + }, + }, + ], + }) + ); + return form; + })(new FormData()), + }).then(response2 => { + const contentType = response2.headers.get('content-type') ?? ''; + if (contentType.includes('application/json')) { + return response2.json().then((json: object) => JSON.stringify(json)); + } + if (contentType.includes('multipart')) { + return response2 + .arrayBuffer() + .then(arrayBuffer => + multipart + .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) + .filter(part => part.name === outputVarName) + .map(part => JSON.parse(part.data.toString())[0]) + .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') + ) + .catch(error => { + console.error('Error:', error); + return ''; + }); + } + return response2.text(); + }); + }) + ); + return fetched; + }; protected initialize(register: Registration): void { register({ method: Method.POST, @@ -87,5 +234,65 @@ export default class FireflyManager extends ApiManager { ); }, }); + register({ + method: Method.POST, + subscription: '/queryFireflyImage', + secureHandler: ({ req, res }) => + this.generateImage(req.body.prompt).then(url => + DashUploadUtils.UploadImage(url ?? '').then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }) + ), + }); + + register({ + method: Method.POST, + subscription: '/queryFireflyImageText', + // eslint-disable-next-line @typescript-eslint/no-unused-vars + secureHandler: ({ req, res }) => + fetch('http://localhost:1050/files/images/testshot.png').then(json => + json.blob().then(file => + this.getImageText(file).then(text => { + _success(res, text); + }) + ) + ), + }); + register({ + method: Method.POST, + subscription: '/expandImage', + secureHandler: ({ req, res }) => + new Promise((resolve, reject) => { + const dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN }); + fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(req.body.file)}`), undefined, (err, contents) => { + if (err) { + console.log('Error: ', err); + reject(); + } else { + dbx.filesUpload({ path: `/Apps/browndash/${path.basename(req.body.file)}`, contents }) + .then(response => { + dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => { + console.log(link.result); + this.expandImage(link.result.link, req.body.prompt).then(text => { + if (text.error_code) _error(res, text.message); + else + DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + resolve(); + }); + }); + }); + }) + .catch(uploadErr => { + console.log(uploadErr); + _error(res, 'upload to dropbox failed'); + reject(); + }); + } + }); + }), + }); } } -- cgit v1.2.3-70-g09d2 From b7105063030ee29ba644d8fbb5878c8ee41fd389 Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 16 Dec 2024 11:52:42 -0500 Subject: fixed api for creating an image using another image as a structure reference. --- src/client/views/ContextMenuItem.tsx | 10 +- src/client/views/PropertiesView.tsx | 4 +- .../CollectionFreeFormLayoutEngines.tsx | 3 - src/client/views/nodes/DocumentView.tsx | 4 +- src/client/views/nodes/ImageBox.tsx | 3 +- src/client/views/smartdraw/DrawingFillHandler.tsx | 70 ++++------- src/server/ApiManagers/FireflyManager.ts | 132 ++++++++------------- 7 files changed, 82 insertions(+), 144 deletions(-) (limited to 'src/server') diff --git a/src/client/views/ContextMenuItem.tsx b/src/client/views/ContextMenuItem.tsx index 6f8f41bdd..218718b18 100644 --- a/src/client/views/ContextMenuItem.tsx +++ b/src/client/views/ContextMenuItem.tsx @@ -1,6 +1,6 @@ import { IconProp } from '@fortawesome/fontawesome-svg-core'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { action, makeObservable, observable, runInAction } from 'mobx'; +import { action, computed, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; import { SnappingManager } from '../util/SnappingManager'; @@ -26,7 +26,7 @@ export class ContextMenuItem extends ObservableReactComponent this._items.push(...(this._props.subitems ?? []))); + @computed get items() { + return this._items.concat(this._props.subitems ?? []); } handleEvent = async (e: React.MouseEvent) => { @@ -91,7 +91,7 @@ export class ContextMenuItem extends ObservableReactComponent ); + const submenu = this.items.map(prop => ); return this.props.event || this._props.noexpand ? this.renderItem(submenu) :
{submenu}
; } } diff --git a/src/client/views/PropertiesView.tsx b/src/client/views/PropertiesView.tsx index 4e12e0b2d..aefdeee17 100644 --- a/src/client/views/PropertiesView.tsx +++ b/src/client/views/PropertiesView.tsx @@ -995,9 +995,7 @@ export class PropertiesView extends ObservableReactComponent { - DrawingFillHandler.drawingToImage(targetDoc, 'fill in the details of this image'); - }, 'createImage')} + onClick={undoable(() => DrawingFillHandler.drawingToImage(targetDoc, 'fill in the details of this image'), 'createImage')} />
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx index 79aad0ef2..272c13546 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormLayoutEngines.tsx @@ -102,7 +102,6 @@ export function computePassLayout(poolData: Map, pivotDoc: Doc replica: '', }); }); - // eslint-disable-next-line no-use-before-define return normalizeResults(panelDim, 12, docMap, poolData, viewDefsToJSX, [], 0, []); } @@ -272,7 +271,6 @@ export function computePivotLayout(poolData: Map, pivotDoc: Do payload: pivotColumnGroups.get(key)?.filters, })); groupNames.push(...dividers); - // eslint-disable-next-line no-use-before-define return normalizeResults(panelDim, maxText, docMap, poolData, viewDefsToJSX, groupNames, 0, []); } @@ -347,7 +345,6 @@ export function computeTimelineLayout(poolData: Map, pivotDoc: if (!stack && (curTime === undefined || Math.abs(x - (curTime - minTime) * scaling) > pivotAxisWidth)) { groupNames.push({ type: 'text', text: toLabel(key), x: x, y: stack * 25, height: fontHeight, fontSize, payload: undefined }); } - // eslint-disable-next-line no-use-before-define layoutDocsAtTime(keyDocs, key); }); if (sortedKeys.length && curTime !== undefined && curTime > sortedKeys[sortedKeys.length - 1]) { diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 4bfa7fc92..8519cda3c 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -52,6 +52,7 @@ import { FormattedTextBox } from './formattedText/FormattedTextBox'; import { PresEffect, PresEffectDirection } from './trails/PresEnums'; import SpringAnimation from './trails/SlideEffect'; import { SpringType, springMappings } from './trails/SpringUtils'; +import { DrawingFillHandler } from '../smartdraw/DrawingFillHandler'; export interface DocumentViewProps extends FieldViewSharedProps { hideDecorations?: boolean; // whether to suppress all DocumentDecorations when doc is selected @@ -551,6 +552,7 @@ export class DocumentViewInternal extends DocComponent DocumentView.SetLightboxDoc(this.Document), icon: 'external-link-alt' }); } appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'map-pin' }); + appearanceItems.push({ description: 'Make Image', event: () => DrawingFillHandler.drawingToImage(this.Document, StrCast(this.Document.title)), icon: 'map-pin' }); !Doc.noviceMode && templateDoc && appearanceItems.push({ description: 'Open Template ', event: () => this._props.addDocTab(templateDoc, OpenWhere.addRight), icon: 'eye' }); !appearance && appearanceItems.length && cm.addItem({ description: 'Appearance...', subitems: appearanceItems, icon: 'compass' }); @@ -1072,7 +1074,7 @@ export class DocumentView extends DocComponent() { public static GetDocImage(doc: Doc) { return DocumentView.getDocumentView(doc) ?.ComponentView?.updateIcon?.() - .then(() => ImageCast(DocCast(doc).icon)); + .then(() => ImageCast(doc?.icon, ImageCast(doc[Doc.LayoutFieldKey(doc)]))); } public get displayName() { return 'DocumentView(' + (this.Document?.title??"") + ')'; } // prettier-ignore diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index d847b7940..03d417e4e 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -40,7 +40,6 @@ import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; import { Upload } from '../../../server/SharedMediaTypes'; -import { ImageUtils } from '../../util/Import & Export/ImageUtils'; export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -361,6 +360,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { } }; + updateIcon = () => new Promise(res => res()); + choosePath = (url: URL) => { if (!url?.href) return ''; const lower = url.href.toLowerCase(); diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 6f111e95b..32416cec5 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -1,50 +1,22 @@ -// import { action, makeObservable } from 'mobx'; -// import { observer } from 'mobx-react'; -// import React from 'react'; -// import { Doc } from '../../../fields/Doc'; -// import { ImageCast } from '../../../fields/Types'; -// import { ImageField } from '../../../fields/URLField'; -// import { Docs } from '../../documents/Documents'; -// import { Networking } from '../../Network'; -// import { makeUserTemplateButtonOrImage } from '../../util/DropConverter'; -// import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; -// import { ImageUtility } from '../nodes/imageEditor/imageEditorUtils/ImageHandler'; -// import { OpenWhere } from '../nodes/OpenWhere'; -// import { ObservableReactComponent } from '../ObservableReactComponent'; +import { Doc } from '../../../fields/Doc'; +import { ImageCast } from '../../../fields/Types'; +import { Upload } from '../../../server/SharedMediaTypes'; +import { Docs } from '../../documents/Documents'; +import { Networking } from '../../Network'; +import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; +import { OpenWhere } from '../nodes/OpenWhere'; -// export class DrawingFillHandler { -// static drawingToImage = async (drawing: Doc, prompt: string) => { -// const imageField = await DocumentView.GetDocImage(drawing); -// if (!imageField) return; -// const { href } = ImageCast(imageField).url; -// const hrefParts = href.split('.'); -// const hrefComplete = `${hrefParts[0]}_o.${hrefParts[1]}`; -// try { -// const response = await fetch(hrefComplete); -// const blob: Blob = await response.blob(); -// const strength: number = 100; -// const img = await Networking.PostToServer('/oldQueryFireflyImage', { prompt, blob, strength }); -// DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, {}), OpenWhere.addRight); -// // Networking.PostToServer('/oldQueryFireflyImage', { prompt, blob, strength }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, {}), OpenWhere.addRight)); -// } catch (error) { -// console.error('Error fetching image:', error); -// return; -// } - -// // const image = new Image(); -// // image.src = imageField.url?.href; -// // // image.onload = async () => { -// // const canvas = document.createElement('canvas'); -// // canvas.width = image.width; -// // canvas.height = image.height; -// // const ctx = canvas.getContext('2d'); -// // if (!ctx) return; -// // ctx.globalCompositeOperation = 'source-over'; -// // ctx.clearRect(0, 0, image.width, image.height); -// // ctx.drawImage(image, 0, 0); -// // const blob: Blob = await ImageUtility.canvasToBlob(canvas); -// // const strength: number = 100; -// // Networking.PostToServer('/oldQueryFireflyImage', { prompt, blob, strength }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img, {}), OpenWhere.addRight)); -// // }; -// }; -// } +export class DrawingFillHandler { + static drawingToImage = (drawing: Doc, prompt: string) => + DocumentView.GetDocImage(drawing)?.then(imageField => { + if (imageField) { + const { href } = ImageCast(imageField).url; + const hrefParts = href.split('.'); + const structureUrl = `${hrefParts[0]}_o.${hrefParts[1]}`; + const strength: number = 100; + Networking.PostToServer('/queryFireflyImageFromStructure', { prompt, structureUrl, strength }).then((info: Upload.ImageInformation) => + DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, {}), OpenWhere.addRight)) // prettier-ignore + } + return false; + }); +} diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 73cf94206..c84e9e8fa 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -20,21 +20,10 @@ export default class FireflyManager extends ApiManager { return undefined; }); - askFireflyOld = (prompt: string = 'a realistic illustration of a cat coding', uploadId: string, strength: number) => { - const fetched = this.getBearerToken().then(response => - response?.json().then((data: { access_token: string }) => { - const body: any = { - prompt: prompt, - structure: { - strength: strength, - imageReference: { - source: { - uploadId: uploadId, - }, - }, - }, - }; - return fetch('https://firefly-api.adobe.io/v3/images/generate', { + generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', structureUrl: string, strength: number) => + this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ ['Content-Type', 'application/json'], @@ -42,40 +31,40 @@ export default class FireflyManager extends ApiManager { ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], - body: JSON.stringify(body), + body: JSON.stringify({ + prompt, + structure: !structureUrl + ? undefined + : { + strength, + imageReference: { + source: { url: structureUrl }, + }, + }, + }), }) - .then(response => response.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) + .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) .catch(error => { console.error('Error:', error); return ''; + }) + ) + ); + + uploadImageToDropbox = (fileUrl: string, dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN })) => + new Promise((res, rej) => + fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => { + if (err) { + console.log('Error: ', err); + rej(); + } else { + dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }).then(response => { + dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => res(link.result.link)); }); + } }) ); - return fetched; - }; - uploadImageToFirefly = (image: Blob) => { - const fetched = this.getBearerToken().then(response => - response?.json().then((data: { access_token: string }) => - fetch('https://firefly-api.adobe.io/v3/uploads', { - method: 'POST', - headers: [ - ['Content-Type', image.type], - ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], - ['Authorization', `Bearer ${data.access_token}`], - ], - body: image, - }) - .then(response => response.json()) - .then(data => data.uploadId) // Extract the uploadId from the response - .catch(error => { - console.error('Error:', error); - return ''; - }) - ) - ); - return fetched; - }; generateImage = (prompt: string = 'a realistic illustration of a cat coding') => { const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => @@ -222,17 +211,16 @@ export default class FireflyManager extends ApiManager { protected initialize(register: Registration): void { register({ method: Method.POST, - subscription: '/oldQueryFireflyImage', - secureHandler: async ({ req, res }) => { - const { prompt, imageBlob, strength = 0.5 } = req.body; - const uploadId = imageBlob ? await this.uploadImageToFirefly(imageBlob) : null; - this.askFireflyOld(prompt, uploadId, strength).then(fire => - DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info); - }) - ); - }, + subscription: '/queryFireflyImageFromStructure', + secureHandler: async ({ req, res }) => + this.uploadImageToDropbox(req.body.structureUrl).then(uploadUrl => + this.generateImageFromStructure(req.body.prompt, uploadUrl, req.body.strength).then(fire => + DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }) + ) + ), }); register({ method: Method.POST, @@ -263,36 +251,16 @@ export default class FireflyManager extends ApiManager { method: Method.POST, subscription: '/expandImage', secureHandler: ({ req, res }) => - new Promise((resolve, reject) => { - const dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN }); - fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(req.body.file)}`), undefined, (err, contents) => { - if (err) { - console.log('Error: ', err); - reject(); - } else { - dbx.filesUpload({ path: `/Apps/browndash/${path.basename(req.body.file)}`, contents }) - .then(response => { - dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => { - console.log(link.result); - this.expandImage(link.result.link, req.body.prompt).then(text => { - if (text.error_code) _error(res, text.message); - else - DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info); - resolve(); - }); - }); - }); - }) - .catch(uploadErr => { - console.log(uploadErr); - _error(res, 'upload to dropbox failed'); - reject(); - }); - } - }); - }), + this.uploadImageToDropbox(req.body.file).then(uploadUrl => + this.expandImage(uploadUrl, req.body.prompt).then(text => { + if (text.error_code) _error(res, text.message); + else + DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }); + }) + ), }); } } -- cgit v1.2.3-70-g09d2 From d5c4751b1522535ad323bb5eb254b7a1a7b4a63f Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 17 Dec 2024 15:03:07 -0500 Subject: added updateIcon to imageBoxes to allow capturing of strokes for use with generative apis --- src/client/views/nodes/ImageBox.tsx | 26 ++++++++++++++++++++++++-- src/server/ApiManagers/FireflyManager.ts | 2 +- 2 files changed, 25 insertions(+), 3 deletions(-) (limited to 'src/server') diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 03d417e4e..25e7b566f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -7,7 +7,7 @@ import { observer } from 'mobx-react'; import { extname } from 'path'; import * as React from 'react'; import ReactLoading from 'react-loading'; -import { ClientUtils, DashColor, returnEmptyString, returnFalse, returnOne, returnZero, setupMoveUpEvents } from '../../../ClientUtils'; +import { ClientUtils, DashColor, returnEmptyString, returnFalse, returnOne, returnZero, setupMoveUpEvents, UpdateIcon } from '../../../ClientUtils'; import { Doc, DocListCast, Opt } from '../../../fields/Doc'; import { DocData } from '../../../fields/DocSymbols'; import { Id } from '../../../fields/FieldSymbols'; @@ -360,7 +360,29 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { } }; - updateIcon = () => new Promise(res => res()); + // updateIcon = () => new Promise(res => res()); + updateIcon = (usePanelDimensions?: boolean) => { + const contentDiv = this._mainCont.current; + return !contentDiv + ? new Promise(res => res()) + : UpdateIcon( + this.layoutDoc[Id] + '_icon_' + new Date().getTime(), + contentDiv, + usePanelDimensions ? this._props.PanelWidth() : NumCast(this.layoutDoc._width), + usePanelDimensions ? this._props.PanelHeight() : NumCast(this.layoutDoc._height), + this._props.PanelWidth(), + this._props.PanelHeight(), + 0, + 1, + false, + '', + (iconFile, nativeWidth, nativeHeight) => { + this.dataDoc.icon = new ImageField(iconFile); + this.dataDoc.icon_nativeWidth = nativeWidth; + this.dataDoc.icon_nativeHeight = nativeHeight; + } + ); + }; choosePath = (url: URL) => { if (!url?.href) return ''; diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index c84e9e8fa..e73795e41 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -239,7 +239,7 @@ export default class FireflyManager extends ApiManager { subscription: '/queryFireflyImageText', // eslint-disable-next-line @typescript-eslint/no-unused-vars secureHandler: ({ req, res }) => - fetch('http://localhost:1050/files/images/testshot.png').then(json => + fetch(req.body.file).then(json => json.blob().then(file => this.getImageText(file).then(text => { _success(res, text); -- cgit v1.2.3-70-g09d2 From c314477466a96171e5365463ca24d12fc8421bcf Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 18 Dec 2024 14:48:39 -0500 Subject: fixed uploading data: images --- src/server/DashUploadUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/server') diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index a06fa3b6e..623172894 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -459,7 +459,7 @@ export namespace DashUploadUtils { } const outputFile = filename || result.filename || ''; - return UploadInspectedImage(result, outputFile, prefix, isLocal().exec(source) ? true : false); + return UploadInspectedImage(result, outputFile, prefix, isLocal().exec(source) || source.startsWith('data:') ? true : false); }; type md5 = 'md5'; -- cgit v1.2.3-70-g09d2 From 0eff48b757ca81860a883d25e147b8a869e5fe00 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Mon, 30 Dec 2024 23:35:24 -0500 Subject: created image regeneration with dialogue --- src/client/apis/gpt/GPT.ts | 39 +++- src/client/documents/Documents.ts | 4 + src/client/views/PropertiesView.scss | 1 + src/client/views/PropertiesView.tsx | 12 +- .../collectionFreeForm/CollectionFreeFormView.tsx | 22 +-- src/client/views/nodes/ImageBox.tsx | 10 ++ src/client/views/nodes/imageEditor/ImageEditor.tsx | 2 +- src/client/views/pdf/AnchorMenu.tsx | 1 + src/client/views/smartdraw/DrawingFillHandler.tsx | 11 +- src/client/views/smartdraw/SmartDrawHandler.scss | 8 +- src/client/views/smartdraw/SmartDrawHandler.tsx | 197 ++++++++++++++------- src/client/views/smartdraw/StickerPalette.tsx | 1 + src/server/ApiManagers/FireflyManager.ts | 20 ++- src/server/DashUploadUtils.ts | 1 - 14 files changed, 238 insertions(+), 91 deletions(-) (limited to 'src/server') diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 03380e4d6..9241eb120 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -249,6 +249,41 @@ const gptHandwriting = async (src: string): Promise => { } }; +const gptDescribeImage = async (image: string): Promise => { + try { + const response = await openai.chat.completions.create({ + model: 'gpt-4o', + temperature: 0, + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: `Identify what this drawing is, naming as many elements and their location in the drawing as possible`, + }, + { + type: 'image_url', + image_url: { + url: `${image}`, + detail: 'low', + }, + }, + ], + }, + ], + }); + if (response.choices[0].message.content) { + console.log('GPT DESCRIPTION', response.choices[0].message.content); + return response.choices[0].message.content; + } + return 'Unknown drawing'; + } catch (err) { + console.log(err); + return 'Error connecting with API'; + } +}; + const gptDrawingColor = async (image: string, coords: string[]): Promise => { try { const response = await openai.chat.completions.create({ @@ -276,11 +311,11 @@ const gptDrawingColor = async (image: string, coords: string[]): Promise if (response.choices[0].message.content) { return response.choices[0].message.content; } - return 'Missing labels'; + return 'Unknown drawing'; } catch (err) { console.log(err); return 'Error connecting with API'; } }; -export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDrawingColor }; +export { gptAPICall, gptImageCall, GPTCallType, gptImageLabel, gptGetEmbedding, gptHandwriting, gptDescribeImage, gptDrawingColor }; diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index c51c1645d..785af3409 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -516,6 +516,10 @@ export class DocumentOptions { card_sort?: STRt = new StrInfo('way cards are sorted in deck view'); card_sort_isDesc?: BOOLt = new BoolInfo('whether the cards are sorted ascending or descending'); + + ai_generated?: boolean; // to mark items as ai generated + firefly_seed?: number; + firefly_prompt?: string; } export const DocOptions = new DocumentOptions(); diff --git a/src/client/views/PropertiesView.scss b/src/client/views/PropertiesView.scss index 693c75ebf..7866e67e7 100644 --- a/src/client/views/PropertiesView.scss +++ b/src/client/views/PropertiesView.scss @@ -642,6 +642,7 @@ .smooth, .color, +.strength-slider, .smooth-slider { margin-top: 7px; } diff --git a/src/client/views/PropertiesView.tsx b/src/client/views/PropertiesView.tsx index aefdeee17..5b24eb7ea 100644 --- a/src/client/views/PropertiesView.tsx +++ b/src/client/views/PropertiesView.tsx @@ -982,6 +982,9 @@ export class PropertiesView extends ObservableReactComponent { + !isNaN(val) && (this.refStrength = val); + }); return (
{!targetDoc.layout_isSvg && this.containsInkDoc && ( @@ -995,9 +998,10 @@ export class PropertiesView extends ObservableReactComponent DrawingFillHandler.drawingToImage(targetDoc, 'fill in the details of this image'), 'createImage')} + onClick={undoable(() => DrawingFillHandler.drawingToImage(targetDoc, this.refStrength, 'fill in the details of this image'), 'createImage')} />
+
{strength}
{ + doc[DocData].drawing_refStrength = Number(value); + }); + } @computed get smoothAmt() { return Number(this.getField('stroke_smoothAmount') || '5'); } // prettier-ignore set smoothAmt(value) { this.selectedStrokes.forEach(doc => { diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index acf72e5cb..4bccdd286 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1295,6 +1295,7 @@ export class CollectionFreeFormView extends CollectionSubView { - SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; - SmartDrawHandler.Instance.AddDrawing = this.addDrawing; - SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; - !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10) : SmartDrawHandler.Instance.hideRegenerate(); - }), - icon: 'pen-to-square', - }); + this.layoutDoc.drawingData != undefined && + optionItems.push({ + description: 'Show Drawing Editor', + event: action(() => { + SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; + SmartDrawHandler.Instance.AddDrawing = this.addDrawing; + SmartDrawHandler.Instance.RemoveDrawing = this.removeDrawing; + !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this._downX, this._downY - 10) : SmartDrawHandler.Instance.hideRegenerate(); + }), + icon: 'pen-to-square', + }); optionItems.push({ description: this.Document.savedAsSticker ? 'Sticker Saved!' : 'Save to Stickers', event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 25e7b566f..8f6a90e61 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -40,6 +40,7 @@ import { FocusViewOptions } from './FocusViewOptions'; import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; import { Upload } from '../../../server/SharedMediaTypes'; +import { SmartDrawHandler } from '../smartdraw/SmartDrawHandler'; export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -351,6 +352,15 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { }), icon: 'pencil-alt', }); + this.layoutDoc.ai_generated && + funcs.push({ + description: 'Regenerate AI Image', + event: action(() => { + console.log('COOOORDS', this.dataDoc.width as number, this.dataDoc.y as number); + !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this.dataDoc.x as number, (this.dataDoc.y as number) - 10) : SmartDrawHandler.Instance.hideRegenerate(); + }), + icon: 'pen-to-square', + }); funcs.push({ description: this.Document.savedAsSticker ? 'Sticker Saved!' : 'Save to Stickers', event: action(undoable(async () => await StickerPalette.addToPalette(this.Document), 'save to palette')), diff --git a/src/client/views/nodes/imageEditor/ImageEditor.tsx b/src/client/views/nodes/imageEditor/ImageEditor.tsx index a39878924..2a8bc034d 100644 --- a/src/client/views/nodes/imageEditor/ImageEditor.tsx +++ b/src/client/views/nodes/imageEditor/ImageEditor.tsx @@ -411,7 +411,7 @@ const ImageEditor = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc let finalImgURL: string = url; // crop the image for these brush modes to remove excess blank space around the image contents if (currCutType == CutMode.IN || currCutType == CutMode.DRAW_IN) { - const croppedData = cropImage(image, minX, maxX, minY, maxY); + const croppedData = cropImage(image, Math.max(minX, 0), Math.min(maxX, image.width), Math.max(minY, 0), Math.min(maxY, image.height)); finalImg = croppedData; finalImgURL = croppedData.src; } diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index fe03f32a5..bb8082061 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -158,6 +158,7 @@ export class AnchorMenu extends AntimodeMenu { docData.drawingColored = opts.autoColor; docData.drawingSize = opts.size; docData.drawingData = gptRes; + docData.ai_generated = true; }); pointerDown = (e: React.PointerEvent) => { diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 48e71bc9f..1a470f995 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -1,21 +1,26 @@ +import { imageUrlToBase64 } from '../../../ClientUtils'; import { Doc } from '../../../fields/Doc'; import { ImageCast } from '../../../fields/Types'; import { Upload } from '../../../server/SharedMediaTypes'; +import { gptDescribeImage } from '../../apis/gpt/GPT'; import { Docs } from '../../documents/Documents'; import { Networking } from '../../Network'; import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { OpenWhere } from '../nodes/OpenWhere'; export class DrawingFillHandler { - static drawingToImage = (drawing: Doc, prompt: string) => + static drawingToImage = (drawing: Doc, strength: number, prompt: string) => DocumentView.GetDocImage(drawing)?.then(imageField => { if (imageField) { const { href } = ImageCast(imageField).url; const hrefParts = href.split('.'); const structureUrl = `${hrefParts.slice(0, -1).join('.')}_o.${hrefParts.lastElement()}`; - const strength: number = 100; - Networking.PostToServer('/queryFireflyImageFromStructure', { prompt, structureUrl, strength }).then((info: Upload.ImageInformation) => + imageUrlToBase64(structureUrl) + .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) + .then((prompt: string) => { + Networking.PostToServer('/queryFireflyImageFromStructure', { prompt, structureUrl, strength }).then((info: Upload.ImageInformation) => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, {}), OpenWhere.addRight)) // prettier-ignore + }); } return false; }); diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss index c25273876..513779512 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.scss +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -12,7 +12,13 @@ } } - .smartdraw-options { + .smartdraw-output-options { + display: flex; + flex-direction: row; + justify-content: center; + } + + .smartdraw-svg-options { margin-top: 5px; display: flex; flex-direction: row; diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 036ac5983..fb1a5771e 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -1,5 +1,5 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { Slider, Switch } from '@mui/material'; +import { Checkbox, Slider, Switch } from '@mui/material'; import { Button, IconButton } from 'browndash-components'; import { action, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; @@ -74,6 +74,8 @@ export class SmartDrawHandler extends ObservableReactComponent { @observable private _autoColor: boolean = true; @observable private _regenInput: string = ''; @observable private _canInteract: boolean = true; + @observable private _generateDrawing: boolean = true; + @observable private _generateImage: boolean = true; @observable public ShowRegenerate: boolean = false; @@ -195,6 +197,7 @@ export class SmartDrawHandler extends ObservableReactComponent { */ @action handleSendClick = async () => { + if (!this._generateImage && !this._generateDrawing) return; this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { @@ -212,7 +215,12 @@ export class SmartDrawHandler extends ObservableReactComponent { this._showOptions = false; }); try { - await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + if (this._generateImage) { + await this.createImageWithFirefly(this._userInput); + } + if (this._generateDrawing) { + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + } this.hideSmartDrawHandler(); runInAction(() => { @@ -240,15 +248,12 @@ export class SmartDrawHandler extends ObservableReactComponent { drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { if (input) { this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; - - Networking.PostToServer('/queryFireflyImage', { prompt: input }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input }), OpenWhere.addRight)); - const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); if (res) { const strokeData = await this.parseSvg(res, startPt, false, autoColor); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - + this._selectedDoc = drawingDoc; this._errorOccurredOnce = false; return strokeData; } else { @@ -258,6 +263,23 @@ export class SmartDrawHandler extends ObservableReactComponent { return undefined; }; + /** + * Calls Firefly API to create an image based on user input + */ + createImageWithFirefly = (input: string, seed?: number) => { + this._lastInput.text = input; + return Networking.PostToServer('/queryFireflyImage', { prompt: input, seed: seed }).then(img => { + const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { + title: input.match(/^(.*?)~~~.*$/)?.[1] || input, + ai_generated: true, + firefly_seed: img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1], + firefly_prompt: input, + }); + DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); + this._selectedDoc = imgDoc; + }); + }; + /** * Regenerates drawings with the option to add a specific regenerate prompt/request. */ @@ -266,27 +288,39 @@ export class SmartDrawHandler extends ObservableReactComponent { if (lastInput) this._lastInput = lastInput; if (lastResponse) this._lastResponse = lastResponse; if (regenInput) this._regenInput = regenInput; - - try { - let res; + if (this._generateDrawing) { + try { + let res; + if (this._regenInput !== '') { + const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + } else { + res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + } + if (!res) { + console.error('GPT call failed'); + return; + } + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + } catch (err) { + console.error('Error regenerating drawing', err); + } + } + if (this._generateImage) { if (this._regenInput !== '') { - const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; - res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); - this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + if (this._selectedDoc) { + const docData = this._selectedDoc[DocData]; + const newPrompt = `${docData.firefly_prompt}, ${this._regenInput}`; + const seed: number = docData?.firefly_seed as number; + await this.createImageWithFirefly(newPrompt, seed); + } } else { - res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + await this.createImageWithFirefly(this._lastInput.text); } - if (!res) { - console.error('GPT call failed'); - return; - } - const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); - this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - return strokeData; - } catch (err) { - console.error('Error regenerating drawing', err); } }; @@ -397,58 +431,87 @@ export class SmartDrawHandler extends ObservableReactComponent { {this._showOptions && (
-
-
- Auto color - this._canInteract && (this._autoColor = !this._autoColor))} - /> -
-
- Complexity - +
+ Generate Ink + this._canInteract && (this._complexity = val as number))} - valueLabelDisplay="auto" + checked={this._generateDrawing} + onChange={() => this._canInteract && (this._generateDrawing = !this._generateDrawing)} />
-
- Size (in pixels) - + Generate Image + this._canInteract && (this._size = val as number))} - valueLabelDisplay="auto" + checked={this._generateImage} + onChange={() => this._canInteract && (this._generateImage = !this._generateImage)} />
-
+ {this._generateDrawing && ( +
+
+ Auto color + this._canInteract && (this._autoColor = !this._autoColor))} + /> +
+
+ Complexity + this._canInteract && (this._complexity = val as number))} + valueLabelDisplay="auto" + /> +
+
+ Size (in pixels) + this._canInteract && (this._size = val as number))} + valueLabelDisplay="auto" + /> +
+
+ )}
)}
diff --git a/src/client/views/smartdraw/StickerPalette.tsx b/src/client/views/smartdraw/StickerPalette.tsx index d56878f10..352a02e32 100644 --- a/src/client/views/smartdraw/StickerPalette.tsx +++ b/src/client/views/smartdraw/StickerPalette.tsx @@ -186,6 +186,7 @@ export class StickerPalette extends ObservableReactComponent { + generateImage = (prompt: string = 'a realistic illustration of a cat coding', seed?: number) => { + let body = `{ "prompt": "${prompt}" }`; + if (seed) { + body = `{ "prompt": "${prompt}", "seeds": [${seed}]}`; + } const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { @@ -76,9 +80,15 @@ export default class FireflyManager extends ApiManager { ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], ['Authorization', `Bearer ${data.access_token}`], ], - body: `{ "prompt": "${prompt}" }`, + body: body, }) - .then(response2 => response2.json().then(json => (json.outputs?.[0] as { image: { url: string } })?.image.url)) + .then(response2 => + response2.json().then(json => { + const seed = json.outputs?.[0]?.seed; + const url = json.outputs?.[0]?.image?.url; + return { seed, url }; + }) + ) .catch(error => { console.error('Error:', error); return undefined; @@ -226,8 +236,8 @@ export default class FireflyManager extends ApiManager { method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.generateImage(req.body.prompt).then(url => - DashUploadUtils.UploadImage(url ?? '').then(info => { + this.generateImage(req.body.prompt, req.body.seed).then(img => + DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); }) diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 623172894..2177c5d97 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -458,7 +458,6 @@ export namespace DashUploadUtils { return { name: result.name, message: result.message }; } const outputFile = filename || result.filename || ''; - return UploadInspectedImage(result, outputFile, prefix, isLocal().exec(source) || source.startsWith('data:') ? true : false); }; -- cgit v1.2.3-70-g09d2 From 4eebcc75a174b82033b5092f837b1477bcb628eb Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Wed, 1 Jan 2025 21:28:23 -0500 Subject: added size option to image generation --- src/client/views/smartdraw/SmartDrawHandler.scss | 64 +++++--- src/client/views/smartdraw/SmartDrawHandler.tsx | 179 ++++++++++++++--------- src/server/ApiManagers/FireflyManager.ts | 10 +- 3 files changed, 157 insertions(+), 96 deletions(-) (limited to 'src/server') diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss index 513779512..4b21c92a5 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.scss +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -1,6 +1,6 @@ .smart-draw-handler { position: absolute; - // width: 265px; + width: 265px; .smart-draw-main { display: flex; @@ -18,36 +18,54 @@ justify-content: center; } - .smartdraw-svg-options { - margin-top: 5px; + .smartdraw-options-container { + padding: 5px; + font-weight: bolder; + text-align: center; display: flex; - flex-direction: row; - justify-content: space-around; + flex-direction: column; - .auto-color { + .smartdraw-options { + font-weight: normal; + margin-top: 5px; display: flex; - flex-direction: column; - align-items: center; - width: 30%; - } + flex-direction: row; + justify-content: space-around; - .complexity { - display: flex; - flex-direction: column; - align-items: center; - width: 31%; - } + .smartdraw-auto-color { + display: flex; + flex-direction: column; + align-items: center; + width: 30%; + margin-top: 3px; + } - .size { - display: flex; - flex-direction: column; - align-items: center; - width: 39%; + .smartdraw-complexity { + display: flex; + flex-direction: column; + align-items: center; + width: 31%; + margin-top: 3px; + } - .size-slider { - width: 80%; + .smartdraw-size { + display: flex; + flex-direction: column; + align-items: center; + width: 39%; + margin-top: 3px; } } + + .smartdraw-dimensions { + font-weight: normal; + margin-top: 7px; + padding-left: 25px; + } + } + + .smartdraw-slider { + width: 65px; } .regenerate-box, diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index fb1a5771e..ed2734c8b 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -1,5 +1,5 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { Checkbox, Slider, Switch } from '@mui/material'; +import { Checkbox, FormControlLabel, Radio, RadioGroup, Slider, Switch } from '@mui/material'; import { Button, IconButton } from 'browndash-components'; import { action, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; @@ -35,6 +35,13 @@ export interface DrawingOptions { y: number; } +enum FireflyImageDimensions { + Square = 'square', + Landscape = 'landscape', + Portrait = 'portrait', + Widescreen = 'widescreen', +} + /** * The SmartDrawHandler allows users to generate drawings with GPT from text input. Users are able to enter * the item to draw, how complex they want the drawing to be, how large the drawing should be, and whether @@ -66,13 +73,16 @@ export class SmartDrawHandler extends ObservableReactComponent { @observable private _scale: number = 0; @observable private _yRelativeToTop: boolean = true; @observable private _isLoading: boolean = false; + @observable private _userInput: string = ''; + @observable private _regenInput: string = ''; @observable private _showOptions: boolean = false; @observable private _showEditBox: boolean = false; @observable private _complexity: number = 5; @observable private _size: number = 200; @observable private _autoColor: boolean = true; - @observable private _regenInput: string = ''; + @observable private _imgDims: FireflyImageDimensions = FireflyImageDimensions.Square; + @observable private _canInteract: boolean = true; @observable private _generateDrawing: boolean = true; @observable private _generateImage: boolean = true; @@ -197,7 +207,7 @@ export class SmartDrawHandler extends ObservableReactComponent { */ @action handleSendClick = async () => { - if (!this._generateImage && !this._generateDrawing) return; + if ((!this.ShowRegenerate && this._userInput == '') || (!this._generateImage && !this._generateDrawing)) return; this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { @@ -207,10 +217,6 @@ export class SmartDrawHandler extends ObservableReactComponent { this._showEditBox = false; }); } else { - if (this._userInput == '') { - this._isLoading = false; - return; - } runInAction(() => { this._showOptions = false; }); @@ -232,7 +238,7 @@ export class SmartDrawHandler extends ObservableReactComponent { this._errorOccurredOnce = false; } else { this._errorOccurredOnce = true; - await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + await this.handleSendClick(); } } } @@ -268,7 +274,26 @@ export class SmartDrawHandler extends ObservableReactComponent { */ createImageWithFirefly = (input: string, seed?: number) => { this._lastInput.text = input; - return Networking.PostToServer('/queryFireflyImage', { prompt: input, seed: seed }).then(img => { + let width = 0; + let height = 0; + switch (this._imgDims) { + case FireflyImageDimensions.Square: + width = 2048; + height = 2048; + break; + case FireflyImageDimensions.Landscape: + width = 2304; + height = 1792; + case FireflyImageDimensions.Portrait: + width = 1792; + height = 2304; + break; + case FireflyImageDimensions.Widescreen: + width = 2688; + height = 1536; + break; + } + return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: width, height: height, seed: seed }).then(img => { const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input.match(/^(.*?)~~~.*$/)?.[1] || input, ai_generated: true, @@ -288,6 +313,18 @@ export class SmartDrawHandler extends ObservableReactComponent { if (lastInput) this._lastInput = lastInput; if (lastResponse) this._lastResponse = lastResponse; if (regenInput) this._regenInput = regenInput; + if (this._generateImage) { + if (this._regenInput !== '') { + if (this._selectedDoc) { + const docData = this._selectedDoc[DocData]; + const newPrompt = `${docData.firefly_prompt}, ${this._regenInput}`; + const seed: number = docData?.firefly_seed as number; + await this.createImageWithFirefly(newPrompt, seed); + } + } else { + await this.createImageWithFirefly(this._lastInput.text); + } + } if (this._generateDrawing) { try { let res; @@ -310,18 +347,6 @@ export class SmartDrawHandler extends ObservableReactComponent { console.error('Error regenerating drawing', err); } } - if (this._generateImage) { - if (this._regenInput !== '') { - if (this._selectedDoc) { - const docData = this._selectedDoc[DocData]; - const newPrompt = `${docData.firefly_prompt}, ${this._regenInput}`; - const seed: number = docData?.firefly_seed as number; - await this.createImageWithFirefly(newPrompt, seed); - } - } else { - await this.createImageWithFirefly(this._lastInput.text); - } - } }; /** @@ -460,55 +485,71 @@ export class SmartDrawHandler extends ObservableReactComponent { {this._generateDrawing && ( -
-
- Auto color - this._canInteract && (this._autoColor = !this._autoColor))} - /> -
-
- Complexity - this._canInteract && (this._complexity = val as number))} - valueLabelDisplay="auto" - /> +
+ Drawing Options +
+
+ Auto color + this._canInteract && (this._autoColor = !this._autoColor))} + /> +
+
+ Complexity + this._canInteract && (this._complexity = val as number))} + valueLabelDisplay="auto" + /> +
+
+ Size (in pixels) + this._canInteract && (this._size = val as number))} + valueLabelDisplay="auto" + /> +
-
- Size (in pixels) - this._canInteract && (this._size = val as number))} - valueLabelDisplay="auto" - /> +
+ )} + {this._generateImage && ( +
+ Image Options +
+ + } onChange={() => this._canInteract && (this._imgDims = FireflyImageDimensions.Square)} label="Square" /> + } onChange={() => this._canInteract && (this._imgDims = FireflyImageDimensions.Landscape)} label="Landscape" /> + } onChange={() => this._canInteract && (this._imgDims = FireflyImageDimensions.Portrait)} label="Portrait" /> + } onChange={() => this._canInteract && (this._imgDims = FireflyImageDimensions.Widescreen)} label="Widescreen" /> +
)} diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index c5348c7db..a41492745 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -65,10 +65,12 @@ export default class FireflyManager extends ApiManager { }) ); - generateImage = (prompt: string = 'a realistic illustration of a cat coding', seed?: number) => { - let body = `{ "prompt": "${prompt}" }`; + generateImage = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, seed?: number) => { + console.log('DIMENSIONS', width, height); + let body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} }`; if (seed) { - body = `{ "prompt": "${prompt}", "seeds": [${seed}]}`; + console.log('RECEIVED SEED', seed); + body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}}, "seeds": [${seed}]}`; } const fetched = this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => @@ -236,7 +238,7 @@ export default class FireflyManager extends ApiManager { method: Method.POST, subscription: '/queryFireflyImage', secureHandler: ({ req, res }) => - this.generateImage(req.body.prompt, req.body.seed).then(img => + this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img => DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); -- cgit v1.2.3-70-g09d2 From 383b0487d5268bd860e514feddf09f4f3eb2fe3f Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Thu, 2 Jan 2025 01:13:50 -0500 Subject: made drawing fill automatically size images --- src/client/util/bezierFit.ts | 1 + .../collectionFreeForm/CollectionFreeFormView.tsx | 2 +- src/client/views/nodes/ImageBox.tsx | 5 +- .../imageEditor/imageEditingUtils/ImageHandler.ts | 312 --------------------- src/client/views/smartdraw/DrawingFillHandler.tsx | 18 +- src/client/views/smartdraw/FireflyConstants.ts | 20 ++ src/client/views/smartdraw/SmartDrawHandler.scss | 3 +- src/client/views/smartdraw/SmartDrawHandler.tsx | 34 +-- src/server/ApiManagers/FireflyManager.ts | 7 +- 9 files changed, 51 insertions(+), 351 deletions(-) delete mode 100644 src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts create mode 100644 src/client/views/smartdraw/FireflyConstants.ts (limited to 'src/server') diff --git a/src/client/util/bezierFit.ts b/src/client/util/bezierFit.ts index 84b27e84c..7ef370d48 100644 --- a/src/client/util/bezierFit.ts +++ b/src/client/util/bezierFit.ts @@ -703,6 +703,7 @@ export function SVGToBezier(name: SVGType, attributes: any): Point[] { coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[1]), Y: parseInt(match[2]) }); coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); + coordList.push({ X: parseInt(match[3]), Y: parseInt(match[4]) }); lastPt = { X: parseInt(match[3]), Y: parseInt(match[4]) }; } else if (match[0].startsWith('C')) { coordList.push({ X: parseInt(match[5]), Y: parseInt(match[6]) }); diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 4bccdd286..ef0b80720 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1995,7 +1995,7 @@ export class CollectionFreeFormView extends CollectionSubView { SmartDrawHandler.Instance.CreateDrawingDoc = this.createDrawingDoc; SmartDrawHandler.Instance.AddDrawing = this.addDrawing; diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 8f6a90e61..7ce429f0f 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -355,9 +355,8 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { this.layoutDoc.ai_generated && funcs.push({ description: 'Regenerate AI Image', - event: action(() => { - console.log('COOOORDS', this.dataDoc.width as number, this.dataDoc.y as number); - !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(this.dataDoc.x as number, (this.dataDoc.y as number) - 10) : SmartDrawHandler.Instance.hideRegenerate(); + event: action(e => { + !SmartDrawHandler.Instance.ShowRegenerate ? SmartDrawHandler.Instance.displayRegenerate(e?.x || 0, e?.y || 0) : SmartDrawHandler.Instance.hideRegenerate(); }), icon: 'pen-to-square', }); diff --git a/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts b/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts deleted file mode 100644 index 514e8a94f..000000000 --- a/src/client/views/nodes/imageEditor/imageEditingUtils/ImageHandler.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { RefObject } from 'react'; -import { bgColor, canvasSize } from '../ImageEditorUtils/imageEditorConstants'; - -export interface APISuccess { - status: 'success'; - urls: string[]; -} - -export interface APIError { - status: 'error'; - message: string; -} - -export class ImageUtility { - /** - * - * @param canvas Canvas to convert - * @returns Blob of canvas - */ - static canvasToBlob = (canvas: HTMLCanvasElement): Promise => - new Promise(resolve => { - canvas.toBlob(blob => { - if (blob) { - resolve(blob); - } - }, 'image/png'); - }); - - // given a square api image, get the cropped img - static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { - // Create a new canvas element - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - const ctx = canvas.getContext('2d'); - if (ctx) { - // Clear the canvas - ctx.clearRect(0, 0, canvas.width, canvas.height); - if (width < height) { - // horizontal padding, x offset - const xOffset = (canvasSize - width) / 2; - ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); - } else { - // vertical padding, y offset - const yOffset = (canvasSize - height) / 2; - ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); - } - return canvas; - } - return undefined; - }; - - // converts an image to a canvas data url - static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise => - new Promise((resolve, reject) => { - const img = new Image(); - img.onload = () => { - const canvas = this.getCroppedImg(img, width, height); - if (canvas) { - const dataUrl = canvas.toDataURL(); - resolve(dataUrl); - } - }; - img.onerror = error => { - reject(error); - }; - img.src = imageSrc; - }); - - // calls the openai api to get image edits - static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise => { - const apiUrl = 'https://api.openai.com/v1/images/edits'; - const fd = new FormData(); - fd.append('image', imgBlob, 'image.png'); - fd.append('mask', maskBlob, 'mask.png'); - fd.append('prompt', prompt); - fd.append('size', '1024x1024'); - fd.append('n', n ? JSON.stringify(n) : '1'); - fd.append('response_format', 'b64_json'); - - try { - const res = await fetch(apiUrl, { - method: 'POST', - headers: { - Authorization: `Bearer ${process.env.OPENAI_KEY}`, - }, - body: fd, - }); - const data = await res.json(); - console.log(data.data); - return { - status: 'success', - urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`), - }; - } catch (err) { - console.log(err); - return { status: 'error', message: 'API error.' }; - } - }; - - // mock api call - static mockGetEdit = async (mockSrc: string): Promise => ({ - status: 'success', - urls: [mockSrc, mockSrc, mockSrc], - }); - - // Gets the canvas rendering context of a canvas - static getCanvasContext = (canvasRef: RefObject): CanvasRenderingContext2D | null => { - if (!canvasRef.current) return null; - const ctx = canvasRef.current.getContext('2d'); - if (!ctx) return null; - return ctx; - }; - - // Helper for downloading the canvas (for debugging) - static downloadCanvas = (canvas: HTMLCanvasElement) => { - const url = canvas.toDataURL(); - const downloadLink = document.createElement('a'); - downloadLink.href = url; - downloadLink.download = 'canvas'; - - downloadLink.click(); - downloadLink.remove(); - }; - - // Download the canvas (for debugging) - static downloadImageCanvas = (imgUrl: string) => { - const img = new Image(); - img.src = imgUrl; - img.onload = () => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); - - this.downloadCanvas(canvas); - }; - }; - - // Clears the canvas - static clearCanvas = (canvasRef: React.RefObject) => { - const ctx = this.getCanvasContext(canvasRef); - if (!ctx || !canvasRef.current) return; - ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); - }; - - // Draws the image to the current canvas - static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject, width: number, height: number) => { - const drawImg = (htmlImg: HTMLImageElement) => { - const ctx = this.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.globalCompositeOperation = 'source-over'; - ctx.clearRect(0, 0, width, height); - ctx.drawImage(htmlImg, 0, 0, width, height); - }; - - if (img.complete) { - drawImg(img); - } else { - img.onload = () => { - drawImg(img); - }; - } - }; - - // Gets the image mask for the openai endpoint - static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - if (!ctx) return undefined; - ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.drawImage(paddedCanvas, 0, 0); - - // extract and set padding data - if (srcCanvas.height > srcCanvas.width) { - // horizontal padding, x offset - const xOffset = (canvasSize - srcCanvas.width) / 2; - ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); - ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); - } else { - // vertical padding, y offset - const yOffset = (canvasSize - srcCanvas.height) / 2; - ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); - ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); - } - return canvas; - }; - - // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) - static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const { data } = imageData; - for (let i = 0; i < canvas.height; i++) { - for (let j = 0; j < xOffset; j++) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceI = i; - const sourceJ = xOffset + (xOffset - j); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - for (let i = 0; i < canvas.height; i++) { - for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceI = i; - const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - ctx.putImageData(imageData, 0, 0); - }; - - // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) - static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const { data } = imageData; - for (let j = 0; j < canvas.width; j++) { - for (let i = 0; i < yOffset; i++) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceJ = j; - const sourceI = yOffset + (yOffset - i); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - for (let j = 0; j < canvas.width; j++) { - for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceJ = j; - const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - ctx.putImageData(imageData, 0, 0); - }; - - // Gets the unaltered (besides filling in padding) version of the image for the api call - static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - if (!ctx) return undefined; - // fix scaling - const scale = Math.min(canvasSize / img.width, canvasSize / img.height); - const width = Math.floor(img.width * scale); - const height = Math.floor(img.height * scale); - ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.fillStyle = bgColor; - ctx.fillRect(0, 0, canvasSize, canvasSize); - - // extract and set padding data - if (img.naturalHeight > img.naturalWidth) { - // horizontal padding, x offset - const xOffset = Math.floor((canvasSize - width) / 2); - ctx.drawImage(img, xOffset, 0, width, height); - - // draw reflected image padding - this.drawHorizontalReflection(ctx, canvas, xOffset); - } else { - // vertical padding, y offset - const yOffset = Math.floor((canvasSize - height) / 2); - ctx.drawImage(img, 0, yOffset, width, height); - - // draw reflected image padding - this.drawVerticalReflection(ctx, canvas, yOffset); - } - return canvas; - }; - - /** - * Converts a url to base64 (tainted canvas workaround) - */ - static urlToBase64 = async (imageUrl: string): Promise => { - try { - const res = await fetch(imageUrl); - const blob = await res.blob(); - - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const base64Data = reader.result?.toString().split(',')[1]; - if (base64Data) { - resolve(base64Data); - } else { - reject(new Error('Failed to convert.')); - } - }; - reader.onerror = () => { - reject(new Error('Error reading image data')); - }; - reader.readAsDataURL(blob); - }); - } catch (err) { - console.error(err); - } - return undefined; - }; -} diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 1a470f995..7a95e27c2 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -1,5 +1,6 @@ import { imageUrlToBase64 } from '../../../ClientUtils'; import { Doc } from '../../../fields/Doc'; +import { DocData } from '../../../fields/DocSymbols'; import { ImageCast } from '../../../fields/Types'; import { Upload } from '../../../server/SharedMediaTypes'; import { gptDescribeImage } from '../../apis/gpt/GPT'; @@ -7,19 +8,32 @@ import { Docs } from '../../documents/Documents'; import { Networking } from '../../Network'; import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { OpenWhere } from '../nodes/OpenWhere'; +import { AspectRatioLimits, FireflyDimensionsMap, FireflyImageDimensions } from './FireflyConstants'; export class DrawingFillHandler { static drawingToImage = (drawing: Doc, strength: number, prompt: string) => DocumentView.GetDocImage(drawing)?.then(imageField => { if (imageField) { + const aspectRatio = (drawing.width as number) / (drawing.height as number); + let dims: { width: number; height: number }; + if (aspectRatio > AspectRatioLimits[FireflyImageDimensions.Widescreen]) { + dims = FireflyDimensionsMap[FireflyImageDimensions.Widescreen]; + } else if (aspectRatio > AspectRatioLimits[FireflyImageDimensions.Landscape]) { + dims = FireflyDimensionsMap[FireflyImageDimensions.Landscape]; + } else if (aspectRatio < AspectRatioLimits[FireflyImageDimensions.Portrait]) { + dims = FireflyDimensionsMap[FireflyImageDimensions.Portrait]; + } else { + dims = FireflyDimensionsMap[FireflyImageDimensions.Square]; + } const { href } = ImageCast(imageField).url; const hrefParts = href.split('.'); const structureUrl = `${hrefParts.slice(0, -1).join('.')}_o.${hrefParts.lastElement()}`; imageUrlToBase64(structureUrl) .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) .then((prompt: string) => { - Networking.PostToServer('/queryFireflyImageFromStructure', { prompt, structureUrl, strength }).then((info: Upload.ImageInformation) => - DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, {}), OpenWhere.addRight)) // prettier-ignore + Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: prompt, width: dims.width, height: dims.height, structureUrl, strength }).then((info: Upload.ImageInformation) => + DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai_generated: true, nativeWidth: dims.width, nativeHeight: dims.height }), OpenWhere.addRight) + ); // prettier-ignore }); } return false; diff --git a/src/client/views/smartdraw/FireflyConstants.ts b/src/client/views/smartdraw/FireflyConstants.ts new file mode 100644 index 000000000..f51305fba --- /dev/null +++ b/src/client/views/smartdraw/FireflyConstants.ts @@ -0,0 +1,20 @@ +export enum FireflyImageDimensions { + Square = 'square', + Landscape = 'landscape', + Portrait = 'portrait', + Widescreen = 'widescreen', +} + +export const FireflyDimensionsMap = { + square: { width: 2048, height: 2048 }, + landscape: { width: 2304, height: 1792 }, + portrait: { width: 1792, height: 2304 }, + widescreen: { width: 2688, height: 1536 }, +}; + +export const AspectRatioLimits = { + square: 1, + landscape: 1.167, + portrait: 0.875, + widescreen: 1.472, +}; diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss index 4b21c92a5..cca7d77c7 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.scss +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -1,6 +1,5 @@ .smart-draw-handler { position: absolute; - width: 265px; .smart-draw-main { display: flex; @@ -16,9 +15,11 @@ display: flex; flex-direction: row; justify-content: center; + margin-top: 3px; } .smartdraw-options-container { + width: 265px; padding: 5px; font-weight: bolder; text-align: center; diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 8cff2174f..6c9470480 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -25,6 +25,7 @@ import { ActiveInkArrowEnd, ActiveInkArrowStart, ActiveInkDash, ActiveInkFillCol import './SmartDrawHandler.scss'; import { Networking } from '../../Network'; import { OpenWhere } from '../nodes/OpenWhere'; +import { FireflyDimensionsMap, FireflyImageDimensions } from './FireflyConstants'; export interface DrawingOptions { text: string; @@ -35,13 +36,6 @@ export interface DrawingOptions { y: number; } -enum FireflyImageDimensions { - Square = 'square', - Landscape = 'landscape', - Portrait = 'portrait', - Widescreen = 'widescreen', -} - /** * The SmartDrawHandler allows users to generate drawings with GPT from text input. Users are able to enter * the item to draw, how complex they want the drawing to be, how large the drawing should be, and whether @@ -274,30 +268,12 @@ export class SmartDrawHandler extends ObservableReactComponent { */ createImageWithFirefly = (input: string, seed?: number) => { this._lastInput.text = input; - let width = 0; - let height = 0; - switch (this._imgDims) { - case FireflyImageDimensions.Square: - width = 2048; - height = 2048; - break; - case FireflyImageDimensions.Landscape: - width = 2304; - height = 1792; - case FireflyImageDimensions.Portrait: - width = 1792; - height = 2304; - break; - case FireflyImageDimensions.Widescreen: - width = 2688; - height = 1536; - break; - } - return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: width, height: height, seed: seed }).then(img => { + const dims = FireflyDimensionsMap[this._imgDims]; + return Networking.PostToServer('/queryFireflyImage', { prompt: input, width: dims.width, height: dims.height, seed: seed }).then(img => { const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input.match(/^(.*?)~~~.*$/)?.[1] || input, - nativeWidth: width, - nativeHeight: height, + nativeWidth: dims.width, + nativeHeight: dims.height, ai_generated: true, firefly_seed: img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1], firefly_prompt: input, diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index a41492745..4c4aac5e0 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -20,7 +20,7 @@ export default class FireflyManager extends ApiManager { return undefined; }); - generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', structureUrl: string, strength: number) => + generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50) => this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => fetch('https://firefly-api.adobe.io/v3/images/generate', { @@ -32,7 +32,8 @@ export default class FireflyManager extends ApiManager { ['Authorization', `Bearer ${data.access_token}`], ], body: JSON.stringify({ - prompt, + prompt: prompt, + size: { width: width, height: height }, structure: !structureUrl ? undefined : { @@ -226,7 +227,7 @@ export default class FireflyManager extends ApiManager { subscription: '/queryFireflyImageFromStructure', secureHandler: async ({ req, res }) => this.uploadImageToDropbox(req.body.structureUrl).then(uploadUrl => - this.generateImageFromStructure(req.body.prompt, uploadUrl, req.body.strength).then(fire => + this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploadUrl, req.body.strength).then(fire => DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); -- cgit v1.2.3-70-g09d2 From 6e7cb570f9bad527cd4772bb5c715dd588fb77df Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Thu, 2 Jan 2025 22:39:25 -0500 Subject: tags can now be used as style presets --- src/client/apis/gpt/GPT.ts | 2 +- src/client/views/DocumentButtonBar.tsx | 18 ++++++++++++++++++ src/client/views/nodes/DocumentView.tsx | 2 ++ src/client/views/smartdraw/DrawingFillHandler.tsx | 10 +++++++--- src/client/views/smartdraw/FireflyConstants.ts | 22 ++++++++++++++++++++++ src/server/ApiManagers/FireflyManager.ts | 7 +++++-- 6 files changed, 55 insertions(+), 6 deletions(-) (limited to 'src/server') diff --git a/src/client/apis/gpt/GPT.ts b/src/client/apis/gpt/GPT.ts index 9241eb120..4b0d58cc1 100644 --- a/src/client/apis/gpt/GPT.ts +++ b/src/client/apis/gpt/GPT.ts @@ -260,7 +260,7 @@ const gptDescribeImage = async (image: string): Promise => { content: [ { type: 'text', - text: `Identify what this drawing is, naming as many elements and their location in the drawing as possible`, + text: `Briefly identify what this drawing is, naming all the drawing elements and their location within the image. Do not include anything about the drawing style.`, }, { type: 'image_url', diff --git a/src/client/views/DocumentButtonBar.tsx b/src/client/views/DocumentButtonBar.tsx index 32bf67df1..b7033af3f 100644 --- a/src/client/views/DocumentButtonBar.tsx +++ b/src/client/views/DocumentButtonBar.tsx @@ -314,6 +314,23 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( ); } + @computed + get aiEditorButton() { + const targetDoc = this.view0?.Document; + return !targetDoc ? null : ( + Edit with AI}> +
{ + CalendarManager.Instance.open(this.view0, targetDoc); + }}> + +
+ + ); + } + @observable _isRecording = false; _stopFunc: () => void = emptyFunction; @computed @@ -484,6 +501,7 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => (
{this.pinButton}
{this.recordButton}
{this.calendarButton}
+
{this.aiEditorButton}
{this.keywordButton}
{!Doc.UserDoc().documentLinksButton_fullMenu ? null :
{this.shareButton}
}
{this.menuButton}
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx index 8519cda3c..048b92c71 100644 --- a/src/client/views/nodes/DocumentView.tsx +++ b/src/client/views/nodes/DocumentView.tsx @@ -1400,6 +1400,8 @@ export class DocumentView extends DocComponent() { NativeHeight = () => this.effectiveNativeHeight; PanelWidth = () => this.panelWidth; PanelHeight = () => this.panelHeight; + ReducedPanelWidth = () => this.panelWidth / 2; + ReducedPanelHeight = () => this.panelWidth / 2; NativeDimScaling = () => this.nativeScaling; hideLinkCount = () => !!this.hideLinkButton; isHovering = () => this._isHovering; diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 7a95e27c2..52652d377 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -8,10 +8,13 @@ import { Docs } from '../../documents/Documents'; import { Networking } from '../../Network'; import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { OpenWhere } from '../nodes/OpenWhere'; -import { AspectRatioLimits, FireflyDimensionsMap, FireflyImageDimensions } from './FireflyConstants'; +import { AspectRatioLimits, FireflyDimensionsMap, FireflyImageDimensions, FireflyStylePresets } from './FireflyConstants'; export class DrawingFillHandler { - static drawingToImage = (drawing: Doc, strength: number, prompt: string) => + static drawingToImage = (drawing: Doc, strength: number, prompt: string) => { + const docData = drawing[DocData]; + const tags: string[] = ((docData?.tags as unknown as string[]) ?? []).map(tag => tag.slice(1)) ?? []; + const styles = tags.filter(tag => FireflyStylePresets.has(tag)); DocumentView.GetDocImage(drawing)?.then(imageField => { if (imageField) { const aspectRatio = (drawing.width as number) / (drawing.height as number); @@ -31,11 +34,12 @@ export class DrawingFillHandler { imageUrlToBase64(structureUrl) .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) .then((prompt: string) => { - Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: prompt, width: dims.width, height: dims.height, structureUrl, strength }).then((info: Upload.ImageInformation) => + Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: prompt, width: dims.width, height: dims.height, structureUrl: structureUrl, strength: strength, styles: styles }).then((info: Upload.ImageInformation) => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai_generated: true, nativeWidth: dims.width, nativeHeight: dims.height }), OpenWhere.addRight) ); // prettier-ignore }); } return false; }); + }; } diff --git a/src/client/views/smartdraw/FireflyConstants.ts b/src/client/views/smartdraw/FireflyConstants.ts index f51305fba..3574039e4 100644 --- a/src/client/views/smartdraw/FireflyConstants.ts +++ b/src/client/views/smartdraw/FireflyConstants.ts @@ -18,3 +18,25 @@ export const AspectRatioLimits = { portrait: 0.875, widescreen: 1.472, }; + +// prettier-ignore +export const FireflyStylePresets = + new Set(['graphic', 'wireframe', + 'vector_look','bw','cool_colors','golden','monochromatic','muted_color','toned_image','vibrant_colors','warm_tone','closeup', + 'knolling','landscape_photography','macrophotography','photographed_through_window','shallow_depth_of_field','shot_from_above', + 'shot_from_below','surface_detail','wide_angle','beautiful','bohemian','chaotic','dais','divine','eclectic','futuristic','kitschy', + 'nostalgic','simple','antique_photo','bioluminescent','bokeh','color_explosion','dark','faded_image','fisheye','gomori_photography', + 'grainy_film','iridescent','isometric','misty','neon','otherworldly_depiction','ultraviolet','underwater', 'backlighting', + 'dramatic_light', 'golden_hour', 'harsh_light','long','low_lighting','multiexposure','studio_light','surreal_lighting', + '3d_patterns','charcoal','claymation','fabric','fur','guilloche_patterns','layered_paper','marble_sculpture','made_of_metal', + 'origami','paper_mache','polka','strange_patterns','wood_carving','yarn','art_deco','art_nouveau','baroque','bauhaus', + 'constructivism','cubism','cyberpunk','fantasy','fauvism', 'film_noir','glitch_art','impressionism','industrialism','maximalism', + 'minimalism','modern_art','modernism','neo','pointillism','psychedelic','science_fiction','steampunk','surrealism','synthetism', + 'synthwave','vaporwave','acrylic_paint','bold_lines','chiaroscuro','color_shift_art','daguerreotype','digital_fractal', + 'doodle_drawing','double_exposure_portrait','fresco','geometric_pen','halftone','ink','light_painting','line_drawing','linocut', + 'oil_paint','paint_spattering','painting','palette_knife','photo_manipulation','scribble_texture','sketch','splattering', + 'stippling_drawing','watercolor','3d','anime','cartoon','cinematic','comic_book','concept_art','cyber_matrix','digital_art', + 'flat_design','geometric','glassmorphism','glitch_graphic','graffiti','hyper_realistic','interior_design','line_gradient', + 'low_poly','newspaper_collage','optical_illusion','pattern_pixel','pixel_art','pop_art','product_photo','psychedelic_background', + 'psychedelic_wonderland','scandinavian','splash_images','stamp','trompe_loeil' + ]); diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 4c4aac5e0..cb5d7c1da 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -20,9 +20,10 @@ export default class FireflyManager extends ApiManager { return undefined; }); - generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50) => + generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50, styles: string[]) => this.getBearerToken().then(response => response?.json().then((data: { access_token: string }) => + //prettier-ignore fetch('https://firefly-api.adobe.io/v3/images/generate', { method: 'POST', headers: [ @@ -42,6 +43,8 @@ export default class FireflyManager extends ApiManager { source: { url: structureUrl }, }, }, + //prettier-ignore + style: { presets: styles } }), }) .then(response2 => response2.json().then(json => JSON.stringify((json.outputs?.[0] as { image: { url: string } })?.image))) @@ -227,7 +230,7 @@ export default class FireflyManager extends ApiManager { subscription: '/queryFireflyImageFromStructure', secureHandler: async ({ req, res }) => this.uploadImageToDropbox(req.body.structureUrl).then(uploadUrl => - this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploadUrl, req.body.strength).then(fire => + this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploadUrl, req.body.strength, req.body.styles).then(fire => DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { if (info instanceof Error) _invalid(res, info.message); else _success(res, info); -- cgit v1.2.3-70-g09d2 From 1d62d867621b293c41ff8488ca5a3bd6010723d5 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Sun, 5 Jan 2025 23:47:18 -0500 Subject: added AI image editor --- src/client/documents/Documents.ts | 6 +- src/client/util/CurrentUserUtils.ts | 2 +- src/client/views/DocumentButtonBar.tsx | 7 +- src/client/views/ViewBoxInterface.ts | 1 + .../collectionFreeForm/CollectionFreeFormView.tsx | 12 +-- src/client/views/nodes/DocumentView.tsx | 22 +++- src/client/views/nodes/ImageBox.scss | 28 +++++ src/client/views/nodes/ImageBox.tsx | 113 ++++++++++++++++++++- src/client/views/pdf/AnchorMenu.tsx | 12 +-- src/client/views/smartdraw/DrawingFillHandler.tsx | 2 +- src/client/views/smartdraw/SmartDrawHandler.tsx | 92 +++++++++-------- src/client/views/smartdraw/StickerPalette.tsx | 17 ++-- src/server/ApiManagers/FireflyManager.ts | 1 - 13 files changed, 235 insertions(+), 80 deletions(-) (limited to 'src/server') diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts index 785af3409..7f1387ff8 100644 --- a/src/client/documents/Documents.ts +++ b/src/client/documents/Documents.ts @@ -517,9 +517,9 @@ export class DocumentOptions { card_sort?: STRt = new StrInfo('way cards are sorted in deck view'); card_sort_isDesc?: BOOLt = new BoolInfo('whether the cards are sorted ascending or descending'); - ai_generated?: boolean; // to mark items as ai generated - firefly_seed?: number; - firefly_prompt?: string; + ai?: string; // to mark items as ai generated + ai_firefly_seed?: number; + ai_firefly_prompt?: string; } export const DocOptions = new DocumentOptions(); diff --git a/src/client/util/CurrentUserUtils.ts b/src/client/util/CurrentUserUtils.ts index 9af79a02e..b41fd09dc 100644 --- a/src/client/util/CurrentUserUtils.ts +++ b/src/client/util/CurrentUserUtils.ts @@ -779,7 +779,7 @@ pie title Minerals in my tap water { title: " Size", toolTip: "Size of area pencil eraser", btnType: ButtonType.NumberSliderButton, toolType: InkProperty.EraserWidth,ignoreClick: true, scripts: {script: '{ return setInkProperty(this.toolType, value, _readOnly_);}'}, funcs: {hidden:"NotRadiusEraser()"}, numBtnMin: 1, linearBtnWidth:40}, { title: "Mask", toolTip: "Make Stroke a Stencil Mask", btnType: ButtonType.ToggleButton, icon: "user-circle", toolType: InkProperty.Mask, scripts: {onClick:'{ return setInkProperty(this.toolType, value, _readOnly_);}'}, funcs: {hidden:"IsNoviceMode()" } }, { title: "Labels", toolTip: "Show Labels Inside Shapes", btnType: ButtonType.ToggleButton, icon: "text-width", toolType: InkProperty.Labels, scripts: {onClick:'{ return setInkProperty(this.toolType, value, _readOnly_);}'}}, - { title: "Smart Draw", toolTip: "Draw with GPT", btnType: ButtonType.ToggleButton, icon: "user-pen", toolType: InkTool.SmartDraw, scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {hidden: "IsNoviceMode()"}}, + { title: "Smart Draw", toolTip: "Draw with AI", btnType: ButtonType.ToggleButton, icon: "user-pen", toolType: InkTool.SmartDraw, scripts: {onClick:'{ return setActiveTool(this.toolType, false, _readOnly_);}'}, funcs: {hidden: "IsNoviceMode()"}}, ]; } diff --git a/src/client/views/DocumentButtonBar.tsx b/src/client/views/DocumentButtonBar.tsx index b7033af3f..d722b28b5 100644 --- a/src/client/views/DocumentButtonBar.tsx +++ b/src/client/views/DocumentButtonBar.tsx @@ -319,12 +319,7 @@ export class DocumentButtonBar extends ObservableReactComponent<{ views: () => ( const targetDoc = this.view0?.Document; return !targetDoc ? null : ( Edit with AI}> -
{ - CalendarManager.Instance.open(this.view0, targetDoc); - }}> +
this.view0?.toggleAIEditor(), 'toggle AI editor')}>
diff --git a/src/client/views/ViewBoxInterface.ts b/src/client/views/ViewBoxInterface.ts index b7980d74e..df08f2564 100644 --- a/src/client/views/ViewBoxInterface.ts +++ b/src/client/views/ViewBoxInterface.ts @@ -60,4 +60,5 @@ export abstract class ViewBoxInterface

extends ObservableReactComponent boolean; dontRegisterView?: () => boolean; // KeyValueBox's don't want to register their views isUnstyledView?: () => boolean; // SchemaView and KeyValue are unstyled -- not titles, no opacity, no animations + componentAIView?: (top: number) => JSX.Element; } diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index ef0b80720..9af698ec7 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -1290,12 +1290,12 @@ export class CollectionFreeFormView extends CollectionSubView> = undefined; // needs to be accessed from DocumentView wrapper class @observable _animateScaleTime: Opt = undefined; // milliseconds for animating between views. defaults to 300 if not uset @observable _animateScalingTo = 0; + @observable public _showAIEditor: boolean = false; + + @action + showAIEditor() { + this._showAIEditor = !this._showAIEditor; + } get _contentDiv() { return this._mainCont.current; } // prettier-ignore get _docView() { return this._props.DocumentView?.(); } // prettier-ignore @@ -552,7 +558,6 @@ export class DocumentViewInternal extends DocComponent DocumentView.SetLightboxDoc(this.Document), icon: 'external-link-alt' }); } appearanceItems.push({ description: 'Pin', event: () => this._props.pinToPres(this.Document, {}), icon: 'map-pin' }); - appearanceItems.push({ description: 'Make Image', event: () => DrawingFillHandler.drawingToImage(this.Document, StrCast(this.Document.title)), icon: 'map-pin' }); !Doc.noviceMode && templateDoc && appearanceItems.push({ description: 'Open Template ', event: () => this._props.addDocTab(templateDoc, OpenWhere.addRight), icon: 'eye' }); !appearance && appearanceItems.length && cm.addItem({ description: 'Appearance...', subitems: appearanceItems, icon: 'compass' }); @@ -711,6 +716,8 @@ export class DocumentViewInternal extends DocComponent this._props.PanelWidth() * 0.6; + rph = () => this.panelHeight() * 0.6; @computed get viewBoxContents() { TraceMobx(); const isInk = this.layoutDoc._layout_isSvg && !this._props.LayoutTemplateString; @@ -720,7 +727,9 @@ export class DocumentViewInternal extends DocComponent + {this._showAIEditor && (this._componentView?.componentAIView?.(this.rph()) ?? null)}

); } @@ -1286,6 +1297,11 @@ export class DocumentView extends DocComponent() { } }; + @action + public toggleAIEditor = () => { + this._docViewInternal && this._docViewInternal.showAIEditor(); + }; + public setTextHtmlOverlay = action((text: string | undefined, effect?: Doc) => { this._htmlOverlayText = text; this._htmlOverlayEffect = effect; diff --git a/src/client/views/nodes/ImageBox.scss b/src/client/views/nodes/ImageBox.scss index 3ffda5a35..03314e90f 100644 --- a/src/client/views/nodes/ImageBox.scss +++ b/src/client/views/nodes/ImageBox.scss @@ -139,3 +139,31 @@ .imageBox-fadeBlocker-hover { opacity: 0; } + +.imageBox-aiView { + padding: 5px; + position: absolute; + overflow: scroll; + text-align: center; + font-weight: bold; + margin-top: 5px; + + .imageBox-aiView-subtitle { + align-self: start; + } + + .imageBox-aiView-regenerate-container, + .imageBox-aiView-options-container { + font-weight: normal; + text-align: start; + } + + .imageBox-aiView-regenerate, + .imageBox-aiView-options { + display: flex; + flex-direction: row; + justify-content: center; + flex-direction: row; + gap: 5px; + } +} diff --git a/src/client/views/nodes/ImageBox.tsx b/src/client/views/nodes/ImageBox.tsx index 7ce429f0f..f00580d77 100644 --- a/src/client/views/nodes/ImageBox.tsx +++ b/src/client/views/nodes/ImageBox.tsx @@ -1,7 +1,7 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Tooltip } from '@mui/material'; import axios from 'axios'; -import { Colors } from 'browndash-components'; +import { Colors, Type } from 'browndash-components'; import { action, computed, IReactionDisposer, makeObservable, observable, ObservableMap, reaction } from 'mobx'; import { observer } from 'mobx-react'; import { extname } from 'path'; @@ -41,6 +41,9 @@ import './ImageBox.scss'; import { OpenWhere } from './OpenWhere'; import { Upload } from '../../../server/SharedMediaTypes'; import { SmartDrawHandler } from '../smartdraw/SmartDrawHandler'; +import { Button } from 'browndash-components'; +import { SettingsManager } from '../../util/SettingsManager'; +import { AiOutlineSend } from 'react-icons/ai'; export class ImageEditorData { // eslint-disable-next-line no-use-before-define @@ -352,7 +355,7 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { }), icon: 'pencil-alt', }); - this.layoutDoc.ai_generated && + this.layoutDoc.ai && funcs.push({ description: 'Regenerate AI Image', event: action(e => { @@ -526,6 +529,112 @@ export class ImageBox extends ViewBoxAnnotatableComponent() { ); } + @observable private _regenInput = ''; + @observable private _canInteract = true; + @observable private _regenerateLoading = false; + + componentAIView = (top: number) => { + const field = Cast(this.dataDoc[this.fieldKey], ImageField); + const showRegenerate = this.Document[DocData].ai; + return ( +
+ Edit Image with AI + {showRegenerate && ( +
+ Regenerate AI Image +
+ this._canInteract && (this._regenInput = e.target.value))} + // onKeyDown={this.handleKeyPress} + placeholder="Prompt (Optional)" + /> +
+
+ )} +
+ {showRegenerate && More Image Options } +
+
+
+
+ ); + }; + @computed get annotationLayer() { TraceMobx(); return
; diff --git a/src/client/views/pdf/AnchorMenu.tsx b/src/client/views/pdf/AnchorMenu.tsx index bb8082061..2e704aa8d 100644 --- a/src/client/views/pdf/AnchorMenu.tsx +++ b/src/client/views/pdf/AnchorMenu.tsx @@ -153,12 +153,12 @@ export class AnchorMenu extends AntimodeMenu { this.AddDrawingAnnotation(drawing); const docData = drawing[DocData]; docData.title = opts.text.match(/^(.*?)~~~.*$/)?.[1] || opts.text; - docData.drawingInput = opts.text; - docData.drawingComplexity = opts.complexity; - docData.drawingColored = opts.autoColor; - docData.drawingSize = opts.size; - docData.drawingData = gptRes; - docData.ai_generated = true; + docData.ai_drawing_input = opts.text; + docData.ai_drawing_complexity = opts.complexity; + docData.ai_drawing_colored = opts.autoColor; + docData.ai_drawing_size = opts.size; + docData.ai_drawing_data = gptRes; + docData.ai = 'gpt'; }); pointerDown = (e: React.PointerEvent) => { diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 52652d377..8e41ee105 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -35,7 +35,7 @@ export class DrawingFillHandler { .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) .then((prompt: string) => { Networking.PostToServer('/queryFireflyImageFromStructure', { prompt: prompt, width: dims.width, height: dims.height, structureUrl: structureUrl, strength: strength, styles: styles }).then((info: Upload.ImageInformation) => - DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai_generated: true, nativeWidth: dims.width, nativeHeight: dims.height }), OpenWhere.addRight) + DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, { ai: 'firefly', ai_firefly_prompt: prompt, nativeWidth: dims.width, nativeHeight: dims.height }), OpenWhere.addRight) ); // prettier-ignore }); } diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 6c9470480..0c67c7a13 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -58,7 +58,7 @@ export class SmartDrawHandler extends ObservableReactComponent { private _lastInput: DrawingOptions = { text: '', complexity: 5, size: 350, autoColor: true, x: 0, y: 0 }; private _lastResponse: string = ''; - private _selectedDoc: Doc | undefined = undefined; + private _selectedDocs: Doc[] = []; private _errorOccurredOnce = false; @observable private _display: boolean = false; @@ -144,14 +144,14 @@ export class SmartDrawHandler extends ObservableReactComponent { */ @action displayRegenerate = (x: number, y: number) => { - this._selectedDoc = DocumentView.SelectedDocs()?.lastElement(); + this._selectedDocs = [DocumentView.SelectedDocs()?.lastElement()]; [this._pageX, this._pageY] = [x, y]; this._display = false; this.ShowRegenerate = true; this._showEditBox = false; - const docData = this._selectedDoc[DocData]; + const docData = this._selectedDocs[0][DocData]; this._lastResponse = StrCast(docData.drawingData); - this._lastInput = { text: StrCast(docData.drawingInput), complexity: NumCast(docData.drawingComplexity), size: NumCast(docData.drawingSize), autoColor: BoolCast(docData.drawingColored), x: this._pageX, y: this._pageY }; + this._lastInput = { text: StrCast(docData.ai_drawing_input), complexity: NumCast(docData.ai_drawing_complexity), size: NumCast(docData.ai_drawing_size), autoColor: BoolCast(docData.ai_drawing_colored), x: this._pageX, y: this._pageY }; }; /** @@ -205,8 +205,9 @@ export class SmartDrawHandler extends ObservableReactComponent { this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { - await this.regenerate(); + await this.regenerate(this._selectedDocs); runInAction(() => { + this._selectedDocs = []; this._regenInput = ''; this._showEditBox = false; }); @@ -253,7 +254,7 @@ export class SmartDrawHandler extends ObservableReactComponent { const strokeData = await this.parseSvg(res, startPt, false, autoColor); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - this._selectedDoc = drawingDoc; + drawingDoc && this._selectedDocs.push(drawingDoc); this._errorOccurredOnce = false; return strokeData; } else { @@ -274,57 +275,62 @@ export class SmartDrawHandler extends ObservableReactComponent { title: input.match(/^(.*?)~~~.*$/)?.[1] || input, nativeWidth: dims.width, nativeHeight: dims.height, - ai_generated: true, - firefly_seed: img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1], - firefly_prompt: input, + ai: 'firefly', + ai_firefly_seed: img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1], + ai_firefly_prompt: input, }); DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); - this._selectedDoc = imgDoc; + this._selectedDocs.push(imgDoc); }); }; /** * Regenerates drawings with the option to add a specific regenerate prompt/request. + * @param doc the drawing Docs to regenerate */ @action - regenerate = async (lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { + regenerate = async (drawingDocs: Doc[], lastInput?: DrawingOptions, lastResponse?: string, regenInput?: string) => { if (lastInput) this._lastInput = lastInput; if (lastResponse) this._lastResponse = lastResponse; if (regenInput) this._regenInput = regenInput; - if (this._generateImage) { - if (this._regenInput !== '') { - if (this._selectedDoc) { - const docData = this._selectedDoc[DocData]; - const newPrompt = `${docData.firefly_prompt}, ${this._regenInput}`; - const seed: number = docData?.firefly_seed as number; - await this.createImageWithFirefly(newPrompt, seed); + await Promise.all( + drawingDocs.map(async doc => { + const docData = doc[DocData]; + if (docData.type == 'image') { + const seed: number = docData?.ai_firefly_seed as number; + if (this._regenInput !== '') { + // if (this._selectedDoc) { + const newPrompt = `${docData.ai_firefly_prompt}, ${this._regenInput}`; + await this.createImageWithFirefly(newPrompt, seed); + // } + } else { + await this.createImageWithFirefly(this._lastInput.text || StrCast(docData.ai_firefly_prompt)); + } } - } else { - await this.createImageWithFirefly(this._lastInput.text); - } - } - if (this._generateDrawing) { - try { - let res; - if (this._regenInput !== '') { - const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; - res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); - this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; - } else { - res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); - } - if (!res) { - console.error('GPT call failed'); - return; + if (docData.type == 'collection') { + try { + let res; + if (this._regenInput !== '') { + const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + } else { + res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + } + if (!res) { + console.error('GPT call failed'); + return; + } + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, doc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + } catch (err) { + console.error('Error regenerating drawing', err); + } } - const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); - this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - } catch (err) { - console.error('Error regenerating drawing', err); - } - } + }) + ); }; /** diff --git a/src/client/views/smartdraw/StickerPalette.tsx b/src/client/views/smartdraw/StickerPalette.tsx index 352a02e32..d23763eb9 100644 --- a/src/client/views/smartdraw/StickerPalette.tsx +++ b/src/client/views/smartdraw/StickerPalette.tsx @@ -142,19 +142,20 @@ export class StickerPalette extends ObservableReactComponent { this._isLoading = true; + const prevDrawings = DocListCast(this._props.Document[DocData].data); this._props.Document[DocData].data = undefined; SmartDrawHandler.Instance.AddDrawing = this.addDrawing; this._canInteract = false; await Promise.all( Array.from({ length: 3 }).map((_, i) => { return this._showRegenerate - ? SmartDrawHandler.Instance.regenerate(this._opts, this._gptRes[i], this._userInput) + ? SmartDrawHandler.Instance.regenerate(prevDrawings, this._opts, this._gptRes[i], this._userInput) : SmartDrawHandler.Instance.drawWithGPT({ X: 0, Y: 0 }, this._userInput, this._opts.complexity, this._opts.size, this._opts.autoColor); }) ); @@ -181,12 +182,12 @@ export class StickerPalette extends ObservableReactComponent { - console.log('DIMENSIONS', width, height); let body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} }`; if (seed) { console.log('RECEIVED SEED', seed); -- cgit v1.2.3-70-g09d2 From 2be78b5319303aef31250088397c87eb301d7c59 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 7 Jan 2025 13:36:46 -0500 Subject: added an endpoint for / that redirects to /home. added a user error message for unknown endpoints. --- src/server/server_Initialization.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src/server') diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index 4dcb32f8b..1bd495965 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -108,7 +108,7 @@ function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { // detect search query and use default search engine res.redirect(req.headers.referer + 'corsProxy/' + encodeURIComponent('http://www.google.com' + relativeUrl)); } else { - res.end(); + res.status(404).json({ error: 'no such file or endpoint: try /home /logout /login' }); } }); } @@ -247,6 +247,10 @@ export default async function InitializeServer(routeSetter: RouteSetter) { const app = buildWithMiddleware(express()); const compiler = webpack(config as webpack.Configuration); + // Default route + app.get('/', (req, res) => { + res.redirect(req.user ? '/home' : '/login'); //res.send('This is the default route.'); + }); // route table managed by express. routes are tested sequentially against each of these map rules. when a match is found, the handler is called to process the request app.use(wdm(compiler, { publicPath: config.output.publicPath })); app.use(whm(compiler)); @@ -259,7 +263,6 @@ export default async function InitializeServer(routeSetter: RouteSetter) { isRelease && !SSL.Loaded && SSL.exit(); routeSetter(new RouteManager(app, isRelease)); // this sets up all the regular supervised routes (things like /home, download/upload api's, pdf, search, session, etc) registerEmbeddedBrowseRelativePathHandler(app); // this allows renered web pages which internally have relative paths to find their content - isRelease && process.env.serverPort && (resolvedPorts.server = Number(process.env.serverPort)); const server = isRelease ? createServer(SSL.Credentials, app) : app; await new Promise(resolve => { -- cgit v1.2.3-70-g09d2 From eabaae11dc5c5e1de463d2903c3c8e9407506c92 Mon Sep 17 00:00:00 2001 From: bobzel Date: Wed, 8 Jan 2025 10:37:09 -0500 Subject: from last --- src/client/views/smartdraw/StickerPalette.tsx | 4 ++-- src/server/server_Initialization.ts | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'src/server') diff --git a/src/client/views/smartdraw/StickerPalette.tsx b/src/client/views/smartdraw/StickerPalette.tsx index e3345f547..d5307974f 100644 --- a/src/client/views/smartdraw/StickerPalette.tsx +++ b/src/client/views/smartdraw/StickerPalette.tsx @@ -1,6 +1,6 @@ +import { Button } from '@dash/components'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { Slider, Switch } from '@mui/material'; -import { Button } from 'browndash-components'; import { action, makeObservable, observable } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; @@ -22,8 +22,8 @@ import { ObservableReactComponent } from '../ObservableReactComponent'; import { DefaultStyleProvider, returnEmptyDocViewList } from '../StyleProvider'; import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { FieldView } from '../nodes/FieldView'; -import './StickerPalette.scss'; import { DrawingOptions, SmartDrawHandler } from './SmartDrawHandler'; +import './StickerPalette.scss'; interface StickerPaletteProps { Document: Doc; diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index 1bd495965..12c5319b7 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -115,7 +115,7 @@ function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { // eslint-disable-next-line @typescript-eslint/no-explicit-any function proxyServe(req: any, requrl: string, response: any) { - // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires + // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const htmlBodyMemoryStream = new (require('memorystream'))(); let wasinBrFormat = false; const sendModifiedBody = () => { @@ -189,7 +189,6 @@ function proxyServe(req: any, requrl: string, response: any) { res.headers['x-permitted-cross-domain-policies'] = 'all'; res.headers['x-frame-options'] = ''; res.headers['content-security-policy'] = ''; - // eslint-disable-next-line no-multi-assign response.headers = response._headers = res.headers; }) .on('end', sendModifiedBody) -- cgit v1.2.3-70-g09d2 From dd0526a0a256fb3f1de9e0a2508566adcd6cd7e9 Mon Sep 17 00:00:00 2001 From: bobzel Date: Thu, 9 Jan 2025 00:02:10 -0500 Subject: fixed sizing of images created by firefly. added error message for failed firefly request. --- src/client/Network.ts | 2 +- .../collections/collectionFreeForm/MarqueeView.tsx | 4 +- src/client/views/smartdraw/DrawingFillHandler.tsx | 4 +- src/server/ApiManagers/FireflyManager.ts | 47 +++++++++++++--------- src/server/RouteManager.ts | 3 +- 5 files changed, 35 insertions(+), 25 deletions(-) (limited to 'src/server') diff --git a/src/client/Network.ts b/src/client/Network.ts index 9afdc844f..3b0406141 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -15,7 +15,7 @@ export namespace Networking { return (await fetch(relativeRoute)).text(); } - export async function PostToServer(relativeRoute: string, body?: unknown) { + export function PostToServer(relativeRoute: string, body?: unknown) { const options = { uri: ClientUtils.prepend(relativeRoute), method: 'POST', diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index ddc50871d..5524fedb3 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -586,7 +586,7 @@ export class MarqueeView extends ObservableReactComponent { const selection: Doc[] = []; const selectFunc = (doc: Doc) => { const layoutDoc = Doc.Layout(doc); @@ -619,7 +619,7 @@ export class MarqueeView extends ObservableReactComponent doc.z !== undefined) .map(selectFunc); return selection; - } + }; @computed get marqueeDiv() { const cpt = this._lassoFreehand || !this._visible ? [0, 0] : [this._downX < this._lastX ? this._downX : this._lastX, this._downY < this._lastY ? this._downY : this._lastY]; diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index fea4acb67..96b21123b 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -38,8 +38,8 @@ export class DrawingFillHandler { { prompt: user_prompt || prompt, width: dims.width, height: dims.height, structureUrl, strength, styles }) .then((info: Upload.ImageInformation) => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, - { ai: 'firefly', ai_firefly_prompt: user_prompt || prompt, nativeWidth: dims.width, nativeHeight: dims.height }), OpenWhere.addRight) - ); // prettier-ignore + { ai: 'firefly', ai_firefly_prompt: user_prompt || prompt, _width: 500, data_nativeWidth: info.nativeWidth, data_nativeHeight: info.nativeHeight }), OpenWhere.addRight) + ).catch(e => alert("create image failed: " + e.toString())); // prettier-ignore }); } return false; diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts index 6daa5840e..a1f8fab8d 100644 --- a/src/server/ApiManagers/FireflyManager.ts +++ b/src/server/ApiManagers/FireflyManager.ts @@ -56,15 +56,22 @@ export default class FireflyManager extends ApiManager { ); uploadImageToDropbox = (fileUrl: string, dbx = new Dropbox({ accessToken: process.env.DROPBOX_TOKEN })) => - new Promise((res, rej) => + new Promise((res, rej) => fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => { if (err) { console.log('Error: ', err); rej(); } else { - dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }).then(response => { - dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }).then(link => res(link.result.link)); - }); + dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }) + .then(response => { + dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }) + .then(link => res(link.result.link)) + .catch(e => res(new Error(e.toString()))); + }) + .catch(e => { + console.log('Dropbox error:', e); + res(new Error(e.toString())); + }); } }) ); @@ -229,12 +236,14 @@ export default class FireflyManager extends ApiManager { subscription: '/queryFireflyImageFromStructure', secureHandler: async ({ req, res }) => this.uploadImageToDropbox(req.body.structureUrl).then(uploadUrl => - this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploadUrl, req.body.strength, req.body.styles).then(fire => - DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info); - }) - ) + uploadUrl instanceof Error + ? _invalid(res, uploadUrl.message) + : this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploadUrl, req.body.strength, req.body.styles).then(fire => + DashUploadUtils.UploadImage(JSON.parse(fire ?? '').url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }) + ) ), }); register({ @@ -267,14 +276,16 @@ export default class FireflyManager extends ApiManager { subscription: '/expandImage', secureHandler: ({ req, res }) => this.uploadImageToDropbox(req.body.file).then(uploadUrl => - this.expandImage(uploadUrl, req.body.prompt).then(text => { - if (text.error_code) _error(res, text.message); - else - DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { - if (info instanceof Error) _invalid(res, info.message); - else _success(res, info); - }); - }) + uploadUrl instanceof Error + ? _invalid(res, uploadUrl.message) + : this.expandImage(uploadUrl, req.body.prompt).then(text => { + if (text.error_code) _error(res, text.message); + else + DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }); + }) ), }); } diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index d8e0455f6..2f6cf80b5 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -39,8 +39,7 @@ export function _success(res: Response, body: any) { } export function _invalid(res: Response, message: string) { - res.statusMessage = message; - res.status(STATUS.BAD_REQUEST).send(); + res.status(STATUS.BAD_REQUEST).send(message); } export function _permissionDenied(res: Response, message?: string) { -- cgit v1.2.3-70-g09d2 From fd922d7898ed7a405ed47a7e48b85c582d787c07 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Fri, 10 Jan 2025 15:07:42 -0500 Subject: working on merge conflicts --- .../collectionFreeForm/CollectionFreeFormView.tsx | 9 ++-- src/client/views/nodes/DocumentView.scss | 2 + src/client/views/nodes/ImageBox.scss | 8 +++- src/client/views/nodes/ImageBox.tsx | 49 ++++++++++++++-------- src/client/views/smartdraw/DrawingFillHandler.tsx | 19 +++++++-- src/server/ApiManagers/FireflyManager.ts | 41 ++++++++++++------ 6 files changed, 89 insertions(+), 39 deletions(-) (limited to 'src/server') diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx index 112bfd178..e1144f21a 100644 --- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx +++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx @@ -2266,7 +2266,6 @@ export class CollectionFreeFormView extends CollectionSubView this._canInteract && (this._drawingFillInput = e.target.value))} />
- Reference Strength this._canInteract && (this._fireflyRefStrength = val as number))} valueLabelDisplay="auto" /> + Reference Strength