From 0eff48b757ca81860a883d25e147b8a869e5fe00 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Mon, 30 Dec 2024 23:35:24 -0500 Subject: created image regeneration with dialogue --- src/client/views/smartdraw/DrawingFillHandler.tsx | 11 +- src/client/views/smartdraw/SmartDrawHandler.scss | 8 +- src/client/views/smartdraw/SmartDrawHandler.tsx | 197 ++++++++++++++-------- src/client/views/smartdraw/StickerPalette.tsx | 1 + 4 files changed, 146 insertions(+), 71 deletions(-) (limited to 'src/client/views/smartdraw') diff --git a/src/client/views/smartdraw/DrawingFillHandler.tsx b/src/client/views/smartdraw/DrawingFillHandler.tsx index 48e71bc9f..1a470f995 100644 --- a/src/client/views/smartdraw/DrawingFillHandler.tsx +++ b/src/client/views/smartdraw/DrawingFillHandler.tsx @@ -1,21 +1,26 @@ +import { imageUrlToBase64 } from '../../../ClientUtils'; import { Doc } from '../../../fields/Doc'; import { ImageCast } from '../../../fields/Types'; import { Upload } from '../../../server/SharedMediaTypes'; +import { gptDescribeImage } from '../../apis/gpt/GPT'; import { Docs } from '../../documents/Documents'; import { Networking } from '../../Network'; import { DocumentView, DocumentViewInternal } from '../nodes/DocumentView'; import { OpenWhere } from '../nodes/OpenWhere'; export class DrawingFillHandler { - static drawingToImage = (drawing: Doc, prompt: string) => + static drawingToImage = (drawing: Doc, strength: number, prompt: string) => DocumentView.GetDocImage(drawing)?.then(imageField => { if (imageField) { const { href } = ImageCast(imageField).url; const hrefParts = href.split('.'); const structureUrl = `${hrefParts.slice(0, -1).join('.')}_o.${hrefParts.lastElement()}`; - const strength: number = 100; - Networking.PostToServer('/queryFireflyImageFromStructure', { prompt, structureUrl, strength }).then((info: Upload.ImageInformation) => + imageUrlToBase64(structureUrl) + .then((hrefBase64: string) => gptDescribeImage(hrefBase64)) + .then((prompt: string) => { + Networking.PostToServer('/queryFireflyImageFromStructure', { prompt, structureUrl, strength }).then((info: Upload.ImageInformation) => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(info.accessPaths.agnostic.client, {}), OpenWhere.addRight)) // prettier-ignore + }); } return false; }); diff --git a/src/client/views/smartdraw/SmartDrawHandler.scss b/src/client/views/smartdraw/SmartDrawHandler.scss index c25273876..513779512 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.scss +++ b/src/client/views/smartdraw/SmartDrawHandler.scss @@ -12,7 +12,13 @@ } } - .smartdraw-options { + .smartdraw-output-options { + display: flex; + flex-direction: row; + justify-content: center; + } + + .smartdraw-svg-options { margin-top: 5px; display: flex; flex-direction: row; diff --git a/src/client/views/smartdraw/SmartDrawHandler.tsx b/src/client/views/smartdraw/SmartDrawHandler.tsx index 036ac5983..fb1a5771e 100644 --- a/src/client/views/smartdraw/SmartDrawHandler.tsx +++ b/src/client/views/smartdraw/SmartDrawHandler.tsx @@ -1,5 +1,5 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { Slider, Switch } from '@mui/material'; +import { Checkbox, Slider, Switch } from '@mui/material'; import { Button, IconButton } from 'browndash-components'; import { action, makeObservable, observable, runInAction } from 'mobx'; import { observer } from 'mobx-react'; @@ -74,6 +74,8 @@ export class SmartDrawHandler extends ObservableReactComponent { @observable private _autoColor: boolean = true; @observable private _regenInput: string = ''; @observable private _canInteract: boolean = true; + @observable private _generateDrawing: boolean = true; + @observable private _generateImage: boolean = true; @observable public ShowRegenerate: boolean = false; @@ -195,6 +197,7 @@ export class SmartDrawHandler extends ObservableReactComponent { */ @action handleSendClick = async () => { + if (!this._generateImage && !this._generateDrawing) return; this._isLoading = true; this._canInteract = false; if (this.ShowRegenerate) { @@ -212,7 +215,12 @@ export class SmartDrawHandler extends ObservableReactComponent { this._showOptions = false; }); try { - await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + if (this._generateImage) { + await this.createImageWithFirefly(this._userInput); + } + if (this._generateDrawing) { + await this.drawWithGPT({ X: this._pageX, Y: this._pageY }, this._userInput, this._complexity, this._size, this._autoColor); + } this.hideSmartDrawHandler(); runInAction(() => { @@ -240,15 +248,12 @@ export class SmartDrawHandler extends ObservableReactComponent { drawWithGPT = async (startPt: { X: number; Y: number }, input: string, complexity: number, size: number, autoColor: boolean) => { if (input) { this._lastInput = { text: input, complexity: complexity, size: size, autoColor: autoColor, x: startPt.X, y: startPt.Y }; - - Networking.PostToServer('/queryFireflyImage', { prompt: input }).then(img => DocumentViewInternal.addDocTabFunc(Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { title: input }), OpenWhere.addRight)); - const res = await gptAPICall(`"${input}", "${complexity}", "${size}"`, GPTCallType.DRAW, undefined, true); if (res) { const strokeData = await this.parseSvg(res, startPt, false, autoColor); const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - + this._selectedDoc = drawingDoc; this._errorOccurredOnce = false; return strokeData; } else { @@ -258,6 +263,23 @@ export class SmartDrawHandler extends ObservableReactComponent { return undefined; }; + /** + * Calls Firefly API to create an image based on user input + */ + createImageWithFirefly = (input: string, seed?: number) => { + this._lastInput.text = input; + return Networking.PostToServer('/queryFireflyImage', { prompt: input, seed: seed }).then(img => { + const imgDoc: Doc = Docs.Create.ImageDocument(img.accessPaths.agnostic.client, { + title: input.match(/^(.*?)~~~.*$/)?.[1] || input, + ai_generated: true, + firefly_seed: img.accessPaths.agnostic.client.match(/\/(\d+)upload/)[1], + firefly_prompt: input, + }); + DocumentViewInternal.addDocTabFunc(imgDoc, OpenWhere.addRight); + this._selectedDoc = imgDoc; + }); + }; + /** * Regenerates drawings with the option to add a specific regenerate prompt/request. */ @@ -266,27 +288,39 @@ export class SmartDrawHandler extends ObservableReactComponent { if (lastInput) this._lastInput = lastInput; if (lastResponse) this._lastResponse = lastResponse; if (regenInput) this._regenInput = regenInput; - - try { - let res; + if (this._generateDrawing) { + try { + let res; + if (this._regenInput !== '') { + const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; + res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); + this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + } else { + res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + } + if (!res) { + console.error('GPT call failed'); + return; + } + const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); + this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); + const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); + drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); + } catch (err) { + console.error('Error regenerating drawing', err); + } + } + if (this._generateImage) { if (this._regenInput !== '') { - const prompt: string = `This is your previously generated svg code: ${this._lastResponse} for the user input "${this._lastInput.text}". Please regenerate it with the provided specifications.`; - res = await gptAPICall(`"${this._regenInput}"`, GPTCallType.DRAW, prompt, true); - this._lastInput.text = `${this._lastInput.text} ~~~ ${this._regenInput}`; + if (this._selectedDoc) { + const docData = this._selectedDoc[DocData]; + const newPrompt = `${docData.firefly_prompt}, ${this._regenInput}`; + const seed: number = docData?.firefly_seed as number; + await this.createImageWithFirefly(newPrompt, seed); + } } else { - res = await gptAPICall(`"${this._lastInput.text}", "${this._lastInput.complexity}", "${this._lastInput.size}"`, GPTCallType.DRAW, undefined, true); + await this.createImageWithFirefly(this._lastInput.text); } - if (!res) { - console.error('GPT call failed'); - return; - } - const strokeData = await this.parseSvg(res, { X: this._lastInput.x, Y: this._lastInput.y }, true, lastInput?.autoColor || this._autoColor); - this.RemoveDrawing !== unimplementedFunction && this.RemoveDrawing(true, this._selectedDoc); - const drawingDoc = strokeData && this.CreateDrawingDoc(strokeData.data, strokeData.lastInput, strokeData.lastRes); - drawingDoc && this.AddDrawing(drawingDoc, this._lastInput, res); - return strokeData; - } catch (err) { - console.error('Error regenerating drawing', err); } }; @@ -397,58 +431,87 @@ export class SmartDrawHandler extends ObservableReactComponent { {this._showOptions && (
-
-
- Auto color - this._canInteract && (this._autoColor = !this._autoColor))} - /> -
-
- Complexity - +
+ Generate Ink + this._canInteract && (this._complexity = val as number))} - valueLabelDisplay="auto" + checked={this._generateDrawing} + onChange={() => this._canInteract && (this._generateDrawing = !this._generateDrawing)} />
-
- Size (in pixels) - + Generate Image + this._canInteract && (this._size = val as number))} - valueLabelDisplay="auto" + checked={this._generateImage} + onChange={() => this._canInteract && (this._generateImage = !this._generateImage)} />
-
+ {this._generateDrawing && ( +
+
+ Auto color + this._canInteract && (this._autoColor = !this._autoColor))} + /> +
+
+ Complexity + this._canInteract && (this._complexity = val as number))} + valueLabelDisplay="auto" + /> +
+
+ Size (in pixels) + this._canInteract && (this._size = val as number))} + valueLabelDisplay="auto" + /> +
+
+ )}
)}
diff --git a/src/client/views/smartdraw/StickerPalette.tsx b/src/client/views/smartdraw/StickerPalette.tsx index d56878f10..352a02e32 100644 --- a/src/client/views/smartdraw/StickerPalette.tsx +++ b/src/client/views/smartdraw/StickerPalette.tsx @@ -186,6 +186,7 @@ export class StickerPalette extends ObservableReactComponent