diff options
Diffstat (limited to 'src/client/views/nodes/generativeFill')
10 files changed, 0 insertions, 1253 deletions
diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.scss b/src/client/views/nodes/generativeFill/GenerativeFill.scss deleted file mode 100644 index c2669a950..000000000 --- a/src/client/views/nodes/generativeFill/GenerativeFill.scss +++ /dev/null @@ -1,97 +0,0 @@ -$navHeight: 5rem; -$canvasSize: 1024px; -$scale: 0.5; - -.generativeFillContainer { - position: absolute; - top: 0; - left: 0; - z-index: 9999; - height: 100vh; - width: 100vw; - display: flex; - flex-direction: column; - overflow: hidden; - - .generativeFillControls { - flex-shrink: 0; - height: $navHeight; - color: #000000; - background-color: #ffffff; - z-index: 999; - width: 100%; - display: flex; - gap: 3rem; - justify-content: space-between; - align-items: center; - border-bottom: 1px solid #c7cdd0; - padding: 0 2rem; - - h1 { - font-size: 1.5rem; - } - } - - .drawingArea { - cursor: none; - touch-action: none; - position: relative; - flex-grow: 1; - display: flex; - justify-content: center; - align-items: center; - width: 100%; - background-color: #f0f4f6; - - canvas { - display: block; - position: absolute; - transform-origin: 50% 50%; - } - - .pointer { - pointer-events: none; - position: absolute; - border-radius: 50%; - width: 50px; - height: 50px; - border: 1px solid #ffffff; - transform: translate(-50%, -50%); - display: flex; - justify-content: center; - align-items: center; - - .innerPointer { - width: 100%; - height: 100%; - border: 1px solid #000000; - border-radius: 50%; - } - } - - .iconContainer { - position: absolute; - top: 2rem; - left: 2rem; - display: flex; - flex-direction: column; - gap: 2rem; - } - - .editsBox { - position: absolute; - top: 2rem; - right: 2rem; - display: flex; - flex-direction: column; - gap: 1rem; - - img { - transition: all 0.2s ease-in-out; - &:hover { - opacity: 0.8; - } - } - } - } -} diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx deleted file mode 100644 index 261eb4bb4..000000000 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ /dev/null @@ -1,687 +0,0 @@ -/* eslint-disable jsx-a11y/label-has-associated-control */ -/* eslint-disable jsx-a11y/no-noninteractive-element-interactions */ -/* eslint-disable jsx-a11y/img-redundant-alt */ -/* eslint-disable jsx-a11y/click-events-have-key-events */ -/* eslint-disable react/function-component-definition */ -import { Checkbox, FormControlLabel, Slider, TextField } from '@mui/material'; -import { IconButton } from 'browndash-components'; -import * as React from 'react'; -import { useEffect, useRef, useState } from 'react'; -import { CgClose } from 'react-icons/cg'; -import { IoMdRedo, IoMdUndo } from 'react-icons/io'; -import { ClientUtils } from '../../../../ClientUtils'; -import { Doc, DocListCast } from '../../../../fields/Doc'; -import { List } from '../../../../fields/List'; -import { NumCast } from '../../../../fields/Types'; -import { Networking } from '../../../Network'; -import { DocUtils } from '../../../documents/DocUtils'; -import { Docs } from '../../../documents/Documents'; -import { CollectionDockingView } from '../../collections/CollectionDockingView'; -import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; -import { ImageEditorData } from '../ImageBox'; -import { OpenWhereMod } from '../OpenWhere'; -import './GenerativeFill.scss'; -import { EditButtons, CutButtons } from './GenerativeFillButtons'; -import { BrushHandler, BrushType } from './generativeFillUtils/BrushHandler'; -import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; -import { PointerHandler } from './generativeFillUtils/PointerHandler'; -import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; -import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces'; -import { DocumentView } from '../DocumentView'; -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { ImageField } from '../../../../fields/URLField'; -import { resolve } from 'url'; - -interface GenerativeFillProps { - imageEditorOpen: boolean; - imageEditorSource: string; - imageRootDoc: Doc | undefined; - addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; -} - -// Added field on image doc: gen_fill_children: List of children Docs - -const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { - const canvasRef = useRef<HTMLCanvasElement>(null); - const canvasBackgroundRef = useRef<HTMLCanvasElement>(null); - const drawingAreaRef = useRef<HTMLDivElement>(null); - const [cursorData, setCursorData] = useState<CursorData>({ - x: 0, - y: 0, - width: 150, - }); - const [isBrushing, setIsBrushing] = useState(false); - const [canvasScale, setCanvasScale] = useState(0.5); - // format: array of [image source, corresponding image Doc] - const [edits, setEdits] = useState<{ url: string; saveRes: Doc | undefined }[]>([]); - const [edited, setEdited] = useState(false); - // const [brushStyle] = useState<BrushStyle>(BrushStyle.ADD); - const [input, setInput] = useState(''); - const [loading, setLoading] = useState(false); - const [canvasDims, setCanvasDims] = useState<ImageDimensions>({ - width: canvasSize, - height: canvasSize, - }); - // whether to create a new collection or not - const [isNewCollection, setIsNewCollection] = useState(true); - // the current image in the main canvas - const currImg = useRef<HTMLImageElement | null>(null); - // the unedited version of each generation (parent) - const originalImg = useRef<HTMLImageElement | null>(null); - const originalDoc = useRef<Doc | null>(null); - // stores history of data urls - const undoStack = useRef<string[]>([]); - // stores redo stack - const redoStack = useRef<string[]>([]); - - // references to keep track of tree structure - const newCollectionRef = useRef<Doc | null>(null); - const parentDoc = useRef<Doc | null>(null); - const childrenDocs = useRef<Doc[]>([]); - - // constants for image cutting - const cutPts = useRef<Point[]>([]); - - // Undo and Redo - const handleUndo = () => { - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx || !currImg.current || !canvasRef.current) return; - - const target = undoStack.current[undoStack.current.length - 1]; - if (!target) { - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); - } else { - redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()]; - const img = new Image(); - img.src = target; - ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); - undoStack.current = undoStack.current.slice(0, -1); - } - }; - - const handleRedo = () => { - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx || !currImg.current || !canvasRef.current) return; - - const target = redoStack.current[redoStack.current.length - 1]; - if (target) { - undoStack.current = [...undoStack.current, canvasRef.current?.toDataURL()]; - const img = new Image(); - img.src = target; - ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); - redoStack.current = redoStack.current.slice(0, -1); - } - }; - - // resets any erase strokes - const handleReset = () => { - if (!canvasRef.current || !currImg.current) return; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.clearRect(0, 0, canvasSize, canvasSize); - undoStack.current = []; - redoStack.current = []; - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); - }; - - // initiate brushing - const handlePointerDown = (e: React.PointerEvent) => { - const canvas = canvasRef.current; - if (!canvas) return; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return; - - undoStack.current = [...undoStack.current, canvasRef.current.toDataURL()]; - redoStack.current = []; - - setIsBrushing(true); - const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); - BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor /* , brushStyle === BrushStyle.SUBTRACT */); - }; - - // stop brushing, push to undo stack - const handlePointerUp = () => { - const ctx = ImageUtility.getCanvasContext(canvasBackgroundRef); - if (!ctx) return; - if (!isBrushing) return; - setIsBrushing(false); - }; - - // handles brushing on pointer movement - useEffect(() => { - if (!isBrushing) return undefined; - const canvas = canvasRef.current; - if (!canvas) return undefined; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return undefined; - - const handlePointerMove = (e: PointerEvent) => { - const currPoint = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); - const lastPoint: Point = { - x: currPoint.x - e.movementX / canvasScale, - y: currPoint.y - e.movementY / canvasScale, - }; - const pts = BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor, BrushType.CUT); - cutPts.current.push(...pts); - }; - - drawingAreaRef.current?.addEventListener('pointermove', handlePointerMove); - return () => { - drawingAreaRef.current?.removeEventListener('pointermove', handlePointerMove); - }; - }, [isBrushing]); - - // first load - useEffect(() => { - const loadInitial = async () => { - if (!imageEditorSource || imageEditorSource === '') return; - const img = new Image(); - const res = await ImageUtility.urlToBase64(imageEditorSource); - if (!res) return; - img.src = `data:image/png;base64,${res}`; - - img.onload = () => { - currImg.current = img; - originalImg.current = img; - const imgWidth = img.naturalWidth; - const imgHeight = img.naturalHeight; - const scale = Math.min(canvasSize / imgWidth, canvasSize / imgHeight); - const width = imgWidth * scale; - const height = imgHeight * scale; - setCanvasDims({ width, height }); - }; - }; - - loadInitial(); - - // cleanup - return () => { - setInput(''); - setEdited(false); - newCollectionRef.current = null; - parentDoc.current = null; - childrenDocs.current = []; - currImg.current = null; - originalImg.current = null; - originalDoc.current = null; - undoStack.current = []; - redoStack.current = []; - ImageUtility.clearCanvas(canvasRef); - }; - }, [canvasRef, imageEditorSource]); - - // once the appropriate dimensions are set, draw the image to the canvas - useEffect(() => { - if (!currImg.current) return; - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); - }, [canvasDims]); - - // handles brush sizing - useEffect(() => { - const handleKeyPress = (e: KeyboardEvent) => { - if (e.key === 'ArrowUp') { - e.preventDefault(); - e.stopPropagation(); - setCursorData(data => ({ ...data, width: data.width + 5 })); - } else if (e.key === 'ArrowDown') { - e.preventDefault(); - e.stopPropagation(); - setCursorData(data => (data.width >= 20 ? { ...data, width: data.width - 5 } : data)); - } - }; - window.addEventListener('keydown', handleKeyPress); - return () => window.removeEventListener('keydown', handleKeyPress); - }, []); - - // handle pinch zoom - useEffect(() => { - const handlePinch = (e: WheelEvent) => { - e.preventDefault(); - e.stopPropagation(); - const delta = e.deltaY; - const scaleFactor = delta > 0 ? 0.98 : 1.02; - setCanvasScale(prevScale => prevScale * scaleFactor); - }; - - drawingAreaRef.current?.addEventListener('wheel', handlePinch, { - passive: false, - }); - return () => drawingAreaRef.current?.removeEventListener('wheel', handlePinch); - }, [drawingAreaRef]); - - // updates the current position of the cursor - const updateCursorData = (e: React.PointerEvent) => { - const drawingArea = drawingAreaRef.current; - if (!drawingArea) return; - const { x, y } = PointerHandler.getPointRelativeToElement(drawingArea, e, 1); - setCursorData(data => ({ - ...data, - x, - y, - })); - }; - - // Get AI Edit - const getEdit = async () => { - const img = currImg.current; - if (!img) return; - const canvas = canvasRef.current; - if (!canvas) return; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return; - setLoading(true); - setEdited(true); - try { - const canvasOriginalImg = ImageUtility.getCanvasImg(img); - if (!canvasOriginalImg) return; - const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); - if (!canvasMask) return; - const maskBlob = await ImageUtility.canvasToBlob(canvasMask); - const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); - const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); - - // create first image - if (!newCollectionRef.current) { - if (!isNewCollection && imageRootDoc) { - // if the parent hasn't been set yet - if (!parentDoc.current) parentDoc.current = imageRootDoc; - } else { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); - - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - - // add the doc to the main freeform - // eslint-disable-next-line no-use-before-define - await createNewImgDoc(originalImg.current, true); - } - } else { - childrenDocs.current = []; - } - - originalImg.current = currImg.current; - originalDoc.current = parentDoc.current; - const { urls } = res as APISuccess; - if (res.status !== 'error') { - const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height))); - const imgRes = await Promise.all( - imgUrls.map(async url => { - // eslint-disable-next-line no-use-before-define - const saveRes = await onSave(url); - return { url, saveRes }; - }) - ); - setEdits(imgRes); - const image = new Image(); - // eslint-disable-next-line prefer-destructuring - image.src = imgUrls[0]; - ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); - currImg.current = image; - parentDoc.current = imgRes[0].saveRes ?? null; - } - } catch (err) { - console.log(err); - } - setLoading(false); - }; - - const cutImage = async () => { - const img = currImg.current; - const canvas = canvasRef.current; - if (!canvas || !img) return; - canvas.width = img.naturalWidth; - canvas.height = img.naturalHeight; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.globalCompositeOperation = 'source-over'; - setLoading(true); - setEdited(true); - // get the original image - const canvasOriginalImg = ImageUtility.getCanvasImg(img); - if (!canvasOriginalImg) return; - // draw the image onto the canvas - ctx.drawImage(img, 0, 0); - // get the mask which i assume is the thing the user draws on - // const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); - // if (!canvasMask) return; - // canvasMask.width = canvas.width; - // canvasMask.height = canvas.height; - // now put the user's path around the mask - if (cutPts.current.length) { - ctx.beginPath(); - ctx.moveTo(cutPts.current[0].x, cutPts.current[0].y); // later check edge case where cutPts is empty - for (let i = 0; i < cutPts.current.length; i++) { - ctx.lineTo(cutPts.current[i].x, cutPts.current[i].y); - } - ctx.closePath(); - ctx.stroke(); - ctx.fill(); - // ctx.clip(); - } - const url = canvas.toDataURL(); // this does the same thing as convert img to canvasurl - if (!newCollectionRef.current) { - if (!isNewCollection && imageRootDoc) { - // if the parent hasn't been set yet - if (!parentDoc.current) parentDoc.current = imageRootDoc; - } else { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - } - } - const image = new Image(); - image.src = url; - await createNewImgDoc(image, true); - // add the doc to the main freeform - // eslint-disable-next-line no-use-before-define - setLoading(false); - cutPts.current.length = 0; - }; - - // adjusts all the img positions to be aligned - const adjustImgPositions = () => { - if (!parentDoc.current) return; - const startY = NumCast(parentDoc.current.y); - const children = DocListCast(parentDoc.current.gen_fill_children); - const len = children.length; - const initialYPositions: number[] = []; - for (let i = 0; i < len; i++) { - initialYPositions.push(startY + i * offsetDistanceY); - } - children.forEach((doc, i) => { - if (len % 2 === 1) { - doc.y = initialYPositions[i] - Math.floor(len / 2) * offsetDistanceY; - } else { - doc.y = initialYPositions[i] - (len / 2 - 1 / 2) * offsetDistanceY; - } - }); - }; - - // creates a new image document and returns its reference - const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise<Doc | undefined> => { - if (!imageRootDoc) return undefined; - const { src } = img; - const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); - const source = ClientUtils.prepend(result.accessPaths.agnostic.client); - - if (firstDoc) { - const x = 0; - const initialY = 0; - const newImg = Docs.Create.ImageDocument(source, { - x: x, - y: initialY, - _height: freeformRenderSize, - _width: freeformRenderSize, - data_nativeWidth: result.nativeWidth, - data_nativeHeight: result.nativeHeight, - }); - if (isNewCollection && newCollectionRef.current) { - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); - } else { - addDoc?.(newImg); - } - parentDoc.current = newImg; - return newImg; - } - if (!parentDoc.current) return undefined; - const x = NumCast(parentDoc.current.x) + freeformRenderSize + offsetX; - const initialY = 0; - - const newImg = Docs.Create.ImageDocument(source, { - x: x, - y: initialY, - _height: freeformRenderSize, - _width: freeformRenderSize, - data_nativeWidth: result.nativeWidth, - data_nativeHeight: result.nativeHeight, - }); - - const parentList = DocListCast(parentDoc.current.gen_fill_children); - if (parentList.length > 0) { - parentList.push(newImg); - parentDoc.current.gen_fill_children = new List<Doc>(parentList); - } else { - parentDoc.current.gen_fill_children = new List<Doc>([newImg]); - } - - DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: `Image edit; Prompt: ${input}` }); - adjustImgPositions(); - - if (isNewCollection && newCollectionRef.current) { - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); - } else { - addDoc?.(newImg); - } - return newImg; - }; - - // Saves an image to the collection - const onSave = async (src: string) => { - const img = new Image(); - img.src = src; - if (!currImg.current || !originalImg.current || !imageRootDoc) return undefined; - try { - const res = await createNewImgDoc(img, false); - return res; - } catch (err) { - console.log(err); - } - return undefined; - }; - - // Closes the editor view - const handleViewClose = () => { - ImageEditorData.Open = false; - ImageEditorData.Source = ''; - if (newCollectionRef.current) { - DocumentView.addViewRenderedCb(newCollectionRef.current, dv => (dv.ComponentView as CollectionFreeFormView)?.fitContentOnce()); - } - setEdits([]); - }; - - return ( - <div className="generativeFillContainer" style={{ display: imageEditorOpen ? 'flex' : 'none' }}> - <div className="generativeFillControls"> - <h1>Image Editor</h1> - {/* <IconButton text="Cut out" icon={<FontAwesomeIcon icon="scissors" />} /> */} - <div style={{ display: 'flex', alignItems: 'center', gap: '1.5rem' }}> - <FormControlLabel - control={ - <Checkbox - // disable once edited has been clicked (doesn't make sense to change after first edit) - disabled={edited} - checked={isNewCollection} - onChange={() => { - setIsNewCollection(prev => !prev); - }} - /> - } - label="Create New Collection" - labelPlacement="end" - sx={{ whiteSpace: 'nowrap' }} - /> - <EditButtons onClick={getEdit} loading={loading} onReset={handleReset} /> - <CutButtons onClick={cutImage} loading={loading} onReset={handleReset} /> - <IconButton color={activeColor} tooltip="close" icon={<CgClose size="16px" />} onClick={handleViewClose} /> - </div> - </div> - {/* Main canvas for editing */} - <div - className="drawingArea" // this only works if pointerevents: none is set on the custom pointer - ref={drawingAreaRef} - onPointerOver={updateCursorData} - onPointerMove={updateCursorData} - onPointerDown={handlePointerDown} - onPointerUp={handlePointerUp}> - <canvas ref={canvasRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} /> - <canvas ref={canvasBackgroundRef} width={canvasDims.width} height={canvasDims.height} style={{ transform: `scale(${canvasScale})` }} /> - <div - className="pointer" - style={{ - left: cursorData.x, - top: cursorData.y, - width: cursorData.width, - height: cursorData.width, - }}> - <div className="innerPointer" /> - </div> - {/* Icons */} - <div className="iconContainer"> - {/* Undo and Redo */} - <IconButton - style={{ cursor: 'pointer' }} - onPointerDown={e => { - e.stopPropagation(); - handleUndo(); - }} - onPointerUp={e => { - e.stopPropagation(); - }} - color={activeColor} - tooltip="Undo" - icon={<IoMdUndo />} - /> - <IconButton - style={{ cursor: 'pointer' }} - onPointerDown={e => { - e.stopPropagation(); - handleRedo(); - }} - onPointerUp={e => { - e.stopPropagation(); - }} - color={activeColor} - tooltip="Redo" - icon={<IoMdRedo />} - /> - <div onPointerDown={e => e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}> - <Slider - sx={{ - '& input[type="range"]': { - WebkitAppearance: 'slider-vertical', - }, - }} - orientation="vertical" - min={25} - max={500} - defaultValue={150} - size="small" - valueLabelDisplay="auto" - onChange={(e: any, val: any) => { - setCursorData(prev => ({ ...prev, width: val as number })); - }} - /> - </div> - <div onPointerDown={e => e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}> - <Slider - sx={{ - '& input[type="range"]': { - WebkitAppearance: 'slider-vertical', - }, - }} - orientation="vertical" - min={1} - max={500} - defaultValue={150} - size="small" - valueLabelDisplay="auto" - onChange={(e: any, val: any) => { - setCursorData(prev => ({ ...prev, width: val as number })); - }} - /> - </div> - </div> - {/* Edits thumbnails */} - <div className="editsBox"> - {edits.map((edit, i) => ( - <img - // eslint-disable-next-line react/no-array-index-key - key={i} - alt="image edits" - width={75} - src={edit.url} - style={{ cursor: 'pointer' }} - onClick={async () => { - const img = new Image(); - img.src = edit.url; - ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); - currImg.current = img; - parentDoc.current = edit.saveRes ?? null; - }} - /> - ))} - {/* Original img thumbnail */} - {edits.length > 0 && ( - <div style={{ position: 'relative' }}> - <label - style={{ - position: 'absolute', - bottom: 10, - left: 10, - color: '#ffffff', - fontSize: '0.8rem', - letterSpacing: '1px', - textTransform: 'uppercase', - }}> - Original - </label> - <img - alt="image stuff" - width={75} - src={originalImg.current?.src} - style={{ cursor: 'pointer' }} - onClick={() => { - if (!originalImg.current) return; - const img = new Image(); - img.src = originalImg.current.src; - ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); - currImg.current = img; - parentDoc.current = originalDoc.current; - }} - /> - </div> - )} - </div> - </div> - <div> - <TextField - value={input} - onChange={(e: any) => setInput(e.target.value)} - disabled={isBrushing} - type="text" - label="Prompt" - placeholder="Prompt..." - InputLabelProps={{ style: { fontSize: '16px' } }} - inputProps={{ style: { fontSize: '16px' } }} - sx={{ - backgroundColor: '#ffffff', - position: 'absolute', - bottom: '16px', - transform: 'translateX(calc(50vw - 50%))', - width: 'calc(100vw - 64px)', - }} - /> - </div> - </div> - ); -}; - -export default GenerativeFill; diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss b/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss deleted file mode 100644 index 0180ef904..000000000 --- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.scss +++ /dev/null @@ -1,4 +0,0 @@ -.generativeFillBtnContainer { - display: flex; - gap: 1rem; -} diff --git a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx b/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx deleted file mode 100644 index fe22b273d..000000000 --- a/src/client/views/nodes/generativeFill/GenerativeFillButtons.tsx +++ /dev/null @@ -1,72 +0,0 @@ -import './GenerativeFillButtons.scss'; -import * as React from 'react'; -import ReactLoading from 'react-loading'; -import { Button, IconButton, Type } from 'browndash-components'; -import { AiOutlineInfo } from 'react-icons/ai'; -import { activeColor } from './generativeFillUtils/generativeFillConstants'; - -interface ButtonContainerProps { - onClick: () => Promise<void>; - loading: boolean; - onReset: () => void; -} - -export function EditButtons({ loading, onClick: getEdit, onReset }: ButtonContainerProps) { - return ( - <div className="generativeFillBtnContainer"> - <Button text="RESET" type={Type.PRIM} color={activeColor} onClick={onReset} /> - {loading ? ( - <Button - text="GET EDITS" - type={Type.TERT} - color={activeColor} - icon={<ReactLoading type="spin" color="#ffffff" width={20} height={20} />} - iconPlacement="right" - onClick={() => { - if (!loading) getEdit(); - }} - /> - ) : ( - <Button - text="GET EDITS" - type={Type.TERT} - color={activeColor} - onClick={() => { - if (!loading) getEdit(); - }} - /> - )} - <IconButton type={Type.SEC} color={activeColor} tooltip="Open Documentation" icon={<AiOutlineInfo size="16px" />} onClick={() => window.open('https://brown-dash.github.io/Dash-Documentation/features/generativeai/#editing', '_blank')} /> - </div> - ); -} - -export function CutButtons({ loading, onClick: cutImage, onReset }: ButtonContainerProps) { - return ( - <div className="generativeFillBtnContainer"> - <Button text="RESET" type={Type.PRIM} color={activeColor} onClick={onReset} /> - {loading ? ( - <Button - text="CUT IMAGE" - type={Type.TERT} - color={activeColor} - icon={<ReactLoading type="spin" color="#ffffff" width={20} height={20} />} - iconPlacement="right" - onClick={() => { - if (!loading) cutImage(); - }} - /> - ) : ( - <Button - text="CUT IMAGE" - type={Type.TERT} - color={activeColor} - onClick={() => { - if (!loading) cutImage(); - }} - /> - )} - <IconButton type={Type.SEC} color={activeColor} tooltip="Open Documentation" icon={<AiOutlineInfo size="16px" />} onClick={() => window.open('https://brown-dash.github.io/Dash-Documentation/features/generativeai/#editing', '_blank')} /> - </div> - ); -} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts deleted file mode 100644 index 8a66d7347..000000000 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/BrushHandler.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { GenerativeFillMathHelpers } from './GenerativeFillMathHelpers'; -import { eraserColor } from './generativeFillConstants'; -import { Point } from './generativeFillInterfaces'; -import { points } from '@turf/turf'; - -export enum BrushType { - GEN_FILL, - CUT, -} - -export class BrushHandler { - static brushCircleOverlay = (x: number, y: number, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string /* , erase: boolean */) => { - ctx.globalCompositeOperation = 'destination-out'; - ctx.fillStyle = fillColor; - ctx.shadowColor = eraserColor; - ctx.shadowBlur = 5; - ctx.beginPath(); - ctx.arc(x, y, brushRadius, 0, 2 * Math.PI); - ctx.fill(); - ctx.closePath(); - }; - - static createBrushPathOverlay = (startPoint: Point, endPoint: Point, brushRadius: number, ctx: CanvasRenderingContext2D, fillColor: string, brushType: BrushType) => { - const dist = GenerativeFillMathHelpers.distanceBetween(startPoint, endPoint); - const pts: Point[] = []; - for (let i = 0; i < dist; i += 5) { - const s = i / dist; - const x = startPoint.x * (1 - s) + endPoint.x * s; - const y = startPoint.y * (1 - s) + endPoint.y * s; - pts.push({ x: startPoint.x, y: startPoint.y }); - BrushHandler.brushCircleOverlay(x, y, brushRadius, ctx, fillColor); - } - return pts; - }; -} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts deleted file mode 100644 index 6da8c3da0..000000000 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/GenerativeFillMathHelpers.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Point } from './generativeFillInterfaces'; - -export class GenerativeFillMathHelpers { - static distanceBetween = (p1: Point, p2: Point) => Math.sqrt((p2.x - p1.x) ** 2 + (p2.y - p1.y) ** 2); - static angleBetween = (p1: Point, p2: Point) => Math.atan2(p2.x - p1.x, p2.y - p1.y); -} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts deleted file mode 100644 index 24dba1778..000000000 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/ImageHandler.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { RefObject } from 'react'; -import { bgColor, canvasSize } from './generativeFillConstants'; - -export interface APISuccess { - status: 'success'; - urls: string[]; -} - -export interface APIError { - status: 'error'; - message: string; -} - -export class ImageUtility { - /** - * - * @param canvas Canvas to convert - * @returns Blob of canvas - */ - static canvasToBlob = (canvas: HTMLCanvasElement): Promise<Blob> => - new Promise(resolve => { - canvas.toBlob(blob => { - if (blob) { - resolve(blob); - } - }, 'image/png'); - }); - - // given a square api image, get the cropped img - static getCroppedImg = (img: HTMLImageElement, width: number, height: number): HTMLCanvasElement | undefined => { - // Create a new canvas element - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - const ctx = canvas.getContext('2d'); - if (ctx) { - // Clear the canvas - ctx.clearRect(0, 0, canvas.width, canvas.height); - if (width < height) { - // horizontal padding, x offset - const xOffset = (canvasSize - width) / 2; - ctx.drawImage(img, xOffset, 0, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); - } else { - // vertical padding, y offset - const yOffset = (canvasSize - height) / 2; - ctx.drawImage(img, 0, yOffset, canvas.width, canvas.height, 0, 0, canvas.width, canvas.height); - } - return canvas; - } - return undefined; - }; - - // converts an image to a canvas data url - static convertImgToCanvasUrl = async (imageSrc: string, width: number, height: number): Promise<string> => - new Promise<string>((resolve, reject) => { - const img = new Image(); - img.onload = () => { - const canvas = this.getCroppedImg(img, width, height); - if (canvas) { - const dataUrl = canvas.toDataURL(); - resolve(dataUrl); - } - }; - img.onerror = error => { - reject(error); - }; - img.src = imageSrc; - }); - - // calls the openai api to get image edits - static getEdit = async (imgBlob: Blob, maskBlob: Blob, prompt: string, n?: number): Promise<APISuccess | APIError> => { - const apiUrl = 'https://api.openai.com/v1/images/edits'; - const fd = new FormData(); - fd.append('image', imgBlob, 'image.png'); - fd.append('mask', maskBlob, 'mask.png'); - fd.append('prompt', prompt); - fd.append('size', '1024x1024'); - fd.append('n', n ? JSON.stringify(n) : '1'); - fd.append('response_format', 'b64_json'); - - try { - const res = await fetch(apiUrl, { - method: 'POST', - headers: { - Authorization: `Bearer ${process.env.OPENAI_KEY}`, - }, - body: fd, - }); - const data = await res.json(); - console.log(data.data); - return { - status: 'success', - urls: (data.data as { b64_json: string }[]).map(urlData => `data:image/png;base64,${urlData.b64_json}`), - }; - } catch (err) { - console.log(err); - return { status: 'error', message: 'API error.' }; - } - }; - - // mock api call - static mockGetEdit = async (mockSrc: string): Promise<APISuccess | APIError> => ({ - status: 'success', - urls: [mockSrc, mockSrc, mockSrc], - }); - - // Gets the canvas rendering context of a canvas - static getCanvasContext = (canvasRef: RefObject<HTMLCanvasElement>): CanvasRenderingContext2D | null => { - if (!canvasRef.current) return null; - const ctx = canvasRef.current.getContext('2d'); - if (!ctx) return null; - return ctx; - }; - - // Helper for downloading the canvas (for debugging) - static downloadCanvas = (canvas: HTMLCanvasElement) => { - const url = canvas.toDataURL(); - const downloadLink = document.createElement('a'); - downloadLink.href = url; - downloadLink.download = 'canvas'; - - downloadLink.click(); - downloadLink.remove(); - }; - - // Download the canvas (for debugging) - static downloadImageCanvas = (imgUrl: string) => { - const img = new Image(); - img.src = imgUrl; - img.onload = () => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - ctx?.drawImage(img, 0, 0, canvasSize, canvasSize); - - this.downloadCanvas(canvas); - }; - }; - - // Clears the canvas - static clearCanvas = (canvasRef: React.RefObject<HTMLCanvasElement>) => { - const ctx = this.getCanvasContext(canvasRef); - if (!ctx || !canvasRef.current) return; - ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); - }; - - // Draws the image to the current canvas - static drawImgToCanvas = (img: HTMLImageElement, canvasRef: React.RefObject<HTMLCanvasElement>, width: number, height: number) => { - const drawImg = (htmlImg: HTMLImageElement) => { - const ctx = this.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.globalCompositeOperation = 'source-over'; - ctx.clearRect(0, 0, width, height); - ctx.drawImage(htmlImg, 0, 0, width, height); - }; - - if (img.complete) { - drawImg(img); - } else { - img.onload = () => { - drawImg(img); - }; - } - }; - - // Gets the image mask for the openai endpoint - static getCanvasMask = (srcCanvas: HTMLCanvasElement, paddedCanvas: HTMLCanvasElement): HTMLCanvasElement | undefined => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - if (!ctx) return undefined; - ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.drawImage(paddedCanvas, 0, 0); - - // extract and set padding data - if (srcCanvas.height > srcCanvas.width) { - // horizontal padding, x offset - const xOffset = (canvasSize - srcCanvas.width) / 2; - ctx?.clearRect(xOffset, 0, srcCanvas.width, srcCanvas.height); - ctx.drawImage(srcCanvas, xOffset, 0, srcCanvas.width, srcCanvas.height); - } else { - // vertical padding, y offset - const yOffset = (canvasSize - srcCanvas.height) / 2; - ctx?.clearRect(0, yOffset, srcCanvas.width, srcCanvas.height); - ctx.drawImage(srcCanvas, 0, yOffset, srcCanvas.width, srcCanvas.height); - } - return canvas; - }; - - // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) - static drawHorizontalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, xOffset: number) => { - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const { data } = imageData; - for (let i = 0; i < canvas.height; i++) { - for (let j = 0; j < xOffset; j++) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceI = i; - const sourceJ = xOffset + (xOffset - j); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - for (let i = 0; i < canvas.height; i++) { - for (let j = canvas.width - 1; j >= canvas.width - 1 - xOffset; j--) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceI = i; - const sourceJ = canvas.width - 1 - xOffset - (xOffset - (canvas.width - j)); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - ctx.putImageData(imageData, 0, 0); - }; - - // Fills in the blank areas of the image with an image reflection (to fill in a square-shaped canvas) - static drawVerticalReflection = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, yOffset: number) => { - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const { data } = imageData; - for (let j = 0; j < canvas.width; j++) { - for (let i = 0; i < yOffset; i++) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceJ = j; - const sourceI = yOffset + (yOffset - i); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - for (let j = 0; j < canvas.width; j++) { - for (let i = canvas.height - 1; i >= canvas.height - 1 - yOffset; i--) { - const targetIdx = 4 * (i * canvas.width + j); - const sourceJ = j; - const sourceI = canvas.height - 1 - yOffset - (yOffset - (canvas.height - i)); - const sourceIdx = 4 * (sourceI * canvas.width + sourceJ); - data[targetIdx] = data[sourceIdx]; - data[targetIdx + 1] = data[sourceIdx + 1]; - data[targetIdx + 2] = data[sourceIdx + 2]; - } - } - ctx.putImageData(imageData, 0, 0); - }; - - // Gets the unaltered (besides filling in padding) version of the image for the api call - static getCanvasImg = (img: HTMLImageElement): HTMLCanvasElement | undefined => { - const canvas = document.createElement('canvas'); - canvas.width = canvasSize; - canvas.height = canvasSize; - const ctx = canvas.getContext('2d'); - if (!ctx) return undefined; - // fix scaling - const scale = Math.min(canvasSize / img.width, canvasSize / img.height); - const width = Math.floor(img.width * scale); - const height = Math.floor(img.height * scale); - ctx?.clearRect(0, 0, canvasSize, canvasSize); - ctx.fillStyle = bgColor; - ctx.fillRect(0, 0, canvasSize, canvasSize); - - // extract and set padding data - if (img.naturalHeight > img.naturalWidth) { - // horizontal padding, x offset - const xOffset = Math.floor((canvasSize - width) / 2); - ctx.drawImage(img, xOffset, 0, width, height); - - // draw reflected image padding - this.drawHorizontalReflection(ctx, canvas, xOffset); - } else { - // vertical padding, y offset - const yOffset = Math.floor((canvasSize - height) / 2); - ctx.drawImage(img, 0, yOffset, width, height); - - // draw reflected image padding - this.drawVerticalReflection(ctx, canvas, yOffset); - } - return canvas; - }; - - /** - * Converts a url to base64 (tainted canvas workaround) - */ - static urlToBase64 = async (imageUrl: string): Promise<string | undefined> => { - try { - const res = await fetch(imageUrl); - const blob = await res.blob(); - - return new Promise<string>((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const base64Data = reader.result?.toString().split(',')[1]; - if (base64Data) { - resolve(base64Data); - } else { - reject(new Error('Failed to convert.')); - } - }; - reader.onerror = () => { - reject(new Error('Error reading image data')); - }; - reader.readAsDataURL(blob); - }); - } catch (err) { - console.error(err); - } - return undefined; - }; -} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts deleted file mode 100644 index 260923a64..000000000 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/PointerHandler.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Point } from './generativeFillInterfaces'; - -export class PointerHandler { - static getPointRelativeToElement = (element: HTMLElement, e: React.PointerEvent | PointerEvent, scale: number): Point => { - const boundingBox = element.getBoundingClientRect(); - return { - x: (e.clientX - boundingBox.x) / scale, - y: (e.clientY - boundingBox.y) / scale, - }; - }; -} diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts deleted file mode 100644 index 4772304bc..000000000 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillConstants.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const canvasSize = 1024; -export const freeformRenderSize = 300; -export const offsetDistanceY = freeformRenderSize + 400; -export const offsetX = 200; -export const newCollectionSize = 500; - -export const activeColor = '#1976d2'; -export const eraserColor = '#e1e9ec'; -export const bgColor = '#f0f4f6'; diff --git a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts b/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts deleted file mode 100644 index 1e7801056..000000000 --- a/src/client/views/nodes/generativeFill/generativeFillUtils/generativeFillInterfaces.ts +++ /dev/null @@ -1,20 +0,0 @@ -export interface CursorData { - x: number; - y: number; - width: number; -} - -export interface Point { - x: number; - y: number; -} - -export enum BrushMode { - ADD, - SUBTRACT, -} - -export interface ImageDimensions { - width: number; - height: number; -} |