From 3b17868560090756caf8b9b0f043ea163f2320e8 Mon Sep 17 00:00:00 2001 From: eleanor-park Date: Sun, 20 Oct 2024 12:44:15 -0400 Subject: changes --- .../views/nodes/generativeFill/GenerativeFill.tsx | 687 --------------------- 1 file changed, 687 deletions(-) delete mode 100644 src/client/views/nodes/generativeFill/GenerativeFill.tsx (limited to 'src/client/views/nodes/generativeFill/GenerativeFill.tsx') diff --git a/src/client/views/nodes/generativeFill/GenerativeFill.tsx b/src/client/views/nodes/generativeFill/GenerativeFill.tsx deleted file mode 100644 index 261eb4bb4..000000000 --- a/src/client/views/nodes/generativeFill/GenerativeFill.tsx +++ /dev/null @@ -1,687 +0,0 @@ -/* eslint-disable jsx-a11y/label-has-associated-control */ -/* eslint-disable jsx-a11y/no-noninteractive-element-interactions */ -/* eslint-disable jsx-a11y/img-redundant-alt */ -/* eslint-disable jsx-a11y/click-events-have-key-events */ -/* eslint-disable react/function-component-definition */ -import { Checkbox, FormControlLabel, Slider, TextField } from '@mui/material'; -import { IconButton } from 'browndash-components'; -import * as React from 'react'; -import { useEffect, useRef, useState } from 'react'; -import { CgClose } from 'react-icons/cg'; -import { IoMdRedo, IoMdUndo } from 'react-icons/io'; -import { ClientUtils } from '../../../../ClientUtils'; -import { Doc, DocListCast } from '../../../../fields/Doc'; -import { List } from '../../../../fields/List'; -import { NumCast } from '../../../../fields/Types'; -import { Networking } from '../../../Network'; -import { DocUtils } from '../../../documents/DocUtils'; -import { Docs } from '../../../documents/Documents'; -import { CollectionDockingView } from '../../collections/CollectionDockingView'; -import { CollectionFreeFormView } from '../../collections/collectionFreeForm'; -import { ImageEditorData } from '../ImageBox'; -import { OpenWhereMod } from '../OpenWhere'; -import './GenerativeFill.scss'; -import { EditButtons, CutButtons } from './GenerativeFillButtons'; -import { BrushHandler, BrushType } from './generativeFillUtils/BrushHandler'; -import { APISuccess, ImageUtility } from './generativeFillUtils/ImageHandler'; -import { PointerHandler } from './generativeFillUtils/PointerHandler'; -import { activeColor, canvasSize, eraserColor, freeformRenderSize, newCollectionSize, offsetDistanceY, offsetX } from './generativeFillUtils/generativeFillConstants'; -import { CursorData, ImageDimensions, Point } from './generativeFillUtils/generativeFillInterfaces'; -import { DocumentView } from '../DocumentView'; -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { ImageField } from '../../../../fields/URLField'; -import { resolve } from 'url'; - -interface GenerativeFillProps { - imageEditorOpen: boolean; - imageEditorSource: string; - imageRootDoc: Doc | undefined; - addDoc: ((doc: Doc | Doc[], annotationKey?: string) => boolean) | undefined; -} - -// Added field on image doc: gen_fill_children: List of children Docs - -const GenerativeFill = ({ imageEditorOpen, imageEditorSource, imageRootDoc, addDoc }: GenerativeFillProps) => { - const canvasRef = useRef(null); - const canvasBackgroundRef = useRef(null); - const drawingAreaRef = useRef(null); - const [cursorData, setCursorData] = useState({ - x: 0, - y: 0, - width: 150, - }); - const [isBrushing, setIsBrushing] = useState(false); - const [canvasScale, setCanvasScale] = useState(0.5); - // format: array of [image source, corresponding image Doc] - const [edits, setEdits] = useState<{ url: string; saveRes: Doc | undefined }[]>([]); - const [edited, setEdited] = useState(false); - // const [brushStyle] = useState(BrushStyle.ADD); - const [input, setInput] = useState(''); - const [loading, setLoading] = useState(false); - const [canvasDims, setCanvasDims] = useState({ - width: canvasSize, - height: canvasSize, - }); - // whether to create a new collection or not - const [isNewCollection, setIsNewCollection] = useState(true); - // the current image in the main canvas - const currImg = useRef(null); - // the unedited version of each generation (parent) - const originalImg = useRef(null); - const originalDoc = useRef(null); - // stores history of data urls - const undoStack = useRef([]); - // stores redo stack - const redoStack = useRef([]); - - // references to keep track of tree structure - const newCollectionRef = useRef(null); - const parentDoc = useRef(null); - const childrenDocs = useRef([]); - - // constants for image cutting - const cutPts = useRef([]); - - // Undo and Redo - const handleUndo = () => { - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx || !currImg.current || !canvasRef.current) return; - - const target = undoStack.current[undoStack.current.length - 1]; - if (!target) { - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); - } else { - redoStack.current = [...redoStack.current, canvasRef.current.toDataURL()]; - const img = new Image(); - img.src = target; - ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); - undoStack.current = undoStack.current.slice(0, -1); - } - }; - - const handleRedo = () => { - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx || !currImg.current || !canvasRef.current) return; - - const target = redoStack.current[redoStack.current.length - 1]; - if (target) { - undoStack.current = [...undoStack.current, canvasRef.current?.toDataURL()]; - const img = new Image(); - img.src = target; - ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); - redoStack.current = redoStack.current.slice(0, -1); - } - }; - - // resets any erase strokes - const handleReset = () => { - if (!canvasRef.current || !currImg.current) return; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.clearRect(0, 0, canvasSize, canvasSize); - undoStack.current = []; - redoStack.current = []; - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); - }; - - // initiate brushing - const handlePointerDown = (e: React.PointerEvent) => { - const canvas = canvasRef.current; - if (!canvas) return; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return; - - undoStack.current = [...undoStack.current, canvasRef.current.toDataURL()]; - redoStack.current = []; - - setIsBrushing(true); - const { x, y } = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); - BrushHandler.brushCircleOverlay(x, y, cursorData.width / 2 / canvasScale, ctx, eraserColor /* , brushStyle === BrushStyle.SUBTRACT */); - }; - - // stop brushing, push to undo stack - const handlePointerUp = () => { - const ctx = ImageUtility.getCanvasContext(canvasBackgroundRef); - if (!ctx) return; - if (!isBrushing) return; - setIsBrushing(false); - }; - - // handles brushing on pointer movement - useEffect(() => { - if (!isBrushing) return undefined; - const canvas = canvasRef.current; - if (!canvas) return undefined; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return undefined; - - const handlePointerMove = (e: PointerEvent) => { - const currPoint = PointerHandler.getPointRelativeToElement(canvas, e, canvasScale); - const lastPoint: Point = { - x: currPoint.x - e.movementX / canvasScale, - y: currPoint.y - e.movementY / canvasScale, - }; - const pts = BrushHandler.createBrushPathOverlay(lastPoint, currPoint, cursorData.width / 2 / canvasScale, ctx, eraserColor, BrushType.CUT); - cutPts.current.push(...pts); - }; - - drawingAreaRef.current?.addEventListener('pointermove', handlePointerMove); - return () => { - drawingAreaRef.current?.removeEventListener('pointermove', handlePointerMove); - }; - }, [isBrushing]); - - // first load - useEffect(() => { - const loadInitial = async () => { - if (!imageEditorSource || imageEditorSource === '') return; - const img = new Image(); - const res = await ImageUtility.urlToBase64(imageEditorSource); - if (!res) return; - img.src = `data:image/png;base64,${res}`; - - img.onload = () => { - currImg.current = img; - originalImg.current = img; - const imgWidth = img.naturalWidth; - const imgHeight = img.naturalHeight; - const scale = Math.min(canvasSize / imgWidth, canvasSize / imgHeight); - const width = imgWidth * scale; - const height = imgHeight * scale; - setCanvasDims({ width, height }); - }; - }; - - loadInitial(); - - // cleanup - return () => { - setInput(''); - setEdited(false); - newCollectionRef.current = null; - parentDoc.current = null; - childrenDocs.current = []; - currImg.current = null; - originalImg.current = null; - originalDoc.current = null; - undoStack.current = []; - redoStack.current = []; - ImageUtility.clearCanvas(canvasRef); - }; - }, [canvasRef, imageEditorSource]); - - // once the appropriate dimensions are set, draw the image to the canvas - useEffect(() => { - if (!currImg.current) return; - ImageUtility.drawImgToCanvas(currImg.current, canvasRef, canvasDims.width, canvasDims.height); - }, [canvasDims]); - - // handles brush sizing - useEffect(() => { - const handleKeyPress = (e: KeyboardEvent) => { - if (e.key === 'ArrowUp') { - e.preventDefault(); - e.stopPropagation(); - setCursorData(data => ({ ...data, width: data.width + 5 })); - } else if (e.key === 'ArrowDown') { - e.preventDefault(); - e.stopPropagation(); - setCursorData(data => (data.width >= 20 ? { ...data, width: data.width - 5 } : data)); - } - }; - window.addEventListener('keydown', handleKeyPress); - return () => window.removeEventListener('keydown', handleKeyPress); - }, []); - - // handle pinch zoom - useEffect(() => { - const handlePinch = (e: WheelEvent) => { - e.preventDefault(); - e.stopPropagation(); - const delta = e.deltaY; - const scaleFactor = delta > 0 ? 0.98 : 1.02; - setCanvasScale(prevScale => prevScale * scaleFactor); - }; - - drawingAreaRef.current?.addEventListener('wheel', handlePinch, { - passive: false, - }); - return () => drawingAreaRef.current?.removeEventListener('wheel', handlePinch); - }, [drawingAreaRef]); - - // updates the current position of the cursor - const updateCursorData = (e: React.PointerEvent) => { - const drawingArea = drawingAreaRef.current; - if (!drawingArea) return; - const { x, y } = PointerHandler.getPointRelativeToElement(drawingArea, e, 1); - setCursorData(data => ({ - ...data, - x, - y, - })); - }; - - // Get AI Edit - const getEdit = async () => { - const img = currImg.current; - if (!img) return; - const canvas = canvasRef.current; - if (!canvas) return; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return; - setLoading(true); - setEdited(true); - try { - const canvasOriginalImg = ImageUtility.getCanvasImg(img); - if (!canvasOriginalImg) return; - const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); - if (!canvasMask) return; - const maskBlob = await ImageUtility.canvasToBlob(canvasMask); - const imgBlob = await ImageUtility.canvasToBlob(canvasOriginalImg); - const res = await ImageUtility.getEdit(imgBlob, maskBlob, input !== '' ? input + ' in the same style' : 'Fill in the image in the same style', 2); - - // create first image - if (!newCollectionRef.current) { - if (!isNewCollection && imageRootDoc) { - // if the parent hasn't been set yet - if (!parentDoc.current) parentDoc.current = imageRootDoc; - } else { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); - - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - - // add the doc to the main freeform - // eslint-disable-next-line no-use-before-define - await createNewImgDoc(originalImg.current, true); - } - } else { - childrenDocs.current = []; - } - - originalImg.current = currImg.current; - originalDoc.current = parentDoc.current; - const { urls } = res as APISuccess; - if (res.status !== 'error') { - const imgUrls = await Promise.all(urls.map(url => ImageUtility.convertImgToCanvasUrl(url, canvasDims.width, canvasDims.height))); - const imgRes = await Promise.all( - imgUrls.map(async url => { - // eslint-disable-next-line no-use-before-define - const saveRes = await onSave(url); - return { url, saveRes }; - }) - ); - setEdits(imgRes); - const image = new Image(); - // eslint-disable-next-line prefer-destructuring - image.src = imgUrls[0]; - ImageUtility.drawImgToCanvas(image, canvasRef, canvasDims.width, canvasDims.height); - currImg.current = image; - parentDoc.current = imgRes[0].saveRes ?? null; - } - } catch (err) { - console.log(err); - } - setLoading(false); - }; - - const cutImage = async () => { - const img = currImg.current; - const canvas = canvasRef.current; - if (!canvas || !img) return; - canvas.width = img.naturalWidth; - canvas.height = img.naturalHeight; - const ctx = ImageUtility.getCanvasContext(canvasRef); - if (!ctx) return; - ctx.globalCompositeOperation = 'source-over'; - setLoading(true); - setEdited(true); - // get the original image - const canvasOriginalImg = ImageUtility.getCanvasImg(img); - if (!canvasOriginalImg) return; - // draw the image onto the canvas - ctx.drawImage(img, 0, 0); - // get the mask which i assume is the thing the user draws on - // const canvasMask = ImageUtility.getCanvasMask(canvas, canvasOriginalImg); - // if (!canvasMask) return; - // canvasMask.width = canvas.width; - // canvasMask.height = canvas.height; - // now put the user's path around the mask - if (cutPts.current.length) { - ctx.beginPath(); - ctx.moveTo(cutPts.current[0].x, cutPts.current[0].y); // later check edge case where cutPts is empty - for (let i = 0; i < cutPts.current.length; i++) { - ctx.lineTo(cutPts.current[i].x, cutPts.current[i].y); - } - ctx.closePath(); - ctx.stroke(); - ctx.fill(); - // ctx.clip(); - } - const url = canvas.toDataURL(); // this does the same thing as convert img to canvasurl - if (!newCollectionRef.current) { - if (!isNewCollection && imageRootDoc) { - // if the parent hasn't been set yet - if (!parentDoc.current) parentDoc.current = imageRootDoc; - } else { - if (!(originalImg.current && imageRootDoc)) return; - // create new collection and add it to the view - newCollectionRef.current = Docs.Create.FreeformDocument([], { - x: NumCast(imageRootDoc.x) + NumCast(imageRootDoc._width) + offsetX, - y: NumCast(imageRootDoc.y), - _width: newCollectionSize, - _height: newCollectionSize, - title: 'Image edit collection', - }); - DocUtils.MakeLink(imageRootDoc, newCollectionRef.current, { link_relationship: 'Image Edit Version History' }); - // opening new tab - CollectionDockingView.AddSplit(newCollectionRef.current, OpenWhereMod.right); - } - } - const image = new Image(); - image.src = url; - await createNewImgDoc(image, true); - // add the doc to the main freeform - // eslint-disable-next-line no-use-before-define - setLoading(false); - cutPts.current.length = 0; - }; - - // adjusts all the img positions to be aligned - const adjustImgPositions = () => { - if (!parentDoc.current) return; - const startY = NumCast(parentDoc.current.y); - const children = DocListCast(parentDoc.current.gen_fill_children); - const len = children.length; - const initialYPositions: number[] = []; - for (let i = 0; i < len; i++) { - initialYPositions.push(startY + i * offsetDistanceY); - } - children.forEach((doc, i) => { - if (len % 2 === 1) { - doc.y = initialYPositions[i] - Math.floor(len / 2) * offsetDistanceY; - } else { - doc.y = initialYPositions[i] - (len / 2 - 1 / 2) * offsetDistanceY; - } - }); - }; - - // creates a new image document and returns its reference - const createNewImgDoc = async (img: HTMLImageElement, firstDoc: boolean): Promise => { - if (!imageRootDoc) return undefined; - const { src } = img; - const [result] = await Networking.PostToServer('/uploadRemoteImage', { sources: [src] }); - const source = ClientUtils.prepend(result.accessPaths.agnostic.client); - - if (firstDoc) { - const x = 0; - const initialY = 0; - const newImg = Docs.Create.ImageDocument(source, { - x: x, - y: initialY, - _height: freeformRenderSize, - _width: freeformRenderSize, - data_nativeWidth: result.nativeWidth, - data_nativeHeight: result.nativeHeight, - }); - if (isNewCollection && newCollectionRef.current) { - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); - } else { - addDoc?.(newImg); - } - parentDoc.current = newImg; - return newImg; - } - if (!parentDoc.current) return undefined; - const x = NumCast(parentDoc.current.x) + freeformRenderSize + offsetX; - const initialY = 0; - - const newImg = Docs.Create.ImageDocument(source, { - x: x, - y: initialY, - _height: freeformRenderSize, - _width: freeformRenderSize, - data_nativeWidth: result.nativeWidth, - data_nativeHeight: result.nativeHeight, - }); - - const parentList = DocListCast(parentDoc.current.gen_fill_children); - if (parentList.length > 0) { - parentList.push(newImg); - parentDoc.current.gen_fill_children = new List(parentList); - } else { - parentDoc.current.gen_fill_children = new List([newImg]); - } - - DocUtils.MakeLink(parentDoc.current, newImg, { link_relationship: `Image edit; Prompt: ${input}` }); - adjustImgPositions(); - - if (isNewCollection && newCollectionRef.current) { - Doc.AddDocToList(newCollectionRef.current, undefined, newImg); - } else { - addDoc?.(newImg); - } - return newImg; - }; - - // Saves an image to the collection - const onSave = async (src: string) => { - const img = new Image(); - img.src = src; - if (!currImg.current || !originalImg.current || !imageRootDoc) return undefined; - try { - const res = await createNewImgDoc(img, false); - return res; - } catch (err) { - console.log(err); - } - return undefined; - }; - - // Closes the editor view - const handleViewClose = () => { - ImageEditorData.Open = false; - ImageEditorData.Source = ''; - if (newCollectionRef.current) { - DocumentView.addViewRenderedCb(newCollectionRef.current, dv => (dv.ComponentView as CollectionFreeFormView)?.fitContentOnce()); - } - setEdits([]); - }; - - return ( -
-
-

Image Editor

- {/* } /> */} -
- { - setIsNewCollection(prev => !prev); - }} - /> - } - label="Create New Collection" - labelPlacement="end" - sx={{ whiteSpace: 'nowrap' }} - /> - - - } onClick={handleViewClose} /> -
-
- {/* Main canvas for editing */} -
- - -
-
-
- {/* Icons */} -
- {/* Undo and Redo */} - { - e.stopPropagation(); - handleUndo(); - }} - onPointerUp={e => { - e.stopPropagation(); - }} - color={activeColor} - tooltip="Undo" - icon={} - /> - { - e.stopPropagation(); - handleRedo(); - }} - onPointerUp={e => { - e.stopPropagation(); - }} - color={activeColor} - tooltip="Redo" - icon={} - /> -
e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}> - { - setCursorData(prev => ({ ...prev, width: val as number })); - }} - /> -
-
e.stopPropagation()} style={{ height: 225, width: '100%', display: 'flex', justifyContent: 'center', cursor: 'pointer' }}> - { - setCursorData(prev => ({ ...prev, width: val as number })); - }} - /> -
-
- {/* Edits thumbnails */} -
- {edits.map((edit, i) => ( - image edits { - const img = new Image(); - img.src = edit.url; - ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); - currImg.current = img; - parentDoc.current = edit.saveRes ?? null; - }} - /> - ))} - {/* Original img thumbnail */} - {edits.length > 0 && ( -
- - image stuff { - if (!originalImg.current) return; - const img = new Image(); - img.src = originalImg.current.src; - ImageUtility.drawImgToCanvas(img, canvasRef, canvasDims.width, canvasDims.height); - currImg.current = img; - parentDoc.current = originalDoc.current; - }} - /> -
- )} -
-
-
- setInput(e.target.value)} - disabled={isBrushing} - type="text" - label="Prompt" - placeholder="Prompt..." - InputLabelProps={{ style: { fontSize: '16px' } }} - inputProps={{ style: { fontSize: '16px' } }} - sx={{ - backgroundColor: '#ffffff', - position: 'absolute', - bottom: '16px', - transform: 'translateX(calc(50vw - 50%))', - width: 'calc(100vw - 64px)', - }} - /> -
-
- ); -}; - -export default GenerativeFill; -- cgit v1.2.3-70-g09d2