diff options
Diffstat (limited to 'src/client/views/collections/collectionFreeForm')
5 files changed, 485 insertions, 48 deletions
diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss new file mode 100644 index 000000000..819c72760 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.scss @@ -0,0 +1,85 @@ +.image-box-container { + display: flex; + align-items: center; + justify-content: center; + width: 100%; + height: 100%; + font-size: 10px; + line-height: 1; + background: none; + z-index: 1000; + padding: 0px; + overflow: auto; + cursor: default; +} + +.image-label-list { + display: flex; + flex-direction: column; + align-items: center; // Centers the content vertically in the flex container + width: 100%; + + > div { + display: flex; + justify-content: space-between; // Puts the content and delete button on opposite ends + align-items: center; + width: 100%; + margin-top: 8px; // Adds space between label rows + background-color: black; + + p { + text-align: center; // Centers the text of the paragraph + font-size: large; + vertical-align: middle; + margin-left: 10px; + } + + .IconButton { + // Styling for the delete button + margin-left: auto; // Pushes the button to the far right + } + } +} + +.image-information-list { + display: flex; + flex-direction: column; + align-items: center; + width: 100%; + margin-top: 10px; +} + +.image-information { + border: 1px solid; + width: 100%; + display: inline-flex; + flex-direction: column; + justify-content: center; + align-items: center; + overflow: hidden; + padding: 2px; + overflow-x: auto; + overflow-y: auto; + + img { + max-width: 200px; + max-height: 200px; + width: auto; + height: auto; + } +} + +.image-information-labels { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + + .image-label { + margin-top: 5px; + margin-bottom: 5px; + padding: 3px; + border-radius: 2px; + border: solid 1px; + } +} diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx new file mode 100644 index 000000000..fec4d3e12 --- /dev/null +++ b/src/client/views/collections/collectionFreeForm/ImageLabelBox.tsx @@ -0,0 +1,342 @@ +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { Colors, IconButton } from 'browndash-components'; +import { action, computed, makeObservable, observable, reaction } from 'mobx'; +import { observer } from 'mobx-react'; +import React from 'react'; +import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; +import { Docs } from '../../../documents/Documents'; +import { DocumentType } from '../../../documents/DocumentTypes'; +import { ViewBoxBaseComponent } from '../../DocComponent'; +import { FieldView, FieldViewProps } from '../../nodes/FieldView'; +import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; +import './ImageLabelBox.scss'; +import { MainView } from '../../MainView'; +import 'ldrs/ring'; +import { ring } from 'ldrs'; +import { SnappingManager } from '../../../util/SnappingManager'; +import { ImageCast } from '../../../../fields/Types'; +import { DocData } from '../../../../fields/DocSymbols'; +import { SettingsManager } from '../../../util/SettingsManager'; +import { CollectionCardView } from '../CollectionCardDeckView'; +import { gptGetEmbedding, gptImageLabel } from '../../../apis/gpt/GPT'; +import { numberRange, Utils } from '../../../../Utils'; +import { List } from '../../../../fields/List'; +import { DragManager } from '../../../util/DragManager'; +import { OpenWhere } from '../../nodes/OpenWhere'; +import similarity from 'compute-cosine-similarity'; +import { DocumentView } from '../../nodes/DocumentView'; + +export class ImageInformationItem {} + +export class ImageLabelBoxData { + static _instance: ImageLabelBoxData; + @observable _docs: Doc[] = []; + @observable _labelGroups: string[] = []; + + constructor() { + makeObservable(this); + ImageLabelBoxData._instance = this; + } + public static get Instance() { + return ImageLabelBoxData._instance ?? new ImageLabelBoxData(); + } + + @action + public setData = (docs: Doc[]) => { + this._docs = docs; + }; + + @action + addLabel = (label: string) => { + label = label.toUpperCase().trim(); + if (label.length > 0) { + if (!this._labelGroups.includes(label)) { + this._labelGroups = [...this._labelGroups, label]; + } + } + }; + + @action + removeLabel = (label: string) => { + const labelUp = label.toUpperCase(); + this._labelGroups = this._labelGroups.filter(group => group !== labelUp); + }; +} + +@observer +export class ImageLabelBox extends ViewBoxBaseComponent<FieldViewProps>() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(ImageLabelBox, fieldKey); + } + + private _dropDisposer?: DragManager.DragDropDisposer; + public static Instance: ImageLabelBox; + private _inputRef = React.createRef<HTMLInputElement>(); + @observable _loading: boolean = false; + private _currentLabel: string = ''; + + protected createDropTarget = (ele: HTMLDivElement) => { + this._dropDisposer?.(); + ele && (this._dropDisposer = DragManager.MakeDropTarget(ele, this.onInternalDrop.bind(this), this.layoutDoc)); + }; + + protected onInternalDrop(e: Event, de: DragManager.DropEvent): boolean { + const { docDragData } = de.complete; + if (docDragData) { + ImageLabelBoxData.Instance.setData(ImageLabelBoxData.Instance._docs.concat(docDragData.droppedDocuments)); + return false; + } + return false; + } + + @computed get _labelGroups() { + return ImageLabelBoxData.Instance._labelGroups; + } + + @computed get _selectedImages() { + // return DocListCast(this.dataDoc.data); + return ImageLabelBoxData.Instance._docs; + } + @observable _displayImageInformation: boolean = false; + + constructor(props: any) { + super(props); + makeObservable(this); + ring.register(); + ImageLabelBox.Instance = this; + } + + // ImageLabelBox.Instance.setData() + /** + * This method is called when the SearchBox component is first mounted. When the user opens + * the search panel, the search input box is automatically selected. This allows the user to + * type in the search input box immediately, without needing clicking on it first. + */ + componentDidMount() { + this.classifyImagesInBox(); + reaction( + () => this._selectedImages, + () => this.classifyImagesInBox() + ); + } + + @action + groupImages = () => { + this.groupImagesInBox(); + }; + + @action + startLoading = () => { + this._loading = true; + }; + + @action + endLoading = () => { + this._loading = false; + }; + + @action + toggleDisplayInformation = () => { + this._displayImageInformation = !this._displayImageInformation; + if (this._displayImageInformation) { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = true)); + } else { + this._selectedImages.forEach(doc => (doc[DocData].showLabels = false)); + } + }; + + @action + submitLabel = () => { + const input = document.getElementById('new-label') as HTMLInputElement; + ImageLabelBoxData.Instance.addLabel(this._currentLabel); + this._currentLabel = ''; + input.value = ''; + }; + + onInputChange = action((e: React.ChangeEvent<HTMLInputElement>) => { + this._currentLabel = e.target.value; + }); + + classifyImagesInBox = async () => { + this.startLoading(); + + // Converts the images into a Base64 format, afterwhich the information is sent to GPT to label them. + + const imageInfos = this._selectedImages.map(async doc => { + if (!doc[DocData].data_labels) { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => + !hrefBase64 ? undefined : + gptImageLabel(hrefBase64).then(labels => + ({ doc, labels }))) ; // prettier-ignore + } + }); + + (await Promise.all(imageInfos)).forEach(imageInfo => { + if (imageInfo) { + imageInfo.doc[DocData].data_labels = new List<string>(); + + const labels = imageInfo.labels.split('\n'); + labels.forEach(label => { + label = label.replace(/^\d+\.\s*|-|\*/, '').trim(); + imageInfo.doc[DocData][`${label}`] = true; + (imageInfo.doc[DocData].data_labels as List<string>).push(label); + }); + } + }); + + this.endLoading(); + }; + + /** + * Groups images to most similar labels. + */ + groupImagesInBox = action(async () => { + this.startLoading(); + + for (const doc of this._selectedImages) { + for (let index = 0; index < (doc[DocData].data_labels as List<string>).length; index++) { + const label = (doc[DocData].data_labels as List<string>)[index]; + const embedding = await gptGetEmbedding(label); + doc[`data_labels_embedding_${index + 1}`] = new List<number>(embedding); + } + } + + const labelToEmbedding = new Map<string, number[]>(); + // Create embeddings for the labels. + await Promise.all(this._labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); + + // For each image, loop through the labels, and calculate similarity. Associate it with the + // most similar one. + this._selectedImages.forEach(doc => { + const embedLists = numberRange((doc[DocData].data_labels as List<string>).length).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); + const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map((l, index) => (embedding && similarity(Array.from(embedding), l)!) || 0)); + const {label: mostSimilarLabelCollect} = + this._labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) + .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, + { label: '', similarityScore: 0, }); // prettier-ignore + doc[DocData].data_label = mostSimilarLabelCollect; // The label most similar to the image's contents. + }); + + this.endLoading(); + + if (this._selectedImages) { + MarqueeOptionsMenu.Instance.groupImages(); + } + + MainView.Instance.closeFlyout(); + }); + + render() { + if (this._loading) { + return ( + <div className="image-box-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + <l-ring size="60" color="white" /> + </div> + ); + } + + if (this._selectedImages.length === 0) { + return ( + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele)}> + <p style={{ fontSize: 'large' }}>In order to classify and sort images, marquee select the desired images and press the 'Classify and Sort Images' button. Then, add the desired groups for the images to be put in.</p> + </div> + ); + } + + return ( + <div className="searchBox-container" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }} ref={ele => this.createDropTarget(ele)}> + <div className="searchBox-bar" style={{ pointerEvents: 'all', color: SnappingManager.userColor, background: SnappingManager.userBackgroundColor }}> + <IconButton + tooltip={'See image information'} + onPointerDown={this.toggleDisplayInformation} + icon={this._displayImageInformation ? <FontAwesomeIcon icon="caret-up" /> : <FontAwesomeIcon icon="caret-down" />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + <input + defaultValue="" + autoComplete="off" + onChange={this.onInputChange} + onKeyDown={e => { + e.key === 'Enter' ? this.submitLabel() : null; + e.stopPropagation(); + }} + type="text" + placeholder="Input groups for images to be put into..." + aria-label="label-input" + id="new-label" + className="searchBox-input" + style={{ width: '100%', borderRadius: '5px' }} + ref={this._inputRef} + /> + <IconButton + tooltip={'Add a label'} + onPointerDown={() => { + const input = document.getElementById('new-label') as HTMLInputElement; + ImageLabelBoxData.Instance.addLabel(this._currentLabel); + this._currentLabel = ''; + input.value = ''; + }} + icon={<FontAwesomeIcon icon="plus" />} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '19px' }} + /> + {this._labelGroups.length > 0 ? <IconButton tooltip={'Group Images'} onPointerDown={this.groupImages} icon={<FontAwesomeIcon icon="object-group" />} color={Colors.MEDIUM_BLUE} style={{ width: '19px' }} /> : <div></div>} + </div> + <div> + <div className="image-label-list"> + {this._labelGroups.map(group => { + return ( + <div key={Utils.GenerateGuid()}> + <p style={{ color: MarqueeOptionsMenu.Instance.userColor }}>{group}</p> + <IconButton + tooltip={'Remove Label'} + onPointerDown={() => { + ImageLabelBoxData.Instance.removeLabel(group); + }} + icon={'x'} + color={MarqueeOptionsMenu.Instance.userColor} + style={{ width: '8px' }} + /> + </div> + ); + })} + </div> + </div> + {this._displayImageInformation ? ( + <div className="image-information-list"> + {this._selectedImages.map(doc => { + const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); + return ( + <div className="image-information" style={{ borderColor: SettingsManager.userColor }} key={Utils.GenerateGuid()}> + <img + src={`${name}_o.${type}`} + onClick={async () => { + await DocumentView.showDocument(doc, { willZoomCentered: true }); + }}></img> + <div className="image-information-labels" onClick={() => this._props.addDocTab(doc, OpenWhere.addRightKeyvalue)}> + {(doc[DocData].data_labels as List<string>).map(label => { + return ( + <div key={Utils.GenerateGuid()} className="image-label" style={{ backgroundColor: SettingsManager.userVariantColor, borderColor: SettingsManager.userColor }}> + {label} + </div> + ); + })} + </div> + </div> + ); + })} + </div> + ) : ( + <div></div> + )} + </div> + ); + } +} + +Docs.Prototypes.TemplateMap.set(DocumentType.IMAGEGROUPER, { + layout: { view: ImageLabelBox, dataField: 'data' }, + options: { acl: '', _width: 400 }, +}); diff --git a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx index 7f27c6b5c..73befb205 100644 --- a/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx +++ b/src/client/views/collections/collectionFreeForm/ImageLabelHandler.tsx @@ -77,7 +77,7 @@ export class ImageLabelHandler extends ObservableReactComponent<{}> { }}> <div> <IconButton tooltip={'Cancel'} onPointerDown={this.hideLabelhandler} icon={<FontAwesomeIcon icon="eye-slash" />} color={MarqueeOptionsMenu.Instance.userColor} style={{ width: '19px' }} /> - <input aria-label="label-input" id="new-label" type="text" style={{ color: 'black' }} /> + <input aria-label="label-input" id="new-label" type="text" placeholder="Input a classification" style={{ color: 'black' }} /> <IconButton tooltip={'Add Label'} onPointerDown={() => { diff --git a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx index f02cd9d45..b94a22d04 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeOptionsMenu.tsx @@ -39,7 +39,7 @@ export class MarqueeOptionsMenu extends AntimodeMenu<AntimodeMenuProps> { <IconButton tooltip="Summarize Documents" onPointerDown={this.summarize} icon={<FontAwesomeIcon icon="compress-arrows-alt" />} color={this.userColor} /> <IconButton tooltip="Delete Documents" onPointerDown={this.delete} icon={<FontAwesomeIcon icon="trash-alt" />} color={this.userColor} /> <IconButton tooltip="Pin selected region" onPointerDown={this.pinWithView} icon={<FontAwesomeIcon icon="map-pin" />} color={this.userColor} /> - <IconButton tooltip="Classify Images" onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> + <IconButton tooltip="Classify and Sort Images" onPointerDown={this.classifyImages} icon={<FontAwesomeIcon icon="object-group" />} color={this.userColor} /> </> ); return this.getElement(buttons); diff --git a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx index dc15c83c5..07e3acb1d 100644 --- a/src/client/views/collections/collectionFreeForm/MarqueeView.tsx +++ b/src/client/views/collections/collectionFreeForm/MarqueeView.tsx @@ -5,7 +5,7 @@ import { observer } from 'mobx-react'; import * as React from 'react'; import { ClientUtils, lightOrDark, returnFalse } from '../../../../ClientUtils'; import { intersectRect, numberRange } from '../../../../Utils'; -import { Doc, NumListCast, Opt } from '../../../../fields/Doc'; +import { Doc, DocListCast, NumListCast, Opt } from '../../../../fields/Doc'; import { AclAdmin, AclAugment, AclEdit, DocData } from '../../../../fields/DocSymbols'; import { Id } from '../../../../fields/FieldSymbols'; import { InkData, InkField, InkTool } from '../../../../fields/InkField'; @@ -36,6 +36,9 @@ import { CollectionFreeFormView } from './CollectionFreeFormView'; import { ImageLabelHandler } from './ImageLabelHandler'; import { MarqueeOptionsMenu } from './MarqueeOptionsMenu'; import './MarqueeView.scss'; +import { MainView } from '../../MainView'; +import { ImageLabelBox, ImageLabelBoxData } from './ImageLabelBox'; +import { SearchBox } from '../../search/SearchBox'; interface MarqueeViewProps { getContainerTransform: () => Transform; @@ -53,6 +56,9 @@ interface MarqueeViewProps { slowLoadDocuments: (files: File[] | string, options: DocumentOptions, generatedDocuments: Doc[], text: string, completed: ((doc: Doc[]) => void) | undefined, addDocument: (doc: Doc | Doc[]) => boolean) => Promise<void>; } +/** + * A component that deals with the marquee select in the freeform canvas. + */ @observer export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps & MarqueeViewProps> { public static CurViewBounds(pinDoc: Doc, panelWidth: number, panelHeight: number) { @@ -60,9 +66,12 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps return { left: NumCast(pinDoc._freeform_panX) - panelWidth / 2 / ps, top: NumCast(pinDoc._freeform_panY) - panelHeight / 2 / ps, width: panelWidth / ps, height: panelHeight / ps }; } + static Instance: MarqueeView; + constructor(props: any) { super(props); makeObservable(this); + MarqueeView.Instance = this; } private _commandExecuted = false; @@ -430,32 +439,14 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps /** * Classifies images and assigns the labels as document fields. - * TODO: Turn into lists of labels instead of individual fields. */ @undoBatch classifyImages = action(async (e: React.MouseEvent | undefined) => { - this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); - - const imageInfos = this._selectedDocs.map(async doc => { - const [name, type] = ImageCast(doc[Doc.LayoutFieldKey(doc)]).url.href.split('.'); - return CollectionCardView.imageUrlToBase64(`${name}_o.${type}`).then(hrefBase64 => - !hrefBase64 ? undefined : - gptImageLabel(hrefBase64).then(labels => - Promise.all(labels.split('\n').map(label => gptGetEmbedding(label))).then(embeddings => - ({ doc, embeddings, labels }))) ); // prettier-ignore - }); - - (await Promise.all(imageInfos)).forEach(imageInfo => { - if (imageInfo && Array.isArray(imageInfo.embeddings)) { - imageInfo.doc[DocData].data_labels = imageInfo.labels; - numberRange(3).forEach(n => { - imageInfo.doc[`data_labels_embedding_${n + 1}`] = new List<number>(imageInfo.embeddings[n]); - }); - } - }); - - if (e) { - ImageLabelHandler.Instance.displayLabelHandler(e.pageX, e.pageY); + const groupButton = DocListCast(Doc.MyLeftSidebarMenu.data).find(d => d.target === Doc.MyImageGrouper); + if (groupButton) { + this._selectedDocs = this.marqueeSelect(false, DocumentType.IMG); + ImageLabelBoxData.Instance.setData(this._selectedDocs); + MainView.Instance.expandFlyout(groupButton); } }); @@ -464,28 +455,47 @@ export class MarqueeView extends ObservableReactComponent<SubCollectionViewProps */ @undoBatch groupImages = action(async () => { - const labelGroups = ImageLabelHandler.Instance._labelGroups; - const labelToEmbedding = new Map<string, number[]>(); - // Create embeddings for the labels. - await Promise.all(labelGroups.map(async label => gptGetEmbedding(label).then(labelEmbedding => labelToEmbedding.set(label, labelEmbedding)))); - - // For each image, loop through the labels, and calculate similarity. Associate it with the - // most similar one. - this._selectedDocs.forEach(doc => { - const embedLists = numberRange(3).map(n => Array.from(NumListCast(doc[`data_labels_embedding_${n + 1}`]))); - const bestEmbedScore = (embedding: Opt<number[]>) => Math.max(...embedLists.map(l => (embedding && similarity(Array.from(embedding), l)) || 0)); - const {label: mostSimilarLabelCollect} = - labelGroups.map(label => ({ label, similarityScore: bestEmbedScore(labelToEmbedding.get(label)) })) - .reduce((prev, cur) => cur.similarityScore < 0.3 || cur.similarityScore <= prev.similarityScore ? prev: cur, - { label: '', similarityScore: 0, }); // prettier-ignore - - numberRange(3).forEach(n => { - doc[`data_labels_embedding_${n + 1}`] = undefined; - }); - doc[DocData].data_label = mostSimilarLabelCollect; - }); - this._props.Document._type_collection = CollectionViewType.Time; - this._props.Document.pivotField = 'data_label'; + const labelGroups: string[] = ImageLabelBoxData.Instance._labelGroups; + const labelToCollection: Map<string, Doc> = new Map(); + const selectedImages = ImageLabelBoxData.Instance._docs; + + // Create new collections associated with each label and get the embeddings for the labels. + let x_offset = 0; + let y_offset = 0; + let row_count = 0; + for (const label of labelGroups) { + const newCollection = this.getCollection([], undefined, false); + newCollection._width = 900; + newCollection._height = 900; + newCollection._x = this.Bounds.left; + newCollection._y = this.Bounds.top; + newCollection._freeform_panX = this.Bounds.left + this.Bounds.width / 2; + newCollection._freeform_panY = this.Bounds.top + this.Bounds.height / 2; + newCollection._x = (newCollection._x as number) + x_offset; + newCollection._y = (newCollection._y as number) + y_offset; + x_offset += (newCollection._width as number) + 40; + row_count += 1; + if (row_count == 3) { + y_offset += (newCollection._height as number) + 40; + x_offset = 0; + row_count = 0; + } + labelToCollection.set(label, newCollection); + this._props.addDocument?.(newCollection); + console.log('added collection!'); + } + + console.log(labelToCollection); + + for (const doc of selectedImages) { + if (doc[DocData].data_label) { + Doc.AddDocToList(labelToCollection.get(doc[DocData].data_label as string)!, undefined, doc); + this._props.removeDocument?.(doc); + } + } + + //this._props.Document._type_collection = CollectionViewType.Time; // Change the collection view to a Time view. + //this._props.Document.pivotField = 'data_label'; // Sets the pivot to be the 'data_label'. }); @undoBatch |