/** * @file ChatBox.tsx * @description This file defines the ChatBox component, which manages user interactions with * an AI assistant. It handles document uploads, chat history, message input, and integration * with the OpenAI API. The ChatBox is MobX-observable and tracks the progress of tasks such as * document analysis and AI-driven summaries. It also maintains real-time chat functionality * with support for follow-up questions and citation management. */ import dotenv from 'dotenv'; import { ObservableSet, action, computed, makeObservable, observable, observe, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { v4 as uuidv4 } from 'uuid'; import { ClientUtils, OmitKeys } from '../../../../../ClientUtils'; import { Doc, DocListCast, Opt } from '../../../../../fields/Doc'; import { DocData, DocViews } from '../../../../../fields/DocSymbols'; import { Id } from '../../../../../fields/FieldSymbols'; import { RichTextField } from '../../../../../fields/RichTextField'; import { ScriptField } from '../../../../../fields/ScriptField'; import { CsvCast, DocCast, NumCast, PDFCast, RTFCast, StrCast, VideoCast, AudioCast } from '../../../../../fields/Types'; import { DocUtils } from '../../../../documents/DocUtils'; import { CollectionViewType, DocumentType } from '../../../../documents/DocumentTypes'; import { Docs, DocumentOptions } from '../../../../documents/Documents'; import { DocServer } from '../../../../DocServer'; import { DocumentManager } from '../../../../util/DocumentManager'; import { ImageUtils } from '../../../../util/Import & Export/ImageUtils'; import { LinkManager } from '../../../../util/LinkManager'; import { CompileError, CompileScript } from '../../../../util/Scripting'; import { DictationButton } from '../../../DictationButton'; import { ViewBoxAnnotatableComponent } from '../../../DocComponent'; import { AudioBox } from '../../AudioBox'; import { DocumentView, DocumentViewInternal } from '../../DocumentView'; import { FieldView, FieldViewProps } from '../../FieldView'; import { PDFBox } from '../../PDFBox'; import { ScriptingBox } from '../../ScriptingBox'; import { VideoBox } from '../../VideoBox'; import { Agent } from '../agentsystem/Agent'; import { supportedDocTypes } from '../types/tool_types'; import { ASSISTANT_ROLE, AssistantMessage, CHUNK_TYPE, Citation, ProcessingInfo, SimplifiedChunk, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { OpenWhere } from '../../OpenWhere'; import { Upload } from '../../../../../server/SharedMediaTypes'; import { AgentDocumentManager } from '../utils/AgentDocumentManager'; import { AiOutlineSend } from 'react-icons/ai'; import { SnappingManager } from '../../../../util/SnappingManager'; import { Button, Size, Type } from '@dash/components'; import { MdLink, MdViewModule } from 'react-icons/md'; import { Tooltip } from '@mui/material'; dotenv.config(); export type parsedDocData = { doc_type: string; data: unknown; _disable_resource_loading?: boolean; _sandbox_iframe?: boolean; _iframe_sandbox?: string; data_useCors?: boolean; }; export type parsedDoc = DocumentOptions & parsedDocData; /** * ChatBox is the main class responsible for managing the interaction between the user and the assistant, * handling documents, and integrating with OpenAI for tasks such as document analysis, chat functionality, * and vector store interactions. */ @observer export class ChatBox extends ViewBoxAnnotatableComponent() { // MobX observable properties to track UI state and data @observable private _history: AssistantMessage[] = []; @observable.deep private _current_message: AssistantMessage | undefined = undefined; @observable private _isLoading: boolean = false; @observable private _uploadProgress: number = 0; @observable private _currentStep: string = ''; @observable private _expandedScratchpadIndex: number | null = null; @observable private _inputValue: string = ''; @observable private _linked_docs_to_add: ObservableSet = observable.set(); @observable private _linked_csv_files: { filename: string; id: string; text: string }[] = []; @observable private _isUploadingDocs: boolean = false; @observable private _citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; @observable private _isFontSizeModalOpen: boolean = false; @observable private _fontSize: 'small' | 'normal' | 'large' | 'xlarge' = 'normal'; @observable private _toolReloadModal: { visible: boolean; toolName: string } = { visible: false, toolName: '' }; // Private properties for managing OpenAI API, vector store, agent, and UI elements private openai!: OpenAI; // Using definite assignment assertion private vectorstore_id: string | undefined; private vectorstore: Vectorstore | undefined; private agent: Agent | undefined; private messagesRef: React.RefObject = React.createRef(); private _textInputRef: HTMLInputElement | undefined | null; private docManager: AgentDocumentManager | undefined; /** * Static method that returns the layout string for the field. * @param fieldKey Key to get the layout string. */ public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ChatBox, fieldKey); } setChatInput = action((input: string) => { this._inputValue = input; }); @action toggleCanvasMode = () => { const newMode = !this.docManager?.getCanvasMode(); this.docManager?.setCanvasMode(newMode); }; /** * Constructor initializes the component, sets up OpenAI, vector store, and agent instances, * and observes changes in the chat history to save the state in dataDoc. * @param props The properties passed to the component. */ constructor(props: FieldViewProps) { super(props); makeObservable(this); } /** * Adds a document to the vectorstore for AI-based analysis. * Handles the upload progress and errors during the process. * @param newLinkedDoc The new document to add. */ @action addDocToVectorstore = async (newLinkedDoc: Doc) => { try { const isAudioOrVideo = VideoCast(newLinkedDoc.data)?.url?.pathname || AudioCast(newLinkedDoc.data)?.url?.pathname; // Set UI state to show the processing overlay runInAction(() => { this._isUploadingDocs = true; this._uploadProgress = 0; this._currentStep = isAudioOrVideo ? 'Preparing media file...' : 'Processing document...'; }); if (!this.docManager || !this.vectorstore) throw new Error('Document manager or vectorstore is not initialized'); // Process the document first to ensure it has a valid ID await this.docManager.processDocument(newLinkedDoc); // Add the document to the vectorstore which will also register chunks await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); // Give a slight delay to show the completion message if (this._uploadProgress === 100) { await new Promise(resolve => setTimeout(resolve, 1000)); } // Reset UI state runInAction(() => { this._isUploadingDocs = false; this._uploadProgress = 0; this._currentStep = ''; }); return true; } catch (err) { console.error('Error adding document to vectorstore:', err); // Show error in UI runInAction(() => { this._currentStep = `Error: ${err instanceof Error ? err.message : 'Failed to process document'}`; }); await new Promise(resolve => setTimeout(resolve, 2000)); // Reset UI state runInAction(() => { this._isUploadingDocs = false; this._uploadProgress = 0; this._currentStep = ''; }); return false; } }; /** * Updates the upload progress and the current step in the UI. * @param progress The percentage of the progress. * @param step The current step name. */ @action updateProgress = (progress: number, step: string) => { // Ensure progress is within expected bounds const validProgress = Math.min(Math.max(0, progress), 100); this._uploadProgress = validProgress; this._currentStep = step; // Force UI update if (process.env.NODE_ENV !== 'production') { console.log(`Progress: ${validProgress}%, Step: ${step}`); } }; //TODO: Update for new chunk_simpl on agentDocument /** * Adds a CSV file for analysis by sending it to OpenAI and generating a summary. * @param newLinkedDoc The linked document representing the CSV file. * @param id Optional ID for the document. */ @action addCSVForAnalysis = async (newLinkedDoc: Doc, id?: string) => { if (!newLinkedDoc.chunk_simpl) { // Convert document text to CSV data const csvData: string = StrCast(newLinkedDoc.text); // Generate a summary using OpenAI API const completion = await this.openai.chat.completions.create({ messages: [ { role: 'system', content: 'You are an AI assistant tasked with summarizing the content of a CSV file. You will be provided with the data from the CSV file and your goal is to generate a concise summary that captures the main themes, trends, and key points represented in the data.', }, { role: 'user', content: `Please provide a comprehensive summary of the CSV file based on the provided data. Ensure the summary highlights the most important information, patterns, and insights. Your response should be in paragraph form and be concise. CSV Data: ${csvData} ********** Summary:`, }, ], model: 'gpt-3.5-turbo', }); const csvId = id ?? uuidv4(); // Add CSV details to linked files this._linked_csv_files.push({ filename: CsvCast(newLinkedDoc.data)?.url.pathname ?? '', id: csvId, text: csvData, }); // Add a chunk for the CSV and assign the summary const chunkToAdd = { chunkId: csvId, chunkType: CHUNK_TYPE.CSV, }; newLinkedDoc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); newLinkedDoc.summary = completion.choices[0].message.content!; } }; /** * Toggles the tool logs, expanding or collapsing the scratchpad at the given index. * @param index Index of the tool log to toggle. */ @action toggleToolLogs = (index: number) => { this._expandedScratchpadIndex = this._expandedScratchpadIndex === index ? null : index; }; /** * Initializes the OpenAI API client using the API key from environment variables. * @returns OpenAI client instance. */ initializeOpenAI() { const configuration: ClientOptions = { apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true, }; this.openai = new OpenAI(configuration); } /** * Adds a scroll event listener to detect user scrolling and handle passive wheel events. */ addScrollListener = () => { if (this.messagesRef?.current) { this.messagesRef.current.addEventListener('wheel', this.onPassiveWheel, { passive: false }); } }; /** * Removes the scroll event listener from the chat messages container. */ removeScrollListener = () => { if (this.messagesRef?.current) { this.messagesRef.current.removeEventListener('wheel', this.onPassiveWheel); } }; /** * Scrolls the chat messages container to the bottom, ensuring the latest message is visible. */ scrollToBottom = () => { // if (this.messagesRef.current) { // this.messagesRef.current.scrollTop = this.messagesRef.current.scrollHeight; // } }; /** * Event handler for detecting wheel scrolling and stopping the event propagation. * @param e The wheel event. */ onPassiveWheel = (e: WheelEvent) => { if (this._props.isContentActive()) { e.stopPropagation(); } }; /** * Sends the user's input to OpenAI, displays the loading indicator, and updates the chat history. * @param event The form submission event. */ @action askGPT = async (event: React.FormEvent): Promise => { event.preventDefault(); if (!this._textInputRef) { console.log('ERROR: text input ref is undefined'); return; } this._inputValue = ''; // Extract the user's message const textInput = this._textInputRef?.value ?? ''; const trimmedText = textInput.trim(); if (trimmedText) { this._textInputRef.value = ''; // Clear the input field try { // Add the user's message to the history this._history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], processing_info: [], }); this._isLoading = true; this._current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], processing_info: [], }; // Define callbacks for real-time processing updates const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { runInAction(() => { if (this._current_message) { this._current_message = { ...this._current_message, processing_info: processingUpdate, }; } }); this.scrollToBottom(); }; const onAnswerUpdate = (answerUpdate: string) => { runInAction(() => { if (this._current_message) { this._current_message = { ...this._current_message, content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }], }; } }); }; if (!this.agent) throw new Error('Agent is not initialized'); // Send the user's question to the assistant and get the final message const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); // Update the history with the final assistant message runInAction(() => { if (this._current_message) { this._history.push({ ...finalMessage }); this._current_message = undefined; this.dataDoc.data = JSON.stringify(this._history); } }); } catch (err) { console.error('Error:', err); // Handle error in processing runInAction(() => this._history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.ERROR, text: `Sorry, I encountered an error while processing your request: ${err} `, citation_ids: null }], processing_info: [], }) ); } finally { runInAction(() => { this._isLoading = false; }); this.scrollToBottom(); } } this.scrollToBottom(); }; /** * Updates the citations for a given message in the chat history. * @param index The index of the message in the history. * @param citations The list of citations to add to the message. */ @action updateMessageCitations = (index: number, citations: Citation[]) => { if (this._history[index]) { this._history[index].citations = citations; } }; /** * Getter to retrieve the current user's name from the client utils. */ @computed get userName() { return ClientUtils.CurrentUserEmail; } /** * Creates a CSV document in the dashboard and adds it for analysis. * @param url The URL of the CSV. * @param title The title of the CSV document. * @param id The unique ID for the document. * @param data The CSV data content. */ @action createCSVInDash = (url: string, title: string, id: string, data: string) => DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) }).then(doc => { if (doc) { LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); this._props.addDocument?.(doc); DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => this.addCSVForAnalysis(doc, id)); } }); @action createImageInDash = async (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => { const newImgSrc = result.accessPaths.agnostic.client.indexOf('dashblobstore') === -1 // ? ClientUtils.prepend(result.accessPaths.agnostic.client) : result.accessPaths.agnostic.client; const doc = Docs.Create.ImageDocument(newImgSrc, options); this.addDocument(ImageUtils.AssignImgInfo(doc, result)); const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); if (doc) { if (this._props.addDocument) this._props.addDocument(doc); else DocumentViewInternal.addDocTabFunc(doc, OpenWhere.addRight); } await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); }; /** * Creates a text document in the dashboard and adds it for analysis. * @param title The title of the doc. * @param text_content The text of the document. * @param options Other optional document options (e.g. color) * @param id The unique ID for the document. */ @action private createCollectionWithChildren = (data: parsedDoc[], insideCol: boolean): Opt[] => data.map(doc => this.whichDoc(doc, insideCol)); @action public whichDoc = (doc: parsedDoc, insideCol: boolean): Opt => { const options = OmitKeys(doc, ['doc_type', 'data']).omit as DocumentOptions; const data = (doc as parsedDocData).data; const ndoc = (() => { switch (doc.doc_type) { default: case supportedDocTypes.note: return Docs.Create.TextDocument(data as string, options); case supportedDocTypes.comparison: return this.createComparison(JSON.parse(data as string) as parsedDoc[], options); case supportedDocTypes.flashcard: return this.createFlashcard(JSON.parse(data as string) as parsedDoc[], options); case supportedDocTypes.deck: return this.createDeck(JSON.parse(data as string) as parsedDoc[], options); case supportedDocTypes.image: return Docs.Create.ImageDocument(data as string, options); case supportedDocTypes.equation: return Docs.Create.EquationDocument(data as string, options); case supportedDocTypes.notetaking: return Docs.Create.NoteTakingDocument([], options); case supportedDocTypes.web: { // Create web document with enhanced safety options const webOptions = { ...options, data_useCors: true }; // If iframe_sandbox was passed from AgentDocumentManager, add it to the options if ('_iframe_sandbox' in options) { webOptions._iframe_sandbox = options._iframe_sandbox; } return Docs.Create.WebDocument(data as string, webOptions); } case supportedDocTypes.dataviz: case supportedDocTypes.table: return Docs.Create.DataVizDocument('/Users/ajshul/Dash-Web/src/server/public/files/csv/0d237e7c-98c9-44d0-aa61-5285fdbcf96c-random_sample.csv.csv', options); case supportedDocTypes.pdf: return Docs.Create.PdfDocument(data as string, options); case supportedDocTypes.video: return Docs.Create.VideoDocument(data as string, options); case supportedDocTypes.diagram: return Docs.Create.DiagramDocument(undefined, { text: data as unknown as RichTextField, ...options}); // text: can take a string or RichTextField but it's typed for RichTextField. // case supportedDocumentTypes.dataviz: // { // const { fileUrl, id } = await Networking.PostToServer('/createCSV', { // filename: (options.title as string).replace(/\s+/g, '') + '.csv', // data: data, // }); // const doc = Docs.Create.DataVizDocument(fileUrl, { ...options, text: RTFCast(data as string) }); // this.addCSVForAnalysis(doc, id); // return doc; // } case supportedDocTypes.script: { const result = !(data as string).trim() ? ({ compiled: false, errors: [] } as CompileError) : CompileScript(data as string, {}); const script_field = result.compiled ? new ScriptField(result, undefined, data as string) : undefined; const sdoc = Docs.Create.ScriptingDocument(script_field, options); DocumentManager.Instance.showDocument(sdoc, { willZoomCentered: true }, () => { const firstView = Array.from(sdoc[DocViews])[0] as DocumentView; (firstView.ComponentView as ScriptingBox)?.onApply?.(); (firstView.ComponentView as ScriptingBox)?.onRun?.(); }); return sdoc; } case supportedDocTypes.collection: { const arr = this.createCollectionWithChildren(JSON.parse(data as string) as parsedDoc[], true).filter(d=>d).map(d => d!); const collOpts = { _width:300, _height: 300, _layout_fitWidth: true, _freeform_backgroundGrid: true, ...options, }; return (() => { switch (options.type_collection) { case CollectionViewType.Tree: return Docs.Create.TreeDocument(arr, collOpts); case CollectionViewType.Stacking: return Docs.Create.StackingDocument(arr, collOpts); case CollectionViewType.Masonry: return Docs.Create.MasonryDocument(arr, collOpts); case CollectionViewType.Card: return Docs.Create.CardDeckDocument(arr, collOpts); case CollectionViewType.Carousel: return Docs.Create.CarouselDocument(arr, collOpts); case CollectionViewType.Carousel3D: return Docs.Create.Carousel3DDocument(arr, collOpts); case CollectionViewType.Multicolumn: return Docs.Create.CarouselDocument(arr, collOpts); default: return Docs.Create.FreeformDocument(arr, collOpts); } })(); } // case supportedDocumentTypes.map: return Docs.Create.MapDocument([], options); // case supportedDocumentTypes.button: return Docs.Create.ButtonDocument(options); // case supportedDocumentTypes.trail: return Docs.Create.PresDocument(options); } // prettier-ignore })(); if (ndoc) { ndoc.x = NumCast((options.x as number) ?? 0) + (insideCol ? 0 : NumCast(this.layoutDoc.x) + NumCast(this.layoutDoc.width)) + 100; ndoc.y = NumCast(options.y as number) + (insideCol ? 0 : NumCast(this.layoutDoc.y)); } return ndoc; }; /** * Creates a deck of flashcards. * * @param {any} data - The data used to generate the flashcards. Can be a string or an object. * @param {DocumentOptions} options - Configuration options for the flashcard deck. * @returns {Doc} A carousel document containing the flashcard deck. */ @action createDeck = (data: parsedDoc[], options: DocumentOptions) => { const flashcardDeck: Doc[] = []; // Process each flashcard document in the `deckData` array if (data.length == 2 && data[0].doc_type == 'text' && data[1].doc_type == 'text') { this.createFlashcard(data, options); } else { data.forEach(doc => { const flashcardDoc = this.createFlashcard((doc as parsedDocData).data as parsedDoc[] | string[], options); if (flashcardDoc) flashcardDeck.push(flashcardDoc); }); } // Create a carousel to contain the flashcard deck return Docs.Create.CarouselDocument(flashcardDeck, { title: options.title || 'Flashcard Deck', _width: options._width || 300, _height: options._height || 300, _layout_fitWidth: false, _layout_autoHeight: true, }); }; /** * Creates a single flashcard document. * * @param {any} data - The data used to generate the flashcard. Can be a string or an object. * @param {any} options - Configuration options for the flashcard. * @returns {Doc | undefined} The created flashcard document, or undefined if the flashcard cannot be created. */ @action createFlashcard = (data: parsedDoc[] | string[], options: DocumentOptions) => { const [front, back] = data; const sideOptions = { _height: 300, ...options }; // Create front and back text documents const side1 = typeof front === 'string' ? Docs.Create.CenteredTextCreator('question', front as string, sideOptions) : this.whichDoc(front, false); const side2 = typeof back === 'string' ? Docs.Create.CenteredTextCreator('answer', back as string, sideOptions) : this.whichDoc(back, false); // Create the flashcard document with both sides return Docs.Create.FlashcardDocument('flashcard', side1, side2, sideOptions); }; /** * Creates a comparison document. * * @param {any} doc - The document data containing left and right components for comparison. * @param {any} options - Configuration options for the comparison document. * @returns {Doc} The created comparison document. */ @action createComparison = (doc: parsedDoc[], options: DocumentOptions) => Docs.Create.ComparisonDocument(options.title as string, { data_back: this.whichDoc(doc[0], false), data_front: this.whichDoc(doc[1], false), _width: options._width, _height: options._height || 300, backgroundColor: options.backgroundColor, }); /** * Event handler to manage citations click in the message components. * @param citation The citation object clicked by the user. */ @action handleCitationClick = async (citation: Citation) => { try { // Extract values from MobX proxy object if needed const chunkId = typeof citation.chunk_id === 'object' ? (citation.chunk_id as any).toString() : citation.chunk_id; // For debugging console.log('Citation clicked:', { chunkId, citation: JSON.stringify(citation, null, 2), }); if (!this.docManager) throw new Error('Document manager is not initialized'); // Get the simplified chunk using the document manager const { foundChunk, doc, dataDoc } = this.docManager.getSimplifiedChunkById(chunkId); console.log('doc: ', doc); console.log('dataDoc: ', dataDoc); if (!foundChunk || !doc) { if (doc) { console.warn(`Chunk not found in document, ${doc.id}, for chunk ID: ${chunkId}`); DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); } else { console.warn(`Chunk not found for chunk ID: ${chunkId}`); } return; } console.log(`Found chunk in document:`, foundChunk); // Handle different chunk types if (foundChunk.chunkType === CHUNK_TYPE.AUDIO || foundChunk.chunkType === CHUNK_TYPE.VIDEO) { const directMatchSegmentStart = this.getDirectMatchingSegmentStart(doc, citation.direct_text || '', foundChunk.indexes || []); if (directMatchSegmentStart) { await this.goToMediaTimestamp(doc, directMatchSegmentStart, foundChunk.chunkType); } else { console.error('No direct matching segment found for the citation.'); } } else if (foundChunk.chunkType === CHUNK_TYPE.TABLE || foundChunk.chunkType === CHUNK_TYPE.IMAGE) { console.log('here: ', foundChunk); this.handleOtherChunkTypes(foundChunk as SimplifiedChunk, citation, doc); } else { if (doc.type === 'web') { DocumentManager.Instance.showDocument(doc, { openLocation: OpenWhere.addRight }, () => {}); return; } this.handleOtherChunkTypes(foundChunk, citation, doc, dataDoc); // Show the chunk text in citation popup const chunkText = citation.direct_text || 'Text content not available'; this.showCitationPopup(chunkText); // Also navigate to the document DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); } } catch (error) { console.error('Error handling citation click:', error); } }; /** * Finds a matching segment in a document based on text content. * @param doc The document to search in * @param citationText The text to find in the document * @param indexesOfSegments Optional indexes of segments to search in * @returns The starting timestamp of the matching segment, or -1 if not found */ getDirectMatchingSegmentStart = (doc: Doc, citationText: string, indexesOfSegments: string[]): number => { if (!doc || !citationText || !this.docManager) return -1; // Get original segments using document manager const original_segments = this.docManager.getOriginalSegments(doc); if (!original_segments || !Array.isArray(original_segments) || original_segments.length === 0) { return -1; } let segments = original_segments; // If specific indexes are provided, filter segments by those indexes if (indexesOfSegments && indexesOfSegments.length > 0) { segments = original_segments.filter(segment => indexesOfSegments.includes(segment.index)); } // If no segments match the indexes, use all segments if (segments.length === 0) { segments = original_segments; } // First try to find an exact match const exactMatch = segments.find(segment => segment.text && segment.text.includes(citationText)); if (exactMatch) { return exactMatch.start; } // If no exact match, find segment with best word overlap const calculateWordOverlap = (text1: string, text2: string): number => { if (!text1 || !text2) return 0; const words1 = text1.toLowerCase().split(/\s+/); const words2 = text2.toLowerCase().split(/\s+/); const wordSet1 = new Set(words1); let overlap = 0; for (const word of words2) { if (wordSet1.has(word)) { overlap++; } } // Return percentage of overlap relative to the shorter text return overlap / Math.min(words1.length, words2.length); }; // Find segment with highest word overlap let bestMatch = null; let highestOverlap = 0; for (const segment of segments) { if (!segment.text) continue; const overlap = calculateWordOverlap(segment.text, citationText); if (overlap > highestOverlap) { highestOverlap = overlap; bestMatch = segment; } } // Only return matches with significant overlap (more than 30%) if (bestMatch && highestOverlap > 0.3) { return bestMatch.start; } // If no good match found, return the start of the first segment as fallback return segments.length > 0 ? segments[0].start : -1; }; /** * Navigates to the given timestamp in the media player. * @param doc The document containing the media file. * @param timestamp The timestamp to navigate to. */ goToMediaTimestamp = async (doc: Doc, timestamp: number, type: 'video' | 'audio') => { try { // Show the media document in the viewer if (type == 'video') { DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { const firstView = Array.from(doc[DocViews])[0] as DocumentView; (firstView.ComponentView as VideoBox)?.Seek?.(timestamp); }); } else { DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { const firstView = Array.from(doc[DocViews])[0] as DocumentView; (firstView.ComponentView as AudioBox)?.playFrom?.(timestamp); }); } console.log(`Navigated to timestamp: ${timestamp}s in document ${doc.id}`); } catch (error) { console.error('Error navigating to media timestamp:', error); } }; /** * Handles non-media chunk types as before. * @param foundChunk The chunk object. * @param citation The citation object. * @param doc The document containing the chunk. */ handleOtherChunkTypes = (foundChunk: SimplifiedChunk, citation: Citation, doc: Doc, dataDoc?: Doc) => { switch (foundChunk.chunkType) { case CHUNK_TYPE.IMAGE: case CHUNK_TYPE.TABLE: { const values = foundChunk.location?.replace(/[[\]]/g, '').split(','); if (values?.length !== 4) { console.error('Location string must contain exactly 4 numbers'); return; } if (foundChunk.startPage === undefined || foundChunk.endPage === undefined) { DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); return; } const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); const annotationKey = '$' + Doc.LayoutDataKey(doc) + '_annotations'; const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); if (existingDoc) { existingDoc.x = x1; existingDoc.y = y1; existingDoc._width = x2 - x1; existingDoc._height = y2 - y1; } const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); //doc.layout_scroll = y1; doc._layout_curPage = foundChunk.startPage + 1; DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); } break; case CHUNK_TYPE.TEXT: { this._citationPopup = { text: citation.direct_text ?? 'No text available', visible: true }; this.startCitationPopupTimer(); // Check if the document is a PDF (has a PDF viewer component) const isPDF = PDFCast(dataDoc!.data) !== null || dataDoc!.type === DocumentType.PDF; // First ensure document is fully visible before trying to access its views this.ensureDocumentIsVisible(dataDoc!, isPDF, citation, foundChunk, doc); } break; case CHUNK_TYPE.CSV: case CHUNK_TYPE.URL: DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { console.log(`Showing web document in viewer with URL: ${foundChunk.url}`); }); break; default: console.error('Unhandled chunk type:', foundChunk.chunkType); break; } }; /** * Ensures a document is fully visible and rendered before performing actions on it * @param doc The document to ensure is visible * @param isPDF Whether this is a PDF document * @param citation The citation information * @param foundChunk The chunk information * @param doc The document to ensure is visible */ ensureDocumentIsVisible = (doc: Doc, isPDF: boolean, citation: Citation, foundChunk: SimplifiedChunk, layoutDoc?: Doc) => { try { // First, check if the document already has views and is rendered const hasViews = doc[DocViews] && doc[DocViews].size > 0; console.log(`Document ${doc.id}: Current state - hasViews: ${hasViews}, isPDF: ${isPDF}`); if (hasViews) { // Document is already rendered, proceed with accessing its view this.processPDFDocumentView(doc, isPDF, citation, foundChunk); return; } else if (layoutDoc) { this.processPDFDocumentView(layoutDoc, isPDF, citation, foundChunk); return; } // If document is not rendered yet, show it and wait for it to be ready console.log(`Document ${doc.id} needs to be shown first`); // Force document to be rendered by using willZoomCentered: true DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { // Wait a bit for the document to be fully rendered (longer than our previous attempts) setTimeout(() => { // Now manually check if document view exists and is valid this.verifyAndProcessDocumentView(doc, isPDF, citation, foundChunk, 1); }, 800); // Increased initial delay }); } catch (error) { console.error('Error ensuring document visibility:', error); // Show the document anyway as a fallback DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }); } }; /** * Verifies document view exists and processes it, with retries if needed */ verifyAndProcessDocumentView = (doc: Doc, isPDF: boolean, citation: Citation, foundChunk: SimplifiedChunk, attempt: number) => { // Diagnostic info console.log(`Verify attempt ${attempt}: Document views for ${doc.id}:`, doc[DocViews] ? `Found ${doc[DocViews].size} views` : 'No views'); // Double-check document exists in current document system const docExists = DocServer.GetCachedRefField(doc[Id]) !== undefined; if (!docExists) { console.warn(`Document ${doc.id} no longer exists in document system`); return; } try { if (!doc[DocViews] || doc[DocViews].size === 0) { if (attempt >= 5) { console.error(`Maximum verification attempts (${attempt}) reached for document ${doc.id}`); // Last resort: force re-creation of the document view if (isPDF) { console.log('Forcing document recreation as last resort'); DocumentManager.Instance.showDocument(doc, { willZoomCentered: true, }); } return; } // Let's try explicitly requesting the document be shown again if (attempt > 2) { console.log(`Attempt ${attempt}: Re-requesting document be shown`); DocumentManager.Instance.showDocument(doc, { willZoomCentered: true, openLocation: attempt % 2 === 0 ? OpenWhere.addRight : undefined, }); } // Use exponential backoff for retries const nextDelay = Math.min(2000, 500 * Math.pow(1.5, attempt)); console.log(`Scheduling retry ${attempt + 1} in ${nextDelay}ms`); setTimeout(() => { this.verifyAndProcessDocumentView(doc, isPDF, citation, foundChunk, attempt + 1); }, nextDelay); return; } this.processPDFDocumentView(doc, isPDF, citation, foundChunk); } catch (error) { console.error(`Error on verification attempt ${attempt}:`, error); if (attempt < 5) { setTimeout( () => { this.verifyAndProcessDocumentView(doc, isPDF, citation, foundChunk, attempt + 1); }, 500 * Math.pow(1.5, attempt) ); } } }; /** * Processes a PDF document view once we're sure it exists */ processPDFDocumentView = (doc: Doc, isPDF: boolean, citation: Citation, foundChunk: SimplifiedChunk) => { try { const views = Array.from(doc[DocViews] || []); if (!views.length) { console.warn('No document views found in document that should have views'); return; } const firstView = views[0] as DocumentView; if (!firstView) { console.warn('First view is invalid'); return; } console.log(`Successfully found document view for ${doc.id}:`, firstView.ComponentView ? `Component: ${firstView.ComponentView.constructor.name}` : 'No component view'); if (!firstView.ComponentView) { console.warn('Component view not available'); return; } // For PDF documents, perform fuzzy search if (isPDF && firstView.ComponentView && citation.direct_text) { const pdfComponent = firstView.ComponentView as PDFBox; this.ensureFuzzySearchAndExecute(pdfComponent, citation.direct_text.trim(), foundChunk.startPage); } } catch (error) { console.error('Error processing PDF document view:', error); } }; /** * Creates an annotation highlight on a PDF document for image citations. * @param x1 X-coordinate of the top-left corner of the highlight. * @param y1 Y-coordinate of the top-left corner of the highlight. * @param x2 X-coordinate of the bottom-right corner of the highlight. * @param y2 Y-coordinate of the bottom-right corner of the highlight. * @param citation The citation object to associate with the highlight. * @param annotationKey The key used to store the annotation. * @param pdfDoc The document where the highlight is created. * @returns The highlighted document. */ createImageCitationHighlight = (x1: number, y1: number, x2: number, y2: number, citation: Citation, annotationKey: string, pdfDoc: Doc): Doc => { const highlight_doc = Docs.Create.FreeformDocument([], { x: x1, y: y1, _width: x2 - x1, _height: y2 - y1, backgroundColor: 'rgba(255, 255, 0, 0.5)', }); highlight_doc[DocData].citation_id = citation.citation_id; highlight_doc.freeform_scale = 1; Doc.AddDocToList(pdfDoc[DocData], annotationKey, highlight_doc); highlight_doc.annotationOn = pdfDoc; Doc.SetContainer(highlight_doc, pdfDoc); return highlight_doc; }; /** * Lifecycle method that triggers when the component updates. * Ensures the chat is scrolled to the bottom when new messages are added. */ componentDidUpdate() { this.scrollToBottom(); } /** * Lifecycle method that triggers when the component mounts. * Initializes scroll listeners, sets up document reactions, and loads chat history from dataDoc if available. */ componentDidMount() { // At mount time, find the DocumentView whose .Document is the collection container. const parentView = this.DocumentView?.()._props.containerViewPath?.().lastElement(); if (!parentView) { console.warn('GPT ChatBox not inside a DocumentView – cannot perform operations on Documents.'); return; } this.docManager = new AgentDocumentManager(this, parentView); // Initialize OpenAI client this.initializeOpenAI(); // Create a unique vectorstore ID for this ChatBox this.vectorstore_id = uuidv4(); // Initialize vectorstore with the document manager this.vectorstore = new Vectorstore(this.vectorstore_id, this.docManager); /* reaction( () => ({ selDoc: DocumentView.Selected().lastElement(), visible: SnappingManager.ChatVisible }), ({ selDoc, visible }) => { const hasChildDocs = visible && selDoc?.ComponentView?.hasChildDocs; if (hasChildDocs) { this._textToDocMap.clear(); this.setCollectionContext(selDoc.Document); this.onGptResponse = (sortResult: string, questionType: GPTDocCommand) => this.processGptResponse(selDoc, this._textToDocMap, sortResult, questionType); this.onQuizRandom = () => this.randomlyChooseDoc(selDoc.Document, hasChildDocs()); this._documentDescriptions = Promise.all(hasChildDocs().map(doc => Doc.getDescription(doc).then(text => this._textToDocMap.set(text.replace(/\n/g, ' ').trim(), doc) && `${DescriptionSeperator}${text}${DescriptionSeperator}`) )).then(docDescriptions => docDescriptions.join()); // prettier-ignore } }, { fireImmediately: true } ); }*/ // Initialize font size from saved preference this.initFontSize(); // Create an agent with the vectorstore this.agent = new Agent(this.vectorstore, this.retrieveFormattedHistory.bind(this), this.retrieveCSVData.bind(this), this.createImageInDash.bind(this), this.createCSVInDash.bind(this), this.docManager); // Set up the tool created callback this.agent.setToolCreatedCallback(this.handleToolCreated); // Reaction to update dataDoc when chat history changes reaction( () => this._history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations, })), serializableHistory => { this.dataDoc.data = JSON.stringify(serializableHistory); } ); this._props.setContentViewBox?.(this); if (this.dataDoc.data) { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { this._history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations, })) ); }); } catch (e) { console.error('Failed to parse history from dataDoc:', e); } } else { // Default welcome message runInAction(() => { this._history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [ { index: 0, type: TEXT_TYPE.NORMAL, text: `Hey, ${this.userName()}! Welcome to Your Friendly Assistant. Link a document or ask questions to get started.`, citation_ids: null, }, ], processing_info: [], }); }); } // Set up reactions for linked documents reaction( () => { const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d); return linkedDocs; }, linked => linked.forEach(doc => this._linked_docs_to_add.add(doc)) ); // Observe changes to linked documents and handle document addition observe(this._linked_docs_to_add, change => { if (change.type === 'add') { if (CsvCast(change.newValue.data)) { this.addCSVForAnalysis(change.newValue); } else { this.addDocToVectorstore(change.newValue); } } else if (change.type === 'delete') { // Handle document removal } }); this.addScrollListener(); // Initialize the document manager by finding existing documents this.docManager.initializeFindDocsFreeform(); // If there are stored doc IDs in our list of docs to add, process them if (this._linked_docs_to_add.size > 0) { this._linked_docs_to_add.forEach(async doc => { await this.docManager!.processDocument(doc); }); } // Add event listeners this.addScrollListener(); } /** * Lifecycle method that triggers when the component unmounts. * Removes scroll listeners to avoid memory leaks. */ componentWillUnmount() { this.removeScrollListener(); } /** * Getter that retrieves all linked CSV files for analysis. */ @computed get linkedCSVs(): { filename: string; id: string; text: string }[] { return this._linked_csv_files; } /** * Getter that formats the entire chat history as a string for the agent's system message. */ @computed get formattedHistory(): string { let history = '\n'; for (const message of this._history) { history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; if (message.loop_summary) { history += `${message.loop_summary}`; } history += `\n`; } history += ''; return history; } // Other helper methods for retrieving document data and processing retrieveCSVData = () => { return this.linkedCSVs; }; retrieveFormattedHistory = (): string => { return this.formattedHistory; }; /** * Handles follow-up questions when the user clicks on them. * Automatically sets the input value to the clicked follow-up question. * @param question The follow-up question clicked by the user. */ @action handleFollowUpClick = (question: string) => { this._inputValue = question; }; /** * Handles tool creation notification and shows the reload modal * @param toolName The name of the tool that was created */ @action handleToolCreated = (toolName: string) => { this._toolReloadModal = { visible: true, toolName: toolName, }; }; /** * Closes the tool reload modal */ @action closeToolReloadModal = () => { this._toolReloadModal = { visible: false, toolName: '', }; }; /** * Handles the reload confirmation and triggers page reload */ @action handleReloadConfirmation = async () => { // Close the modal first this.closeToolReloadModal(); try { // Perform the deferred tool save operation const saveSuccess = await this.agent?.performDeferredToolSave(); if (saveSuccess) { console.log('Tool saved successfully, proceeding with reload...'); } else { console.warn('Tool save failed, but proceeding with reload anyway...'); } } catch (error) { console.error('Error during deferred tool save:', error); } // Trigger page reload to rebuild webpack and load the new tool setTimeout(() => { window.location.reload(); }, 100); }; _dictation: DictationButton | null = null; /** * Toggles the font size modal visibility */ @action toggleFontSizeModal = () => { this._isFontSizeModalOpen = !this._isFontSizeModalOpen; }; /** * Changes the font size and applies it to the chat interface * @param size The new font size to apply */ @action changeFontSize = (size: 'small' | 'normal' | 'large' | 'xlarge') => { this._fontSize = size; this._isFontSizeModalOpen = false; // Save preference to localStorage if needed if (typeof window !== 'undefined') { localStorage.setItem('chatbox-font-size', size); } }; /** * Initializes font size from saved preference */ initFontSize = () => { if (typeof window !== 'undefined') { const savedSize = localStorage.getItem('chatbox-font-size'); if (savedSize && ['small', 'normal', 'large', 'xlarge'].includes(savedSize)) { this._fontSize = savedSize as 'small' | 'normal' | 'large' | 'xlarge'; } } }; /** * Renders a font size icon SVG */ renderFontSizeIcon = () => ( ); /** * Shows the citation popup with the given text. * @param text The text to display in the popup. */ @action showCitationPopup = (text: string) => { this._citationPopup = { text: text || 'No text available', visible: true, }; this.startCitationPopupTimer(); }; /** * Closes the citation popup. */ @action closeCitationPopup = () => { this._citationPopup.visible = false; }; /** * Starts the auto-close timer for the citation popup. */ startCitationPopupTimer = () => { // Auto-close the popup after 5 seconds setTimeout(() => this.closeCitationPopup(), 5000); }; /** * Retry PDF search with exponential backoff */ retryPdfSearch = (doc: Doc, citation: Citation, foundChunk: SimplifiedChunk, isPDF: boolean, attempt: number) => { if (attempt > 5) { console.error('Maximum retry attempts reached for PDF search'); return; } const delay = Math.min(2000, 500 * Math.pow(1.5, attempt)); // Exponential backoff with max delay of 2 seconds setTimeout(() => { try { if (!doc[DocViews] || doc[DocViews].size === 0) { this.retryPdfSearch(doc, citation, foundChunk, isPDF, attempt + 1); return; } const views = Array.from(doc[DocViews]); if (!views.length) { this.retryPdfSearch(doc, citation, foundChunk, isPDF, attempt + 1); return; } const firstView = views[0] as DocumentView; if (!firstView || !firstView.ComponentView) { this.retryPdfSearch(doc, citation, foundChunk, isPDF, attempt + 1); return; } const pdfComponent = firstView.ComponentView as PDFBox; if (isPDF && pdfComponent && citation.direct_text) { console.log(`PDF component found on attempt ${attempt}, executing search...`); this.ensureFuzzySearchAndExecute(pdfComponent, citation.direct_text.trim(), foundChunk.startPage); } } catch (error) { console.error(`Error on retry attempt ${attempt}:`, error); this.retryPdfSearch(doc, citation, foundChunk, isPDF, attempt + 1); } }, delay); }; /** * Ensures fuzzy search is enabled in PDFBox and performs a search * @param pdfComponent The PDFBox component * @param searchText The text to search for * @param startPage Optional page to navigate to before searching */ private ensureFuzzySearchAndExecute = (pdfComponent: PDFBox, searchText: string, startPage?: number) => { if (!pdfComponent) { console.warn('PDF component is undefined, cannot perform search'); return; } if (!searchText?.trim()) { console.warn('Search text is empty, skipping search'); return; } try { // Check if the component has required methods if (typeof pdfComponent.gotoPage !== 'function' || typeof pdfComponent.toggleFuzzySearch !== 'function' || typeof pdfComponent.search !== 'function') { console.warn('PDF component missing required methods'); return; } // Navigate to the page if specified if (typeof startPage === 'number') { pdfComponent.gotoPage(startPage + 1); } // Always try to enable fuzzy search try { // PDFBox.tsx toggles fuzzy search state internally // We'll call it once to make sure it's enabled pdfComponent.toggleFuzzySearch(); } catch (toggleError) { console.warn('Error toggling fuzzy search:', toggleError); } // Add a sufficient delay to ensure PDF is fully loaded before searching setTimeout(() => { try { console.log('Performing fuzzy search for text:', searchText); pdfComponent.search(searchText); } catch (searchError) { console.error('Error performing search:', searchError); } }, 1000); // Increased delay for better reliability } catch (error) { console.error('Error in fuzzy search setup:', error); } }; /** * Main render method for the ChatBox */ render() { const fontSizeClass = `font-size-${this._fontSize}`; return (
{this._isUploadingDocs && (
{Math.round(this._uploadProgress)}%
{this._currentStep}
)}

{this.userName()}'s AI Assistant

{this.docManager?.getCanvasMode() ? 'Click to limit scope to linked documents' : 'Click to expand scope to all documents on canvas'}
} placement="bottom">
{this.docManager?.getCanvasMode() ? : }
{this.renderFontSizeIcon()}
{this._isFontSizeModalOpen && (
this.changeFontSize('small')}> Small Aa
this.changeFontSize('normal')}> Normal Aa
this.changeFontSize('large')}> Large Aa
this.changeFontSize('xlarge')}> Extra Large Aa
)}
{this._history.map((message, index) => ( ))} {this._current_message && ( )}
{ this._textInputRef = r; }} type="text" name="messageInput" autoComplete="off" placeholder="Type your message here..." value={this._inputValue} onChange={e => this.setChatInput(e.target.value)} disabled={this._isLoading} />
{this._citationPopup.text}
)} {/* Tool Reload Modal */} {this._toolReloadModal.visible && (

Tool Created Successfully!

The tool {this._toolReloadModal.toolName} has been created and saved successfully.

To make the tool available for future use, the page needs to be reloaded to rebuild the application bundle.

Click "Reload Page" to complete the tool installation.

)} ); } } /** * Register the ChatBox component as the template for CHAT document types. */ Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { layout: { view: ChatBox, dataField: 'data' }, options: { acl: '', _layout_nativeDimEditable: true, _layout_fitWidth: true, chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '', _forceActive: true }, });