import { action, computed, makeObservable, observable, observe, reaction, runInAction, ObservableSet } from 'mobx'; import { observer } from 'mobx-react'; import OpenAI, { ClientOptions } from 'openai'; import * as React from 'react'; import { Doc, DocListCast } from '../../../../fields/Doc'; import { CsvCast, DocCast, PDFCast, RTFCast, StrCast } from '../../../../fields/Types'; import { DocumentType } from '../../../documents/DocumentTypes'; import { Docs } from '../../../documents/Documents'; import { LinkManager } from '../../../util/LinkManager'; import { ViewBoxAnnotatableComponent } from '../../DocComponent'; import { FieldView, FieldViewProps } from '../FieldView'; import './ChatBox.scss'; import MessageComponentBox from './MessageComponent'; import { ASSISTANT_ROLE, AssistantMessage, AI_Document, Citation, CHUNK_TYPE, RAGChunk, getChunkType, TEXT_TYPE, SimplifiedChunk, ProcessingInfo, MessageContent } from './types'; import { Vectorstore } from './vectorstore/Vectorstore'; import { Agent } from './Agent'; import dotenv from 'dotenv'; import { DocData, DocViews } from '../../../../fields/DocSymbols'; import { AnswerParser } from './AnswerParser'; import { DocumentManager } from '../../../util/DocumentManager'; import { v4 as uuidv4 } from 'uuid'; import { chunk } from 'lodash'; import { DocUtils } from '../../../documents/DocUtils'; import { createRef } from 'react'; import { ClientUtils } from '../../../../ClientUtils'; import { ProgressBar } from './ProgressBar'; dotenv.config(); @observer export class ChatBox extends ViewBoxAnnotatableComponent() { @observable history: AssistantMessage[] = []; @observable.deep current_message: AssistantMessage | undefined = undefined; @observable isLoading: boolean = false; @observable uploadProgress: number = 0; // Track progress percentage @observable currentStep: string = ''; // Track current step name @observable expandedScratchpadIndex: number | null = null; @observable inputValue: string = ''; @observable private linked_docs_to_add: ObservableSet = observable.set(); @observable private linked_csv_files: { filename: string; id: string; text: string }[] = []; @observable private isUploadingDocs: boolean = false; private openai: OpenAI; private vectorstore_id: string; private vectorstore: Vectorstore; private agent: Agent; // Add the ChatBot instance private _oldWheel: HTMLDivElement | null = null; private messagesRef: React.RefObject; public static LayoutString(fieldKey: string) { return FieldView.LayoutString(ChatBox, fieldKey); } constructor(props: FieldViewProps) { super(props); makeObservable(this); this.openai = this.initializeOpenAI(); if (StrCast(this.dataDoc.vectorstore_id) == '') { console.log('new_id'); this.vectorstore_id = uuidv4(); this.dataDoc.vectorstore_id = this.vectorstore_id; } else { this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); } this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createCSVInDash); this.messagesRef = React.createRef(); reaction( () => this.history.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations })), serializableHistory => { this.dataDoc.data = JSON.stringify(serializableHistory); } ); } @action addDocToVectorstore = async (newLinkedDoc: Doc) => { this.uploadProgress = 0; this.currentStep = 'Initializing...'; this.isUploadingDocs = true; try { await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); } catch (error) { console.error('Error uploading document:', error); this.currentStep = 'Error during upload'; } finally { this.isUploadingDocs = false; this.uploadProgress = 0; this.currentStep = ''; } }; @action updateProgress = (progress: number, step: string) => { console.log('Progress:', progress, step); this.uploadProgress = progress; this.currentStep = step; }; @action addCSVForAnalysis = async (newLinkedDoc: Doc, id?: string) => { console.log('adding csv file for analysis'); if (!newLinkedDoc.chunk_simpl) { const csvData: string = StrCast(newLinkedDoc.text); console.log('CSV Data:', csvData); const completion = await this.openai.chat.completions.create({ messages: [ { role: 'system', content: 'You are an AI assistant tasked with summarizing the content of a CSV file. You will be provided with the data from the CSV file and your goal is to generate a concise summary that captures the main themes, trends, and key points represented in the data.', }, { role: 'user', content: `Please provide a comprehensive summary of the CSV file based on the provided data. Ensure the summary highlights the most important information, patterns, and insights. Your response should be in paragraph form and be concise. CSV Data: ${csvData} ********** Summary:`, }, ], model: 'gpt-3.5-turbo', }); console.log('CSV Data:', csvData); const csvId = id ?? uuidv4(); this.linked_csv_files.push({ filename: CsvCast(newLinkedDoc.data).url.pathname, id: csvId, text: csvData, }); console.log(this.linked_csv_files); const chunkToAdd = { chunkId: csvId, chunkType: CHUNK_TYPE.CSV, }; newLinkedDoc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); newLinkedDoc.summary = completion.choices[0].message.content!; } }; @action toggleToolLogs = (index: number) => { this.expandedScratchpadIndex = this.expandedScratchpadIndex === index ? null : index; }; initializeOpenAI() { console.log(process.env.OPENAI_KEY); const configuration: ClientOptions = { apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true, }; return new OpenAI(configuration); } addScrollListener = () => { if (this.messagesRef.current) { this.messagesRef.current.addEventListener('wheel', this.onPassiveWheel, { passive: false }); } }; removeScrollListener = () => { if (this.messagesRef.current) { this.messagesRef.current.removeEventListener('wheel', this.onPassiveWheel); } }; scrollToBottom = () => { if (this.messagesRef.current) { this.messagesRef.current.scrollTop = this.messagesRef.current.scrollHeight; } }; onPassiveWheel = (e: WheelEvent) => { if (this._props.isContentActive()) { e.stopPropagation(); } }; @action askGPT = async (event: React.FormEvent): Promise => { event.preventDefault(); this.inputValue = ''; const textInput = event.currentTarget.elements.namedItem('messageInput') as HTMLInputElement; const trimmedText = textInput.value.trim(); if (trimmedText) { try { textInput.value = ''; this.history.push({ role: ASSISTANT_ROLE.USER, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: trimmedText, citation_ids: null }], processing_info: [] }); this.isLoading = true; this.current_message = { role: ASSISTANT_ROLE.ASSISTANT, content: [], citations: [], processing_info: [] }; const onProcessingUpdate = (processingUpdate: ProcessingInfo[]) => { runInAction(() => { if (this.current_message) { this.current_message = { ...this.current_message, processing_info: processingUpdate }; } }); this.scrollToBottom(); }; const onAnswerUpdate = (answerUpdate: string) => { runInAction(() => { if (this.current_message) { this.current_message = { ...this.current_message, content: [{ text: answerUpdate, type: TEXT_TYPE.NORMAL, index: 0, citation_ids: [] }] }; } }); }; const finalMessage = await this.agent.askAgent(trimmedText, onProcessingUpdate, onAnswerUpdate); runInAction(() => { if (this.current_message) { this.history.push({ ...finalMessage }); this.current_message = undefined; this.dataDoc.data = JSON.stringify(this.history); } }); } catch (err) { console.error('Error:', err); this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.ERROR, text: 'Sorry, I encountered an error while processing your request.', citation_ids: null }], processing_info: [] }); } finally { this.isLoading = false; this.scrollToBottom(); } } this.scrollToBottom(); }; @action updateMessageCitations = (index: number, citations: Citation[]) => { if (this.history[index]) { this.history[index].citations = citations; } }; @action addLinkedUrlDoc = async (url: string, id: string) => { const doc = Docs.Create.WebDocument(url); const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); const chunkToAdd = { chunkId: id, chunkType: CHUNK_TYPE.URL, }; doc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); }; @computed get userName() { return ClientUtils.CurrentUserEmail; } @action createCSVInDash = async (url: string, title: string, id: string, data: string) => { console.log('Creating CSV in Dash:', url, title); const doc = DocCast(await DocUtils.DocumentFromType('csv', url, { title: title, text: RTFCast(data) })); const linkDoc = Docs.Create.LinkDocument(this.Document, doc); LinkManager.Instance.addLink(linkDoc); doc && this._props.addDocument?.(doc); await DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); this.addCSVForAnalysis(doc, id); }; @action handleCitationClick = (citation: Citation) => { console.log('Citation clicked:', citation); const currentLinkedDocs: Doc[] = this.linkedDocs; const chunkId = citation.chunk_id; for (let doc of currentLinkedDocs) { if (doc.chunk_simpl) { const docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; console.log(docChunkSimpl); const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkId === chunkId); console.log(foundChunk); if (foundChunk) { console.log(getChunkType(foundChunk.chunkType)); switch (foundChunk.chunkType) { case CHUNK_TYPE.IMAGE: case CHUNK_TYPE.TABLE: const values = foundChunk.location?.replace(/[\[\]]/g, '').split(','); if (values?.length !== 4) { console.error('Location string must contain exactly 4 numbers'); return; } const x1 = parseFloat(values[0]) * Doc.NativeWidth(doc); const y1 = parseFloat(values[1]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); const x2 = parseFloat(values[2]) * Doc.NativeWidth(doc); const y2 = parseFloat(values[3]) * Doc.NativeHeight(doc) + foundChunk.startPage * Doc.NativeHeight(doc); const annotationKey = Doc.LayoutFieldKey(doc) + '_annotations'; const existingDoc = DocListCast(doc[DocData][annotationKey]).find(d => d.citation_id === citation.citation_id); const highlightDoc = existingDoc ?? this.createImageCitationHighlight(x1, y1, x2, y2, citation, annotationKey, doc); DocumentManager.Instance.showDocument(highlightDoc, { willZoomCentered: true }, () => {}); break; case CHUNK_TYPE.TEXT: DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { const firstView = Array.from(doc[DocViews])[0]; firstView.ComponentView?.search?.(citation.direct_text); }); break; case CHUNK_TYPE.URL: DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { const firstView = Array.from(doc[DocViews])[0]; }); break; case CHUNK_TYPE.CSV: DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { const firstView = Array.from(doc[DocViews])[0]; }); break; default: console.log('Chunk type not supported', foundChunk.chunkType); break; } } } } }; createImageCitationHighlight = (x1: number, y1: number, x2: number, y2: number, citation: Citation, annotationKey: string, pdfDoc: Doc): Doc => { const highlight_doc = Docs.Create.FreeformDocument([], { x: x1, y: y1, _width: x2 - x1, _height: y2 - y1, backgroundColor: 'rgba(255, 255, 0, 0.5)', }); highlight_doc[DocData].citation_id = citation.citation_id; Doc.AddDocToList(pdfDoc[DocData], annotationKey, highlight_doc); highlight_doc.annotationOn = pdfDoc; Doc.SetContainer(highlight_doc, pdfDoc); return highlight_doc; }; componentDidUpdate() { this.scrollToBottom(); } componentDidMount() { this._props.setContentViewBox?.(this); if (this.dataDoc.data) { try { const storedHistory = JSON.parse(StrCast(this.dataDoc.data)); runInAction(() => { this.history.push( ...storedHistory.map((msg: AssistantMessage) => ({ role: msg.role, content: msg.content, follow_up_questions: msg.follow_up_questions, citations: msg.citations, })) ); }); } catch (e) { console.error('Failed to parse history from dataDoc:', e); } } else { runInAction(() => { this.history.push({ role: ASSISTANT_ROLE.ASSISTANT, content: [{ index: 0, type: TEXT_TYPE.NORMAL, text: `Hey, ${this.userName()} Welcome to the Your Friendly Assistant! Link a document or ask questions about anything to get started.`, citation_ids: null }], processing_info: [], }); }); } reaction( () => { const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d); return linkedDocs; }, linked => linked.forEach(doc => this.linked_docs_to_add.add(doc)) ); observe(this.linked_docs_to_add, change => { if (change.type === 'add') { if (PDFCast(change.newValue.data)) { this.addDocToVectorstore(change.newValue); } else if (CsvCast(change.newValue.data)) { this.addCSVForAnalysis(change.newValue); } } else if (change.type === 'delete') { console.log('Deleted docs: ', change.oldValue); } }); this.addScrollListener(); } componentWillUnmount() { this.removeScrollListener(); } @computed get linkedDocs() { return LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d); } @computed get docIds() { return LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d) .filter(d => d.ai_doc_id) .map(d => StrCast(d.ai_doc_id)); } @computed get summaries(): string { return ( LinkManager.Instance.getAllRelatedLinks(this.Document) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d) .filter(d => d.summary) .map((doc, index) => { if (PDFCast(doc.data)) { return `${doc.summary}`; } else if (CsvCast(doc.data)) { return `${doc.summary}`; } else { return `${index + 1}) ${doc.summary}`; } }) .join('\n') + '\n' ); } @computed get linkedCSVs(): { filename: string; id: string; text: string }[] { return this.linked_csv_files; } @computed get formattedHistory(): string { let history = '\n'; for (const message of this.history) { history += `<${message.role}>${message.content.map(content => content.text).join(' ')}`; if (message.loop_summary) { history += `${message.loop_summary}`; } history += `\n`; } history += ''; return history; } retrieveSummaries = () => { return this.summaries; }; retrieveCSVData = () => { return this.linkedCSVs; }; retrieveFormattedHistory = () => { return this.formattedHistory; }; retrieveDocIds = () => { return this.docIds; }; @action handleFollowUpClick = (question: string) => { console.log('Follow-up question clicked:', question); this.inputValue = question; }; render() { return (
{this.isUploadingDocs && (
{this.currentStep}
)}

{this.userName()}'s AI Assistant

{this.history.map((message, index) => ( ))} {this.current_message && ( )}
(this.inputValue = e.target.value)} />
); } } Docs.Prototypes.TemplateMap.set(DocumentType.CHAT, { layout: { view: ChatBox, dataField: 'data' }, options: { acl: '', chat: '', chat_history: '', chat_thread_id: '', chat_assistant_id: '', chat_vector_store_id: '' }, });