From a99d8df26d331d87bae4f27cd52ce5ec4d97fa7a Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Thu, 3 Apr 2025 11:45:02 -0400 Subject: attempt --- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 900 +++++++++++++++++++++ 1 file changed, 900 insertions(+) create mode 100644 src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts new file mode 100644 index 000000000..9c3a1fbb5 --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -0,0 +1,900 @@ +import { Doc, FieldType } from '../../../../../fields/Doc'; +import { DocData } from '../../../../../fields/DocSymbols'; +import { Observation } from '../types/types'; +import { ParametersType, ToolInfo } from '../types/tool_types'; +import { BaseTool } from './BaseTool'; +import { DocumentOptions } from '../../../../documents/Documents'; +import { CollectionFreeFormDocumentView } from '../../../nodes/CollectionFreeFormDocumentView'; +import { v4 as uuidv4 } from 'uuid'; +import { LinkManager } from '../../../../util/LinkManager'; +import { DocCast, StrCast } from '../../../../../fields/Types'; + +// Parameter definitions for DocumentMetadataTool +const documentMetadataToolParams = [ + { + name: 'action', + type: 'string', + description: 'The action to perform: "get" for retrieving metadata, "edit" for modifying document metadata, or "list" for a simple document list', + required: true, + }, + { + name: 'documentId', + type: 'string', + description: 'The ID of the document to get metadata from or edit. If not provided when getting metadata, information about all documents will be returned.', + required: false, + }, + { + name: 'fieldName', + type: 'string', + description: 'When editing, the name of the field to modify (with or without leading underscore).', + required: false, + }, + { + name: 'fieldValue', + type: 'string', + description: 'When editing, the new value to set for the specified field. Numeric values will be converted to strings automatically.', + required: false, + }, + { + name: 'documentLocation', + type: 'string', + description: 'When editing, specify where to modify the field: "layout" (default), "data", or "auto" (determines automatically based on existing fields).', + required: false, + } +] as const; + +type DocumentMetadataToolParamsType = typeof documentMetadataToolParams; + +// Detailed description with usage guidelines for the DocumentMetadataTool +const toolDescription = `Extracts and modifies metadata from documents in the same Freeform view as the ChatBox. +This tool helps you work with document properties, understand available fields, and edit document metadata. + +The Dash document system organizes fields in two locations: +1. Layout documents: contain visual properties like position, dimensions, and appearance +2. Data documents: contain the actual content and document-specific data + +This tool provides the following capabilities: +- Get metadata from all documents in the current Freeform view +- Get metadata from a specific document +- Edit metadata fields on documents (in either layout or data documents) +- List all available documents in the current view +- Understand which fields are stored where (layout vs data document) +- Get detailed information about all available document fields`; + +// Extensive usage guidelines for the tool +const citationRules = `USAGE GUIDELINES: +To GET document metadata: +- Use action="get" with optional documentId to return metadata for one or all documents +- Returns field values, field definitions, and location information (layout vs data document) + +To EDIT document metadata: +- Use action="edit" with required documentId, fieldName, and fieldValue parameters +- Optionally specify documentLocation="layout"|"data"|"auto" (default is "auto") +- The tool will determine the correct document location automatically unless specified +- Field names can be provided with or without leading underscores (e.g., both "width" and "_width" work) +- Common fields like "width" and "height" are automatically mapped to "_width" and "_height" +- Numeric values are accepted for appropriate fields (width, height, etc.) +- The tool will apply the edit to the correct document (layout or data) + +To LIST available documents: +- Use action="list" to get a simple list of all documents in the current view +- This is useful when you need to identify documents before getting details or editing them + +Editing fields follows these rules: +1. If documentLocation is specified, the field is modified in that location +2. If documentLocation="auto", the tool checks if the field exists on layout or data document first +3. If the field doesn't exist in either document, it's added to the location specified or layout document by default +4. Fields with leading underscores are automatically handled correctly + +Examples: +- To list all documents: { action: "list" } +- To get all document metadata: { action: "get" } +- To get metadata for a specific document: { action: "get", documentId: "doc123" } +- To edit a field: { action: "edit", documentId: "doc123", fieldName: "backgroundColor", fieldValue: "#ff0000" } +- To edit a width property: { action: "edit", documentId: "doc123", fieldName: "width", fieldValue: 300 } +- To edit a field in the data document: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "New content", documentLocation: "data" }`; + +const documentMetadataToolInfo: ToolInfo = { + name: 'documentMetadata', + description: toolDescription, + parameterRules: documentMetadataToolParams, + citationRules: citationRules, +}; + +/** + * A tool for extracting and modifying metadata from documents in a Freeform view. + * This tool collects metadata from both layout and data documents in a Freeform view + * and allows for editing document fields in the correct location. + */ +export class DocumentMetadataTool extends BaseTool { + private freeformView: any; + private chatBox: any; + private chatBoxDocument: Doc | null = null; + private documentsById: Map = new Map(); + private layoutDocsById: Map = new Map(); + private dataDocsById: Map = new Map(); + private fieldMetadata: Record = {}; + private readonly DOCUMENT_ID_FIELD = '_dash_document_id'; + + constructor(chatBox: any) { + super(documentMetadataToolInfo); + this.chatBox = chatBox; + + // Store a direct reference to the ChatBox document + if (chatBox && chatBox.Document) { + this.chatBoxDocument = chatBox.Document; + if (this.chatBoxDocument && this.chatBoxDocument.id) { + console.log('DocumentMetadataTool initialized with ChatBox Document:', this.chatBoxDocument.id); + } else { + console.log('DocumentMetadataTool initialized with ChatBox Document (no ID)'); + } + } else if (chatBox && chatBox.props && chatBox.props.Document) { + this.chatBoxDocument = chatBox.props.Document; + if (this.chatBoxDocument && this.chatBoxDocument.id) { + console.log('DocumentMetadataTool initialized with ChatBox props.Document:', this.chatBoxDocument.id); + } else { + console.log('DocumentMetadataTool initialized with ChatBox props.Document (no ID)'); + } + } else { + console.warn('DocumentMetadataTool initialized without valid ChatBox Document reference'); + } + + this.initializeFieldMetadata(); + } + + /** + * Extracts field metadata from DocumentOptions class + */ + private initializeFieldMetadata() { + // Parse DocumentOptions to extract field definitions + const documentOptionsInstance = new DocumentOptions(); + const documentOptionsEntries = Object.entries(documentOptionsInstance); + + for (const [fieldName, fieldInfo] of documentOptionsEntries) { + // Extract field information + const fieldData: Record = { + name: fieldName, + withoutUnderscore: fieldName.startsWith('_') ? fieldName.substring(1) : fieldName, + description: '', + type: 'unknown', + required: false, + defaultValue: undefined, + possibleValues: [], + }; + + // Check if fieldInfo has description property (it's likely a FInfo instance) + if (fieldInfo && typeof fieldInfo === 'object' && 'description' in fieldInfo) { + fieldData.description = fieldInfo.description; + + // Extract field type if available + if ('fieldType' in fieldInfo) { + fieldData.type = fieldInfo.fieldType; + } + + // Extract possible values if available + if ('values' in fieldInfo && Array.isArray(fieldInfo.values)) { + fieldData.possibleValues = fieldInfo.values; + } + } + + this.fieldMetadata[fieldName] = fieldData; + } + } + + /** + * Gets all documents in the same Freeform view as the ChatBox + * Uses the LinkManager to get all linked documents, similar to how ChatBox does it + */ + private findDocumentsInFreeformView() { + // Reset collections + this.documentsById.clear(); + this.layoutDocsById.clear(); + this.dataDocsById.clear(); + + try { + // Use the LinkManager approach which is proven to work in ChatBox + if (this.chatBoxDocument) { + console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument.id); + + // Get directly linked documents via LinkManager + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.chatBoxDocument) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.chatBoxDocument!))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + console.log(`Found ${linkedDocs.length} linked documents via LinkManager`); + + // Process the linked documents + linkedDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + + // Include the ChatBox document itself + this.processDocument(this.chatBoxDocument); + + // If we have access to the Document's parent, try to find sibling documents + if (this.chatBoxDocument.parent) { + const parent = this.chatBoxDocument.parent; + console.log('Found parent document, checking for siblings'); + + // Check if parent is a Doc type and has a childDocs function + if (parent && typeof parent === 'object' && 'childDocs' in parent && typeof parent.childDocs === 'function') { + try { + const siblingDocs = parent.childDocs(); + if (Array.isArray(siblingDocs)) { + console.log(`Found ${siblingDocs.length} sibling documents via parent.childDocs()`); + siblingDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + } catch (e) { + console.warn('Error accessing parent.childDocs:', e); + } + } + } + } else if (this.chatBox && this.chatBox.linkedDocs) { + // If we have direct access to the linkedDocs computed property from ChatBox + console.log('Using ChatBox.linkedDocs directly'); + const linkedDocs = this.chatBox.linkedDocs; + if (Array.isArray(linkedDocs)) { + console.log(`Found ${linkedDocs.length} documents via ChatBox.linkedDocs`); + linkedDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + + // Process the ChatBox document if available + if (this.chatBox.Document) { + this.processDocument(this.chatBox.Document); + } + } else { + console.warn('No ChatBox document reference available for finding linked documents'); + } + + console.log(`DocumentMetadataTool found ${this.documentsById.size} total documents`); + + // If we didn't find any documents, try a fallback method + if (this.documentsById.size === 0 && this.chatBox) { + console.log('No documents found, trying fallback method'); + + // Try to access any field that might contain documents + if (this.chatBox.props && this.chatBox.props.documents) { + const documents = this.chatBox.props.documents; + if (Array.isArray(documents)) { + console.log(`Found ${documents.length} documents via ChatBox.props.documents`); + documents.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + } + } + } catch (error) { + console.error('Error finding documents in Freeform view:', error); + } + } + + /** + * Process a document by ensuring it has an ID and adding it to the appropriate collections + * @param doc The document to process + */ + private processDocument(doc: Doc) { + // Ensure document has a persistent ID + const docId = this.ensureDocumentId(doc); + + // Only add if we haven't already processed this document + if (!this.documentsById.has(docId)) { + this.documentsById.set(docId, doc); + + // Get layout doc (the document itself or its layout) + const layoutDoc = Doc.Layout(doc); + if (layoutDoc) { + this.layoutDocsById.set(docId, layoutDoc); + } + + // Get data doc + const dataDoc = doc[DocData]; + if (dataDoc) { + this.dataDocsById.set(docId, dataDoc); + } + } + } + + /** + * Ensures a document has a persistent ID stored in its metadata + * @param doc The document to ensure has an ID + * @returns The document's ID + */ + private ensureDocumentId(doc: Doc): string { + let docId: string | undefined; + + // First try to get the ID from our custom field + if (doc[this.DOCUMENT_ID_FIELD]) { + docId = String(doc[this.DOCUMENT_ID_FIELD]); + return docId; + } + + // Try different ways to get a document ID + + // 1. Try the direct id property if it exists + if (doc.id && typeof doc.id === 'string') { + docId = doc.id; + } + // 2. Try doc._id if it exists + else if (doc._id && typeof doc._id === 'string') { + docId = doc._id; + } + // 3. Try doc.data?.id if it exists + else if (doc.data && typeof doc.data === 'object' && 'id' in doc.data && typeof doc.data.id === 'string') { + docId = doc.data.id; + } + // 4. If none of the above work, generate a UUID + else { + docId = uuidv4(); + console.log(`Generated new UUID for document with title: ${doc.title || 'Untitled'}`); + } + + // Store the ID in the document's metadata so it persists + try { + doc[this.DOCUMENT_ID_FIELD] = docId; + } catch (e) { + console.warn(`Could not assign ID to document property`, e); + } + + return docId; + } + + /** + * Extracts metadata from a specific document + * @param docId The ID of the document to extract metadata from + * @returns An object containing the document's metadata + */ + private extractDocumentMetadata(docId: string) { + const doc = this.documentsById.get(docId); + if (!doc) { + return null; + } + + const layoutDoc = this.layoutDocsById.get(docId); + const dataDoc = this.dataDocsById.get(docId); + + const metadata: Record = { + id: docId, + title: doc.title || '', + type: doc.type || '', + fields: { + layout: {}, + data: {}, + }, + fieldLocationMap: {}, + }; + + // Process all known field definitions + Object.keys(this.fieldMetadata).forEach(fieldName => { + const fieldDef = this.fieldMetadata[fieldName]; + const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check if field exists on layout document + let layoutValue = undefined; + if (layoutDoc) { + layoutValue = layoutDoc[fieldName]; + if (layoutValue !== undefined) { + // Field exists on layout document + metadata.fields.layout[fieldName] = this.formatFieldValue(layoutValue); + metadata.fieldLocationMap[strippedName] = 'layout'; + } + } + + // Check if field exists on data document + let dataValue = undefined; + if (dataDoc) { + dataValue = dataDoc[fieldName]; + if (dataValue !== undefined) { + // Field exists on data document + metadata.fields.data[fieldName] = this.formatFieldValue(dataValue); + if (!metadata.fieldLocationMap[strippedName]) { + metadata.fieldLocationMap[strippedName] = 'data'; + } + } + } + + // For fields with stripped names (without leading underscore), + // also check if they exist on documents without the underscore + if (fieldName.startsWith('_')) { + const nonUnderscoreFieldName = fieldName.substring(1); + + if (layoutDoc) { + const nonUnderscoreLayoutValue = layoutDoc[nonUnderscoreFieldName]; + if (nonUnderscoreLayoutValue !== undefined) { + metadata.fields.layout[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreLayoutValue); + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'layout'; + } + } + + if (dataDoc) { + const nonUnderscoreDataValue = dataDoc[nonUnderscoreFieldName]; + if (nonUnderscoreDataValue !== undefined) { + metadata.fields.data[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreDataValue); + if (!metadata.fieldLocationMap[nonUnderscoreFieldName]) { + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'data'; + } + } + } + } + }); + + // Add common field aliases for easier discovery + // This helps users understand both width and _width refer to the same property + if (metadata.fields.layout._width !== undefined && metadata.fields.layout.width === undefined) { + metadata.fields.layout.width = metadata.fields.layout._width; + metadata.fieldLocationMap.width = 'layout'; + } + + if (metadata.fields.layout._height !== undefined && metadata.fields.layout.height === undefined) { + metadata.fields.layout.height = metadata.fields.layout._height; + metadata.fieldLocationMap.height = 'layout'; + } + + return metadata; + } + + /** + * Edits a specific field on a document + * @param docId The ID of the document to edit + * @param fieldName The name of the field to edit + * @param fieldValue The new value for the field + * @param documentLocation Where to edit the field: 'layout', 'data', or 'auto' + * @returns Object with success status, message, and additional information + */ + private editDocumentField(docId: string, fieldName: string, fieldValue: string, documentLocation: string = 'auto'): { + success: boolean; + message: string; + fieldName?: string; + originalFieldName?: string; + newValue?: any; + } { + // Normalize field name (handle with/without underscore) + let normalizedFieldName = fieldName.startsWith('_') ? fieldName : fieldName; + const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Handle common field name aliases (width → _width, height → _height) + // Many document fields use '_' prefix for layout properties + if (fieldName === 'width') { + normalizedFieldName = '_width'; + } else if (fieldName === 'height') { + normalizedFieldName = '_height'; + } + + // Get the documents + const doc = this.documentsById.get(docId); + if (!doc) { + return { success: false, message: `Document with ID ${docId} not found` }; + } + + const layoutDoc = this.layoutDocsById.get(docId); + const dataDoc = this.dataDocsById.get(docId); + + if (!layoutDoc && !dataDoc) { + return { success: false, message: `Could not find layout or data document for document with ID ${docId}` }; + } + + try { + // Convert the field value to the appropriate type based on field metadata + const convertedValue = this.convertFieldValue(normalizedFieldName, fieldValue); + + // Determine where to set the field value + let targetDoc: Doc | undefined; + let targetLocation: string; + + if (documentLocation === 'layout') { + // Explicitly set on layout document + targetDoc = layoutDoc; + targetLocation = 'layout'; + } else if (documentLocation === 'data') { + // Explicitly set on data document + targetDoc = dataDoc; + targetLocation = 'data'; + } else { + // Auto-detect where to set the field + + // Check if field exists on layout document (with or without underscore) + const existsOnLayout = layoutDoc && + (layoutDoc[normalizedFieldName] !== undefined || + layoutDoc[strippedFieldName] !== undefined); + + // Check if field exists on data document (with or without underscore) + const existsOnData = dataDoc && + (dataDoc[normalizedFieldName] !== undefined || + dataDoc[strippedFieldName] !== undefined); + + if (existsOnLayout) { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } else if (existsOnData) { + targetDoc = dataDoc; + targetLocation = 'data'; + } else { + // Field doesn't exist on either document, default to layout document + targetDoc = layoutDoc || dataDoc; + targetLocation = layoutDoc ? 'layout' : 'data'; + } + } + + if (!targetDoc) { + return { success: false, message: `Target document (${documentLocation}) not available` }; + } + + // Set the field value on the target document + targetDoc[normalizedFieldName] = convertedValue; + + return { + success: true, + message: `Successfully updated field '${normalizedFieldName}' on ${targetLocation} document (ID: ${docId})`, + fieldName: normalizedFieldName, + originalFieldName: fieldName, + newValue: convertedValue + }; + } catch (error) { + console.error('Error editing document field:', error); + return { + success: false, + message: `Error updating field: ${error instanceof Error ? error.message : String(error)}` + }; + } + } + + /** + * Converts a string field value to the appropriate type based on field metadata + * @param fieldName The name of the field + * @param fieldValue The string value to convert + * @returns The converted value with the appropriate type + */ + private convertFieldValue(fieldName: string, fieldValue: any): any { + // If fieldValue is already a number, we don't need to convert it from string + if (typeof fieldValue === 'number') { + return fieldValue; + } + + // If fieldValue is not a string (and not a number), convert it to string + if (typeof fieldValue !== 'string') { + fieldValue = String(fieldValue); + } + + // Get field metadata + const normalizedFieldName = fieldName.startsWith('_') ? fieldName : `_${fieldName}`; + const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check both versions of the field name in metadata + const fieldMeta = this.fieldMetadata[normalizedFieldName] || this.fieldMetadata[strippedFieldName]; + + // Special handling for width and height without metadata + if (!fieldMeta && (fieldName === '_width' || fieldName === '_height' || fieldName === 'width' || fieldName === 'height')) { + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } + + if (!fieldMeta) { + // If no metadata found, just return the string value + return fieldValue; + } + + // Convert based on field type + const fieldType = fieldMeta.type; + + if (fieldType === 'boolean') { + // Convert to boolean + return fieldValue.toLowerCase() === 'true'; + } else if (fieldType === 'number') { + // Convert to number + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } else if (fieldType === 'date') { + // Try to convert to date (stored as number timestamp) + try { + return new Date(fieldValue).getTime(); + } catch (e) { + return fieldValue; + } + } else if (fieldType.includes('list') || fieldType.includes('array')) { + // Try to parse as JSON array + try { + return JSON.parse(fieldValue); + } catch (e) { + return fieldValue; + } + } else if (fieldType === 'json' || fieldType === 'object') { + // Try to parse as JSON object + try { + return JSON.parse(fieldValue); + } catch (e) { + return fieldValue; + } + } + + // Default to string + return fieldValue; + } + + /** + * Formats a field value for JSON output + * @param value The field value to format + * @returns A JSON-friendly representation of the field value + */ + private formatFieldValue(value: any): any { + if (value === undefined || value === null) { + return null; + } + + // Handle Doc objects + if (value instanceof Doc) { + return { + type: 'Doc', + id: value.id || this.ensureDocumentId(value), + title: value.title || '', + docType: value.type || '', + }; + } + + // Handle arrays and complex objects + if (typeof value === 'object') { + // If the object has a toString method, use it + if (value.toString && value.toString !== Object.prototype.toString) { + return value.toString(); + } + + try { + // Try to convert to JSON string + return JSON.stringify(value); + } catch (e) { + return '[Complex Object]'; + } + } + + // Return primitive values as is + return value; + } + + /** + * Executes the DocumentMetadataTool to extract or edit metadata from documents in the Freeform view + * @param args The parameters for the tool execution + * @returns A promise that resolves to an array of Observation objects + */ + async execute(args: ParametersType): Promise { + console.log('DocumentMetadataTool executing with args:', args); + + // Add diagnostic information about the ChatBox instance + if (this.chatBox) { + console.log('ChatBox instance available:', { + hasDocument: !!this.chatBoxDocument, + chatBoxProps: this.chatBox.props ? Object.keys(this.chatBox.props) : 'No props', + linkedDocsMethod: typeof this.chatBox.linkedDocs === 'function' ? 'Exists (function)' : + typeof this.chatBox.linkedDocs === 'object' ? 'Exists (object)' : 'Not available' + }); + } else { + console.warn('No ChatBox instance available'); + } + + // Find all documents in the Freeform view + this.findDocumentsInFreeformView(); + + // Debug diagnostic information about all documents found + if (this.documentsById.size > 0) { + console.log('Documents found:', Array.from(this.documentsById.entries()).map(([id, doc]) => ({ + id, + title: doc.title || 'Untitled', + type: doc.type || 'Unknown', + hasLayout: !!this.layoutDocsById.get(id), + hasData: !!this.dataDocsById.get(id) + }))); + } + + // Check if we found any documents + if (this.documentsById.size === 0) { + console.error('No documents found in Freeform view'); + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: 'No documents found in the current view. Unable to extract or edit metadata.', + chatBoxDocumentId: this.chatBoxDocument ? this.chatBoxDocument.id : 'No ChatBox document', + diagnostics: { + chatBoxAvailable: !!this.chatBox, + documentAvailable: !!this.chatBoxDocument, + toolInitializedWith: this.chatBox ? + (this.chatBox.constructor && this.chatBox.constructor.name) || 'Unknown constructor' + : 'No ChatBox' + } + }, null, 2) + }]; + } + + let result: Record; + + // Determine which action to perform + const action = args.action?.toLowerCase(); + + if (action === 'list') { + // Just return a simple list of available documents for quick reference + const documentList = Array.from(this.documentsById.entries()).map(([id, doc]) => ({ + id, + title: doc.title || 'Untitled', + type: doc.type || 'Unknown' + })); + + result = { + success: true, + message: `Found ${documentList.length} documents in the current view`, + documents: documentList + }; + } + else if (action === 'edit') { + // Edit document metadata + if (!args.documentId) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: 'Document ID is required for edit operations', + availableDocuments: Array.from(this.documentsById.keys()) + }, null, 2) + }]; + } + + if (!args.fieldName) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: 'Field name is required for edit operations', + }, null, 2) + }]; + } + + if (args.fieldValue === undefined) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: 'Field value is required for edit operations', + }, null, 2) + }]; + } + + // Check if the document exists + if (!this.documentsById.has(args.documentId)) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: `Document with ID ${args.documentId} not found`, + availableDocuments: Array.from(this.documentsById.keys()) + }, null, 2) + }]; + } + + // Convert fieldValue to string if it's not already + // This is to support the Agent passing numeric values directly + const fieldValue = typeof args.fieldValue === 'string' ? + args.fieldValue : + String(args.fieldValue); + + // Perform the edit + const editResult = this.editDocumentField( + args.documentId, + args.fieldName, + fieldValue, + args.documentLocation || 'auto' + ); + + // If successful, also get the updated metadata + if (editResult.success) { + const documentMetadata = this.extractDocumentMetadata(args.documentId); + result = { + ...editResult, + document: documentMetadata, + }; + } else { + result = editResult; + } + } else if (action === 'get') { + // Get document metadata + if (args.documentId) { + // Check if the document exists + if (!this.documentsById.has(args.documentId)) { + return [{ + type: 'text', + text: JSON.stringify({ + success: false, + message: `Document with ID ${args.documentId} not found`, + availableDocuments: Array.from(this.documentsById.keys()) + }, null, 2) + }]; + } + + // Get metadata for a specific document + const documentMetadata = this.extractDocumentMetadata(args.documentId); + result = { + success: !!documentMetadata, + message: documentMetadata ? 'Document metadata retrieved successfully' : `Document with ID ${args.documentId} not found`, + document: documentMetadata, + fieldDefinitions: this.fieldMetadata, + }; + } else { + // Get metadata for all documents + const documentsMetadata: Record = {}; + for (const docId of this.documentsById.keys()) { + documentsMetadata[docId] = this.extractDocumentMetadata(docId); + } + + result = { + success: true, + message: `Retrieved metadata for ${this.documentsById.size} documents`, + documents: documentsMetadata, + fieldDefinitions: this.fieldMetadata, + documentCount: this.documentsById.size, + fieldMetadataCount: Object.keys(this.fieldMetadata).length, + }; + } + } else { + // Invalid action + result = { + success: false, + message: `Invalid action: ${action}. Valid actions are "list", "get", and "edit".`, + availableActions: ["list", "get", "edit"] + }; + } + + // Log the result to the console + console.log('Document Metadata Tool Result:', result); + + // Return the result as an observation + // Convert to string format as the Observation type only supports 'text' or 'image_url' types + return [{ + type: 'text', + text: JSON.stringify(result, null, 2) + }]; + } + + /** + * Validates the input parameters for the DocumentMetadataTool + * This custom validator allows numbers to be passed for fieldValue while maintaining + * compatibility with the standard validation + * + * @param params The parameters to validate + * @returns True if the parameters are valid, false otherwise + */ + inputValidator(params: ParametersType): boolean { + // Default validation for required fields + if (params.action === undefined) { + return false; + } + + // For edit action, documentId, fieldName, and fieldValue are required + if (params.action === 'edit') { + if (!params.documentId || !params.fieldName || params.fieldValue === undefined) { + return false; + } + } + + // For get action with documentId, documentId is required + if (params.action === 'get' && params.documentId === '') { + return false; + } + + // Allow for numeric fieldValue even though the type is defined as string + if (params.fieldValue !== undefined && typeof params.fieldValue === 'number') { + console.log('Numeric fieldValue detected, will be converted to string'); + // We'll convert it later, so don't fail validation + return true; + } + + return true; + } +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 5400cf24ded3bde5ba319bb9ab50602ce0fb9927 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Thu, 3 Apr 2025 12:15:09 -0400 Subject: fix: handles dependent fields and boolean values --- .../views/nodes/chatbot/agentsystem/Agent.ts | 12 +- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 497 +++++++++++---------- 2 files changed, 261 insertions(+), 248 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index b5cdb8cf1..0610dc198 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -461,12 +461,12 @@ export class Agent { } console.log(actionInput); - // Special handling for documentMetadata tool with numeric fieldValue - if (action === 'documentMetadata' && - 'fieldValue' in actionInput && - typeof actionInput.fieldValue === 'number') { - // Convert number to string to pass validation - actionInput.fieldValue = String(actionInput.fieldValue); + // Special handling for documentMetadata tool with numeric or boolean fieldValue + if (action === 'documentMetadata' && 'fieldValue' in actionInput) { + if (typeof actionInput.fieldValue === 'number' || typeof actionInput.fieldValue === 'boolean') { + // Convert number or boolean to string to pass validation + actionInput.fieldValue = String(actionInput.fieldValue); + } } for (const param of this.tools[action].parameterRules) { diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 9c3a1fbb5..13b8315da 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -1,7 +1,7 @@ import { Doc, FieldType } from '../../../../../fields/Doc'; import { DocData } from '../../../../../fields/DocSymbols'; import { Observation } from '../types/types'; -import { ParametersType, ToolInfo } from '../types/tool_types'; +import { ParametersType, ToolInfo, Parameter } from '../types/tool_types'; import { BaseTool } from './BaseTool'; import { DocumentOptions } from '../../../../documents/Documents'; import { CollectionFreeFormDocumentView } from '../../../nodes/CollectionFreeFormDocumentView'; @@ -9,41 +9,35 @@ import { v4 as uuidv4 } from 'uuid'; import { LinkManager } from '../../../../util/LinkManager'; import { DocCast, StrCast } from '../../../../../fields/Types'; -// Parameter definitions for DocumentMetadataTool -const documentMetadataToolParams = [ +// Define the parameters for the DocumentMetadataTool +const parameterDefinitions: ReadonlyArray = [ { name: 'action', type: 'string', - description: 'The action to perform: "get" for retrieving metadata, "edit" for modifying document metadata, or "list" for a simple document list', required: true, + description: 'The action to perform: "get" to retrieve metadata, "edit" to modify metadata, or "list" to enumerate documents', }, { name: 'documentId', type: 'string', - description: 'The ID of the document to get metadata from or edit. If not provided when getting metadata, information about all documents will be returned.', required: false, + description: 'The ID of the document to get or edit metadata for. Required for "edit", optional for "get", ignored for "list"', }, { name: 'fieldName', type: 'string', - description: 'When editing, the name of the field to modify (with or without leading underscore).', required: false, + description: 'The name of the field to edit. Required for "edit" action. Field names can be provided with or without leading underscores', }, { name: 'fieldValue', type: 'string', - description: 'When editing, the new value to set for the specified field. Numeric values will be converted to strings automatically.', - required: false, - }, - { - name: 'documentLocation', - type: 'string', - description: 'When editing, specify where to modify the field: "layout" (default), "data", or "auto" (determines automatically based on existing fields).', required: false, + description: 'The new value for the field. Required for "edit" action. Can be a string, number, or boolean value depending on the field type', } ] as const; -type DocumentMetadataToolParamsType = typeof documentMetadataToolParams; +type DocumentMetadataToolParamsType = typeof parameterDefinitions; // Detailed description with usage guidelines for the DocumentMetadataTool const toolDescription = `Extracts and modifies metadata from documents in the same Freeform view as the ChatBox. @@ -59,7 +53,17 @@ This tool provides the following capabilities: - Edit metadata fields on documents (in either layout or data documents) - List all available documents in the current view - Understand which fields are stored where (layout vs data document) -- Get detailed information about all available document fields`; +- Get detailed information about all available document fields +- Support for all value types: strings, numbers, and booleans + +IMPORTANT: Some fields have dependencies that must be handled for edits to work correctly: +- When editing "height", first set "layout_autoHeight" to false (as a boolean value, not a string) +- When editing "width", first set "layout_autoWidth" to false (as a boolean value, not a string) +- Check document metadata to identify other similar dependencies + +Example: To change document height, first disable auto-height: +1. { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } +2. { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 300 }`; // Extensive usage guidelines for the tool const citationRules = `USAGE GUIDELINES: @@ -69,22 +73,32 @@ To GET document metadata: To EDIT document metadata: - Use action="edit" with required documentId, fieldName, and fieldValue parameters -- Optionally specify documentLocation="layout"|"data"|"auto" (default is "auto") -- The tool will determine the correct document location automatically unless specified +- The tool will determine the correct document location automatically - Field names can be provided with or without leading underscores (e.g., both "width" and "_width" work) - Common fields like "width" and "height" are automatically mapped to "_width" and "_height" -- Numeric values are accepted for appropriate fields (width, height, etc.) -- The tool will apply the edit to the correct document (layout or data) +- All value types are supported: strings, numbers, and booleans +- The tool will apply the edit to the correct document (layout or data) based on existing fields + +HANDLING DEPENDENT FIELDS: +- When editing some fields, you may need to update related dependent fields +- For example, when changing "height", you should also set "layout_autoHeight" to false: + 1. First: { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } + 2. Then: { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 300 } +- Always check for dependent fields that might affect your edits, such as: + - height → layout_autoHeight (set to false to allow manual height) + - width → layout_autoWidth (set to false to allow manual width) + - Other auto-sizing related properties To LIST available documents: - Use action="list" to get a simple list of all documents in the current view - This is useful when you need to identify documents before getting details or editing them Editing fields follows these rules: -1. If documentLocation is specified, the field is modified in that location -2. If documentLocation="auto", the tool checks if the field exists on layout or data document first -3. If the field doesn't exist in either document, it's added to the location specified or layout document by default -4. Fields with leading underscores are automatically handled correctly +1. First checks if the field exists on the layout document using Doc.Get +2. If it exists on the layout document, it's updated there +3. If it has an underscore prefix (_), it's created/updated on the layout document +4. Otherwise, the field is created/updated on the data document +5. Fields with leading underscores are automatically handled correctly Examples: - To list all documents: { action: "list" } @@ -92,12 +106,15 @@ Examples: - To get metadata for a specific document: { action: "get", documentId: "doc123" } - To edit a field: { action: "edit", documentId: "doc123", fieldName: "backgroundColor", fieldValue: "#ff0000" } - To edit a width property: { action: "edit", documentId: "doc123", fieldName: "width", fieldValue: 300 } -- To edit a field in the data document: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "New content", documentLocation: "data" }`; +- To disable auto-height: { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } +- To edit height with dependent field: + { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } + { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 200 }`; const documentMetadataToolInfo: ToolInfo = { name: 'documentMetadata', description: toolDescription, - parameterRules: documentMetadataToolParams, + parameterRules: parameterDefinitions, citationRules: citationRules, }; @@ -449,16 +466,16 @@ export class DocumentMetadataTool extends BaseTool): Promise { - console.log('DocumentMetadataTool executing with args:', args); - - // Add diagnostic information about the ChatBox instance - if (this.chatBox) { - console.log('ChatBox instance available:', { - hasDocument: !!this.chatBoxDocument, - chatBoxProps: this.chatBox.props ? Object.keys(this.chatBox.props) : 'No props', - linkedDocsMethod: typeof this.chatBox.linkedDocs === 'function' ? 'Exists (function)' : - typeof this.chatBox.linkedDocs === 'object' ? 'Exists (object)' : 'Not available' - }); - } else { - console.warn('No ChatBox instance available'); - } + console.log('DocumentMetadataTool: Executing with args:', args); // Find all documents in the Freeform view this.findDocumentsInFreeformView(); - // Debug diagnostic information about all documents found - if (this.documentsById.size > 0) { - console.log('Documents found:', Array.from(this.documentsById.entries()).map(([id, doc]) => ({ - id, - title: doc.title || 'Untitled', - type: doc.type || 'Unknown', - hasLayout: !!this.layoutDocsById.get(id), - hasData: !!this.dataDocsById.get(id) - }))); - } - - // Check if we found any documents - if (this.documentsById.size === 0) { - console.error('No documents found in Freeform view'); - return [{ - type: 'text', - text: JSON.stringify({ - success: false, - message: 'No documents found in the current view. Unable to extract or edit metadata.', - chatBoxDocumentId: this.chatBoxDocument ? this.chatBoxDocument.id : 'No ChatBox document', - diagnostics: { - chatBoxAvailable: !!this.chatBox, - documentAvailable: !!this.chatBoxDocument, - toolInitializedWith: this.chatBox ? - (this.chatBox.constructor && this.chatBox.constructor.name) || 'Unknown constructor' - : 'No ChatBox' - } - }, null, 2) - }]; - } - - let result: Record; - - // Determine which action to perform - const action = args.action?.toLowerCase(); - - if (action === 'list') { - // Just return a simple list of available documents for quick reference - const documentList = Array.from(this.documentsById.entries()).map(([id, doc]) => ({ - id, - title: doc.title || 'Untitled', - type: doc.type || 'Unknown' - })); - - result = { - success: true, - message: `Found ${documentList.length} documents in the current view`, - documents: documentList - }; - } - else if (action === 'edit') { - // Edit document metadata - if (!args.documentId) { - return [{ - type: 'text', - text: JSON.stringify({ - success: false, - message: 'Document ID is required for edit operations', - availableDocuments: Array.from(this.documentsById.keys()) - }, null, 2) - }]; - } - - if (!args.fieldName) { - return [{ - type: 'text', - text: JSON.stringify({ - success: false, - message: 'Field name is required for edit operations', - }, null, 2) - }]; - } - - if (args.fieldValue === undefined) { + try { + // Validate required input parameters based on action + if (!this.inputValidator(args)) { return [{ - type: 'text', - text: JSON.stringify({ - success: false, - message: 'Field value is required for edit operations', - }, null, 2) + type: 'text', + text: `Error: Invalid or missing parameters for action "${args.action}". ${this.getParameterRequirementsByAction(String(args.action))}` }]; } - // Check if the document exists - if (!this.documentsById.has(args.documentId)) { + // Ensure the action is valid and convert to string + const action = String(args.action); + if (!['get', 'edit', 'list'].includes(action)) { return [{ - type: 'text', - text: JSON.stringify({ - success: false, - message: `Document with ID ${args.documentId} not found`, - availableDocuments: Array.from(this.documentsById.keys()) - }, null, 2) + type: 'text', + text: 'Error: Invalid action. Valid actions are "get", "edit", or "list".' }]; } + + // Safely convert documentId to string or undefined + const documentId = args.documentId ? String(args.documentId) : undefined; - // Convert fieldValue to string if it's not already - // This is to support the Agent passing numeric values directly - const fieldValue = typeof args.fieldValue === 'string' ? - args.fieldValue : - String(args.fieldValue); - - // Perform the edit - const editResult = this.editDocumentField( - args.documentId, - args.fieldName, - fieldValue, - args.documentLocation || 'auto' - ); - - // If successful, also get the updated metadata - if (editResult.success) { - const documentMetadata = this.extractDocumentMetadata(args.documentId); - result = { - ...editResult, - document: documentMetadata, - }; - } else { - result = editResult; - } - } else if (action === 'get') { - // Get document metadata - if (args.documentId) { - // Check if the document exists - if (!this.documentsById.has(args.documentId)) { + // Perform the specified action + switch (action) { + case 'get': { + // Get metadata for a specific document or all documents + const result = this.getDocumentMetadata(documentId); + console.log('DocumentMetadataTool: Get metadata result:', result); return [{ - type: 'text', - text: JSON.stringify({ - success: false, - message: `Document with ID ${args.documentId} not found`, - availableDocuments: Array.from(this.documentsById.keys()) - }, null, 2) + type: 'text', + text: `Document metadata ${documentId ? 'for document ' + documentId : ''} retrieved successfully:\n${JSON.stringify(result, null, 2)}` }]; } - // Get metadata for a specific document - const documentMetadata = this.extractDocumentMetadata(args.documentId); - result = { - success: !!documentMetadata, - message: documentMetadata ? 'Document metadata retrieved successfully' : `Document with ID ${args.documentId} not found`, - document: documentMetadata, - fieldDefinitions: this.fieldMetadata, - }; - } else { - // Get metadata for all documents - const documentsMetadata: Record = {}; - for (const docId of this.documentsById.keys()) { - documentsMetadata[docId] = this.extractDocumentMetadata(docId); + case 'edit': { + // Edit a specific field on a document + if (!documentId || !args.fieldName) { + return [{ + type: 'text', + text: 'Error: Document ID, field name, and field value are required for edit actions.' + }]; + } + + // Ensure document exists + if (!this.documentsById.has(documentId)) { + return [{ + type: 'text', + text: `Error: Document with ID ${documentId} not found.` + }]; + } + + // Get fieldValue in its original form - we'll handle conversion in editDocumentField + let fieldValue = args.fieldValue; + + // Only convert to string if it's neither boolean nor number + if (typeof fieldValue !== 'boolean' && typeof fieldValue !== 'number') { + fieldValue = String(fieldValue); + } + + const fieldName = String(args.fieldName); + + // Edit the field + const result = this.editDocumentField( + documentId, + fieldName, + fieldValue + ); + + console.log('DocumentMetadataTool: Edit field result:', result); + + if (!result.success) { + return [{ type: 'text', text: result.message }]; + } + + // Include warning if present + let responseText = result.message; + if (result.warning) { + responseText += `\n\n${result.warning}`; + } + + // Get the updated metadata to return + const updatedMetadata = this.getDocumentMetadata(documentId); + + return [{ + type: 'text', + text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}` + }]; } - result = { - success: true, - message: `Retrieved metadata for ${this.documentsById.size} documents`, - documents: documentsMetadata, - fieldDefinitions: this.fieldMetadata, - documentCount: this.documentsById.size, - fieldMetadataCount: Object.keys(this.fieldMetadata).length, - }; + case 'list': { + // List all available documents in simple format + const docs = Array.from(this.documentsById.entries()).map(([id, doc]) => ({ + id, + title: doc.title || 'Untitled Document', + type: doc.type || 'Unknown Type' + })); + + if (docs.length === 0) { + return [{ + type: 'text', + text: 'No documents found in the current view.' + }]; + } + + return [{ + type: 'text', + text: `Found ${docs.length} document(s) in the current view:\n${JSON.stringify(docs, null, 2)}` + }]; + } + + default: + return [{ + type: 'text', + text: 'Error: Unknown action. Valid actions are "get", "edit", or "list".' + }]; } - } else { - // Invalid action - result = { - success: false, - message: `Invalid action: ${action}. Valid actions are "list", "get", and "edit".`, - availableActions: ["list", "get", "edit"] - }; + } catch (error) { + console.error('DocumentMetadataTool execution error:', error); + return [{ + type: 'text', + text: `Error executing DocumentMetadataTool: ${error instanceof Error ? error.message : String(error)}` + }]; } - - // Log the result to the console - console.log('Document Metadata Tool Result:', result); - - // Return the result as an observation - // Convert to string format as the Observation type only supports 'text' or 'image_url' types - return [{ - type: 'text', - text: JSON.stringify(result, null, 2) - }]; } /** * Validates the input parameters for the DocumentMetadataTool - * This custom validator allows numbers to be passed for fieldValue while maintaining - * compatibility with the standard validation + * This custom validator allows numbers and booleans to be passed for fieldValue + * while maintaining compatibility with the standard validation * * @param params The parameters to validate * @returns True if the parameters are valid, false otherwise @@ -888,13 +847,67 @@ export class DocumentMetadataTool extends BaseTool = {}; + for (const docId of this.documentsById.keys()) { + documentsMetadata[docId] = this.extractDocumentMetadata(docId); + } + + return { + documentCount: this.documentsById.size, + documents: documentsMetadata, + fieldDefinitions: this.fieldMetadata + }; + } + } } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 6e646b38523c84f0285863865e2c929870105e2a Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Thu, 3 Apr 2025 12:30:58 -0400 Subject: better editing of texts --- .../views/nodes/chatbot/agentsystem/Agent.ts | 32 +- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 421 ++++++++++++++++++--- 2 files changed, 390 insertions(+), 63 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 0610dc198..3c76cf348 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -462,10 +462,34 @@ export class Agent { console.log(actionInput); // Special handling for documentMetadata tool with numeric or boolean fieldValue - if (action === 'documentMetadata' && 'fieldValue' in actionInput) { - if (typeof actionInput.fieldValue === 'number' || typeof actionInput.fieldValue === 'boolean') { - // Convert number or boolean to string to pass validation - actionInput.fieldValue = String(actionInput.fieldValue); + if (action === 'documentMetadata') { + // Handle single field edit + if ('fieldValue' in actionInput) { + if (typeof actionInput.fieldValue === 'number' || typeof actionInput.fieldValue === 'boolean') { + // Convert number or boolean to string to pass validation + actionInput.fieldValue = String(actionInput.fieldValue); + } + } + + // Handle fieldEdits parameter (for multiple field edits) + if ('fieldEdits' in actionInput && actionInput.fieldEdits) { + try { + // If it's already an array, stringify it to ensure it passes validation + if (Array.isArray(actionInput.fieldEdits)) { + actionInput.fieldEdits = JSON.stringify(actionInput.fieldEdits); + } + // If it's an object but not an array, it might be a single edit - convert to array and stringify + else if (typeof actionInput.fieldEdits === 'object') { + actionInput.fieldEdits = JSON.stringify([actionInput.fieldEdits]); + } + // Otherwise, ensure it's a string for the validator + else if (typeof actionInput.fieldEdits !== 'string') { + actionInput.fieldEdits = String(actionInput.fieldEdits); + } + } catch (error) { + console.error('Error processing fieldEdits:', error); + // Don't fail validation here, let the tool handle it + } } } diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 13b8315da..6354e8df3 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -15,25 +15,31 @@ const parameterDefinitions: ReadonlyArray = [ name: 'action', type: 'string', required: true, - description: 'The action to perform: "get" to retrieve metadata, "edit" to modify metadata, or "list" to enumerate documents', + description: 'The action to perform: "get" to retrieve metadata, "edit" to modify metadata, "list" to enumerate documents, or "getFieldOptions" to retrieve all available field options', }, { name: 'documentId', type: 'string', required: false, - description: 'The ID of the document to get or edit metadata for. Required for "edit", optional for "get", ignored for "list"', + description: 'The ID of the document to get or edit metadata for. Required for "edit", optional for "get", ignored for "list" and "getFieldOptions"', }, { name: 'fieldName', type: 'string', required: false, - description: 'The name of the field to edit. Required for "edit" action. Field names can be provided with or without leading underscores', + description: 'The name of the field to edit. Required for single field edits. Ignored if fieldEdits is provided', }, { name: 'fieldValue', type: 'string', required: false, - description: 'The new value for the field. Required for "edit" action. Can be a string, number, or boolean value depending on the field type', + description: 'The new value for the field. Required for single field edits. Can be a string, number, or boolean value depending on the field type', + }, + { + name: 'fieldEdits', + type: 'string', + required: false, + description: 'JSON array of field edits for editing multiple fields at once. Each item should have fieldName and fieldValue. Example: [{"fieldName":"layout_autoHeight","fieldValue":false},{"fieldName":"height","fieldValue":300}]', } ] as const; @@ -51,19 +57,34 @@ This tool provides the following capabilities: - Get metadata from all documents in the current Freeform view - Get metadata from a specific document - Edit metadata fields on documents (in either layout or data documents) +- Edit multiple fields at once (useful for updating dependent fields together) - List all available documents in the current view +- Retrieve all available field options with metadata (IMPORTANT: always call this before editing) - Understand which fields are stored where (layout vs data document) - Get detailed information about all available document fields - Support for all value types: strings, numbers, and booleans +IMPORTANT: Before editing any document metadata, first call 'getFieldOptions' to understand: +- Which fields are available +- The data type of each field +- Special dependencies between fields (like layout_autoHeight and height) +- Proper naming conventions (with or without underscores) + IMPORTANT: Some fields have dependencies that must be handled for edits to work correctly: - When editing "height", first set "layout_autoHeight" to false (as a boolean value, not a string) - When editing "width", first set "layout_autoWidth" to false (as a boolean value, not a string) - Check document metadata to identify other similar dependencies +- You can edit dependent fields in a single operation using the fieldEdits parameter Example: To change document height, first disable auto-height: 1. { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } -2. { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 300 }`; +2. { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 300 } + +OR using multi-field edit (recommended for dependent fields): +{ action: "edit", documentId: "doc123", fieldEdits: [ + { fieldName: "layout_autoHeight", fieldValue: false }, + { fieldName: "height", fieldValue: 300 } +]}`; // Extensive usage guidelines for the tool const citationRules = `USAGE GUIDELINES: @@ -71,19 +92,37 @@ To GET document metadata: - Use action="get" with optional documentId to return metadata for one or all documents - Returns field values, field definitions, and location information (layout vs data document) +To GET ALL FIELD OPTIONS (call this first): +- Use action="getFieldOptions" to retrieve metadata about all available document fields +- No additional parameters are required +- Returns structured metadata with field names, types, descriptions, and dependencies +- ALWAYS call this before attempting to edit document metadata +- Use this information to understand which fields need special handling + To EDIT document metadata: -- Use action="edit" with required documentId, fieldName, and fieldValue parameters +- Use action="edit" with required documentId, and either: + 1. fieldName + fieldValue for single field edits, OR + 2. fieldEdits for updating multiple fields at once - The tool will determine the correct document location automatically - Field names can be provided with or without leading underscores (e.g., both "width" and "_width" work) - Common fields like "width" and "height" are automatically mapped to "_width" and "_height" - All value types are supported: strings, numbers, and booleans - The tool will apply the edit to the correct document (layout or data) based on existing fields +RECOMMENDED WORKFLOW: +1. First call action="list" to identify available documents +2. Then call action="getFieldOptions" to understand available fields +3. Get document metadata with action="get" to see current values +4. Edit fields with action="edit" using proper dependencies + HANDLING DEPENDENT FIELDS: - When editing some fields, you may need to update related dependent fields -- For example, when changing "height", you should also set "layout_autoHeight" to false: - 1. First: { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } - 2. Then: { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 300 } +- For example, when changing "height", you should also set "layout_autoHeight" to false +- Use the fieldEdits parameter to update dependent fields in a single operation (recommended): + { action: "edit", documentId: "doc123", fieldEdits: [ + { fieldName: "layout_autoHeight", fieldValue: false }, + { fieldName: "height", fieldValue: 300 } + ]} - Always check for dependent fields that might affect your edits, such as: - height → layout_autoHeight (set to false to allow manual height) - width → layout_autoWidth (set to false to allow manual width) @@ -101,15 +140,18 @@ Editing fields follows these rules: 5. Fields with leading underscores are automatically handled correctly Examples: +- To get field options: { action: "getFieldOptions" } - To list all documents: { action: "list" } - To get all document metadata: { action: "get" } - To get metadata for a specific document: { action: "get", documentId: "doc123" } -- To edit a field: { action: "edit", documentId: "doc123", fieldName: "backgroundColor", fieldValue: "#ff0000" } +- To edit a single field: { action: "edit", documentId: "doc123", fieldName: "backgroundColor", fieldValue: "#ff0000" } - To edit a width property: { action: "edit", documentId: "doc123", fieldName: "width", fieldValue: 300 } - To disable auto-height: { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } -- To edit height with dependent field: - { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } - { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 200 }`; +- To edit height with its dependent field together (recommended): + { action: "edit", documentId: "doc123", fieldEdits: [ + { fieldName: "layout_autoHeight", fieldValue: false }, + { fieldName: "height", fieldValue: 200 } + ]}`; const documentMetadataToolInfo: ToolInfo = { name: 'documentMetadata', @@ -686,6 +728,112 @@ export class DocumentMetadataTool extends BaseTool = { + fieldCount: Object.keys(this.fieldMetadata).length, + fields: {}, + fieldsByType: { + string: [], + number: [], + boolean: [], + doc: [], + list: [], + date: [], + enumeration: [], + other: [] + }, + fieldNameMappings: {}, + commonFields: { + appearance: [], + position: [], + size: [], + content: [], + behavior: [], + layout: [] + } + }; + + // Process each field in the metadata + Object.entries(this.fieldMetadata).forEach(([fieldName, fieldInfo]) => { + const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Add to fieldNameMappings + if (fieldName.startsWith('_')) { + result.fieldNameMappings[strippedName] = fieldName; + } + + // Create structured field metadata + const fieldData: Record = { + name: fieldName, + displayName: strippedName, + description: fieldInfo.description || '', + type: fieldInfo.fieldType || 'unknown', + possibleValues: fieldInfo.values || [], + }; + + // Add field to fields collection + result.fields[fieldName] = fieldData; + + // Categorize by field type + const type = fieldInfo.fieldType?.toLowerCase() || 'unknown'; + if (type === 'string') { + result.fieldsByType.string.push(fieldName); + } else if (type === 'number') { + result.fieldsByType.number.push(fieldName); + } else if (type === 'boolean') { + result.fieldsByType.boolean.push(fieldName); + } else if (type === 'doc') { + result.fieldsByType.doc.push(fieldName); + } else if (type === 'list') { + result.fieldsByType.list.push(fieldName); + } else if (type === 'date') { + result.fieldsByType.date.push(fieldName); + } else if (type === 'enumeration') { + result.fieldsByType.enumeration.push(fieldName); + } else { + result.fieldsByType.other.push(fieldName); + } + + // Categorize by field purpose + if (fieldName.includes('width') || fieldName.includes('height') || fieldName.includes('size')) { + result.commonFields.size.push(fieldName); + } else if (fieldName.includes('color') || fieldName.includes('background') || fieldName.includes('border')) { + result.commonFields.appearance.push(fieldName); + } else if (fieldName.includes('x') || fieldName.includes('y') || fieldName.includes('position') || fieldName.includes('pan')) { + result.commonFields.position.push(fieldName); + } else if (fieldName.includes('text') || fieldName.includes('title') || fieldName.includes('data')) { + result.commonFields.content.push(fieldName); + } else if (fieldName.includes('action') || fieldName.includes('click') || fieldName.includes('event')) { + result.commonFields.behavior.push(fieldName); + } else if (fieldName.includes('layout')) { + result.commonFields.layout.push(fieldName); + } + }); + + // Add special section for auto-sizing related fields + result.autoSizingFields = { + height: { + autoHeightField: '_layout_autoHeight', + heightField: '_height', + displayName: 'height', + usage: 'To manually set height, first set layout_autoHeight to false' + }, + width: { + autoWidthField: '_layout_autoWidth', + widthField: '_width', + displayName: 'width', + usage: 'To manually set width, first set layout_autoWidth to false' + } + }; + + return result; + } + /** * Executes the document metadata tool * @param args The arguments for the tool @@ -708,10 +856,10 @@ export class DocumentMetadataTool extends BaseTool { + responseText += `- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}\n`; + + // Add any warnings + if (result.warning) { + responseText += ` Warning: ${result.warning}\n`; + } + }); + } else { + responseText = `Errors occurred while editing fields on document ${documentId}:\n`; + results.forEach(result => { + if (result.success) { + responseText += `- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}\n`; + + // Add any warnings + if (result.warning) { + responseText += ` Warning: ${result.warning}\n`; + } + } else { + responseText += `- Error editing '${result.originalFieldName}': ${result.message}\n`; + } + }); + } + + // Get the updated metadata to return + const updatedMetadata = this.getDocumentMetadata(documentId); + + return [{ + type: 'text', + text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}` + }]; + } catch (error) { + return [{ + type: 'text', + text: `Error processing fieldEdits: ${error instanceof Error ? error.message : String(error)}` + }]; + } + } else { + // Single field edit (original behavior) + if (!args.fieldName) { + return [{ + type: 'text', + text: 'Error: Field name and field value are required for edit actions.' + }]; + } + + // Get fieldValue in its original form - we'll handle conversion in editDocumentField + let fieldValue = args.fieldValue; + + // Only convert to string if it's neither boolean nor number + if (typeof fieldValue !== 'boolean' && typeof fieldValue !== 'number') { + fieldValue = String(fieldValue); + } + + const fieldName = String(args.fieldName); + + // Edit the field + const result = this.editDocumentField( + documentId, + fieldName, + fieldValue + ); + + console.log('DocumentMetadataTool: Edit field result:', result); + + if (!result.success) { + return [{ type: 'text', text: result.message }]; + } + + // Include warning if present + let responseText = result.message; + if (result.warning) { + responseText += `\n\n${result.warning}`; + } + + // Get the updated metadata to return + const updatedMetadata = this.getDocumentMetadata(documentId); + + return [{ + type: 'text', + text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}` + }]; } - - // Get the updated metadata to return - const updatedMetadata = this.getDocumentMetadata(documentId); - - return [{ - type: 'text', - text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}` - }]; } case 'list': { @@ -806,10 +1059,20 @@ export class DocumentMetadataTool extends BaseTool Date: Thu, 3 Apr 2025 12:46:22 -0400 Subject: better text editing? --- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 80 ++++++++++++++++++++++ 1 file changed, 80 insertions(+) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 6354e8df3..093ab248d 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -109,6 +109,12 @@ To EDIT document metadata: - All value types are supported: strings, numbers, and booleans - The tool will apply the edit to the correct document (layout or data) based on existing fields +SPECIAL FIELD HANDLING: +- Text fields: When editing the 'text' field, provide simple plain text + Example: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "Hello world" } + The tool will automatically convert your text to the proper RichTextField format +- Width/Height: Set layout_autoHeight/layout_autoWidth to false before editing + RECOMMENDED WORKFLOW: 1. First call action="list" to identify available documents 2. Then call action="getFieldOptions" to understand available fields @@ -146,6 +152,7 @@ Examples: - To get metadata for a specific document: { action: "get", documentId: "doc123" } - To edit a single field: { action: "edit", documentId: "doc123", fieldName: "backgroundColor", fieldValue: "#ff0000" } - To edit a width property: { action: "edit", documentId: "doc123", fieldName: "width", fieldValue: 300 } +- To edit text content: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "Simple plain text goes here" } - To disable auto-height: { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } - To edit height with its dependent field together (recommended): { action: "edit", documentId: "doc123", fieldEdits: [ @@ -634,6 +641,34 @@ export class DocumentMetadataTool extends BaseTool { + if (node.text) { + plainText += node.text; + } + if (node.content && Array.isArray(node.content)) { + node.content.forEach((child: any) => extractText(child)); + } + }; + + extractText(rtfObj.doc); + + // If we successfully extracted text, show it, but also preserve the original value + if (plainText) { + return { + type: 'RichText', + text: plainText, + length: plainText.length, + // Don't include the full value as it can be very large + }; + } + } + } catch (e) { + // If parsing fails, just treat as a regular string + } + } + // Handle arrays and complex objects if (typeof value === 'object') { // If the object has a toString method, use it @@ -831,6 +900,17 @@ export class DocumentMetadataTool extends BaseTool Date: Fri, 4 Apr 2025 11:38:42 -0400 Subject: feat: create and edit are unified --- .../views/nodes/chatbot/agentsystem/Agent.ts | 2 +- .../nodes/chatbot/tools/CreateDocumentTool.ts | 2 +- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 165 +++++++++++++++++++-- 3 files changed, 158 insertions(+), 11 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 3c76cf348..bab2081b3 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -87,7 +87,7 @@ export class Agent { noTool: new NoTool(), imageCreationTool: new ImageCreationTool(createImage), // createTextDoc: new CreateTextDocTool(addLinkedDoc), - createDoc: new CreateDocTool(addLinkedDoc), + // createDoc: new CreateDocTool(addLinkedDoc), // createAnyDocument: new CreateAnyDocumentTool(addLinkedDoc), // dictionary: new DictionaryTool(), documentMetadata: new DocumentMetadataTool(this), diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts index 284879a4a..b3bf6dfb2 100644 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts @@ -9,7 +9,7 @@ import { CollectionViewType } from '../../../../documents/DocumentTypes'; */ export enum supportedDocTypes { flashcard = 'flashcard', - text = 'text', + note = 'note', html = 'html', equation = 'equation', functionplot = 'functionplot', diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 093ab248d..c74e502e7 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -8,6 +8,8 @@ import { CollectionFreeFormDocumentView } from '../../../nodes/CollectionFreeFor import { v4 as uuidv4 } from 'uuid'; import { LinkManager } from '../../../../util/LinkManager'; import { DocCast, StrCast } from '../../../../../fields/Types'; +import { supportedDocTypes } from './CreateDocumentTool'; +import { parsedDoc } from '../chatboxcomponents/ChatBox'; // Define the parameters for the DocumentMetadataTool const parameterDefinitions: ReadonlyArray = [ @@ -15,13 +17,13 @@ const parameterDefinitions: ReadonlyArray = [ name: 'action', type: 'string', required: true, - description: 'The action to perform: "get" to retrieve metadata, "edit" to modify metadata, "list" to enumerate documents, or "getFieldOptions" to retrieve all available field options', + description: 'The action to perform: "get" to retrieve metadata, "edit" to modify metadata, "list" to enumerate documents, "getFieldOptions" to retrieve all available field options, or "create" to create a new document', }, { name: 'documentId', type: 'string', required: false, - description: 'The ID of the document to get or edit metadata for. Required for "edit", optional for "get", ignored for "list" and "getFieldOptions"', + description: 'The ID of the document to get or edit metadata for. Required for "edit", optional for "get", ignored for "list", "getFieldOptions", and "create"', }, { name: 'fieldName', @@ -40,14 +42,32 @@ const parameterDefinitions: ReadonlyArray = [ type: 'string', required: false, description: 'JSON array of field edits for editing multiple fields at once. Each item should have fieldName and fieldValue. Example: [{"fieldName":"layout_autoHeight","fieldValue":false},{"fieldName":"height","fieldValue":300}]', + }, + { + name: 'title', + type: 'string', + required: false, + description: 'The title of the document to create. Required for "create" action', + }, + { + name: 'data', + type: 'string', + required: false, + description: 'The data content for the document to create. Required for "create" action', + }, + { + name: 'doc_type', + type: 'string', + required: false, + description: `The type of document to create. Required for "create" action. Options: ${Object.keys(supportedDocTypes).join(',')}`, } ] as const; type DocumentMetadataToolParamsType = typeof parameterDefinitions; // Detailed description with usage guidelines for the DocumentMetadataTool -const toolDescription = `Extracts and modifies metadata from documents in the same Freeform view as the ChatBox. -This tool helps you work with document properties, understand available fields, and edit document metadata. +const toolDescription = `Extracts and modifies metadata from documents in the same Freeform view as the ChatBox, and can create new documents. +This tool helps you work with document properties, understand available fields, edit document metadata, and create new documents. The Dash document system organizes fields in two locations: 1. Layout documents: contain visual properties like position, dimensions, and appearance @@ -63,6 +83,13 @@ This tool provides the following capabilities: - Understand which fields are stored where (layout vs data document) - Get detailed information about all available document fields - Support for all value types: strings, numbers, and booleans +- Create new documents with basic properties + +DOCUMENT CREATION: +- Use action="create" to create new documents with a simplified approach +- Required parameters: title, data, and doc_type +- The tool will create the document with sensible defaults and link it to the current view +- After creation, you can use the edit action to update its properties IMPORTANT: Before editing any document metadata, first call 'getFieldOptions' to understand: - Which fields are available @@ -99,6 +126,14 @@ To GET ALL FIELD OPTIONS (call this first): - ALWAYS call this before attempting to edit document metadata - Use this information to understand which fields need special handling +To CREATE a new document: +- Use action="create" with the following required parameters: + - title: The title of the document to create + - data: The content data for the document (text content, URL, etc.) + - doc_type: The type of document to create (text, web, image, etc.) +- Example: { action: "create", title: "My Notes", data: "This is the content", doc_type: "text" } +- After creation, you can edit the document with more specific properties + To EDIT document metadata: - Use action="edit" with required documentId, and either: 1. fieldName + fieldValue for single field edits, OR @@ -120,6 +155,10 @@ RECOMMENDED WORKFLOW: 2. Then call action="getFieldOptions" to understand available fields 3. Get document metadata with action="get" to see current values 4. Edit fields with action="edit" using proper dependencies +OR +1. Create a new document with action="create" +2. Get its ID from the response +3. Edit the document's properties with action="edit" HANDLING DEPENDENT FIELDS: - When editing some fields, you may need to update related dependent fields @@ -154,6 +193,8 @@ Examples: - To edit a width property: { action: "edit", documentId: "doc123", fieldName: "width", fieldValue: 300 } - To edit text content: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "Simple plain text goes here" } - To disable auto-height: { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } +- To create a text document: { action: "create", title: "My Notes", data: "This is my note content", doc_type: "text" } +- To create a web document: { action: "create", title: "Google", data: "https://www.google.com", doc_type: "web" } - To edit height with its dependent field together (recommended): { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, @@ -936,10 +977,10 @@ export class DocumentMetadataTool extends BaseTool Date: Fri, 4 Apr 2025 11:43:58 -0400 Subject: specifying note document and removing unnecessary tools --- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 4 +- .../nodes/chatbot/tools/CreateDocumentTool.ts | 497 --------------------- .../nodes/chatbot/tools/CreateTextDocumentTool.ts | 57 --- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 3 +- src/client/views/nodes/chatbot/types/tool_types.ts | 26 ++ 5 files changed, 30 insertions(+), 557 deletions(-) delete mode 100644 src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts delete mode 100644 src/client/views/nodes/chatbot/tools/CreateTextDocumentTool.ts (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 065c2780c..3152fe923 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -35,7 +35,7 @@ import { PDFBox } from '../../PDFBox'; import { ScriptingBox } from '../../ScriptingBox'; import { VideoBox } from '../../VideoBox'; import { Agent } from '../agentsystem/Agent'; -import { supportedDocTypes } from '../tools/CreateDocumentTool'; +import { supportedDocTypes } from '../types/tool_types'; import { ASSISTANT_ROLE, AssistantMessage, CHUNK_TYPE, Citation, ProcessingInfo, SimplifiedChunk, TEXT_TYPE } from '../types/types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import './ChatBox.scss'; @@ -461,7 +461,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent() { const ndoc = (() => { switch (doc.doc_type) { default: - case supportedDocTypes.text: return Docs.Create.TextDocument(data as string, options); + case supportedDocTypes.note: return Docs.Create.TextDocument(data as string, options); case supportedDocTypes.comparison: return this.createComparison(JSON.parse(data as string) as parsedDoc[], options); case supportedDocTypes.flashcard: return this.createFlashcard(JSON.parse(data as string) as parsedDoc[], options); case supportedDocTypes.deck: return this.createDeck(JSON.parse(data as string) as parsedDoc[], options); diff --git a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts deleted file mode 100644 index b3bf6dfb2..000000000 --- a/src/client/views/nodes/chatbot/tools/CreateDocumentTool.ts +++ /dev/null @@ -1,497 +0,0 @@ -import { BaseTool } from './BaseTool'; -import { Observation } from '../types/types'; -import { Parameter, ParametersType, ToolInfo } from '../types/tool_types'; -import { parsedDoc } from '../chatboxcomponents/ChatBox'; -import { CollectionViewType } from '../../../../documents/DocumentTypes'; - -/** - * List of supported document types that can be created via text LLM. - */ -export enum supportedDocTypes { - flashcard = 'flashcard', - note = 'note', - html = 'html', - equation = 'equation', - functionplot = 'functionplot', - dataviz = 'dataviz', - notetaking = 'notetaking', - audio = 'audio', - video = 'video', - pdf = 'pdf', - rtf = 'rtf', - message = 'message', - collection = 'collection', - image = 'image', - deck = 'deck', - web = 'web', - comparison = 'comparison', - diagram = 'diagram', - script = 'script', -} -/** - * Tthe CreateDocTool class is responsible for creating - * documents of various types (e.g., text, flashcards, collections) and organizing them in a - * structured manner. The tool supports creating dashboards with diverse document types and - * ensures proper placement of documents without overlap. - */ - -// Example document structure for various document types -const example = [ - { - doc_type: supportedDocTypes.equation, - title: 'quadratic', - data: 'x^2 + y^2 = 3', - _width: 300, - _height: 300, - x: 0, - y: 0, - }, - { - doc_type: supportedDocTypes.collection, - title: 'Advanced Biology', - data: [ - { - doc_type: supportedDocTypes.text, - title: 'Cell Structure', - data: 'Cells are the basic building blocks of all living organisms.', - _width: 300, - _height: 300, - x: 500, - y: 0, - }, - ], - backgroundColor: '#00ff00', - _width: 600, - _height: 600, - x: 600, - y: 0, - type_collection: 'tree', - }, - { - doc_type: supportedDocTypes.image, - title: 'experiment', - data: 'https://plus.unsplash.com/premium_photo-1694819488591-a43907d1c5cc?q=80&w=2628&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D', - _width: 300, - _height: 300, - x: 600, - y: 300, - }, - { - doc_type: supportedDocTypes.deck, - title: 'Chemistry', - data: [ - { - doc_type: supportedDocTypes.flashcard, - title: 'Photosynthesis', - data: [ - { - doc_type: supportedDocTypes.text, - title: 'front_Photosynthesis', - data: 'What is photosynthesis?', - _width: 300, - _height: 300, - x: 100, - y: 600, - }, - { - doc_type: supportedDocTypes.text, - title: 'back_photosynthesis', - data: 'The process by which plants make food.', - _width: 300, - _height: 300, - x: 100, - y: 700, - }, - ], - backgroundColor: '#00ff00', - _width: 300, - _height: 300, - x: 300, - y: 1000, - }, - { - doc_type: supportedDocTypes.flashcard, - title: 'Photosynthesis', - data: [ - { - doc_type: supportedDocTypes.text, - title: 'front_Photosynthesis', - data: 'What is photosynthesis?', - _width: 300, - _height: 300, - x: 200, - y: 800, - }, - { - doc_type: supportedDocTypes.text, - title: 'back_photosynthesis', - data: 'The process by which plants make food.', - _width: 300, - _height: 300, - x: 100, - y: -100, - }, - ], - backgroundColor: '#00ff00', - _width: 300, - _height: 300, - x: 10, - y: 70, - }, - ], - backgroundColor: '#00ff00', - _width: 600, - _height: 600, - x: 200, - y: 800, - }, - { - doc_type: supportedDocTypes.web, - title: 'Brown University Wikipedia', - data: 'https://en.wikipedia.org/wiki/Brown_University', - _width: 300, - _height: 300, - x: 1000, - y: 2000, - }, - { - doc_type: supportedDocTypes.comparison, - title: 'WWI vs. WWII', - data: [ - { - doc_type: supportedDocTypes.text, - title: 'WWI', - data: 'From 1914 to 1918, fighting took place across several continents, at sea and, for the first time, in the air.', - _width: 300, - _height: 300, - x: 100, - y: 100, - }, - { - doc_type: supportedDocTypes.text, - title: 'WWII', - data: 'A devastating global conflict spanning from 1939 to 1945, saw the Allied powers fight against the Axis powers.', - _width: 300, - _height: 300, - x: 100, - y: 100, - }, - ], - _width: 300, - _height: 300, - x: 100, - y: 100, - }, - { - doc_type: supportedDocTypes.collection, - title: 'Science Collection', - data: [ - { - doc_type: supportedDocTypes.flashcard, - title: 'Photosynthesis', - data: [ - { - doc_type: supportedDocTypes.text, - title: 'front_Photosynthesis', - data: 'What is photosynthesis?', - _width: 300, - _height: 300, - }, - { - doc_type: supportedDocTypes.text, - title: 'back_photosynthesis', - data: 'The process by which plants make food.', - _width: 300, - _height: 300, - }, - ], - backgroundColor: '#00ff00', - _width: 300, - _height: 300, - }, - { - doc_type: supportedDocTypes.web, - title: 'Brown University Wikipedia', - data: 'https://en.wikipedia.org/wiki/Brown_University', - _width: 300, - _height: 300, - x: 1100, - y: 1100, - }, - { - doc_type: supportedDocTypes.text, - title: 'Water Cycle', - data: 'The continuous movement of water on, above, and below the Earth’s surface.', - _width: 300, - _height: 300, - x: 1500, - y: 500, - }, - { - doc_type: supportedDocTypes.collection, - title: 'Advanced Biology', - data: [ - { - doc_type: 'text', - title: 'Cell Structure', - data: 'Cells are the basic building blocks of all living organisms.', - _width: 300, - _height: 300, - }, - ], - backgroundColor: '#00ff00', - _width: 600, - _height: 600, - x: 1100, - y: 500, - type_collection: 'stacking', - }, - ], - _width: 600, - _height: 600, - x: 500, - y: 500, - type_collection: 'carousel', - }, -]; - -// Stringify the entire structure for transmission if needed -const finalJsonString = JSON.stringify(example); - -const standardOptions = ['title', 'backgroundColor']; -/** - * Description of document options and data field for each type. - */ -const documentTypesInfo: { [key in supportedDocTypes]: { options: string[]; dataDescription: string } } = { - comparison: { - options: [...standardOptions, 'fontColor', 'text_align'], - dataDescription: 'an array of two documents of any kind that can be compared.', - }, - deck: { - options: [...standardOptions, 'fontColor', 'text_align'], - dataDescription: 'an array of flashcard docs', - }, - flashcard: { - options: [...standardOptions, 'fontColor', 'text_align'], - dataDescription: 'an array of two strings. the first string contains a question, and the second string contains an answer', - }, - text: { - options: [...standardOptions, 'fontColor', 'text_align'], - dataDescription: 'The text content of the document.', - }, - web: { - options: [], - dataDescription: 'A URL to a webpage. Example: https://en.wikipedia.org/wiki/Brown_University', - }, - html: { - options: [], - dataDescription: 'The HTML-formatted text content of the document.', - }, - equation: { - options: [...standardOptions, 'fontColor'], - dataDescription: 'The equation content represented as a MathML string.', - }, - functionplot: { - options: [...standardOptions, 'function_definition'], - dataDescription: 'The function definition(s) for plotting. Provide as a string or array of function definitions.', - }, - dataviz: { - options: [...standardOptions, 'chartType'], - dataDescription: 'A string of comma-separated values representing the CSV data.', - }, - notetaking: { - options: standardOptions, - dataDescription: 'An array of related text documents with small amounts of text.', - }, - rtf: { - options: standardOptions, - dataDescription: 'The rich text content in RTF format.', - }, - image: { - options: standardOptions, - dataDescription: `A url string that must end with '.png', '.jpeg', '.gif', or '.jpg'`, - }, - pdf: { - options: standardOptions, - dataDescription: 'the pdf content as a PDF file url.', - }, - audio: { - options: standardOptions, - dataDescription: 'The audio content as a file url.', - }, - video: { - options: standardOptions, - dataDescription: 'The video content as a file url.', - }, - message: { - options: standardOptions, - dataDescription: 'The message content of the document.', - }, - diagram: { - options: standardOptions, - dataDescription: 'diagram content as a text string in Mermaid format.', - }, - script: { - options: standardOptions, - dataDescription: 'The compilable JavaScript code. Use this for creating scripts.', - }, - collection: { - options: [...standardOptions, 'type_collection'], - dataDescription: 'A collection of Docs represented as an array.', - }, -}; - -// Parameters for creating individual documents -const createDocToolParams: { name: string; type: 'string' | 'number' | 'boolean' | 'string[]' | 'number[]'; description: string; required: boolean }[] = [ - { - name: 'data', - type: 'string', // Accepts either string or array, supporting individual and nested data - description: - 'the data that describes the Document contents. For collections this is an' + - `Array of documents in stringified JSON format. Each item in the array should be an individual stringified JSON object. ` + - `Creates any type of document with the provided options and data. Supported document types are: ${Object.keys(documentTypesInfo).join(', ')}. - dataviz is a csv table tool, so for CSVs, use dataviz. Here are the options for each type: - ` + - Object.entries(documentTypesInfo) - .map( - ([doc_type, info]) => - ` - ${info.dataDescription} - ` + - info.options.map(option => ``).join('\n') + - ` - - ` - ) - .join('\n') + - ` An example of the structure of a collection is:` + - finalJsonString, // prettier-ignore, - required: true, - }, - { - name: 'doc_type', - type: 'string', - description: `The type of the document. Options: ${Object.keys(documentTypesInfo).join(',')}.`, - required: true, - }, - { - name: 'title', - type: 'string', - description: 'The title of the document.', - required: true, - }, - { - name: 'x', - type: 'number', - description: 'The x location of the document; 0 <= x.', - required: true, - }, - { - name: 'y', - type: 'number', - description: 'The y location of the document; 0 <= y.', - required: true, - }, - { - name: 'backgroundColor', - type: 'string', - description: 'The background color of the document as a hex string.', - required: false, - }, - { - name: 'fontColor', - type: 'string', - description: 'The font color of the document as a hex string.', - required: false, - }, - { - name: '_width', - type: 'number', - description: 'The width of the document in pixels.', - required: true, - }, - { - name: '_height', - type: 'number', - description: 'The height of the document in pixels.', - required: true, - }, - { - name: 'type_collection', - type: 'string', - description: `the visual style for a collection doc. Options include: ${Object.values(CollectionViewType).join(',')}.`, - required: false, - }, -] as const; - -type CreateDocToolParamsType = typeof createDocToolParams; - -const createDocToolInfo: ToolInfo = { - name: 'createDoc', - description: `Creates one or more documents that best fit the user’s request. - If the user requests a "dashboard," first call the search tool and then generate a variety of document types individually, with absolutely a minimum of 20 documents - with two stacks of flashcards that are small and it should have a couple nested freeform collections of things, each with different content and color schemes. - For example, create multiple individual documents, including ${Object.keys(documentTypesInfo) - .map(t => '"' + t + '"') - .join(',')} - If the "doc_type" parameter is missing, set it to an empty string (""). - Use Decks instead of Flashcards for dashboards. Decks should have at least three flashcards. - Really think about what documents are useful to the user. If they ask for a dashboard about the skeletal system, include flashcards, as they would be helpful. - Arrange the documents in a grid layout, ensuring that the x and y coordinates are calculated so no documents overlap but they should be directly next to each other with 20 padding in between. - Take into account the width and height of each document, spacing them appropriately to prevent collisions. - Use a systematic approach, such as placing each document in a grid cell based on its order, where cell dimensions match the document dimensions plus a fixed margin for spacing. - Do not nest all documents within a single collection unless explicitly requested by the user. - Instead, create a set of independent documents with diverse document types. Each type should appear separately unless specified otherwise. - Use the "data" parameter for document content and include title, color, and document dimensions. - Ensure web documents use URLs from the search tool if relevant. Each document in a dashboard should be unique and well-differentiated in type and content, - without repetition of similar types in any single collection. - When creating a dashboard, ensure that it consists of a broad range of document types. - Include a variety of documents, such as text, web, deck, comparison, image, and equation documents, - each with distinct titles and colors, following the user’s preferences. - Do not overuse collections or nest all document types within a single collection; instead, represent document types individually. Use this example for reference: - ${finalJsonString} . - Which documents are created should be random with different numbers of each document type and different for each dashboard. - Must use search tool before creating a dashboard.`, - parameterRules: createDocToolParams, - citationRules: 'No citation needed.', -}; - -// Tool class for creating documents -export class CreateDocTool extends BaseTool< - { - name: string; - type: 'string' | 'number' | 'boolean' | 'string[]' | 'number[]'; - description: string; - required: boolean; - }[] -> { - private _addLinkedDoc: (doc: parsedDoc) => void; - - constructor(addLinkedDoc: (doc: parsedDoc) => void) { - super(createDocToolInfo); - this._addLinkedDoc = addLinkedDoc; - } - - override inputValidator(inputParam: ParametersType) { - return !!inputParam.data; - } - // Executes the tool logic for creating documents - async execute( - args: ParametersType< - { - name: 'string'; - type: 'string' | 'number' | 'boolean' | 'string[]' | 'number[]'; - description: 'string'; - required: boolean; - }[] - > - ): Promise { - try { - const parsedDocs = args instanceof Array ? args : Object.keys(args).length === 1 && 'data' in args ? JSON.parse(args.data as string) : [args]; - parsedDocs.forEach((pdoc: parsedDoc) => this._addLinkedDoc({ ...pdoc, _layout_fitWidth: false, _layout_autoHeight: true })); - return [{ type: 'text', text: 'Created document.' }]; - } catch (error) { - return [{ type: 'text', text: 'Error creating text document, ' + error }]; - } - } -} diff --git a/src/client/views/nodes/chatbot/tools/CreateTextDocumentTool.ts b/src/client/views/nodes/chatbot/tools/CreateTextDocumentTool.ts deleted file mode 100644 index 16dc938bb..000000000 --- a/src/client/views/nodes/chatbot/tools/CreateTextDocumentTool.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { parsedDoc } from '../chatboxcomponents/ChatBox'; -import { ParametersType, ToolInfo } from '../types/tool_types'; -import { Observation } from '../types/types'; -import { BaseTool } from './BaseTool'; -const createTextDocToolParams = [ - { - name: 'text_content', - type: 'string', - description: 'The text content that the document will display', - required: true, - }, - { - name: 'title', - type: 'string', - description: 'The title of the document', - required: true, - }, - // { - // name: 'background_color', - // type: 'string', - // description: 'The background color of the document as a hex string', - // required: false, - // }, - // { - // name: 'font_color', - // type: 'string', - // description: 'The font color of the document as a hex string', - // required: false, - // }, -] as const; - -type CreateTextDocToolParamsType = typeof createTextDocToolParams; - -const createTextDocToolInfo: ToolInfo = { - name: 'createTextDoc', - description: 'Creates a text document with the provided content and title. Use if the user wants to create a textbox or text document of some sort. Can use after a search or other tool to save information.', - citationRules: 'No citation needed.', - parameterRules: createTextDocToolParams, -}; - -export class CreateTextDocTool extends BaseTool { - private _addLinkedDoc: (doc: parsedDoc) => void; - - constructor(addLinkedDoc: (doc: parsedDoc) => void) { - super(createTextDocToolInfo); - this._addLinkedDoc = addLinkedDoc; - } - - async execute(args: ParametersType): Promise { - try { - this._addLinkedDoc({ doc_type: 'text', data: args.text_content, title: args.title }); - return [{ type: 'text', text: 'Created text document.' }]; - } catch (error) { - return [{ type: 'text', text: 'Error creating text document, ' + error }]; - } - } -} diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index c74e502e7..a9fb45b5a 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -8,9 +8,10 @@ import { CollectionFreeFormDocumentView } from '../../../nodes/CollectionFreeFor import { v4 as uuidv4 } from 'uuid'; import { LinkManager } from '../../../../util/LinkManager'; import { DocCast, StrCast } from '../../../../../fields/Types'; -import { supportedDocTypes } from './CreateDocumentTool'; +import { supportedDocTypes } from '../types/tool_types'; import { parsedDoc } from '../chatboxcomponents/ChatBox'; + // Define the parameters for the DocumentMetadataTool const parameterDefinitions: ReadonlyArray = [ { diff --git a/src/client/views/nodes/chatbot/types/tool_types.ts b/src/client/views/nodes/chatbot/types/tool_types.ts index 6ae48992d..6a0b5e708 100644 --- a/src/client/views/nodes/chatbot/types/tool_types.ts +++ b/src/client/views/nodes/chatbot/types/tool_types.ts @@ -50,3 +50,29 @@ export type ParamType

= P['type'] extends keyof TypeMap ? T export type ParametersType

> = { [K in P[number] as K['name']]: ParamType; }; + + +/** + * List of supported document types that can be created via text LLM. + */ +export enum supportedDocTypes { + flashcard = 'flashcard', + note = 'note', + html = 'html', + equation = 'equation', + functionplot = 'functionplot', + dataviz = 'dataviz', + notetaking = 'notetaking', + audio = 'audio', + video = 'video', + pdf = 'pdf', + rtf = 'rtf', + message = 'message', + collection = 'collection', + image = 'image', + deck = 'deck', + web = 'web', + comparison = 'comparison', + diagram = 'diagram', + script = 'script', +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 6e8f05495dfcf7d64bdc424503f874fece85e291 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Thu, 10 Apr 2025 12:36:07 -0400 Subject: trying again --- .../views/nodes/chatbot/agentsystem/Agent.ts | 14 +- src/client/views/nodes/chatbot/guides/guide.md | 647 +++++++++++++++++++++ .../views/nodes/chatbot/tools/CreateAnyDocTool.ts | 158 ----- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 561 +++++++++--------- src/server/chunker/pdf_chunker.py | 5 +- 5 files changed, 952 insertions(+), 433 deletions(-) create mode 100644 src/client/views/nodes/chatbot/guides/guide.md delete mode 100644 src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index bab2081b3..b166254bb 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -8,7 +8,6 @@ import { StreamedAnswerParser } from '../response_parsers/StreamedAnswerParser'; import { BaseTool } from '../tools/BaseTool'; import { CalculateTool } from '../tools/CalculateTool'; //import { CreateAnyDocumentTool } from '../tools/CreateAnyDocTool'; -import { CreateDocTool } from '../tools/CreateDocumentTool'; import { DataAnalysisTool } from '../tools/DataAnalysisTool'; import { DocumentMetadataTool } from '../tools/DocumentMetadataTool'; import { ImageCreationTool } from '../tools/ImageCreationTool'; @@ -83,13 +82,8 @@ export class Agent { dataAnalysis: new DataAnalysisTool(csvData), websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), searchTool: new SearchTool(addLinkedUrlDoc), - // createCSV: new CreateCSVTool(createCSVInDash), noTool: new NoTool(), - imageCreationTool: new ImageCreationTool(createImage), - // createTextDoc: new CreateTextDocTool(addLinkedDoc), - // createDoc: new CreateDocTool(addLinkedDoc), - // createAnyDocument: new CreateAnyDocumentTool(addLinkedDoc), - // dictionary: new DictionaryTool(), + //imageCreationTool: new ImageCreationTool(createImage), documentMetadata: new DocumentMetadataTool(this), }; } @@ -103,7 +97,7 @@ export class Agent { * @param maxTurns The maximum number of turns to allow in the conversation. * @returns The final response from the assistant. */ - async askAgent(question: string, onProcessingUpdate: (processingUpdate: ProcessingInfo[]) => void, onAnswerUpdate: (answerUpdate: string) => void, maxTurns: number = 30): Promise { + async askAgent(question: string, onProcessingUpdate: (processingUpdate: ProcessingInfo[]) => void, onAnswerUpdate: (answerUpdate: string) => void, maxTurns: number = 50): Promise { console.log(`Starting query: ${question}`); const MAX_QUERY_LENGTH = 1000; // adjust the limit as needed @@ -470,7 +464,7 @@ export class Agent { actionInput.fieldValue = String(actionInput.fieldValue); } } - + // Handle fieldEdits parameter (for multiple field edits) if ('fieldEdits' in actionInput && actionInput.fieldEdits) { try { @@ -521,7 +515,7 @@ export class Agent { /** * Reinitializes the DocumentMetadataTool with a direct reference to the ChatBox instance. * This ensures that the tool can properly access the ChatBox document and find related documents. - * + * * @param chatBox The ChatBox instance to pass to the DocumentMetadataTool */ public reinitializeDocumentMetadataTool(chatBox: any): void { diff --git a/src/client/views/nodes/chatbot/guides/guide.md b/src/client/views/nodes/chatbot/guides/guide.md new file mode 100644 index 000000000..2af76490d --- /dev/null +++ b/src/client/views/nodes/chatbot/guides/guide.md @@ -0,0 +1,647 @@ +# Dash Agent Tool Development Guide + +**Table of Contents** + +1. [Introduction: The Role and Potential of Tools](#1-introduction-the-role-and-potential-of-tools) + - Beyond Information Retrieval: Action and Creation + - The Agent as an Extension of the User within Dash +2. [Core Agent Architecture Deep Dive](#2-core-agent-architecture-deep-dive) + - The ReAct-Inspired Interaction Loop: Rationale and Flow + - XML Structure: Why XML? Parsing and LLM Guidance + - Stages (``) and Roles (`role="..."`): Enforcing Order + - Message Management (`messages`, `interMessages`): Building Context + - State Handling: Agent's Internal State vs. Tool Statelessness + - Key Components Revisited (`Agent.ts`, `prompts.ts`, `BaseTool.ts`, Parsers) + - Role of `prompts.ts`: Template and Dynamic Content Injection + - Limits and Safeguards (`maxTurns`) +3. [Anatomy of a Dash Agent Tool (Detailed Breakdown)](#3-anatomy-of-a-dash-agent-tool-detailed-breakdown) + - The `BaseTool` Abstract Class: Foundation and Contract + - `ToolInfo`: Defining Identity and LLM Instructions + - `name`: Uniqueness and LLM Invocation Trigger + - `description`: The LLM's Primary Guide - _Dynamically Injected into Prompt_ + - `parameterRules`: The Input Contract (In Depth) + - `citationRules`: Controlling Grounding in the Final Answer + - The `execute` Method: Heart of the Tool + - Asynchronous Nature (`async/await`) + - Receiving Arguments (`args: ParametersType

`) + - Performing the Core Logic (API calls, Dash functions) + - Returning `Observation[]`: The Output Contract (In Depth) + - The `inputValidator` Method: Handling Edge Cases +4. [The Agent-Tool Interaction Flow (Annotated XML Trace)](#4-the-agent-tool-interaction-flow-annotated-xml-trace) + - Detailed Step-by-Step with `Agent.ts` actions highlighted +5. [Step-by-Step Guide: Creating a New Tool](#5-step-by-step-guide-creating-a-new-tool) + - Step 1: Define Goal, Scope, Inputs, Outputs, Dash Interactions, Side Effects + - Step 2: Create the Tool Class File (Directory Structure) + - Step 3: Define Parameters (`parameterRules`) - Type Handling, Arrays + - Step 4: Define Tool Information (`ToolInfo`) - Crafting the _Crucial_ `description` + - Step 5: Implement `execute` - Defensive Coding, Using Injected Functions, Error Handling Pattern + - Step 6: Format Output (`Observation[]`) - Chunk Structure, `chunk_type`, IDs + - Step 7: Register Tool in `Agent.ts` - _This makes the tool available to the prompt_ + - Step 8: Verify Prompt Integration (No Manual Editing Needed) + - Step 9: Testing Your Tool - Strategies and What to Look For +6. [Deep Dive: Advanced Concepts & Patterns](#6-deep-dive-advanced-concepts--patterns) + - Handling Complex Data Types (Arrays, Objects) in Parameters/Observations + - Binary Data Handling (e.g., Base64 in Chunks) + - Managing Long-Running Tasks (Beyond simple `await`) + - Tools Needing Dash Context (Passing `this` vs. specific functions) + - The Role of `chunk_id` and `chunk_type` +7. [Best Practices and Advanced Considerations](#7-best-practices-and-advanced-considerations) + - Error Handling & Reporting (Specific Error Chunks) + - Security Considerations (Input Sanitization, API Key Management, Output Filtering) + - Performance Optimization (Minimize `execute` workload) + - Idempotency: Designing for Retries + - Tool Granularity: Single Responsibility Principle + - Context Window Management (Concise Descriptions are Key) + - User Experience (Tool output clarity) + - Maintainability and Code Comments +8. [Debugging Strategies](#8-debugging-strategies) + - Console Logging within `execute` + - Inspecting `interMessages` in `Agent.ts` + - Testing Tool Logic Standalone + - Analyzing LLM Failures (Incorrect tool choice -> Check `description`, bad parameters) +9. [Example: `CreateDashNoteTool`](#9-example-createdashnotetool) +10. [Glossary of Key Terms](#10-glossary-of-key-terms) +11. [Conclusion](#11-conclusion) + +--- + +## 1. Introduction: The Role and Potential of Tools + +Welcome, Dash team member! This guide will walk you through creating new tools for the Dash Agent. The Agent is designed to interact with users, understand their queries, and leverage specialized **Tools** to perform actions or retrieve information that the core Large Language Model (LLM) cannot do on its own. + +Tools extend the Agent's capabilities beyond simple conversation. They allow the Agent to: + +- Interact with external APIs (e.g., web search, calculators, image generation). +- Access and process data specific to the user's Dash environment (e.g., querying document metadata, analyzing linked CSVs). +- Perform actions within Dash (e.g., creating new documents, adding links, modifying metadata). + +By building new tools, you directly enhance the Agent's utility and integration within the Dash ecosystem. + +### Beyond Information Retrieval: Action and Creation + +While tools like `RAGTool` and `SearchTool` retrieve information, others _act_. `CalculateTool` performs computations, `ImageCreationTool` generates content, and importantly, tools like `DocumentMetadataTool` and your custom tools can **modify the Dash environment**, creating documents, adding links, or changing properties. + +### The Agent as an Extension of the User within Dash + +Think of the Agent, equipped with tools, as an intelligent assistant that can perform tasks _on behalf of the user_ directly within their Dash workspace. This deep integration is a key differentiator. + +--- + +## 2. Core Agent Architecture Deep Dive + +Understanding the "why" behind the architecture helps in tool development. + +### The ReAct-Inspired Interaction Loop: Rationale and Flow + +The Agent operates based on a loop inspired by the ReAct (Reason + Act) framework. The LLM alternates between: + +- **Reasoning (``):** Analyzing the query and deciding the next step. +- **Acting (``, ``):** Selecting and preparing to use a tool, or formulating a final answer (``). +- **Observing (``):** Receiving the results from a tool execution. + +This structure (Reason -> Act -> Observe -> Reason...) forces the LLM to break down complex tasks into manageable steps, making the process more reliable and auditable than letting the LLM generate a monolithic plan upfront. + +### XML Structure: Why XML? Parsing and LLM Guidance + +- **Why XML?** LLMs are generally adept at generating well-formed XML. XML's explicit start/end tags make parsing by `Agent.ts` (using libraries like `fast-xml-parser`) more robust and less prone to LLM "hallucinations" breaking the structure compared to formats like JSON in some complex scenarios. +- **LLM Guidance:** The strict XML schema defined in the system prompt provides clear guardrails for the LLM's output, constraining it to valid actions and formats. + +### Stages (``) and Roles (`role="..."`): Enforcing Order + +The `` ensures sequential processing. The `role` attribute indicates the source (e.g., `user`, `assistant`) and dictates control flow. `Agent.ts` _waits_ for a `user` (or `system-error-reporter`) stage after sending an `assistant` stage, enforcing the turn-based nature. The LLM is explicitly told only to generate `assistant` stages. + +### Message Management (`messages`, `interMessages`): Building Context + +- `messages`: The user-facing chat history (persisted in the Dash Doc `data` field). +- `interMessages`: The **internal, complete context** sent to the LLM for each turn. It includes the system prompt, user queries, _all intermediate thoughts, actions, rules, inputs, and observations_. This ensures the LLM has the full history of the current reasoning chain. It grows with each step in the loop. + +### State Handling: Agent's Internal State vs. Tool Statelessness + +- `Agent.ts` manages the conversational state (`interMessages`, current turn number, `processingInfo`, etc.). +- **Tools should be designed to be stateless.** They receive inputs via `args`, perform their action, and return results. Any persistent state relevant to the user's work should reside within Dash Docs/Fobs, accessible perhaps via tools like `DocumentMetadataTool` or specific functions passed to the tool. + +### Key Components Revisited (`Agent.ts`, `prompts.ts`, `BaseTool.ts`, Parsers) + +- `Agent.ts`: The central controller. Parses XML, validates actions, manages the loop, calls `tool.execute`, formats `Observation`s. Handles the streaming updates for the _final_ answer via `StreamedAnswerParser`. Holds the registry of available tools (`this.tools`). +- `prompts.ts` (`getReactPrompt`): Generates the system prompt for the LLM. It acts as a **template** defining the Agent's overall task, rules, and the required XML structure. Crucially, it **dynamically injects the list of available tools** (including their names and descriptions) based on the tools registered in the `Agent.ts` instance at runtime. **_You do not manually add tool descriptions here._** +- `BaseTool.ts`: The abstract class defining the _interface_ all tools must adhere to. Contains properties like `name` and `description` used by `getReactPrompt`. +- Parsers (`AnswerParser`, `StreamedAnswerParser`): Handle the final `` tag, extracting structured content, citations, etc., for UI display (`ChatBox.tsx`). + +### Limits and Safeguards (`maxTurns`) + +`Agent.ts` includes a `maxTurns` limit (default 30) to prevent infinite loops if the LLM gets stuck or fails to reach an `` stage. + +--- + +## 3. Anatomy of a Dash Agent Tool (Detailed Breakdown) + +All tools inherit from the abstract class `BaseTool`. + +### The `BaseTool` Abstract Class: Foundation and Contract + +- Located in `src/components/views/nodes/chatbot/agentsystem/tools/BaseTool.ts`. +- Generic `BaseTool

>`: `P` represents the specific, readonly array of `Parameter` definitions for _your_ tool, ensuring type safety for the `args` in `execute`. +- Defines the public properties (`name`, `description`, `parameterRules`, `citationRules`) and the abstract `execute` method that all tools must implement. + +### `ToolInfo`: Defining Identity and LLM Instructions + +- A configuration object (`{ name: string; description: string; parameterRules: P; citationRules: string; }`) passed to the `BaseTool` constructor. +- `name: string`: + - The **unique identifier** for your tool (e.g., `dictionaryLookup`, `createDashNote`). + - Must match the key used when registering the tool in `Agent.ts`'s `this.tools` map. + - This is the string the LLM will output in the `` tag to invoke your tool. + - Keep it concise and descriptive (camelCase recommended). +- `description: string`: The LLM's Primary Guide - _Dynamically Injected into Prompt_. + - This text is extracted from your `ToolInfo` object when `getReactPrompt` is called. + - It's **the text the LLM sees** to understand your tool's purpose and when to use it. + - **Crafting this is critical.** Make it extremely clear, concise, and accurate. Explicitly state: + - What the tool _does_. + - What _inputs_ it needs (briefly). + - What _output_ it provides. + - _Crucially_, under what circumstances the Agent should _choose_ this tool over others. (e.g., "Use this tool to create _new_ Dash notes, not for editing existing ones.") +- `parameterRules: P` (where `P extends ReadonlyArray`): + - The readonly array defining the **exact inputs** your `execute` method expects. + - Each element is a `Parameter` object (`{ name: string; type: 'string' | ... ; required: boolean; description: string; max_inputs?: number }`): + - `name`: Name of the parameter (e.g., `wordToDefine`, `noteContent`). Used as the key in the `args` object passed to `execute`. + - `type`: `'string' | 'number' | 'boolean' | 'string[]' | 'number[]' | 'boolean[]'`. `Agent.ts` uses this for basic validation and parsing (specifically for arrays). + - `required`: `true` if the LLM _must_ provide this parameter for the tool to function. `Agent.ts` checks this before calling `execute` (unless `inputValidator` overrides). + - `description`: Explanation of the parameter _for the LLM_. Guides the LLM on _what value_ to provide. Be specific (e.g., "The exact URL to scrape", "A search query suitable for web search"). + - `max_inputs?`: Optional. For array types (`string[]`, etc.), suggests a limit to the LLM on the number of items to provide. +- `citationRules: string`: + - Instructions for the LLM on how to construct the `` block within the final `` tag _when information obtained from this specific tool is used_. + - Directly influences the grounding and verifiability of the Agent's final response. + - Be explicit about the format: "Cite using `Optional text snippet`". + - Specify what goes into `chunk_id` (e.g., "Use the ID provided in the observation chunk"), `type` (a constant string representing your tool's output type), and whether the text snippet should be included (often empty for URLs, calculations). + - If no citation is appropriate (e.g., calculator, ephemeral action), state clearly: "No citation needed for this tool's output." + +### The `execute` Method: Heart of the Tool + +- `abstract execute(args: ParametersType

): Promise;` +- **Asynchronous Nature (`async/await`):** Must be `async` because tool actions often involve I/O (network requests, database access via Dash functions, filesystem). Must return a `Promise`. +- **Receiving Arguments (`args: ParametersType

`):** + - Receives a single argument `args`. This object's keys are your defined parameter names (from `parameterRules`), and the values are those provided by the LLM in the `` block. + - The type `ParametersType

` infers the structure of `args` based on your specific `parameterRules` definition (`P`), providing TypeScript type safety. + - `Agent.ts` performs basic validation (required fields) and type coercion (for arrays) before calling `execute`. However, **always perform defensive checks** within `execute` (e.g., check if required args are truly present and not empty strings, check types if crucial). +- **Performing the Core Logic:** This is where your tool does its work. Examples: + - Call external APIs (using `axios`, `fetch`, or specific SDKs). + - Call Dash functions passed via the constructor (e.g., `this._createDocInDash(...)`, `this._addLinkedUrlDoc(...)`). + - Perform calculations or data transformations. + - Interact with other backend systems if necessary. +- **Returning `Observation[]`: The Output Contract (In Depth)** + - The method **must** resolve to an array of `Observation` objects. Even if there's only one piece of output, return it in an array: `[observation]`. + - Each `Observation` object usually has the structure `{ type: 'text', text: string }`. Other types might be possible but `text` is standard for LLM interaction. + - The `text` string **must** contain the tool's output formatted within one or more `` tags. This is how the Agent passes structured results back to the LLM. + - Format: `OUTPUT_DATA` + - `chunk_id`: A unique identifier for this specific piece of output (use `uuidv4()`). Essential for linking citations back to observations. + - `chunk_type`: A string literal describing the _semantic type_ of the data (e.g., `'search_result_url'`, `'calculation_result'`, `'note_creation_status'`, `'error'`, `'metadata_info'`). Helps the LLM interpret the result. Choose consistent and descriptive names. + - `OUTPUT_DATA`: The actual result from your tool. Can be simple text, JSON stringified data, etc. Keep it concise if possible for the LLM context. + - **Return errors** using the same format, but with `chunk_type="error"` and a descriptive error message inside the chunk tag. This allows the Agent loop to continue gracefully and potentially inform the LLM or user. + +### The `inputValidator` Method: Handling Edge Cases + +- `inputValidator(inputParam: ParametersType) { return false; }` (Default implementation in `BaseTool`). +- Override this method _only_ if your tool needs complex input validation logic beyond simple `required` checks (e.g., dependencies between parameters). +- If you override it to return `true`, `Agent.ts` will skip its standard check for missing _required_ parameters. Your `execute` method becomes fully responsible for validating the `args` object. +- Use case example: `DocumentMetadataTool` uses it to allow either `fieldName`/`fieldValue` OR `fieldEdits` to be provided for the "edit" action. + +--- + +## 4. The Agent-Tool Interaction Flow (Annotated XML Trace) + +Let's trace the `dictionaryLookup` example with `Agent.ts` actions: + +1. **User Input:** User types "What is hypermedia?" and submits. + + - `// ChatBox.tsx calls agent.askAgent("What is hypermedia?")` + - `// Agent.ts adds stage 1 to interMessages:` + + ```xml + + What is hypermedia? + + ``` + + - `// Agent.ts calls LLM with interMessages.` + +2. **LLM Thought & Action:** LLM processes the query and system prompt (which includes the dynamically generated description for `dictionaryLookup`). + + - `// LLM responds with stage 2:` + + ```xml + + The user is asking for a definition. The dictionaryLookup tool is appropriate for this. + dictionaryLookup + + ``` + + - `// Agent.ts parses stage 2. Finds dictionaryLookup.` + - `// Agent.ts retrieves the dictionaryTool instance: tool = this.tools['dictionaryLookup'].` + - `// Agent.ts gets parameter rules: rules = tool.getActionRule().` + +3. **Agent Provides Rules:** `Agent.ts` formats the rules into XML. + + - `// Agent.ts adds stage 3 to interMessages:` + + ```xml + + + dictionaryLookup + Looks up the definition of a given English word. + Cite the definition using the provided chunk_id and type="dictionary_definition". Leave citation content empty. + + + string + The word to define. + true + + + + + ``` + + - `// Agent.ts calls LLM with updated interMessages.` + +4. **LLM Provides Inputs:** LLM uses the rules to formulate the required inputs. + + - `// LLM responds with stage 4:` + + ```xml + + + Looking up the definition for the word 'hypermedia'. + + hypermedia + + + + ``` + + - `// Agent.ts parses stage 4. Finds . Extracts inputs: { word: 'hypermedia' }.` + - `// Agent.ts validates required params (finds 'word'). Checks tool.inputValidator (returns false). OK.` + - `// Agent.ts calls: const observations = await dictionaryTool.execute({ word: 'hypermedia' });` + +5. **Tool Execution -> Observation:** The `dictionaryTool.execute` method runs, calls an API, gets the definition, and formats the `Observation`. + + - `// dictionaryTool.execute resolves Promise with:` + + ```javascript + [{ type: 'text', text: 'Hypermedia is an extension of hypertext allowing...' }]; + ``` + + - `// Agent.ts receives the resolved Observation[].` + +6. **Agent Provides Observation:** `Agent.ts` formats the observation(s) into the next stage. + + - `// Agent.ts adds stage 5 to interMessages:` + + ```xml + + + Hypermedia is an extension of hypertext allowing... + + + ``` + + - `// Agent.ts calls LLM with updated interMessages.` + +7. **LLM Generates Final Answer:** LLM uses the full context (query, thoughts, actions, observation) and the `citationRules` from the `dictionaryLookup` tool. + - `// LLM responds with stage 6:` + ```xml + + I have received the definition from the dictionary tool. I will now format the final answer according to the specified structure, including the citation. + + Hypermedia is an extension of hypertext allowing graphics, audio, video, plain text and hyperlinks to intertwine to create a generally non-linear medium of information. + + + + + Can you give an example of a hypermedia system? + How does hypermedia differ from multimedia? + Who coined the term hypermedia? + + Used dictionaryLookup tool to define 'hypermedia'. + + + ``` + - `// Agent.ts parses stage 6. Finds . Calls AnswerParser.` + - `// Agent.ts updates ChatBox UI (`\_history.push(...)`). Loop ends.` + +--- + +## 5. Step-by-Step Guide: Creating a New Tool + +Let's use the example of creating a `CreateDashNoteTool`. + +### Step 1: Define Goal, Scope, Inputs, Outputs, Dash Interactions, Side Effects + +- **Goal:** Allow Agent to create a new text note (`DocumentType.TEXT` or equivalent) in Dash. +- **Scope:** Creates a _simple_ note with title and text content. Does not handle complex formatting, linking (beyond default linking to ChatBox if handled by the creation function), or specific placement beyond a potential default offset. +- **Inputs:** `noteTitle` (string, required), `noteContent` (string, required). +- **Outputs (Observation):** Confirmation message with new note's Dash Document ID, or an error message. +- **Dash Interactions:** Calls a function capable of creating Dash documents (e.g., `createDocInDash` passed via constructor). +- **Side Effects:** A new Dash text document is created in the user's space and potentially linked to the ChatBox. + +### Step 2: Create the Tool Class File (Directory Structure) + +- Create file: `src/components/views/nodes/chatbot/agentsystem/tools/CreateDashNoteTool.ts` +- Ensure it's within the `tools` subdirectory. + +### Step 3: Define Parameters (`parameterRules`) - Type Handling, Arrays + +- Use `as const` for the array to allow TypeScript to infer literal types, which aids `ParametersType`. +- Define `noteTitle` and `noteContent` as required strings. + +```typescript +import { Parameter } from '../types/tool_types'; + +const createDashNoteToolParams = [ + { + name: 'noteTitle', + type: 'string', + required: true, + description: 'The title for the new Dash note document. Cannot be empty.', + }, + { + name: 'noteContent', + type: 'string', + required: true, + description: 'The text content for the new Dash note. Can be an empty string.', // Specify if empty content is allowed + }, +] as const; // Use 'as const' for precise typing + +// Infer the type for args object in execute +type CreateDashNoteToolParamsType = typeof createDashNoteToolParams; +``` + +### Step 4: Define Tool Information (`ToolInfo`) - Crafting the _Crucial_ `description` + +- This object's `description` is key for the LLM. + +```typescript +import { ToolInfo, ParametersType } from '../types/tool_types'; + +// Assuming createDashNoteToolParams and CreateDashNoteToolParamsType are defined above + +const createDashNoteToolInfo: ToolInfo = { + name: 'createDashNote', // Must match registration key in Agent.ts + description: + 'Creates a *new*, simple text note document within the current Dash view. Requires a title and text content. The note will be linked to the ChatBox and placed nearby with default dimensions. Use this when the user asks to create a new note, save information, or write something down persistently in Dash.', + parameterRules: createDashNoteToolParams, + citationRules: 'This tool creates a document. The observation confirms success and provides the new document ID. No citation is typically needed in the final answer unless confirming the action.', +}; +``` + +### Step 5: Implement `execute` - Defensive Coding, Using Injected Functions, Error Handling Pattern + +- Implement the `execute` method within your class. +- Wrap logic in `try...catch`. +- Validate inputs defensively. +- Check injected dependencies (`this._createDocInDash`). +- Call the Dash function. +- Handle the return value. +- Format success or error `Observation`. + +```typescript +import { BaseTool } from './BaseTool'; +import { Observation } from '../types/types'; +import { supportedDocTypes } from '../types/tool_types'; +import { parsedDoc } from '../chatboxcomponents/ChatBox'; // May need adjustment based on actual path +import { Doc } from '../../../../../../fields/Doc'; // Adjust path as needed +import { v4 as uuidv4 } from 'uuid'; +import { RTFCast } from '../../../../../../fields/Types'; // Adjust path as needed + +// Assuming createDashNoteToolParams, CreateDashNoteToolParamsType, createDashNoteToolInfo are defined above + +export class CreateDashNoteTool extends BaseTool { + // Dependency: Function to create a document in Dash + private _createDocInDash: (doc: parsedDoc) => Doc | undefined; + + // Constructor to inject dependencies + constructor(createDocInDash: (doc: parsedDoc) => Doc | undefined) { + super(createDashNoteToolInfo); + if (typeof createDocInDash !== 'function') { + console.error('CreateDashNoteTool Error: createDocInDash function dependency not provided during instantiation!'); + // Consider throwing an error or setting a flag to prevent execution + } + this._createDocInDash = createDocInDash; + } + + async execute(args: ParametersType): Promise { + const chunkId = uuidv4(); // Unique ID for this observation + const { noteTitle, noteContent } = args; + + // --- Input Validation --- + if (typeof noteTitle !== 'string' || !noteTitle.trim()) { + return [{ type: 'text', text: `Invalid input: Note title must be a non-empty string.` }]; + } + if (typeof noteContent !== 'string') { + // Assuming empty content IS allowed based on description + // If not allowed, return error here. + return [{ type: 'text', text: `Invalid input: Note content must be a string.` }]; + } + if (!this._createDocInDash) { + return [{ type: 'text', text: `Tool Configuration Error: Document creation function not available.` }]; + } + // --- End Validation --- + + try { + const trimmedTitle = noteTitle.trim(); + + // Prepare the document object for the creation function + const noteDoc: parsedDoc = { + doc_type: supportedDocTypes.note, // Use the correct type for a text note + title: trimmedTitle, + data: RTFCast(noteContent) as unknown as string, // Ensure data is correctly formatted if needed + // Example default properties: + _width: 300, + _layout_fitWidth: false, + _layout_autoHeight: true, + backgroundColor: '#FFFFE0', // Light yellow background + // Add x, y coordinates if desired, potentially relative to ChatBox if context is available + }; + + console.log(`CreateDashNoteTool: Attempting to create doc:`, { title: noteDoc.title, type: noteDoc.doc_type }); // Avoid logging full content + + // Call the injected Dash function + const createdDoc = this._createDocInDash(noteDoc); + + // Check the result + if (createdDoc && createdDoc.id) { + const successMessage = `Successfully created note titled "${trimmedTitle}" with ID: ${createdDoc.id}. It has been added to your current view.`; + console.log(`CreateDashNoteTool: Success - ${successMessage}`); + // Return observation confirming success + return [{ type: 'text', text: `${successMessage}` }]; + } else { + console.error('CreateDashNoteTool Error: _createDocInDash returned undefined or document without an ID.'); + throw new Error('Dash document creation failed or did not return a valid document ID.'); + } + } catch (error) { + console.error(`CreateDashNoteTool: Error creating note titled "${noteTitle.trim()}":`, error); + const errorMessage = error instanceof Error ? error.message : 'An unknown error occurred during note creation.'; + // Return observation indicating error + return [{ type: 'text', text: `Error creating note: ${errorMessage}` }]; + } + } +} +``` + +### Step 6: Format Output (`Observation[]`) - Chunk Structure, `chunk_type`, IDs + +- Ensure the `text` field within the returned `Observation` contains `...`. +- Use a specific `chunk_type` (e.g., `note_creation_status`, `error`). +- Generate a unique `chunk_id` using `uuidv4()`. +- The text inside the chunk should be informative for the LLM and potentially for debugging. + +### Step 7: Register Tool in `Agent.ts` - _This makes the tool available to the prompt_ + +- Import your tool class at the top of `Agent.ts`: + ```typescript + import { CreateDashNoteTool } from '../tools/CreateDashNoteTool'; + ``` +- In the `Agent` constructor, instantiate your tool within the `this.tools = { ... };` block. Ensure the key matches `ToolInfo.name` and pass any required dependencies (like the `createDocInDash` function). + + ```typescript + constructor( + _vectorstore: Vectorstore, + summaries: () => string, + history: () => string, + csvData: () => { filename: string; id: string; text: string }[], + addLinkedUrlDoc: (url: string, id: string) => void, + createImage: (result: any, options: any) => void, // Use specific types if known + createDocInDashFunc: (doc: parsedDoc) => Doc | undefined, // Renamed for clarity + createCSVInDash: (url: string, title: string, id: string, data: string) => void + ) { + // ... existing initializations (OpenAI client, vectorstore, etc.) ... + this.vectorstore = _vectorstore; + this._summaries = summaries; + this._history = history; + this._csvData = csvData; + + this.tools = { + calculate: new CalculateTool(), + rag: new RAGTool(this.vectorstore), + dataAnalysis: new DataAnalysisTool(csvData), + websiteInfoScraper: new WebsiteInfoScraperTool(addLinkedUrlDoc), + searchTool: new SearchTool(addLinkedUrlDoc), + noTool: new NoTool(), + imageCreationTool: new ImageCreationTool(createImage), + documentMetadata: new DocumentMetadataTool(this), // Pass ChatBox instance if needed by tool + // Register the new tool here: + createDashNote: new CreateDashNoteTool(createDocInDashFunc), // Pass the required function + }; + // ... rest of constructor + } + ``` + +- **Verify Dependencies:** Ensure that the `createDocInDashFunc` parameter (or however you name it) is actually being passed into the `Agent` constructor when it's instantiated (likely within `ChatBox.tsx`). Trace the dependency chain. + +### Step 8: Verify Prompt Integration (No Manual Editing Needed) + +- **No manual changes are needed in `prompts.ts`**. The `getReactPrompt` function dynamically builds the `` section from `this.tools`. +- **Verify (Recommended):** Temporarily add `console.log(systemPrompt)` in `Agent.ts` right after `const systemPrompt = getReactPrompt(...)` within the `askAgent` method. Run a query. Examine the console output to confirm the system prompt includes your tool's `` and `<description>` within the `<tools>` block. Remove the log afterward. + +### Step 9: Testing Your Tool - Strategies and What to Look For + +- **Functional Tests:** Use specific prompts like "Create a note called 'Ideas' with content 'Test 1'." Check the Dash UI for the note and the chat for the success message/ID. +- **Edge Case Tests:** Test empty titles (should fail validation), empty content (should succeed if allowed), titles/content with special characters or excessive length. +- **LLM Interaction Tests:** Use less direct prompts like "Save this thought: Remember to buy milk." Does the LLM correctly identify the need for your tool and extract/request the title and content? +- **Failure Tests:** If possible, simulate failure in the dependency (`createDocInDash`) to ensure the `error` chunk is returned correctly. +- **Console/Debugging:** Use `console.log` within `execute` and inspect `interMessages` in `Agent.ts` to trace the flow and identify issues. + +--- + +## 6. Deep Dive: Advanced Concepts & Patterns + +### Handling Complex Data Types (Arrays, Objects) in Parameters/Observations + +- **Parameters:** For complex inputs, define the parameter `type` as `string` in `parameterRules`. In the `description`, instruct the LLM to provide a **valid JSON string**. Inside your `execute` method, use `JSON.parse()` within a `try...catch` block to parse this string. Handle potential parsing errors gracefully (return an `error` chunk). +- **Observations:** To return structured data, `JSON.stringify` your object/array and embed this string _inside_ the `<chunk>` tag. Use a specific `chunk_type` (e.g., `json_data_analysis`). The LLM might need guidance (via prompt engineering) on how to interpret and use this JSON data effectively in its final response. + +### Binary Data Handling (e.g., Base64 in Chunks) + +- **Avoid large binary data in observations.** Context windows are limited. +- **Preferred:** Save binary data server-side (e.g., using `DashUploadUtils` or similar) or reference existing Dash media docs. Return a **reference** (URL, Doc ID, file path accessible by Dash) within the `<chunk>`. +- **If absolutely necessary:** For small images/data needed _directly_ by the LLM, Base64 encode it inside the chunk: `<chunk chunk_id="..." chunk_type="base64_image_png">BASE64_STRING</chunk>`. + +### Managing Long-Running Tasks (Beyond simple `await`) + +- The agent's `askAgent` loop `await`s `tool.execute()`. Tasks taking more than ~5-10 seconds degrade user experience. Very long tasks risk timeouts. +- **Limitation:** The current architecture doesn't have built-in support for asynchronous background jobs with status polling. +- **Possible (Complex) Workaround:** + 1. Tool `execute` initiates a long-running _external_ process (like the Python PDF chunker) or backend job. + 2. `execute` _immediately_ returns an `Observation` like `<chunk chunk_type="task_initiated" job_id="JOB123">Processing started. Use status check tool with ID JOB123.</chunk>`. + 3. Requires a _separate_ `StatusCheckTool` that takes a `job_id` and queries the external process/backend for status. + 4. This adds significant complexity to the agent's reasoning flow. Use only if absolutely necessary. + +### Tools Needing Dash Context (Passing `this` vs. specific functions) + +- **Specific Functions (Preferred):** Pass only necessary functions from `ChatBox`/Dash utilities. Promotes modularity and testability. Requires updating constructors if needs change. +- **`ChatBox` Instance (`this`) (As in `DocumentMetadataTool`):** Provides broad access to `ChatBox` state (`Document`, `layoutDoc`, computed properties) and methods. Easier for tools with complex Dash interactions but increases coupling and makes testing harder. +- **Decision:** Start with specific functions. Escalate to passing `this` only if the tool's requirements become extensive and unmanageable via individual function injection. + +### The Role of `chunk_id` and `chunk_type` + +- `chunk_id` (e.g., `uuidv4()`): **Traceability & Citation.** Uniquely identifies a piece of data returned by a tool. Allows the final `<answer>`'s `<citation>` tag to precisely reference the source observation via this ID. Essential for debugging and grounding. +- `chunk_type`: **Semantic Meaning.** Tells the LLM _what kind_ of information the chunk contains (e.g., `url`, `calculation_result`, `error`, `note_creation_status`). Guides the LLM in processing the observation and formatting the final answer appropriately. Use consistent and descriptive type names. + +--- + +## 7. Best Practices and Advanced Considerations + +- **Error Handling & Reporting:** Return errors in structured `<chunk chunk_type="error">...</chunk>` format. Include context in the message (e.g., "API call failed for URL: [url]", "Invalid value for parameter: [param_name]"). +- **Security:** + - **Input Sanitization:** **Crucial.** If tool inputs influence API calls, file paths, database queries, etc., validate and sanitize them rigorously. Do not trust LLM output implicitly. + - **API Keys:** Use server-side environment variables (`process.env`) for keys used in backend routes called by tools. Avoid exposing keys directly in client-side tool code if possible. + - **Output Filtering:** Be mindful of sensitive data. Don't leak PII or internal details in observations or error messages. +- **Performance Optimization:** Keep `execute` logic efficient. Minimize blocking operations. Use asynchronous patterns correctly. +- **Idempotency:** Design tools (especially those causing side effects like creation/modification) to be safe if run multiple times with the same input, if possible. +- **Tool Granularity (SRP):** Aim for tools that do one thing well. Complex workflows can be achieved by the LLM chaining multiple focused tools. +- **Context Window Management:** Write concise but clear tool `description`s. Keep `Observation` data relevant and succinct. +- **User Experience:** Tool output (via observations) influences the final answer. Ensure returned data is clear and `citationRules` guide the LLM to produce understandable results. +- **Maintainability:** Use clear code, comments for complex logic, TypeScript types, and follow project conventions. + +--- + +## 8. Debugging Strategies + +1. **`console.log`:** Liberally use `console.log` inside your tool's `execute` method to inspect `args`, intermediate variables, API responses, and the `Observation[]` object just before returning. +2. **Inspect `interMessages`:** Temporarily modify `Agent.ts` (e.g., in the `askAgent` `while` loop) to `console.log(JSON.stringify(this.interMessages, null, 2))` before each LLM call. This shows the exact XML context the LLM sees and its raw XML response. Pinpoint where the conversation deviates or breaks. +3. **Test Standalone:** Create a simple test script (`.ts` file run with `ts-node` or similar). Import your tool. Create mock objects/functions for its dependencies (e.g., `const mockCreateDoc = (doc) => ({ id: 'mock-doc-123', ...doc });`). Instantiate your tool with mocks: `const tool = new YourTool(mockCreateDoc);`. Call `await tool.execute(testArgs);` and assert the output. This isolates tool logic. +4. **Analyzing LLM Failures:** Use the `interMessages` log: + - **Wrong Tool Chosen:** LLM's `<thought>` selects the wrong tool, or uses `<action>noTool</action>` inappropriately. -> **Refine your tool's `description`** in `ToolInfo` for clarity and better differentiation. + - **Missing/Incorrect Parameters:** LLM fails to provide required parameters in `<inputs>`, or provides wrong values. -> **Refine parameter `description`s** in `parameterRules`. Check the `<action_input>` stage in the log. + - **Ignoring Observation/Bad Answer:** LLM gets the correct `<observation>` but generates a poor `<answer>` (ignores data, bad citation). -> Check `chunk_type`, data format inside the chunk, and **clarify `citationRules`**. Simplify observation data if needed. + - **XML Formatting Errors:** LLM returns malformed XML. -> This might require adjusting the system prompt's structure rules or adding more robust parsing/error handling in `Agent.ts`. + +--- + +## 9. Example: `CreateDashNoteTool` + +The code provided in Step 5 serves as a practical example, demonstrating dependency injection, input validation, calling a Dash function, and formatting success/error observations within the required `<chunk>` structure. Ensure the dependency (`createDocInDashFunc`) is correctly passed during `Agent` instantiation in `ChatBox.tsx`. + +--- + +## 10. Glossary of Key Terms + +- **Agent (`Agent.ts`):** The orchestrator class managing the LLM interaction loop and tool usage. +- **Tool (`BaseTool.ts`):** A class extending `BaseTool` to provide specific functionality (API calls, Dash actions). +- **LLM (Large Language Model):** The AI model providing reasoning and text generation (e.g., GPT-4o). +- **ReAct Loop:** The core interaction pattern: Reason -> Act -> Observe. +- **XML Structure:** The tag-based format (`<stage>`, `<thought>`, etc.) for LLM communication. +- **`interMessages`:** The internal, complete conversational context sent to the LLM. +- **`ToolInfo`:** Configuration object (`name`, `description`, `parameterRules`, `citationRules`) defining a tool. **Source of dynamic prompt content for the tool list.** +- **`parameterRules`:** Array defining a tool's expected input parameters. +- **`citationRules`:** Instructions for the LLM on citing a tool's output. +- **`execute`:** The primary asynchronous method within a tool containing its core logic. +- **`Observation`:** The structured object (`{ type: 'text', text: '<chunk>...' }`) returned by `execute`. +- **`<chunk>`:** The required XML-like wrapper within an Observation's `text`, containing `chunk_id` and `chunk_type`. +- **`chunk_type`:** Semantic identifier for the data type within a `<chunk>`. +- **System Prompt (`getReactPrompt`):** The foundational instructions for the LLM, acting as a **template dynamically populated** with registered tool descriptions. +- **Dash Functions:** Capabilities from the Dash environment (e.g., `createDocInDash`) injected into tools. +- **Stateless Tool:** A tool whose output depends solely on current inputs, not past interactions. + +--- + +## 11. Conclusion + +This guide provides a detailed framework for extending the Dash Agent with custom tools. By adhering to the `BaseTool` structure, understanding the agent's interaction flow, crafting clear `ToolInfo` descriptions, implementing robust `execute` methods, and correctly registering your tool in `Agent.ts`, you can build powerful integrations that leverage both AI and the unique capabilities of the Dash hypermedia environment. Remember that testing and careful consideration of dependencies, errors, and security are crucial for creating reliable tools. diff --git a/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts b/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts deleted file mode 100644 index 754d230c8..000000000 --- a/src/client/views/nodes/chatbot/tools/CreateAnyDocTool.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { toLower } from 'lodash'; -import { Doc } from '../../../../../fields/Doc'; -import { Id } from '../../../../../fields/FieldSymbols'; -import { DocumentOptions } from '../../../../documents/Documents'; -import { parsedDoc } from '../chatboxcomponents/ChatBox'; -import { ParametersType, ToolInfo } from '../types/tool_types'; -import { Observation } from '../types/types'; -import { BaseTool } from './BaseTool'; -import { supportedDocTypes } from './CreateDocumentTool'; - -const standardOptions = ['title', 'backgroundColor']; -/** - * Description of document options and data field for each type. - */ -const documentTypesInfo: { [key in supportedDocTypes]: { options: string[]; dataDescription: string } } = { - [supportedDocTypes.flashcard]: { - options: [...standardOptions, 'fontColor', 'text_align'], - dataDescription: 'an array of two strings. the first string contains a question, and the second string contains an answer', - }, - [supportedDocTypes.text]: { - options: [...standardOptions, 'fontColor', 'text_align'], - dataDescription: 'The text content of the document.', - }, - [supportedDocTypes.html]: { - options: [], - dataDescription: 'The HTML-formatted text content of the document.', - }, - [supportedDocTypes.equation]: { - options: [...standardOptions, 'fontColor'], - dataDescription: 'The equation content as a string.', - }, - [supportedDocTypes.functionplot]: { - options: [...standardOptions, 'function_definition'], - dataDescription: 'The function definition(s) for plotting. Provide as a string or array of function definitions.', - }, - [supportedDocTypes.dataviz]: { - options: [...standardOptions, 'chartType'], - dataDescription: 'A string of comma-separated values representing the CSV data.', - }, - [supportedDocTypes.notetaking]: { - options: standardOptions, - dataDescription: 'The initial content or structure for note-taking.', - }, - [supportedDocTypes.rtf]: { - options: standardOptions, - dataDescription: 'The rich text content in RTF format.', - }, - [supportedDocTypes.image]: { - options: standardOptions, - dataDescription: 'The image content as an image file URL.', - }, - [supportedDocTypes.pdf]: { - options: standardOptions, - dataDescription: 'the pdf content as a PDF file url.', - }, - [supportedDocTypes.audio]: { - options: standardOptions, - dataDescription: 'The audio content as a file url.', - }, - [supportedDocTypes.video]: { - options: standardOptions, - dataDescription: 'The video content as a file url.', - }, - [supportedDocTypes.message]: { - options: standardOptions, - dataDescription: 'The message content of the document.', - }, - [supportedDocTypes.diagram]: { - options: ['title', 'backgroundColor'], - dataDescription: 'diagram content as a text string in Mermaid format.', - }, - [supportedDocTypes.script]: { - options: ['title', 'backgroundColor'], - dataDescription: 'The compilable JavaScript code. Use this for creating scripts.', - }, -}; - -const createAnyDocumentToolParams = [ - { - name: 'document_type', - type: 'string', - description: `The type of the document to create. Supported types are: ${Object.values(supportedDocTypes).join(', ')}`, - required: true, - }, - { - name: 'data', - type: 'string', - description: 'The content or data of the document. The exact format depends on the document type.', - required: true, - }, - { - name: 'options', - type: 'string', - required: false, - description: `A JSON string representing the document options. Available options depend on the document type. For example: - ${Object.entries(documentTypesInfo).map( ([doc_type, info]) => ` -- For '${doc_type}' documents, options include: ${info.options.join(', ')}`) - .join('\n')}`, // prettier-ignore - }, -] as const; - -type CreateAnyDocumentToolParamsType = typeof createAnyDocumentToolParams; - -const createAnyDocToolInfo: ToolInfo<CreateAnyDocumentToolParamsType> = { - name: 'createAnyDocument', - description: - `Creates any type of document with the provided options and data. - Supported document types are: ${Object.values(supportedDocTypes).join(', ')}. - dataviz is a csv table tool, so for CSVs, use dataviz. Here are the options for each type: - <supported_document_types>` + - Object.entries(documentTypesInfo) - .map( - ([doc_type, info]) => - `<document_type name="${doc_type}"> - <data_description>${info.dataDescription}</data_description> - <options>` + - info.options.map(option => `<option>${option}</option>`).join('\n') + - `</options> - </document_type>` - ) - .join('\n') + - `</supported_document_types>`, - parameterRules: createAnyDocumentToolParams, - citationRules: 'No citation needed.', -}; - -export class CreateAnyDocumentTool extends BaseTool<CreateAnyDocumentToolParamsType> { - private _addLinkedDoc: (doc: parsedDoc) => Doc | undefined; - - constructor(addLinkedDoc: (doc: parsedDoc) => Doc | undefined) { - super(createAnyDocToolInfo); - this._addLinkedDoc = addLinkedDoc; - } - - async execute(args: ParametersType<CreateAnyDocumentToolParamsType>): Promise<Observation[]> { - try { - const documentType = toLower(args.document_type) as unknown as supportedDocTypes; - const info = documentTypesInfo[documentType]; - - if (info === undefined) { - throw new Error(`Unsupported document type: ${documentType}. Supported types are: ${Object.values(supportedDocTypes).join(', ')}.`); - } - - if (!args.data) { - throw new Error(`Data is required for ${documentType} documents. ${info.dataDescription}`); - } - - const options: DocumentOptions = !args.options ? {} : JSON.parse(args.options); - - // Call the function to add the linked document (add default title that can be overriden if set in options) - const doc = this._addLinkedDoc({ doc_type: documentType, data: args.data, title: `New ${documentType.charAt(0).toUpperCase() + documentType.slice(1)} Document`, ...options }); - - return [{ type: 'text', text: `Created ${documentType} document with ID ${doc?.[Id]}.` }]; - } catch (error) { - return [{ type: 'text', text: 'Error creating document: ' + (error as Error).message }]; - } - } -} diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index a9fb45b5a..a3d86287d 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -11,7 +11,6 @@ import { DocCast, StrCast } from '../../../../../fields/Types'; import { supportedDocTypes } from '../types/tool_types'; import { parsedDoc } from '../chatboxcomponents/ChatBox'; - // Define the parameters for the DocumentMetadataTool const parameterDefinitions: ReadonlyArray<Parameter> = [ { @@ -61,7 +60,7 @@ const parameterDefinitions: ReadonlyArray<Parameter> = [ type: 'string', required: false, description: `The type of document to create. Required for "create" action. Options: ${Object.keys(supportedDocTypes).join(',')}`, - } + }, ] as const; type DocumentMetadataToolParamsType = typeof parameterDefinitions; @@ -105,14 +104,14 @@ IMPORTANT: Some fields have dependencies that must be handled for edits to work - You can edit dependent fields in a single operation using the fieldEdits parameter Example: To change document height, first disable auto-height: -1. { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } -2. { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 300 } +1. {... inputs: { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false }} +2. {... inputs: { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 300 }} OR using multi-field edit (recommended for dependent fields): -{ action: "edit", documentId: "doc123", fieldEdits: [ +{... inputs: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 300 } -]}`; +]}}`; // Extensive usage guidelines for the tool const citationRules = `USAGE GUIDELINES: @@ -132,7 +131,7 @@ To CREATE a new document: - title: The title of the document to create - data: The content data for the document (text content, URL, etc.) - doc_type: The type of document to create (text, web, image, etc.) -- Example: { action: "create", title: "My Notes", data: "This is the content", doc_type: "text" } +- Example: {...inputs: { action: "create", title: "My Notes", data: "This is the content", doc_type: "text" }} - After creation, you can edit the document with more specific properties To EDIT document metadata: @@ -147,7 +146,7 @@ To EDIT document metadata: SPECIAL FIELD HANDLING: - Text fields: When editing the 'text' field, provide simple plain text - Example: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "Hello world" } + Example: {...inputs: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "Hello world" }} The tool will automatically convert your text to the proper RichTextField format - Width/Height: Set layout_autoHeight/layout_autoWidth to false before editing @@ -165,10 +164,10 @@ HANDLING DEPENDENT FIELDS: - When editing some fields, you may need to update related dependent fields - For example, when changing "height", you should also set "layout_autoHeight" to false - Use the fieldEdits parameter to update dependent fields in a single operation (recommended): - { action: "edit", documentId: "doc123", fieldEdits: [ + {...inputs: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 300 } - ]} +]}} - Always check for dependent fields that might affect your edits, such as: - height → layout_autoHeight (set to false to allow manual height) - width → layout_autoWidth (set to false to allow manual width) @@ -200,8 +199,8 @@ Examples: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 200 } - ]}`; - + ]} +- IMPORTANT: MULTI STEP WORKFLOWS ARE NOT ONLY ALLOWED BUT ENCOURAGED. TAKE THINGS 1 STEP AT A TIME.`; const documentMetadataToolInfo: ToolInfo<DocumentMetadataToolParamsType> = { name: 'documentMetadata', description: toolDescription, @@ -227,7 +226,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp constructor(chatBox: any) { super(documentMetadataToolInfo); this.chatBox = chatBox; - + // Store a direct reference to the ChatBox document if (chatBox && chatBox.Document) { this.chatBoxDocument = chatBox.Document; @@ -246,7 +245,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } else { console.warn('DocumentMetadataTool initialized without valid ChatBox Document reference'); } - + this.initializeFieldMetadata(); } @@ -273,12 +272,12 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp // Check if fieldInfo has description property (it's likely a FInfo instance) if (fieldInfo && typeof fieldInfo === 'object' && 'description' in fieldInfo) { fieldData.description = fieldInfo.description; - + // Extract field type if available if ('fieldType' in fieldInfo) { fieldData.type = fieldInfo.fieldType; } - + // Extract possible values if available if ('values' in fieldInfo && Array.isArray(fieldInfo.values)) { fieldData.possibleValues = fieldInfo.values; @@ -303,30 +302,30 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp // Use the LinkManager approach which is proven to work in ChatBox if (this.chatBoxDocument) { console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument.id); - + // Get directly linked documents via LinkManager const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.chatBoxDocument) .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.chatBoxDocument!))) .map(d => DocCast(d?.annotationOn, d)) .filter(d => d); - + console.log(`Found ${linkedDocs.length} linked documents via LinkManager`); - + // Process the linked documents linkedDocs.forEach((doc: Doc) => { if (doc) { this.processDocument(doc); } }); - + // Include the ChatBox document itself this.processDocument(this.chatBoxDocument); - + // If we have access to the Document's parent, try to find sibling documents if (this.chatBoxDocument.parent) { const parent = this.chatBoxDocument.parent; console.log('Found parent document, checking for siblings'); - + // Check if parent is a Doc type and has a childDocs function if (parent && typeof parent === 'object' && 'childDocs' in parent && typeof parent.childDocs === 'function') { try { @@ -356,7 +355,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } }); } - + // Process the ChatBox document if available if (this.chatBox.Document) { this.processDocument(this.chatBox.Document); @@ -366,11 +365,11 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } console.log(`DocumentMetadataTool found ${this.documentsById.size} total documents`); - + // If we didn't find any documents, try a fallback method if (this.documentsById.size === 0 && this.chatBox) { console.log('No documents found, trying fallback method'); - + // Try to access any field that might contain documents if (this.chatBox.props && this.chatBox.props.documents) { const documents = this.chatBox.props.documents; @@ -396,17 +395,17 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp private processDocument(doc: Doc) { // Ensure document has a persistent ID const docId = this.ensureDocumentId(doc); - + // Only add if we haven't already processed this document if (!this.documentsById.has(docId)) { this.documentsById.set(docId, doc); - + // Get layout doc (the document itself or its layout) const layoutDoc = Doc.Layout(doc); if (layoutDoc) { this.layoutDocsById.set(docId, layoutDoc); } - + // Get data doc const dataDoc = doc[DocData]; if (dataDoc) { @@ -422,15 +421,15 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp */ private ensureDocumentId(doc: Doc): string { let docId: string | undefined; - + // First try to get the ID from our custom field if (doc[this.DOCUMENT_ID_FIELD]) { docId = String(doc[this.DOCUMENT_ID_FIELD]); return docId; } - + // Try different ways to get a document ID - + // 1. Try the direct id property if it exists if (doc.id && typeof doc.id === 'string') { docId = doc.id; @@ -448,14 +447,14 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp docId = uuidv4(); console.log(`Generated new UUID for document with title: ${doc.title || 'Untitled'}`); } - + // Store the ID in the document's metadata so it persists try { doc[this.DOCUMENT_ID_FIELD] = docId; } catch (e) { console.warn(`Could not assign ID to document property`, e); } - + return docId; } @@ -472,7 +471,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp const layoutDoc = this.layoutDocsById.get(docId); const dataDoc = this.dataDocsById.get(docId); - + const metadata: Record<string, any> = { id: docId, title: doc.title || '', @@ -488,7 +487,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp Object.keys(this.fieldMetadata).forEach(fieldName => { const fieldDef = this.fieldMetadata[fieldName]; const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; - + // Check if field exists on layout document let layoutValue = undefined; if (layoutDoc) { @@ -499,7 +498,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp metadata.fieldLocationMap[strippedName] = 'layout'; } } - + // Check if field exists on data document let dataValue = undefined; if (dataDoc) { @@ -512,12 +511,12 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } } } - + // For fields with stripped names (without leading underscore), // also check if they exist on documents without the underscore if (fieldName.startsWith('_')) { const nonUnderscoreFieldName = fieldName.substring(1); - + if (layoutDoc) { const nonUnderscoreLayoutValue = layoutDoc[nonUnderscoreFieldName]; if (nonUnderscoreLayoutValue !== undefined) { @@ -525,7 +524,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp metadata.fieldLocationMap[nonUnderscoreFieldName] = 'layout'; } } - + if (dataDoc) { const nonUnderscoreDataValue = dataDoc[nonUnderscoreFieldName]; if (nonUnderscoreDataValue !== undefined) { @@ -544,7 +543,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp metadata.fields.layout.width = metadata.fields.layout._width; metadata.fieldLocationMap.width = 'layout'; } - + if (metadata.fields.layout._height !== undefined && metadata.fields.layout.height === undefined) { metadata.fields.layout.height = metadata.fields.layout._height; metadata.fieldLocationMap.height = 'layout'; @@ -560,18 +559,22 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp * @param fieldValue The new value for the field (string, number, or boolean) * @returns Object with success status, message, and additional information */ - private editDocumentField(docId: string, fieldName: string, fieldValue: string | number | boolean): { - success: boolean; - message: string; - fieldName?: string; - originalFieldName?: string; + private editDocumentField( + docId: string, + fieldName: string, + fieldValue: string | number | boolean + ): { + success: boolean; + message: string; + fieldName?: string; + originalFieldName?: string; newValue?: any; warning?: string; } { // Normalize field name (handle with/without underscore) let normalizedFieldName = fieldName.startsWith('_') ? fieldName : fieldName; const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; - + // Handle common field name aliases (width → _width, height → _height) // Many document fields use '_' prefix for layout properties if (fieldName === 'width') { @@ -579,36 +582,36 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } else if (fieldName === 'height') { normalizedFieldName = '_height'; } - + // Get the documents const doc = this.documentsById.get(docId); if (!doc) { return { success: false, message: `Document with ID ${docId} not found` }; } - + const layoutDoc = this.layoutDocsById.get(docId); const dataDoc = this.dataDocsById.get(docId); - + if (!layoutDoc && !dataDoc) { return { success: false, message: `Could not find layout or data document for document with ID ${docId}` }; } - + try { // Convert the field value to the appropriate type based on field metadata const convertedValue = this.convertFieldValue(normalizedFieldName, fieldValue); - + let targetDoc: Doc | undefined; let targetLocation: string; - + // First, check if field exists on layout document using Doc.Get if (layoutDoc) { const fieldExistsOnLayout = Doc.Get(layoutDoc, normalizedFieldName, true) !== undefined; - + // If it exists on layout document, update it there if (fieldExistsOnLayout) { targetDoc = layoutDoc; targetLocation = 'layout'; - } + } // If it has an underscore prefix, it's likely a layout property even if not yet set else if (normalizedFieldName.startsWith('_')) { targetDoc = layoutDoc; @@ -618,13 +621,13 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp else if (dataDoc) { targetDoc = dataDoc; targetLocation = 'data'; - } + } // If no data document available, default to layout else { targetDoc = layoutDoc; targetLocation = 'layout'; } - } + } // If no layout document, use data document else if (dataDoc) { targetDoc = dataDoc; @@ -632,26 +635,26 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } else { return { success: false, message: `No valid document found for editing` }; } - + if (!targetDoc) { return { success: false, message: `Target document not available` }; } - + // Set the field value on the target document targetDoc[normalizedFieldName] = convertedValue; - - return { - success: true, + + return { + success: true, message: `Successfully updated field '${normalizedFieldName}' on ${targetLocation} document (ID: ${docId})`, fieldName: normalizedFieldName, originalFieldName: fieldName, - newValue: convertedValue + newValue: convertedValue, }; } catch (error) { console.error('Error editing document field:', error); - return { - success: false, - message: `Error updating field: ${error instanceof Error ? error.message : String(error)}` + return { + success: false, + message: `Error updating field: ${error instanceof Error ? error.message : String(error)}`, }; } } @@ -667,7 +670,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp if (typeof fieldValue === 'number' || typeof fieldValue === 'boolean') { return fieldValue; } - + // If fieldValue is a string "true" or "false", convert to boolean if (typeof fieldValue === 'string') { if (fieldValue.toLowerCase() === 'true') { @@ -677,12 +680,12 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp return false; } } - + // If fieldValue is not a string (and not a number or boolean), convert it to string if (typeof fieldValue !== 'string') { fieldValue = String(fieldValue); } - + // Special handling for text field - convert to proper RichTextField format if (fieldName === 'text') { try { @@ -710,28 +713,28 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp return JSON.stringify(rtf); } } - + // Get field metadata const normalizedFieldName = fieldName.startsWith('_') ? fieldName : `_${fieldName}`; const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; - + // Check both versions of the field name in metadata const fieldMeta = this.fieldMetadata[normalizedFieldName] || this.fieldMetadata[strippedFieldName]; - + // Special handling for width and height without metadata if (!fieldMeta && (fieldName === '_width' || fieldName === '_height' || fieldName === 'width' || fieldName === 'height')) { const num = Number(fieldValue); return isNaN(num) ? fieldValue : num; } - + if (!fieldMeta) { // If no metadata found, just return the string value return fieldValue; } - + // Convert based on field type const fieldType = fieldMeta.type; - + if (fieldType === 'boolean') { // Convert to boolean return fieldValue.toLowerCase() === 'true'; @@ -761,7 +764,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp return fieldValue; } } - + // Default to string return fieldValue; } @@ -775,7 +778,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp if (value === undefined || value === null) { return null; } - + // Handle Doc objects if (value instanceof Doc) { return { @@ -785,7 +788,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp docType: value.type || '', }; } - + // Handle RichTextField (try to extract plain text) if (typeof value === 'string' && value.includes('"type":"doc"') && value.includes('"content":')) { try { @@ -802,9 +805,9 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp node.content.forEach((child: any) => extractText(child)); } }; - + extractText(rtfObj.doc); - + // If we successfully extracted text, show it, but also preserve the original value if (plainText) { return { @@ -819,14 +822,14 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp // If parsing fails, just treat as a regular string } } - + // Handle arrays and complex objects if (typeof value === 'object') { // If the object has a toString method, use it if (value.toString && value.toString !== Object.prototype.toString) { return value.toString(); } - + try { // Try to convert to JSON string return JSON.stringify(value); @@ -834,7 +837,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp return '[Complex Object]'; } } - + // Return primitive values as is return value; } @@ -856,7 +859,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp list: [], date: [], enumeration: [], - other: [] + other: [], }, fieldNameMappings: {}, commonFields: { @@ -865,19 +868,19 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp size: [], content: [], behavior: [], - layout: [] - } + layout: [], + }, }; // Process each field in the metadata Object.entries(this.fieldMetadata).forEach(([fieldName, fieldInfo]) => { const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; - + // Add to fieldNameMappings if (fieldName.startsWith('_')) { result.fieldNameMappings[strippedName] = fieldName; } - + // Create structured field metadata const fieldData: Record<string, any> = { name: fieldName, @@ -886,10 +889,10 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp type: fieldInfo.fieldType || 'unknown', possibleValues: fieldInfo.values || [], }; - + // Add field to fields collection result.fields[fieldName] = fieldData; - + // Categorize by field type const type = fieldInfo.fieldType?.toLowerCase() || 'unknown'; if (type === 'string') { @@ -909,7 +912,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } else { result.fieldsByType.other.push(fieldName); } - + // Categorize by field purpose if (fieldName.includes('width') || fieldName.includes('height') || fieldName.includes('size')) { result.commonFields.size.push(fieldName); @@ -925,23 +928,23 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp result.commonFields.layout.push(fieldName); } }); - + // Add special section for auto-sizing related fields result.autoSizingFields = { height: { autoHeightField: '_layout_autoHeight', heightField: '_height', displayName: 'height', - usage: 'To manually set height, first set layout_autoHeight to false' + usage: 'To manually set height, first set layout_autoHeight to false', }, width: { autoWidthField: '_layout_autoWidth', widthField: '_width', displayName: 'width', - usage: 'To manually set width, first set layout_autoWidth to false' - } + usage: 'To manually set width, first set layout_autoWidth to false', + }, }; - + // Add special section for text field format result.specialFields = { text: { @@ -949,10 +952,10 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp description: 'Document text content', format: 'RichTextField', note: 'When setting text, provide plain text - it will be automatically converted to the correct format', - example: 'For setting: "Hello world" (plain text); For getting: Will be converted to plaintext for display' - } + example: 'For setting: "Hello world" (plain text); For getting: Will be converted to plaintext for display', + }, }; - + return result; } @@ -963,121 +966,129 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp */ async execute(args: ParametersType<DocumentMetadataToolParamsType>): Promise<Observation[]> { console.log('DocumentMetadataTool: Executing with args:', args); - + // Find all documents in the Freeform view this.findDocumentsInFreeformView(); - + try { // Validate required input parameters based on action if (!this.inputValidator(args)) { - return [{ - type: 'text', - text: `Error: Invalid or missing parameters for action "${args.action}". ${this.getParameterRequirementsByAction(String(args.action))}` - }]; + return [ + { + type: 'text', + text: `Error: Invalid or missing parameters for action "${args.action}". ${this.getParameterRequirementsByAction(String(args.action))}`, + }, + ]; } - + // Ensure the action is valid and convert to string const action = String(args.action); if (!['get', 'edit', 'list', 'getFieldOptions', 'create'].includes(action)) { - return [{ - type: 'text', - text: 'Error: Invalid action. Valid actions are "get", "edit", "list", "getFieldOptions", or "create".' - }]; + return [ + { + type: 'text', + text: 'Error: Invalid action. Valid actions are "get", "edit", "list", "getFieldOptions", or "create".', + }, + ]; } // Safely convert documentId to string or undefined const documentId = args.documentId ? String(args.documentId) : undefined; - + // Perform the specified action switch (action) { case 'get': { // Get metadata for a specific document or all documents const result = this.getDocumentMetadata(documentId); console.log('DocumentMetadataTool: Get metadata result:', result); - return [{ - type: 'text', - text: `Document metadata ${documentId ? 'for document ' + documentId : ''} retrieved successfully:\n${JSON.stringify(result, null, 2)}` - }]; + return [ + { + type: 'text', + text: `Document metadata ${documentId ? 'for document ' + documentId : ''} retrieved successfully:\n${JSON.stringify(result, null, 2)}`, + }, + ]; } - + case 'edit': { // Edit a specific field on a document if (!documentId) { - return [{ - type: 'text', - text: 'Error: Document ID is required for edit actions.' - }]; + return [ + { + type: 'text', + text: 'Error: Document ID is required for edit actions.', + }, + ]; } - + // Ensure document exists if (!this.documentsById.has(documentId)) { - return [{ - type: 'text', - text: `Error: Document with ID ${documentId} not found.` - }]; + return [ + { + type: 'text', + text: `Error: Document with ID ${documentId} not found.`, + }, + ]; } - + // Check if we're doing a multi-field edit or a single field edit if (args.fieldEdits) { try { // Parse fieldEdits array const edits = JSON.parse(String(args.fieldEdits)); if (!Array.isArray(edits) || edits.length === 0) { - return [{ - type: 'text', - text: 'Error: fieldEdits must be a non-empty array of field edits.' - }]; + return [ + { + type: 'text', + text: 'Error: fieldEdits must be a non-empty array of field edits.', + }, + ]; } - + // Track results for all edits - const results: { - success: boolean; - message: string; - fieldName?: string; - originalFieldName?: string; + const results: { + success: boolean; + message: string; + fieldName?: string; + originalFieldName?: string; newValue?: any; warning?: string; }[] = []; - + let allSuccessful = true; - + // Process each edit for (const edit of edits) { // Get fieldValue in its original form let fieldValue = edit.fieldValue; - + // Only convert to string if it's neither boolean nor number if (typeof fieldValue !== 'boolean' && typeof fieldValue !== 'number') { fieldValue = String(fieldValue); } - + const fieldName = String(edit.fieldName); - + // Edit the field - const result = this.editDocumentField( - documentId, - fieldName, - fieldValue - ); - + const result = this.editDocumentField(documentId, fieldName, fieldValue); + console.log(`DocumentMetadataTool: Edit field result for ${fieldName}:`, result); - + // Add to results results.push(result); - + // Update success status if (!result.success) { allSuccessful = false; } } - + // Format response based on results let responseText = ''; if (allSuccessful) { responseText = `Successfully edited ${results.length} fields on document ${documentId}:\n`; results.forEach(result => { responseText += `- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}\n`; - + // Add any warnings if (result.warning) { responseText += ` Warning: ${result.warning}\n`; @@ -1088,7 +1099,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp results.forEach(result => { if (result.success) { responseText += `- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}\n`; - + // Add any warnings if (result.warning) { responseText += ` Warning: ${result.warning}\n`; @@ -1098,120 +1109,134 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } }); } - + // Get the updated metadata to return const updatedMetadata = this.getDocumentMetadata(documentId); - - return [{ - type: 'text', - text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}` - }]; + + return [ + { + type: 'text', + text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}`, + }, + ]; } catch (error) { - return [{ - type: 'text', - text: `Error processing fieldEdits: ${error instanceof Error ? error.message : String(error)}` - }]; + return [ + { + type: 'text', + text: `Error processing fieldEdits: ${error instanceof Error ? error.message : String(error)}`, + }, + ]; } } else { // Single field edit (original behavior) if (!args.fieldName) { - return [{ - type: 'text', - text: 'Error: Field name and field value are required for edit actions.' - }]; + return [ + { + type: 'text', + text: 'Error: Field name and field value are required for edit actions.', + }, + ]; } - + // Get fieldValue in its original form - we'll handle conversion in editDocumentField let fieldValue = args.fieldValue; - + // Only convert to string if it's neither boolean nor number if (typeof fieldValue !== 'boolean' && typeof fieldValue !== 'number') { fieldValue = String(fieldValue); } - + const fieldName = String(args.fieldName); - + // Edit the field - const result = this.editDocumentField( - documentId, - fieldName, - fieldValue - ); - + const result = this.editDocumentField(documentId, fieldName, fieldValue); + console.log('DocumentMetadataTool: Edit field result:', result); - + if (!result.success) { return [{ type: 'text', text: result.message }]; } - + // Include warning if present let responseText = result.message; if (result.warning) { responseText += `\n\n${result.warning}`; } - + // Get the updated metadata to return const updatedMetadata = this.getDocumentMetadata(documentId); - - return [{ - type: 'text', - text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}` - }]; + + return [ + { + type: 'text', + text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}`, + }, + ]; } } - + case 'list': { // List all available documents in simple format const docs = Array.from(this.documentsById.entries()).map(([id, doc]) => ({ id, title: doc.title || 'Untitled Document', - type: doc.type || 'Unknown Type' + type: doc.type || 'Unknown Type', })); - + if (docs.length === 0) { - return [{ - type: 'text', - text: 'No documents found in the current view.' - }]; + return [ + { + type: 'text', + text: 'No documents found in the current view.', + }, + ]; } - - return [{ - type: 'text', - text: `Found ${docs.length} document(s) in the current view:\n${JSON.stringify(docs, null, 2)}` - }]; + + return [ + { + type: 'text', + text: `Found ${docs.length} document(s) in the current view:\n${JSON.stringify(docs, null, 2)}`, + }, + ]; } - + case 'getFieldOptions': { // Get all available field options with metadata const fieldOptions = this.getAllFieldMetadata(); - - return [{ - type: 'text', - text: `Document field options retrieved successfully.\nThis information should be consulted before editing document fields to understand available options and dependencies:\n${JSON.stringify(fieldOptions, null, 2)}` - }]; + + return [ + { + type: 'text', + text: `Document field options retrieved successfully.\nThis information should be consulted before editing document fields to understand available options and dependencies:\n${JSON.stringify(fieldOptions, null, 2)}`, + }, + ]; } - + case 'create': { // Create a new document if (!args.title || !args.data || !args.doc_type) { - return [{ - type: 'text', - text: 'Error: Title, data, and doc_type are required for create action.' - }]; + return [ + { + type: 'text', + text: 'Error: Title, data, and doc_type are required for create action.', + }, + ]; } - + const docType = String(args.doc_type); const title = String(args.title); const data = String(args.data); - + // Validate doc_type if (!this.isValidDocType(docType)) { - return [{ - type: 'text', - text: `Error: Invalid doc_type. Valid options are: ${Object.keys(supportedDocTypes).join(',')}` - }]; + return [ + { + type: 'text', + text: `Error: Invalid doc_type. Valid options are: ${Object.keys(supportedDocTypes).join(',')}`, + }, + ]; } - + try { // Create simple document with just title and data const simpleDoc: parsedDoc = { @@ -1223,35 +1248,40 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp _width: 300, _height: 300, _layout_fitWidth: false, - _layout_autoHeight: true + _layout_autoHeight: true, }; - + // Use the chatBox's createDocInDash method to create and link the document if (!this.chatBox || !this.chatBox.createDocInDash) { - return [{ - type: 'text', - text: 'Error: Could not access document creation functionality.' - }]; + return [ + { + type: 'text', + text: 'Error: Could not access document creation functionality.', + }, + ]; } - + const createdDoc = this.chatBox.createDocInDash(simpleDoc); - + if (!createdDoc) { - return [{ - type: 'text', - text: 'Error: Failed to create document.' - }]; + return [ + { + type: 'text', + text: 'Error: Failed to create document.', + }, + ]; } - + // Update our local document maps with the new document this.processDocument(createdDoc); - + // Get the created document's metadata const createdMetadata = this.getDocumentMetadata(createdDoc.id); - - return [{ - type: 'text', - text: `Document created successfully. + + return [ + { + type: 'text', + text: `Document created successfully. Document ID: ${createdDoc.id} Type: ${docType} Title: "${title}" @@ -1266,36 +1296,43 @@ Next steps: 4. For text documents, you can edit the content with: { action: "edit", documentId: "${createdDoc.id}", fieldName: "text", fieldValue: "New content" } Full metadata for the created document: -${JSON.stringify(createdMetadata, null, 2)}` - }]; +${JSON.stringify(createdMetadata, null, 2)}`, + }, + ]; } catch (error) { - return [{ - type: 'text', - text: `Error creating document: ${error instanceof Error ? error.message : String(error)}` - }]; + return [ + { + type: 'text', + text: `Error creating document: ${error instanceof Error ? error.message : String(error)}`, + }, + ]; } } - + default: - return [{ - type: 'text', - text: 'Error: Unknown action. Valid actions are "get", "edit", "list", "getFieldOptions", or "create".' - }]; + return [ + { + type: 'text', + text: 'Error: Unknown action. Valid actions are "get", "edit", "list", "getFieldOptions", or "create".', + }, + ]; } } catch (error) { console.error('DocumentMetadataTool execution error:', error); - return [{ - type: 'text', - text: `Error executing DocumentMetadataTool: ${error instanceof Error ? error.message : String(error)}` - }]; + return [ + { + type: 'text', + text: `Error executing DocumentMetadataTool: ${error instanceof Error ? error.message : String(error)}`, + }, + ]; } } /** * Validates the input parameters for the DocumentMetadataTool - * This custom validator allows numbers and booleans to be passed for fieldValue + * This custom validator allows numbers and booleans to be passed for fieldValue * while maintaining compatibility with the standard validation - * + * * @param params The parameters to validate * @returns True if the parameters are valid, false otherwise */ @@ -1304,12 +1341,12 @@ ${JSON.stringify(createdMetadata, null, 2)}` if (params.action === undefined) { return false; } - + // For create action, validate required parameters if (params.action === 'create') { return !!(params.title && params.data && params.doc_type); } - + // For edit action, validate either single field edit or multiple field edits if (params.action === 'edit') { // If fieldEdits is provided, it must be valid and we'll ignore fieldName/fieldValue @@ -1317,13 +1354,13 @@ ${JSON.stringify(createdMetadata, null, 2)}` try { // Parse fieldEdits and validate its structure const edits = JSON.parse(String(params.fieldEdits)); - + // Ensure it's an array if (!Array.isArray(edits)) { console.log('fieldEdits is not an array'); return false; } - + // Ensure each item has fieldName and fieldValue for (const edit of edits) { if (!edit.fieldName) { @@ -1335,7 +1372,7 @@ ${JSON.stringify(createdMetadata, null, 2)}` return false; } } - + // Everything looks good with fieldEdits return !!params.documentId; // Just ensure documentId is provided } catch (error) { @@ -1349,22 +1386,22 @@ ${JSON.stringify(createdMetadata, null, 2)}` } } } - + // For get action with documentId, documentId is required if (params.action === 'get' && params.documentId === '') { return false; } - + // getFieldOptions action doesn't require any additional parameters if (params.action === 'getFieldOptions') { return true; } - + // list action doesn't require any additional parameters if (params.action === 'list') { return true; } - + // Allow for numeric or boolean fieldValue even though the type is defined as string if (params.fieldValue !== undefined) { if (typeof params.fieldValue === 'number') { @@ -1372,14 +1409,14 @@ ${JSON.stringify(createdMetadata, null, 2)}` // We'll convert it later, so don't fail validation return true; } - + if (typeof params.fieldValue === 'boolean') { console.log('Boolean fieldValue detected, will be converted appropriately'); // We'll handle boolean conversion in the execute method return true; } } - + return true; } @@ -1424,11 +1461,11 @@ ${JSON.stringify(createdMetadata, null, 2)}` for (const docId of this.documentsById.keys()) { documentsMetadata[docId] = this.extractDocumentMetadata(docId); } - + return { documentCount: this.documentsById.size, documents: documentsMetadata, - fieldDefinitions: this.fieldMetadata + fieldDefinitions: this.fieldMetadata, }; } } @@ -1441,4 +1478,4 @@ ${JSON.stringify(createdMetadata, null, 2)}` private isValidDocType(docType: string): boolean { return Object.values(supportedDocTypes).includes(docType as supportedDocTypes); } -} \ No newline at end of file +} diff --git a/src/server/chunker/pdf_chunker.py b/src/server/chunker/pdf_chunker.py index 9d4cfb1b2..feb437f1f 100644 --- a/src/server/chunker/pdf_chunker.py +++ b/src/server/chunker/pdf_chunker.py @@ -602,12 +602,11 @@ class PDFChunker: try: response = self.client.messages.create( - model='claude-3-5-sonnet-20240620', + model='claude-3-7-sonnet-20250219', system=prompt, max_tokens=400 * len(images), # Increased token limit for more detailed summaries messages=messages, temperature=0, - extra_headers={"anthropic-beta": "max-tokens-3-5-sonnet-2024-07-15"} ) # Parse the response @@ -767,7 +766,7 @@ class Document: client = OpenAI() # Initialize OpenAI client for text generation completion = client.chat.completions.create( - model="gpt-3.5-turbo", # Specify the language model + model="gpt-4o", # Specify the language model messages=[ {"role": "system", "content": "You are an AI assistant tasked with summarizing a document. You are provided with important chunks from the document and provide a summary, as best you can, of what the document will contain overall. Be concise and brief with your response."}, -- cgit v1.2.3-70-g09d2 From aa3f1228567102dffe38aa5c9e916dfcb41218fc Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" <Shulman.aj@gmail.com> Date: Thu, 10 Apr 2025 12:41:42 -0400 Subject: getting rid of single field edits. --- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 316 ++++++++------------- 1 file changed, 122 insertions(+), 194 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index a3d86287d..eeb9091f8 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -25,23 +25,12 @@ const parameterDefinitions: ReadonlyArray<Parameter> = [ required: false, description: 'The ID of the document to get or edit metadata for. Required for "edit", optional for "get", ignored for "list", "getFieldOptions", and "create"', }, - { - name: 'fieldName', - type: 'string', - required: false, - description: 'The name of the field to edit. Required for single field edits. Ignored if fieldEdits is provided', - }, - { - name: 'fieldValue', - type: 'string', - required: false, - description: 'The new value for the field. Required for single field edits. Can be a string, number, or boolean value depending on the field type', - }, { name: 'fieldEdits', type: 'string', required: false, - description: 'JSON array of field edits for editing multiple fields at once. Each item should have fieldName and fieldValue. Example: [{"fieldName":"layout_autoHeight","fieldValue":false},{"fieldName":"height","fieldValue":300}]', + description: + 'JSON array of field edits for editing fields. Each item should have fieldName and fieldValue. For single field edits, use an array with one item. Example: [{"fieldName":"layout_autoHeight","fieldValue":false},{"fieldName":"height","fieldValue":300}]', }, { name: 'title', @@ -101,13 +90,9 @@ IMPORTANT: Some fields have dependencies that must be handled for edits to work - When editing "height", first set "layout_autoHeight" to false (as a boolean value, not a string) - When editing "width", first set "layout_autoWidth" to false (as a boolean value, not a string) - Check document metadata to identify other similar dependencies -- You can edit dependent fields in a single operation using the fieldEdits parameter - -Example: To change document height, first disable auto-height: -1. {... inputs: { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false }} -2. {... inputs: { action: "edit", documentId: "doc123", fieldName: "height", fieldValue: 300 }} +- All edits are done using the fieldEdits parameter which accepts an array of fields to modify -OR using multi-field edit (recommended for dependent fields): +Example: To change document height, disable auto-height and set height in a single operation: {... inputs: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 300 } @@ -135,9 +120,9 @@ To CREATE a new document: - After creation, you can edit the document with more specific properties To EDIT document metadata: -- Use action="edit" with required documentId, and either: - 1. fieldName + fieldValue for single field edits, OR - 2. fieldEdits for updating multiple fields at once +- Use action="edit" with required parameters: + - documentId: The ID of the document to edit + - fieldEdits: JSON array of fields to edit, each with fieldName and fieldValue - The tool will determine the correct document location automatically - Field names can be provided with or without leading underscores (e.g., both "width" and "_width" work) - Common fields like "width" and "height" are automatically mapped to "_width" and "_height" @@ -146,7 +131,7 @@ To EDIT document metadata: SPECIAL FIELD HANDLING: - Text fields: When editing the 'text' field, provide simple plain text - Example: {...inputs: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "Hello world" }} + Example: {...inputs: { action: "edit", documentId: "doc123", fieldEdits: [{ fieldName: "text", fieldValue: "Hello world" }] }} The tool will automatically convert your text to the proper RichTextField format - Width/Height: Set layout_autoHeight/layout_autoWidth to false before editing @@ -163,7 +148,7 @@ OR HANDLING DEPENDENT FIELDS: - When editing some fields, you may need to update related dependent fields - For example, when changing "height", you should also set "layout_autoHeight" to false -- Use the fieldEdits parameter to update dependent fields in a single operation (recommended): +- Use the fieldEdits parameter to update dependent fields in a single operation: {...inputs: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 300 } @@ -189,13 +174,13 @@ Examples: - To list all documents: { action: "list" } - To get all document metadata: { action: "get" } - To get metadata for a specific document: { action: "get", documentId: "doc123" } -- To edit a single field: { action: "edit", documentId: "doc123", fieldName: "backgroundColor", fieldValue: "#ff0000" } -- To edit a width property: { action: "edit", documentId: "doc123", fieldName: "width", fieldValue: 300 } -- To edit text content: { action: "edit", documentId: "doc123", fieldName: "text", fieldValue: "Simple plain text goes here" } -- To disable auto-height: { action: "edit", documentId: "doc123", fieldName: "layout_autoHeight", fieldValue: false } +- To edit a single field: { action: "edit", documentId: "doc123", fieldEdits: [{ fieldName: "backgroundColor", fieldValue: "#ff0000" }] } +- To edit a width property: { action: "edit", documentId: "doc123", fieldEdits: [{ fieldName: "width", fieldValue: 300 }] } +- To edit text content: { action: "edit", documentId: "doc123", fieldEdits: [{ fieldName: "text", fieldValue: "Simple plain text goes here" }] } +- To disable auto-height: { action: "edit", documentId: "doc123", fieldEdits: [{ fieldName: "layout_autoHeight", fieldValue: false }] } - To create a text document: { action: "create", title: "My Notes", data: "This is my note content", doc_type: "text" } - To create a web document: { action: "create", title: "Google", data: "https://www.google.com", doc_type: "web" } -- To edit height with its dependent field together (recommended): +- To edit height with its dependent field together: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 200 } @@ -1030,137 +1015,92 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp ]; } - // Check if we're doing a multi-field edit or a single field edit - if (args.fieldEdits) { - try { - // Parse fieldEdits array - const edits = JSON.parse(String(args.fieldEdits)); - if (!Array.isArray(edits) || edits.length === 0) { - return [ - { - type: 'text', - text: 'Error: fieldEdits must be a non-empty array of field edits.', - }, - ]; - } - - // Track results for all edits - const results: { - success: boolean; - message: string; - fieldName?: string; - originalFieldName?: string; - newValue?: any; - warning?: string; - }[] = []; - - let allSuccessful = true; - - // Process each edit - for (const edit of edits) { - // Get fieldValue in its original form - let fieldValue = edit.fieldValue; - - // Only convert to string if it's neither boolean nor number - if (typeof fieldValue !== 'boolean' && typeof fieldValue !== 'number') { - fieldValue = String(fieldValue); - } - - const fieldName = String(edit.fieldName); - - // Edit the field - const result = this.editDocumentField(documentId, fieldName, fieldValue); - - console.log(`DocumentMetadataTool: Edit field result for ${fieldName}:`, result); - - // Add to results - results.push(result); - - // Update success status - if (!result.success) { - allSuccessful = false; - } - } - - // Format response based on results - let responseText = ''; - if (allSuccessful) { - responseText = `Successfully edited ${results.length} fields on document ${documentId}:\n`; - results.forEach(result => { - responseText += `- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}\n`; - - // Add any warnings - if (result.warning) { - responseText += ` Warning: ${result.warning}\n`; - } - }); - } else { - responseText = `Errors occurred while editing fields on document ${documentId}:\n`; - results.forEach(result => { - if (result.success) { - responseText += `- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}\n`; - - // Add any warnings - if (result.warning) { - responseText += ` Warning: ${result.warning}\n`; - } - } else { - responseText += `- Error editing '${result.originalFieldName}': ${result.message}\n`; - } - }); - } - - // Get the updated metadata to return - const updatedMetadata = this.getDocumentMetadata(documentId); + // Check for fieldEdits parameter + if (!args.fieldEdits) { + return [ + { + type: 'text', + text: 'Error: fieldEdits is required for edit actions. Please provide a JSON array of field edits.', + }, + ]; + } + try { + // Parse fieldEdits array + const edits = JSON.parse(String(args.fieldEdits)); + if (!Array.isArray(edits) || edits.length === 0) { return [ { type: 'text', - text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}`, - }, - ]; - } catch (error) { - return [ - { - type: 'text', - text: `Error processing fieldEdits: ${error instanceof Error ? error.message : String(error)}`, - }, - ]; - } - } else { - // Single field edit (original behavior) - if (!args.fieldName) { - return [ - { - type: 'text', - text: 'Error: Field name and field value are required for edit actions.', + text: 'Error: fieldEdits must be a non-empty array of field edits.', }, ]; } - // Get fieldValue in its original form - we'll handle conversion in editDocumentField - let fieldValue = args.fieldValue; + // Track results for all edits + const results: { + success: boolean; + message: string; + fieldName?: string; + originalFieldName?: string; + newValue?: any; + warning?: string; + }[] = []; + + let allSuccessful = true; + + // Process each edit + for (const edit of edits) { + // Get fieldValue in its original form + let fieldValue = edit.fieldValue; + + // Only convert to string if it's neither boolean nor number + if (typeof fieldValue !== 'boolean' && typeof fieldValue !== 'number') { + fieldValue = String(fieldValue); + } - // Only convert to string if it's neither boolean nor number - if (typeof fieldValue !== 'boolean' && typeof fieldValue !== 'number') { - fieldValue = String(fieldValue); - } + const fieldName = String(edit.fieldName); - const fieldName = String(args.fieldName); + // Edit the field + const result = this.editDocumentField(documentId, fieldName, fieldValue); - // Edit the field - const result = this.editDocumentField(documentId, fieldName, fieldValue); + console.log(`DocumentMetadataTool: Edit field result for ${fieldName}:`, result); - console.log('DocumentMetadataTool: Edit field result:', result); + // Add to results + results.push(result); - if (!result.success) { - return [{ type: 'text', text: result.message }]; + // Update success status + if (!result.success) { + allSuccessful = false; + } } - // Include warning if present - let responseText = result.message; - if (result.warning) { - responseText += `\n\n${result.warning}`; + // Format response based on results + let responseText = ''; + if (allSuccessful) { + responseText = `Successfully edited ${results.length} fields on document ${documentId}:\n`; + results.forEach(result => { + responseText += `- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}\n`; + + // Add any warnings + if (result.warning) { + responseText += ` Warning: ${result.warning}\n`; + } + }); + } else { + responseText = `Errors occurred while editing fields on document ${documentId}:\n`; + results.forEach(result => { + if (result.success) { + responseText += `- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}\n`; + + // Add any warnings + if (result.warning) { + responseText += ` Warning: ${result.warning}\n`; + } + } else { + responseText += `- Error editing '${result.originalFieldName}': ${result.message}\n`; + } + }); } // Get the updated metadata to return @@ -1172,6 +1112,13 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp text: `${responseText}\nUpdated metadata:\n${JSON.stringify(updatedMetadata, null, 2)}`, }, ]; + } catch (error) { + return [ + { + type: 'text', + text: `Error processing fieldEdits: ${error instanceof Error ? error.message : String(error)}`, + }, + ]; } } @@ -1291,9 +1238,9 @@ You can now use the "edit" action to modify additional properties of this docume Next steps: 1. Use the "getFieldOptions" action to understand available editable/addable fields/properties and their dependencies. -2. To modify this document, use: { action: "edit", documentId: "${createdDoc.id}", fieldName: "property", fieldValue: "value" } +2. To modify this document, use: { action: "edit", documentId: "${createdDoc.id}", fieldEdits: [{"fieldName":"property","fieldValue":"value"}] } 3. To add styling, consider setting backgroundColor, fontColor, or other properties -4. For text documents, you can edit the content with: { action: "edit", documentId: "${createdDoc.id}", fieldName: "text", fieldValue: "New content" } +4. For text documents, you can edit the content with: { action: "edit", documentId: "${createdDoc.id}", fieldEdits: [{"fieldName":"text","fieldValue":"New content"}] } Full metadata for the created document: ${JSON.stringify(createdMetadata, null, 2)}`, @@ -1347,43 +1294,39 @@ ${JSON.stringify(createdMetadata, null, 2)}`, return !!(params.title && params.data && params.doc_type); } - // For edit action, validate either single field edit or multiple field edits + // For edit action, validate fieldEdits is provided if (params.action === 'edit') { - // If fieldEdits is provided, it must be valid and we'll ignore fieldName/fieldValue - if (params.fieldEdits) { - try { - // Parse fieldEdits and validate its structure - const edits = JSON.parse(String(params.fieldEdits)); - - // Ensure it's an array - if (!Array.isArray(edits)) { - console.log('fieldEdits is not an array'); - return false; - } + if (!params.documentId || !params.fieldEdits) { + return false; + } - // Ensure each item has fieldName and fieldValue - for (const edit of edits) { - if (!edit.fieldName) { - console.log('An edit is missing fieldName'); - return false; - } - if (edit.fieldValue === undefined) { - console.log('An edit is missing fieldValue'); - return false; - } - } + try { + // Parse fieldEdits and validate its structure + const edits = JSON.parse(String(params.fieldEdits)); - // Everything looks good with fieldEdits - return !!params.documentId; // Just ensure documentId is provided - } catch (error) { - console.log('Error parsing fieldEdits:', error); + // Ensure it's an array + if (!Array.isArray(edits)) { + console.log('fieldEdits is not an array'); return false; } - } else { - // Traditional single field edit - if (!params.documentId || !params.fieldName || params.fieldValue === undefined) { - return false; + + // Ensure each item has fieldName and fieldValue + for (const edit of edits) { + if (!edit.fieldName) { + console.log('An edit is missing fieldName'); + return false; + } + if (edit.fieldValue === undefined) { + console.log('An edit is missing fieldValue'); + return false; + } } + + // Everything looks good with fieldEdits + return true; + } catch (error) { + console.log('Error parsing fieldEdits:', error); + return false; } } @@ -1402,21 +1345,6 @@ ${JSON.stringify(createdMetadata, null, 2)}`, return true; } - // Allow for numeric or boolean fieldValue even though the type is defined as string - if (params.fieldValue !== undefined) { - if (typeof params.fieldValue === 'number') { - console.log('Numeric fieldValue detected, will be converted to string'); - // We'll convert it later, so don't fail validation - return true; - } - - if (typeof params.fieldValue === 'boolean') { - console.log('Boolean fieldValue detected, will be converted appropriately'); - // We'll handle boolean conversion in the execute method - return true; - } - } - return true; } @@ -1434,7 +1362,7 @@ ${JSON.stringify(createdMetadata, null, 2)}`, case 'get': return 'The "get" action accepts an optional documentId parameter.'; case 'edit': - return 'The "edit" action requires documentId, fieldName, and fieldValue parameters, or documentId and fieldEdits parameters for multi-field edits.'; + return 'The "edit" action requires documentId and fieldEdits parameters. fieldEdits must be a JSON array of field edits.'; case 'list': return 'The "list" action does not require any additional parameters.'; case 'getFieldOptions': -- cgit v1.2.3-70-g09d2 From 9de0e9844d5cbfbb385da34545006e617873b07a Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" <Shulman.aj@gmail.com> Date: Thu, 10 Apr 2025 12:41:58 -0400 Subject: adding edit and create at the same time --- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 171 +++++++++++++++++++-- 1 file changed, 154 insertions(+), 17 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index eeb9091f8..9781d7a51 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -30,7 +30,7 @@ const parameterDefinitions: ReadonlyArray<Parameter> = [ type: 'string', required: false, description: - 'JSON array of field edits for editing fields. Each item should have fieldName and fieldValue. For single field edits, use an array with one item. Example: [{"fieldName":"layout_autoHeight","fieldValue":false},{"fieldName":"height","fieldValue":300}]', + 'JSON array of field edits for editing fields. Each item should have fieldName and fieldValue. For single field edits, use an array with one item. Can be used with both "edit" and "create" actions. Example: [{"fieldName":"layout_autoHeight","fieldValue":false},{"fieldName":"height","fieldValue":300}]', }, { name: 'title', @@ -73,10 +73,12 @@ This tool provides the following capabilities: - Get detailed information about all available document fields - Support for all value types: strings, numbers, and booleans - Create new documents with basic properties +- Create new documents and immediately configure additional properties in a single operation DOCUMENT CREATION: - Use action="create" to create new documents with a simplified approach - Required parameters: title, data, and doc_type +- Optional parameter: fieldEdits to set additional properties during creation - The tool will create the document with sensible defaults and link it to the current view - After creation, you can use the edit action to update its properties @@ -96,7 +98,19 @@ Example: To change document height, disable auto-height and set height in a sing {... inputs: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 300 } -]}}`; +]}} + +Example: To create a document and set additional properties in a single operation: +{... inputs: { + action: "create", + title: "My Styled Note", + data: "This is my note content", + doc_type: "text", + fieldEdits: [ + { fieldName: "backgroundColor", fieldValue: "#f0f0f0" }, + { fieldName: "borderWidth", fieldValue: 1 } + ] +}}`; // Extensive usage guidelines for the tool const citationRules = `USAGE GUIDELINES: @@ -116,8 +130,10 @@ To CREATE a new document: - title: The title of the document to create - data: The content data for the document (text content, URL, etc.) - doc_type: The type of document to create (text, web, image, etc.) -- Example: {...inputs: { action: "create", title: "My Notes", data: "This is the content", doc_type: "text" }} -- After creation, you can edit the document with more specific properties +- Optionally include fieldEdits to set additional properties during creation +- Example with just basic properties: {...inputs: { action: "create", title: "My Notes", data: "This is the content", doc_type: "text" }} +- Example with additional properties: {...inputs: { action: "create", title: "My Notes", data: "This is the content", doc_type: "text", fieldEdits: [{ fieldName: "backgroundColor", fieldValue: "#f0f0f0" }] }} +- The fieldEdits parameter allows you to create and configure a document in a single operation To EDIT document metadata: - Use action="edit" with required parameters: @@ -141,9 +157,8 @@ RECOMMENDED WORKFLOW: 3. Get document metadata with action="get" to see current values 4. Edit fields with action="edit" using proper dependencies OR -1. Create a new document with action="create" -2. Get its ID from the response -3. Edit the document's properties with action="edit" +1. Create a new document with fieldEdits to configure it in one step: + { action: "create", title: "My Document", data: "Content", doc_type: "text", fieldEdits: [{ fieldName: "backgroundColor", fieldValue: "#f0f0f0" }] } HANDLING DEPENDENT FIELDS: - When editing some fields, you may need to update related dependent fields @@ -180,6 +195,7 @@ Examples: - To disable auto-height: { action: "edit", documentId: "doc123", fieldEdits: [{ fieldName: "layout_autoHeight", fieldValue: false }] } - To create a text document: { action: "create", title: "My Notes", data: "This is my note content", doc_type: "text" } - To create a web document: { action: "create", title: "Google", data: "https://www.google.com", doc_type: "web" } +- To create a document with custom styling: { action: "create", title: "Styled Note", data: "Content here", doc_type: "text", fieldEdits: [{ fieldName: "backgroundColor", fieldValue: "#f0f0f0" }] } - To edit height with its dependent field together: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, @@ -1222,28 +1238,115 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp // Update our local document maps with the new document this.processDocument(createdDoc); + // Apply any fieldEdits if provided + let fieldEditsResults = null; + if (args.fieldEdits) { + try { + // Parse fieldEdits array + const edits = JSON.parse(String(args.fieldEdits)); + if (Array.isArray(edits) && edits.length > 0) { + console.log(`DocumentMetadataTool: Applying ${edits.length} field edits to newly created document`); + + // Track results for all edits + const results: { + success: boolean; + message: string; + fieldName?: string; + originalFieldName?: string; + newValue?: any; + warning?: string; + }[] = []; + + let allSuccessful = true; + + // Process each edit + for (const edit of edits) { + // Get fieldValue in its original form + let fieldValue = edit.fieldValue; + + // Only convert to string if it's neither boolean nor number + if (typeof fieldValue !== 'boolean' && typeof fieldValue !== 'number') { + fieldValue = String(fieldValue); + } + + const fieldName = String(edit.fieldName); + + // Edit the field + const result = this.editDocumentField(createdDoc.id, fieldName, fieldValue); + + console.log(`DocumentMetadataTool: Edit field result for ${fieldName}:`, result); + + // Add to results + results.push(result); + + // Update success status + if (!result.success) { + allSuccessful = false; + } + } + + // Format edit results + fieldEditsResults = { + count: edits.length, + success: allSuccessful, + results, + }; + } + } catch (error) { + console.error('Error applying fieldEdits during document creation:', error); + fieldEditsResults = { + error: `Error applying fieldEdits: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + // Get the created document's metadata const createdMetadata = this.getDocumentMetadata(createdDoc.id); - return [ - { - type: 'text', - text: `Document created successfully. + // Build the response text with fieldEdits results if available + let responseText = `Document created successfully. Document ID: ${createdDoc.id} Type: ${docType} Title: "${title}" -The document has been created with default dimensions and positioning. -You can now use the "edit" action to modify additional properties of this document. +The document has been created with default dimensions and positioning.`; + + // Add field edits results if available + if (fieldEditsResults) { + if ('error' in fieldEditsResults) { + responseText += `\n\nAttempted to apply additional field edits but encountered an error: ${fieldEditsResults.error}`; + } else { + responseText += `\n\nAdditional properties applied: ${fieldEditsResults.count} field(s) ${fieldEditsResults.success ? 'successfully updated' : 'with some errors'}`; + + if (fieldEditsResults.results && fieldEditsResults.results.length > 0) { + responseText += '\nField edit results:'; + fieldEditsResults.results.forEach(result => { + if (result.success) { + responseText += `\n- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}`; + if (result.warning) { + responseText += ` (Warning: ${result.warning})`; + } + } else { + responseText += `\n- Error editing '${result.originalFieldName}': ${result.message}`; + } + }); + } + } + } -Next steps: + responseText += `\n\nNext steps: 1. Use the "getFieldOptions" action to understand available editable/addable fields/properties and their dependencies. 2. To modify this document, use: { action: "edit", documentId: "${createdDoc.id}", fieldEdits: [{"fieldName":"property","fieldValue":"value"}] } 3. To add styling, consider setting backgroundColor, fontColor, or other properties 4. For text documents, you can edit the content with: { action: "edit", documentId: "${createdDoc.id}", fieldEdits: [{"fieldName":"text","fieldValue":"New content"}] } Full metadata for the created document: -${JSON.stringify(createdMetadata, null, 2)}`, +${JSON.stringify(createdMetadata, null, 2)}`; + + return [ + { + type: 'text', + text: responseText, }, ]; } catch (error) { @@ -1291,7 +1394,41 @@ ${JSON.stringify(createdMetadata, null, 2)}`, // For create action, validate required parameters if (params.action === 'create') { - return !!(params.title && params.data && params.doc_type); + // Validate required base parameters + if (!(params.title && params.data && params.doc_type)) { + return false; + } + + // If fieldEdits is provided, validate its structure + if (params.fieldEdits) { + try { + // Parse fieldEdits and validate its structure + const edits = JSON.parse(String(params.fieldEdits)); + + // Ensure it's an array + if (!Array.isArray(edits)) { + console.log('fieldEdits is not an array'); + return false; + } + + // Ensure each item has fieldName and fieldValue + for (const edit of edits) { + if (!edit.fieldName) { + console.log('An edit is missing fieldName'); + return false; + } + if (edit.fieldValue === undefined) { + console.log('An edit is missing fieldValue'); + return false; + } + } + } catch (error) { + console.log('Error parsing fieldEdits:', error); + return false; + } + } + + return true; } // For edit action, validate fieldEdits is provided @@ -1368,7 +1505,7 @@ ${JSON.stringify(createdMetadata, null, 2)}`, case 'getFieldOptions': return 'The "getFieldOptions" action does not require any additional parameters. It returns metadata about all available document fields.'; case 'create': - return 'The "create" action requires title, data, and doc_type parameters.'; + return 'The "create" action requires title, data, and doc_type parameters. You can optionally include fieldEdits to set additional properties during creation.'; default: return `Unknown action "${action}". Valid actions are "get", "edit", "list", "getFieldOptions", or "create".`; } -- cgit v1.2.3-70-g09d2 From fcee0e51396e85f3deac939ea5d836ac35e2caf7 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" <Shulman.aj@gmail.com> Date: Thu, 10 Apr 2025 12:42:25 -0400 Subject: Revert "adding edit and create at the same time" This reverts commit 9de0e9844d5cbfbb385da34545006e617873b07a. --- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 171 ++------------------- 1 file changed, 17 insertions(+), 154 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 9781d7a51..eeb9091f8 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -30,7 +30,7 @@ const parameterDefinitions: ReadonlyArray<Parameter> = [ type: 'string', required: false, description: - 'JSON array of field edits for editing fields. Each item should have fieldName and fieldValue. For single field edits, use an array with one item. Can be used with both "edit" and "create" actions. Example: [{"fieldName":"layout_autoHeight","fieldValue":false},{"fieldName":"height","fieldValue":300}]', + 'JSON array of field edits for editing fields. Each item should have fieldName and fieldValue. For single field edits, use an array with one item. Example: [{"fieldName":"layout_autoHeight","fieldValue":false},{"fieldName":"height","fieldValue":300}]', }, { name: 'title', @@ -73,12 +73,10 @@ This tool provides the following capabilities: - Get detailed information about all available document fields - Support for all value types: strings, numbers, and booleans - Create new documents with basic properties -- Create new documents and immediately configure additional properties in a single operation DOCUMENT CREATION: - Use action="create" to create new documents with a simplified approach - Required parameters: title, data, and doc_type -- Optional parameter: fieldEdits to set additional properties during creation - The tool will create the document with sensible defaults and link it to the current view - After creation, you can use the edit action to update its properties @@ -98,19 +96,7 @@ Example: To change document height, disable auto-height and set height in a sing {... inputs: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 300 } -]}} - -Example: To create a document and set additional properties in a single operation: -{... inputs: { - action: "create", - title: "My Styled Note", - data: "This is my note content", - doc_type: "text", - fieldEdits: [ - { fieldName: "backgroundColor", fieldValue: "#f0f0f0" }, - { fieldName: "borderWidth", fieldValue: 1 } - ] -}}`; +]}}`; // Extensive usage guidelines for the tool const citationRules = `USAGE GUIDELINES: @@ -130,10 +116,8 @@ To CREATE a new document: - title: The title of the document to create - data: The content data for the document (text content, URL, etc.) - doc_type: The type of document to create (text, web, image, etc.) -- Optionally include fieldEdits to set additional properties during creation -- Example with just basic properties: {...inputs: { action: "create", title: "My Notes", data: "This is the content", doc_type: "text" }} -- Example with additional properties: {...inputs: { action: "create", title: "My Notes", data: "This is the content", doc_type: "text", fieldEdits: [{ fieldName: "backgroundColor", fieldValue: "#f0f0f0" }] }} -- The fieldEdits parameter allows you to create and configure a document in a single operation +- Example: {...inputs: { action: "create", title: "My Notes", data: "This is the content", doc_type: "text" }} +- After creation, you can edit the document with more specific properties To EDIT document metadata: - Use action="edit" with required parameters: @@ -157,8 +141,9 @@ RECOMMENDED WORKFLOW: 3. Get document metadata with action="get" to see current values 4. Edit fields with action="edit" using proper dependencies OR -1. Create a new document with fieldEdits to configure it in one step: - { action: "create", title: "My Document", data: "Content", doc_type: "text", fieldEdits: [{ fieldName: "backgroundColor", fieldValue: "#f0f0f0" }] } +1. Create a new document with action="create" +2. Get its ID from the response +3. Edit the document's properties with action="edit" HANDLING DEPENDENT FIELDS: - When editing some fields, you may need to update related dependent fields @@ -195,7 +180,6 @@ Examples: - To disable auto-height: { action: "edit", documentId: "doc123", fieldEdits: [{ fieldName: "layout_autoHeight", fieldValue: false }] } - To create a text document: { action: "create", title: "My Notes", data: "This is my note content", doc_type: "text" } - To create a web document: { action: "create", title: "Google", data: "https://www.google.com", doc_type: "web" } -- To create a document with custom styling: { action: "create", title: "Styled Note", data: "Content here", doc_type: "text", fieldEdits: [{ fieldName: "backgroundColor", fieldValue: "#f0f0f0" }] } - To edit height with its dependent field together: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, @@ -1238,115 +1222,28 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp // Update our local document maps with the new document this.processDocument(createdDoc); - // Apply any fieldEdits if provided - let fieldEditsResults = null; - if (args.fieldEdits) { - try { - // Parse fieldEdits array - const edits = JSON.parse(String(args.fieldEdits)); - if (Array.isArray(edits) && edits.length > 0) { - console.log(`DocumentMetadataTool: Applying ${edits.length} field edits to newly created document`); - - // Track results for all edits - const results: { - success: boolean; - message: string; - fieldName?: string; - originalFieldName?: string; - newValue?: any; - warning?: string; - }[] = []; - - let allSuccessful = true; - - // Process each edit - for (const edit of edits) { - // Get fieldValue in its original form - let fieldValue = edit.fieldValue; - - // Only convert to string if it's neither boolean nor number - if (typeof fieldValue !== 'boolean' && typeof fieldValue !== 'number') { - fieldValue = String(fieldValue); - } - - const fieldName = String(edit.fieldName); - - // Edit the field - const result = this.editDocumentField(createdDoc.id, fieldName, fieldValue); - - console.log(`DocumentMetadataTool: Edit field result for ${fieldName}:`, result); - - // Add to results - results.push(result); - - // Update success status - if (!result.success) { - allSuccessful = false; - } - } - - // Format edit results - fieldEditsResults = { - count: edits.length, - success: allSuccessful, - results, - }; - } - } catch (error) { - console.error('Error applying fieldEdits during document creation:', error); - fieldEditsResults = { - error: `Error applying fieldEdits: ${error instanceof Error ? error.message : String(error)}`, - }; - } - } - // Get the created document's metadata const createdMetadata = this.getDocumentMetadata(createdDoc.id); - // Build the response text with fieldEdits results if available - let responseText = `Document created successfully. + return [ + { + type: 'text', + text: `Document created successfully. Document ID: ${createdDoc.id} Type: ${docType} Title: "${title}" -The document has been created with default dimensions and positioning.`; - - // Add field edits results if available - if (fieldEditsResults) { - if ('error' in fieldEditsResults) { - responseText += `\n\nAttempted to apply additional field edits but encountered an error: ${fieldEditsResults.error}`; - } else { - responseText += `\n\nAdditional properties applied: ${fieldEditsResults.count} field(s) ${fieldEditsResults.success ? 'successfully updated' : 'with some errors'}`; - - if (fieldEditsResults.results && fieldEditsResults.results.length > 0) { - responseText += '\nField edit results:'; - fieldEditsResults.results.forEach(result => { - if (result.success) { - responseText += `\n- Field '${result.originalFieldName}': updated to ${JSON.stringify(result.newValue)}`; - if (result.warning) { - responseText += ` (Warning: ${result.warning})`; - } - } else { - responseText += `\n- Error editing '${result.originalFieldName}': ${result.message}`; - } - }); - } - } - } +The document has been created with default dimensions and positioning. +You can now use the "edit" action to modify additional properties of this document. - responseText += `\n\nNext steps: +Next steps: 1. Use the "getFieldOptions" action to understand available editable/addable fields/properties and their dependencies. 2. To modify this document, use: { action: "edit", documentId: "${createdDoc.id}", fieldEdits: [{"fieldName":"property","fieldValue":"value"}] } 3. To add styling, consider setting backgroundColor, fontColor, or other properties 4. For text documents, you can edit the content with: { action: "edit", documentId: "${createdDoc.id}", fieldEdits: [{"fieldName":"text","fieldValue":"New content"}] } Full metadata for the created document: -${JSON.stringify(createdMetadata, null, 2)}`; - - return [ - { - type: 'text', - text: responseText, +${JSON.stringify(createdMetadata, null, 2)}`, }, ]; } catch (error) { @@ -1394,41 +1291,7 @@ ${JSON.stringify(createdMetadata, null, 2)}`; // For create action, validate required parameters if (params.action === 'create') { - // Validate required base parameters - if (!(params.title && params.data && params.doc_type)) { - return false; - } - - // If fieldEdits is provided, validate its structure - if (params.fieldEdits) { - try { - // Parse fieldEdits and validate its structure - const edits = JSON.parse(String(params.fieldEdits)); - - // Ensure it's an array - if (!Array.isArray(edits)) { - console.log('fieldEdits is not an array'); - return false; - } - - // Ensure each item has fieldName and fieldValue - for (const edit of edits) { - if (!edit.fieldName) { - console.log('An edit is missing fieldName'); - return false; - } - if (edit.fieldValue === undefined) { - console.log('An edit is missing fieldValue'); - return false; - } - } - } catch (error) { - console.log('Error parsing fieldEdits:', error); - return false; - } - } - - return true; + return !!(params.title && params.data && params.doc_type); } // For edit action, validate fieldEdits is provided @@ -1505,7 +1368,7 @@ ${JSON.stringify(createdMetadata, null, 2)}`; case 'getFieldOptions': return 'The "getFieldOptions" action does not require any additional parameters. It returns metadata about all available document fields.'; case 'create': - return 'The "create" action requires title, data, and doc_type parameters. You can optionally include fieldEdits to set additional properties during creation.'; + return 'The "create" action requires title, data, and doc_type parameters.'; default: return `Unknown action "${action}". Valid actions are "get", "edit", "list", "getFieldOptions", or "create".`; } -- cgit v1.2.3-70-g09d2 From 951fe11f0978b36946c5c0c7f2d2cac7b597d311 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" <Shulman.aj@gmail.com> Date: Tue, 22 Apr 2025 12:20:22 -0400 Subject: improved document creation and limited context window --- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 46 +++++++++++----------- 1 file changed, 22 insertions(+), 24 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index eeb9091f8..08351143b 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -386,7 +386,8 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp this.documentsById.set(docId, doc); // Get layout doc (the document itself or its layout) - const layoutDoc = Doc.Layout(doc); + // TODO: Check if this works. + const layoutDoc = doc; if (layoutDoc) { this.layoutDocsById.set(docId, layoutDoc); } @@ -448,17 +449,13 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp * @param docId The ID of the document to extract metadata from * @returns An object containing the document's metadata */ - private extractDocumentMetadata(docId: string) { - const doc = this.documentsById.get(docId); - if (!doc) { - return null; - } - - const layoutDoc = this.layoutDocsById.get(docId); - const dataDoc = this.dataDocsById.get(docId); + private extractDocumentMetadata(doc?: Doc) { + if (!doc) return null; + const layoutDoc = doc; + const dataDoc = doc[DocData]; const metadata: Record<string, any> = { - id: docId, + id: doc.dash_document_id || doc.id || '', title: doc.title || '', type: doc.type || '', fields: { @@ -840,11 +837,11 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp string: [], number: [], boolean: [], - doc: [], - list: [], - date: [], - enumeration: [], - other: [], + //doc: [], + //list: [], + //date: [], + //enumeration: [], + //other: [], }, fieldNameMappings: {}, commonFields: { @@ -887,15 +884,15 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } else if (type === 'boolean') { result.fieldsByType.boolean.push(fieldName); } else if (type === 'doc') { - result.fieldsByType.doc.push(fieldName); + //result.fieldsByType.doc.push(fieldName); } else if (type === 'list') { - result.fieldsByType.list.push(fieldName); + //result.fieldsByType.list.push(fieldName); } else if (type === 'date') { - result.fieldsByType.date.push(fieldName); + //result.fieldsByType.date.push(fieldName); } else if (type === 'enumeration') { - result.fieldsByType.enumeration.push(fieldName); + //result.fieldsByType.enumeration.push(fieldName); } else { - result.fieldsByType.other.push(fieldName); + //result.fieldsByType.other.push(fieldName); } // Categorize by field purpose @@ -1223,7 +1220,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp this.processDocument(createdDoc); // Get the created document's metadata - const createdMetadata = this.getDocumentMetadata(createdDoc.id); + const createdMetadata = this.extractDocumentMetadata(createdDoc); return [ { @@ -1381,13 +1378,14 @@ ${JSON.stringify(createdMetadata, null, 2)}`, */ private getDocumentMetadata(documentId?: string): any { if (documentId) { + const doc = this.documentsById.get(documentId); // Get metadata for a specific document - return this.extractDocumentMetadata(documentId); + return this.extractDocumentMetadata(doc); } else { // Get metadata for all documents const documentsMetadata: Record<string, any> = {}; - for (const docId of this.documentsById.keys()) { - documentsMetadata[docId] = this.extractDocumentMetadata(docId); + for (const doc of this.documentsById.values()) { + documentsMetadata.add(this.extractDocumentMetadata(doc)); } return { -- cgit v1.2.3-70-g09d2 From 321977e670cbdf10f6c49fc9071e3260a8bd4aae Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" <Shulman.aj@gmail.com> Date: Thu, 24 Apr 2025 12:06:11 -0400 Subject: Linking docs now works with visible docs --- .../views/nodes/chatbot/agentsystem/Agent.ts | 17 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 28 +- .../views/nodes/chatbot/tools/CreateLinksTool.ts | 68 ++ .../nodes/chatbot/tools/DocumentMetadataTool.ts | 884 +------------------- .../nodes/chatbot/utils/AgentDocumentManager.ts | 923 +++++++++++++++++++++ 5 files changed, 1032 insertions(+), 888 deletions(-) create mode 100644 src/client/views/nodes/chatbot/tools/CreateLinksTool.ts create mode 100644 src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 43faf5bf4..5af021dbf 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -20,10 +20,12 @@ import { getReactPrompt } from './prompts'; //import { DictionaryTool } from '../tools/DictionaryTool'; import { ChatCompletionMessageParam } from 'openai/resources'; import { Doc } from '../../../../../fields/Doc'; -import { parsedDoc } from '../chatboxcomponents/ChatBox'; +import { ChatBox, parsedDoc } from '../chatboxcomponents/ChatBox'; import { WebsiteInfoScraperTool } from '../tools/WebsiteInfoScraperTool'; import { Upload } from '../../../../../server/SharedMediaTypes'; import { RAGTool } from '../tools/RAGTool'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; +import { CreateLinksTool } from '../tools/CreateLinksTool'; //import { CreateTextDocTool } from '../tools/CreateTextDocumentTool'; dotenv.config(); @@ -47,6 +49,7 @@ export class Agent { private processingInfo: ProcessingInfo[] = []; private streamedAnswerParser: StreamedAnswerParser = new StreamedAnswerParser(); private tools: Record<string, BaseTool<ReadonlyArray<Parameter>>>; + private _docManager: AgentDocumentManager; /** * The constructor initializes the agent with the vector store and toolset, and sets up the OpenAI client. @@ -64,9 +67,9 @@ export class Agent { csvData: () => { filename: string; id: string; text: string }[], addLinkedUrlDoc: (url: string, id: string) => void, createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void, - addLinkedDoc: (doc: parsedDoc) => Doc | undefined, // eslint-disable-next-line @typescript-eslint/no-unused-vars - createCSVInDash: (url: string, title: string, id: string, data: string) => void + createCSVInDash: (url: string, title: string, id: string, data: string) => void, + chatBox: ChatBox ) { // Initialize OpenAI client with API key from environment this.client = new OpenAI({ apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true }); @@ -74,6 +77,7 @@ export class Agent { this._history = history; this._summaries = summaries; this._csvData = csvData; + this._docManager = new AgentDocumentManager(chatBox); // Define available tools for the assistant this.tools = { @@ -84,7 +88,8 @@ export class Agent { searchTool: new SearchTool(addLinkedUrlDoc), noTool: new NoTool(), //imageCreationTool: new ImageCreationTool(createImage), - documentMetadata: new DocumentMetadataTool(this), + documentMetadata: new DocumentMetadataTool(this._docManager), + createLinks: new CreateLinksTool(this._docManager), }; } @@ -541,9 +546,9 @@ export class Agent { * * @param chatBox The ChatBox instance to pass to the DocumentMetadataTool */ - public reinitializeDocumentMetadataTool(chatBox: any): void { + public reinitializeDocumentMetadataTool(): void { if (this.tools && this.tools.documentMetadata) { - this.tools.documentMetadata = new DocumentMetadataTool(chatBox); + this.tools.documentMetadata = new DocumentMetadataTool(this._docManager); console.log('Agent: Reinitialized DocumentMetadataTool with ChatBox instance'); } else { console.warn('Agent: Could not reinitialize DocumentMetadataTool - tool not found'); diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 33419e559..e09b4313f 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -107,11 +107,11 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); } this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); - this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createImageInDash, this.createDocInDash, this.createCSVInDash); + this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.createImageInDash, this.createCSVInDash, this); // Reinitialize the DocumentMetadataTool with a direct reference to this ChatBox instance // This ensures the tool can properly access documents in the same Freeform view - this.agent.reinitializeDocumentMetadataTool(this); + this.agent.reinitializeDocumentMetadataTool(); this.messagesRef = React.createRef<HTMLDivElement>(); @@ -446,7 +446,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { private createCollectionWithChildren = (data: parsedDoc[], insideCol: boolean): Opt<Doc>[] => data.map(doc => this.whichDoc(doc, insideCol)); @action - whichDoc = (doc: parsedDoc, insideCol: boolean): Opt<Doc> => { + public whichDoc = (doc: parsedDoc, insideCol: boolean): Opt<Doc> => { const options = OmitKeys(doc, ['doct_type', 'data']).omit as DocumentOptions; const data = (doc as parsedDocData).data; const ndoc = (() => { @@ -515,28 +515,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return ndoc; }; - /** - * Creates a document in the dashboard. - * - * @param {string} doc_type - The type of document to create. - * @param {string} data - The data used to generate the document. - * @param {DocumentOptions} options - Configuration options for the document. - * @returns {Promise<void>} A promise that resolves once the document is created and displayed. - */ - @action - createDocInDash = (pdoc: parsedDoc) => { - const linkAndShowDoc = (doc: Opt<Doc>) => { - if (doc) { - LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.Document, doc)); - this._props.addDocument?.(doc); - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - } - }; - const doc = this.whichDoc(pdoc, false); - if (doc) linkAndShowDoc(doc); - return doc; - }; - /** * Creates a deck of flashcards. * diff --git a/src/client/views/nodes/chatbot/tools/CreateLinksTool.ts b/src/client/views/nodes/chatbot/tools/CreateLinksTool.ts new file mode 100644 index 000000000..c2850a8ce --- /dev/null +++ b/src/client/views/nodes/chatbot/tools/CreateLinksTool.ts @@ -0,0 +1,68 @@ +import { Observation } from '../types/types'; +import { ParametersType, ToolInfo } from '../types/tool_types'; +import { BaseTool } from './BaseTool'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; + +const createLinksToolParams = [ + { + name: 'document_ids', + type: 'string[]', + description: 'List of document IDs to create links between. All documents will be linked to each other.', + required: true, + }, +] as const; + +type CreateLinksToolParamsType = typeof createLinksToolParams; + +const createLinksToolInfo: ToolInfo<CreateLinksToolParamsType> = { + name: 'createLinks', + description: 'Creates visual links between multiple documents in the dashboard. This allows related documents to be connected visually with lines that users can see.', + citationRules: 'No citation needed.', + parameterRules: createLinksToolParams, +}; + +export class CreateLinksTool extends BaseTool<CreateLinksToolParamsType> { + private _documentManager: AgentDocumentManager; + + constructor(documentManager: AgentDocumentManager) { + super(createLinksToolInfo); + this._documentManager = documentManager; + } + + async execute(args: ParametersType<CreateLinksToolParamsType>): Promise<Observation[]> { + try { + // Validate that we have at least 2 documents to link + if (args.document_ids.length < 2) { + return [{ type: 'text', text: 'Error: At least 2 document IDs are required to create links.' }]; + } + + // Validate that all documents exist + const missingDocIds = args.document_ids.filter(id => !this._documentManager.has(id)); + if (missingDocIds.length > 0) { + return [ + { + type: 'text', + text: `Error: The following document IDs were not found: ${missingDocIds.join(', ')}`, + }, + ]; + } + + // Create links between all documents with the specified relationship + const createdLinks = this._documentManager.addLinks(args.document_ids); + + return [ + { + type: 'text', + text: `Successfully created ${createdLinks.length} visual links between ${args.document_ids.length}.`, + }, + ]; + } catch (error) { + return [ + { + type: 'text', + text: `Error creating links: ${error instanceof Error ? error.message : String(error)}`, + }, + ]; + } + } +} diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 08351143b..4b751acc0 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -10,6 +10,7 @@ import { LinkManager } from '../../../../util/LinkManager'; import { DocCast, StrCast } from '../../../../../fields/Types'; import { supportedDocTypes } from '../types/tool_types'; import { parsedDoc } from '../chatboxcomponents/ChatBox'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; // Define the parameters for the DocumentMetadataTool const parameterDefinitions: ReadonlyArray<Parameter> = [ @@ -199,746 +200,12 @@ const documentMetadataToolInfo: ToolInfo<DocumentMetadataToolParamsType> = { * and allows for editing document fields in the correct location. */ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsType> { - private freeformView: any; - private chatBox: any; - private chatBoxDocument: Doc | null = null; - private documentsById: Map<string, Doc> = new Map(); - private layoutDocsById: Map<string, Doc> = new Map(); - private dataDocsById: Map<string, Doc> = new Map(); - private fieldMetadata: Record<string, any> = {}; - private readonly DOCUMENT_ID_FIELD = '_dash_document_id'; - - constructor(chatBox: any) { - super(documentMetadataToolInfo); - this.chatBox = chatBox; - - // Store a direct reference to the ChatBox document - if (chatBox && chatBox.Document) { - this.chatBoxDocument = chatBox.Document; - if (this.chatBoxDocument && this.chatBoxDocument.id) { - console.log('DocumentMetadataTool initialized with ChatBox Document:', this.chatBoxDocument.id); - } else { - console.log('DocumentMetadataTool initialized with ChatBox Document (no ID)'); - } - } else if (chatBox && chatBox.props && chatBox.props.Document) { - this.chatBoxDocument = chatBox.props.Document; - if (this.chatBoxDocument && this.chatBoxDocument.id) { - console.log('DocumentMetadataTool initialized with ChatBox props.Document:', this.chatBoxDocument.id); - } else { - console.log('DocumentMetadataTool initialized with ChatBox props.Document (no ID)'); - } - } else { - console.warn('DocumentMetadataTool initialized without valid ChatBox Document reference'); - } - - this.initializeFieldMetadata(); - } - - /** - * Extracts field metadata from DocumentOptions class - */ - private initializeFieldMetadata() { - // Parse DocumentOptions to extract field definitions - const documentOptionsInstance = new DocumentOptions(); - const documentOptionsEntries = Object.entries(documentOptionsInstance); - - for (const [fieldName, fieldInfo] of documentOptionsEntries) { - // Extract field information - const fieldData: Record<string, any> = { - name: fieldName, - withoutUnderscore: fieldName.startsWith('_') ? fieldName.substring(1) : fieldName, - description: '', - type: 'unknown', - required: false, - defaultValue: undefined, - possibleValues: [], - }; - - // Check if fieldInfo has description property (it's likely a FInfo instance) - if (fieldInfo && typeof fieldInfo === 'object' && 'description' in fieldInfo) { - fieldData.description = fieldInfo.description; - - // Extract field type if available - if ('fieldType' in fieldInfo) { - fieldData.type = fieldInfo.fieldType; - } - - // Extract possible values if available - if ('values' in fieldInfo && Array.isArray(fieldInfo.values)) { - fieldData.possibleValues = fieldInfo.values; - } - } - - this.fieldMetadata[fieldName] = fieldData; - } - } - - /** - * Gets all documents in the same Freeform view as the ChatBox - * Uses the LinkManager to get all linked documents, similar to how ChatBox does it - */ - private findDocumentsInFreeformView() { - // Reset collections - this.documentsById.clear(); - this.layoutDocsById.clear(); - this.dataDocsById.clear(); - - try { - // Use the LinkManager approach which is proven to work in ChatBox - if (this.chatBoxDocument) { - console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument.id); - - // Get directly linked documents via LinkManager - const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.chatBoxDocument) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.chatBoxDocument!))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d); - - console.log(`Found ${linkedDocs.length} linked documents via LinkManager`); - - // Process the linked documents - linkedDocs.forEach((doc: Doc) => { - if (doc) { - this.processDocument(doc); - } - }); - - // Include the ChatBox document itself - this.processDocument(this.chatBoxDocument); - - // If we have access to the Document's parent, try to find sibling documents - if (this.chatBoxDocument.parent) { - const parent = this.chatBoxDocument.parent; - console.log('Found parent document, checking for siblings'); - - // Check if parent is a Doc type and has a childDocs function - if (parent && typeof parent === 'object' && 'childDocs' in parent && typeof parent.childDocs === 'function') { - try { - const siblingDocs = parent.childDocs(); - if (Array.isArray(siblingDocs)) { - console.log(`Found ${siblingDocs.length} sibling documents via parent.childDocs()`); - siblingDocs.forEach((doc: Doc) => { - if (doc) { - this.processDocument(doc); - } - }); - } - } catch (e) { - console.warn('Error accessing parent.childDocs:', e); - } - } - } - } else if (this.chatBox && this.chatBox.linkedDocs) { - // If we have direct access to the linkedDocs computed property from ChatBox - console.log('Using ChatBox.linkedDocs directly'); - const linkedDocs = this.chatBox.linkedDocs; - if (Array.isArray(linkedDocs)) { - console.log(`Found ${linkedDocs.length} documents via ChatBox.linkedDocs`); - linkedDocs.forEach((doc: Doc) => { - if (doc) { - this.processDocument(doc); - } - }); - } - - // Process the ChatBox document if available - if (this.chatBox.Document) { - this.processDocument(this.chatBox.Document); - } - } else { - console.warn('No ChatBox document reference available for finding linked documents'); - } - - console.log(`DocumentMetadataTool found ${this.documentsById.size} total documents`); - - // If we didn't find any documents, try a fallback method - if (this.documentsById.size === 0 && this.chatBox) { - console.log('No documents found, trying fallback method'); - - // Try to access any field that might contain documents - if (this.chatBox.props && this.chatBox.props.documents) { - const documents = this.chatBox.props.documents; - if (Array.isArray(documents)) { - console.log(`Found ${documents.length} documents via ChatBox.props.documents`); - documents.forEach((doc: Doc) => { - if (doc) { - this.processDocument(doc); - } - }); - } - } - } - } catch (error) { - console.error('Error finding documents in Freeform view:', error); - } - } - - /** - * Process a document by ensuring it has an ID and adding it to the appropriate collections - * @param doc The document to process - */ - private processDocument(doc: Doc) { - // Ensure document has a persistent ID - const docId = this.ensureDocumentId(doc); - - // Only add if we haven't already processed this document - if (!this.documentsById.has(docId)) { - this.documentsById.set(docId, doc); - - // Get layout doc (the document itself or its layout) - // TODO: Check if this works. - const layoutDoc = doc; - if (layoutDoc) { - this.layoutDocsById.set(docId, layoutDoc); - } - - // Get data doc - const dataDoc = doc[DocData]; - if (dataDoc) { - this.dataDocsById.set(docId, dataDoc); - } - } - } - - /** - * Ensures a document has a persistent ID stored in its metadata - * @param doc The document to ensure has an ID - * @returns The document's ID - */ - private ensureDocumentId(doc: Doc): string { - let docId: string | undefined; - - // First try to get the ID from our custom field - if (doc[this.DOCUMENT_ID_FIELD]) { - docId = String(doc[this.DOCUMENT_ID_FIELD]); - return docId; - } - - // Try different ways to get a document ID - - // 1. Try the direct id property if it exists - if (doc.id && typeof doc.id === 'string') { - docId = doc.id; - } - // 2. Try doc._id if it exists - else if (doc._id && typeof doc._id === 'string') { - docId = doc._id; - } - // 3. Try doc.data?.id if it exists - else if (doc.data && typeof doc.data === 'object' && 'id' in doc.data && typeof doc.data.id === 'string') { - docId = doc.data.id; - } - // 4. If none of the above work, generate a UUID - else { - docId = uuidv4(); - console.log(`Generated new UUID for document with title: ${doc.title || 'Untitled'}`); - } - - // Store the ID in the document's metadata so it persists - try { - doc[this.DOCUMENT_ID_FIELD] = docId; - } catch (e) { - console.warn(`Could not assign ID to document property`, e); - } - - return docId; - } - - /** - * Extracts metadata from a specific document - * @param docId The ID of the document to extract metadata from - * @returns An object containing the document's metadata - */ - private extractDocumentMetadata(doc?: Doc) { - if (!doc) return null; - const layoutDoc = doc; - const dataDoc = doc[DocData]; - - const metadata: Record<string, any> = { - id: doc.dash_document_id || doc.id || '', - title: doc.title || '', - type: doc.type || '', - fields: { - layout: {}, - data: {}, - }, - fieldLocationMap: {}, - }; - - // Process all known field definitions - Object.keys(this.fieldMetadata).forEach(fieldName => { - const fieldDef = this.fieldMetadata[fieldName]; - const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; - - // Check if field exists on layout document - let layoutValue = undefined; - if (layoutDoc) { - layoutValue = layoutDoc[fieldName]; - if (layoutValue !== undefined) { - // Field exists on layout document - metadata.fields.layout[fieldName] = this.formatFieldValue(layoutValue); - metadata.fieldLocationMap[strippedName] = 'layout'; - } - } - - // Check if field exists on data document - let dataValue = undefined; - if (dataDoc) { - dataValue = dataDoc[fieldName]; - if (dataValue !== undefined) { - // Field exists on data document - metadata.fields.data[fieldName] = this.formatFieldValue(dataValue); - if (!metadata.fieldLocationMap[strippedName]) { - metadata.fieldLocationMap[strippedName] = 'data'; - } - } - } - - // For fields with stripped names (without leading underscore), - // also check if they exist on documents without the underscore - if (fieldName.startsWith('_')) { - const nonUnderscoreFieldName = fieldName.substring(1); - - if (layoutDoc) { - const nonUnderscoreLayoutValue = layoutDoc[nonUnderscoreFieldName]; - if (nonUnderscoreLayoutValue !== undefined) { - metadata.fields.layout[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreLayoutValue); - metadata.fieldLocationMap[nonUnderscoreFieldName] = 'layout'; - } - } - - if (dataDoc) { - const nonUnderscoreDataValue = dataDoc[nonUnderscoreFieldName]; - if (nonUnderscoreDataValue !== undefined) { - metadata.fields.data[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreDataValue); - if (!metadata.fieldLocationMap[nonUnderscoreFieldName]) { - metadata.fieldLocationMap[nonUnderscoreFieldName] = 'data'; - } - } - } - } - }); - - // Add common field aliases for easier discovery - // This helps users understand both width and _width refer to the same property - if (metadata.fields.layout._width !== undefined && metadata.fields.layout.width === undefined) { - metadata.fields.layout.width = metadata.fields.layout._width; - metadata.fieldLocationMap.width = 'layout'; - } - - if (metadata.fields.layout._height !== undefined && metadata.fields.layout.height === undefined) { - metadata.fields.layout.height = metadata.fields.layout._height; - metadata.fieldLocationMap.height = 'layout'; - } - - return metadata; - } + private _docManager: AgentDocumentManager; - /** - * Edits a specific field on a document - * @param docId The ID of the document to edit - * @param fieldName The name of the field to edit - * @param fieldValue The new value for the field (string, number, or boolean) - * @returns Object with success status, message, and additional information - */ - private editDocumentField( - docId: string, - fieldName: string, - fieldValue: string | number | boolean - ): { - success: boolean; - message: string; - fieldName?: string; - originalFieldName?: string; - newValue?: any; - warning?: string; - } { - // Normalize field name (handle with/without underscore) - let normalizedFieldName = fieldName.startsWith('_') ? fieldName : fieldName; - const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; - - // Handle common field name aliases (width → _width, height → _height) - // Many document fields use '_' prefix for layout properties - if (fieldName === 'width') { - normalizedFieldName = '_width'; - } else if (fieldName === 'height') { - normalizedFieldName = '_height'; - } - - // Get the documents - const doc = this.documentsById.get(docId); - if (!doc) { - return { success: false, message: `Document with ID ${docId} not found` }; - } - - const layoutDoc = this.layoutDocsById.get(docId); - const dataDoc = this.dataDocsById.get(docId); - - if (!layoutDoc && !dataDoc) { - return { success: false, message: `Could not find layout or data document for document with ID ${docId}` }; - } - - try { - // Convert the field value to the appropriate type based on field metadata - const convertedValue = this.convertFieldValue(normalizedFieldName, fieldValue); - - let targetDoc: Doc | undefined; - let targetLocation: string; - - // First, check if field exists on layout document using Doc.Get - if (layoutDoc) { - const fieldExistsOnLayout = Doc.Get(layoutDoc, normalizedFieldName, true) !== undefined; - - // If it exists on layout document, update it there - if (fieldExistsOnLayout) { - targetDoc = layoutDoc; - targetLocation = 'layout'; - } - // If it has an underscore prefix, it's likely a layout property even if not yet set - else if (normalizedFieldName.startsWith('_')) { - targetDoc = layoutDoc; - targetLocation = 'layout'; - } - // Otherwise, look for or create on data document - else if (dataDoc) { - targetDoc = dataDoc; - targetLocation = 'data'; - } - // If no data document available, default to layout - else { - targetDoc = layoutDoc; - targetLocation = 'layout'; - } - } - // If no layout document, use data document - else if (dataDoc) { - targetDoc = dataDoc; - targetLocation = 'data'; - } else { - return { success: false, message: `No valid document found for editing` }; - } - - if (!targetDoc) { - return { success: false, message: `Target document not available` }; - } - - // Set the field value on the target document - targetDoc[normalizedFieldName] = convertedValue; - - return { - success: true, - message: `Successfully updated field '${normalizedFieldName}' on ${targetLocation} document (ID: ${docId})`, - fieldName: normalizedFieldName, - originalFieldName: fieldName, - newValue: convertedValue, - }; - } catch (error) { - console.error('Error editing document field:', error); - return { - success: false, - message: `Error updating field: ${error instanceof Error ? error.message : String(error)}`, - }; - } - } - - /** - * Converts a string field value to the appropriate type based on field metadata - * @param fieldName The name of the field - * @param fieldValue The string value to convert - * @returns The converted value with the appropriate type - */ - private convertFieldValue(fieldName: string, fieldValue: any): any { - // If fieldValue is already a number or boolean, we don't need to convert it from string - if (typeof fieldValue === 'number' || typeof fieldValue === 'boolean') { - return fieldValue; - } - - // If fieldValue is a string "true" or "false", convert to boolean - if (typeof fieldValue === 'string') { - if (fieldValue.toLowerCase() === 'true') { - return true; - } - if (fieldValue.toLowerCase() === 'false') { - return false; - } - } - - // If fieldValue is not a string (and not a number or boolean), convert it to string - if (typeof fieldValue !== 'string') { - fieldValue = String(fieldValue); - } - - // Special handling for text field - convert to proper RichTextField format - if (fieldName === 'text') { - try { - // Check if it's already a valid JSON RichTextField - JSON.parse(fieldValue); - return fieldValue; - } catch (e) { - // It's a plain text string, so convert it to RichTextField format - const rtf = { - doc: { - type: 'doc', - content: [ - { - type: 'paragraph', - content: [ - { - type: 'text', - text: fieldValue, - }, - ], - }, - ], - }, - }; - return JSON.stringify(rtf); - } - } - - // Get field metadata - const normalizedFieldName = fieldName.startsWith('_') ? fieldName : `_${fieldName}`; - const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; - - // Check both versions of the field name in metadata - const fieldMeta = this.fieldMetadata[normalizedFieldName] || this.fieldMetadata[strippedFieldName]; - - // Special handling for width and height without metadata - if (!fieldMeta && (fieldName === '_width' || fieldName === '_height' || fieldName === 'width' || fieldName === 'height')) { - const num = Number(fieldValue); - return isNaN(num) ? fieldValue : num; - } - - if (!fieldMeta) { - // If no metadata found, just return the string value - return fieldValue; - } - - // Convert based on field type - const fieldType = fieldMeta.type; - - if (fieldType === 'boolean') { - // Convert to boolean - return fieldValue.toLowerCase() === 'true'; - } else if (fieldType === 'number') { - // Convert to number - const num = Number(fieldValue); - return isNaN(num) ? fieldValue : num; - } else if (fieldType === 'date') { - // Try to convert to date (stored as number timestamp) - try { - return new Date(fieldValue).getTime(); - } catch (e) { - return fieldValue; - } - } else if (fieldType.includes('list') || fieldType.includes('array')) { - // Try to parse as JSON array - try { - return JSON.parse(fieldValue); - } catch (e) { - return fieldValue; - } - } else if (fieldType === 'json' || fieldType === 'object') { - // Try to parse as JSON object - try { - return JSON.parse(fieldValue); - } catch (e) { - return fieldValue; - } - } - - // Default to string - return fieldValue; - } - - /** - * Formats a field value for JSON output - * @param value The field value to format - * @returns A JSON-friendly representation of the field value - */ - private formatFieldValue(value: any): any { - if (value === undefined || value === null) { - return null; - } - - // Handle Doc objects - if (value instanceof Doc) { - return { - type: 'Doc', - id: value.id || this.ensureDocumentId(value), - title: value.title || '', - docType: value.type || '', - }; - } - - // Handle RichTextField (try to extract plain text) - if (typeof value === 'string' && value.includes('"type":"doc"') && value.includes('"content":')) { - try { - const rtfObj = JSON.parse(value); - // If this looks like a rich text field structure - if (rtfObj.doc && rtfObj.doc.content) { - // Recursively extract text from the content - let plainText = ''; - const extractText = (node: any) => { - if (node.text) { - plainText += node.text; - } - if (node.content && Array.isArray(node.content)) { - node.content.forEach((child: any) => extractText(child)); - } - }; - - extractText(rtfObj.doc); - - // If we successfully extracted text, show it, but also preserve the original value - if (plainText) { - return { - type: 'RichText', - text: plainText, - length: plainText.length, - // Don't include the full value as it can be very large - }; - } - } - } catch (e) { - // If parsing fails, just treat as a regular string - } - } - - // Handle arrays and complex objects - if (typeof value === 'object') { - // If the object has a toString method, use it - if (value.toString && value.toString !== Object.prototype.toString) { - return value.toString(); - } - - try { - // Try to convert to JSON string - return JSON.stringify(value); - } catch (e) { - return '[Complex Object]'; - } - } - - // Return primitive values as is - return value; - } - - /** - * Extracts all field metadata from DocumentOptions - * @returns A structured object containing metadata about all available document fields - */ - private getAllFieldMetadata() { - // Start with our already populated fieldMetadata from the DocumentOptions class - const result: Record<string, any> = { - fieldCount: Object.keys(this.fieldMetadata).length, - fields: {}, - fieldsByType: { - string: [], - number: [], - boolean: [], - //doc: [], - //list: [], - //date: [], - //enumeration: [], - //other: [], - }, - fieldNameMappings: {}, - commonFields: { - appearance: [], - position: [], - size: [], - content: [], - behavior: [], - layout: [], - }, - }; - - // Process each field in the metadata - Object.entries(this.fieldMetadata).forEach(([fieldName, fieldInfo]) => { - const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; - - // Add to fieldNameMappings - if (fieldName.startsWith('_')) { - result.fieldNameMappings[strippedName] = fieldName; - } - - // Create structured field metadata - const fieldData: Record<string, any> = { - name: fieldName, - displayName: strippedName, - description: fieldInfo.description || '', - type: fieldInfo.fieldType || 'unknown', - possibleValues: fieldInfo.values || [], - }; - - // Add field to fields collection - result.fields[fieldName] = fieldData; - - // Categorize by field type - const type = fieldInfo.fieldType?.toLowerCase() || 'unknown'; - if (type === 'string') { - result.fieldsByType.string.push(fieldName); - } else if (type === 'number') { - result.fieldsByType.number.push(fieldName); - } else if (type === 'boolean') { - result.fieldsByType.boolean.push(fieldName); - } else if (type === 'doc') { - //result.fieldsByType.doc.push(fieldName); - } else if (type === 'list') { - //result.fieldsByType.list.push(fieldName); - } else if (type === 'date') { - //result.fieldsByType.date.push(fieldName); - } else if (type === 'enumeration') { - //result.fieldsByType.enumeration.push(fieldName); - } else { - //result.fieldsByType.other.push(fieldName); - } - - // Categorize by field purpose - if (fieldName.includes('width') || fieldName.includes('height') || fieldName.includes('size')) { - result.commonFields.size.push(fieldName); - } else if (fieldName.includes('color') || fieldName.includes('background') || fieldName.includes('border')) { - result.commonFields.appearance.push(fieldName); - } else if (fieldName.includes('x') || fieldName.includes('y') || fieldName.includes('position') || fieldName.includes('pan')) { - result.commonFields.position.push(fieldName); - } else if (fieldName.includes('text') || fieldName.includes('title') || fieldName.includes('data')) { - result.commonFields.content.push(fieldName); - } else if (fieldName.includes('action') || fieldName.includes('click') || fieldName.includes('event')) { - result.commonFields.behavior.push(fieldName); - } else if (fieldName.includes('layout')) { - result.commonFields.layout.push(fieldName); - } - }); - - // Add special section for auto-sizing related fields - result.autoSizingFields = { - height: { - autoHeightField: '_layout_autoHeight', - heightField: '_height', - displayName: 'height', - usage: 'To manually set height, first set layout_autoHeight to false', - }, - width: { - autoWidthField: '_layout_autoWidth', - widthField: '_width', - displayName: 'width', - usage: 'To manually set width, first set layout_autoWidth to false', - }, - }; - - // Add special section for text field format - result.specialFields = { - text: { - name: 'text', - description: 'Document text content', - format: 'RichTextField', - note: 'When setting text, provide plain text - it will be automatically converted to the correct format', - example: 'For setting: "Hello world" (plain text); For getting: Will be converted to plaintext for display', - }, - }; - - return result; + constructor(docManager: AgentDocumentManager) { + super(documentMetadataToolInfo); + this._docManager = docManager; + this._docManager.initializeFindDocsFreeform(); } /** @@ -950,7 +217,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp console.log('DocumentMetadataTool: Executing with args:', args); // Find all documents in the Freeform view - this.findDocumentsInFreeformView(); + this._docManager.initializeFindDocsFreeform(); try { // Validate required input parameters based on action @@ -981,7 +248,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp switch (action) { case 'get': { // Get metadata for a specific document or all documents - const result = this.getDocumentMetadata(documentId); + const result = this._docManager.getDocumentMetadata(documentId); console.log('DocumentMetadataTool: Get metadata result:', result); return [ { @@ -1003,7 +270,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } // Ensure document exists - if (!this.documentsById.has(documentId)) { + if (!this._docManager.has(documentId)) { return [ { type: 'text', @@ -1059,7 +326,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp const fieldName = String(edit.fieldName); // Edit the field - const result = this.editDocumentField(documentId, fieldName, fieldValue); + const result = this._docManager.editDocumentField(documentId, fieldName, fieldValue); console.log(`DocumentMetadataTool: Edit field result for ${fieldName}:`, result); @@ -1101,7 +368,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } // Get the updated metadata to return - const updatedMetadata = this.getDocumentMetadata(documentId); + const updatedMetadata = this._docManager.getDocumentMetadata(documentId); return [ { @@ -1120,33 +387,12 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } case 'list': { - // List all available documents in simple format - const docs = Array.from(this.documentsById.entries()).map(([id, doc]) => ({ - id, - title: doc.title || 'Untitled Document', - type: doc.type || 'Unknown Type', - })); - - if (docs.length === 0) { - return [ - { - type: 'text', - text: 'No documents found in the current view.', - }, - ]; - } - - return [ - { - type: 'text', - text: `Found ${docs.length} document(s) in the current view:\n${JSON.stringify(docs, null, 2)}`, - }, - ]; + this._docManager.listDocs(); } case 'getFieldOptions': { // Get all available field options with metadata - const fieldOptions = this.getAllFieldMetadata(); + const fieldOptions = this._docManager.getAllFieldMetadata(); return [ { @@ -1171,61 +417,27 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp const title = String(args.title); const data = String(args.data); - // Validate doc_type - if (!this.isValidDocType(docType)) { + const createdDoc = this._docManager.createDocInDash(docType, title, data); + + if (!createdDoc) { return [ { type: 'text', - text: `Error: Invalid doc_type. Valid options are: ${Object.keys(supportedDocTypes).join(',')}`, + text: 'Error: Failed to create document.', }, ]; } - try { - // Create simple document with just title and data - const simpleDoc: parsedDoc = { - doc_type: docType, - title: title, - data: data, - x: 0, - y: 0, - _width: 300, - _height: 300, - _layout_fitWidth: false, - _layout_autoHeight: true, - }; - - // Use the chatBox's createDocInDash method to create and link the document - if (!this.chatBox || !this.chatBox.createDocInDash) { - return [ - { - type: 'text', - text: 'Error: Could not access document creation functionality.', - }, - ]; - } - - const createdDoc = this.chatBox.createDocInDash(simpleDoc); - - if (!createdDoc) { - return [ - { - type: 'text', - text: 'Error: Failed to create document.', - }, - ]; - } + // Update our local document maps with the new document + this._docManager.processDocument(createdDoc); - // Update our local document maps with the new document - this.processDocument(createdDoc); + // Get the created document's metadata + const createdMetadata = this._docManager.extractDocumentMetadata(this._docManager.createAgentDoc(createdDoc)); - // Get the created document's metadata - const createdMetadata = this.extractDocumentMetadata(createdDoc); - - return [ - { - type: 'text', - text: `Document created successfully. + return [ + { + type: 'text', + text: `Document created successfully. Document ID: ${createdDoc.id} Type: ${docType} Title: "${title}" @@ -1241,16 +453,8 @@ Next steps: Full metadata for the created document: ${JSON.stringify(createdMetadata, null, 2)}`, - }, - ]; - } catch (error) { - return [ - { - type: 'text', - text: `Error creating document: ${error instanceof Error ? error.message : String(error)}`, - }, - ]; - } + }, + ]; } default: @@ -1370,38 +574,4 @@ ${JSON.stringify(createdMetadata, null, 2)}`, return `Unknown action "${action}". Valid actions are "get", "edit", "list", "getFieldOptions", or "create".`; } } - - /** - * Gets metadata for a specific document or all documents - * @param documentId Optional ID of a specific document to get metadata for - * @returns Document metadata or metadata for all documents - */ - private getDocumentMetadata(documentId?: string): any { - if (documentId) { - const doc = this.documentsById.get(documentId); - // Get metadata for a specific document - return this.extractDocumentMetadata(doc); - } else { - // Get metadata for all documents - const documentsMetadata: Record<string, any> = {}; - for (const doc of this.documentsById.values()) { - documentsMetadata.add(this.extractDocumentMetadata(doc)); - } - - return { - documentCount: this.documentsById.size, - documents: documentsMetadata, - fieldDefinitions: this.fieldMetadata, - }; - } - } - - /** - * Helper method to validate a document type and ensure it's a valid supportedDocType - * @param docType The document type to validate - * @returns True if the document type is valid, false otherwise - */ - private isValidDocType(docType: string): boolean { - return Object.values(supportedDocTypes).includes(docType as supportedDocTypes); - } } diff --git a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts new file mode 100644 index 000000000..c954226e4 --- /dev/null +++ b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts @@ -0,0 +1,923 @@ +import { ChatBox } from '../chatboxcomponents/ChatBox'; +import { Doc, FieldType, Opt } from '../../../../../fields/Doc'; +import { DocData } from '../../../../../fields/DocSymbols'; +import { Observation } from '../types/types'; +import { ParametersType, ToolInfo, Parameter } from '../types/tool_types'; +import { BaseTool } from '../tools/BaseTool'; +import { Docs, DocumentOptions } from '../../../../documents/Documents'; +import { CollectionFreeFormDocumentView } from '../../CollectionFreeFormDocumentView'; +import { v4 as uuidv4 } from 'uuid'; +import { LinkManager, UPDATE_SERVER_CACHE } from '../../../../util/LinkManager'; +import { DocCast, StrCast } from '../../../../../fields/Types'; +import { supportedDocTypes } from '../types/tool_types'; +import { parsedDoc } from '../chatboxcomponents/ChatBox'; +import { faThumbTackSlash } from '@fortawesome/free-solid-svg-icons'; +import { DocumentManager } from '../../../../util/DocumentManager'; +import { DocumentView } from '../../DocumentView'; + +/** + * Interface representing a document in the freeform view + */ +interface AgentDocument { + layoutDoc: Doc; + dataDoc: Doc; +} + +/** + * Class to manage documents in a freeform view + */ +export class AgentDocumentManager { + private documentsById: Map<string, AgentDocument>; + private chatBox: ChatBox; + private chatBoxDocument: Doc | null = null; + private fieldMetadata: Record<string, any> = {}; + private readonly DOCUMENT_ID_FIELD = '_dash_document_id'; + + /** + * Creates a new DocumentManager + * @param templateDocument The document that serves as a template for new documents + */ + constructor(chatBox: ChatBox) { + this.documentsById = new Map<string, AgentDocument>(); + this.chatBox = chatBox; + this.chatBoxDocument = chatBox.Document; + this.processDocument(this.chatBoxDocument); + this.initializeFieldMetadata(); + } + + /** + * Extracts field metadata from DocumentOptions class + */ + private initializeFieldMetadata() { + // Parse DocumentOptions to extract field definitions + const documentOptionsInstance = new DocumentOptions(); + const documentOptionsEntries = Object.entries(documentOptionsInstance); + + for (const [fieldName, fieldInfo] of documentOptionsEntries) { + // Extract field information + const fieldData: Record<string, any> = { + name: fieldName, + withoutUnderscore: fieldName.startsWith('_') ? fieldName.substring(1) : fieldName, + description: '', + type: 'unknown', + required: false, + defaultValue: undefined, + possibleValues: [], + }; + + // Check if fieldInfo has description property (it's likely a FInfo instance) + if (fieldInfo && typeof fieldInfo === 'object' && 'description' in fieldInfo) { + fieldData.description = fieldInfo.description; + + // Extract field type if available + if ('fieldType' in fieldInfo) { + fieldData.type = fieldInfo.fieldType; + } + + // Extract possible values if available + if ('values' in fieldInfo && Array.isArray(fieldInfo.values)) { + fieldData.possibleValues = fieldInfo.values; + } + } + + this.fieldMetadata[fieldName] = fieldData; + } + } + + /** + * Gets all documents in the same Freeform view as the ChatBox + * Uses the LinkManager to get all linked documents, similar to how ChatBox does it + */ + public initializeFindDocsFreeform() { + // Reset collections + this.documentsById.clear(); + + try { + // Use the LinkManager approach which is proven to work in ChatBox + if (this.chatBoxDocument) { + console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument.id); + + // Get directly linked documents via LinkManager + const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.chatBoxDocument) + .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.chatBoxDocument!))) + .map(d => DocCast(d?.annotationOn, d)) + .filter(d => d); + + console.log(`Found ${linkedDocs.length} linked documents via LinkManager`); + + // Process the linked documents + linkedDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + + // Include the ChatBox document itself + this.processDocument(this.chatBoxDocument); + + // If we have access to the Document's parent, try to find sibling documents + if (this.chatBoxDocument.parent) { + const parent = this.chatBoxDocument.parent; + console.log('Found parent document, checking for siblings'); + + // Check if parent is a Doc type and has a childDocs function + if (parent && typeof parent === 'object' && 'childDocs' in parent && typeof parent.childDocs === 'function') { + try { + const siblingDocs = parent.childDocs(); + if (Array.isArray(siblingDocs)) { + console.log(`Found ${siblingDocs.length} sibling documents via parent.childDocs()`); + siblingDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + } catch (e) { + console.warn('Error accessing parent.childDocs:', e); + } + } + } + } else if (this.chatBox && this.chatBox.linkedDocs) { + // If we have direct access to the linkedDocs computed property from ChatBox + console.log('Using ChatBox.linkedDocs directly'); + const linkedDocs = this.chatBox.linkedDocs; + if (Array.isArray(linkedDocs)) { + console.log(`Found ${linkedDocs.length} documents via ChatBox.linkedDocs`); + linkedDocs.forEach((doc: Doc) => { + if (doc) { + this.processDocument(doc); + } + }); + } + + // Process the ChatBox document if available + if (this.chatBox.Document) { + this.processDocument(this.chatBox.Document); + } + } else { + console.warn('No ChatBox document reference available for finding linked documents'); + } + + console.log(`DocumentMetadataTool found ${this.documentsById.size} total documents`); + } catch (error) { + console.error('Error finding documents in Freeform view:', error); + } + } + + /** + * Process a document by ensuring it has an ID and adding it to the appropriate collections + * @param doc The document to process + */ + public processDocument(doc: Doc) { + // Ensure document has a persistent ID + const docId = this.ensureDocumentId(doc); + // Only add if we haven't already processed this document + if (!this.documentsById.has(docId)) { + this.documentsById.set(docId, { layoutDoc: doc, dataDoc: doc[DocData] }); + } + } + + /** + * Ensures a document has a persistent ID stored in its metadata + * @param doc The document to ensure has an ID + * @returns The document's ID + */ + private ensureDocumentId(doc: Doc): string { + let docId: string | undefined; + + // First try to get the ID from our custom field + if (doc[this.DOCUMENT_ID_FIELD]) { + docId = String(doc[this.DOCUMENT_ID_FIELD]); + return docId; + } + + // Try different ways to get a document ID + + // 1. Try the direct id property if it exists + if (doc.id && typeof doc.id === 'string') { + docId = doc.id; + } + // 2. Try doc._id if it exists + else if (doc._id && typeof doc._id === 'string') { + docId = doc._id; + } + // 3. Try doc.data?.id if it exists + else if (doc.data && typeof doc.data === 'object' && 'id' in doc.data && typeof doc.data.id === 'string') { + docId = doc.data.id; + } + // 4. If none of the above work, generate a UUID + else { + docId = uuidv4(); + console.log(`Generated new UUID for document with title: ${doc.title || 'Untitled'}`); + } + + // Store the ID in the document's metadata so it persists + try { + doc[this.DOCUMENT_ID_FIELD] = docId; + } catch (e) { + console.warn(`Could not assign ID to document property`, e); + } + + return docId; + } + + /** + * Extracts metadata from a specific document + * @param docId The ID of the document to extract metadata from + * @returns An object containing the document's metadata + */ + public extractDocumentMetadata(doc?: AgentDocument) { + if (!doc) return null; + const layoutDoc = doc.layoutDoc; + const dataDoc = doc.dataDoc; + + const metadata: Record<string, any> = { + id: layoutDoc.dash_document_id || layoutDoc.id || '', + title: layoutDoc.title || '', + type: layoutDoc.type || '', + fields: { + layout: {}, + data: {}, + }, + fieldLocationMap: {}, + }; + + // Process all known field definitions + Object.keys(this.fieldMetadata).forEach(fieldName => { + const fieldDef = this.fieldMetadata[fieldName]; + const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check if field exists on layout document + let layoutValue = undefined; + if (layoutDoc) { + layoutValue = layoutDoc[fieldName]; + if (layoutValue !== undefined) { + // Field exists on layout document + metadata.fields.layout[fieldName] = this.formatFieldValue(layoutValue); + metadata.fieldLocationMap[strippedName] = 'layout'; + } + } + + // Check if field exists on data document + let dataValue = undefined; + if (dataDoc) { + dataValue = dataDoc[fieldName]; + if (dataValue !== undefined) { + // Field exists on data document + metadata.fields.data[fieldName] = this.formatFieldValue(dataValue); + if (!metadata.fieldLocationMap[strippedName]) { + metadata.fieldLocationMap[strippedName] = 'data'; + } + } + } + + // For fields with stripped names (without leading underscore), + // also check if they exist on documents without the underscore + if (fieldName.startsWith('_')) { + const nonUnderscoreFieldName = fieldName.substring(1); + + if (layoutDoc) { + const nonUnderscoreLayoutValue = layoutDoc[nonUnderscoreFieldName]; + if (nonUnderscoreLayoutValue !== undefined) { + metadata.fields.layout[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreLayoutValue); + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'layout'; + } + } + + if (dataDoc) { + const nonUnderscoreDataValue = dataDoc[nonUnderscoreFieldName]; + if (nonUnderscoreDataValue !== undefined) { + metadata.fields.data[nonUnderscoreFieldName] = this.formatFieldValue(nonUnderscoreDataValue); + if (!metadata.fieldLocationMap[nonUnderscoreFieldName]) { + metadata.fieldLocationMap[nonUnderscoreFieldName] = 'data'; + } + } + } + } + }); + + // Add common field aliases for easier discovery + // This helps users understand both width and _width refer to the same property + if (metadata.fields.layout._width !== undefined && metadata.fields.layout.width === undefined) { + metadata.fields.layout.width = metadata.fields.layout._width; + metadata.fieldLocationMap.width = 'layout'; + } + + if (metadata.fields.layout._height !== undefined && metadata.fields.layout.height === undefined) { + metadata.fields.layout.height = metadata.fields.layout._height; + metadata.fieldLocationMap.height = 'layout'; + } + + return metadata; + } + + /** + * Formats a field value for JSON output + * @param value The field value to format + * @returns A JSON-friendly representation of the field value + */ + private formatFieldValue(value: any): any { + if (value === undefined || value === null) { + return null; + } + + // Handle Doc objects + if (value instanceof Doc) { + return { + type: 'Doc', + id: value.id || this.ensureDocumentId(value), + title: value.title || '', + docType: value.type || '', + }; + } + + // Handle RichTextField (try to extract plain text) + if (typeof value === 'string' && value.includes('"type":"doc"') && value.includes('"content":')) { + try { + const rtfObj = JSON.parse(value); + // If this looks like a rich text field structure + if (rtfObj.doc && rtfObj.doc.content) { + // Recursively extract text from the content + let plainText = ''; + const extractText = (node: any) => { + if (node.text) { + plainText += node.text; + } + if (node.content && Array.isArray(node.content)) { + node.content.forEach((child: any) => extractText(child)); + } + }; + + extractText(rtfObj.doc); + + // If we successfully extracted text, show it, but also preserve the original value + if (plainText) { + return { + type: 'RichText', + text: plainText, + length: plainText.length, + // Don't include the full value as it can be very large + }; + } + } + } catch (e) { + // If parsing fails, just treat as a regular string + } + } + + // Handle arrays and complex objects + if (typeof value === 'object') { + // If the object has a toString method, use it + if (value.toString && value.toString !== Object.prototype.toString) { + return value.toString(); + } + + try { + // Try to convert to JSON string + return JSON.stringify(value); + } catch (e) { + return '[Complex Object]'; + } + } + + // Return primitive values as is + return value; + } + + /** + * Converts a string field value to the appropriate type based on field metadata + * @param fieldName The name of the field + * @param fieldValue The string value to convert + * @returns The converted value with the appropriate type + */ + private convertFieldValue(fieldName: string, fieldValue: any): any { + // If fieldValue is already a number or boolean, we don't need to convert it from string + if (typeof fieldValue === 'number' || typeof fieldValue === 'boolean') { + return fieldValue; + } + + // If fieldValue is a string "true" or "false", convert to boolean + if (typeof fieldValue === 'string') { + if (fieldValue.toLowerCase() === 'true') { + return true; + } + if (fieldValue.toLowerCase() === 'false') { + return false; + } + } + + // If fieldValue is not a string (and not a number or boolean), convert it to string + if (typeof fieldValue !== 'string') { + fieldValue = String(fieldValue); + } + + // Special handling for text field - convert to proper RichTextField format + if (fieldName === 'text') { + try { + // Check if it's already a valid JSON RichTextField + JSON.parse(fieldValue); + return fieldValue; + } catch (e) { + // It's a plain text string, so convert it to RichTextField format + const rtf = { + doc: { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: fieldValue, + }, + ], + }, + ], + }, + }; + return JSON.stringify(rtf); + } + } + + // Get field metadata + const normalizedFieldName = fieldName.startsWith('_') ? fieldName : `_${fieldName}`; + const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Check both versions of the field name in metadata + const fieldMeta = this.fieldMetadata[normalizedFieldName] || this.fieldMetadata[strippedFieldName]; + + // Special handling for width and height without metadata + if (!fieldMeta && (fieldName === '_width' || fieldName === '_height' || fieldName === 'width' || fieldName === 'height')) { + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } + + if (!fieldMeta) { + // If no metadata found, just return the string value + return fieldValue; + } + + // Convert based on field type + const fieldType = fieldMeta.type; + + if (fieldType === 'boolean') { + // Convert to boolean + return fieldValue.toLowerCase() === 'true'; + } else if (fieldType === 'number') { + // Convert to number + const num = Number(fieldValue); + return isNaN(num) ? fieldValue : num; + } else if (fieldType === 'date') { + // Try to convert to date (stored as number timestamp) + try { + return new Date(fieldValue).getTime(); + } catch (e) { + return fieldValue; + } + } else if (fieldType.includes('list') || fieldType.includes('array')) { + // Try to parse as JSON array + try { + return JSON.parse(fieldValue); + } catch (e) { + return fieldValue; + } + } else if (fieldType === 'json' || fieldType === 'object') { + // Try to parse as JSON object + try { + return JSON.parse(fieldValue); + } catch (e) { + return fieldValue; + } + } + + // Default to string + return fieldValue; + } + + /** + * Extracts all field metadata from DocumentOptions + * @returns A structured object containing metadata about all available document fields + */ + public getAllFieldMetadata() { + // Start with our already populated fieldMetadata from the DocumentOptions class + const result: Record<string, any> = { + fieldCount: Object.keys(this.fieldMetadata).length, + fields: {}, + fieldsByType: { + string: [], + number: [], + boolean: [], + //doc: [], + //list: [], + //date: [], + //enumeration: [], + //other: [], + }, + fieldNameMappings: {}, + commonFields: { + appearance: [], + position: [], + size: [], + content: [], + behavior: [], + layout: [], + }, + }; + + // Process each field in the metadata + Object.entries(this.fieldMetadata).forEach(([fieldName, fieldInfo]) => { + const strippedName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Add to fieldNameMappings + if (fieldName.startsWith('_')) { + result.fieldNameMappings[strippedName] = fieldName; + } + + // Create structured field metadata + const fieldData: Record<string, any> = { + name: fieldName, + displayName: strippedName, + description: fieldInfo.description || '', + type: fieldInfo.fieldType || 'unknown', + possibleValues: fieldInfo.values || [], + }; + + // Add field to fields collection + result.fields[fieldName] = fieldData; + + // Categorize by field type + const type = fieldInfo.fieldType?.toLowerCase() || 'unknown'; + if (type === 'string') { + result.fieldsByType.string.push(fieldName); + } else if (type === 'number') { + result.fieldsByType.number.push(fieldName); + } else if (type === 'boolean') { + result.fieldsByType.boolean.push(fieldName); + } else if (type === 'doc') { + //result.fieldsByType.doc.push(fieldName); + } else if (type === 'list') { + //result.fieldsByType.list.push(fieldName); + } else if (type === 'date') { + //result.fieldsByType.date.push(fieldName); + } else if (type === 'enumeration') { + //result.fieldsByType.enumeration.push(fieldName); + } else { + //result.fieldsByType.other.push(fieldName); + } + + // Categorize by field purpose + if (fieldName.includes('width') || fieldName.includes('height') || fieldName.includes('size')) { + result.commonFields.size.push(fieldName); + } else if (fieldName.includes('color') || fieldName.includes('background') || fieldName.includes('border')) { + result.commonFields.appearance.push(fieldName); + } else if (fieldName.includes('x') || fieldName.includes('y') || fieldName.includes('position') || fieldName.includes('pan')) { + result.commonFields.position.push(fieldName); + } else if (fieldName.includes('text') || fieldName.includes('title') || fieldName.includes('data')) { + result.commonFields.content.push(fieldName); + } else if (fieldName.includes('action') || fieldName.includes('click') || fieldName.includes('event')) { + result.commonFields.behavior.push(fieldName); + } else if (fieldName.includes('layout')) { + result.commonFields.layout.push(fieldName); + } + }); + + // Add special section for auto-sizing related fields + result.autoSizingFields = { + height: { + autoHeightField: '_layout_autoHeight', + heightField: '_height', + displayName: 'height', + usage: 'To manually set height, first set layout_autoHeight to false', + }, + width: { + autoWidthField: '_layout_autoWidth', + widthField: '_width', + displayName: 'width', + usage: 'To manually set width, first set layout_autoWidth to false', + }, + }; + + // Add special section for text field format + result.specialFields = { + text: { + name: 'text', + description: 'Document text content', + format: 'RichTextField', + note: 'When setting text, provide plain text - it will be automatically converted to the correct format', + example: 'For setting: "Hello world" (plain text); For getting: Will be converted to plaintext for display', + }, + }; + + return result; + } + + /** + * Edits a specific field on a document + * @param docId The ID of the document to edit + * @param fieldName The name of the field to edit + * @param fieldValue The new value for the field (string, number, or boolean) + * @returns Object with success status, message, and additional information + */ + public editDocumentField( + docId: string, + fieldName: string, + fieldValue: string | number | boolean + ): { + success: boolean; + message: string; + fieldName?: string; + originalFieldName?: string; + newValue?: any; + warning?: string; + } { + // Normalize field name (handle with/without underscore) + let normalizedFieldName = fieldName.startsWith('_') ? fieldName : fieldName; + const strippedFieldName = fieldName.startsWith('_') ? fieldName.substring(1) : fieldName; + + // Handle common field name aliases (width → _width, height → _height) + // Many document fields use '_' prefix for layout properties + if (fieldName === 'width') { + normalizedFieldName = '_width'; + } else if (fieldName === 'height') { + normalizedFieldName = '_height'; + } + + // Get the documents + const doc = this.documentsById.get(docId); + if (!doc) { + return { success: false, message: `Document with ID ${docId} not found` }; + } + + const { layoutDoc, dataDoc } = this.documentsById.get(docId) ?? { layoutDoc: null, dataDoc: null }; + + if (!layoutDoc && !dataDoc) { + return { success: false, message: `Could not find layout or data document for document with ID ${docId}` }; + } + + try { + // Convert the field value to the appropriate type based on field metadata + const convertedValue = this.convertFieldValue(normalizedFieldName, fieldValue); + + let targetDoc: Doc | undefined; + let targetLocation: string; + + // First, check if field exists on layout document using Doc.Get + if (layoutDoc) { + const fieldExistsOnLayout = Doc.Get(layoutDoc, normalizedFieldName, true) !== undefined; + + // If it exists on layout document, update it there + if (fieldExistsOnLayout) { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } + // If it has an underscore prefix, it's likely a layout property even if not yet set + else if (normalizedFieldName.startsWith('_')) { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } + // Otherwise, look for or create on data document + else if (dataDoc) { + targetDoc = dataDoc; + targetLocation = 'data'; + } + // If no data document available, default to layout + else { + targetDoc = layoutDoc; + targetLocation = 'layout'; + } + } + // If no layout document, use data document + else if (dataDoc) { + targetDoc = dataDoc; + targetLocation = 'data'; + } else { + return { success: false, message: `No valid document found for editing` }; + } + + if (!targetDoc) { + return { success: false, message: `Target document not available` }; + } + + // Set the field value on the target document + targetDoc[normalizedFieldName] = convertedValue; + + return { + success: true, + message: `Successfully updated field '${normalizedFieldName}' on ${targetLocation} document (ID: ${docId})`, + fieldName: normalizedFieldName, + originalFieldName: fieldName, + newValue: convertedValue, + }; + } catch (error) { + console.error('Error editing document field:', error); + return { + success: false, + message: `Error updating field: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + /** + * Gets metadata for a specific document or all documents + * @param documentId Optional ID of a specific document to get metadata for + * @returns Document metadata or metadata for all documents + */ + public getDocumentMetadata(documentId?: string): any { + if (documentId) { + const doc = this.documentsById.get(documentId); + // Get metadata for a specific document + return this.extractDocumentMetadata(doc); + } else { + // Get metadata for all documents + const documentsMetadata: Record<string, any> = {}; + for (const doc of this.documentsById.values()) { + documentsMetadata.add(this.extractDocumentMetadata(doc)); + } + + return { + documentCount: this.documentsById.size, + documents: documentsMetadata, + fieldDefinitions: this.fieldMetadata, + }; + } + } + + /** + * Adds links between documents based on their IDs + * @param docIds Array of document IDs to link + * @param relationship Optional relationship type for the links + * @returns Array of created link documents + */ + public addLinks(docIds: string[]): Doc[] { + const createdLinks: Doc[] = []; + // Use string keys for Set instead of arrays which don't work as expected as keys + const alreadyLinked = new Set<string>(); + + // Iterate over the document IDs and add links + docIds.forEach(docId1 => { + const doc1 = this.documentsById.get(docId1); + docIds.forEach(docId2 => { + if (docId1 === docId2) return; // Skip self-linking + + // Create a consistent key regardless of document order + const linkKey = [docId1, docId2].sort().join('_'); + if (alreadyLinked.has(linkKey)) return; + + const doc2 = this.documentsById.get(docId2); + if (doc1?.layoutDoc && doc2?.layoutDoc) { + try { + // Create a link document between doc1 and doc2 + const linkDoc = Docs.Create.LinkDocument(doc1.layoutDoc, doc2.layoutDoc); + + // Set a default color if relationship doesn't specify one + if (!linkDoc.color) { + linkDoc.color = 'lightBlue'; // Default blue color + } + + // Ensure link is visible by setting essential properties + linkDoc.link_visible = true; + linkDoc.link_enabled = true; + linkDoc.link_autoMove = true; + linkDoc.link_showDirected = true; + + // Set the embedContainer to ensure visibility + // This is shown in the image as a key difference between visible/non-visible links + if (this.chatBoxDocument && this.chatBoxDocument.parent && typeof this.chatBoxDocument.parent === 'object' && 'title' in this.chatBoxDocument.parent) { + linkDoc.embedContainer = String(this.chatBoxDocument.parent.title); + } else if (doc1.layoutDoc.parent && typeof doc1.layoutDoc.parent === 'object' && 'title' in doc1.layoutDoc.parent) { + linkDoc.embedContainer = String(doc1.layoutDoc.parent.title); + } else { + // Default to a tab name if we can't find one + linkDoc.embedContainer = 'Untitled Tab 1'; + } + + // Add the link to the document system + LinkManager.Instance.addLink(linkDoc); + + const ancestor = DocumentView.linkCommonAncestor(linkDoc); + ancestor?.ComponentView?.addDocument?.(linkDoc); + // Add to user document list to make it visible in the UI + Doc.AddDocToList(Doc.UserDoc(), 'links', linkDoc); + + // Create a visual link for display + if (this.chatBoxDocument) { + // Make sure the docs are visible in the UI + this.chatBox._props.addDocument?.(doc1.layoutDoc); + this.chatBox._props.addDocument?.(doc2.layoutDoc); + + // Use DocumentManager to ensure documents are visible + DocumentManager.Instance.showDocument(doc1.layoutDoc, { willZoomCentered: false }); + DocumentManager.Instance.showDocument(doc2.layoutDoc, { willZoomCentered: false }); + } + + createdLinks.push(linkDoc); + alreadyLinked.add(linkKey); + } catch (error) { + console.error('Error creating link between documents:', error); + } + } + }); + }); + + // Force update of the UI to show new links + setTimeout(() => { + try { + // Update server cache to ensure links are persisted + UPDATE_SERVER_CACHE && typeof UPDATE_SERVER_CACHE === 'function' && UPDATE_SERVER_CACHE(); + } catch (e) { + console.warn('Could not update server cache after creating links:', e); + } + }, 100); + + return createdLinks; + } + /** + * Helper method to validate a document type and ensure it's a valid supportedDocType + * @param docType The document type to validate + * @returns True if the document type is valid, false otherwise + */ + private isValidDocType(docType: string): boolean { + return Object.values(supportedDocTypes).includes(docType as supportedDocTypes); + } + /** + * Creates a document in the dashboard. + * + * @param {string} doc_type - The type of document to create. + * @param {string} data - The data used to generate the document. + * @param {DocumentOptions} options - Configuration options for the document. + * @returns {Promise<void>} A promise that resolves once the document is created and displayed. + */ + createDocInDash = (docType: string, title: string, data: string) => { + // Validate doc_type + if (!this.isValidDocType(docType)) { + throw new Error(`Invalid document type: ${docType}`); + } + + try { + // Create simple document with just title and data + const simpleDoc: parsedDoc = { + doc_type: docType, + title: title, + data: data, + x: 0, + y: 0, + _width: 300, + _height: 300, + _layout_fitWidth: false, + _layout_autoHeight: true, + }; + + // Use the chatBox's createDocInDash method to create and link the document + if (!this.chatBox) { + throw new Error('ChatBox instance not available for creating document'); + } + const linkAndShowDoc = (doc: Opt<Doc>) => { + if (doc) { + LinkManager.Instance.addLink(Docs.Create.LinkDocument(this.chatBoxDocument!, doc)); + this.chatBox._props.addDocument?.(doc); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + } + }; + const doc = this.chatBox.whichDoc(simpleDoc, false); + if (doc) linkAndShowDoc(doc); + return doc; + } catch (error) { + throw new Error(`Error creating document: ${error}`); + } + }; + + public has(docId: string) { + return this.documentsById.has(docId); + } + + public listDocs() { + // List all available documents in simple format + const docs = Array.from(this.documentsById.entries()).map(([id, doc]) => ({ + id, + title: doc.layoutDoc.title || 'Untitled Document', + type: doc.layoutDoc.type || doc.dataDoc.type || 'Unknown Type', + })); + + if (docs.length === 0) { + return [ + { + type: 'text', + text: 'No documents found in the current view.', + }, + ]; + } + + return [ + { + type: 'text', + text: `Found ${docs.length} document(s) in the current view:\n${JSON.stringify(docs, null, 2)}`, + }, + ]; + } + + public createAgentDoc(doc: Doc) { + // Ideally check if Doc is already in there. + const agentDoc = { layoutDoc: doc, dataDoc: doc[DocData] }; + this.documentsById.set(this.ensureDocumentId(doc), agentDoc); + return agentDoc; + } +} -- cgit v1.2.3-70-g09d2 From 3ef3d40506348d9fd537cc8f4aea975b9770689f Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" <Shulman.aj@gmail.com> Date: Sun, 27 Apr 2025 13:14:49 -0400 Subject: new attempt with new citation unification --- .../views/nodes/chatbot/agentsystem/Agent.ts | 5 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 450 +++++++++++++-------- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 16 +- src/client/views/nodes/chatbot/tools/SearchTool.ts | 18 +- src/client/views/nodes/chatbot/types/types.ts | 1 + .../nodes/chatbot/utils/AgentDocumentManager.ts | 168 +++++--- .../views/nodes/chatbot/vectorstore/Vectorstore.ts | 130 ++++-- 7 files changed, 510 insertions(+), 278 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index c021d141e..80fdb6533 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -65,12 +65,9 @@ export class Agent { summaries: () => string, history: () => string, csvData: () => { filename: string; id: string; text: string }[], - addLinkedUrlDoc: (url: string, id: string) => void, getLinkedUrlDocId: (url: string) => string[], createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void, - // eslint-disable-next-line @typescript-eslint/no-unused-vars createCSVInDash: (url: string, title: string, id: string, data: string) => void, - chatBox: ChatBox, docManager: AgentDocumentManager ) { // Initialize OpenAI client with API key from environment @@ -87,7 +84,7 @@ export class Agent { rag: new RAGTool(this.vectorstore), dataAnalysis: new DataAnalysisTool(csvData), websiteInfoScraper: new WebsiteInfoScraperTool(getLinkedUrlDocId), - searchTool: new SearchTool(addLinkedUrlDoc), + searchTool: new SearchTool(this._docManager), noTool: new NoTool(), //imageCreationTool: new ImageCreationTool(createImage), documentMetadata: new DocumentMetadataTool(this._docManager), diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 43765c1ce..35dbee3e9 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -71,7 +71,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @observable private _citationPopup: { text: string; visible: boolean } = { text: '', visible: false }; // Private properties for managing OpenAI API, vector store, agent, and UI elements - private openai: OpenAI; + private openai!: OpenAI; // Using definite assignment assertion private vectorstore_id: string; private vectorstore: Vectorstore; private agent: Agent; @@ -98,25 +98,34 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ constructor(props: FieldViewProps) { super(props); - makeObservable(this); // Enable MobX observables + makeObservable(this); - // Initialize OpenAI, vectorstore, and agent - this.openai = this.initializeOpenAI(); - if (StrCast(this.dataDoc.vectorstore_id) == '') { - this.vectorstore_id = uuidv4(); - this.dataDoc.vectorstore_id = this.vectorstore_id; - } else { - this.vectorstore_id = StrCast(this.dataDoc.vectorstore_id); - } - this.vectorstore = new Vectorstore(this.vectorstore_id, this.retrieveDocIds); + this.messagesRef = React.createRef(); this.docManager = new AgentDocumentManager(this); - this.agent = new Agent(this.vectorstore, this.retrieveSummaries, this.retrieveFormattedHistory, this.retrieveCSVData, this.addLinkedUrlDoc, this.getLinkedUrlDocIds, this.createImageInDash, this.createCSVInDash, this, this.docManager); - // Reinitialize the DocumentMetadataTool with a direct reference to this ChatBox instance - // This ensures the tool can properly access documents in the same Freeform view - this.agent.reinitializeDocumentMetadataTool(); + // Initialize OpenAI client + this.initializeOpenAI(); + + // Create a unique vectorstore ID for this ChatBox + this.vectorstore_id = uuidv4(); + + // Initialize vectorstore with the document manager + this.vectorstore = new Vectorstore(this.vectorstore_id, this.docManager); + + // Create an agent with the vectorstore + this.agent = new Agent( + this.vectorstore, + this.retrieveSummaries.bind(this), + this.retrieveFormattedHistory.bind(this), + this.retrieveCSVData.bind(this), + this.retrieveDocIds.bind(this), + this.createImageInDash.bind(this), + this.createCSVInDash.bind(this), + this.docManager + ); - this.messagesRef = React.createRef<HTMLDivElement>(); + // Add event listeners + this.addScrollListener(); // Reaction to update dataDoc when chat history changes reaction( @@ -140,22 +149,25 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action addDocToVectorstore = async (newLinkedDoc: Doc) => { - this._uploadProgress = 0; - this._currentStep = 'Initializing...'; - this._isUploadingDocs = true; - try { - // Add the document to the vectorstore + this._isUploadingDocs = true; + + // Process the document first to ensure it has a valid ID + this.docManager.processDocument(newLinkedDoc); + + // Add the document to the vectorstore which will also register chunks await this.vectorstore.addAIDoc(newLinkedDoc, this.updateProgress); - } catch (error) { - console.error('Error uploading document:', error); - this._currentStep = 'Error during upload'; - } finally { - runInAction(() => { - this._isUploadingDocs = false; - this._uploadProgress = 0; - this._currentStep = ''; - }); + + // No longer needed as documents are tracked by the AgentDocumentManager + // this._linked_docs_to_add.add(newLinkedDoc); + + this._isUploadingDocs = false; + + return true; + } catch (err) { + console.error('Error adding document to vectorstore:', err); + this._isUploadingDocs = false; + return false; } }; @@ -238,7 +250,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { apiKey: process.env.OPENAI_KEY, dangerouslyAllowBrowser: true, }; - return new OpenAI(configuration); + this.openai = new OpenAI(configuration); } /** @@ -375,49 +387,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } }; - /** - * Adds a linked document from a URL for future reference and analysis. - * @param url The URL of the document to add. - * @param id The unique identifier for the document. - */ - @action - addLinkedUrlDoc = async (url: string, id: string) => { - const doc = Docs.Create.WebDocument(url, { data_useCors: true }); - this.docManager.addCustomId(doc, id); - const linkDoc = Docs.Create.LinkDocument(this.Document, doc); - LinkManager.Instance.addLink(linkDoc); - - const chunkToAdd = { - chunkId: id, - chunkType: CHUNK_TYPE.URL, - url: url, - }; - - doc.chunk_simpl = JSON.stringify({ chunks: [chunkToAdd] }); - this.docManager.processDocument(doc); - }; - - /** - * Retrieves the IDs of linked url documents. - * @returns An array of document IDs. - */ - @action - getLinkedUrlDocIds = () => { - const linkedDocs: Doc[] = this.linkedDocs; - const linkedUrlDocIds: string[] = []; - - for (const doc of linkedDocs) { - if (doc.chunk_simpl) { - const docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; - const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkType === CHUNK_TYPE.URL); - if (foundChunk) { - linkedUrlDocIds.push(foundChunk.chunkId); - } - } - } - return linkedUrlDocIds; - }; - /** * Getter to retrieve the current user's name from the client utils. */ @@ -613,82 +582,224 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action handleCitationClick = async (citation: Citation) => { - const currentLinkedDocs: Doc[] = this.linkedDocs; - const chunkId = citation.chunk_id; + try { + // Extract values from MobX proxy object if needed + const chunkId = typeof citation.chunk_id === 'object' ? (citation.chunk_id as any).toString() : citation.chunk_id; + + // For debugging + console.log('Citation clicked:', { + chunkId, + citation: JSON.stringify(citation, null, 2), + }); - for (const doc of currentLinkedDocs) { - if (doc.chunk_simpl) { - const docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; - const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkId === chunkId); + // Try to find the document + const linkedDocs = this.linkedDocs; + let doc: Doc | undefined; - if (foundChunk) { - // Handle media chunks specifically + // First try to find the document using the document manager's chunk ID lookup + const parentDocId = this.docManager.getDocIdByChunkId(chunkId); + if (parentDocId) { + doc = this.docManager.getDocument(parentDocId); + console.log(`Found document by chunk ID lookup: ${parentDocId}`); + } - if (doc.ai_type == 'video' || doc.ai_type == 'audio') { - const directMatchSegmentStart = this.getDirectMatchingSegmentStart(doc, citation.direct_text || '', foundChunk.indexes || []); + // If not found, fall back to searching through linked docs (maintains compatibility) + if (!doc) { + for (const linkedDoc of linkedDocs) { + if (linkedDoc.chunk_simpl) { + try { + const docChunkSimpl = JSON.parse(StrCast(linkedDoc.chunk_simpl)) as { chunks: SimplifiedChunk[] }; + const foundChunk = docChunkSimpl.chunks.find(chunk => chunk.chunkId === chunkId); + if (foundChunk) { + doc = linkedDoc; + console.log(`Found document by iterating through linked docs`); + break; + } + } catch (e) { + console.error(`Error parsing chunk_simpl for doc ${linkedDoc.id}:`, e); + } + } + } + } - if (directMatchSegmentStart) { - // Navigate to the segment's start time in the media player - await this.goToMediaTimestamp(doc, directMatchSegmentStart, doc.ai_type); - } else { - console.error('No direct matching segment found for the citation.'); + if (!doc) { + console.warn(`Document not found for citation with chunk_id: ${chunkId}`); + return; + } + + // Process the chunk data + let docChunkSimpl: { chunks: SimplifiedChunk[] } = { chunks: [] }; + try { + docChunkSimpl = JSON.parse(StrCast(doc.chunk_simpl) || '{"chunks":[]}'); + } catch (e) { + console.error(`Error parsing chunk_simpl for the found document:`, e); + return; + } + + const foundChunk = docChunkSimpl.chunks.find((chunk: SimplifiedChunk) => chunk.chunkId === chunkId); + + // Handle different chunk types + if (foundChunk) { + console.log(`Found chunk in document:`, foundChunk); + + // Handle video chunks + if (foundChunk.chunkType === CHUNK_TYPE.VIDEO) { + if (foundChunk.start_time !== undefined) { + await this.goToMediaTimestamp(doc, foundChunk.start_time, 'video'); + } else { + console.warn('Video chunk missing start_time:', foundChunk); + } + } + // Handle audio chunks - note that we're using string comparison since 'audio' isn't in CHUNK_TYPE enum + else if (String(foundChunk.chunkType).toLowerCase() === 'audio') { + if (foundChunk.start_time !== undefined) { + await this.goToMediaTimestamp(doc, foundChunk.start_time, 'audio'); + } else { + console.warn('Audio chunk missing start_time:', foundChunk); + } + } + // Handle table or image chunks + else if (foundChunk.chunkType === CHUNK_TYPE.TABLE || foundChunk.chunkType === CHUNK_TYPE.IMAGE) { + this.handleOtherChunkTypes(foundChunk, citation, doc); + } + // Handle text chunks + else if (foundChunk.chunkType === CHUNK_TYPE.TEXT) { + // Find text from the document's chunks metadata + let chunkText = ''; + + try { + // We already parsed the chunks earlier, so use that + const matchingChunk = docChunkSimpl.chunks.find(c => c.chunkId === foundChunk.chunkId); + if (matchingChunk && 'text' in matchingChunk) { + // If the text property exists on the chunk (even though it's not in the type) + chunkText = String(matchingChunk['text'] || ''); } + } catch (e) { + console.error('Error getting chunk text:', e); + } + + // Default text if none found + if (!chunkText) { + chunkText = 'Text content not available'; + } + + this._citationPopup = { + text: chunkText, + visible: true, + }; + } + // Handle URL chunks + else if (foundChunk.chunkType === CHUNK_TYPE.URL) { + if (foundChunk.url) { + // Instead of opening the URL in a new window, show the document in the viewer + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + console.log(`Navigated to web document with URL: ${foundChunk.url}`); } else { - // Handle other chunk types as before - this.handleOtherChunkTypes(foundChunk, citation, doc); + console.warn('URL chunk missing URL:', foundChunk); } } + } else if (doc?.original_segments) { + // Handle original segments for media files + let original_segments: any[] = []; + try { + original_segments = JSON.parse(StrCast(doc.original_segments)); + } catch (e) { + console.error(`Error parsing original_segments:`, e); + return; + } + + // Check if there's direct text to find in the segments + if (citation.direct_text) { + // Find the segment that contains the direct text + const start = this.getDirectMatchingSegmentStart(doc, citation.direct_text, []); + if (start !== -1) { + await this.goToMediaTimestamp(doc, start, doc.ai_type === 'audio' ? 'audio' : 'video'); + } + } + } else { + console.warn('Unable to find chunk or segments for citation', citation); } + } catch (error) { + console.error('Error handling citation click:', error); } }; + /** + * Finds a matching segment in a document based on text content. + * @param doc The document to search in + * @param citationText The text to find in the document + * @param indexesOfSegments Optional indexes of segments to search in + * @returns The starting timestamp of the matching segment, or -1 if not found + */ getDirectMatchingSegmentStart = (doc: Doc, citationText: string, indexesOfSegments: string[]): number => { - const originalSegments = JSON.parse(StrCast(doc.original_segments!)).map((segment: any, index: number) => ({ - index: index.toString(), - text: segment.text, - start: segment.start, - end: segment.end, - })); - - if (!Array.isArray(originalSegments) || originalSegments.length === 0 || !Array.isArray(indexesOfSegments)) { - return 0; + if (!doc || !citationText) return -1; + + // Get original segments from the document + const original_segments = doc.original_segments ? JSON.parse(StrCast(doc.original_segments)) : []; + + if (!original_segments || !Array.isArray(original_segments) || original_segments.length === 0) { + return -1; } - // Create itemsToSearch array based on indexesOfSegments - const itemsToSearch = indexesOfSegments.map((indexStr: string) => { - const index = parseInt(indexStr, 10); - const segment = originalSegments[index]; - return { text: segment.text, start: segment.start }; - }); + let segments = original_segments; - console.log('Constructed itemsToSearch:', itemsToSearch); + // If specific indexes are provided, filter segments by those indexes + if (indexesOfSegments && indexesOfSegments.length > 0) { + segments = original_segments.filter((segment: any) => indexesOfSegments.includes(segment.index)); + } + + // If no segments match the indexes, use all segments + if (segments.length === 0) { + segments = original_segments; + } - // Helper function to calculate word overlap score + // First try to find an exact match + const exactMatch = segments.find((segment: any) => segment.text && segment.text.includes(citationText)); + + if (exactMatch) { + return exactMatch.start; + } + + // If no exact match, find segment with best word overlap const calculateWordOverlap = (text1: string, text2: string): number => { - const words1 = new Set(text1.toLowerCase().split(/\W+/)); - const words2 = new Set(text2.toLowerCase().split(/\W+/)); - const intersection = new Set([...words1].filter(word => words2.has(word))); - return intersection.size / Math.max(words1.size, words2.size); // Jaccard similarity + if (!text1 || !text2) return 0; + + const words1 = text1.toLowerCase().split(/\s+/); + const words2 = text2.toLowerCase().split(/\s+/); + const wordSet1 = new Set(words1); + + let overlap = 0; + for (const word of words2) { + if (wordSet1.has(word)) { + overlap++; + } + } + + // Return percentage of overlap relative to the shorter text + return overlap / Math.min(words1.length, words2.length); }; - // Search for the best matching segment - let bestMatchStart = 0; - let bestScore = 0; - - console.log(`Searching for best match for query: "${citationText}"`); - itemsToSearch.forEach(item => { - const score = calculateWordOverlap(citationText, item.text); - console.log(`Comparing query to segment: "${item.text}" | Score: ${score}`); - if (score > bestScore) { - bestScore = score; - bestMatchStart = item.start; + // Find segment with highest word overlap + let bestMatch = null; + let highestOverlap = 0; + + for (const segment of segments) { + if (!segment.text) continue; + + const overlap = calculateWordOverlap(segment.text, citationText); + if (overlap > highestOverlap) { + highestOverlap = overlap; + bestMatch = segment; } - }); + } - console.log('Best match found with score:', bestScore, '| Start time:', bestMatchStart); + // Only return matches with significant overlap (more than 30%) + if (bestMatch && highestOverlap > 0.3) { + return bestMatch.start; + } - // Return the start time of the best match - return bestMatchStart; + // If no good match found, return the start of the first segment as fallback + return segments.length > 0 ? segments[0].start : -1; }; /** @@ -772,7 +883,9 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { break; case CHUNK_TYPE.CSV: case CHUNK_TYPE.URL: - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => { + console.log(`Showing web document in viewer with URL: ${foundChunk.url}`); + }); break; default: console.error('Unhandled chunk type:', foundChunk.chunkType); @@ -879,6 +992,16 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } }); this.addScrollListener(); + + // Initialize the document manager by finding existing documents + this.docManager.initializeFindDocsFreeform(); + + // If there are stored doc IDs in our list of docs to add, process them + if (this._linked_docs_to_add.size > 0) { + this._linked_docs_to_add.forEach(doc => { + this.docManager.processDocument(doc); + }); + } } /** @@ -892,28 +1015,28 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { /** * Getter that retrieves all linked documents for the current document. */ - @computed - get linkedDocs() { - return LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d); + @computed get linkedDocs(): Doc[] { + const docIds = this.docManager.listDocs(); + const docs: Doc[] = []; + + // Get documents from the document manager using the getDocument method + docIds.forEach(id => { + const doc = this.docManager.getDocument(id); + if (doc) { + docs.push(doc); + } + }); + + return docs; } /** - * Getter that retrieves document IDs of linked documents that have AI-related content. + * Getter that retrieves document IDs of linked documents that have PDF_chunker–parsed content. */ @computed - get docIds() { - return LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d) - .filter(d => { - console.log(d.ai_doc_id); - return d.ai_doc_id; - }) - .map(d => StrCast(d.ai_doc_id)); + get docIds(): string[] { + // Use the document manager to get all document IDs + return Array.from(this.docManager.listDocs()); } /** @@ -921,23 +1044,18 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @computed get summaries(): string { - return ( - LinkManager.Instance.getAllRelatedLinks(this.Document) - .map(d => DocCast(LinkManager.getOppositeAnchor(d, this.Document))) - .map(d => DocCast(d?.annotationOn, d)) - .filter(d => d) - .filter(d => d.summary) - .map((doc, index) => { - if (PDFCast(doc.data)) { - return `<summary file_name="${PDFCast(doc.data).url.pathname}" applicable_tools=["rag"]>${doc.summary}</summary>`; - } else if (CsvCast(doc.data)) { - return `<summary file_name="${CsvCast(doc.data).url.pathname}" applicable_tools=["dataAnalysis"]>${doc.summary}</summary>`; - } else { - return `${index + 1}) ${doc.summary}`; - } - }) - .join('\n') + '\n' - ); + const linkedDocs = Array.from(this.docManager.listDocs()) + .map(id => { + const doc = this.docManager.extractDocumentMetadata(id); + if (doc && doc.fields && (doc.fields.layout.summary || doc.fields.data.summary)) { + return doc.fields.layout.summary || doc.fields.data.summary; + } + return null; + }) + .filter(Boolean) + .join('\n\n'); + + return linkedDocs; } /** @@ -965,7 +1083,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // Other helper methods for retrieving document data and processing - retrieveSummaries = () => { + retrieveSummaries = (): string => { return this.summaries; }; @@ -973,12 +1091,12 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return this.linkedCSVs; }; - retrieveFormattedHistory = () => { + retrieveFormattedHistory = (): string => { return this.formattedHistory; }; - retrieveDocIds = () => { - return this.docIds; + retrieveDocIds = (): string[] => { + return Array.from(this.docManager.listDocs()); }; /** diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 4b751acc0..e6c2421e5 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -417,9 +417,9 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp const title = String(args.title); const data = String(args.data); - const createdDoc = this._docManager.createDocInDash(docType, title, data); + const id = this._docManager.createDocInDash(docType, data, { title: title }); - if (!createdDoc) { + if (!id) { return [ { type: 'text', @@ -427,18 +427,14 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp }, ]; } - - // Update our local document maps with the new document - this._docManager.processDocument(createdDoc); - // Get the created document's metadata - const createdMetadata = this._docManager.extractDocumentMetadata(this._docManager.createAgentDoc(createdDoc)); + const createdMetadata = this._docManager.extractDocumentMetadata(id); return [ { type: 'text', text: `Document created successfully. -Document ID: ${createdDoc.id} +Document ID: ${id} Type: ${docType} Title: "${title}" @@ -447,9 +443,9 @@ You can now use the "edit" action to modify additional properties of this docume Next steps: 1. Use the "getFieldOptions" action to understand available editable/addable fields/properties and their dependencies. -2. To modify this document, use: { action: "edit", documentId: "${createdDoc.id}", fieldEdits: [{"fieldName":"property","fieldValue":"value"}] } +2. To modify this document, use: { action: "edit", documentId: "${id}", fieldEdits: [{"fieldName":"property","fieldValue":"value"}] } 3. To add styling, consider setting backgroundColor, fontColor, or other properties -4. For text documents, you can edit the content with: { action: "edit", documentId: "${createdDoc.id}", fieldEdits: [{"fieldName":"text","fieldValue":"New content"}] } +4. For text documents, you can edit the content with: { action: "edit", documentId: "${id}", fieldEdits: [{"fieldName":"text","fieldValue":"New content"}] } Full metadata for the created document: ${JSON.stringify(createdMetadata, null, 2)}`, diff --git a/src/client/views/nodes/chatbot/tools/SearchTool.ts b/src/client/views/nodes/chatbot/tools/SearchTool.ts index 2ee30f0cf..53f5fc109 100644 --- a/src/client/views/nodes/chatbot/tools/SearchTool.ts +++ b/src/client/views/nodes/chatbot/tools/SearchTool.ts @@ -3,6 +3,9 @@ import { Networking } from '../../../../Network'; import { BaseTool } from './BaseTool'; import { Observation } from '../types/types'; import { ParametersType, ToolInfo } from '../types/tool_types'; +import { Agent } from 'http'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; +import { StrCast } from '../../../../../fields/Types'; const searchToolParams = [ { @@ -25,12 +28,12 @@ const searchToolInfo: ToolInfo<SearchToolParamsType> = { }; export class SearchTool extends BaseTool<SearchToolParamsType> { - private _addLinkedUrlDoc: (url: string, id: string) => void; + private _docManager: AgentDocumentManager; private _max_results: number; - constructor(addLinkedUrlDoc: (url: string, id: string) => void, max_results: number = 3) { + constructor(docManager: AgentDocumentManager, max_results: number = 3) { super(searchToolInfo); - this._addLinkedUrlDoc = addLinkedUrlDoc; + this._docManager = docManager; this._max_results = max_results; } @@ -46,8 +49,13 @@ export class SearchTool extends BaseTool<SearchToolParamsType> { max_results: this._max_results, })) as { results: { url: string; snippet: string }[] }; const data = results.map((result: { url: string; snippet: string }) => { - const id = uuidv4(); - this._addLinkedUrlDoc(result.url, id); + // Create a web document with the URL + const id = this._docManager.createDocInDash('web', result.url, { + title: `Search Result: ${result.url}`, + text_html: result.snippet, + data_useCors: true, + }); + return { type: 'text' as const, text: `<chunk chunk_id="${id}" chunk_type="url"><url>${result.url}</url><overview>${result.snippet}</overview></chunk>`, diff --git a/src/client/views/nodes/chatbot/types/types.ts b/src/client/views/nodes/chatbot/types/types.ts index 882e74ebb..dcb132ec7 100644 --- a/src/client/views/nodes/chatbot/types/types.ts +++ b/src/client/views/nodes/chatbot/types/types.ts @@ -108,6 +108,7 @@ export interface SimplifiedChunk { start_time?: number; end_time?: number; indexes?: string[]; + text?: string; } export interface AI_Document { diff --git a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts index 4eeac3c6a..c3beebcde 100644 --- a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts +++ b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts @@ -165,22 +165,18 @@ export class AgentDocumentManager { } } - public addCustomId(doc: Doc, id: string) { - doc.id = id; - doc.DOCUMENT_ID_FIELD = id; - } - /** * Process a document by ensuring it has an ID and adding it to the appropriate collections * @param doc The document to process */ - public processDocument(doc: Doc) { + public processDocument(doc: Doc): string { // Ensure document has a persistent ID const docId = this.ensureDocumentId(doc); // Only add if we haven't already processed this document if (!this.documentsById.has(docId)) { this.documentsById.set(docId, { layoutDoc: doc, dataDoc: doc[DocData] }); } + return docId; } /** @@ -232,7 +228,9 @@ export class AgentDocumentManager { * @param docId The ID of the document to extract metadata from * @returns An object containing the document's metadata */ - public extractDocumentMetadata(doc?: AgentDocument) { + public extractDocumentMetadata(id: string) { + if (!id) return null; + const doc = this.documentsById.get(id); if (!doc) return null; const layoutDoc = doc.layoutDoc; const dataDoc = doc.dataDoc; @@ -729,16 +727,14 @@ export class AgentDocumentManager { */ public getDocumentMetadata(documentId?: string): any { if (documentId) { - const doc = this.documentsById.get(documentId); - // Get metadata for a specific document - return this.extractDocumentMetadata(doc); + console.log(`Returning document metadata for docID, ${documentId}:`, this.extractDocumentMetadata(documentId)); + return this.extractDocumentMetadata(documentId); } else { // Get metadata for all documents const documentsMetadata: Record<string, any> = {}; - for (const doc of this.documentsById.values()) { - documentsMetadata.add(this.extractDocumentMetadata(doc) ?? { documentId: doc.layoutDoc.id, title: doc.layoutDoc.title, type: doc.layoutDoc.type }); + for (const documentId of this.documentsById.keys()) { + documentsMetadata.add(this.extractDocumentMetadata(documentId)); } - return { documentCount: this.documentsById.size, documents: documentsMetadata, @@ -845,14 +841,15 @@ export class AgentDocumentManager { return Object.values(supportedDocTypes).includes(docType as supportedDocTypes); } /** - * Creates a document in the dashboard. + * Creates a document in the dashboard and returns its ID. + * This is a public API used by tools like SearchTool. * - * @param {string} doc_type - The type of document to create. - * @param {string} data - The data used to generate the document. - * @param {DocumentOptions} options - Configuration options for the document. - * @returns {Promise<void>} A promise that resolves once the document is created and displayed. + * @param docType The type of document to create + * @param data The data for the document + * @param options Optional configuration options + * @returns The ID of the created document */ - createDocInDash = (docType: string, title: string, data: string) => { + public createDocInDash(docType: string, data: string, options?: any): string { // Validate doc_type if (!this.isValidDocType(docType)) { throw new Error(`Invalid document type: ${docType}`); @@ -862,10 +859,10 @@ export class AgentDocumentManager { // Create simple document with just title and data const simpleDoc: parsedDoc = { doc_type: docType, - title: title, + title: options?.title ?? `Untitled Document ${this.documentsById.size + 1}`, data: data, - x: 0, - y: 0, + x: options?.x ?? 0, + y: options?.y ?? 0, _width: 300, _height: 300, _layout_fitWidth: false, @@ -884,46 +881,111 @@ export class AgentDocumentManager { } }; const doc = this.chatBox.whichDoc(simpleDoc, false); - if (doc) linkAndShowDoc(doc); - return doc; + if (doc) { + linkAndShowDoc(doc); + const id = this.processDocument(doc); + return id; + } else { + throw new Error(`Error creating document. Created document not found.`); + } } catch (error) { throw new Error(`Error creating document: ${error}`); } - }; + } public has(docId: string) { return this.documentsById.has(docId); } - public listDocs() { - // List all available documents in simple format - const docs = Array.from(this.documentsById.entries()).map(([id, doc]) => ({ - id, - title: doc.layoutDoc.title || 'Untitled Document', - type: doc.layoutDoc.type || doc.dataDoc.type || 'Unknown Type', - })); - - if (docs.length === 0) { - return [ - { - type: 'text', - text: 'No documents found in the current view.', - }, - ]; - } - - return [ - { - type: 'text', - text: `Found ${docs.length} document(s) in the current view:\n${JSON.stringify(docs, null, 2)}`, - }, - ]; + /** + * Returns a list of all document IDs in the manager. + * @returns An array of document IDs (strings). + */ + public listDocs(): string[] { + return Array.from(this.documentsById.keys()); + } + + /** + * Adds a document with a custom ID to the manager + * @param doc The document to add + * @param customId The custom ID to assign to the document + * @returns The customId that was assigned + */ + public addCustomId(doc: Doc, customId: string): string { + if (!doc) { + console.error('Cannot add null document with custom ID'); + return ''; + } + + // Set the custom ID in the document's metadata + doc[this.DOCUMENT_ID_FIELD] = customId; + + // Store the document in our map + this.documentsById.set(customId, { + layoutDoc: doc, + dataDoc: doc, + }); + + return customId; } - public createAgentDoc(doc: Doc) { - // Ideally check if Doc is already in there. - const agentDoc = { layoutDoc: doc, dataDoc: doc[DocData] }; - this.documentsById.set(this.ensureDocumentId(doc), agentDoc); - return agentDoc; + /** + * Gets a document by its ID + * @param docId The ID of the document to retrieve + * @returns The document if found, undefined otherwise + */ + public getDocument(docId: string): Doc | undefined { + const docInfo = this.documentsById.get(docId); + return docInfo?.layoutDoc; + } + + /** + * Registers chunk IDs associated with a document in the manager + * @param docId The parent document ID + * @param chunkIds Array of chunk IDs associated with this document + */ + public registerChunkIds(docId: string, chunkIds: string[]): void { + // Get the document if it exists + const docInfo = this.documentsById.get(docId); + if (!docInfo) { + console.warn(`Cannot register chunks for unknown document ID: ${docId}`); + return; + } + + // Store chunk IDs on the document for future reference + const doc = docInfo.layoutDoc; + if (!doc.chunk_ids) { + doc.chunk_ids = JSON.stringify(chunkIds); + } else { + // Merge with existing chunk IDs if they exist + const existingIds = JSON.parse(doc.chunk_ids as string); + const updatedIds = [...new Set([...existingIds, ...chunkIds])]; // Remove duplicates + doc.chunk_ids = JSON.stringify(updatedIds); + } + + // Ensure each chunk ID can be linked back to its parent document + chunkIds.forEach(chunkId => { + // Store a mapping from chunk ID to parent document ID + // This allows us to easily find a document by any of its chunk IDs + if (!this.documentsById.has(chunkId)) { + this.documentsById.set(chunkId, { + layoutDoc: doc, + dataDoc: docInfo.dataDoc, + }); + } + }); + } + + /** + * Gets a document ID by a chunk ID + * @param chunkId The chunk ID to look up + * @returns The parent document ID if found + */ + public getDocIdByChunkId(chunkId: string): string | undefined { + const docInfo = this.documentsById.get(chunkId); + if (docInfo) { + return docInfo.layoutDoc[this.DOCUMENT_ID_FIELD] as string; + } + return undefined; } } diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts index afd34f28d..4bb61d8b2 100644 --- a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -15,7 +15,7 @@ import { Networking } from '../../../../Network'; import { AI_Document, CHUNK_TYPE, RAGChunk } from '../types/types'; import OpenAI from 'openai'; import { Embedding } from 'openai/resources'; -import { PineconeEnvironmentVarsNotSupportedError } from '@pinecone-database/pinecone/dist/errors'; +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; dotenv.config(); @@ -29,7 +29,7 @@ export class Vectorstore { private openai: OpenAI; // OpenAI client for generating embeddings. private indexName: string = 'pdf-chatbot'; // Default name for the index. private _id: string; // Unique ID for the Vectorstore instance. - private _doc_ids: () => string[]; // List of document IDs handled by this instance. + private docManager: AgentDocumentManager; // Document manager for handling documents documents: AI_Document[] = []; // Store the documents indexed in the vectorstore. @@ -37,9 +37,9 @@ export class Vectorstore { * Initializes the Pinecone and OpenAI clients, sets up the document ID list, * and initializes the Pinecone index. * @param id The unique identifier for the vectorstore instance. - * @param doc_ids A function that returns a list of document IDs. + * @param docManager An instance of AgentDocumentManager to handle document management. */ - constructor(id: string, doc_ids: () => string[]) { + constructor(id: string, docManager: AgentDocumentManager) { const pineconeApiKey = process.env.PINECONE_API_KEY; if (!pineconeApiKey) { throw new Error('PINECONE_API_KEY is not defined.'); @@ -49,7 +49,7 @@ export class Vectorstore { this.pinecone = new Pinecone({ apiKey: pineconeApiKey }); this.openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY, dangerouslyAllowBrowser: true }); this._id = id; - this._doc_ids = doc_ids; + this.docManager = docManager; this.initializeIndex(); } @@ -109,15 +109,25 @@ export class Vectorstore { const isAudioOrVideo = local_file_path.endsWith('.mp3') || local_file_path.endsWith('.mp4'); let result: AI_Document & { doc_id: string }; + if (isAudioOrVideo) { console.log('Processing media file...'); const response = await Networking.PostToServer('/processMediaFile', { fileName: path.basename(local_file_path) }); - const segmentedTranscript = response.condensed; + + // Type assertion to handle the response properties + const typedResponse = response as { + condensed: Array<{ text: string; indexes: string[]; start: number; end: number }>; + full: Array<unknown>; + summary: string; + }; + + const segmentedTranscript = typedResponse.condensed; console.log(segmentedTranscript); - const summary = response.summary; + const summary = typedResponse.summary; doc.summary = summary; + // Generate embeddings for each chunk - const texts = segmentedTranscript.map((chunk: any) => chunk.text); + const texts = segmentedTranscript.map(chunk => chunk.text); try { const embeddingsResponse = await this.openai.embeddings.create({ @@ -126,10 +136,19 @@ export class Vectorstore { encoding_format: 'float', }); - doc.original_segments = JSON.stringify(response.full); + doc.original_segments = JSON.stringify(typedResponse.full); doc.ai_type = local_file_path.endsWith('.mp3') ? 'audio' : 'video'; const doc_id = uuidv4(); + // Register the document with the AgentDocumentManager + this.docManager.addCustomId(doc, doc_id); + + // Generate chunk IDs upfront so we can register them + const chunkIds = segmentedTranscript.map(() => uuidv4()); + + // Register all chunk IDs with the document manager + this.docManager.registerChunkIds(doc_id, chunkIds); + // Add transcript and embeddings to metadata result = { doc_id, @@ -137,13 +156,13 @@ export class Vectorstore { file_name: local_file_path, num_pages: 0, summary: '', - chunks: segmentedTranscript.map((chunk: any, index: number) => ({ - id: uuidv4(), + chunks: segmentedTranscript.map((chunk, index) => ({ + id: chunkIds[index], // Use pre-generated chunk ID values: (embeddingsResponse.data as Embedding[])[index].embedding, // Assign embedding metadata: { indexes: chunk.indexes, original_document: local_file_path, - doc_id: doc_id, + doc_id: doc_id, // Ensure doc_id is consistent file_path: local_file_path, start_time: chunk.start, end_time: chunk.end, @@ -159,20 +178,24 @@ export class Vectorstore { } doc.segmented_transcript = JSON.stringify(segmentedTranscript); - // Simplify chunks for storage + // Simplify chunks for storage - ensure simplified chunks use EXACTLY the same IDs const simplifiedChunks = result.chunks.map(chunk => ({ - chunkId: chunk.id, + chunkId: chunk.id, // Use the exact same ID as the full chunk start_time: chunk.metadata.start_time, end_time: chunk.metadata.end_time, indexes: chunk.metadata.indexes, chunkType: CHUNK_TYPE.VIDEO, text: chunk.metadata.text, + doc_id: chunk.metadata.doc_id, // Include parent doc_id for completeness })); doc.chunk_simpl = JSON.stringify({ chunks: simplifiedChunks }); } else { - // Existing document processing logic remains unchanged + // Process regular document console.log('Processing regular document...'); - const { jobId } = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); + const createDocumentResponse = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); + + // Type assertion for the response + const { jobId } = createDocumentResponse as { jobId: string }; while (true) { await new Promise(resolve => setTimeout(resolve, 2000)); @@ -188,6 +211,16 @@ export class Vectorstore { progressCallback(progressResponseJson.progress, progressResponseJson.step); } } + + // Register the document with the AgentDocumentManager + this.docManager.addCustomId(doc, result.doc_id); + + // Collect all chunk IDs + const chunkIds = result.chunks.map(chunk => chunk.id); + + // Register chunks with the document manager + this.docManager.registerChunkIds(result.doc_id, chunkIds); + if (!doc.chunk_simpl) { doc.chunk_simpl = JSON.stringify({ chunks: [] }); } @@ -196,12 +229,13 @@ export class Vectorstore { result.chunks.forEach((chunk: RAGChunk) => { const chunkToAdd = { - chunkId: chunk.id, + chunkId: chunk.id, // Ensure we use the exact same ID startPage: chunk.metadata.start_page, endPage: chunk.metadata.end_page, location: chunk.metadata.location, chunkType: chunk.metadata.type as CHUNK_TYPE, text: chunk.metadata.text, + doc_id: chunk.metadata.doc_id, // Include parent doc_id for consistency }; const new_chunk_simpl = JSON.parse(StrCast(doc.chunk_simpl)); new_chunk_simpl.chunks = new_chunk_simpl.chunks.concat(chunkToAdd); @@ -298,39 +332,55 @@ export class Vectorstore { let queryEmbedding = queryEmbeddingResponse.data[0].embedding; - // Extract the embedding from the response. + // Get document IDs from the AgentDocumentManager + const docIds = Array.from(this.docManager.listDocs()); + console.log('Using document IDs for retrieval:', docIds); - console.log(this._doc_ids()); // Query the Pinecone index using the embedding and filter by document IDs. + // We'll query based on document IDs that are registered in the document manager const queryResponse: QueryResponse = await this.index.query({ vector: queryEmbedding, filter: { - doc_id: { $in: this._doc_ids() }, + doc_id: { $in: docIds }, }, topK, includeValues: true, includeMetadata: true, }); - console.log(queryResponse); - - // Map the results into RAGChunks and return them. - return queryResponse.matches.map( - match => - ({ - id: match.id, - values: match.values as number[], - metadata: match.metadata as { - text: string; - type: string; - original_document: string; - file_path: string; - doc_id: string; - location: string; - start_page: number; - end_page: number; - }, - }) as RAGChunk - ); + console.log(`Found ${queryResponse.matches.length} matching chunks`); + + // For each retrieved chunk, ensure its document ID is registered in the document manager + // This maintains compatibility with existing code while ensuring consistency + const processedMatches = queryResponse.matches.map(match => { + const chunk = { + id: match.id, + values: match.values as number[], + metadata: match.metadata as { + text: string; + type: string; + original_document: string; + file_path: string; + doc_id: string; + location: string; + start_page: number; + end_page: number; + }, + } as RAGChunk; + + // Ensure the document manager knows about this chunk + // This is important for maintaining backwards compatibility + if (chunk.id && !this.docManager.getDocIdByChunkId(chunk.id)) { + // If the chunk ID isn't registered but we have a doc_id in metadata + if (chunk.metadata.doc_id && this.docManager.has(chunk.metadata.doc_id)) { + // Register the chunk with its parent document + this.docManager.registerChunkIds(chunk.metadata.doc_id, [chunk.id]); + } + } + + return chunk; + }); + + return processedMatches; } catch (error) { console.error(`Error retrieving chunks: ${error}`); return []; -- cgit v1.2.3-70-g09d2 From a5d7f5c38192b91b7df3bd6ecace5ba7365449a6 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" <Shulman.aj@gmail.com> Date: Sun, 11 May 2025 13:42:00 -0400 Subject: Made it so chunk Ids are seperately managed and made sure the doc id is sonsistent and not created in python spawn --- src/client/views/nodes/WebBox.scss | 241 ++++---- src/client/views/nodes/WebBox.tsx | 605 +++++++-------------- src/client/views/nodes/WebBoxRenderer.js | 103 ++++ .../views/nodes/chatbot/agentsystem/prompts.ts | 4 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 36 +- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 38 +- src/client/views/nodes/chatbot/tools/RAGTool.ts | 5 +- .../nodes/chatbot/utils/AgentDocumentManager.ts | 213 +++----- .../views/nodes/chatbot/vectorstore/Vectorstore.ts | 33 +- src/server/ApiManagers/AssistantManager.ts | 8 +- src/server/chunker/pdf_chunker.py | 13 +- 11 files changed, 548 insertions(+), 751 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/WebBox.scss b/src/client/views/nodes/WebBox.scss index a1991d1d0..77d7716f4 100644 --- a/src/client/views/nodes/WebBox.scss +++ b/src/client/views/nodes/WebBox.scss @@ -1,9 +1,13 @@ @use '../global/globalCssVariables.module.scss' as global; .webBox { + height: 100%; + width: 100%; + top: 0; + left: 0; position: relative; + display: flex; overflow: hidden; - aspect-ratio: 1 / 1; // Explicitly enforce square aspect ratio .webBox-sideResizer { position: absolute; @@ -16,119 +20,6 @@ .webBox-background { width: 100%; height: 100%; - position: absolute; - top: 0; - left: 0; - } - - // Simple container for screenshot - .webBox-screenshot-container { - width: 100%; - } - - .webBox-screenshot { - width: 100%; - height: auto; // Maintain aspect ratio - display: block; - pointer-events: none; - } - - .webBox-loading { - padding: 20px; - text-align: center; - color: #666; - background-color: #f5f5f5; - min-height: 200px; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - } - - .webBox-loading-spinner { - margin-top: 15px; - color: #1976d2; - font-size: 24px; - } - - .webBox-error { - padding: 20px; - color: #d32f2f; - text-align: center; - background-color: #ffebee; - min-height: 200px; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - gap: 15px; - } - - .webBox-placeholder { - padding: 20px; - text-align: center; - color: #757575; - background-color: #fafafa; - min-height: 200px; - display: flex; - align-items: center; - justify-content: center; - } - - // Basic container layout - .webBox-container { - width: 100%; - height: 100%; - position: relative; - } - - // Simple scrollable container - vertical only - .webBox-outerContent { - width: 100%; - position: relative; - overflow-y: auto; - overflow-x: hidden; - background-color: #f5f5f5; - - // Improve scrollbar styling - &::-webkit-scrollbar-thumb { - background-color: #888; - border-radius: 6px; - } - - &::-webkit-scrollbar { - width: 8px; - background-color: #f5f5f5; - } - } - - .webBox-innerContent { - width: 100%; - background-color: #f5f5f5; - } - - .webBox-htmlSpan { - position: absolute; - top: 0; - left: 0; - cursor: text; - padding: 15px; - width: 100%; - height: 100%; - } - - .webBox-annotationLayer { - position: absolute; - transform-origin: left top; - top: 0; - width: 100%; - pointer-events: none; - mix-blend-mode: multiply; - } - - .webBox-annotationBox { - position: absolute; - background-color: rgba(245, 230, 95, 0.616); } .webBox-ui { @@ -177,14 +68,14 @@ } } - .webBox-refreshButton { + .webBox-nextIcon, + .webBox-prevIcon { background: #121721; + color: white; height: 20px; width: 25px; display: flex; - position: absolute; - bottom: 0; - right: 40px; + position: relative; align-items: center; justify-content: center; border-radius: 3px; @@ -192,6 +83,10 @@ padding: 0px; } + .webBox-overlayButton:hover { + background: none; + } + .webBox-overlayCont { position: absolute; width: calc(100% - 40px); @@ -223,7 +118,8 @@ justify-content: center; border-radius: 3px; pointer-events: all; - z-index: 1; + z-index: 1; // so it appears on top of the document's title, if shown + box-shadow: global.$standard-box-shadow; transition: 0.2s; @@ -238,6 +134,89 @@ opacity: 0.1; } + .webBox-annotationLayer { + position: absolute; + transform-origin: left top; + top: 0; + width: 100%; + pointer-events: none; + mix-blend-mode: multiply; // bcz: makes text fuzzy! + } + + .webBox-annotationBox { + position: absolute; + background-color: rgba(245, 230, 95, 0.616); + } + + .webBox-container { + transform-origin: top left; + width: 100%; + height: 100%; + position: absolute; + + .webBox-htmlSpan { + position: absolute; + top: 0; + left: 0; + cursor: text; + padding: 15px; + height: 100%; + } + + .webBox-cont { + pointer-events: none; + } + + .webBox-cont, + .webBox-cont-interactive { + padding: 0vw; + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + transform-origin: top left; + + .webBox-iframe { + width: 100%; + height: 100%; + position: absolute; + top: 0; + left: 0; + body { + ::selection { + color: white; + background: orange; + } + } + } + } + + .webBox-cont-interactive { + span { + user-select: text !important; + } + } + + .webBox-outerContent { + width: 100%; + height: 100%; + position: absolute; + transform-origin: top left; + top: 0; + left: 0; + overflow: auto; + + .webBox-innerContent { + position: relative; + } + } + + div.webBox-outerContent::-webkit-scrollbar-thumb { + cursor: nw-resize; + } + } + .webBox-overlay { width: 100%; height: 100%; @@ -277,13 +256,37 @@ width: 100%; height: 100%; position: absolute; + pointer-events: all; .indicator { position: absolute; + transition: background-color 0.2s ease; + border-radius: 2px; &.active { background-color: rgba(0, 0, 0, 0.1); + box-shadow: 0 0 2px rgba(0, 0, 0, 0.2); } } } + + // Add styles to hide font errors and improve user experience + .font-error-hidden { + font-family: + system-ui, + -apple-system, + BlinkMacSystemFont, + 'Segoe UI', + Roboto, + Arial, + sans-serif !important; + } + + // Change iframe behavior when resource loading errors occur + iframe.webBox-iframe { + &.loading-error { + // Make full content accessible when external resources fail + pointer-events: all !important; + } + } } diff --git a/src/client/views/nodes/WebBox.tsx b/src/client/views/nodes/WebBox.tsx index 045af7ecd..1e158f484 100644 --- a/src/client/views/nodes/WebBox.tsx +++ b/src/client/views/nodes/WebBox.tsx @@ -4,7 +4,6 @@ import { htmlToText } from 'html-to-text'; import { action, computed, IReactionDisposer, makeObservable, observable, ObservableMap, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import * as React from 'react'; -import axios from 'axios'; import * as WebRequest from 'web-request'; import { addStyleSheet, addStyleSheetRule, clearStyleSheetRules, ClientUtils, DivHeight, getWordAtPoint, lightOrDark, returnFalse, returnOne, returnZero, setupMoveUpEvents, smoothScroll } from '../../../ClientUtils'; import { Doc, DocListCast, Field, FieldType, Opt, StrListCast } from '../../../fields/Doc'; @@ -70,20 +69,23 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { private _scrollTimer: NodeJS.Timeout | undefined; private _getAnchor: (savedAnnotations: Opt<ObservableMap<number, HTMLDivElement[]>>, addAsAnnotation: boolean) => Opt<Doc> = () => undefined; - @observable private _webUrl = ''; // url of the page we want to display - @observable private _hackHide = false; + @observable private _webUrl = ''; // url of the src parameter of the embedded iframe but not necessarily the rendered page - eg, when following a link, the rendered page changes but we don't want the src parameter to also change as that would cause an unnecessary re-render. + @observable private _hackHide = false; // apparently changing the value of the 'sandbox' prop doesn't necessarily apply it to the active iframe. so thisforces the ifrmae to be rebuilt when allowScripts is toggled @observable private _searching: boolean = false; @observable private _showSidebar = false; @observable private _webPageHasBeenRendered = false; @observable private _marqueeing: number[] | undefined = undefined; - @observable private _screenshotUrl: string | null = null; // URL to the screenshot image - @observable private _fullHeight: number = 0; // Full height of the webpage screenshot - @observable private _isLoadingScreenshot: boolean = false; // Loading state for the screenshot + get marqueeing() { + return this._marqueeing; + } + set marqueeing(val) { + val && this._marqueeref.current?.onInitiateSelection(val); + !val && this._marqueeref.current?.onTerminateSelection(); + this._marqueeing = val; + } @observable private _iframe: HTMLIFrameElement | null = null; @observable private _savedAnnotations = new ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>(); @observable private _scrollHeight = NumCast(this.layoutDoc.scrollHeight); - @observable private _screenshotError: string | null = null; // Error message if screenshot fails - @observable private _loadingFromCache: boolean = false; @computed get _url() { return this.webField?.toString() || ''; } @@ -143,38 +145,31 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }; updateIcon = async () => { - if (!this._screenshotUrl) { - // If we don't have a screenshot yet, capture one first - await this.captureWebScreenshot(); - } - + if (!this._iframe) return new Promise<void>(res => res()); const scrollTop = NumCast(this.layoutDoc._layout_scrollTop); const nativeWidth = NumCast(this.layoutDoc.nativeWidth); const nativeHeight = (nativeWidth * this._props.PanelHeight()) / this._props.PanelWidth(); - + let htmlString = this._iframe.contentDocument && new XMLSerializer().serializeToString(this._iframe.contentDocument); + if (!htmlString) { + htmlString = await fetch(ClientUtils.CorsProxy(this.webField!.href)).then(response => response.text()); + } this.layoutDoc.thumb = undefined; this.Document.thumbLockout = true; // lock to prevent multiple thumb updates. - - try { - // If we have a screenshot, use it directly for the thumbnail - if (this._screenshotUrl) { - return ClientUtils.convertDataUri(this._screenshotUrl, this.layoutDoc[Id] + '_icon_' + new Date().getTime(), true, this.layoutDoc[Id] + '_icon_').then(returnedfilename => { + return (CreateImage(this._webUrl.endsWith('/') ? this._webUrl.substring(0, this._webUrl.length - 1) : this._webUrl, this._iframe.contentDocument?.styleSheets ?? [], htmlString, nativeWidth, nativeHeight, scrollTop) as Promise<string>) + .then((dataUrl: string) => { + if (dataUrl.includes('<!DOCTYPE')) { + console.log('BAD DATA IN THUMB CREATION'); + return; + } + return ClientUtils.convertDataUri(dataUrl, this.layoutDoc[Id] + '_icon_' + new Date().getTime(), true, this.layoutDoc[Id] + '_icon_').then(returnedfilename => { this.Document.thumbLockout = false; this.layoutDoc.thumb = new ImageField(returnedfilename); this.layoutDoc.thumbScrollTop = scrollTop; this.layoutDoc.thumbNativeWidth = nativeWidth; this.layoutDoc.thumbNativeHeight = nativeHeight; }); - } else { - console.log('No screenshot available for thumbnail generation'); - this.Document.thumbLockout = false; - return Promise.resolve(); - } - } catch (error) { - console.error('Error creating thumbnail:', error); - this.Document.thumbLockout = false; - return Promise.reject(error); - } + }) + .catch((error: object) => console.error('oops, something went wrong!', error)); }; componentDidMount() { @@ -243,64 +238,13 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }, { fireImmediately: true } ); - - // Check if we have a cached screenshot URL in metadata - if (this._url) { - this._webUrl = this._url; - const cachedScreenshotUrl = StrCast(this.dataDoc[this.fieldKey + '_screenshotUrl']); - const cachedHeight = NumCast(this.dataDoc[this.fieldKey + '_screenshotHeight']); - - if (cachedScreenshotUrl && cachedHeight) { - // Use cached screenshot - this._loadingFromCache = true; - this._isLoadingScreenshot = true; - - // Verify the cached screenshot exists by loading the image - const img = new Image(); - img.onload = action(() => { - this._screenshotUrl = cachedScreenshotUrl; - this._fullHeight = cachedHeight; - this._scrollHeight = cachedHeight; - this._webPageHasBeenRendered = true; - this._isLoadingScreenshot = false; - this._loadingFromCache = false; - - // Apply dimensions and initial scroll - if (this.layoutDoc._layout_autoHeight) { - this.layoutDoc._nativeHeight = this._fullHeight; - this._props.setHeight?.(this._fullHeight * (this._props.NativeDimScaling?.() || 1)); - } - - if (this._initialScroll !== undefined) { - this.setScrollPos(this._initialScroll); - } - - console.log(`Loaded cached screenshot: ${this._screenshotUrl}`); - }); - - img.onerror = action(() => { - // If image fails to load, capture a new screenshot - console.log('Cached screenshot not found, capturing new one'); - this._loadingFromCache = false; - this.captureWebScreenshot(); - }); - - img.src = cachedScreenshotUrl; - } else { - // No cached screenshot, capture a new one - this.captureWebScreenshot(); - } - } } componentWillUnmount() { - // Clean up timers - if (this._scrollTimer) { - clearTimeout(this._scrollTimer); - this._scrollTimer = undefined; - } - - // Clean up reaction disposers + this._iframetimeout && clearTimeout(this._iframetimeout); + this._iframetimeout = undefined; Object.values(this._disposers).forEach(disposer => disposer?.()); + // this._iframe?.removeEventListener('wheel', this.iframeWheel, true); + // this._iframe?.contentDocument?.removeEventListener("pointerup", this.iframeUp); } private _selectionText: string = ''; @@ -414,6 +358,59 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { _textAnnotationCreator: (() => ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]>) | undefined; savedAnnotationsCreator: () => ObservableMap<number, (HTMLDivElement & { marqueeing?: boolean })[]> = () => this._textAnnotationCreator?.() || this._savedAnnotations; + @action + iframeMove = (e: PointerEvent) => { + const theclick = this.props + .ScreenToLocalTransform() + .inverse() + .transformPoint(e.clientX, e.clientY - NumCast(this.layoutDoc.layout_scrollTop)); + this._marqueeref.current?.onMove(theclick); + }; + @action + iframeUp = (e: PointerEvent) => { + this._iframe?.contentDocument?.removeEventListener('pointermove', this.iframeMove); + this.marqueeing = undefined; + this._getAnchor = AnchorMenu.Instance?.GetAnchor; // need to save AnchorMenu's getAnchor since a subsequent selection on another doc will overwrite this value + this._textAnnotationCreator = undefined; + this.DocumentView?.()?.cleanupPointerEvents(); // pointerup events aren't generated on containing document view, so we have to invoke it here. + if (this._iframe?.contentWindow && this._iframe.contentDocument && !this._iframe.contentWindow.getSelection()?.isCollapsed) { + const mainContBounds = ClientUtils.GetScreenTransform(this._mainCont.current!); + const scale = (this._props.NativeDimScaling?.() || 1) * mainContBounds.scale; + const sel = this._iframe.contentWindow.getSelection(); + if (sel) { + this._selectionText = sel.toString(); + AnchorMenu.Instance.setSelectedText(sel.toString()); + this._textAnnotationCreator = () => this.createTextAnnotation(sel, !sel.isCollapsed ? sel.getRangeAt(0) : undefined); + AnchorMenu.Instance.jumpTo(e.clientX * scale + mainContBounds.translateX, e.clientY * scale + mainContBounds.translateY - NumCast(this.layoutDoc._layout_scrollTop) * scale); + // Changing which document to add the annotation to (the currently selected WebBox) + GPTPopup.Instance.setSidebarFieldKey(`${this._props.fieldKey}_${this._urlHash ? this._urlHash + '_' : ''}sidebar`); + GPTPopup.Instance.addDoc = this.sidebarAddDocument; + } + } else { + const theclick = this.props + .ScreenToLocalTransform() + .inverse() + .transformPoint(e.clientX, e.clientY - NumCast(this.layoutDoc.layout_scrollTop)); + if (!this._marqueeref.current?.isEmpty) this._marqueeref.current?.onEnd(theclick[0], theclick[1]); + else { + if (!(e.target as HTMLElement)?.tagName?.includes('INPUT')) this.finishMarquee(theclick[0], theclick[1]); + this._getAnchor = AnchorMenu.Instance?.GetAnchor; + this.marqueeing = undefined; + } + + ContextMenu.Instance.closeMenu(); + ContextMenu.Instance.setIgnoreEvents(false); + if (e?.button === 2 || e?.altKey) { + e?.preventDefault(); + e?.stopPropagation(); + setTimeout(() => { + // if menu comes up right away, the down event can still be active causing a menu item to be selected + this.specificContextMenu(); + this.DocumentView?.().onContextMenu(undefined, theclick[0], theclick[1]); + }); + } + } + }; @action webClipDown = (e: React.PointerEvent) => { e.stopPropagation(); @@ -508,6 +505,98 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this._scrollHeight = this._iframe?.contentDocument?.body?.scrollHeight ?? 0; this.addWebStyleSheetRule(this.addWebStyleSheet(this._iframe?.contentDocument), '::selection', { color: 'white', background: 'orange' }, ''); + // Add error handler to suppress font CORS errors + if (this._iframe?.contentWindow) { + try { + // Track if any resource errors occurred + let hasResourceErrors = false; + + // Override the console.error to filter out font CORS errors + const win = this._iframe.contentWindow as Window & { console: Console }; + const originalConsoleError = win.console.error; + win.console.error = (...args: unknown[]) => { + const errorMsg = args.map(arg => String(arg)).join(' '); + if (errorMsg.includes('Access to font') && errorMsg.includes('has been blocked by CORS policy')) { + // Mark that we have font errors + hasResourceErrors = true; + // Ignore font CORS errors + return; + } + // Also catch other resource loading errors + if (errorMsg.includes('ERR_FAILED') || errorMsg.includes('ERR_BLOCKED_BY_CLIENT')) { + hasResourceErrors = true; + } + originalConsoleError.apply(win.console, args); + }; + + // Listen for resource loading errors + this._iframe.contentWindow.addEventListener( + 'error', + (e: Event) => { + const target = e.target as HTMLElement; + if (target instanceof HTMLElement) { + // If it's a resource that failed to load + if (target.tagName === 'LINK' || target.tagName === 'IMG' || target.tagName === 'SCRIPT') { + hasResourceErrors = true; + // Apply error class after a short delay to allow initial content to load + setTimeout(() => { + if (this._iframe && hasResourceErrors) { + this._iframe.classList.add('loading-error'); + } + }, 1000); + } + } + }, + true + ); + + // Add fallback CSS for fonts that fail to load + const style = this._iframe.contentDocument?.createElement('style'); + if (style) { + style.textContent = ` + @font-face { + font-family: 'CORS-fallback-serif'; + src: local('Times New Roman'), local('Georgia'), serif; + } + @font-face { + font-family: 'CORS-fallback-sans'; + src: local('Arial'), local('Helvetica'), sans-serif; + } + /* Fallback for all fonts that fail to load */ + @font-face { + font-display: swap !important; + } + + /* Add a script to find and fix elements with failed fonts */ + @font-face { + font-family: '__failed_font__'; + src: local('Arial'); + unicode-range: U+0000; + } + `; + this._iframe.contentDocument?.head.appendChild(style); + + // Add a script to detect and fix font loading issues + const script = this._iframe.contentDocument?.createElement('script'); + if (script) { + script.textContent = ` + // Fix font loading issues with fallbacks + setTimeout(function() { + document.querySelectorAll('*').forEach(function(el) { + if (window.getComputedStyle(el).fontFamily.includes('__failed_font__')) { + el.classList.add('font-error-hidden'); + } + }); + }, 1000); + `; + this._iframe.contentDocument?.head.appendChild(script); + } + } + } catch (e) { + console.log('Error setting up font error handling:', e); + } + } + let href: Opt<string>; try { href = iframe?.contentWindow?.location.href; @@ -658,23 +747,15 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.dataDoc[this.fieldKey + '_history'] = new List<string>([...history, this._url]); this.dataDoc[this.fieldKey] = new WebField(new URL(future.pop()!)); this._scrollHeight = 0; - - // Reset screenshot state for new URL - this._screenshotUrl = null; - this._fullHeight = 0; - this._isLoadingScreenshot = false; - if (this._webUrl === this._url) { this._webUrl = curUrl; setTimeout( action(() => { this._webUrl = this._url; - this.captureWebScreenshot(); // Capture screenshot for new URL }) ); } else { this._webUrl = this._url; - this.captureWebScreenshot(); // Capture screenshot for new URL } return true; } @@ -694,18 +775,11 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { else this.dataDoc[this.fieldKey + '_future'] = new List<string>([...future, this._url]); this.dataDoc[this.fieldKey] = new WebField(new URL(history.pop()!)); this._scrollHeight = 0; - - // Reset screenshot state for new URL - this._screenshotUrl = null; - this._fullHeight = 0; - this._isLoadingScreenshot = false; - if (this._webUrl === this._url) { this._webUrl = curUrl; setTimeout(action(() => (this._webUrl = this._url))); } else { this._webUrl = this._url; - this.captureWebScreenshot(); // Capture screenshot for new URL } return true; } @@ -724,11 +798,10 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.layoutDoc.thumbNativeWidth = undefined; this.layoutDoc.thumbNativeHeight = undefined; } - + } + if (!preview) { if (!dontUpdateIframe) { this._webUrl = this._url; - // Capture screenshot when URL changes - this.captureWebScreenshot(); } } } catch { @@ -737,85 +810,6 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return true; }; - @action - captureWebScreenshot = async () => { - if (!this._url || this._loadingFromCache) return; - - try { - this._isLoadingScreenshot = true; - this._screenshotError = null; - - console.log(`Capturing screenshot for URL: ${this._url}`); - - try { - const response = await axios.post('/captureWebScreenshot', { - url: this._url, - width: NumCast(this.Document.nativeWidth, 1200), - height: NumCast(this.Document.nativeHeight, 800), - fullPage: true, // Request a full page screenshot - }); - - runInAction(() => { - this._screenshotUrl = response.data.screenshotUrl; - this._fullHeight = response.data.fullHeight; - this._scrollHeight = response.data.fullHeight; - this._webPageHasBeenRendered = true; - this._isLoadingScreenshot = false; - - // Store screenshot URL and height in document metadata - this.dataDoc[this.fieldKey + '_screenshotUrl'] = response.data.screenshotUrl; - this.dataDoc[this.fieldKey + '_screenshotHeight'] = response.data.fullHeight; - - // Update native dimensions to match the screenshot - if (!this.dataDoc[this.fieldKey + '_nativeWidth']) { - this.dataDoc[this.fieldKey + '_nativeWidth'] = 1200; // Default width - } - - if (!this.dataDoc[this.fieldKey + '_nativeHeight']) { - this.dataDoc[this.fieldKey + '_nativeHeight'] = this._fullHeight; - } - - // Set document height if needed - if (this.layoutDoc._layout_autoHeight) { - this.layoutDoc._nativeHeight = this._fullHeight; - this._props.setHeight?.(this._fullHeight * (this._props.NativeDimScaling?.() || 1)); - } - - // Apply initial scroll if needed - if (this._initialScroll !== undefined) { - this.setScrollPos(this._initialScroll); - } - - console.log(`Screenshot captured successfully: ${this._screenshotUrl} with height: ${this._fullHeight}px`); - }); - } catch (error: any) { - // Handle error from the API - console.error('Error capturing screenshot:', error); - let errorMessage = 'Failed to capture webpage screenshot'; - - // Try to extract detailed error message from response - if (error.response && error.response.data && error.response.data.error) { - errorMessage = error.response.data.error; - } else if (error.message) { - errorMessage = error.message; - } - - runInAction(() => { - this._screenshotError = errorMessage; - this._isLoadingScreenshot = false; - }); - } - } catch (error: any) { - // Handle unexpected errors - runInAction(() => { - console.error('Unexpected error in captureWebScreenshot:', error); - this._screenshotError = 'An unexpected error occurred'; - this._isLoadingScreenshot = false; - }); - } - }; - - @action onWebUrlDrop = (e: React.DragEvent) => { const { dataTransfer } = e; const html = dataTransfer.getData('text/html'); @@ -830,28 +824,13 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { setData = (data: FieldType | Promise<RefField | undefined>) => { if (!(typeof data === 'string') && !(data instanceof WebField)) return false; if (Field.toString(data) === this._url) return false; - - // Reset state for new URL this._scrollHeight = 0; - this._screenshotUrl = null; - this._fullHeight = 0; - this._isLoadingScreenshot = false; - - // Clear stored screenshot metadata for the previous URL - this.dataDoc[this.fieldKey + '_screenshotUrl'] = undefined; - this.dataDoc[this.fieldKey + '_screenshotHeight'] = undefined; - const oldUrl = this._url; const history = Cast(this.dataDoc[this.fieldKey + '_history'], listSpec('string'), []); const weburl = new WebField(Field.toString(data)); this.dataDoc[this.fieldKey + '_future'] = new List<string>([]); this.dataDoc[this.fieldKey + '_history'] = new List<string>([...(history || []), oldUrl]); this.dataDoc[this.fieldKey] = weburl; - - // Capture screenshot for the new URL - this._webUrl = weburl.toString(); - this.captureWebScreenshot(); - return true; }; onWebUrlValueKeyDown = (e: React.KeyboardEvent) => { @@ -868,14 +847,26 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { description: (this.layoutDoc[this.fieldKey + '_useCors'] ? "Don't Use" : 'Use') + ' Cors', event: () => { this.layoutDoc[this.fieldKey + '_useCors'] = !this.layoutDoc[this.fieldKey + '_useCors']; - // Re-capture screenshot with the new setting - this.captureWebScreenshot(); }, icon: 'snowflake', }); - - // Remove the "Allow Scripts" option since it's not relevant for screenshots - + funcs.push({ + description: (this.dataDoc[this.fieldKey + '_allowScripts'] ? 'Prevent' : 'Allow') + ' Scripts', + event: () => { + this.dataDoc[this.fieldKey + '_allowScripts'] = !this.dataDoc[this.fieldKey + '_allowScripts']; + if (this._iframe) { + runInAction(() => { + this._hackHide = true; + }); + setTimeout( + action(() => { + this._hackHide = false; + }) + ); + } + }, + icon: 'snowflake', + }); funcs.push({ description: (!this.layoutDoc.layout_reflowHorizontal ? 'Force' : 'Prevent') + ' Reflow', event: () => { @@ -887,21 +878,7 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }, icon: 'snowflake', }); - - // Add a refresh option to re-capture the screenshot - funcs.push({ - description: 'Refresh Screenshot', - event: () => this.captureWebScreenshot(), - icon: 'sync-alt', - }); - - !Doc.noviceMode && - funcs.push({ - description: 'Update Icon', - event: () => this.updateIcon(), - icon: 'portrait', - }); - + !Doc.noviceMode && funcs.push({ description: 'Update Icon', event: () => this.updateIcon(), icon: 'portrait' }); cm.addItem({ description: 'Options...', subitems: funcs, icon: 'asterisk' }); } }; @@ -913,7 +890,7 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { */ @action onMarqueeDown = (e: React.PointerEvent) => { - const sel = window.document.getSelection(); + const sel = this._url ? this._iframe?.contentDocument?.getSelection() : window.document.getSelection(); this._textAnnotationCreator = undefined; if (sel?.empty) sel.empty(); // Chrome @@ -948,7 +925,6 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @computed get urlContent() { if (this.ScreenToLocalBoxXf().Scale > 25) return <div />; - setTimeout( action(() => { if (this._initialScroll === undefined && !this._webPageHasBeenRendered) { @@ -957,10 +933,7 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this._webPageHasBeenRendered = true; }) ); - const field = this.dataDoc[this._props.fieldKey]; - - // Handle HTML field (text content) if (field instanceof HtmlField) { return ( <span @@ -977,8 +950,6 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { /> ); } - - // Handle WebField (screenshot of webpage) if (field instanceof WebField) { const url = this.layoutDoc[this.fieldKey + '_useCors'] ? '/corsproxy/' + this._webUrl : this._webUrl; const scripts = this.dataDoc[this.fieldKey + '_allowScripts'] || this._webUrl.includes('wikipedia.org') || this._webUrl.includes('google.com') || this._webUrl.startsWith('https://bing'); @@ -1198,7 +1169,7 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { TraceMobx(); // const previewScale = this._previewNativeWidth ? 1 - this.sidebarWidth() / this._previewNativeWidth : 1; const pointerEvents = this.layoutDoc._lockedPosition ? 'none' : (this._props.pointerEvents?.() as Property.PointerEvents | undefined); - const scale = this._props.NativeDimScaling?.() || 1; + // const scale = previewScale * (this._props.NativeDimScaling?.() || 1); return ( <div className="webBox-outerContent" @@ -1207,16 +1178,11 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { height: '100%', //`${100 / scale}%`, pointerEvents, }} + // when active, block wheel events from propagating since they're handled by the iframe onWheel={this.onZoomWheel} onScroll={() => this.setDashScrollTop(this._outerRef.current?.scrollTop || 0)} onPointerDown={this.onMarqueeDown}> - <div - className="webBox-innerContent" - style={{ - width: '100%', - pointerEvents, - backgroundColor: '#f5f5f5', - }}> + <div className="webBox-innerContent" style={{ height: (this._webPageHasBeenRendered && this._scrollHeight > this._props.PanelHeight() && this._scrollHeight) || '100%', pointerEvents }}> {this.content} <div style={{ display: SnappingManager.CanEmbed ? 'none' : undefined, mixBlendMode: 'multiply' }}>{this.renderTransparentAnnotations}</div> {this.renderOpaqueAnnotations} @@ -1258,13 +1224,6 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { <FontAwesomeIcon icon={this._searching ? 'times' : 'search'} size="lg" /> </div> </button> - - {/* Refresh button */} - <button type="button" className="webBox-overlayButton webBox-refreshButton" title="Refresh webpage" onClick={() => this.captureWebScreenshot()}> - <div className="webBox-overlayButton-iconCont" onPointerDown={e => e.stopPropagation()}> - <FontAwesomeIcon icon="sync" size="lg" /> - </div> - </button> </div> ); } @@ -1293,25 +1252,16 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { annotationPointerEvents = () => (this._props.isContentActive() && (SnappingManager.IsDragging || Doc.ActiveTool !== InkTool.None) ? 'all' : 'none'); render() { TraceMobx(); - const containerWidth = NumCast(this.layoutDoc._width) || this._props.PanelWidth(); + const previewScale = this._previewNativeWidth ? 1 - this.sidebarWidth() / this._previewNativeWidth : 1; const pointerEvents = this.layoutDoc._lockedPosition ? 'none' : (this._props.pointerEvents?.() as Property.PointerEvents); - const scale = this._props.NativeDimScaling?.() || 1; - - // Force the component to be square - this.layoutDoc._height = containerWidth; - this.layoutDoc._width = containerWidth; - this.layoutDoc._forceActive = true; - + const scale = previewScale * (this._props.NativeDimScaling?.() || 1); return ( <div className="webBox" ref={this._mainCont} style={{ - pointerEvents: this.pointerEvents(), + pointerEvents: this.pointerEvents(), // position: SnappingManager.IsDragging ? 'absolute' : undefined, - width: `${containerWidth}px`, - height: `${containerWidth}px`, - aspectRatio: '1 / 1', // Explicitly enforce square aspect ratio }}> <div className="webBox-background" style={{ backgroundColor: this._props.styleProvider?.(this.layoutDoc, this._props, StyleProp.BackgroundColor) as string }} /> <div @@ -1376,15 +1326,6 @@ export class WebBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { </div> ); } - - get marqueeing() { - return this._marqueeing; - } - set marqueeing(val) { - val && this._marqueeref.current?.onInitiateSelection(val); - !val && this._marqueeref.current?.onTerminateSelection(); - this._marqueeing = val; - } } // eslint-disable-next-line prefer-arrow-callback ScriptingGlobals.add(function urlHash(url: string) { @@ -1395,149 +1336,3 @@ Docs.Prototypes.TemplateMap.set(DocumentType.WEB, { layout: { view: WebBox, dataField: 'data' }, options: { acl: '', _height: 300, _layout_fitWidth: true, _layout_nativeDimEditable: true, _layout_reflowVertical: true, waitForDoubleClickToClick: 'always', systemIcon: 'BsGlobe' }, }); - -// Add CSS styles for screenshot mode -const webBoxStyles = ` -.webBox-screenshot-container { - width: 100%; - position: relative; - overflow: visible; - display: flex; - align-items: flex-start; - justify-content: center; - background-color: #f5f5f5; -} - -.webBox-screenshot { - width: 100%; - pointer-events: none; - display: block; - user-select: none; - object-fit: contain; - transition: opacity 0.3s ease; -} - -.webBox-loading { - padding: 20px; - text-align: center; - color: #666; - background-color: #f5f5f5; - border-radius: 4px; - min-height: 200px; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; -} - -.webBox-loading-message { - font-size: 16px; - margin-bottom: 15px; - color: #555; -} - -.webBox-loading-spinner { - margin-top: 10px; - color: #1976d2; -} - -.webBox-error { - padding: 20px; - color: #d32f2f; - text-align: center; - background-color: #ffebee; - border-radius: 4px; - min-height: 200px; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - gap: 15px; -} - -.webBox-error-icon { - color: #d32f2f; - margin-bottom: 10px; -} - -.webBox-error-message { - color: #d32f2f; - font-size: 14px; - max-width: 80%; - line-height: 1.5; -} - -.webBox-error-actions { - margin-top: 10px; -} - -.webBox-retry-button { - background-color: #f44336; - color: white; - border: none; - padding: 8px 16px; - border-radius: 4px; - cursor: pointer; - font-size: 14px; - transition: background-color 0.3s; -} - -.webBox-retry-button:hover { - background-color: #d32f2f; -} - -.webBox-placeholder { - padding: 20px; - text-align: center; - color: #757575; - background-color: #fafafa; - border-radius: 4px; - min-height: 200px; - display: flex; - align-items: center; - justify-content: center; -} - -.webBox-refreshButton { - margin-right: 5px; -} - -.webBox-innerContent { - position: relative; - width: 100%; - background-color: #f5f5f5; - overflow: visible; -} - -.webBox-outerContent { - overflow: auto; - width: 100%; - background-color: #f5f5f5; - position: relative; -} - -.webBox-container { - position: relative; - display: flex; - flex-direction: column; - height: 100%; - background-color: white; - border-radius: 4px; - overflow: hidden; -} - -.webBox { - position: relative; - height: 100%; - width: 100%; - overflow: hidden; - background-color: white; - border-radius: 4px; - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24); -} -`; - -// Add the styles to the document -const styleEl = document.createElement('style'); -styleEl.textContent = webBoxStyles; -document.head.appendChild(styleEl); diff --git a/src/client/views/nodes/WebBoxRenderer.js b/src/client/views/nodes/WebBoxRenderer.js index ef465c453..31e0ef5e4 100644 --- a/src/client/views/nodes/WebBoxRenderer.js +++ b/src/client/views/nodes/WebBoxRenderer.js @@ -145,6 +145,29 @@ const ForeignHtmlRenderer = function (styleSheets) { return urlsFound; }; + /** + * Extracts font-face URLs from CSS rules + * @param {String} cssRuleStr + * @returns {String[]} + */ + const getFontFaceUrlsFromCss = function (cssRuleStr) { + const fontFaceUrls = []; + // Find @font-face blocks + const fontFaceBlocks = cssRuleStr.match(/@font-face\s*{[^}]*}/g) || []; + + fontFaceBlocks.forEach(block => { + // Extract URLs from src properties + const urls = block.match(/src\s*:\s*[^;]*/g) || []; + urls.forEach(srcDeclaration => { + // Find all url() references in the src declaration + const fontUrls = getUrlsFromCssString(srcDeclaration); + fontFaceUrls.push(...fontUrls); + }); + }); + + return fontFaceUrls; + }; + /** * * @param {String} html @@ -158,6 +181,61 @@ const ForeignHtmlRenderer = function (styleSheets) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string }; + /** + * Create a fallback font-face rule for handling CORS errors + * @returns {String} + */ + const createFallbackFontFaceRules = function () { + return ` + @font-face { + font-family: 'CORS-fallback-serif'; + src: local('Times New Roman'), local('Georgia'), serif; + } + @font-face { + font-family: 'CORS-fallback-sans'; + src: local('Arial'), local('Helvetica'), sans-serif; + } + /* Add fallback font handling */ + [data-font-error] { + font-family: 'CORS-fallback-sans', sans-serif !important; + } + [data-font-error="serif"] { + font-family: 'CORS-fallback-serif', serif !important; + } + `; + }; + + /** + * Clean up and optimize CSS for better rendering + * @param {String} cssStyles + * @returns {String} + */ + const optimizeCssForRendering = function (cssStyles) { + // Add fallback font-face rules + const enhanced = cssStyles + createFallbackFontFaceRules(); + + // Replace problematic font-face declarations with proxied versions + let optimized = enhanced.replace(/(url\(['"]?)(https?:\/\/[^)'"]+)(['"]?\))/gi, (match, prefix, url, suffix) => { + // If it's a font file, proxy it + if (url.match(/\.(woff2?|ttf|eot|otf)(\?.*)?$/i)) { + return `${prefix}${CorsProxy(url)}${suffix}`; + } + return match; + }); + + // Add error handling for fonts + optimized += ` + /* Suppress font CORS errors in console */ + @supports (font-display: swap) { + @font-face { + font-display: swap !important; + } + } + `; + + return optimized; + }; + /** * * @param {String} contentHtml @@ -175,6 +253,7 @@ const ForeignHtmlRenderer = function (styleSheets) { // copy styles let cssStyles = ''; const urlsFoundInCss = []; + const fontUrlsInCss = []; for (let i = 0; i < styleSheets.length; i += 1) { try { @@ -182,6 +261,7 @@ const ForeignHtmlRenderer = function (styleSheets) { for (let j = 0; j < rules.length; j += 1) { const cssRuleStr = rules[j].cssText; urlsFoundInCss.push(...getUrlsFromCssString(cssRuleStr)); + fontUrlsInCss.push(...getFontFaceUrlsFromCss(cssRuleStr)); cssStyles += cssRuleStr; } } catch (e) { @@ -189,6 +269,9 @@ const ForeignHtmlRenderer = function (styleSheets) { } } + // Optimize and enhance CSS + cssStyles = optimizeCssForRendering(cssStyles); + // const fetchedResourcesFromStylesheets = await getMultipleResourcesAsBase64(webUrl, urlsFoundInCss); // for (let i = 0; i < fetchedResourcesFromStylesheets.length; i++) { // const r = fetchedResourcesFromStylesheets[i]; @@ -203,6 +286,26 @@ const ForeignHtmlRenderer = function (styleSheets) { .replace(/<div class="mediaset"><\/div>/g, '') // when scripting isn't available (ie, rendering web pages here), <noscript> tags should become <div>'s. But for Brown CS, there's a layout problem if you leave the empty <mediaset> tag .replace(/<link[^>]*>/g, '') // don't need to keep any linked style sheets because we've already processed all style sheets above .replace(/srcset="([^ "]*)[^"]*"/g, 'src="$1"'); // instead of converting each item in the srcset to a data url, just convert the first one and use that + + // Add script to handle font loading errors + contentHtml += ` + <script> + // Handle font loading errors with fallbacks + document.addEventListener('DOMContentLoaded', function() { + // Mark elements with font issues + document.querySelectorAll('*').forEach(function(el) { + const style = window.getComputedStyle(el); + const fontFamily = style.getPropertyValue('font-family'); + if (fontFamily && !fontFamily.includes('serif') && !fontFamily.includes('sans')) { + el.setAttribute('data-font-error', 'sans'); + } else if (fontFamily && fontFamily.includes('serif')) { + el.setAttribute('data-font-error', 'serif'); + } + }); + }); + </script> + `; + const urlsFoundInHtml = getImageUrlsFromFromHtml(contentHtml).filter(url => !url.startsWith('data:')); return getMultipleResourcesAsBase64(webUrl, urlsFoundInHtml).then(fetchedResources => { for (let i = 0; i < fetchedResources.length; i += 1) { diff --git a/src/client/views/nodes/chatbot/agentsystem/prompts.ts b/src/client/views/nodes/chatbot/agentsystem/prompts.ts index e551ef830..fcb4ab450 100644 --- a/src/client/views/nodes/chatbot/agentsystem/prompts.ts +++ b/src/client/views/nodes/chatbot/agentsystem/prompts.ts @@ -103,9 +103,9 @@ export function getReactPrompt(tools: BaseTool<ReadonlyArray<Parameter>>[], summ <note>If no external tool is required, use 'no_tool', but if there might be relevant external information, use the appropriate tool.</note> </tools> - <summaries> + <available_documents> ${summaries()} - </summaries> + </available_documents> <chat_history> ${chatHistory} diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index d919b5f7f..34a1ade2e 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -650,16 +650,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { citation: JSON.stringify(citation, null, 2), }); - // Try to find the document - let doc: Doc | undefined; - // First try to find the document using the document manager's chunk ID lookup - const parentDocId = this.docManager.getDocIdByChunkId(chunkId); - if (parentDocId) { - doc = this.docManager.getDocument(parentDocId); - console.log(`Found document by chunk ID lookup: ${parentDocId}`); - } - + const doc: Doc | undefined = this.docManager.getDocByChunkId(chunkId); if (!doc) { console.warn(`Document not found for citation with chunk_id: ${chunkId}`); return; @@ -989,32 +981,13 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { componentWillUnmount() { this.removeScrollListener(); } - - /** - * Getter that retrieves all linked documents for the current document. - */ - @computed get linkedDocs(): Doc[] { - const docIds = this.docManager.listDocs(); - const docs: Doc[] = []; - - // Get documents from the document manager using the getDocument method - docIds.forEach(id => { - const doc = this.docManager.getDocument(id); - if (doc) { - docs.push(doc); - } - }); - - return docs; - } - /** * Getter that retrieves document IDs of linked documents that have PDF_chunker–parsed content. */ @computed get docIds(): string[] { // Use the document manager to get all document IDs - return Array.from(this.docManager.listDocs()); + return Array.from(this.docManager.listDocs); } /** @@ -1023,7 +996,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { @computed get summaries(): string { // Use the document manager to get all summaries - return this.docManager.getAllDocumentSummaries(); + console.log(this.docManager.listDocs); + return JSON.stringify(this.docManager.listDocs); } /** @@ -1064,7 +1038,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { }; retrieveDocIds = (): string[] => { - return Array.from(this.docManager.listDocs()); + return Array.from(this.docManager.docIds); }; /** diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index e6c2421e5..5297292bf 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -18,13 +18,13 @@ const parameterDefinitions: ReadonlyArray<Parameter> = [ name: 'action', type: 'string', required: true, - description: 'The action to perform: "get" to retrieve metadata, "edit" to modify metadata, "list" to enumerate documents, "getFieldOptions" to retrieve all available field options, or "create" to create a new document', + description: 'The action to perform: "get" to retrieve metadata, "edit" to modify metadata, "getFieldOptions" to retrieve all available field options, or "create" to create a new document', }, { name: 'documentId', type: 'string', required: false, - description: 'The ID of the document to get or edit metadata for. Required for "edit", optional for "get", ignored for "list", "getFieldOptions", and "create"', + description: 'The ID of the document to get or edit metadata for. Required for "edit", optional for "get", ignored for "getFieldOptions", and "create"', }, { name: 'fieldEdits', @@ -68,7 +68,6 @@ This tool provides the following capabilities: - Get metadata from a specific document - Edit metadata fields on documents (in either layout or data documents) - Edit multiple fields at once (useful for updating dependent fields together) -- List all available documents in the current view - Retrieve all available field options with metadata (IMPORTANT: always call this before editing) - Understand which fields are stored where (layout vs data document) - Get detailed information about all available document fields @@ -137,8 +136,8 @@ SPECIAL FIELD HANDLING: - Width/Height: Set layout_autoHeight/layout_autoWidth to false before editing RECOMMENDED WORKFLOW: -1. First call action="list" to identify available documents -2. Then call action="getFieldOptions" to understand available fields +0. Understand the currently available documents that were provided as <available_documents> in the prompt +1. Call action="getFieldOptions" to understand available fields 3. Get document metadata with action="get" to see current values 4. Edit fields with action="edit" using proper dependencies OR @@ -159,10 +158,6 @@ HANDLING DEPENDENT FIELDS: - width → layout_autoWidth (set to false to allow manual width) - Other auto-sizing related properties -To LIST available documents: -- Use action="list" to get a simple list of all documents in the current view -- This is useful when you need to identify documents before getting details or editing them - Editing fields follows these rules: 1. First checks if the field exists on the layout document using Doc.Get 2. If it exists on the layout document, it's updated there @@ -172,7 +167,6 @@ Editing fields follows these rules: Examples: - To get field options: { action: "getFieldOptions" } -- To list all documents: { action: "list" } - To get all document metadata: { action: "get" } - To get metadata for a specific document: { action: "get", documentId: "doc123" } - To edit a single field: { action: "edit", documentId: "doc123", fieldEdits: [{ fieldName: "backgroundColor", fieldValue: "#ff0000" }] } @@ -186,7 +180,8 @@ Examples: { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 200 } ]} -- IMPORTANT: MULTI STEP WORKFLOWS ARE NOT ONLY ALLOWED BUT ENCOURAGED. TAKE THINGS 1 STEP AT A TIME.`; +- IMPORTANT: MULTI STEP WORKFLOWS ARE NOT ONLY ALLOWED BUT ENCOURAGED. TAKE THINGS 1 STEP AT A TIME. +- IMPORTANT: WHEN CITING A DOCUMENT, MAKE THE CHUNK ID THE DOCUMENT ID. WHENEVER YOU CITE A DOCUMENT, ALWAYS MAKE THE CITATION TYPE "text", THE "direct_text" FIELD BLANK, AND THE "chunk_id" FIELD THE DOCUMENT ID.`; const documentMetadataToolInfo: ToolInfo<DocumentMetadataToolParamsType> = { name: 'documentMetadata', description: toolDescription, @@ -232,11 +227,11 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp // Ensure the action is valid and convert to string const action = String(args.action); - if (!['get', 'edit', 'list', 'getFieldOptions', 'create'].includes(action)) { + if (!['get', 'edit', 'getFieldOptions', 'create'].includes(action)) { return [ { type: 'text', - text: 'Error: Invalid action. Valid actions are "get", "edit", "list", "getFieldOptions", or "create".', + text: 'Error: Invalid action. Valid actions are "get", "edit", "getFieldOptions", or "create".', }, ]; } @@ -386,10 +381,6 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp } } - case 'list': { - this._docManager.listDocs(); - } - case 'getFieldOptions': { // Get all available field options with metadata const fieldOptions = this._docManager.getAllFieldMetadata(); @@ -457,7 +448,7 @@ ${JSON.stringify(createdMetadata, null, 2)}`, return [ { type: 'text', - text: 'Error: Unknown action. Valid actions are "get", "edit", "list", "getFieldOptions", or "create".', + text: 'Error: Unknown action. Valid actions are "get", "edit", "getFieldOptions", or "create".', }, ]; } @@ -537,11 +528,6 @@ ${JSON.stringify(createdMetadata, null, 2)}`, return true; } - // list action doesn't require any additional parameters - if (params.action === 'list') { - return true; - } - return true; } @@ -552,7 +538,7 @@ ${JSON.stringify(createdMetadata, null, 2)}`, */ private getParameterRequirementsByAction(action?: string): string { if (!action) { - return 'Please specify an action: "get", "edit", "list", "getFieldOptions", or "create".'; + return 'Please specify an action: "get", "edit", "getFieldOptions", or "create".'; } switch (action.toLowerCase()) { @@ -560,14 +546,12 @@ ${JSON.stringify(createdMetadata, null, 2)}`, return 'The "get" action accepts an optional documentId parameter.'; case 'edit': return 'The "edit" action requires documentId and fieldEdits parameters. fieldEdits must be a JSON array of field edits.'; - case 'list': - return 'The "list" action does not require any additional parameters.'; case 'getFieldOptions': return 'The "getFieldOptions" action does not require any additional parameters. It returns metadata about all available document fields.'; case 'create': return 'The "create" action requires title, data, and doc_type parameters.'; default: - return `Unknown action "${action}". Valid actions are "get", "edit", "list", "getFieldOptions", or "create".`; + return `Unknown action "${action}". Valid actions are "get", "edit", "getFieldOptions", or "create".`; } } } diff --git a/src/client/views/nodes/chatbot/tools/RAGTool.ts b/src/client/views/nodes/chatbot/tools/RAGTool.ts index ef374ed22..90b803d21 100644 --- a/src/client/views/nodes/chatbot/tools/RAGTool.ts +++ b/src/client/views/nodes/chatbot/tools/RAGTool.ts @@ -3,6 +3,7 @@ import { Observation, RAGChunk } from '../types/types'; import { ParametersType, ToolInfo } from '../types/tool_types'; import { Vectorstore } from '../vectorstore/Vectorstore'; import { BaseTool } from './BaseTool'; +import { DocumentMetadataTool } from './DocumentMetadataTool'; const ragToolParams = [ { @@ -17,7 +18,7 @@ type RAGToolParamsType = typeof ragToolParams; const ragToolInfo: ToolInfo<RAGToolParamsType> = { name: 'rag', - description: 'Performs a RAG (Retrieval-Augmented Generation) search on user documents and returns a set of document chunks (text or images) to provide a grounded response based on user documents.', + description: `Performs a RAG (Retrieval-Augmented Generation) search on user documents (only PDF, audio, and video are supported—for information about other document types, use the ${DocumentMetadataTool.name} tool) and returns a set of document chunks (text or images) to provide a grounded response based on user documents.`, citationRules: `When using the RAG tool, the structure must adhere to the format described in the ReAct prompt. Below are additional guidelines specifically for RAG-based responses: 1. **Grounded Text Guidelines**: @@ -75,7 +76,7 @@ export class RAGTool extends BaseTool<RAGToolParamsType> { async getFormattedChunks(relevantChunks: RAGChunk[]): Promise<Observation[]> { try { - const { formattedChunks } = await Networking.PostToServer('/formatChunks', { relevantChunks }) as { formattedChunks: Observation[]} + const { formattedChunks } = (await Networking.PostToServer('/formatChunks', { relevantChunks })) as { formattedChunks: Observation[] }; if (!formattedChunks) { throw new Error('Failed to format chunks'); diff --git a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts index 14cffcb70..c8a6bb16b 100644 --- a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts +++ b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts @@ -1,4 +1,4 @@ -import { action, makeObservable, observable, ObservableMap, reaction, runInAction } from 'mobx'; +import { action, computed, makeObservable, observable, ObservableMap, reaction, runInAction } from 'mobx'; import { observer } from 'mobx-react'; import { v4 as uuidv4 } from 'uuid'; import { Doc, StrListCast } from '../../../../../fields/Doc'; @@ -31,7 +31,7 @@ export class AgentDocumentManager { private chatBox: ChatBox; private chatBoxDocument: Doc | null = null; private fieldMetadata: Record<string, any> = {}; - private readonly DOCUMENT_ID_FIELD = '_dash_document_id'; + @observable private documentIdsFromChunkIds: ObservableMap<string, string>; /** * Creates a new DocumentManager @@ -40,8 +40,17 @@ export class AgentDocumentManager { constructor(chatBox: ChatBox) { makeObservable(this); const agentDoc = DocCast(chatBox.Document.agentDocument) ?? new Doc(); + const chunkIds = DocCast(agentDoc.chunkIds) ?? new Doc(); + agentDoc.title = chatBox.Document.title + '_agentDocument'; + chunkIds.title = '_chunkIds'; chatBox.Document.agentDocument = agentDoc; + DocCast(chatBox.Document.agentDocument)!.chunkIds = chunkIds; + this.documentIdsFromChunkIds = StrListCast(chunkIds.mapping).reduce((mapping, content) => { + const [chunkId, docId] = content.split(':'); + mapping.set(chunkId, docId); + return mapping; + }, new ObservableMap<string, string>()); this.documentsById = StrListCast(agentDoc.mapping).reduce((mapping, content) => { const [id, layoutId, docId] = content.split(':'); const layoutDoc = DocServer.GetCachedRefField(layoutId); @@ -66,6 +75,19 @@ export class AgentDocumentManager { } //{ fireImmediately: true } ); + reaction( + () => this.documentIdsFromChunkIds.values(), + () => { + if (this.chatBoxDocument && DocCast(this.chatBoxDocument.agentDocument)) { + // Store the mapping with chunkId:docId format for consistency + const chunkIdsDoc = DocCast(DocCast(this.chatBoxDocument.agentDocument)!.chunkIds); + if (chunkIdsDoc) { + chunkIdsDoc.mapping = new List<string>(Array.from(this.documentIdsFromChunkIds.entries()).map(([chunkId, docId]) => `${chunkId}:${docId}`)); + } + } + } + //{ fireImmediately: true } + ); this.processDocument(this.chatBoxDocument); this.initializeFieldMetadata(); } @@ -120,7 +142,7 @@ export class AgentDocumentManager { try { // Use the LinkManager approach which is proven to work in ChatBox if (this.chatBoxDocument) { - console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument.id); + console.log('Finding documents linked to ChatBox document with ID:', this.chatBoxDocument[Id]); // Get directly linked documents via LinkManager const linkedDocs = LinkManager.Instance.getAllRelatedLinks(this.chatBoxDocument) @@ -134,57 +156,10 @@ export class AgentDocumentManager { linkedDocs.forEach((doc: Doc | undefined) => { if (doc) { this.processDocument(doc); - console.log('Processed linked document:', doc.id, doc.title, doc.type); + console.log('Processed linked document:', doc[Id], doc.title, doc.type); } }); - - // Include the ChatBox document itself - this.processDocument(this.chatBoxDocument); - - // If we have access to the Document's parent, try to find sibling documents - if (this.chatBoxDocument.parent) { - const parent = this.chatBoxDocument.parent; - console.log('Found parent document, checking for siblings'); - - // Check if parent is a Doc type and has a childDocs function - if (parent && typeof parent === 'object' && 'childDocs' in parent && typeof parent.childDocs === 'function') { - try { - const siblingDocs = parent.childDocs(); - if (Array.isArray(siblingDocs)) { - console.log(`Found ${siblingDocs.length} sibling documents via parent.childDocs()`); - siblingDocs.forEach((doc: Doc) => { - if (doc) { - this.processDocument(doc); - } - }); - } - } catch (e) { - console.warn('Error accessing parent.childDocs:', e); - } - } - } - } else if (this.chatBox && this.chatBox.linkedDocs) { - // If we have direct access to the linkedDocs computed property from ChatBox - console.log('Using ChatBox.linkedDocs directly'); - const linkedDocs = this.chatBox.linkedDocs; - if (Array.isArray(linkedDocs)) { - console.log(`Found ${linkedDocs.length} documents via ChatBox.linkedDocs`); - linkedDocs.forEach((doc: Doc) => { - if (doc) { - this.processDocument(doc); - } - }); - } - - // Process the ChatBox document if available - if (this.chatBox.Document) { - this.processDocument(this.chatBox.Document); - } - } else { - console.warn('No ChatBox document reference available for finding linked documents'); } - - console.log(`DocumentMetadataTool found ${this.documentsById.size} total documents`); } catch (error) { console.error('Error finding documents in Freeform view:', error); } @@ -201,6 +176,7 @@ export class AgentDocumentManager { // Only add if we haven't already processed this document if (!this.documentsById.has(docId)) { this.documentsById.set(docId, { layoutDoc: doc, dataDoc: doc[DocData] }); + console.log('Added document to documentsById:', doc[Id], docId, doc[Id], doc[DocData][Id]); } return docId; } @@ -213,37 +189,12 @@ export class AgentDocumentManager { private ensureDocumentId(doc: Doc): string { let docId: string | undefined; - // First try to get the ID from our custom field - if (doc[this.DOCUMENT_ID_FIELD]) { - docId = String(doc[this.DOCUMENT_ID_FIELD]); - return docId; - } - - // Try different ways to get a document ID - // 1. Try the direct id property if it exists - if (doc.id && typeof doc.id === 'string') { - docId = doc.id; - } - // 2. Try doc._id if it exists - else if (doc._id && typeof doc._id === 'string') { - docId = doc._id; - } - // 3. Try doc.data?.id if it exists - else if (doc.data && typeof doc.data === 'object' && 'id' in doc.data && typeof doc.data.id === 'string') { - docId = doc.data.id; - } - // 4. If none of the above work, generate a UUID - else { - docId = uuidv4(); - console.log(`Generated new UUID for document with title: ${doc.title || 'Untitled'}`); - } - - // Store the ID in the document's metadata so it persists - try { - doc[this.DOCUMENT_ID_FIELD] = docId; - } catch (e) { - console.warn(`Could not assign ID to document property`, e); + if (doc[Id]) { + console.log('Found document ID (normal):', doc[Id]); + docId = doc[Id]; + } else { + throw new Error('No document ID found'); } return docId; @@ -256,13 +207,13 @@ export class AgentDocumentManager { */ public extractDocumentMetadata(id: string) { if (!id) return null; - const doc = this.documentsById.get(id); - if (!doc) return null; - const layoutDoc = doc.layoutDoc; - const dataDoc = doc.dataDoc; + const agentDoc = this.documentsById.get(id); + if (!agentDoc) return null; + const layoutDoc = agentDoc.layoutDoc; + const dataDoc = agentDoc.dataDoc; const metadata: Record<string, any> = { - id: layoutDoc.dash_document_id || layoutDoc.id || '', + id: layoutDoc[Id] || dataDoc[Id] || '', title: layoutDoc.title || '', type: layoutDoc.type || '', fields: { @@ -355,7 +306,7 @@ export class AgentDocumentManager { if (value instanceof Doc) { return { type: 'Doc', - id: value.id || this.ensureDocumentId(value), + id: value[Id] || this.ensureDocumentId(value), title: value.title || '', docType: value.type || '', }; @@ -1011,33 +962,17 @@ export class AgentDocumentManager { * Returns a list of all document IDs in the manager. * @returns An array of document IDs (strings). */ - public listDocs(): string[] { - return Array.from(this.documentsById.keys()); + @computed + public get listDocs(): string[] { + console.log( + Array.from(this.documentsById.entries()).map(([id, agentDoc]) => JSON.stringify({ id, title: agentDoc.layoutDoc.title, type: agentDoc.layoutDoc.type, summary: agentDoc.layoutDoc.summary || 'No summary available for this document.' })) + ); + return Array.from(this.documentsById.entries()).map(([id, agentDoc]) => JSON.stringify({ id, title: agentDoc.layoutDoc.title, type: agentDoc.layoutDoc.type, summary: agentDoc.layoutDoc.summary || 'No summary available for this document.' })); } - /** - * Adds a document with a custom ID to the manager - * @param doc The document to add - * @param customId The custom ID to assign to the document - * @returns The customId that was assigned - */ - @action - public addCustomId(doc: Doc, customId: string): string { - if (!doc) { - console.error('Cannot add null document with custom ID'); - return ''; - } - - // Set the custom ID in the document's metadata - doc[this.DOCUMENT_ID_FIELD] = customId; - - // Store the document in our map - this.documentsById.set(customId, { - layoutDoc: doc, - dataDoc: doc, - }); - - return customId; + @computed + public get docIds(): string[] { + return Array.from(this.documentsById.keys()); } /** @@ -1078,11 +1013,8 @@ export class AgentDocumentManager { // Ensure each chunk ID can be linked back to its parent document // Store a mapping from chunk ID to parent document ID // This allows us to easily find a document by any of its chunk IDs - if (!this.documentsById.has(chunkId)) { - this.documentsById.set(chunkId, { - layoutDoc: doc, - dataDoc: docInfo.dataDoc, - }); + if (!this.documentIdsFromChunkIds.has(chunkId) && doc) { + this.documentIdsFromChunkIds.set(chunkId, doc[Id]); } } } @@ -1092,11 +1024,25 @@ export class AgentDocumentManager { * @param chunkId The chunk ID to look up * @returns The parent document ID if found */ - public getDocIdByChunkId(chunkId: string): string | undefined { - const docInfo = this.documentsById.get(chunkId); + public getDocByChunkId(chunkId: string): Doc | undefined { + // First, look up the document ID using the chunk ID mapping + const docId = this.documentIdsFromChunkIds.get(chunkId); + console.log('this.documentIdsFromChunkIds', this.documentIdsFromChunkIds); + console.log('docId', docId); + if (!docId) { + if (this.documentsById.has(chunkId)) { + return this.documentsById.get(chunkId)?.layoutDoc; + } else { + console.error('No document found for chunkId and docId', chunkId); + return undefined; + } + } + // Then get the document using the document ID + const docInfo = this.documentsById.get(docId); if (docInfo) { - return docInfo.layoutDoc[this.DOCUMENT_ID_FIELD] as string; + return docInfo.layoutDoc; } + console.error('No document found for docId', docId); return undefined; } @@ -1157,7 +1103,7 @@ export class AgentDocumentManager { return baseChunk; } }); - + console.log('simplifiedChunks', simplifiedChunks); // Update the document with all simplified chunks at once doc.chunk_simpl = JSON.stringify({ chunks: simplifiedChunks }); @@ -1165,32 +1111,25 @@ export class AgentDocumentManager { } /** - * Gets the simplified chunks from a document - * @param doc The document to get simplified chunks from - * @returns Array of simplified chunks or empty array if none exist + * Gets a specific simplified chunk by ID + * @param doc The document containing chunks + * @param chunkId The ID of the chunk to retrieve + * @returns The simplified chunk if found, undefined otherwise */ - public getSimplifiedChunks(doc: Doc): any[] { + public getSimplifiedChunkById(doc: Doc, chunkId: string): any | undefined { + let chunks: any[] = []; if (!doc || !doc.chunk_simpl) { + chunks = []; + console.warn('No chunk found for chunkId', chunkId, '. Checking if document exists in documentsById.'); return []; } - try { const parsed = JSON.parse(StrCast(doc.chunk_simpl)); - return parsed.chunks || []; + chunks = parsed.chunks || []; } catch (e) { console.error('Error parsing simplified chunks:', e); return []; } - } - - /** - * Gets a specific simplified chunk by ID - * @param doc The document containing chunks - * @param chunkId The ID of the chunk to retrieve - * @returns The simplified chunk if found, undefined otherwise - */ - public getSimplifiedChunkById(doc: Doc, chunkId: string): any | undefined { - const chunks = this.getSimplifiedChunks(doc); return chunks.find(chunk => chunk.chunkId === chunkId); } diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts index 3df1294e9..1349df483 100644 --- a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -16,6 +16,7 @@ import { AI_Document, CHUNK_TYPE, RAGChunk } from '../types/types'; import OpenAI from 'openai'; import { Embedding } from 'openai/resources'; import { AgentDocumentManager } from '../utils/AgentDocumentManager'; +import { Id } from '../../../../../fields/FieldSymbols'; dotenv.config(); @@ -24,13 +25,12 @@ dotenv.config(); * and OpenAI text-embedding-3-large for text embedding. It handles AI document management, uploads, and query-based retrieval. */ export class Vectorstore { - private pinecone: Pinecone; // Pinecone client for managing the vector index. + private pinecone!: Pinecone; // Pinecone client for managing the vector index. private index!: Index; // The specific Pinecone index used for document chunks. - private openai: OpenAI; // OpenAI client for generating embeddings. + private openai!: OpenAI; // OpenAI client for generating embeddings. private indexName: string = 'pdf-chatbot'; // Default name for the index. - private _id: string; // Unique ID for the Vectorstore instance. - private docManager: AgentDocumentManager; // Document manager for handling documents - + private _id!: string; // Unique ID for the Vectorstore instance. + private docManager!: AgentDocumentManager; // Document manager for handling documents documents: AI_Document[] = []; // Store the documents indexed in the vectorstore. /** @@ -143,10 +143,8 @@ export class Vectorstore { progressCallback(85, 'Embeddings generated. Finalizing document...'); doc.original_segments = JSON.stringify(typedResponse.full); - const doc_id = uuidv4(); - - // Register the document with the AgentDocumentManager - this.docManager.addCustomId(doc, doc_id); + const doc_id = doc[Id]; + console.log('doc_id in vectorstore', doc_id); // Generate chunk IDs upfront so we can register them const chunkIds = segmentedTranscript.map(() => uuidv4()); @@ -191,7 +189,7 @@ export class Vectorstore { } else { // Process regular document console.log('Processing regular document...'); - const createDocumentResponse = await Networking.PostToServer('/createDocument', { file_path: local_file_path }); + const createDocumentResponse = await Networking.PostToServer('/createDocument', { file_path: local_file_path, doc_id: doc[Id] }); // Type assertion for the response const { jobId } = createDocumentResponse as { jobId: string }; @@ -211,12 +209,13 @@ export class Vectorstore { } } - // Register the document with the AgentDocumentManager - this.docManager.addCustomId(doc, result.doc_id); - // Collect all chunk IDs const chunkIds = result.chunks.map(chunk => chunk.id); + if (result.doc_id !== doc[Id]) { + console.log('doc_id in vectorstore', result.doc_id, 'does not match doc_id in doc', doc[Id]); + } + // Register chunks with the document manager this.docManager.registerChunkIds(result.doc_id, chunkIds); @@ -319,16 +318,14 @@ export class Vectorstore { const queryEmbedding = queryEmbeddingResponse.data[0].embedding; - // Get document IDs from the AgentDocumentManager - const docIds = Array.from(this.docManager.listDocs()); - console.log('Using document IDs for retrieval:', docIds); + console.log('Using document IDs for retrieval:', this.docManager.docIds); // Query the Pinecone index using the embedding and filter by document IDs. // We'll query based on document IDs that are registered in the document manager const queryResponse: QueryResponse = await this.index.query({ vector: queryEmbedding, filter: { - doc_id: { $in: docIds }, + doc_id: { $in: this.docManager.docIds }, }, topK, includeValues: true, @@ -356,7 +353,7 @@ export class Vectorstore { // Ensure the document manager knows about this chunk // This is important for maintaining backwards compatibility - if (chunk.id && !this.docManager.getDocIdByChunkId(chunk.id)) { + if (chunk.id && !this.docManager.getDocByChunkId(chunk.id)) { // If the chunk ID isn't registered but we have a doc_id in metadata if (chunk.metadata.doc_id && this.docManager.has(chunk.metadata.doc_id)) { // Register the chunk with its parent document diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index 6d2779163..378f14094 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -559,7 +559,7 @@ export default class AssistantManager extends ApiManager { method: Method.POST, subscription: '/createDocument', secureHandler: async ({ req, res }) => { - const { file_path } = req.body; + const { file_path, doc_id } = req.body; const public_path = path.join(publicDirectory, file_path); // Resolve the file path in the public directory const file_name = path.basename(file_path); // Extract the file name from the path @@ -572,7 +572,7 @@ export default class AssistantManager extends ApiManager { // Spawn the Python process and track its progress/output // eslint-disable-next-line no-use-before-define - spawnPythonProcess(jobId, public_path); + spawnPythonProcess(jobId, public_path, doc_id); // Send the job ID back to the client for tracking res.send({ jobId }); @@ -850,7 +850,7 @@ export default class AssistantManager extends ApiManager { * @param file_name The name of the file to process. * @param file_path The filepath of the file to process. */ -function spawnPythonProcess(jobId: string, file_path: string) { +function spawnPythonProcess(jobId: string, file_path: string, doc_id: string) { const venvPath = path.join(__dirname, '../chunker/venv'); const requirementsPath = path.join(__dirname, '../chunker/requirements.txt'); const pythonScriptPath = path.join(__dirname, '../chunker/pdf_chunker.py'); @@ -860,7 +860,7 @@ function spawnPythonProcess(jobId: string, file_path: string) { function runPythonScript() { const pythonPath = process.platform === 'win32' ? path.join(venvPath, 'Scripts', 'python') : path.join(venvPath, 'bin', 'python3'); - const pythonProcess = spawn(pythonPath, [pythonScriptPath, jobId, file_path, outputDirectory]); + const pythonProcess = spawn(pythonPath, [pythonScriptPath, jobId, file_path, outputDirectory, doc_id]); let pythonOutput = ''; let stderrOutput = ''; diff --git a/src/server/chunker/pdf_chunker.py b/src/server/chunker/pdf_chunker.py index e9b9ef2b3..e34753176 100644 --- a/src/server/chunker/pdf_chunker.py +++ b/src/server/chunker/pdf_chunker.py @@ -622,7 +622,7 @@ class Document: Represents a document being processed, such as a PDF, handling chunking, embedding, and summarization. """ - def __init__(self, file_path: str, file_name: str, job_id: str, output_folder: str): + def __init__(self, file_path: str, file_name: str, job_id: str, output_folder: str, doc_id: str): """ Initialize the Document with file data, file name, and job ID. @@ -635,7 +635,7 @@ class Document: self.file_path = file_path self.job_id = job_id self.type = self._get_document_type(file_name) # Determine the document type (PDF, CSV, etc.) - self.doc_id = job_id # Use the job ID as the document ID + self.doc_id = doc_id # Use the job ID as the document ID self.chunks = [] # List to hold text and visual chunks self.num_pages = 0 # Number of pages in the document (if applicable) self.summary = "" # The generated summary for the document @@ -755,7 +755,7 @@ class Document: "doc_id": self.doc_id }, indent=2) # Convert the document's attributes to JSON format -def process_document(file_path, job_id, output_folder): +def process_document(file_path, job_id, output_folder, doc_id): """ Top-level function to process a document and return the JSON output. @@ -763,26 +763,27 @@ def process_document(file_path, job_id, output_folder): :param job_id: The job ID for this document processing task. :return: The processed document's data in JSON format. """ - new_document = Document(file_path, file_path, job_id, output_folder) + new_document = Document(file_path, file_path, job_id, output_folder, doc_id) return new_document.to_json() def main(): """ Main entry point for the script, called with arguments from Node.js. """ - if len(sys.argv) != 4: + if len(sys.argv) != 5: print(json.dumps({"error": "Invalid arguments"}), file=sys.stderr) return job_id = sys.argv[1] file_path = sys.argv[2] output_folder = sys.argv[3] # Get the output folder from arguments + doc_id = sys.argv[4] try: os.makedirs(output_folder, exist_ok=True) # Process the document - document_result = process_document(file_path, job_id, output_folder) # Pass output_folder + document_result = process_document(file_path, job_id, output_folder,doc_id) # Pass output_folder # Output the final result as JSON to stdout print(document_result) -- cgit v1.2.3-70-g09d2 From e141307dbd9b951f76c908610e7b89e296ad92b8 Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" <Shulman.aj@gmail.com> Date: Sun, 11 May 2025 17:18:18 -0400 Subject: chanegd everything to be more consistent - made both web related tools use doc manager and chunk Ids --- .../views/nodes/chatbot/agentsystem/Agent.ts | 9 +- .../nodes/chatbot/chatboxcomponents/ChatBox.tsx | 50 ++---- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 2 +- src/client/views/nodes/chatbot/tools/SearchTool.ts | 26 +-- .../nodes/chatbot/tools/WebsiteInfoScraperTool.ts | 30 ++-- .../views/nodes/chatbot/tools/WikipediaTool.ts | 2 +- src/client/views/nodes/chatbot/types/types.ts | 1 + .../nodes/chatbot/utils/AgentDocumentManager.ts | 192 +++++---------------- .../views/nodes/chatbot/vectorstore/Vectorstore.ts | 21 +-- src/server/ApiManagers/AssistantManager.ts | 160 ++++++++++++----- 10 files changed, 215 insertions(+), 278 deletions(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/agentsystem/Agent.ts b/src/client/views/nodes/chatbot/agentsystem/Agent.ts index 24471bf5b..86d40864e 100644 --- a/src/client/views/nodes/chatbot/agentsystem/Agent.ts +++ b/src/client/views/nodes/chatbot/agentsystem/Agent.ts @@ -63,10 +63,8 @@ export class Agent { */ constructor( _vectorstore: Vectorstore, - summaries: () => string, history: () => string, csvData: () => { filename: string; id: string; text: string }[], - getLinkedUrlDocId: (url: string) => string[], createImage: (result: Upload.FileInformation & Upload.InspectionResults, options: DocumentOptions) => void, createCSVInDash: (url: string, title: string, id: string, data: string) => void, docManager: AgentDocumentManager @@ -83,7 +81,7 @@ export class Agent { calculate: new CalculateTool(), rag: new RAGTool(this.vectorstore), dataAnalysis: new DataAnalysisTool(csvData), - websiteInfoScraper: new WebsiteInfoScraperTool(getLinkedUrlDocId), + websiteInfoScraper: new WebsiteInfoScraperTool(this._docManager), searchTool: new SearchTool(this._docManager), noTool: new NoTool(), //imageCreationTool: new ImageCreationTool(createImage), @@ -125,11 +123,8 @@ export class Agent { // Retrieve chat history and generate system prompt const chatHistory = this._history(); // Get document summaries directly from document manager - const documentSummaries = this._docManager.getAllDocumentSummaries(); - // Create a function that returns document summaries for the prompt - const getSummaries = () => documentSummaries; // Generate the system prompt with the summaries - const systemPrompt = getReactPrompt(Object.values(this.tools), getSummaries, chatHistory); + const systemPrompt = getReactPrompt(Object.values(this.tools), () => JSON.stringify(this._docManager.listDocs), chatHistory); // Initialize intermediate messages this.interMessages = [{ role: 'system', content: systemPrompt }]; diff --git a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx index 6349e554e..867e78860 100644 --- a/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx +++ b/src/client/views/nodes/chatbot/chatboxcomponents/ChatBox.tsx @@ -121,16 +121,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { this.vectorstore = new Vectorstore(this.vectorstore_id, this.docManager); // Create an agent with the vectorstore - this.agent = new Agent( - this.vectorstore, - this.retrieveSummaries.bind(this), - this.retrieveFormattedHistory.bind(this), - this.retrieveCSVData.bind(this), - this.retrieveDocIds.bind(this), - this.createImageInDash.bind(this), - this.createCSVInDash.bind(this), - this.docManager - ); + this.agent = new Agent(this.vectorstore, this.retrieveFormattedHistory.bind(this), this.retrieveCSVData.bind(this), this.createImageInDash.bind(this), this.createCSVInDash.bind(this), this.docManager); // Add event listeners this.addScrollListener(); @@ -228,6 +219,7 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } }; + //TODO: Update for new chunk_simpl on agentDocument /** * Adds a CSV file for analysis by sending it to OpenAI and generating a summary. * @param newLinkedDoc The linked document representing the CSV file. @@ -650,18 +642,15 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { citation: JSON.stringify(citation, null, 2), }); - // First try to find the document using the document manager's chunk ID lookup - const doc: Doc | undefined = this.docManager.getDocByChunkId(chunkId); - if (!doc) { - console.warn(`Document not found for citation with chunk_id: ${chunkId}`); - return; - } - // Get the simplified chunk using the document manager - const foundChunk = this.docManager.getSimplifiedChunkById(doc, chunkId); + const { foundChunk, doc } = this.docManager.getSimplifiedChunkById(chunkId); if (!foundChunk) { - console.warn(`Chunk not found in document for chunk ID: ${chunkId}`); - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + if (doc) { + console.warn(`Chunk not found in document, ${doc.id}, for chunk ID: ${chunkId}`); + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + } else { + console.warn(`Chunk not found for chunk ID: ${chunkId}`); + } return; } @@ -678,6 +667,10 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { } else if (foundChunk.chunkType === CHUNK_TYPE.TABLE || foundChunk.chunkType === CHUNK_TYPE.IMAGE) { this.handleOtherChunkTypes(foundChunk, citation, doc); } else { + if (doc.type === 'web') { + DocumentManager.Instance.showDocument(doc, { openLocation: OpenWhere.addRight }, () => {}); + return; + } // Show the chunk text in citation popup let chunkText = citation.direct_text || 'Text content not available'; this.showCitationPopup(chunkText); @@ -986,16 +979,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return Array.from(this.docManager.listDocs); } - /** - * Getter that retrieves summaries of all linked documents. - */ - @computed - get summaries(): string { - // Use the document manager to get all summaries - console.log(this.docManager.listDocs); - return JSON.stringify(this.docManager.listDocs); - } - /** * Getter that retrieves all linked CSV files for analysis. */ @@ -1022,7 +1005,8 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { // Other helper methods for retrieving document data and processing retrieveSummaries = (): string => { - return this.docManager.getAllDocumentSummaries(); + console.log(this.docManager.listDocs); + return JSON.stringify(this.docManager.listDocs); }; retrieveCSVData = () => { @@ -1033,10 +1017,6 @@ export class ChatBox extends ViewBoxAnnotatableComponent<FieldViewProps>() { return this.formattedHistory; }; - retrieveDocIds = (): string[] => { - return Array.from(this.docManager.docIds); - }; - /** * Handles follow-up questions when the user clicks on them. * Automatically sets the input value to the clicked follow-up question. diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 5297292bf..405949c1e 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -408,7 +408,7 @@ export class DocumentMetadataTool extends BaseTool<DocumentMetadataToolParamsTyp const title = String(args.title); const data = String(args.data); - const id = this._docManager.createDocInDash(docType, data, { title: title }); + const id = await this._docManager.createDocInDash(docType, data, { title: title }); if (!id) { return [ diff --git a/src/client/views/nodes/chatbot/tools/SearchTool.ts b/src/client/views/nodes/chatbot/tools/SearchTool.ts index 53f5fc109..43f14ea83 100644 --- a/src/client/views/nodes/chatbot/tools/SearchTool.ts +++ b/src/client/views/nodes/chatbot/tools/SearchTool.ts @@ -48,19 +48,21 @@ export class SearchTool extends BaseTool<SearchToolParamsType> { query, max_results: this._max_results, })) as { results: { url: string; snippet: string }[] }; - const data = results.map((result: { url: string; snippet: string }) => { - // Create a web document with the URL - const id = this._docManager.createDocInDash('web', result.url, { - title: `Search Result: ${result.url}`, - text_html: result.snippet, - data_useCors: true, - }); + const data = await Promise.all( + results.map(async (result: { url: string; snippet: string }) => { + // Create a web document with the URL + const id = await this._docManager.createDocInDash('web', result.url, { + title: `Search Result: ${result.url}`, + text_html: result.snippet, + data_useCors: true, + }); - return { - type: 'text' as const, - text: `<chunk chunk_id="${id}" chunk_type="url"><url>${result.url}</url><overview>${result.snippet}</overview></chunk>`, - }; - }); + return { + type: 'text' as const, + text: `<chunk chunk_id="${id}" chunk_type="url"><url>${result.url}</url><overview>${result.snippet}</overview></chunk>`, + }; + }) + ); return data; } catch (error) { console.log(error); diff --git a/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts b/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts index 3c7b4e3db..495a985cb 100644 --- a/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts +++ b/src/client/views/nodes/chatbot/tools/WebsiteInfoScraperTool.ts @@ -3,12 +3,14 @@ import { Networking } from '../../../../Network'; import { BaseTool } from './BaseTool'; import { Observation } from '../types/types'; import { ParametersType, ToolInfo } from '../types/tool_types'; - +import { AgentDocumentManager } from '../utils/AgentDocumentManager'; +import { Doc } from '../../../../../fields/Doc'; +import { StrCast, WebCast } from '../../../../../fields/Types'; const websiteInfoScraperToolParams = [ { - name: 'urls', + name: 'chunk_ids', type: 'string[]', - description: 'The URLs of the websites to scrape', + description: 'The chunk_ids of the urls to scrape from the SearchTool.', required: true, max_inputs: 3, }, @@ -66,11 +68,11 @@ const websiteInfoScraperToolInfo: ToolInfo<WebsiteInfoScraperToolParamsType> = { }; export class WebsiteInfoScraperTool extends BaseTool<WebsiteInfoScraperToolParamsType> { - private _getLinkedUrlDocId: (url: string) => string[]; + private _docManager: AgentDocumentManager; - constructor(getLinkedUrlDocIds: (url: string) => string[]) { + constructor(docManager: AgentDocumentManager) { super(websiteInfoScraperToolInfo); - this._getLinkedUrlDocId = getLinkedUrlDocIds; + this._docManager = docManager; } /** @@ -79,10 +81,13 @@ export class WebsiteInfoScraperTool extends BaseTool<WebsiteInfoScraperToolParam * @param maxRetries Maximum number of retry attempts * @returns The scraped content or error message */ - private async scrapeWithRetry(url: string, maxRetries = 2): Promise<Observation> { + private async scrapeWithRetry(chunkDoc: Doc, maxRetries = 2): Promise<Observation> { let lastError = ''; let retryCount = 0; - + const url = WebCast(chunkDoc.data!)!.url.href; + console.log(url); + console.log(chunkDoc); + console.log(chunkDoc.data); // Validate URL format try { new URL(url); // This will throw if URL is invalid @@ -110,7 +115,6 @@ export class WebsiteInfoScraperTool extends BaseTool<WebsiteInfoScraperToolParam } const { website_plain_text } = response as { website_plain_text: string }; - const id = this._getLinkedUrlDocId(url); // Validate content quality if (!website_plain_text) { @@ -126,7 +130,7 @@ export class WebsiteInfoScraperTool extends BaseTool<WebsiteInfoScraperToolParam if (retryCount === maxRetries) { return { type: 'text', - text: `<chunk chunk_id="${id}" chunk_type="url">\n${website_plain_text}\nNote: Limited content was retrieved from this URL.\n</chunk>`, + text: `<chunk chunk_id="${chunkDoc.id}" chunk_type="url">\n${website_plain_text}\nNote: Limited content was retrieved from this URL.\n</chunk>`, } as Observation; } @@ -138,7 +142,7 @@ export class WebsiteInfoScraperTool extends BaseTool<WebsiteInfoScraperToolParam // Process and return content if it looks good return { type: 'text', - text: `<chunk chunk_id="${id}" chunk_type="url">\n${website_plain_text}\n</chunk>`, + text: `<chunk chunk_id="${chunkDoc.id}" chunk_type="url">\n${website_plain_text}\n</chunk>`, } as Observation; } catch (error) { lastError = error instanceof Error ? error.message : 'Unknown error'; @@ -156,10 +160,10 @@ export class WebsiteInfoScraperTool extends BaseTool<WebsiteInfoScraperToolParam } async execute(args: ParametersType<WebsiteInfoScraperToolParamsType>): Promise<Observation[]> { - const urls = args.urls; + const chunk_ids = args.chunk_ids; // Create an array of promises, each one handling a website scrape for a URL - const scrapingPromises = urls.map(url => this.scrapeWithRetry(url)); + const scrapingPromises = chunk_ids.map(chunk_id => this.scrapeWithRetry(this._docManager.getDocument(chunk_id)!)); // Wait for all scraping promises to resolve const results = await Promise.all(scrapingPromises); diff --git a/src/client/views/nodes/chatbot/tools/WikipediaTool.ts b/src/client/views/nodes/chatbot/tools/WikipediaTool.ts index ee815532a..ec5d83e52 100644 --- a/src/client/views/nodes/chatbot/tools/WikipediaTool.ts +++ b/src/client/views/nodes/chatbot/tools/WikipediaTool.ts @@ -32,7 +32,7 @@ export class WikipediaTool extends BaseTool<WikipediaToolParamsType> { async execute(args: ParametersType<WikipediaToolParamsType>): Promise<Observation[]> { try { - const { text } = await Networking.PostToServer('/getWikipediaSummary', { title: args.title }); + const { text } = (await Networking.PostToServer('/getWikipediaSummary', { title: args.title })) as { text: string }; const id = uuidv4(); const url = `https://en.wikipedia.org/wiki/${args.title.replace(/ /g, '_')}`; this._addLinkedUrlDoc(url, id); diff --git a/src/client/views/nodes/chatbot/types/types.ts b/src/client/views/nodes/chatbot/types/types.ts index 90b5e7e11..0d1804b2d 100644 --- a/src/client/views/nodes/chatbot/types/types.ts +++ b/src/client/views/nodes/chatbot/types/types.ts @@ -101,6 +101,7 @@ export interface RAGChunk { export interface SimplifiedChunk { chunkId: string; + doc_id: string; startPage?: number; endPage?: number; location?: string; diff --git a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts index c8a6bb16b..5a09b945b 100644 --- a/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts +++ b/src/client/views/nodes/chatbot/utils/AgentDocumentManager.ts @@ -13,7 +13,7 @@ import { LinkManager, UPDATE_SERVER_CACHE } from '../../../../util/LinkManager'; import { DocumentView } from '../../DocumentView'; import { ChatBox, parsedDoc } from '../chatboxcomponents/ChatBox'; import { supportedDocTypes } from '../types/tool_types'; -import { CHUNK_TYPE, RAGChunk } from '../types/types'; +import { CHUNK_TYPE, RAGChunk, SimplifiedChunk } from '../types/types'; /** * Interface representing a document in the freeform view @@ -31,7 +31,7 @@ export class AgentDocumentManager { private chatBox: ChatBox; private chatBoxDocument: Doc | null = null; private fieldMetadata: Record<string, any> = {}; - @observable private documentIdsFromChunkIds: ObservableMap<string, string>; + @observable private simplifiedChunks: ObservableMap<string, SimplifiedChunk>; /** * Creates a new DocumentManager @@ -40,17 +40,21 @@ export class AgentDocumentManager { constructor(chatBox: ChatBox) { makeObservable(this); const agentDoc = DocCast(chatBox.Document.agentDocument) ?? new Doc(); - const chunkIds = DocCast(agentDoc.chunkIds) ?? new Doc(); + const chunk_simpl = DocCast(agentDoc.chunk_simpl) ?? new Doc(); agentDoc.title = chatBox.Document.title + '_agentDocument'; - chunkIds.title = '_chunkIds'; + chunk_simpl.title = '_chunk_simpl'; chatBox.Document.agentDocument = agentDoc; - DocCast(chatBox.Document.agentDocument)!.chunkIds = chunkIds; - this.documentIdsFromChunkIds = StrListCast(chunkIds.mapping).reduce((mapping, content) => { - const [chunkId, docId] = content.split(':'); - mapping.set(chunkId, docId); + DocCast(chatBox.Document.agentDocument)!.chunk_simpl = chunk_simpl; + + this.simplifiedChunks = StrListCast(chunk_simpl.mapping).reduce((mapping, chunks) => { + StrListCast(chunks).forEach(chunk => { + const parsed = JSON.parse(StrCast(chunk)); + mapping.set(parsed.chunkId, parsed); + }); return mapping; - }, new ObservableMap<string, string>()); + }, new ObservableMap<string, SimplifiedChunk>()); + this.documentsById = StrListCast(agentDoc.mapping).reduce((mapping, content) => { const [id, layoutId, docId] = content.split(':'); const layoutDoc = DocServer.GetCachedRefField(layoutId); @@ -76,14 +80,10 @@ export class AgentDocumentManager { //{ fireImmediately: true } ); reaction( - () => this.documentIdsFromChunkIds.values(), + () => this.simplifiedChunks.values(), () => { if (this.chatBoxDocument && DocCast(this.chatBoxDocument.agentDocument)) { - // Store the mapping with chunkId:docId format for consistency - const chunkIdsDoc = DocCast(DocCast(this.chatBoxDocument.agentDocument)!.chunkIds); - if (chunkIdsDoc) { - chunkIdsDoc.mapping = new List<string>(Array.from(this.documentIdsFromChunkIds.entries()).map(([chunkId, docId]) => `${chunkId}:${docId}`)); - } + DocCast(DocCast(this.chatBoxDocument.agentDocument)!.chunk_simpl)!.mapping = new List<string>(Array.from(this.simplifiedChunks.values()).map(chunk => JSON.stringify(chunk))); } } //{ fireImmediately: true } @@ -831,7 +831,8 @@ export class AgentDocumentManager { * @param options Optional configuration options * @returns The ID of the created document */ - public createDocInDash(docType: string, data: string, options?: any): string { + + public async createDocInDash(docType: string, data: string, options?: any): Promise<string> { // Validate doc_type if (!this.isValidDocType(docType)) { throw new Error(`Invalid document type: ${docType}`); @@ -877,14 +878,15 @@ export class AgentDocumentManager { // Create link and add it to the document system const linkDoc = Docs.Create.LinkDocument(this.chatBoxDocument, doc); LinkManager.Instance.addLink(linkDoc); - - // Add document to view - this.chatBox._props.addDocument?.(doc); - - // Show document - defer actual display to prevent immediate resource loading - setTimeout(() => { - DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); - }, 100); + if (doc.type !== 'web') { + // Add document to view + this.chatBox._props.addDocument?.(doc); + + // Show document - defer actual display to prevent immediate resource loading + setTimeout(() => { + DocumentManager.Instance.showDocument(doc, { willZoomCentered: true }, () => {}); + }, 100); + } } }); @@ -985,67 +987,6 @@ export class AgentDocumentManager { return docInfo?.layoutDoc; } - /** - * Registers chunk IDs associated with a document in the manager - * @param docId The parent document ID - * @param chunkIds Array of chunk IDs associated with this document - */ - @action - public registerChunkIds(docId: string, chunkIds: string[]): void { - // Get the document if it exists - const docInfo = this.documentsById.get(docId); - if (!docInfo) { - console.warn(`Cannot register chunks for unknown document ID: ${docId}`); - return; - } - - // Store chunk IDs on the document for future reference - const doc = docInfo.layoutDoc; - if (!doc.chunk_ids) { - doc.chunk_ids = JSON.stringify(chunkIds); - } else { - // Merge with existing chunk IDs if they exist - const existingIds = JSON.parse(doc.chunk_ids as string); - const updatedIds = [...new Set([...existingIds, ...chunkIds])]; // Remove duplicates - doc.chunk_ids = JSON.stringify(updatedIds); - } - for (const chunkId of chunkIds) { - // Ensure each chunk ID can be linked back to its parent document - // Store a mapping from chunk ID to parent document ID - // This allows us to easily find a document by any of its chunk IDs - if (!this.documentIdsFromChunkIds.has(chunkId) && doc) { - this.documentIdsFromChunkIds.set(chunkId, doc[Id]); - } - } - } - - /** - * Gets a document ID by a chunk ID - * @param chunkId The chunk ID to look up - * @returns The parent document ID if found - */ - public getDocByChunkId(chunkId: string): Doc | undefined { - // First, look up the document ID using the chunk ID mapping - const docId = this.documentIdsFromChunkIds.get(chunkId); - console.log('this.documentIdsFromChunkIds', this.documentIdsFromChunkIds); - console.log('docId', docId); - if (!docId) { - if (this.documentsById.has(chunkId)) { - return this.documentsById.get(chunkId)?.layoutDoc; - } else { - console.error('No document found for chunkId and docId', chunkId); - return undefined; - } - } - // Then get the document using the document ID - const docInfo = this.documentsById.get(docId); - if (docInfo) { - return docInfo.layoutDoc; - } - console.error('No document found for docId', docId); - return undefined; - } - /** * Adds simplified chunks to a document for citation handling * @param doc The document to add simplified chunks to @@ -1053,21 +994,13 @@ export class AgentDocumentManager { * @param docType The type of document (e.g., 'pdf', 'video', 'audio', etc.) * @returns The updated document with simplified chunks */ - public addSimplifiedChunks(doc: Doc, chunks: RAGChunk[], docType: string): Doc { - if (!doc) { - console.error('Cannot add simplified chunks to null document'); - return doc; - } - - // Initialize empty chunks array if not exists - if (!doc.chunk_simpl) { - doc.chunk_simpl = JSON.stringify({ chunks: [] }); - } - + @action + public addSimplifiedChunks(chunks: RAGChunk[], docType: string) { + console.log('chunks', chunks, 'simplifiedChunks', this.simplifiedChunks); // Create array of simplified chunks based on document type - const simplifiedChunks = chunks.map(chunk => { + for (const chunk of chunks) { // Common properties across all chunk types - const baseChunk = { + const baseChunk: SimplifiedChunk = { chunkId: chunk.id, text: chunk.metadata.text, doc_id: chunk.metadata.doc_id, @@ -1076,38 +1009,33 @@ export class AgentDocumentManager { // Add type-specific properties if (docType === 'video' || docType === 'audio') { - return { + this.simplifiedChunks.set(chunk.id, { ...baseChunk, start_time: chunk.metadata.start_time, end_time: chunk.metadata.end_time, indexes: chunk.metadata.indexes, chunkType: docType === 'video' ? CHUNK_TYPE.VIDEO : CHUNK_TYPE.AUDIO, - }; + } as SimplifiedChunk); } else if (docType === 'pdf') { - return { + this.simplifiedChunks.set(chunk.id, { ...baseChunk, startPage: chunk.metadata.start_page, endPage: chunk.metadata.end_page, location: chunk.metadata.location, - }; + } as SimplifiedChunk); } else if (docType === 'csv') { - return { + this.simplifiedChunks.set(chunk.id, { ...baseChunk, rowStart: (chunk.metadata as any).row_start, rowEnd: (chunk.metadata as any).row_end, colStart: (chunk.metadata as any).col_start, colEnd: (chunk.metadata as any).col_end, - }; + } as SimplifiedChunk); } else { // Default for other document types - return baseChunk; + this.simplifiedChunks.set(chunk.id, baseChunk as SimplifiedChunk); } - }); - console.log('simplifiedChunks', simplifiedChunks); - // Update the document with all simplified chunks at once - doc.chunk_simpl = JSON.stringify({ chunks: simplifiedChunks }); - - return doc; + } } /** @@ -1116,21 +1044,10 @@ export class AgentDocumentManager { * @param chunkId The ID of the chunk to retrieve * @returns The simplified chunk if found, undefined otherwise */ - public getSimplifiedChunkById(doc: Doc, chunkId: string): any | undefined { - let chunks: any[] = []; - if (!doc || !doc.chunk_simpl) { - chunks = []; - console.warn('No chunk found for chunkId', chunkId, '. Checking if document exists in documentsById.'); - return []; - } - try { - const parsed = JSON.parse(StrCast(doc.chunk_simpl)); - chunks = parsed.chunks || []; - } catch (e) { - console.error('Error parsing simplified chunks:', e); - return []; - } - return chunks.find(chunk => chunk.chunkId === chunkId); + public getSimplifiedChunkById(chunkId: string): any | undefined { + console.log('chunkId', chunkId, 'simplifiedChunks', this.simplifiedChunks); + console.log('doc', this.getDocument(this.simplifiedChunks.get(chunkId)?.doc_id || '')); + return { foundChunk: this.simplifiedChunks.get(chunkId), doc: this.getDocument(this.simplifiedChunks.get(chunkId)?.doc_id || '') }; } /** @@ -1150,27 +1067,4 @@ export class AgentDocumentManager { return []; } } - - /** - * Gets all document summaries combined into a single string - * @returns String containing all document summaries - */ - public getAllDocumentSummaries(): string { - const summaries = Array.from(this.documentsById.keys()) - .map(id => { - const doc = this.getDocument(id); - if (doc) { - // Try to get summary from either the document or its data document - const summary = doc.summary || (doc[DocData] && doc[DocData].summary); - if (summary) { - return StrCast(summary); - } - } - return null; - }) - .filter(Boolean) - .join('\n\n'); - - return summaries; - } } diff --git a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts index 1349df483..f1fae6f11 100644 --- a/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts +++ b/src/client/views/nodes/chatbot/vectorstore/Vectorstore.ts @@ -148,10 +148,6 @@ export class Vectorstore { // Generate chunk IDs upfront so we can register them const chunkIds = segmentedTranscript.map(() => uuidv4()); - - // Register all chunk IDs with the document manager - this.docManager.registerChunkIds(doc_id, chunkIds); - // Add transcript and embeddings to metadata result = { doc_id, @@ -185,7 +181,7 @@ export class Vectorstore { doc.segmented_transcript = JSON.stringify(segmentedTranscript); // Use doc manager to add simplified chunks const docType = local_file_path.endsWith('.mp3') ? 'audio' : 'video'; - this.docManager.addSimplifiedChunks(doc, result.chunks, docType); + this.docManager.addSimplifiedChunks(result.chunks, docType); } else { // Process regular document console.log('Processing regular document...'); @@ -216,13 +212,10 @@ export class Vectorstore { console.log('doc_id in vectorstore', result.doc_id, 'does not match doc_id in doc', doc[Id]); } - // Register chunks with the document manager - this.docManager.registerChunkIds(result.doc_id, chunkIds); - // Use doc manager to add simplified chunks - determine document type from file extension const fileExt = path.extname(local_file_path).toLowerCase(); const docType = fileExt === '.pdf' ? 'pdf' : fileExt === '.csv' ? 'csv' : 'text'; - this.docManager.addSimplifiedChunks(doc, result.chunks, docType); + this.docManager.addSimplifiedChunks(result.chunks, docType); doc.summary = result.summary; doc.ai_purpose = result.purpose; @@ -351,16 +344,6 @@ export class Vectorstore { }, } as RAGChunk; - // Ensure the document manager knows about this chunk - // This is important for maintaining backwards compatibility - if (chunk.id && !this.docManager.getDocByChunkId(chunk.id)) { - // If the chunk ID isn't registered but we have a doc_id in metadata - if (chunk.metadata.doc_id && this.docManager.has(chunk.metadata.doc_id)) { - // Register the chunk with its parent document - this.docManager.registerChunkIds(chunk.metadata.doc_id, [chunk.id]); - } - } - return chunk; }); diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index 378f14094..b7ce4f663 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -514,30 +514,37 @@ export default class AssistantManager extends ApiManager { await browser.close(); browser = null; - // Use a try-catch block specifically for JSDOM parsing + let extractedText = ''; + + // First try with Readability try { // Parse HTML content using JSDOM const dom = new JSDOM(htmlContent, { url }); // Extract readable content using Mozilla's Readability API - const reader = new Readability(dom.window.document); + const reader = new Readability(dom.window.document, { + // Readability configuration to focus on text content + charThreshold: 100, + keepClasses: false, + }); const article = reader.parse(); - if (article) { - const plainText = article.textContent; - res.send({ website_plain_text: plainText }); + if (article && article.textContent) { + extractedText = article.textContent; } else { - // If Readability fails, fallback to extracting main content - const mainContent = await extractMainContent(htmlContent); - res.send({ website_plain_text: mainContent }); + // If Readability doesn't return useful content, try alternate method + extractedText = await extractEnhancedContent(htmlContent); } } catch (parsingError) { - console.error('Error parsing website content:', parsingError); - - // Fallback to a simplified extraction method - const mainContent = await extractMainContent(htmlContent); - res.send({ website_plain_text: mainContent }); + console.error('Error parsing website content with Readability:', parsingError); + // Fallback to enhanced content extraction + extractedText = await extractEnhancedContent(htmlContent); } + + // Clean up the extracted text + extractedText = cleanupText(extractedText); + + res.send({ website_plain_text: extractedText }); } catch (error) { console.error('Error scraping website:', error); @@ -985,48 +992,119 @@ function spawnPythonProcess(jobId: string, file_path: string, doc_id: string) { } /** - * Extracts main content from HTML by removing scripts, styles, and non-content elements - * Used as a fallback when Readability fails + * Enhanced content extraction that focuses on meaningful text content. * @param html The HTML content to process - * @returns Extracted main text content + * @returns Extracted and cleaned text content */ -async function extractMainContent(html: string): Promise<string> { +async function extractEnhancedContent(html: string): Promise<string> { try { - // Create a simple DOM to extract content + // Create DOM to extract content const dom = new JSDOM(html, { runScripts: 'outside-only' }); const document = dom.window.document; - // Remove scripts, styles, and other non-content elements - const elementsToRemove = ['script', 'style', 'iframe', 'noscript', 'svg', 'header', 'footer', 'nav', 'aside', 'ads', 'banner', 'form', 'button', 'input']; - - elementsToRemove.forEach(tag => { - const elements = document.querySelectorAll(tag); + // Remove all non-content elements + const elementsToRemove = [ + 'script', + 'style', + 'iframe', + 'noscript', + 'svg', + 'canvas', + 'header', + 'footer', + 'nav', + 'aside', + 'form', + 'button', + 'input', + 'select', + 'textarea', + 'meta', + 'link', + 'img', + 'video', + 'audio', + '.ad', + '.ads', + '.advertisement', + '.banner', + '.cookie', + '.popup', + '.modal', + '.newsletter', + '[role="banner"]', + '[role="navigation"]', + '[role="complementary"]', + ]; + + elementsToRemove.forEach(selector => { + const elements = document.querySelectorAll(selector); elements.forEach(el => el.remove()); }); - // Try to find the main content container using common selectors - const mainSelectors = ['main', 'article', '#content', '.content', '#main', '.main', '.post-content', '.article-content', '.entry-content']; - - let mainContent = ''; - - // Try each selector to find main content - for (const selector of mainSelectors) { - const element = document.querySelector(selector); - if (element && element.textContent && element.textContent.trim().length > 100) { - mainContent = element.textContent; - break; + // Get all text paragraphs with meaningful content + const contentElements = [ + ...Array.from(document.querySelectorAll('p')), + ...Array.from(document.querySelectorAll('h1')), + ...Array.from(document.querySelectorAll('h2')), + ...Array.from(document.querySelectorAll('h3')), + ...Array.from(document.querySelectorAll('h4')), + ...Array.from(document.querySelectorAll('h5')), + ...Array.from(document.querySelectorAll('h6')), + ...Array.from(document.querySelectorAll('li')), + ...Array.from(document.querySelectorAll('td')), + ...Array.from(document.querySelectorAll('article')), + ...Array.from(document.querySelectorAll('section')), + ...Array.from(document.querySelectorAll('div:not([class]):not([id])')), + ]; + + // Extract text from content elements that have meaningful text + let contentParts: string[] = []; + contentElements.forEach(el => { + const text = el.textContent?.trim(); + // Only include elements with substantial text (more than just a few characters) + if (text && text.length > 10 && !contentParts.includes(text)) { + contentParts.push(text); } - } + }); - // If no main content found with selectors, use body content - if (!mainContent || mainContent.length < 200) { - mainContent = document.body.textContent || ''; + // If no significant content found with selective approach, fallback to body + if (contentParts.length < 3) { + return document.body.textContent || ''; } - // Clean up the text - return mainContent.replace(/\s+/g, ' ').replace(/\n+/g, '\n').trim(); + return contentParts.join('\n\n'); } catch (error) { - console.error('Error extracting main content:', error); + console.error('Error extracting enhanced content:', error); return 'Failed to extract content from the webpage.'; } } + +/** + * Cleans up extracted text to improve readability and focus on useful content. + * @param text The raw extracted text + * @returns Cleaned and formatted text + */ +function cleanupText(text: string): string { + if (!text) return ''; + + return ( + text + // Remove excessive whitespace and normalize line breaks + .replace(/\s+/g, ' ') + .replace(/\n\s*\n\s*\n+/g, '\n\n') + // Remove common boilerplate phrases + .replace(/cookie policy|privacy policy|terms of service|all rights reserved|copyright ©/gi, '') + // Remove email addresses + .replace(/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, '') + // Remove URLs + .replace(/https?:\/\/[^\s]+/g, '') + // Remove social media handles + .replace(/@[a-zA-Z0-9_]+/g, '') + // Clean up any remaining HTML tags that might have been missed + .replace(/<[^>]*>/g, '') + // Fix spacing issues after cleanup + .replace(/ +/g, ' ') + .trim() + ); +} -- cgit v1.2.3-70-g09d2 From 0a6f3fc649b37e273a501302c1dd645a5e9a18ac Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" <Shulman.aj@gmail.com> Date: Mon, 12 May 2025 11:49:49 -0400 Subject: Added example --- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 197 ++++++++++++++++++++- 1 file changed, 196 insertions(+), 1 deletion(-) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index 405949c1e..b4c21f6e3 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -96,7 +96,202 @@ Example: To change document height, disable auto-height and set height in a sing {... inputs: { action: "edit", documentId: "doc123", fieldEdits: [ { fieldName: "layout_autoHeight", fieldValue: false }, { fieldName: "height", fieldValue: 300 } -]}}`; +]}} + + +Here is a detailed example: + +<interaction description="Using DocumentMetadataTool to create and align a diagram with its notes"> + <stage number="1" role="user"> + <query>Create a flow-chart of neurotransmitter release at a synapse, and place a notes box directly underneath that explains each step.</query> + </stage> + + <!-- CREATE THE DIAGRAM --> + <stage number="2" role="assistant"> + <thought> + I will create a DiagramBox containing a Mermaid flow-chart. After that, I'll make a notes document, read the diagram’s metadata, and then move the notes so it lines up beneath the chart. + </thought> + <action>documentMetadata</action> + </stage> + + <stage number="3" role="user"> + <action_rules>***Action rules omitted***</action_rules> + </stage> + + <stage number="4" role="assistant"> + <action_input> + <action_input_description>Creating a DiagramBox with a Mermaid chart of neurotransmitter release.</action_input_description> + <inputs> + <action>create</action> + <title>Synaptic Release Flowchart + graph TD; A(Vesicle docks)-->B(Ca²⁺ influx); B-->C(Vesicle fuses); C-->D(Neurotransmitter released) + diagram + + + + + + + + Document created successfully. + Document ID: 9f2c43c8-0a16-4c12-987b-b813d2f7b1ee + Type: diagram + Title: "Synaptic Release Flowchart" + + The document has been created with default dimensions and positioning. + … + Full metadata for the created document: + { /* huge JSON omitted for brevity */ } + + + + + + + + I will create a plain-text notes document that summarises the diagram’s steps. + + documentMetadata + + + + ***Action rules omitted*** + + + + + Creating a text document for explanatory notes. + + create + Synaptic Release – Notes + This diagram shows vesicle docking, calcium-triggered fusion, and neurotransmitter release into the synaptic cleft. + note + + + + + + + + Document created successfully. + Document ID: c8e3f4b9-7d2e-462a-a444-61e8e9ff0c7f + Type: note + Title: "Synaptic Release – Notes" + … + Full metadata for the created document: + { /* huge JSON omitted for brevity */ } + + + + + + + + I'll fetch the diagram’s width, height, and position so I can align the notes box underneath it. + + documentMetadata + + + + ***Action rules omitted*** + + + + + Getting metadata for the diagram document. + + get + 9f2c43c8-0a16-4c12-987b-b813d2f7b1ee + + + + + + + + { + "x": 100, + "y": 100, + "width": 600, + "height": 300, + "layout_autoWidth": false, + "layout_autoHeight": false + } + + + + + + + + The diagram ends at y = 100 + 300 = 400. + I’ll move the notes to x = 100, y = 420 (a 20-px gap) and expand its width to 600 px. + + documentMetadata + + + + ***Action rules omitted*** + + + + + Editing the notes document’s position and size. + + edit + c8e3f4b9-7d2e-462a-a444-61e8e9ff0c7f + [ + {"fieldName":"layout_autoWidth","fieldValue":false}, + {"fieldName":"width","fieldValue":600}, + {"fieldName":"x","fieldValue":100}, + {"fieldName":"y","fieldValue":420} + ] + + + + + + + + Successfully edited 4 fields on document c8e3f4b9-7d2e-462a-a444-61e8e9ff0c7f. + New position: (x:100, y:420) Width:600 + + + + + + + + Both documents are now created and perfectly aligned; I can summarise the result. + + + + A **DiagramBox** depicting neurotransmitter release was created at **(100 px, 100 px)**, sized **600 × 300 px**. + + + A **notes document** was then resized to **600 px** wide and moved to **(100 px, 420 px)**—20 px below the diagram—to keep the explanation directly beneath the visualization. + + + This layout ensures viewers can read the synopsis while referring to the flow-chart above. + + + + + + + Would you like to tweak the diagram’s styling (e.g., colours or fonts)? + Should I link external references or papers in the notes? + Do you want similar diagrams for other neural processes? + + + The assistant used **DocumentMetadataTool** four times: + 1) **create** DiagramBox → 2) **create** notes document → 3) **get** diagram metadata → 4) **edit** notes position/size. + This demonstrates creating, inspecting, and aligning documents within a Freeform view. + + + + +`; // Extensive usage guidelines for the tool const citationRules = `USAGE GUIDELINES: -- cgit v1.2.3-70-g09d2 From 9437753fdebfc7c4b172eeda53610c08abe7287a Mon Sep 17 00:00:00 2001 From: "A.J. Shulman" Date: Mon, 12 May 2025 16:30:10 -0400 Subject: added mermaid guide --- .../nodes/chatbot/tools/DocumentMetadataTool.ts | 101 +++++++++++++++++++++ 1 file changed, 101 insertions(+) (limited to 'src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts') diff --git a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts index b4c21f6e3..a55f901e1 100644 --- a/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts +++ b/src/client/views/nodes/chatbot/tools/DocumentMetadataTool.ts @@ -291,6 +291,106 @@ Here is a detailed example: + + + + + Mermaid mindmaps are hierarchical diagrams used to visually organize ideas. Nodes are created using indentation to show parent-child relationships. + + This is an experimental feature in Mermaid and may change in future versions. + + + + + + + + mindmap declares the diagram. + Indentation determines the hierarchy. + Each level must be indented more than its parent. + + + + + Nodes can be styled with various shapes similar to flowchart syntax. + + id[Square Text] + id(Rounded Square) + id((Circle)) + id))Bang(( + id)Cloud( + id{{Hexagon}} + Default shape without any brackets + + + + + Nodes can include icons using the ::icon(class) syntax. + + + + Icon fonts must be included by the site administrator for proper rendering. + + + + Add custom styling classes using :::. + + + + Classes must be defined in your website or application CSS. + + + + Supports markdown-style strings for rich text, line breaks, and auto-wrapping. + + + + + + + Indentation is relative, not absolute — Mermaid will infer hierarchy based on surrounding context even with inconsistent spacing. + + + + + From Mermaid v11, mindmaps are included natively. For older versions, use external imports with lazy loading. + + + + import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs'; + + ]]> + + + + `; // Extensive usage guidelines for the tool @@ -336,6 +436,7 @@ RECOMMENDED WORKFLOW: 3. Get document metadata with action="get" to see current values 4. Edit fields with action="edit" using proper dependencies OR +0. Understand the state of the currently available documents and their metadata using action="get" (this includes spacial positioning). 1. Create a new document with action="create" 2. Get its ID from the response 3. Edit the document's properties with action="edit" -- cgit v1.2.3-70-g09d2