diff options
Diffstat (limited to 'src/client/views/nodes/ChatBox/vectorstore')
| -rw-r--r-- | src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts | 16 |
1 files changed, 7 insertions, 9 deletions
diff --git a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts index b3e3f8679..ab0b6e617 100644 --- a/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts +++ b/src/client/views/nodes/ChatBox/vectorstore/VectorstoreUpload.ts @@ -3,7 +3,7 @@ import { CohereClient } from 'cohere-ai'; import { EmbedResponse } from 'cohere-ai/api'; import dotenv from 'dotenv'; -import { Chunk, AI_Document, convertToAIDocument, CHUNK_TYPE } from '../types'; +import { Chunk, AI_Document, CHUNK_TYPE } from '../types'; import { Doc } from '../../../../../fields/Doc'; import { DocData } from '../../../../../fields/DocSymbols'; import { CsvCast, PDFCast, StrCast } from '../../../../../fields/Types'; @@ -76,7 +76,7 @@ export class Vectorstore { console.log('Document JSON:', document_json); //const ai_document: AI_Document = convertToAIDocument(document_json); this.documents.push(document_json); - await this.indexDocument(convertToAIDocument(document_json)); + await this.indexDocument(JSON.parse(JSON.stringify(document_json, (key, value) => (value === null || value === undefined ? undefined : value)))); console.log(`Document added: ${document_json.file_name}`); doc.summary = document_json.summary; doc.ai_purpose = document_json.purpose; @@ -88,13 +88,13 @@ export class Vectorstore { if (doc.chunk_simpl === undefined || doc.chunk_simpl === null || doc.chunk_simpl === '' || doc.chunk_simpl === '[]') { doc.chunk_simpl = JSON.stringify({ text_chunks: [], image_chunks: [] }); } - let new_chunk_simpl: { text_chunks: { chunk_id: string; start_page: number; end_page: number }[]; image_chunks: { chunk_id: string; location: string; page: number; page_width: number; page_height: number }[] } = { + let new_chunk_simpl: { text_chunks: { chunk_id: string; start_page: number; end_page: number }[]; image_chunks: { chunk_id: string; location: string; page: number }[] } = { text_chunks: [], image_chunks: [], }; document_json.chunks.forEach((chunk: Chunk) => { - let chunk_to_add: { chunk_id: string; start_page: number; end_page: number }[] | { chunk_id: string; location: string; page: number; page_width: number; page_height: number }[]; + let chunk_to_add: { chunk_id: string; start_page: number; end_page: number }[] | { chunk_id: string; location: string; page: number }[]; switch (chunk.metadata.type) { case CHUNK_TYPE.TEXT: chunk_to_add = [{ chunk_id: chunk.id, start_page: chunk.metadata.start_page, end_page: chunk.metadata.end_page }]; @@ -105,9 +105,7 @@ export class Vectorstore { case CHUNK_TYPE.IMAGE: case CHUNK_TYPE.TABLE: console.log('Location:', chunk.metadata.location); - console.log('Height:', chunk.metadata.page_height); - console.log('Width:', chunk.metadata.page_width); - chunk_to_add = [{ chunk_id: chunk.id, location: chunk.metadata.location, page: chunk.metadata.start_page, page_width: chunk.metadata.page_width, page_height: chunk.metadata.page_height }]; + chunk_to_add = [{ chunk_id: chunk.id, location: chunk.metadata.location, page: chunk.metadata.start_page }]; new_chunk_simpl = JSON.parse(StrCast(doc.chunk_simpl)); new_chunk_simpl.image_chunks = new_chunk_simpl.image_chunks.concat(chunk_to_add); doc.chunk_simpl = JSON.stringify(new_chunk_simpl); @@ -120,9 +118,9 @@ export class Vectorstore { } } - private async indexDocument(document: AI_Document) { + private async indexDocument(document: any) { console.log('Uploading vectors to content namespace...'); - const pineconeRecords: PineconeRecord<RecordMetadata>[] = document.chunks.map( + const pineconeRecords: PineconeRecord<RecordMetadata>[] = (document.chunks as Chunk[]).map( chunk => ({ id: chunk.id, |