1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
import { Networking } from '../../../../Network';
import { Observation, RAGChunk } from '../types/types';
import { ParametersType, ToolInfo } from '../types/tool_types';
import { Vectorstore } from '../vectorstore/Vectorstore';
import { BaseTool } from './BaseTool';
import { DocumentMetadataTool } from './DocumentMetadataTool';
const ragToolParams = [
{
name: 'hypothetical_document_chunk',
type: 'string',
description: "A detailed prompt representing an ideal chunk to embed and compare against document vectors to retrieve the most relevant content for answering the user's query.",
required: true,
},
{
name: 'doc_ids',
type: 'string[]',
description: 'An optional array of document IDs to retrieve chunks from. If you want to retrieve chunks from all documents, leave this as an empty array: [] (DO NOT LEAVE THIS EMPTY).',
required: false,
},
] as const;
type RAGToolParamsType = typeof ragToolParams;
const ragToolInfo: ToolInfo<RAGToolParamsType> = {
name: 'rag',
description: `Performs a RAG (Retrieval-Augmented Generation) search on user documents (only PDF, audio, and video are supported—for information about other document types, use the ${DocumentMetadataTool.name} tool) and returns a set of document chunks (text or images) to provide a grounded response based on user documents.`,
citationRules: `When using the RAG tool, the structure must adhere to the format described in the ReAct prompt. Below are additional guidelines specifically for RAG-based responses:
1. **Grounded Text Guidelines**:
- Each <grounded_text> tag must correspond to exactly one citation, ensuring a one-to-one relationship.
- Always cite a **subset** of the chunk, never the full text. The citation should be as short as possible while providing the relevant information (typically one to two sentences).
- Do not paraphrase the chunk text in the citation; use the original subset directly from the chunk. IT MUST BE EXACT AND WORD FOR WORD FROM THE ORIGINAL CHUNK!
- If multiple citations are needed for different sections of the response, create new <grounded_text> tags for each.
- !!!IMPORTANT: For video transcript citations, use a subset of the exact text from the transcript as the citation content. It should be just before the start of the section of the transcript that is relevant to the grounded_text tag.
2. **Citation Guidelines**:
- The citation must include only the relevant excerpt from the chunk being referenced.
- Use unique citation indices and reference the chunk_id for the source of the information.
- For text chunks, the citation content must reflect the **exact subset** of the original chunk that is relevant to the grounded_text tag.
**Example**:
<answer>
<grounded_text citation_index="1">
Artificial Intelligence is revolutionizing various sectors, with healthcare seeing transformations in diagnosis and treatment planning.
</grounded_text>
<grounded_text citation_index="2">
Based on recent data, AI has drastically improved mammogram analysis, achieving 99% accuracy at a rate 30 times faster than human radiologists.
</grounded_text>
<citations>
<citation index="1" chunk_id="abc123" type="text">Artificial Intelligence is revolutionizing various industries, especially in healthcare.</citation>
<citation index="2" chunk_id="abc124" type="table"></citation>
</citations>
<follow_up_questions>
<question>How can AI enhance patient outcomes in fields outside radiology?</question>
<question>What are the challenges in implementing AI systems across different hospitals?</question>
<question>How might AI-driven advancements impact healthcare costs?</question>
</follow_up_questions>
</answer>
***NOTE***:
- !!!IMPORTANT: Prefer to cite visual elements (i.e. table, chart, image etc.) over text, if they both can be used. Only if a visual element is not going to be helpful, then use text. Otherwise, use a visual element!
- Use as many citations as possible (even when one would be sufficient), thus keeping text as grounded as possible.
- When using text citations, keep the EXACT TEXT FROM THE CHUNK—WORD FOR WORD—DO NOT EMIT ANYTHING OR ADD ANYTHING. DO NOT PARAPHRASE! DO NOT CITE TEXT CONTENT FROM A TABLE OR IMAGE—INSTEAD CITE THE TABLE OR IMAGE ITSELF!
- Cite from as many documents as possible and always use MORE, and as granular, citations as possible.
- If you see a table in an image of another table (it is on the same page, but not the table that is circled in red), use the RAG tool again and specifically try to search for that table by using, as the hypothetical_document_chunk, a summary of the table you are trying to find. DO NOT JUST CITE THE TABLE IN RED ON THE SAME PAGE!
- CITATION TEXT MUST BE EXACTLY AS IT APPEARS IN THE CHUNK. DO NOT PARAPHRASE!`,
parameterRules: ragToolParams,
};
export class RAGTool extends BaseTool<RAGToolParamsType> {
constructor(private vectorstore: Vectorstore) {
super(ragToolInfo);
}
async execute(args: ParametersType<RAGToolParamsType>): Promise<Observation[]> {
const relevantChunks = await this.vectorstore.retrieve(args.hypothetical_document_chunk, undefined, args.doc_ids ?? undefined);
const formattedChunks = await this.getFormattedChunks(relevantChunks);
return formattedChunks;
}
async getFormattedChunks(relevantChunks: RAGChunk[]): Promise<Observation[]> {
try {
const { formattedChunks } = (await Networking.PostToServer('/formatChunks', { relevantChunks })) as { formattedChunks: Observation[] };
if (!formattedChunks) {
throw new Error('Failed to format chunks');
}
return formattedChunks;
} catch (error) {
console.error('Error formatting chunks:', error);
throw error;
}
}
}
|
