From 6f9b8f9b393d411a17f7954b6cc36618efe698e2 Mon Sep 17 00:00:00 2001
From: "A.J. Shulman" <Shulman.aj@gmail.com>
Date: Thu, 15 Aug 2024 13:16:32 -0400
Subject: implemented search tool and other tools but scraping doesn't work

---
 .../nodes/ChatBox/tools/WebsiteInfoScraperTool.ts  | 35 ++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts

(limited to 'src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts')

diff --git a/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts b/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts
new file mode 100644
index 000000000..59fd47b7a
--- /dev/null
+++ b/src/client/views/nodes/ChatBox/tools/WebsiteInfoScraperTool.ts
@@ -0,0 +1,35 @@
+import { Networking } from '../../../../Network';
+import { BaseTool } from './BaseTool';
+import { v4 as uuidv4 } from 'uuid';
+
+export class WebsiteInfoScraperTool extends BaseTool<{ url: string }> {
+    private _addLinkedUrlDoc: (url: string, id: string) => void;
+
+    constructor(addLinkedUrlDoc: (url: string, id: string) => void) {
+        super(
+            'websiteInfoScraper',
+            'Scrape detailed information from a specific website identified as the most relevant',
+            {
+                url: {
+                    type: 'string',
+                    description: 'The URL of the website to scrape',
+                    required: true,
+                },
+            },
+            'Provide the URL of the website that you have identified as the most relevant from the previous search. This tool will scrape and process detailed information from that specific website. It will also create a document from the scraped content for future reference.',
+            'Returns the full HTML content from the provided URL and creates a document from the content for further analysis.'
+        );
+        this._addLinkedUrlDoc = addLinkedUrlDoc;
+    }
+
+    async execute(args: { url: string }): Promise<any> {
+        try {
+            const { html } = await Networking.PostToServer('/scrapeWebsite', { url: args.url });
+            const id = uuidv4();
+            this._addLinkedUrlDoc(args.url, id);
+            return [{ type: 'text', text: `<chunk chunk_id=${id} chunk_type=text> ${html} </chunk>` }];
+        } catch (error) {
+            return [{ type: 'text', text: 'An error occurred while scraping the website.' }];
+        }
+    }
+}
-- 
cgit v1.2.3-70-g09d2