trying new image generation plus new implementaion of video and audio

1 files changed, 74 insertions, 0 deletions

diff --git a/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts
new file mode 100644
index 000000000..cf9e8cfc8
--- /dev/null
+++ b/src/client/views/nodes/chatbot/tools/ImageCreationTool.ts
@@ -0,0 +1,74 @@
+import { v4 as uuidv4 } from 'uuid';
+import { Networking } from '../../../../Network';
+import { BaseTool } from './BaseTool';
+import { Observation } from '../types/types';
+import { ParametersType, ToolInfo } from '../types/tool_types';
+import { DocumentOptions } from '../../../../documents/Documents';
+
+const imageCreationToolParams = [
+    {
+        name: 'image_prompt',
+        type: 'string',
+        description: 'The prompt for the image to be created. This should be a string that describes the image to be created in extreme detail for an AI image generator.',
+        required: true,
+    },
+] as const;
+
+type ImageCreationToolParamsType = typeof imageCreationToolParams;
+
+const imageCreationToolInfo: ToolInfo<ImageCreationToolParamsType> = {
+    name: 'imageCreationTool',
+    citationRules: 'No citation needed. Cannot cite image generation for a response.',
+    parameterRules: imageCreationToolParams,
+    description: 'Create an image of any style, content, or design, based on a prompt. The prompt should be a detailed description of the image to be created.',
+};
+
+export class ImageCreationTool extends BaseTool<ImageCreationToolParamsType> {
+    private _addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void;
+    constructor(addLinkedDoc: (doc_type: string, data: string | undefined, options: DocumentOptions, id: string) => void) {
+        super(imageCreationToolInfo);
+        this._addLinkedDoc = addLinkedDoc;
+    }
+
+    async execute(args: ParametersType<ImageCreationToolParamsType>): Promise<Observation[]> {
+        const image_prompt = args.image_prompt;
+
+        console.log(`Generating image for prompt: ${image_prompt}`);
+        // Create an array of promises, each one handling a search for a query
+        try {
+            try {
+                const { image_url } = await Networking.PostToServer('/generateImage', {
+                    image_prompt,
+                });
+                if (res) {
+                    const result = await Networking.PostToServer('/uploadRemoteImage', { sources: res });
+                    const source = ClientUtils.prepend(result[0].accessPaths.agnostic.client);
+                    return source;
+                }
+            } catch (e) {
+                console.log(e);
+            }
+
+            const { base64_data, image_path } = await Networking.PostToServer('/generateImage', {
+                image_prompt,
+            });
+            const id = uuidv4();
+
+            this._addLinkedDoc('image', image_path, {}, id);
+            return [
+                {
+                    type: 'image_url',
+                    image_url: { url: `data:image/jpeg;base64,${base64_data}` },
+                },
+            ];
+        } catch (error) {
+            console.log(error);
+            return [
+                {
+                    type: 'text',
+                    text: `An error occurred while generating image.`,
+                },
+            ];
+        }
+    }
+}