diff options
Diffstat (limited to 'src/server/ApiManagers')
| -rw-r--r-- | src/server/ApiManagers/AssistantManager.ts | 58 | ||||
| -rw-r--r-- | src/server/ApiManagers/DataVizManager.ts | 2 | ||||
| -rw-r--r-- | src/server/ApiManagers/FireflyManager.ts | 407 | ||||
| -rw-r--r-- | src/server/ApiManagers/UploadManager.ts | 35 |
4 files changed, 456 insertions, 46 deletions
diff --git a/src/server/ApiManagers/AssistantManager.ts b/src/server/ApiManagers/AssistantManager.ts index c5ba4b830..c41f697db 100644 --- a/src/server/ApiManagers/AssistantManager.ts +++ b/src/server/ApiManagers/AssistantManager.ts @@ -9,28 +9,22 @@ */ import { Readability } from '@mozilla/readability'; -import axios, { AxiosResponse } from 'axios'; +import axios from 'axios'; import { spawn } from 'child_process'; import * as fs from 'fs'; import { writeFile } from 'fs'; import { google } from 'googleapis'; import { JSDOM } from 'jsdom'; +import OpenAI from 'openai'; import * as path from 'path'; import * as puppeteer from 'puppeteer'; import { promisify } from 'util'; import * as uuid from 'uuid'; import { AI_Document } from '../../client/views/nodes/chatbot/types/types'; +import { DashUploadUtils } from '../DashUploadUtils'; import { Method } from '../RouteManager'; import { filesDirectory, publicDirectory } from '../SocketData'; import ApiManager, { Registration } from './ApiManager'; -import { getServerPath } from '../../client/util/reportManager/reportManagerUtils'; -import { file } from 'jszip'; -import ffmpegInstaller from '@ffmpeg-installer/ffmpeg'; -import ffmpeg from 'fluent-ffmpeg'; -import OpenAI from 'openai'; -import * as xmlbuilder from 'xmlbuilder'; -import { last } from 'lodash'; -import { DashUploadUtils } from '../DashUploadUtils'; // Enumeration of directories where different file types are stored export enum Directory { @@ -133,8 +127,6 @@ export default class AssistantManager extends ApiManager { const { query, max_results } = req.body; const MIN_VALID_RESULTS_RATIO = 0.75; // 3/4 threshold let startIndex = 1; // Start at the first result initially - let validResults: any[] = []; - const fetchSearchResults = async (start: number) => { return customsearch.cse.list({ q: query, @@ -146,20 +138,27 @@ export default class AssistantManager extends ApiManager { }); }; - const filterResultsByXFrameOptions = async (results: any[]) => { + const filterResultsByXFrameOptions = async ( + results: { + url: string | null | undefined; + snippet: string | null | undefined; + }[] + ) => { const filteredResults = await Promise.all( - results.map(async result => { - try { - const urlResponse: AxiosResponse = await axios.head(result.url, { timeout: 5000 }); - const xFrameOptions = urlResponse.headers['x-frame-options']; - if (xFrameOptions && xFrameOptions.toUpperCase() === 'SAMEORIGIN') { - return result; + results + .filter(result => result.url) + .map(async result => { + try { + const urlResponse = await axios.head(result.url!, { timeout: 5000 }); + const xFrameOptions = urlResponse.headers['x-frame-options']; + if (xFrameOptions && xFrameOptions.toUpperCase() === 'SAMEORIGIN') { + return result; + } + } catch (error) { + console.error(`Error checking x-frame-options for URL: ${result.url}`, error); } - } catch (error) { - console.error(`Error checking x-frame-options for URL: ${result.url}`, error); - } - return null; // Exclude the result if it doesn't match - }) + return null; // Exclude the result if it doesn't match + }) ); return filteredResults.filter(result => result !== null); // Remove null results }; @@ -167,14 +166,14 @@ export default class AssistantManager extends ApiManager { try { // Fetch initial search results let response = await fetchSearchResults(startIndex); - let initialResults = + const initialResults = response.data.items?.map(item => ({ url: item.link, snippet: item.snippet, })) || []; // Filter the initial results - validResults = await filterResultsByXFrameOptions(initialResults); + let validResults = await filterResultsByXFrameOptions(initialResults); // If valid results are less than 3/4 of max_results, fetch more results while (validResults.length < max_results * MIN_VALID_RESULTS_RATIO) { @@ -288,7 +287,7 @@ export default class AssistantManager extends ApiManager { // Step 2: Transcribe audio using OpenAI Whisper console.log('Transcribing audio...'); const transcription = await openai.audio.transcriptions.create({ - file: fs.createReadStream(audioPath) as any, + file: fs.createReadStream(audioPath), model: 'whisper-1', response_format: 'verbose_json', timestamp_granularities: ['segment'], @@ -298,7 +297,7 @@ export default class AssistantManager extends ApiManager { // Step 3: Extract concise JSON console.log('Extracting concise JSON...'); - const originalSegments = transcription.segments?.map((segment: any, index: number) => ({ + const originalSegments = transcription.segments?.map((segment, index) => ({ index: index.toString(), text: segment.text, start: segment.start, @@ -767,7 +766,7 @@ function spawnPythonProcess(jobId: string, file_name: string, file_path: string) try { const errorOutput = JSON.parse(lastLine); jobResults[jobId] = errorOutput; - } catch (err) { + } catch { jobResults[jobId] = { error: 'Python process failed' }; } } else { @@ -829,6 +828,3 @@ function spawnPythonProcess(jobId: string, file_name: string, file_path: string) runPythonScript(); } } -function customFfmpeg(filePath: string) { - throw new Error('Function not implemented.'); -} diff --git a/src/server/ApiManagers/DataVizManager.ts b/src/server/ApiManagers/DataVizManager.ts index 88f22992d..d2028f23b 100644 --- a/src/server/ApiManagers/DataVizManager.ts +++ b/src/server/ApiManagers/DataVizManager.ts @@ -9,7 +9,7 @@ export default class DataVizManager extends ApiManager { register({ method: Method.GET, subscription: '/csvData', - secureHandler: async ({ req, res }) => { + secureHandler: ({ req, res }) => { const uri = req.query.uri as string; return new Promise<void>(resolve => { diff --git a/src/server/ApiManagers/FireflyManager.ts b/src/server/ApiManagers/FireflyManager.ts new file mode 100644 index 000000000..160a94d40 --- /dev/null +++ b/src/server/ApiManagers/FireflyManager.ts @@ -0,0 +1,407 @@ +import axios from 'axios'; +import { Dropbox } from 'dropbox'; +import * as fs from 'fs'; +import * as multipart from 'parse-multipart-data'; +import * as path from 'path'; +import { DashUserModel } from '../authentication/DashUserModel'; +import { DashUploadUtils } from '../DashUploadUtils'; +import { _error, _invalid, _success, Method } from '../RouteManager'; +import { Directory, filesDirectory } from '../SocketData'; +import ApiManager, { Registration } from './ApiManager'; + +export default class FireflyManager extends ApiManager { + getBearerToken = () => + fetch('https://ims-na1.adobelogin.com/ims/token/v3', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: `grant_type=client_credentials&client_id=${process.env._CLIENT_FIREFLY_CLIENT_ID}&client_secret=${process.env._CLIENT_FIREFLY_SECRET}&scope=openid,AdobeID,session,additional_info,read_organizations,firefly_api,ff_apis`, + }).catch(error => { + console.error('Error:', error); + return undefined; + }); + + generateImageFromStructure = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, structureUrl: string, strength: number = 50, styles: string[], styleUrl: string | undefined) => + this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => + //prettier-ignore + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: JSON.stringify({ + prompt, + numVariations: 4, + detailLevel: 'preview', + modelVersion: 'image3_fast', + size: { width, height }, + structure: !structureUrl + ? undefined + : { + strength, + imageReference: { + source: { url: structureUrl }, + }, + }, + // prettier-ignore + style: { + presets: styles, + imageReference : !styleUrl + ? undefined + : { + source: { url: styleUrl }, + } + } + }), + }) + .then(response2 => response2.json().then(json => + { + if (json.outputs?.length) + return (json.outputs as {image: {url:string }}[]).map(output => output.image); + throw new Error(JSON.stringify(json)); + }) + ) + ) + ); + + uploadImageToDropbox = (fileUrl: string, user: DashUserModel | undefined, dbx = new Dropbox({ accessToken: user?.dropboxToken || '' })) => + new Promise<string | Error>((res, rej) => + fs.readFile(path.join(filesDirectory, `${Directory.images}/${path.basename(fileUrl)}`), undefined, (err, contents) => { + if (err) { + console.log('Error: ', err); + rej(); + } else { + dbx.filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }) + .then(response => { + dbx.filesGetTemporaryLink({ path: response.result.path_display ?? '' }) + .then(link => res(link.result.link)) + .catch(e => res(new Error(e.toString()))); + }) + .catch(e => { + if (user?.dropboxRefresh) { + console.log('*********** try refresh dropbox for: ' + user.email + ' ***********'); + this.refreshDropboxToken(user).then(token => { + if (!token) { + console.log('Dropbox error: cannot refresh token'); + res(new Error(e.toString())); + } else { + const dbxNew = new Dropbox({ accessToken: user.dropboxToken || '' }); + dbxNew + .filesUpload({ path: `/Apps/browndash/${path.basename(fileUrl)}`, contents }) + .then(response => { + dbxNew + .filesGetTemporaryLink({ path: response.result.path_display ?? '' }) + .then(link => res(link.result.link)) + .catch(linkErr => res(new Error(linkErr.toString()))); + }) + .catch(uploadErr => { + console.log('Dropbox error:', uploadErr); + res(new Error(uploadErr.toString())); + }); + } + }); + } else { + console.log('Dropbox error:', e); + res(new Error(e.toString())); + } + }); + } + }) + ); + + generateImage = (prompt: string = 'a realistic illustration of a cat coding', width: number = 2048, height: number = 2048, seed?: number) => { + let body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}} }`; + if (seed) { + console.log('RECEIVED SEED', seed); + body = `{ "prompt": "${prompt}", "size": { "width": ${width}, "height": ${height}}, "seeds": [${seed}]}`; + } + const fetched = this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => + fetch('https://firefly-api.adobe.io/v3/images/generate', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: body, + }) + .then(response2 => response2.json().then(json => ({ seed: json.outputs?.[0]?.seed, url: json.outputs?.[0]?.image?.url }))) + .catch(error => { + console.error('Error:', error); + return undefined; + }) + ) + ); + return fetched; + }; + expandImage = (imgUrl: string, prompt?: string) => { + const dropboxImgUrl = imgUrl; + const fetched = this.getBearerToken().then(response => + response + ?.json() + .then((data: { access_token: string }) => { + return fetch('https://firefly-api.adobe.io/v3/images/expand', { + method: 'POST', + headers: [ + ['Content-Type', 'application/json'], + ['Accept', 'application/json'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + ['Authorization', `Bearer ${data.access_token}`], + ], + body: JSON.stringify({ + image: { + source: { + url: dropboxImgUrl, + }, + }, + numVariations: 1, + seeds: [0], + size: { + width: 3048, + height: 2048, + }, + prompt: prompt ?? 'cloudy skies', + placement: { + inset: { + left: 0, + top: 0, + right: 0, + bottom: 0, + }, + alignment: { + horizontal: 'center', + vertical: 'center', + }, + }, + }), + }); + }) + .then(resp => resp.json()) + ); + return fetched; + }; + getImageText = (imageBlob: Blob) => { + const inputFileVarName = 'infile'; + const outputVarName = 'result'; + const fetched = this.getBearerToken().then(response => + response?.json().then((data: { access_token: string }) => { + return fetch('https://sensei.adobe.io/services/v2/predict', { + method: 'POST', + headers: [ + ['Prefer', 'respond-async, wait=59'], + ['x-api-key', process.env._CLIENT_FIREFLY_CLIENT_ID ?? ''], + // ['content-type', 'multipart/form-data'], // bcz: Don't set this!! content-type will get set automatically including the Boundary string + ['Authorization', `Bearer ${data.access_token}`], + ], + body: ((form: FormData) => { + form.set(inputFileVarName, imageBlob); + form.set( + 'contentAnalyzerRequests', + JSON.stringify({ + 'sensei:name': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + 'sensei:invocation_mode': 'asynchronous', + 'sensei:invocation_batch': false, + 'sensei:engines': [ + { + 'sensei:execution_info': { + 'sensei:engine': 'Feature:cintel-object-detection:Service-b9ace8b348b6433e9e7d82371aa16690', + }, + 'sensei:inputs': { + documents: [ + { + 'sensei:multipart_field_name': inputFileVarName, + 'dc:format': 'image/png', + }, + ], + }, + 'sensei:params': { + correct_with_dictionary: true, + }, + 'sensei:outputs': { + result: { + 'sensei:multipart_field_name': outputVarName, + 'dc:format': 'application/json', + }, + }, + }, + ], + }) + ); + return form; + })(new FormData()), + }).then(response2 => { + const contentType = response2.headers.get('content-type') ?? ''; + if (contentType.includes('application/json')) { + return response2.json().then((json: object) => JSON.stringify(json)); + } + if (contentType.includes('multipart')) { + return response2 + .arrayBuffer() + .then(arrayBuffer => + multipart + .parse(Buffer.from(arrayBuffer), 'Boundary' + (response2.headers.get('content-type')?.match(/=Boundary(.*);/)?.[1] ?? '')) + .filter(part => part.name === outputVarName) + .map(part => JSON.parse(part.data.toString())[0]) + .reduce((text, json) => text + (json?.is_text_present ? json.tags.map((tag: { text: string }) => tag.text).join(' ') : ''), '') + ) + .catch(error => { + console.error('Error:', error); + return ''; + }); + } + return response2.text(); + }); + }) + ); + return fetched; + }; + + refreshDropboxToken = (user: DashUserModel) => + axios + .post( + 'https://api.dropbox.com/oauth2/token', + new URLSearchParams({ + refresh_token: user.dropboxRefresh || '', + grant_type: 'refresh_token', + client_id: process.env._CLIENT_DROPBOX_CLIENT_ID || '', + client_secret: process.env._CLIENT_DROPBOX_SECRET || '', + }).toString() + ) + .then(refresh => { + console.log('***** dropbox token refreshed for ' + user?.email + ' ******* '); + user.dropboxToken = refresh.data.access_token; + user.save(); + return user.dropboxToken; + }) + .catch(e => { + console.log(e); + return undefined; + }); + + protected initialize(register: Registration): void { + register({ + method: Method.POST, + subscription: '/queryFireflyImageFromStructure', + secureHandler: ({ req, res }) => + new Promise<void>(resolver => { + (req.body.styleUrl ? this.uploadImageToDropbox(req.body.styleUrl, req.user as DashUserModel) : Promise.resolve(undefined)) + .then(styleUrl => { + if (styleUrl instanceof Error) { + _invalid(res, styleUrl.message); + throw new Error('Error uploading images to dropbox'); + } + this.uploadImageToDropbox(req.body.structure, req.user as DashUserModel) + .then(structureUrl => { + if (structureUrl instanceof Error) { + _invalid(res, structureUrl.message); + throw new Error('Error uploading images to dropbox'); + } + return { styleUrl, structureUrl }; + }) + .then(uploads => + this.generateImageFromStructure(req.body.prompt, req.body.width, req.body.height, uploads.structureUrl, req.body.strength, req.body.presets, uploads.styleUrl) + .then(images => { + Promise.all((images ?? [new Error('no images were generated')]).map(fire => (fire instanceof Error ? fire : DashUploadUtils.UploadImage(fire.url)))) + .then(dashImages => { + if (dashImages.every(img => img instanceof Error)) _invalid(res, dashImages[0]!.message); + else _success(res, JSON.stringify(dashImages.filter(img => !(img instanceof Error)))); + }) + .then(resolver); + }) + .catch(e => { + _invalid(res, e.message); + resolver(); + }) + ); + }) + .catch(() => { + /* do nothing */ + resolver(); + }); + }), + }); + register({ + method: Method.POST, + subscription: '/queryFireflyImage', + secureHandler: ({ req, res }) => + this.generateImage(req.body.prompt, req.body.width, req.body.height, req.body.seed).then(img => + DashUploadUtils.UploadImage(img?.url ?? '', undefined, img?.seed).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }) + ), + }); + + register({ + method: Method.POST, + subscription: '/queryFireflyImageText', + secureHandler: ({ req, res }) => + fetch(req.body.file).then(json => + json.blob().then(file => + this.getImageText(file).then(text => { + _success(res, text); + }) + ) + ), + }); + register({ + method: Method.POST, + subscription: '/expandImage', + secureHandler: ({ req, res }) => + this.uploadImageToDropbox(req.body.file, req.user as DashUserModel).then(uploadUrl => + uploadUrl instanceof Error + ? _invalid(res, uploadUrl.message) + : this.expandImage(uploadUrl, req.body.prompt).then(text => { + if (text.error_code) _error(res, text.message); + else + DashUploadUtils.UploadImage(text.outputs[0].image.url).then(info => { + if (info instanceof Error) _invalid(res, info.message); + else _success(res, info); + }); + }) + ), + }); + + // construct this url and send user to it. It will allow them to authorize their dropbox account and will send the resulting token to our endpoint /refreshDropbox + // https://www.dropbox.com/oauth2/authorize?client_id=DROPBOX_CLIENT_ID&response_type=code&token_access_type=offline&redirect_uri=http://localhost:1050/refreshDropbox + // see: https://dropbox.tech/developers/using-oauth-2-0-with-offline-access + // + register({ + method: Method.GET, + subscription: '/refreshDropbox', + secureHandler: ({ req, res }) => { + const user = req.user as DashUserModel; + console.log(`******************* dropbox authorized for ${user?.email} ******************`); + _success(res, 'dropbox authorized for ' + user?.email); + + const data = new URLSearchParams({ + code: req.query.code as string, + grant_type: 'authorization_code', + client_id: process.env._CLIENT_DROPBOX_CLIENT_ID ?? '', + client_secret: process.env._CLIENT_DROPBOX_SECRET ?? '', + redirect_uri: 'http://localhost:1050/refreshDropbox', + }); + axios + .post('https://api.dropbox.com/oauth2/token', data.toString()) + .then(response => { + console.log('***** dropbox token (and refresh) received for ' + user?.email + ' ******* '); + user.dropboxToken = response.data.access_token; + user.dropboxRefresh = response.data.refresh_token; + user.save(); + + setTimeout(() => this.refreshDropboxToken(user), response.data.expires_in - 600); + }) + .catch(e => { + console.log(e); + }); + }, + }); + } +} diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 868373474..c9d5df547 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -70,10 +70,16 @@ export default class UploadManager extends ApiManager { ]); } else { fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`)); + // original filenames with '.'s, such as a Macbook screenshot, can be a problem - their extension is not kept in formidable's newFilename. + // This makes sure that the extension is preserved in the newFilename. + const fixNewFilename = (f: formidable.File) => { + if (path.extname(f.originalFilename ?? '') !== path.extname(f.newFilename)) f.newFilename = f.newFilename + path.extname(f.originalFilename ?? ''); + return f; + }; const results = ( await Promise.all( Array.from(Object.keys(files)).map( - async key => (!files[key] ? undefined : DashUploadUtils.upload(files[key]![0] /* , key */)) // key is the guid used by the client to track upload progress. + async key => (!files[key] ? undefined : DashUploadUtils.upload(fixNewFilename(files[key][0]) /* , key */)) // key is the guid used by the client to track upload progress. ) ) ).filter(result => result && !(result.result instanceof Error)); @@ -147,13 +153,10 @@ export default class UploadManager extends ApiManager { if (doc.id) { doc.id = getId(doc.id); } - // eslint-disable-next-line no-restricted-syntax for (const key in doc.fields) { - // eslint-disable-next-line no-continue if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue; const field = doc.fields[key]; - // eslint-disable-next-line no-continue if (field === undefined || field === null) continue; if (field.__type === 'Doc') { @@ -182,11 +185,9 @@ export default class UploadManager extends ApiManager { let docids: string[] = []; let linkids: string[] = []; try { - // eslint-disable-next-line no-restricted-syntax for (const name in files) { if (Object.prototype.hasOwnProperty.call(files, name)) { const f = files[name]; - // eslint-disable-next-line no-continue if (!f) continue; const path2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set? const zip = new AdmZip(path2.filepath); @@ -273,14 +274,20 @@ export default class UploadManager extends ApiManager { .filter(f => regex.test(f)) .map(f => fs.unlinkSync(serverPath + f)); } - imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => { - const ext = path.extname(savedName).toLowerCase(); - const outputPath = serverPathToFile(Directory.images, filename + ext); - if (AcceptableMedia.imageFormats.includes(ext)) { - workerResample(savedName, outputPath, origSuffix, false); - } - res.send(clientPathToFile(Directory.images, filename + ext)); - }); + imageDataUri + .outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))) + .then((savedName: string) => { + const ext = path.extname(savedName).toLowerCase(); + const outputPath = serverPathToFile(Directory.images, filename + ext); + if (AcceptableMedia.imageFormats.includes(ext)) { + workerResample(savedName, outputPath, origSuffix, false); + } + res.send(clientPathToFile(Directory.images, filename + ext)); + }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .catch((e: any) => { + res.status(404).json({ error: e.toString() }); + }); }, }); } |
