diff options
Diffstat (limited to 'src/server')
46 files changed, 2450 insertions, 2067 deletions
diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts index 55b50cc12..520ebb42e 100644 --- a/src/server/ActionUtilities.ts +++ b/src/server/ActionUtilities.ts @@ -1,14 +1,14 @@ import { exec } from 'child_process'; import { Color, yellow } from 'colors'; import { createWriteStream, exists, mkdir, readFile, unlink, writeFile } from 'fs'; -import * as nodemailer from "nodemailer"; -import { MailOptions } from "nodemailer/lib/json-transport"; +import * as nodemailer from 'nodemailer'; +import { MailOptions } from 'nodemailer/lib/json-transport'; import * as path from 'path'; -import { rimraf } from "rimraf"; +import { rimraf } from 'rimraf'; import { ExecOptions } from 'shelljs'; import * as Mail from 'nodemailer/lib/mailer'; -const projectRoot = path.resolve(__dirname, "../../"); +const projectRoot = path.resolve(__dirname, '../../'); export function pathFromRoot(relative?: string) { if (!relative) { return projectRoot; @@ -16,36 +16,37 @@ export function pathFromRoot(relative?: string) { return path.resolve(projectRoot, relative); } -export async function fileDescriptorFromStream(path: string) { - const logStream = createWriteStream(path); - return new Promise<number>(resolve => logStream.on("open", resolve)); +export async function fileDescriptorFromStream(filePath: string) { + const logStream = createWriteStream(filePath); + return new Promise<number>(resolve => { + logStream.on('open', resolve); + }); } -export const command_line = (command: string, fromDirectory?: string) => { - return new Promise<string>((resolve, reject) => { +export const commandLine = (command: string, fromDirectory?: string) => + new Promise<string>((resolve, reject) => { const options: ExecOptions = {}; if (fromDirectory) { options.cwd = fromDirectory ? path.resolve(projectRoot, fromDirectory) : projectRoot; } - exec(command, options, (err, stdout) => err ? reject(err) : resolve(stdout)); + exec(command, options, (err, stdout) => (err ? reject(err) : resolve(stdout))); }); -}; -export const read_text_file = (relativePath: string) => { +export const readTextFile = (relativePath: string) => { const target = path.resolve(__dirname, relativePath); return new Promise<string>((resolve, reject) => { - readFile(target, (err, data) => err ? reject(err) : resolve(data.toString())); + readFile(target, (err, data) => (err ? reject(err) : resolve(data.toString()))); }); }; -export const write_text_file = (relativePath: string, contents: any) => { +export const writeTextFile = (relativePath: string, contents: any) => { const target = path.resolve(__dirname, relativePath); return new Promise<void>((resolve, reject) => { - writeFile(target, contents, (err) => err ? reject(err) : resolve()); + writeFile(target, contents, err => (err ? reject(err) : resolve())); }); }; -export type Messager<T> = (outcome: { result: T | undefined, error: Error | null }) => string; +export type Messager<T> = (outcome: { result: T | undefined; error: Error | null }) => string; export interface LogData<T> { startMessage: string; @@ -55,70 +56,80 @@ export interface LogData<T> { color?: Color; } +function logHelper(content: string, color: Color | string) { + if (typeof color === 'string') { + console.log(color, content); + } else { + console.log(color(content)); + } +} + let current = Math.ceil(Math.random() * 20); -export async function log_execution<T>({ startMessage, endMessage, action, color }: LogData<T>): Promise<T | undefined> { - let result: T | undefined = undefined, error: Error | null = null; - const resolvedColor = color || `\x1b[${31 + ++current % 6}m%s\x1b[0m`; - log_helper(`${startMessage}...`, resolvedColor); +export async function logExecution<T>({ startMessage, endMessage, action, color }: LogData<T>): Promise<T | undefined> { + let result: T | undefined; + let error: Error | null = null; + const resolvedColor = color || `\x1b[${31 + (++current % 6)}m%s\x1b[0m`; + logHelper(`${startMessage}...`, resolvedColor); try { result = await action(); } catch (e: any) { error = e; } finally { - log_helper(typeof endMessage === "string" ? endMessage : endMessage({ result, error }), resolvedColor); + logHelper(typeof endMessage === 'string' ? endMessage : endMessage({ result, error }), resolvedColor); } return result; } - -function log_helper(content: string, color: Color | string) { - if (typeof color === "string") { - console.log(color, content); - } else { - console.log(color(content)); - } -} - export function logPort(listener: string, port: number) { console.log(`${listener} listening on port ${yellow(String(port))}`); } export function msToTime(duration: number) { - const milliseconds = Math.floor((duration % 1000) / 100), - seconds = Math.floor((duration / 1000) % 60), - minutes = Math.floor((duration / (1000 * 60)) % 60), - hours = Math.floor((duration / (1000 * 60 * 60)) % 24); + const milliseconds = Math.floor((duration % 1000) / 100); + const seconds = Math.floor((duration / 1000) % 60); + const minutes = Math.floor((duration / (1000 * 60)) % 60); + const hours = Math.floor((duration / (1000 * 60 * 60)) % 24); - const hoursS = (hours < 10) ? "0" + hours : hours; - const minutesS = (minutes < 10) ? "0" + minutes : minutes; - const secondsS = (seconds < 10) ? "0" + seconds : seconds; + const hoursS = hours < 10 ? '0' + hours : hours; + const minutesS = minutes < 10 ? '0' + minutes : minutes; + const secondsS = seconds < 10 ? '0' + seconds : seconds; - return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds; + return hoursS + ':' + minutesS + ':' + secondsS + '.' + milliseconds; } -export const createIfNotExists = async (path: string) => { - if (await new Promise<boolean>(resolve => exists(path, resolve))) { +export const createIfNotExists = async (filePath: string) => { + if ( + await new Promise<boolean>(resolve => { + exists(filePath, resolve); + }) + ) { return true; } - return new Promise<boolean>(resolve => mkdir(path, error => resolve(error === null))); + return new Promise<boolean>(resolve => { + mkdir(filePath, error => resolve(error === null)); + }); }; export async function Prune(rootDirectory: string): Promise<boolean> { // const error = await new Promise<Error>(resolve => rimraf(rootDirectory).then(resolve)); - await new Promise<void>(resolve => rimraf(rootDirectory).then(() => resolve())); + await new Promise<void>(resolve => { + rimraf(rootDirectory).then(() => resolve()); + }); // return error === null; return true; } -export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => unlink(mediaPath, error => resolve(error === null))); +export const Destroy = (mediaPath: string) => + new Promise<boolean>(resolve => { + unlink(mediaPath, error => resolve(error === null)); + }); export namespace Email { - const smtpTransport = nodemailer.createTransport({ service: 'Gmail', auth: { user: 'browndashptc@gmail.com', - pass: 'TsarNicholas#2' - } + pass: 'TsarNicholas#2', + }, }); export interface DispatchOptions<T extends string | string[]> { @@ -135,16 +146,18 @@ export namespace Email { export async function dispatchAll({ to, subject, content, attachments }: DispatchOptions<string[]>) { const failures: DispatchFailure[] = []; - await Promise.all(to.map(async recipient => { - let error: Error | null; - const resolved = attachments ? "length" in attachments ? attachments : [attachments] : undefined; - if ((error = await Email.dispatch({ to: recipient, subject, content, attachments: resolved })) !== null) { - failures.push({ - recipient, - error - }); - } - })); + await Promise.all( + to.map(async recipient => { + const resolved = attachments ? ('length' in attachments ? attachments : [attachments]) : undefined; + const error = await Email.dispatch({ to: recipient, subject, content, attachments: resolved }); + if (error !== null) { + failures.push({ + recipient, + error, + }); + } + }) + ); return failures.length ? failures : undefined; } @@ -153,10 +166,11 @@ export namespace Email { to, from: 'browndashptc@gmail.com', subject, - text: `Hello ${to.split("@")[0]},\n\n${content}`, - attachments + text: `Hello ${to.split('@')[0]},\n\n${content}`, + attachments, } as MailOptions; - return new Promise<Error | null>(resolve => smtpTransport.sendMail(mailOptions, resolve)); + return new Promise<Error | null>(resolve => { + smtpTransport.sendMail(mailOptions, resolve); + }); } - -}
\ No newline at end of file +} diff --git a/src/server/ApiManagers/ApiManager.ts b/src/server/ApiManagers/ApiManager.ts index 27e9de065..f55495b2e 100644 --- a/src/server/ApiManagers/ApiManager.ts +++ b/src/server/ApiManagers/ApiManager.ts @@ -1,4 +1,4 @@ -import { RouteInitializer } from "../RouteManager"; +import { RouteInitializer } from '../RouteManager'; export type Registration = (initializer: RouteInitializer) => void; @@ -8,4 +8,4 @@ export default abstract class ApiManager { public register(register: Registration) { this.initialize(register); } -}
\ No newline at end of file +} diff --git a/src/server/ApiManagers/DataVizManager.ts b/src/server/ApiManagers/DataVizManager.ts index 0d43130d1..88f22992d 100644 --- a/src/server/ApiManagers/DataVizManager.ts +++ b/src/server/ApiManagers/DataVizManager.ts @@ -1,14 +1,14 @@ -import { csvParser, csvToString } from "../DataVizUtils"; -import { Method, _success } from "../RouteManager"; -import ApiManager, { Registration } from "./ApiManager"; -import { Directory, serverPathToFile } from "./UploadManager"; import * as path from 'path'; +import { csvParser, csvToString } from '../DataVizUtils'; +import { Method, _success } from '../RouteManager'; +import { Directory, serverPathToFile } from '../SocketData'; +import ApiManager, { Registration } from './ApiManager'; export default class DataVizManager extends ApiManager { protected initialize(register: Registration): void { register({ method: Method.GET, - subscription: "/csvData", + subscription: '/csvData', secureHandler: async ({ req, res }) => { const uri = req.query.uri as string; @@ -19,8 +19,7 @@ export default class DataVizManager extends ApiManager { _success(res, parsedCsv); resolve(); }); - } + }, }); } - -}
\ No newline at end of file +} diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts index c6c4ca464..9ad334c1b 100644 --- a/src/server/ApiManagers/DeleteManager.ts +++ b/src/server/ApiManagers/DeleteManager.ts @@ -1,12 +1,12 @@ -import ApiManager, { Registration } from './ApiManager'; -import { Method, _permission_denied } from '../RouteManager'; -import { WebSocket } from '../websocket'; -import { Database } from '../database'; +import { mkdirSync } from 'fs'; import { rimraf } from 'rimraf'; -import { filesDirectory } from '..'; +import { filesDirectory } from '../SocketData'; import { DashUploadUtils } from '../DashUploadUtils'; -import { mkdirSync } from 'fs'; +import { Method } from '../RouteManager'; import RouteSubscriber from '../RouteSubscriber'; +import { Database } from '../database'; +import { WebSocket } from '../websocket'; +import ApiManager, { Registration } from './ApiManager'; export default class DeleteManager extends ApiManager { protected initialize(register: Registration): void { @@ -24,9 +24,11 @@ export default class DeleteManager extends ApiManager { switch (target) { case 'all': all = true; + // eslint-disable-next-line no-fallthrough case 'database': await WebSocket.doDelete(false); if (!all) break; + // eslint-disable-next-line no-fallthrough case 'files': rimraf.sync(filesDirectory); mkdirSync(filesDirectory); diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts index 2175b6db6..5ee21fb44 100644 --- a/src/server/ApiManagers/DownloadManager.ts +++ b/src/server/ApiManagers/DownloadManager.ts @@ -1,13 +1,13 @@ -import ApiManager, { Registration } from "./ApiManager"; -import { Method } from "../RouteManager"; -import RouteSubscriber from "../RouteSubscriber"; import * as Archiver from 'archiver'; import * as express from 'express'; -import { Database } from "../database"; -import * as path from "path"; -import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; -import { publicDirectory } from ".."; -import { serverPathToFile, Directory } from "./UploadManager"; +import * as path from 'path'; +import { URL } from 'url'; +import { DashUploadUtils, SizeSuffix } from '../DashUploadUtils'; +import { Method } from '../RouteManager'; +import RouteSubscriber from '../RouteSubscriber'; +import { Directory, publicDirectory, serverPathToFile } from '../SocketData'; +import { Database } from '../database'; +import ApiManager, { Registration } from './ApiManager'; export type Hierarchy = { [id: string]: string | Hierarchy }; export type ZipMutator = (file: Archiver.Archiver) => void | Promise<void>; @@ -16,147 +16,45 @@ export interface DocumentElements { title: string; } -export default class DownloadManager extends ApiManager { - - protected initialize(register: Registration): void { - - /** - * Let's say someone's using Dash to organize images in collections. - * This lets them export the hierarchy they've built to their - * own file system in a useful format. - * - * This handler starts with a single document id (interesting only - * if it's that of a collection). It traverses the database, captures - * the nesting of only nested images or collections, writes - * that to a zip file and returns it to the client for download. - */ - register({ - method: Method.GET, - subscription: new RouteSubscriber("imageHierarchyExport").add('docId'), - secureHandler: async ({ req, res }) => { - const id = req.params.docId; - const hierarchy: Hierarchy = {}; - await buildHierarchyRecursive(id, hierarchy); - return BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); - } - }); - - register({ - method: Method.GET, - subscription: new RouteSubscriber("downloadId").add("docId"), - secureHandler: async ({ req, res }) => { - return BuildAndDispatchZip(res, async zip => { - const { id, docs, files } = await getDocs(req.params.docId); - const docString = JSON.stringify({ id, docs }); - zip.append(docString, { name: "doc.json" }); - files.forEach(val => { - zip.file(publicDirectory + val, { name: val.substring(1) }); - }); - }); - } - }); - - register({ - method: Method.GET, - subscription: new RouteSubscriber("serializeDoc").add("docId"), - secureHandler: async ({ req, res }) => { - const { docs, files } = await getDocs(req.params.docId); - res.send({ docs, files: Array.from(files) }); - } - }); - - } - -} - -async function getDocs(id: string) { - const files = new Set<string>(); - const docs: { [id: string]: any } = {}; - const fn = (doc: any): string[] => { - const id = doc.id; - if (typeof id === "string" && id.endsWith("Proto")) { - //Skip protos - return []; - } - const ids: string[] = []; - for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { continue; } - const field = doc.fields[key]; - if (field === undefined || field === null) { continue; } - - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { - ids.push(field.fieldId); - } else if (field.__type === "script" || field.__type === "computed") { - field.captures && ids.push(field.captures.fieldId); - } else if (field.__type === "list") { - ids.push(...fn(field)); - } else if (typeof field === "string") { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; - let match: string[] | null; - while ((match = re.exec(field)) !== null) { - ids.push(match[1]); - } - } else if (field.__type === "RichTextField") { - const re = /"href"\s*:\s*"(.*?)"/g; - let match: string[] | null; - while ((match = re.exec(field.Data)) !== null) { - const urlString = match[1]; - const split = new URL(urlString).pathname.split("doc/"); - if (split.length > 1) { - ids.push(split[split.length - 1]); - } - } - const re2 = /"src"\s*:\s*"(.*?)"/g; - while ((match = re2.exec(field.Data)) !== null) { - const urlString = match[1]; - const pathname = new URL(urlString).pathname; - files.add(pathname); - } - } else if (["audio", "image", "video", "pdf", "web", "map"].includes(field.__type)) { - const url = new URL(field.url); - const pathname = url.pathname; - files.add(pathname); - } - } - - if (doc.id) { - docs[doc.id] = doc; - } - return ids; - }; - await Database.Instance.visit([id], fn); - return { id, docs, files }; -} - /** - * This utility function factors out the process - * of creating a zip file and sending it back to the client - * by piping it into a response. - * - * Learn more about piping and readable / writable streams here! - * https://www.freecodecamp.org/news/node-js-streams-everything-you-need-to-know-c9141306be93/ - * - * @param res the writable stream response object that will transfer the generated zip file - * @param mutator the callback function used to actually modify and insert information into the zip instance + * This is a very specific utility method to help traverse the database + * to parse data and titles out of images and collections alone. + * + * We don't know if the document id given to is corresponds to a view document or a data + * document. If it's a data document, the response from the database will have + * a data field. If not, call recursively on the proto, and resolve with *its* data + * + * @param targetId the id of the Dash document whose data is being requests + * @returns the data of the document, as well as its title */ -export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMutator): Promise<void> { - res.set('Content-disposition', `attachment;`); - res.set('Content-Type', "application/zip"); - const zip = Archiver('zip'); - zip.pipe(res); - await mutator(zip); - return zip.finalize(); +async function getData(targetId: string): Promise<DocumentElements> { + return new Promise<DocumentElements>((resolve, reject) => { + Database.Instance.getDocument(targetId, async (result: any) => { + const { data, proto, title } = result.fields; + if (data) { + if (data.url) { + resolve({ data: data.url, title }); + } else if (data.fields) { + resolve({ data: data.fields, title }); + } else { + reject(); + } + } else if (proto) { + getData(proto.fieldId).then(resolve, reject); + } else { + reject(); + } + }); + }); } /** * This function starts with a single document id as a seed, * typically that of a collection, and then descends the entire tree - * of image or collection documents that are reachable from that seed. + * of image or collection documents that are reachable from that seed. * @param seedId the id of the root of the subtree we're trying to capture, interesting only if it's a collection * @param hierarchy the data structure we're going to use to record the nesting of the collections and images as we descend - */ - -/* + Below is an example of the JSON hierarchy built from two images contained inside a collection titled 'a nested collection', following the general recursive structure shown immediately below { @@ -190,74 +88,175 @@ async function buildHierarchyRecursive(seedId: string, hierarchy: Hierarchy): Pr } /** - * This is a very specific utility method to help traverse the database - * to parse data and titles out of images and collections alone. - * - * We don't know if the document id given to is corresponds to a view document or a data - * document. If it's a data document, the response from the database will have - * a data field. If not, call recursively on the proto, and resolve with *its* data - * - * @param targetId the id of the Dash document whose data is being requests - * @returns the data of the document, as well as its title + * This utility function factors out the process + * of creating a zip file and sending it back to the client + * by piping it into a response. + * + * Learn more about piping and readable / writable streams here! + * https://www.freecodecamp.org/news/node-js-streams-everything-you-need-to-know-c9141306be93/ + * + * @param res the writable stream response object that will transfer the generated zip file + * @param mutator the callback function used to actually modify and insert information into the zip instance */ -async function getData(targetId: string): Promise<DocumentElements> { - return new Promise<DocumentElements>((resolve, reject) => { - Database.Instance.getDocument(targetId, async (result: any) => { - const { data, proto, title } = result.fields; - if (data) { - if (data.url) { - resolve({ data: data.url, title }); - } else if (data.fields) { - resolve({ data: data.fields, title }); - } else { - reject(); - } - } else if (proto) { - getData(proto.fieldId).then(resolve, reject); - } else { - reject(); - } - }); - }); +export async function BuildAndDispatchZip(res: express.Response, mutator: ZipMutator): Promise<void> { + res.set('Content-disposition', `attachment;`); + res.set('Content-Type', 'application/zip'); + const zip = Archiver('zip'); + zip.pipe(res); + await mutator(zip); + return zip.finalize(); } /** - * + * * @param file the zip file to which we write the files * @param hierarchy the data structure from which we read, defining the nesting of the documents in the zip * @param prefix lets us create nested folders in the zip file by continually appending to the end * of the prefix with each layer of recursion. - * + * * Function Call #1 => "Dash Export" * Function Call #2 => "Dash Export/a nested collection" * Function Call #3 => "Dash Export/a nested collection/lowest level collection" * ... */ -async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hierarchy, prefix = "Dash Export"): Promise<void> { - for (const documentTitle of Object.keys(hierarchy)) { - const result = hierarchy[documentTitle]; - // base case or leaf node, we've hit a url (image) - if (typeof result === "string") { - let path: string; - let matches: RegExpExecArray | null; - if ((matches = /\:\d+\/files\/images\/(upload\_[\da-z]{32}.*)/g.exec(result)) !== null) { - // image already exists on our server - path = serverPathToFile(Directory.images, matches[1]); +async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hierarchy, prefix = 'Dash Export'): Promise<void> { + // eslint-disable-next-line no-restricted-syntax + for (const documentTitle in hierarchy) { + if (Object.prototype.hasOwnProperty.call(hierarchy, documentTitle)) { + const result = hierarchy[documentTitle]; + // base case or leaf node, we've hit a url (image) + if (typeof result === 'string') { + let fPath: string; + const matches = /:\d+\/files\/images\/(upload_[\da-z]{32}.*)/g.exec(result); + if (matches !== null) { + // image already exists on our server + fPath = serverPathToFile(Directory.images, matches[1]); + } else { + // the image doesn't already exist on our server (may have been dragged + // and dropped in the browser and thus hosted remotely) so we upload it + // to our server and point the zip file to it, so it can bundle up the bytes + // eslint-disable-next-line no-await-in-loop + const information = await DashUploadUtils.UploadImage(result); + fPath = information instanceof Error ? '' : information.accessPaths[SizeSuffix.Original].server; + } + // write the file specified by the path to the directory in the + // zip file given by the prefix. + if (fPath) { + file.file(fPath, { name: documentTitle, prefix }); + } } else { - // the image doesn't already exist on our server (may have been dragged - // and dropped in the browser and thus hosted remotely) so we upload it - // to our server and point the zip file to it, so it can bundle up the bytes - const information = await DashUploadUtils.UploadImage(result); - path = information instanceof Error ? "" : information.accessPaths[SizeSuffix.Original].server; + // we've hit a collection, so we have to recurse + // eslint-disable-next-line no-await-in-loop + await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`); } - // write the file specified by the path to the directory in the - // zip file given by the prefix. - if (path) { - file.file(path, { name: documentTitle, prefix }); + } + } +} + +async function getDocs(docId: string) { + const files = new Set<string>(); + const docs: { [id: string]: any } = {}; + const fn = (doc: any): string[] => { + const { id } = doc; + if (typeof id === 'string' && id.endsWith('Proto')) { + // Skip protos + return []; + } + const ids: string[] = []; + // eslint-disable-next-line no-restricted-syntax + for (const key in doc.fields) { + // eslint-disable-next-line no-continue + if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue; + + const field = doc.fields[key]; + // eslint-disable-next-line no-continue + if (field === undefined || field === null) continue; + + if (field.__type === 'proxy' || field.__type === 'prefetch_proxy') { + ids.push(field.fieldId); + } else if (field.__type === 'script' || field.__type === 'computed') { + field.captures && ids.push(field.captures.fieldId); + } else if (field.__type === 'list') { + ids.push(...fn(field)); + } else if (typeof field === 'string') { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w-]*)"/g; + for (let match = re.exec(field); match !== null; match = re.exec(field)) { + ids.push(match[1]); + } + } else if (field.__type === 'RichTextField') { + const re = /"href"\s*:\s*"(.*?)"/g; + for (let match = re.exec(field.data); match !== null; match = re.exec(field.Data)) { + const urlString = match[1]; + const split = new URL(urlString).pathname.split('doc/'); + if (split.length > 1) { + ids.push(split[split.length - 1]); + } + } + const re2 = /"src"\s*:\s*"(.*?)"/g; + for (let match = re2.exec(field.Data); match !== null; match = re2.exec(field.Data)) { + const urlString = match[1]; + const { pathname } = new URL(urlString); + files.add(pathname); + } + } else if (['audio', 'image', 'video', 'pdf', 'web', 'map'].includes(field.__type)) { + const { pathname } = new URL(field.url); + files.add(pathname); } - } else { - // we've hit a collection, so we have to recurse - await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`); } + + if (doc.id) { + docs[doc.id] = doc; + } + return ids; + }; + await Database.Instance.visit([docId], fn); + return { id: docId, docs, files }; +} + +export default class DownloadManager extends ApiManager { + protected initialize(register: Registration): void { + /** + * Let's say someone's using Dash to organize images in collections. + * This lets them export the hierarchy they've built to their + * own file system in a useful format. + * + * This handler starts with a single document id (interesting only + * if it's that of a collection). It traverses the database, captures + * the nesting of only nested images or collections, writes + * that to a zip file and returns it to the client for download. + */ + register({ + method: Method.GET, + subscription: new RouteSubscriber('imageHierarchyExport').add('docId'), + secureHandler: async ({ req, res }) => { + const id = req.params.docId; + const hierarchy: Hierarchy = {}; + await buildHierarchyRecursive(id, hierarchy); + return BuildAndDispatchZip(res, zip => writeHierarchyRecursive(zip, hierarchy)); + }, + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber('downloadId').add('docId'), + secureHandler: async ({ req, res }) => + BuildAndDispatchZip(res, async zip => { + const { id, docs, files } = await getDocs(req.params.docId); + const docString = JSON.stringify({ id, docs }); + zip.append(docString, { name: 'doc.json' }); + files.forEach(val => { + zip.file(publicDirectory + val, { name: val.substring(1) }); + }); + }), + }); + + register({ + method: Method.GET, + subscription: new RouteSubscriber('serializeDoc').add('docId'), + secureHandler: async ({ req, res }) => { + const { docs, files } = await getDocs(req.params.docId); + res.send({ docs, files: Array.from(files) }); + }, + }); } -}
\ No newline at end of file +} diff --git a/src/server/ApiManagers/GeneralGoogleManager.ts b/src/server/ApiManagers/GeneralGoogleManager.ts index f94b77cac..12913b1ef 100644 --- a/src/server/ApiManagers/GeneralGoogleManager.ts +++ b/src/server/ApiManagers/GeneralGoogleManager.ts @@ -1,51 +1,49 @@ -import ApiManager, { Registration } from "./ApiManager"; -import { Method, _permission_denied } from "../RouteManager"; -import { GoogleApiServerUtils } from "../apis/google/GoogleApiServerUtils"; -import RouteSubscriber from "../RouteSubscriber"; -import { Database } from "../database"; +import ApiManager, { Registration } from './ApiManager'; +import { Method } from '../RouteManager'; +import { GoogleApiServerUtils } from '../apis/google/GoogleApiServerUtils'; +import RouteSubscriber from '../RouteSubscriber'; +import { Database } from '../database'; const EndpointHandlerMap = new Map<GoogleApiServerUtils.Action, GoogleApiServerUtils.ApiRouter>([ - ["create", (api, params) => api.create(params)], - ["retrieve", (api, params) => api.get(params)], - ["update", (api, params) => api.batchUpdate(params)], + ['create', (api, params) => api.create(params)], + ['retrieve', (api, params) => api.get(params)], + ['update', (api, params) => api.batchUpdate(params)], ]); export default class GeneralGoogleManager extends ApiManager { - protected initialize(register: Registration): void { - register({ method: Method.GET, - subscription: "/readGoogleAccessToken", + subscription: '/readGoogleAccessToken', secureHandler: async ({ user, res }) => { - const { credentials } = (await GoogleApiServerUtils.retrieveCredentials(user.id)); + const { credentials } = await GoogleApiServerUtils.retrieveCredentials(user.id); if (!credentials?.access_token) { return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); } return res.send(credentials); - } + }, }); register({ method: Method.POST, - subscription: "/writeGoogleAccessToken", + subscription: '/writeGoogleAccessToken', secureHandler: async ({ user, req, res }) => { res.send(await GoogleApiServerUtils.processNewUser(user.id, req.body.authenticationCode)); - } + }, }); register({ method: Method.GET, - subscription: "/revokeGoogleAccessToken", + subscription: '/revokeGoogleAccessToken', secureHandler: async ({ user, res }) => { await Database.Auxiliary.GoogleAccessToken.Revoke(user.id); res.send(); - } + }, }); register({ method: Method.POST, - subscription: new RouteSubscriber("googleDocs").add("sector", "action"), + subscription: new RouteSubscriber('googleDocs').add('sector', 'action'), secureHandler: async ({ req, res, user }) => { const sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; const action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; @@ -61,8 +59,7 @@ export default class GeneralGoogleManager extends ApiManager { return; } res.send(undefined); - } + }, }); - } -}
\ No newline at end of file +} diff --git a/src/server/ApiManagers/MongoStore.js b/src/server/ApiManagers/MongoStore.js new file mode 100644 index 000000000..5d91c2805 --- /dev/null +++ b/src/server/ApiManagers/MongoStore.js @@ -0,0 +1,413 @@ +const __createBinding = + (this && this.__createBinding) || + (Object.create + ? function (o, m, k, k2) { + if (k2 === undefined) k2 = k; + let desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ('get' in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { + enumerable: true, + get: function () { + return m[k]; + }, + }; + } + Object.defineProperty(o, k2, desc); + } + : function (o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; + }); +const __setModuleDefault = + (this && this.__setModuleDefault) || + (Object.create + ? function (o, v) { + Object.defineProperty(o, 'default', { enumerable: true, value: v }); + } + : function (o, v) { + o.default = v; + }); +const __importStar = + (this && this.__importStar) || + function (mod) { + if (mod && mod.__esModule) return mod; + const result = {}; + if (mod != null) for (const k in mod) if (k !== 'default' && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; + }; +const __importDefault = + (this && this.__importDefault) || + function (mod) { + return mod && mod.__esModule ? mod : { default: mod }; + }; +Object.defineProperty(exports, '__esModule', { value: true }); +const console_1 = require('console'); +const util_1 = __importDefault(require('util')); +const session = __importStar(require('express-session')); +const mongodb_1 = require('mongodb'); +const debug_1 = __importDefault(require('debug')); +const debug = (0, debug_1.default)('connect-mongo'); +// eslint-disable-next-line @typescript-eslint/no-empty-function +const noop = () => {}; +const unit = a => a; +function defaultSerializeFunction(session) { + // Copy each property of the session to a new object + const obj = {}; + let prop; + for (prop in session) { + if (prop === 'cookie') { + // Convert the cookie instance to an object, if possible + // This gets rid of the duplicate object under session.cookie.data property + // @ts-ignore FIXME: + obj.cookie = session.cookie.toJSON + ? // @ts-ignore FIXME: + session.cookie.toJSON() + : session.cookie; + } else { + // @ts-ignore FIXME: + obj[prop] = session[prop]; + } + } + return obj; +} +function computeTransformFunctions(options) { + if (options.serialize || options.unserialize) { + return { + serialize: options.serialize || defaultSerializeFunction, + unserialize: options.unserialize || unit, + }; + } + if (options.stringify === false) { + return { + serialize: defaultSerializeFunction, + unserialize: unit, + }; + } + // Default case + return { + serialize: JSON.stringify, + unserialize: JSON.parse, + }; +} +class MongoStore extends session.Store { + constructor({ collectionName = 'sessions', ttl = 1209600, mongoOptions = {}, autoRemove = 'native', autoRemoveInterval = 10, touchAfter = 0, stringify = true, crypto, ...required }) { + super(); + this.crypto = null; + debug('create MongoStore instance'); + const options = { + collectionName, + ttl, + mongoOptions, + autoRemove, + autoRemoveInterval, + touchAfter, + stringify, + crypto: { + ...{ + secret: false, + algorithm: 'aes-256-gcm', + hashing: 'sha512', + encodeas: 'base64', + key_size: 32, + iv_size: 16, + at_size: 16, + }, + ...crypto, + }, + ...required, + }; + // Check params + (0, console_1.assert)(options.mongoUrl || options.clientPromise || options.client, 'You must provide either mongoUrl|clientPromise|client in options'); + (0, console_1.assert)(options.createAutoRemoveIdx === null || options.createAutoRemoveIdx === undefined, 'options.createAutoRemoveIdx has been reverted to autoRemove and autoRemoveInterval'); + (0, console_1.assert)(!options.autoRemoveInterval || options.autoRemoveInterval <= 71582, /* (Math.pow(2, 32) - 1) / (1000 * 60) */ 'autoRemoveInterval is too large. options.autoRemoveInterval is in minutes but not seconds nor mills'); + this.transformFunctions = computeTransformFunctions(options); + let _clientP; + if (options.mongoUrl) { + _clientP = mongodb_1.MongoClient.connect(options.mongoUrl, options.mongoOptions); + } else if (options.clientPromise) { + _clientP = options.clientPromise; + } else if (options.client) { + _clientP = Promise.resolve(options.client); + } else { + throw new Error('Cannot init client. Please provide correct options'); + } + (0, console_1.assert)(!!_clientP, 'Client is null|undefined'); + this.clientP = _clientP; + this.options = options; + this.collectionP = _clientP.then(async con => { + const collection = con.db(options.dbName).collection(options.collectionName); + await this.setAutoRemove(collection); + return collection; + }); + if (options.crypto.secret) { + this.crypto = require('kruptein')(options.crypto); + } + } + static create(options) { + return new MongoStore(options); + } + setAutoRemove(collection) { + const removeQuery = () => ({ + expires: { + $lt: new Date(), + }, + }); + switch (this.options.autoRemove) { + case 'native': + debug('Creating MongoDB TTL index'); + return collection.createIndex( + { expires: 1 }, + { + background: true, + expireAfterSeconds: 0, + } + ); + case 'interval': + debug('create Timer to remove expired sessions'); + this.timer = setInterval( + () => + collection.deleteMany(removeQuery(), { + writeConcern: { + w: 0, + j: false, + }, + }), + this.options.autoRemoveInterval * 1000 * 60 + ); + this.timer.unref(); + return Promise.resolve(); + case 'disabled': + default: + return Promise.resolve(); + } + } + computeStorageId(sessionId) { + if (this.options.transformId && typeof this.options.transformId === 'function') { + return this.options.transformId(sessionId); + } + return sessionId; + } + /** + * promisify and bind the `this.crypto.get` function. + * Please check !!this.crypto === true before using this getter! + */ + get cryptoGet() { + if (!this.crypto) { + throw new Error('Check this.crypto before calling this.cryptoGet!'); + } + return util_1.default.promisify(this.crypto.get).bind(this.crypto); + } + /** + * Decrypt given session data + * @param session session data to be decrypt. Mutate the input session. + */ + async decryptSession(session) { + if (this.crypto && session) { + const plaintext = await this.cryptoGet(this.options.crypto.secret, session.session).catch(err => { + throw new Error(err); + }); + // @ts-ignore + session.session = JSON.parse(plaintext); + } + } + /** + * Get a session from the store given a session ID (sid) + * @param sid session ID + */ + get(sid, callback) { + (async () => { + try { + debug(`MongoStore#get=${sid}`); + const collection = await this.collectionP; + const session = await collection.findOne({ + _id: this.computeStorageId(sid), + $or: [{ expires: { $exists: false } }, { expires: { $gt: new Date() } }], + }); + if (this.crypto && session) { + await this.decryptSession(session).catch(err => callback(err)); + } + const s = session && this.transformFunctions.unserialize(session.session); + if (this.options.touchAfter > 0 && (session === null || session === void 0 ? void 0 : session.lastModified)) { + s.lastModified = session.lastModified; + } + this.emit('get', sid); + callback(null, s === undefined ? null : s); + } catch (error) { + callback(error); + } + })(); + } + /** + * Upsert a session into the store given a session ID (sid) and session (session) object. + * @param sid session ID + * @param session session object + */ + set(sid, session, callback = noop) { + (async () => { + let _a; + try { + debug(`MongoStore#set=${sid}`); + // Removing the lastModified prop from the session object before update + // @ts-ignore + if (this.options.touchAfter > 0 && (session === null || session === void 0 ? void 0 : session.lastModified)) { + // @ts-ignore + delete session.lastModified; + } + const s = { + _id: this.computeStorageId(sid), + session: this.transformFunctions.serialize(session), + }; + // Expire handling + if ((_a = session === null || session === void 0 ? void 0 : session.cookie) === null || _a === void 0 ? void 0 : _a.expires) { + s.expires = new Date(session.cookie.expires); + } else { + // If there's no expiration date specified, it is + // browser-session cookie or there is no cookie at all, + // as per the connect docs. + // + // So we set the expiration to two-weeks from now + // - as is common practice in the industry (e.g Django) - + // or the default specified in the options. + s.expires = new Date(Date.now() + this.options.ttl * 1000); + } + // Last modify handling + if (this.options.touchAfter > 0) { + s.lastModified = new Date(); + } + if (this.crypto) { + const cryptoSet = util_1.default.promisify(this.crypto.set).bind(this.crypto); + const data = await cryptoSet(this.options.crypto.secret, s.session).catch(err => { + throw new Error(err); + }); + s.session = data; + } + const collection = await this.collectionP; + const rawResp = await collection.updateOne( + { _id: s._id }, + { $set: s }, + { + upsert: true, + writeConcern: this.options.writeOperationOptions, + } + ); + if (rawResp.upsertedCount > 0) { + this.emit('create', sid); + } else { + this.emit('update', sid); + } + this.emit('set', sid); + } catch (error) { + return callback(error); + } + return callback(null); + })(); + } + touch(sid, session, callback = noop) { + (async () => { + let _a; + try { + debug(`MongoStore#touch=${sid}`); + const updateFields = {}; + const touchAfter = this.options.touchAfter * 1000; + const lastModified = session.lastModified ? session.lastModified.getTime() : 0; + const currentDate = new Date(); + // If the given options has a touchAfter property, check if the + // current timestamp - lastModified timestamp is bigger than + // the specified, if it's not, don't touch the session + if (touchAfter > 0 && lastModified > 0) { + const timeElapsed = currentDate.getTime() - lastModified; + if (timeElapsed < touchAfter) { + debug(`Skip touching session=${sid}`); + return callback(null); + } + updateFields.lastModified = currentDate; + } + if ((_a = session === null || session === void 0 ? void 0 : session.cookie) === null || _a === void 0 ? void 0 : _a.expires) { + updateFields.expires = new Date(session.cookie.expires); + } else { + updateFields.expires = new Date(Date.now() + this.options.ttl * 1000); + } + const collection = await this.collectionP; + const rawResp = await collection.updateOne({ _id: this.computeStorageId(sid) }, { $set: updateFields }, { writeConcern: this.options.writeOperationOptions }); + if (rawResp.matchedCount === 0) { + return callback(new Error('Unable to find the session to touch')); + } else { + this.emit('touch', sid, session); + return callback(null); + } + } catch (error) { + return callback(error); + } + })(); + } + /** + * Get all sessions in the store as an array + */ + all(callback) { + (async () => { + try { + debug('MongoStore#all()'); + const collection = await this.collectionP; + const sessions = collection.find({ + $or: [{ expires: { $exists: false } }, { expires: { $gt: new Date() } }], + }); + const results = []; + for await (const session of sessions) { + if (this.crypto && session) { + await this.decryptSession(session); + } + results.push(this.transformFunctions.unserialize(session.session)); + } + this.emit('all', results); + callback(null, results); + } catch (error) { + callback(error); + } + })(); + } + /** + * Destroy/delete a session from the store given a session ID (sid) + * @param sid session ID + */ + destroy(sid, callback = noop) { + debug(`MongoStore#destroy=${sid}`); + this.collectionP + .then(colleciton => colleciton.deleteOne({ _id: this.computeStorageId(sid) }, { writeConcern: this.options.writeOperationOptions })) + .then(() => { + this.emit('destroy', sid); + callback(null); + }) + .catch(err => callback(err)); + } + /** + * Get the count of all sessions in the store + */ + length(callback) { + debug('MongoStore#length()'); + this.collectionP + .then(collection => collection.countDocuments()) + .then(c => callback(null, c)) + // @ts-ignore + .catch(err => callback(err)); + } + /** + * Delete all sessions from the store. + */ + clear(callback = noop) { + debug('MongoStore#clear()'); + this.collectionP + .then(collection => collection.drop()) + .then(() => callback(null)) + .catch(err => callback(err)); + } + /** + * Close database connection + */ + close() { + debug('MongoStore#close()'); + return this.clientP.then(c => c.close()); + } +} +exports.default = MongoStore; +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiTW9uZ29TdG9yZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9saWIvTW9uZ29TdG9yZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0FBQUEscUNBQWdDO0FBQ2hDLGdEQUF1QjtBQUN2Qix5REFBMEM7QUFDMUMscUNBS2dCO0FBQ2hCLGtEQUF5QjtBQUd6QixNQUFNLEtBQUssR0FBRyxJQUFBLGVBQUssRUFBQyxlQUFlLENBQUMsQ0FBQTtBQWdFcEMsZ0VBQWdFO0FBQ2hFLE1BQU0sSUFBSSxHQUFHLEdBQUcsRUFBRSxHQUFFLENBQUMsQ0FBQTtBQUNyQixNQUFNLElBQUksR0FBbUIsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQTtBQUVyQyxTQUFTLHdCQUF3QixDQUMvQixPQUE0QjtJQUU1QixvREFBb0Q7SUFDcEQsTUFBTSxHQUFHLEdBQUcsRUFBRSxDQUFBO0lBQ2QsSUFBSSxJQUFJLENBQUE7SUFDUixLQUFLLElBQUksSUFBSSxPQUFPLEVBQUU7UUFDcEIsSUFBSSxJQUFJLEtBQUssUUFBUSxFQUFFO1lBQ3JCLHdEQUF3RDtZQUN4RCwyRUFBMkU7WUFDM0Usb0JBQW9CO1lBQ3BCLEdBQUcsQ0FBQyxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQyxNQUFNO2dCQUNoQyxDQUFDLENBQUMsb0JBQW9CO29CQUNwQixPQUFPLENBQUMsTUFBTSxDQUFDLE1BQU0sRUFBRTtnQkFDekIsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUE7U0FDbkI7YUFBTTtZQUNMLG9CQUFvQjtZQUNwQixHQUFHLENBQUMsSUFBSSxDQUFDLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFBO1NBQzFCO0tBQ0Y7SUFFRCxPQUFPLEdBQTBCLENBQUE7QUFDbkMsQ0FBQztBQUVELFNBQVMseUJBQXlCLENBQUMsT0FBbUM7SUFDcEUsSUFBSSxPQUFPLENBQUMsU0FBUyxJQUFJLE9BQU8sQ0FBQyxXQUFXLEVBQUU7UUFDNUMsT0FBTztZQUNMLFNBQVMsRUFBRSxPQUFPLENBQUMsU0FBUyxJQUFJLHdCQUF3QjtZQUN4RCxXQUFXLEVBQUUsT0FBTyxDQUFDLFdBQVcsSUFBSSxJQUFJO1NBQ3pDLENBQUE7S0FDRjtJQUVELElBQUksT0FBTyxDQUFDLFNBQVMsS0FBSyxLQUFLLEVBQUU7UUFDL0IsT0FBTztZQUNMLFNBQVMsRUFBRSx3QkFBd0I7WUFDbkMsV0FBVyxFQUFFLElBQUk7U0FDbEIsQ0FBQTtLQUNGO0lBQ0QsZUFBZTtJQUNmLE9BQU87UUFDTCxTQUFTLEVBQUUsSUFBSSxDQUFDLFNBQVM7UUFDekIsV0FBVyxFQUFFLElBQUksQ0FBQyxLQUFLO0tBQ3hCLENBQUE7QUFDSCxDQUFDO0FBRUQsTUFBcUIsVUFBVyxTQUFRLE9BQU8sQ0FBQyxLQUFLO0lBWW5ELFlBQVksRUFDVixjQUFjLEdBQUcsVUFBVSxFQUMzQixHQUFHLEdBQUcsT0FBTyxFQUNiLFlBQVksR0FBRyxFQUFFLEVBQ2pCLFVBQVUsR0FBRyxRQUFRLEVBQ3JCLGtCQUFrQixHQUFHLEVBQUUsRUFDdkIsVUFBVSxHQUFHLENBQUMsRUFDZCxTQUFTLEdBQUcsSUFBSSxFQUNoQixNQUFNLEVBQ04sR0FBRyxRQUFRLEVBQ1M7UUFDcEIsS0FBSyxFQUFFLENBQUE7UUFyQkQsV0FBTSxHQUFvQixJQUFJLENBQUE7UUFzQnBDLEtBQUssQ0FBQyw0QkFBNEIsQ0FBQyxDQUFBO1FBQ25DLE1BQU0sT0FBTyxHQUErQjtZQUMxQyxjQUFjO1lBQ2QsR0FBRztZQUNILFlBQVk7WUFDWixVQUFVO1lBQ1Ysa0JBQWtCO1lBQ2xCLFVBQVU7WUFDVixTQUFTO1lBQ1QsTUFBTSxFQUFFO2dCQUNOLEdBQUc7b0JBQ0QsTUFBTSxFQUFFLEtBQUs7b0JBQ2IsU0FBUyxFQUFFLGFBQWE7b0JBQ3hCLE9BQU8sRUFBRSxRQUFRO29CQUNqQixRQUFRLEVBQUUsUUFBUTtvQkFDbEIsUUFBUSxFQUFFLEVBQUU7b0JBQ1osT0FBTyxFQUFFLEVBQUU7b0JBQ1gsT0FBTyxFQUFFLEVBQUU7aUJBQ1o7Z0JBQ0QsR0FBRyxNQUFNO2FBQ1Y7WUFDRCxHQUFHLFFBQVE7U0FDWixDQUFBO1FBQ0QsZUFBZTtRQUNmLElBQUEsZ0JBQU0sRUFDSixPQUFPLENBQUMsUUFBUSxJQUFJLE9BQU8sQ0FBQyxhQUFhLElBQUksT0FBTyxDQUFDLE1BQU0sRUFDM0Qsa0VBQWtFLENBQ25FLENBQUE7UUFDRCxJQUFBLGdCQUFNLEVBQ0osT0FBTyxDQUFDLG1CQUFtQixLQUFLLElBQUk7WUFDbEMsT0FBTyxDQUFDLG1CQUFtQixLQUFLLFNBQVMsRUFDM0Msb0ZBQW9GLENBQ3JGLENBQUE7UUFDRCxJQUFBLGdCQUFNLEVBQ0osQ0FBQyxPQUFPLENBQUMsa0JBQWtCLElBQUksT0FBTyxDQUFDLGtCQUFrQixJQUFJLEtBQUs7UUFDbEUseUNBQXlDLENBQUMscUdBQXFHLENBQ2hKLENBQUE7UUFDRCxJQUFJLENBQUMsa0JBQWtCLEdBQUcseUJBQXlCLENBQUMsT0FBTyxDQUFDLENBQUE7UUFDNUQsSUFBSSxRQUE4QixDQUFBO1FBQ2xDLElBQUksT0FBTyxDQUFDLFFBQVEsRUFBRTtZQUNwQixRQUFRLEdBQUcscUJBQVcsQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLFFBQVEsRUFBRSxPQUFPLENBQUMsWUFBWSxDQUFDLENBQUE7U0FDdkU7YUFBTSxJQUFJLE9BQU8sQ0FBQyxhQUFhLEVBQUU7WUFDaEMsUUFBUSxHQUFHLE9BQU8sQ0FBQyxhQUFhLENBQUE7U0FDakM7YUFBTSxJQUFJLE9BQU8sQ0FBQyxNQUFNLEVBQUU7WUFDekIsUUFBUSxHQUFHLE9BQU8sQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFBO1NBQzNDO2FBQU07WUFDTCxNQUFNLElBQUksS0FBSyxDQUFDLG9EQUFvRCxDQUFDLENBQUE7U0FDdEU7UUFDRCxJQUFBLGdCQUFNLEVBQUMsQ0FBQyxDQUFDLFFBQVEsRUFBRSwwQkFBMEIsQ0FBQyxDQUFBO1FBQzlDLElBQUksQ0FBQyxPQUFPLEdBQUcsUUFBUSxDQUFBO1FBQ3ZCLElBQUksQ0FBQyxPQUFPLEdBQUcsT0FBTyxDQUFBO1FBQ3RCLElBQUksQ0FBQyxXQUFXLEdBQUcsUUFBUSxDQUFDLElBQUksQ0FBQyxLQUFLLEVBQUUsR0FBRyxFQUFFLEVBQUU7WUFDN0MsTUFBTSxVQUFVLEdBQUcsR0FBRztpQkFDbkIsRUFBRSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUM7aUJBQ2xCLFVBQVUsQ0FBc0IsT0FBTyxDQUFDLGNBQWMsQ0FBQyxDQUFBO1lBQzFELE1BQU0sSUFBSSxDQUFDLGFBQWEsQ0FBQyxVQUFVLENBQUMsQ0FBQTtZQUNwQyxPQUFPLFVBQVUsQ0FBQTtRQUNuQixDQUFDLENBQUMsQ0FBQTtRQUNGLElBQUksT0FBTyxDQUFDLE1BQU0sQ0FBQyxNQUFNLEVBQUU7WUFDekIsSUFBSSxDQUFDLE1BQU0sR0FBRyxPQUFPLENBQUMsVUFBVSxDQUFDLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFBO1NBQ2xEO0lBQ0gsQ0FBQztJQUVELE1BQU0sQ0FBQyxNQUFNLENBQUMsT0FBNEI7UUFDeEMsT0FBTyxJQUFJLFVBQVUsQ0FBQyxPQUFPLENBQUMsQ0FBQTtJQUNoQyxDQUFDO0lBRU8sYUFBYSxDQUNuQixVQUEyQztRQUUzQyxNQUFNLFdBQVcsR0FBRyxHQUFHLEVBQUUsQ0FBQyxDQUFDO1lBQ3pCLE9BQU8sRUFBRTtnQkFDUCxHQUFHLEVBQUUsSUFBSSxJQUFJLEVBQUU7YUFDaEI7U0FDRixDQUFDLENBQUE7UUFDRixRQUFRLElBQUksQ0FBQyxPQUFPLENBQUMsVUFBVSxFQUFFO1lBQy9CLEtBQUssUUFBUTtnQkFDWCxLQUFLLENBQUMsNEJBQTRCLENBQUMsQ0FBQTtnQkFDbkMsT0FBTyxVQUFVLENBQUMsV0FBVyxDQUMzQixFQUFFLE9BQU8sRUFBRSxDQUFDLEVBQUUsRUFDZDtvQkFDRSxVQUFVLEVBQUUsSUFBSTtvQkFDaEIsa0JBQWtCLEVBQUUsQ0FBQztpQkFDdEIsQ0FDRixDQUFBO1lBQ0gsS0FBSyxVQUFVO2dCQUNiLEtBQUssQ0FBQyx5Q0FBeUMsQ0FBQyxDQUFBO2dCQUNoRCxJQUFJLENBQUMsS0FBSyxHQUFHLFdBQVcsQ0FDdEIsR0FBRyxFQUFFLENBQ0gsVUFBVSxDQUFDLFVBQVUsQ0FBQyxXQUFXLEVBQUUsRUFBRTtvQkFDbkMsWUFBWSxFQUFFO3dCQUNaLENBQUMsRUFBRSxDQUFDO3dCQUNKLENBQUMsRUFBRSxLQUFLO3FCQUNUO2lCQUNGLENBQUMsRUFDSixJQUFJLENBQUMsT0FBTyxDQUFDLGtCQUFrQixHQUFHLElBQUksR0FBRyxFQUFFLENBQzVDLENBQUE7Z0JBQ0QsSUFBSSxDQUFDLEtBQUssQ0FBQyxLQUFLLEVBQUUsQ0FBQTtnQkFDbEIsT0FBTyxPQUFPLENBQUMsT0FBTyxFQUFFLENBQUE7WUFDMUIsS0FBSyxVQUFVLENBQUM7WUFDaEI7Z0JBQ0UsT0FBTyxPQUFPLENBQUMsT0FBTyxFQUFFLENBQUE7U0FDM0I7SUFDSCxDQUFDO0lBRU8sZ0JBQWdCLENBQUMsU0FBaUI7UUFDeEMsSUFDRSxJQUFJLENBQUMsT0FBTyxDQUFDLFdBQVc7WUFDeEIsT0FBTyxJQUFJLENBQUMsT0FBTyxDQUFDLFdBQVcsS0FBSyxVQUFVLEVBQzlDO1lBQ0EsT0FBTyxJQUFJLENBQUMsT0FBTyxDQUFDLFdBQVcsQ0FBQyxTQUFTLENBQUMsQ0FBQTtTQUMzQztRQUNELE9BQU8sU0FBUyxDQUFBO0lBQ2xCLENBQUM7SUFFRDs7O09BR0c7SUFDSCxJQUFZLFNBQVM7UUFDbkIsSUFBSSxDQUFDLElBQUksQ0FBQyxNQUFNLEVBQUU7WUFDaEIsTUFBTSxJQUFJLEtBQUssQ0FBQyxrREFBa0QsQ0FBQyxDQUFBO1NBQ3BFO1FBQ0QsT0FBTyxjQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQTtJQUMxRCxDQUFDO0lBRUQ7OztPQUdHO0lBQ0ssS0FBSyxDQUFDLGNBQWMsQ0FDMUIsT0FBK0M7UUFFL0MsSUFBSSxJQUFJLENBQUMsTUFBTSxJQUFJLE9BQU8sRUFBRTtZQUMxQixNQUFNLFNBQVMsR0FBRyxNQUFNLElBQUksQ0FBQyxTQUFTLENBQ3BDLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLE1BQWdCLEVBQ3BDLE9BQU8sQ0FBQyxPQUFPLENBQ2hCLENBQUMsS0FBSyxDQUFDLENBQUMsR0FBRyxFQUFFLEVBQUU7Z0JBQ2QsTUFBTSxJQUFJLEtBQUssQ0FBQyxHQUFHLENBQUMsQ0FBQTtZQUN0QixDQUFDLENBQUMsQ0FBQTtZQUNGLGFBQWE7WUFDYixPQUFPLENBQUMsT0FBTyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsU0FBUyxDQUFDLENBQUE7U0FDeEM7SUFDSCxDQUFDO0lBRUQ7OztPQUdHO0lBQ0gsR0FBRyxDQUNELEdBQVcsRUFDWCxRQUFrRTtRQUVsRSxDQUFDO1FBQUEsQ0FBQyxLQUFLLElBQUksRUFBRTtZQUNYLElBQUk7Z0JBQ0YsS0FBSyxDQUFDLGtCQUFrQixHQUFHLEVBQUUsQ0FBQyxDQUFBO2dCQUM5QixNQUFNLFVBQVUsR0FBRyxNQUFNLElBQUksQ0FBQyxXQUFXLENBQUE7Z0JBQ3pDLE1BQU0sT0FBTyxHQUFHLE1BQU0sVUFBVSxDQUFDLE9BQU8sQ0FBQztvQkFDdkMsR0FBRyxFQUFFLElBQUksQ0FBQyxnQkFBZ0IsQ0FBQyxHQUFHLENBQUM7b0JBQy9CLEdBQUcsRUFBRTt3QkFDSCxFQUFFLE9BQU8sRUFBRSxFQUFFLE9BQU8sRUFBRSxLQUFLLEVBQUUsRUFBRTt3QkFDL0IsRUFBRSxPQUFPLEVBQUUsRUFBRSxHQUFHLEVBQUUsSUFBSSxJQUFJLEVBQUUsRUFBRSxFQUFFO3FCQUNqQztpQkFDRixDQUFDLENBQUE7Z0JBQ0YsSUFBSSxJQUFJLENBQUMsTUFBTSxJQUFJLE9BQU8sRUFBRTtvQkFDMUIsTUFBTSxJQUFJLENBQUMsY0FBYyxDQUN2QixPQUF5QyxDQUMxQyxDQUFDLEtBQUssQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUE7aUJBQ2hDO2dCQUNELE1BQU0sQ0FBQyxHQUNMLE9BQU8sSUFBSSxJQUFJLENBQUMsa0JBQWtCLENBQUMsV0FBVyxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUMsQ0FBQTtnQkFDakUsSUFBSSxJQUFJLENBQUMsT0FBTyxDQUFDLFVBQVUsR0FBRyxDQUFDLEtBQUksT0FBTyxhQUFQLE9BQU8sdUJBQVAsT0FBTyxDQUFFLFlBQVksQ0FBQSxFQUFFO29CQUN4RCxDQUFDLENBQUMsWUFBWSxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUE7aUJBQ3RDO2dCQUNELElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxDQUFBO2dCQUNyQixRQUFRLENBQUMsSUFBSSxFQUFFLENBQUMsS0FBSyxTQUFTLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUE7YUFDM0M7WUFBQyxPQUFPLEtBQUssRUFBRTtnQkFDZCxRQUFRLENBQUMsS0FBSyxDQUFDLENBQUE7YUFDaEI7UUFDSCxDQUFDLENBQUMsRUFBRSxDQUFBO0lBQ04sQ0FBQztJQUVEOzs7O09BSUc7SUFDSCxHQUFHLENBQ0QsR0FBVyxFQUNYLE9BQTRCLEVBQzVCLFdBQStCLElBQUk7UUFFbkMsQ0FBQztRQUFBLENBQUMsS0FBSyxJQUFJLEVBQUU7O1lBQ1gsSUFBSTtnQkFDRixLQUFLLENBQUMsa0JBQWtCLEdBQUcsRUFBRSxDQUFDLENBQUE7Z0JBQzlCLHVFQUF1RTtnQkFDdkUsYUFBYTtnQkFDYixJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsVUFBVSxHQUFHLENBQUMsS0FBSSxPQUFPLGFBQVAsT0FBTyx1QkFBUCxPQUFPLENBQUUsWUFBWSxDQUFBLEVBQUU7b0JBQ3hELGFBQWE7b0JBQ2IsT0FBTyxPQUFPLENBQUMsWUFBWSxDQUFBO2lCQUM1QjtnQkFDRCxNQUFNLENBQUMsR0FBd0I7b0JBQzdCLEdBQUcsRUFBRSxJQUFJLENBQUMsZ0JBQWdCLENBQUMsR0FBRyxDQUFDO29CQUMvQixPQUFPLEVBQUUsSUFBSSxDQUFDLGtCQUFrQixDQUFDLFNBQVMsQ0FBQyxPQUFPLENBQUM7aUJBQ3BELENBQUE7Z0JBQ0Qsa0JBQWtCO2dCQUNsQixJQUFJLE1BQUEsT0FBTyxhQUFQLE9BQU8sdUJBQVAsT0FBTyxDQUFFLE1BQU0sMENBQUUsT0FBTyxFQUFFO29CQUM1QixDQUFDLENBQUMsT0FBTyxHQUFHLElBQUksSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsT0FBTyxDQUFDLENBQUE7aUJBQzdDO3FCQUFNO29CQUNMLGlEQUFpRDtvQkFDakQsdURBQXVEO29CQUN2RCwyQkFBMkI7b0JBQzNCLEVBQUU7b0JBQ0YsaURBQWlEO29CQUNqRCx5REFBeUQ7b0JBQ3pELDJDQUEyQztvQkFDM0MsQ0FBQyxDQUFDLE9BQU8sR0FBRyxJQUFJLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxHQUFHLEdBQUcsSUFBSSxDQUFDLENBQUE7aUJBQzNEO2dCQUNELHVCQUF1QjtnQkFDdkIsSUFBSSxJQUFJLENBQUMsT0FBTyxDQUFDLFVBQVUsR0FBRyxDQUFDLEVBQUU7b0JBQy9CLENBQUMsQ0FBQyxZQUFZLEdBQUcsSUFBSSxJQUFJLEVBQUUsQ0FBQTtpQkFDNUI7Z0JBQ0QsSUFBSSxJQUFJLENBQUMsTUFBTSxFQUFFO29CQUNmLE1BQU0sU0FBUyxHQUFHLGNBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFBO29CQUNuRSxNQUFNLElBQUksR0FBRyxNQUFNLFNBQVMsQ0FDMUIsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsTUFBZ0IsRUFDcEMsQ0FBQyxDQUFDLE9BQU8sQ0FDVixDQUFDLEtBQUssQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFO3dCQUNkLE1BQU0sSUFBSSxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUE7b0JBQ3RCLENBQUMsQ0FBQyxDQUFBO29CQUNGLENBQUMsQ0FBQyxPQUFPLEdBQUcsSUFBc0MsQ0FBQTtpQkFDbkQ7Z0JBQ0QsTUFBTSxVQUFVLEdBQUcsTUFBTSxJQUFJLENBQUMsV0FBVyxDQUFBO2dCQUN6QyxNQUFNLE9BQU8sR0FBRyxNQUFNLFVBQVUsQ0FBQyxTQUFTLENBQ3hDLEVBQUUsR0FBRyxFQUFFLENBQUMsQ0FBQyxHQUFHLEVBQUUsRUFDZCxFQUFFLElBQUksRUFBRSxDQUFDLEVBQUUsRUFDWDtvQkFDRSxNQUFNLEVBQUUsSUFBSTtvQkFDWixZQUFZLEVBQUUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxxQkFBcUI7aUJBQ2pELENBQ0YsQ0FBQTtnQkFDRCxJQUFJLE9BQU8sQ0FBQyxhQUFhLEdBQUcsQ0FBQyxFQUFFO29CQUM3QixJQUFJLENBQUMsSUFBSSxDQUFDLFFBQVEsRUFBRSxHQUFHLENBQUMsQ0FBQTtpQkFDekI7cUJBQU07b0JBQ0wsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRLEVBQUUsR0FBRyxDQUFDLENBQUE7aUJBQ3pCO2dCQUNELElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxDQUFBO2FBQ3RCO1lBQUMsT0FBTyxLQUFLLEVBQUU7Z0JBQ2QsT0FBTyxRQUFRLENBQUMsS0FBSyxDQUFDLENBQUE7YUFDdkI7WUFDRCxPQUFPLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQTtRQUN2QixDQUFDLENBQUMsRUFBRSxDQUFBO0lBQ04sQ0FBQztJQUVELEtBQUssQ0FDSCxHQUFXLEVBQ1gsT0FBc0QsRUFDdEQsV0FBK0IsSUFBSTtRQUVuQyxDQUFDO1FBQUEsQ0FBQyxLQUFLLElBQUksRUFBRTs7WUFDWCxJQUFJO2dCQUNGLEtBQUssQ0FBQyxvQkFBb0IsR0FBRyxFQUFFLENBQUMsQ0FBQTtnQkFDaEMsTUFBTSxZQUFZLEdBSWQsRUFBRSxDQUFBO2dCQUNOLE1BQU0sVUFBVSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsVUFBVSxHQUFHLElBQUksQ0FBQTtnQkFDakQsTUFBTSxZQUFZLEdBQUcsT0FBTyxDQUFDLFlBQVk7b0JBQ3ZDLENBQUMsQ0FBQyxPQUFPLENBQUMsWUFBWSxDQUFDLE9BQU8sRUFBRTtvQkFDaEMsQ0FBQyxDQUFDLENBQUMsQ0FBQTtnQkFDTCxNQUFNLFdBQVcsR0FBRyxJQUFJLElBQUksRUFBRSxDQUFBO2dCQUU5QiwrREFBK0Q7Z0JBQy9ELDREQUE0RDtnQkFDNUQsc0RBQXNEO2dCQUN0RCxJQUFJLFVBQVUsR0FBRyxDQUFDLElBQUksWUFBWSxHQUFHLENBQUMsRUFBRTtvQkFDdEMsTUFBTSxXQUFXLEdBQUcsV0FBVyxDQUFDLE9BQU8sRUFBRSxHQUFHLFlBQVksQ0FBQTtvQkFDeEQsSUFBSSxXQUFXLEdBQUcsVUFBVSxFQUFFO3dCQUM1QixLQUFLLENBQUMseUJBQXlCLEdBQUcsRUFBRSxDQUFDLENBQUE7d0JBQ3JDLE9BQU8sUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFBO3FCQUN0QjtvQkFDRCxZQUFZLENBQUMsWUFBWSxHQUFHLFdBQVcsQ0FBQTtpQkFDeEM7Z0JBRUQsSUFBSSxNQUFBLE9BQU8sYUFBUCxPQUFPLHVCQUFQLE9BQU8sQ0FBRSxNQUFNLDBDQUFFLE9BQU8sRUFBRTtvQkFDNUIsWUFBWSxDQUFDLE9BQU8sR0FBRyxJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLE9BQU8sQ0FBQyxDQUFBO2lCQUN4RDtxQkFBTTtvQkFDTCxZQUFZLENBQUMsT0FBTyxHQUFHLElBQUksSUFBSSxDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLEdBQUcsR0FBRyxJQUFJLENBQUMsQ0FBQTtpQkFDdEU7Z0JBQ0QsTUFBTSxVQUFVLEdBQUcsTUFBTSxJQUFJLENBQUMsV0FBVyxDQUFBO2dCQUN6QyxNQUFNLE9BQU8sR0FBRyxNQUFNLFVBQVUsQ0FBQyxTQUFTLENBQ3hDLEVBQUUsR0FBRyxFQUFFLElBQUksQ0FBQyxnQkFBZ0IsQ0FBQyxHQUFHLENBQUMsRUFBRSxFQUNuQyxFQUFFLElBQUksRUFBRSxZQUFZLEVBQUUsRUFDdEIsRUFBRSxZQUFZLEVBQUUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxxQkFBcUIsRUFBRSxDQUNyRCxDQUFBO2dCQUNELElBQUksT0FBTyxDQUFDLFlBQVksS0FBSyxDQUFDLEVBQUU7b0JBQzlCLE9BQU8sUUFBUSxDQUFDLElBQUksS0FBSyxDQUFDLHFDQUFxQyxDQUFDLENBQUMsQ0FBQTtpQkFDbEU7cUJBQU07b0JBQ0wsSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLEVBQUUsR0FBRyxFQUFFLE9BQU8sQ0FBQyxDQUFBO29CQUNoQyxPQUFPLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQTtpQkFDdEI7YUFDRjtZQUFDLE9BQU8sS0FBSyxFQUFFO2dCQUNkLE9BQU8sUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFBO2FBQ3ZCO1FBQ0gsQ0FBQyxDQUFDLEVBQUUsQ0FBQTtJQUNOLENBQUM7SUFFRDs7T0FFRztJQUNILEdBQUcsQ0FDRCxRQU1TO1FBRVQsQ0FBQztRQUFBLENBQUMsS0FBSyxJQUFJLEVBQUU7WUFDWCxJQUFJO2dCQUNGLEtBQUssQ0FBQyxrQkFBa0IsQ0FBQyxDQUFBO2dCQUN6QixNQUFNLFVBQVUsR0FBRyxNQUFNLElBQUksQ0FBQyxXQUFXLENBQUE7Z0JBQ3pDLE1BQU0sUUFBUSxHQUFHLFVBQVUsQ0FBQyxJQUFJLENBQUM7b0JBQy9CLEdBQUcsRUFBRTt3QkFDSCxFQUFFLE9BQU8sRUFBRSxFQUFFLE9BQU8sRUFBRSxLQUFLLEVBQUUsRUFBRTt3QkFDL0IsRUFBRSxPQUFPLEVBQUUsRUFBRSxHQUFHLEVBQUUsSUFBSSxJQUFJLEVBQUUsRUFBRSxFQUFFO3FCQUNqQztpQkFDRixDQUFDLENBQUE7Z0JBQ0YsTUFBTSxPQUFPLEdBQTBCLEVBQUUsQ0FBQTtnQkFDekMsSUFBSSxLQUFLLEVBQUUsTUFBTSxPQUFPLElBQUksUUFBUSxFQUFFO29CQUNwQyxJQUFJLElBQUksQ0FBQyxNQUFNLElBQUksT0FBTyxFQUFFO3dCQUMxQixNQUFNLElBQUksQ0FBQyxjQUFjLENBQUMsT0FBeUMsQ0FBQyxDQUFBO3FCQUNyRTtvQkFDRCxPQUFPLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxXQUFXLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUE7aUJBQ25FO2dCQUNELElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFBO2dCQUN6QixRQUFRLENBQUMsSUFBSSxFQUFFLE9BQU8sQ0FBQyxDQUFBO2FBQ3hCO1lBQUMsT0FBTyxLQUFLLEVBQUU7Z0JBQ2QsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFBO2FBQ2hCO1FBQ0gsQ0FBQyxDQUFDLEVBQUUsQ0FBQTtJQUNOLENBQUM7SUFFRDs7O09BR0c7SUFDSCxPQUFPLENBQUMsR0FBVyxFQUFFLFdBQStCLElBQUk7UUFDdEQsS0FBSyxDQUFDLHNCQUFzQixHQUFHLEVBQUUsQ0FBQyxDQUFBO1FBQ2xDLElBQUksQ0FBQyxXQUFXO2FBQ2IsSUFBSSxDQUFDLENBQUMsVUFBVSxFQUFFLEVBQUUsQ0FDbkIsVUFBVSxDQUFDLFNBQVMsQ0FDbEIsRUFBRSxHQUFHLEVBQUUsSUFBSSxDQUFDLGdCQUFnQixDQUFDLEdBQUcsQ0FBQyxFQUFFLEVBQ25DLEVBQUUsWUFBWSxFQUFFLElBQUksQ0FBQyxPQUFPLENBQUMscUJBQXFCLEVBQUUsQ0FDckQsQ0FDRjthQUNBLElBQUksQ0FBQyxHQUFHLEVBQUU7WUFDVCxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSxHQUFHLENBQUMsQ0FBQTtZQUN6QixRQUFRLENBQUMsSUFBSSxDQUFDLENBQUE7UUFDaEIsQ0FBQyxDQUFDO2FBQ0QsS0FBSyxDQUFDLENBQUMsR0FBRyxFQUFFLEVBQUUsQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQTtJQUNsQyxDQUFDO0lBRUQ7O09BRUc7SUFDSCxNQUFNLENBQUMsUUFBNEM7UUFDakQsS0FBSyxDQUFDLHFCQUFxQixDQUFDLENBQUE7UUFDNUIsSUFBSSxDQUFDLFdBQVc7YUFDYixJQUFJLENBQUMsQ0FBQyxVQUFVLEVBQUUsRUFBRSxDQUFDLFVBQVUsQ0FBQyxjQUFjLEVBQUUsQ0FBQzthQUNqRCxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLFFBQVEsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQUFDLENBQUM7WUFDL0IsYUFBYTthQUNaLEtBQUssQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUE7SUFDbEMsQ0FBQztJQUVEOztPQUVHO0lBQ0gsS0FBSyxDQUFDLFdBQStCLElBQUk7UUFDdkMsS0FBSyxDQUFDLG9CQUFvQixDQUFDLENBQUE7UUFDM0IsSUFBSSxDQUFDLFdBQVc7YUFDYixJQUFJLENBQUMsQ0FBQyxVQUFVLEVBQUUsRUFBRSxDQUFDLFVBQVUsQ0FBQyxJQUFJLEVBQUUsQ0FBQzthQUN2QyxJQUFJLENBQUMsR0FBRyxFQUFFLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDO2FBQzFCLEtBQUssQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUE7SUFDbEMsQ0FBQztJQUVEOztPQUVHO0lBQ0gsS0FBSztRQUNILEtBQUssQ0FBQyxvQkFBb0IsQ0FBQyxDQUFBO1FBQzNCLE9BQU8sSUFBSSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxLQUFLLEVBQUUsQ0FBQyxDQUFBO0lBQzVDLENBQUM7Q0FDRjtBQW5hRCw2QkFtYUMifQ== diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 92c10975f..f43ed6ac9 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -1,6 +1,7 @@ +/* eslint-disable no-use-before-define */ import { exec } from 'child_process'; import { cyan, green, red, yellow } from 'colors'; -import { log_execution } from '../ActionUtilities'; +import { logExecution } from '../ActionUtilities'; import { Method } from '../RouteManager'; import RouteSubscriber from '../RouteSubscriber'; import { Search } from '../Search'; @@ -17,8 +18,10 @@ export class SearchManager extends ApiManager { switch (action) { case 'start': case 'stop': - const status = req.params.action === 'start'; - SolrManager.SetRunning(status); + { + const status = req.params.action === 'start'; + SolrManager.SetRunning(status); + } break; case 'update': await SolrManager.update(); @@ -66,13 +69,13 @@ export namespace SolrManager { export async function update() { console.log(green('Beginning update...')); - await log_execution<void>({ + await logExecution<void>({ startMessage: 'Clearing existing Solr information...', endMessage: 'Solr information successfully cleared', action: Search.clear, color: cyan, }); - const cursor = await log_execution({ + const cursor = await logExecution({ startMessage: 'Connecting to and querying for all documents from database...', endMessage: ({ result, error }) => { const success = error === null && result !== undefined; @@ -95,7 +98,7 @@ export namespace SolrManager { if (doc.__type !== 'Doc') { return; } - const fields = doc.fields; + const { fields } = doc; if (!fields) { return; } @@ -118,7 +121,7 @@ export namespace SolrManager { } } await cursor?.forEach(updateDoc); - const result = await log_execution({ + const result = await logExecution({ startMessage: `Dispatching updates for ${updates.length} documents`, endMessage: 'Dispatched updates complete', action: () => Search.updateDocuments(updates), diff --git a/src/server/ApiManagers/SessionManager.ts b/src/server/ApiManagers/SessionManager.ts index e37f8c6db..bebe50a62 100644 --- a/src/server/ApiManagers/SessionManager.ts +++ b/src/server/ApiManagers/SessionManager.ts @@ -1,67 +1,64 @@ -import ApiManager, { Registration } from "./ApiManager"; -import { Method, _permission_denied, AuthorizedCore, SecureHandler } from "../RouteManager"; -import RouteSubscriber from "../RouteSubscriber"; -import { sessionAgent } from ".."; -import { DashSessionAgent } from "../DashSession/DashSessionAgent"; +import ApiManager, { Registration } from './ApiManager'; +import { Method, _permissionDenied, AuthorizedCore, SecureHandler } from '../RouteManager'; +import RouteSubscriber from '../RouteSubscriber'; +import { sessionAgent } from '..'; +import { DashSessionAgent } from '../DashSession/DashSessionAgent'; -const permissionError = "You are not authorized!"; +const permissionError = 'You are not authorized!'; export default class SessionManager extends ApiManager { + private secureSubscriber = (root: string, ...params: string[]) => new RouteSubscriber(root).add('session_key', ...params); - private secureSubscriber = (root: string, ...params: string[]) => new RouteSubscriber(root).add("session_key", ...params); - - private authorizedAction = (handler: SecureHandler) => { - return (core: AuthorizedCore) => { - const { req: { params }, res } = core; - if (!process.env.MONITORED) { - return res.send("This command only makes sense in the context of a monitored session."); - } - if (params.session_key !== process.env.session_key) { - return _permission_denied(res, permissionError); - } - return handler(core); - }; - } + private authorizedAction = (handler: SecureHandler) => (core: AuthorizedCore) => { + const { + req: { params }, + res, + } = core; + if (!process.env.MONITORED) { + return res.send('This command only makes sense in the context of a monitored session.'); + } + if (params.session_key !== process.env.session_key) { + return _permissionDenied(res, permissionError); + } + return handler(core); + }; protected initialize(register: Registration): void { - register({ method: Method.GET, - subscription: this.secureSubscriber("debug", "to?"), + subscription: this.secureSubscriber('debug', 'to?'), secureHandler: this.authorizedAction(async ({ req: { params }, res }) => { const to = params.to || DashSessionAgent.notificationRecipient; - const { error } = await sessionAgent.serverWorker.emit("debug", { to }); + const { error } = await sessionAgent.serverWorker.emit('debug', { to }); res.send(error ? error.message : `Your request was successful: the server captured and compressed (but did not save) a new back up. It was sent to ${to}.`); - }) + }), }); register({ method: Method.GET, - subscription: this.secureSubscriber("backup"), + subscription: this.secureSubscriber('backup'), secureHandler: this.authorizedAction(async ({ res }) => { - const { error } = await sessionAgent.serverWorker.emit("backup"); - res.send(error ? error.message : "Your request was successful: the server successfully created a new back up."); - }) + const { error } = await sessionAgent.serverWorker.emit('backup'); + res.send(error ? error.message : 'Your request was successful: the server successfully created a new back up.'); + }), }); register({ method: Method.GET, - subscription: this.secureSubscriber("kill"), + subscription: this.secureSubscriber('kill'), secureHandler: this.authorizedAction(({ res }) => { - res.send("Your request was successful: the server and its session have been killed."); - sessionAgent.killSession("an authorized user has manually ended the server session via the /kill route"); - }) + res.send('Your request was successful: the server and its session have been killed.'); + sessionAgent.killSession('an authorized user has manually ended the server session via the /kill route'); + }), }); register({ method: Method.GET, - subscription: this.secureSubscriber("deleteSession"), + subscription: this.secureSubscriber('deleteSession'), secureHandler: this.authorizedAction(async ({ res }) => { - const { error } = await sessionAgent.serverWorker.emit("delete"); - res.send(error ? error.message : "Your request was successful: the server successfully deleted the database. Return to /home."); - }) + const { error } = await sessionAgent.serverWorker.emit('delete'); + res.send(error ? error.message : 'Your request was successful: the server successfully deleted the database. Return to /home.'); + }), }); - } - -}
\ No newline at end of file +} diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2306b6589..4cb3d8baf 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -1,50 +1,27 @@ +import * as AdmZip from 'adm-zip'; import * as formidable from 'formidable'; -import { createReadStream, createWriteStream, unlink, writeFile } from 'fs'; -import * as path from 'path'; +import * as fs from 'fs'; +import { createReadStream, createWriteStream, unlink } from 'fs'; +import * as imageDataUri from 'image-data-uri'; import Jimp from 'jimp'; -import { filesDirectory, publicDirectory } from '..'; +import * as path from 'path'; +import * as uuid from 'uuid'; import { retrocycle } from '../../decycler/decycler'; +import { DashVersion } from '../../fields/DocSymbols'; import { DashUploadUtils, InjectSize, SizeSuffix } from '../DashUploadUtils'; -import { Database } from '../database'; import { Method, _success } from '../RouteManager'; -import RouteSubscriber from '../RouteSubscriber'; import { AcceptableMedia, Upload } from '../SharedMediaTypes'; +import { clientPathToFile, Directory, pathToDirectory, publicDirectory, serverPathToFile } from '../SocketData'; +import { Database } from '../database'; import ApiManager, { Registration } from './ApiManager'; import { SolrManager } from './SearchManager'; -import * as uuid from 'uuid'; -import { DashVersion } from '../../fields/DocSymbols'; -import * as AdmZip from 'adm-zip'; -import * as imageDataUri from 'image-data-uri'; -import * as fs from 'fs'; - -export enum Directory { - parsed_files = 'parsed_files', - images = 'images', - videos = 'videos', - pdfs = 'pdfs', - text = 'text', - audio = 'audio', - csv = 'csv', -} - -export function serverPathToFile(directory: Directory, filename: string) { - return path.normalize(`${filesDirectory}/${directory}/${filename}`); -} - -export function pathToDirectory(directory: Directory) { - return path.normalize(`${filesDirectory}/${directory}`); -} - -export function clientPathToFile(directory: Directory, filename: string) { - return `/files/${directory}/${filename}`; -} export default class UploadManager extends ApiManager { protected initialize(register: Registration): void { register({ method: Method.POST, subscription: '/ping', - secureHandler: async ({ req, res }) => { + secureHandler: async ({ /* req, */ res }) => { _success(res, { message: DashVersion, date: new Date() }); }, }); @@ -78,31 +55,33 @@ export default class UploadManager extends ApiManager { form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ${filesize}`))); return new Promise<void>(resolve => { form.parse(req, async (_err, _fields, files) => { - const results: Upload.FileResponse[] = []; if (_err?.message) { - results.push({ - source: { - filepath: '', - originalFilename: 'none', - newFilename: 'none', - mimetype: 'text', - size: 0, - hashAlgorithm: 'md5', - toJSON: () => ({ name: 'none', size: 0, length: 0, mtime: new Date(), filepath: '', originalFilename: 'none', newFilename: 'none', mimetype: 'text' }), + _success(res, [ + { + source: { + filepath: '', + originalFilename: 'none', + newFilename: 'none', + mimetype: 'text', + size: 0, + hashAlgorithm: 'md5', + toJSON: () => ({ name: 'none', size: 0, length: 0, mtime: new Date(), filepath: '', originalFilename: 'none', newFilename: 'none', mimetype: 'text' }), + }, + result: { name: 'failed upload', message: `${_err.message}` }, }, - result: { name: 'failed upload', message: `${_err.message}` }, - }); - } - fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`)); + ]); + } else { + fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `resampling images`)); + const results = ( + await Promise.all( + Array.from(Object.keys(files)).map( + async key => (!files[key] ? undefined : DashUploadUtils.upload(files[key]![0] /* , key */)) // key is the guid used by the client to track upload progress. + ) + ) + ).filter(result => result && !(result.result instanceof Error)); - for (const key in files) { - const f = files[key]; - if (f) { - const result = await DashUploadUtils.upload(f[0], key); // key is the guid used by the client to track upload progress. - result && !(result.result instanceof Error) && results.push(result); - } + _success(res, results); } - _success(res, results); resolve(); }); }); @@ -113,17 +92,14 @@ export default class UploadManager extends ApiManager { method: Method.POST, subscription: '/uploadYoutubeVideo', secureHandler: async ({ req, res }) => { - //req.readableBuffer.head.data - return new Promise<void>(async resolve => { - req.addListener('data', async args => { - const payload = String.fromCharCode.apply(String, args); - const { videoId, overwriteId } = JSON.parse(payload); - const results: Upload.FileResponse[] = []; - const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId); - result && results.push(result); - _success(res, results); - resolve(); - }); + // req.readableBuffer.head.data + req.addListener('data', async args => { + const payload = String.fromCharCode(...args); // .apply(String, args); + const { videoId, overwriteId } = JSON.parse(payload); + const results: Upload.FileResponse[] = []; + const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId); + result && results.push(result); + _success(res, results); }); }, }); @@ -132,49 +108,10 @@ export default class UploadManager extends ApiManager { method: Method.POST, subscription: '/queryYoutubeProgress', secureHandler: async ({ req, res }) => { - return new Promise<void>(async resolve => { - req.addListener('data', args => { - const payload = String.fromCharCode.apply(String, args); - const videoId = JSON.parse(payload).videoId; - _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId, req.user) }); - resolve(); - }); - }); - }, - }); - - register({ - method: Method.POST, - subscription: new RouteSubscriber('youtubeScreenshot'), - secureHandler: async ({ req, res }) => { - const { id, timecode } = req.body; - const convert = (raw: string) => { - const number = Math.floor(Number(raw)); - const seconds = number % 60; - const minutes = (number - seconds) / 60; - return `${minutes}m${seconds}s`; - }; - const suffix = timecode ? `&t=${convert(timecode)}` : ``; - const targetUrl = `https://www.youtube.com/watch?v=${id}${suffix}`; - const buffer = await captureYoutubeScreenshot(targetUrl); - if (!buffer) { - return res.send(); - } - const resolvedName = `youtube_capture_${id}_${suffix}.png`; - const resolvedPath = serverPathToFile(Directory.images, resolvedName); - return new Promise<void>(resolve => { - writeFile(resolvedPath, buffer, async error => { - if (error) { - return res.send(); - } - await DashUploadUtils.outputResizedImages(resolvedPath, resolvedName, pathToDirectory(Directory.images)); - res.send({ - accessPaths: { - agnostic: DashUploadUtils.getAccessPaths(Directory.images, resolvedName), - }, - } as Upload.FileInformation); - resolve(); - }); + req.addListener('data', args => { + const payload = String.fromCharCode(...args); // .apply(String, args); + const { videoId } = JSON.parse(payload); + _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) }); }); }, }); @@ -186,7 +123,8 @@ export default class UploadManager extends ApiManager { const { sources } = req.body; if (Array.isArray(sources)) { const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source))); - return res.send(results); + res.send(results); + return; } res.send(); }, @@ -203,20 +141,22 @@ export default class UploadManager extends ApiManager { const getId = (id: string): string => { if (!remap || id.endsWith('Proto')) return id; if (id in ids) return ids[id]; - return (ids[id] = uuid.v4()); + ids[id] = uuid.v4(); + return ids[id]; }; - const mapFn = (doc: any) => { + const mapFn = (docIn: any) => { + const doc = docIn; if (doc.id) { doc.id = getId(doc.id); } + // eslint-disable-next-line no-restricted-syntax for (const key in doc.fields) { - if (!doc.fields.hasOwnProperty(key)) { - continue; - } + // eslint-disable-next-line no-continue + if (!Object.prototype.hasOwnProperty.call(doc.fields, key)) continue; + const field = doc.fields[key]; - if (field === undefined || field === null) { - continue; - } + // eslint-disable-next-line no-continue + if (field === undefined || field === null) continue; if (field.__type === 'Doc') { mapFn(field); @@ -229,78 +169,80 @@ export default class UploadManager extends ApiManager { } else if (field.__type === 'list') { mapFn(field); } else if (typeof field === 'string') { - const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w\-]*)"/g; - doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); + const re = /("(?:dataD|d)ocumentId"\s*:\s*")([\w-]*)"/g; + doc.fields[key] = (field as any).replace(re, (match: any, p1: string, p2: string) => `${p1}${getId(p2)}"`); } else if (field.__type === 'RichTextField') { const re = /("href"\s*:\s*")(.*?)"/g; - field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => { - return `${p1}${getId(p2)}"`; - }); + field.Data = field.Data.replace(re, (match: any, p1: string, p2: string) => `${p1}${getId(p2)}"`); } } }; return new Promise<void>(resolve => { form.parse(req, async (_err, fields, files) => { - remap = Object.keys(fields).some(key => key === 'remap' && !fields.remap?.includes('false')); //.remap !== 'false'; // bcz: looking to see if the field 'remap' is set to 'false' + remap = Object.keys(fields).some(key => key === 'remap' && !fields.remap?.includes('false')); // .remap !== 'false'; // bcz: looking to see if the field 'remap' is set to 'false' let id: string = ''; let docids: string[] = []; let linkids: string[] = []; try { - for (const name in files) { - const f = files[name]; - if (!f) continue; - const path_2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set? - const zip = new AdmZip(path_2.filepath); - zip.getEntries().forEach((entry: any) => { - let entryName = entry.entryName.replace(/%%%/g, '/'); - if (!entryName.startsWith('files/')) { - return; - } - const extension = path.extname(entryName); - const pathname = publicDirectory + '/' + entry.entryName; - const targetname = publicDirectory + '/' + entryName; - try { - zip.extractEntryTo(entry.entryName, publicDirectory, true, false); - createReadStream(pathname).pipe(createWriteStream(targetname)); - Jimp.read(pathname).then(img => { - DashUploadUtils.imageResampleSizes(extension).forEach(({ width, suffix }) => { - const outputPath = InjectSize(targetname, suffix); - if (!width) createReadStream(pathname).pipe(createWriteStream(outputPath)); - else img = img.resize(width, Jimp.AUTO).write(outputPath); + // eslint-disable-next-line no-restricted-syntax + for (const name in Object.keys(files)) { + if (Object.prototype.hasOwnProperty.call(files, name)) { + const f = files[name]; + // eslint-disable-next-line no-continue + if (!f) continue; + const path2 = f[0]; // what about the rest of the array? are we guaranteed only one value is set? + const zip = new AdmZip(path2.filepath); + zip.getEntries().forEach((entry: any) => { + const entryName = entry.entryName.replace(/%%%/g, '/'); + if (!entryName.startsWith('files/')) { + return; + } + const extension = path.extname(entryName); + const pathname = publicDirectory + '/' + entry.entryName; + const targetname = publicDirectory + '/' + entryName; + try { + zip.extractEntryTo(entry.entryName, publicDirectory, true, false); + createReadStream(pathname).pipe(createWriteStream(targetname)); + Jimp.read(pathname).then(imgIn => { + let img = imgIn; + DashUploadUtils.imageResampleSizes(extension).forEach(({ width, suffix }) => { + const outputPath = InjectSize(targetname, suffix); + if (!width) createReadStream(pathname).pipe(createWriteStream(outputPath)); + else img = img.resize(width, Jimp.AUTO).write(outputPath); + }); + unlink(pathname, () => {}); }); - unlink(pathname, () => {}); - }); - } catch (e) { - console.log(e); - } - }); - const json = zip.getEntry('docs.json'); - if (json) { - try { - const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); - const { docs, links } = data; - id = getId(data.id); - const rdocs = Object.keys(docs).map(key => docs[key]); - const ldocs = Object.keys(links).map(key => links[key]); - [...rdocs, ...ldocs].forEach(mapFn); - docids = rdocs.map(doc => doc.id); - linkids = ldocs.map(link => link.id); - await Promise.all( - [...rdocs, ...ldocs].map( - doc => - new Promise<void>(res => { - // overwrite mongo doc with json doc contents - Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true); - }) - ) - ); - } catch (e) { - console.log(e); + } catch (e) { + console.log(e); + } + }); + const json = zip.getEntry('docs.json'); + if (json) { + try { + const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); + const { docs, links } = data; + id = getId(data.id); + const rdocs = Object.keys(docs).map(key => docs[key]); + const ldocs = Object.keys(links).map(key => links[key]); + [...rdocs, ...ldocs].forEach(mapFn); + docids = rdocs.map(doc => doc.id); + linkids = ldocs.map(link => link.id); + // eslint-disable-next-line no-await-in-loop + await Promise.all( + [...rdocs, ...ldocs].map( + doc => + new Promise<void>(dbRes => { + // overwrite mongo doc with json doc contents + Database.Instance.replace(doc.id, doc, err => dbRes(err && console.log(err)), true); + }) + ) + ); + } catch (e) { + console.log(e); + } } + unlink(path2.filepath, () => {}); } - unlink(path_2.filepath, () => {}); } SolrManager.update(); res.send(JSON.stringify({ id, docids, linkids } || 'error')); @@ -319,9 +261,8 @@ export default class UploadManager extends ApiManager { secureHandler: async ({ req, res }) => { const { source } = req.body; if (typeof source === 'string') { - return res.send(await DashUploadUtils.InspectImage(source)); - } - res.send({}); + res.send(await DashUploadUtils.InspectImage(source)); + } else res.send({}); }, }); @@ -329,7 +270,7 @@ export default class UploadManager extends ApiManager { method: Method.POST, subscription: '/uploadURI', secureHandler: ({ req, res }) => { - const uri: any = req.body.uri; + const { uri } = req.body; const filename = req.body.name; const origSuffix = req.body.nosuffix ? SizeSuffix.None : SizeSuffix.Original; const deleteFiles = req.body.replaceRootFilename; @@ -338,23 +279,24 @@ export default class UploadManager extends ApiManager { return; } if (deleteFiles) { - const path = serverPathToFile(Directory.images, ''); + const serverPath = serverPathToFile(Directory.images, ''); const regex = new RegExp(`${deleteFiles}.*`); - fs.readdirSync(path) + fs.readdirSync(serverPath) .filter((f: any) => regex.test(f)) - .map((f: any) => fs.unlinkSync(path + f)); + .map((f: any) => fs.unlinkSync(serverPath + f)); } - return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => { + imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => { const ext = path.extname(savedName).toLowerCase(); if (AcceptableMedia.imageFormats.includes(ext)) { - Jimp.read(savedName).then(img => + Jimp.read(savedName).then(imgIn => { + let img = imgIn; (!origSuffix ? [{ width: 400, suffix: SizeSuffix.Medium }] : Object.values(DashUploadUtils.Sizes)) // .forEach(({ width, suffix }) => { const outputPath = serverPathToFile(Directory.images, InjectSize(filename, suffix) + ext); if (!width) createReadStream(savedName).pipe(createWriteStream(outputPath)); else img = img.resize(width, Jimp.AUTO).write(outputPath); - }) - ); + }); + }); } res.send(clientPathToFile(Directory.images, filename + ext)); }); @@ -362,35 +304,3 @@ export default class UploadManager extends ApiManager { }); } } -function delay(ms: number) { - return new Promise(resolve => setTimeout(resolve, ms)); -} -/** - * On success, returns a buffer containing the bytes of a screenshot - * of the video (optionally, at a timecode) specified by @param targetUrl. - * - * On failure, returns undefined. - */ -async function captureYoutubeScreenshot(targetUrl: string) { - // const browser = await launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] }); - // const page = await browser.newPage(); - // // await page.setViewport({ width: 1920, height: 1080 }); - - // // await page.goto(targetUrl, { waitUntil: 'domcontentloaded' as any }); - - // const videoPlayer = await page.$('.html5-video-player'); - // videoPlayer && await page.focus("video"); - // await delay(7000); - // const ad = await page.$('.ytp-ad-skip-button-text'); - // await ad?.click(); - // await videoPlayer?.click(); - // await delay(1000); - // // hide youtube player controls. - // await page.evaluate(() => (document.querySelector('.ytp-chrome-bottom') as HTMLElement).style.display = 'none'); - - // const buffer = await videoPlayer?.screenshot({ encoding: "binary" }); - // await browser.close(); - - // return buffer; - return null; -} diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index 0431b9bcf..b587340e2 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -1,16 +1,14 @@ -import ApiManager, { Registration } from './ApiManager'; -import { Method } from '../RouteManager'; -import { Database } from '../database'; -import { msToTime } from '../ActionUtilities'; import * as bcrypt from 'bcrypt-nodejs'; +import { check, validationResult } from 'express-validator'; +import { Utils } from '../../Utils'; import { Opt } from '../../fields/Doc'; -import { WebSocket } from '../websocket'; -import { resolvedPorts } from '../server_Initialization'; import { DashVersion } from '../../fields/DocSymbols'; -import { Utils } from '../../Utils'; -import { check, validationResult } from 'express-validator'; +import { msToTime } from '../ActionUtilities'; +import { Method } from '../RouteManager'; +import { resolvedPorts, socketMap, timeMap } from '../SocketData'; +import { Database } from '../database'; +import ApiManager, { Registration } from './ApiManager'; -export const timeMap: { [id: string]: number } = {}; interface ActivityUnit { user: string; duration: number; @@ -32,9 +30,10 @@ export default class UserManager extends ApiManager { method: Method.POST, subscription: '/setCacheDocumentIds', secureHandler: async ({ user, req, res }) => { + const userModel = user; const result: any = {}; - user.cacheDocumentIds = req.body.cacheDocumentIds; - user.save().then(undefined, err => { + userModel.cacheDocumentIds = req.body.cacheDocumentIds; + userModel.save().then(undefined, (err: any) => { if (err) { result.error = [{ msg: 'Error while caching documents' }]; } @@ -90,17 +89,19 @@ export default class UserManager extends ApiManager { method: Method.POST, subscription: '/internalResetPassword', secureHandler: async ({ user, req, res }) => { + const userModel = user; const result: any = {}; - const { curr_pass, new_pass, new_confirm } = req.body; + // eslint-disable-next-line camelcase + const { curr_pass, new_pass } = req.body; // perhaps should assert whether curr password is entered correctly const validated = await new Promise<Opt<boolean>>(resolve => { - bcrypt.compare(curr_pass, user.password, (err, passwords_match) => { - if (err || !passwords_match) { + bcrypt.compare(curr_pass, userModel.password, (err, passwordsMatch) => { + if (err || !passwordsMatch) { result.error = [{ msg: 'Incorrect current password' }]; res.send(result); resolve(undefined); } else { - resolve(passwords_match); + resolve(passwordsMatch); } }); }); @@ -111,10 +112,11 @@ export default class UserManager extends ApiManager { check('new_pass', 'Password must be at least 4 characters long') .run(req) - .then(chcekcres => console.log(chcekcres)); //.len({ min: 4 }); + .then(chcekcres => console.log(chcekcres)); // .len({ min: 4 }); check('new_confirm', 'Passwords do not match') .run(req) - .then(theres => console.log(theres)); //.equals(new_pass); + .then(theres => console.log(theres)); // .equals(new_pass); + // eslint-disable-next-line camelcase if (curr_pass === new_pass) { result.error = [{ msg: 'Current and new password are the same' }]; } @@ -125,12 +127,13 @@ export default class UserManager extends ApiManager { // will only change password if there are no errors. if (!result.error) { - user.password = new_pass; - user.passwordResetToken = undefined; - user.passwordResetExpires = undefined; + // eslint-disable-next-line camelcase + userModel.password = new_pass; + userModel.passwordResetToken = undefined; + userModel.passwordResetExpires = undefined; } - user.save().then(undefined, err => { + userModel.save().then(undefined, err => { if (err) { result.error = [{ msg: 'Error while saving new password' }]; } @@ -149,13 +152,16 @@ export default class UserManager extends ApiManager { const activeTimes: ActivityUnit[] = []; const inactiveTimes: ActivityUnit[] = []; + // eslint-disable-next-line no-restricted-syntax for (const user in timeMap) { - const time = timeMap[user]; - const socketPair = Array.from(WebSocket.socketMap).find(pair => pair[1] === user); - if (socketPair && !socketPair[0].disconnected) { - const duration = now - time; - const target = duration / 1000 < 60 * 5 ? activeTimes : inactiveTimes; - target.push({ user, duration }); + if (Object.prototype.hasOwnProperty.call(timeMap, user)) { + const time = timeMap[user]; + const socketPair = Array.from(socketMap).find(pair => pair[1] === user); + if (socketPair && !socketPair[0].disconnected) { + const duration = now - time; + const target = duration / 1000 < 60 * 5 ? activeTimes : inactiveTimes; + target.push({ user, duration }); + } } } diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index e657866ce..8ad421a30 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -1,6 +1,7 @@ -import ApiManager, { Registration } from "./ApiManager"; -import { Method } from "../RouteManager"; import { exec } from 'child_process'; +import ApiManager, { Registration } from './ApiManager'; +import { Method } from '../RouteManager'; + // import { IBM_Recommender } from "../../client/apis/IBM_Recommender"; // import { Recommender } from "../Recommender"; @@ -8,9 +9,7 @@ import { exec } from 'child_process'; // recommender.testModel(); export default class UtilManager extends ApiManager { - protected initialize(register: Registration): void { - // register({ // method: Method.POST, // subscription: "/IBMAnalysis", @@ -33,26 +32,25 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, - subscription: "/pull", - secureHandler: async ({ res }) => { - return new Promise<void>(resolve => { + subscription: '/pull', + secureHandler: async ({ res }) => + new Promise<void>(resolve => { exec('"C:\\Program Files\\Git\\git-bash.exe" -c "git pull"', err => { if (err) { res.send(err.message); return; } - res.redirect("/"); + res.redirect('/'); resolve(); }); - }); - } + }), }); register({ method: Method.GET, - subscription: "/version", - secureHandler: ({ res }) => { - return new Promise<void>(resolve => { + subscription: '/version', + secureHandler: ({ res }) => + new Promise<void>(resolve => { exec('"C:\\Program Files\\Git\\bin\\git.exe" rev-parse HEAD', (err, stdout) => { if (err) { res.send(err.message); @@ -61,10 +59,7 @@ export default class UtilManager extends ApiManager { res.send(stdout); }); resolve(); - }); - } + }), }); - } - -}
\ No newline at end of file +} diff --git a/src/server/Client.ts b/src/server/Client.ts index e6f953712..f67999c5b 100644 --- a/src/server/Client.ts +++ b/src/server/Client.ts @@ -1,4 +1,4 @@ -import { computed } from "mobx"; +import { computed } from 'mobx'; export class Client { private _guid: string; @@ -7,5 +7,7 @@ export class Client { this._guid = guid; } - @computed public get GUID(): string { return this._guid; } -}
\ No newline at end of file + @computed public get GUID(): string { + return this._guid; + } +} diff --git a/src/server/DashSession/DashSessionAgent.ts b/src/server/DashSession/DashSessionAgent.ts index 1ef7a131d..f937c17ad 100644 --- a/src/server/DashSession/DashSessionAgent.ts +++ b/src/server/DashSession/DashSessionAgent.ts @@ -1,18 +1,19 @@ -import { Email, pathFromRoot } from '../ActionUtilities'; -import { red, yellow, green, cyan } from 'colors'; -import { get } from 'request-promise'; -import { Utils } from '../../Utils'; -import { WebSocket } from '../websocket'; -import { MessageStore } from '../Message'; -import { launchServer, onWindows } from '..'; -import { readdirSync, statSync, createWriteStream, readFileSync, unlinkSync } from 'fs'; import * as Archiver from 'archiver'; +import { cyan, green, red, yellow } from 'colors'; +import { createWriteStream, readFileSync, readdirSync, statSync, unlinkSync } from 'fs'; import { resolve } from 'path'; +import { get } from 'request-promise'; import { rimraf } from 'rimraf'; +import { launchServer, onWindows } from '..'; +import { Utils } from '../../Utils'; +import { ServerUtils } from '../../ServerUtils'; +import { Email, pathFromRoot } from '../ActionUtilities'; +import { MessageStore } from '../Message'; +import { WebSocket } from '../websocket'; import { AppliedSessionAgent, ExitHandler } from './Session/agents/applied_session_agent'; -import { ServerWorker } from './Session/agents/server_worker'; import { Monitor } from './Session/agents/monitor'; -import { MessageHandler, ErrorLike } from './Session/agents/promisified_ipc_manager'; +import { ErrorLike, MessageHandler } from './Session/agents/promisified_ipc_manager'; +import { ServerWorker } from './Session/agents/server_worker'; /** * If we're the monitor (master) thread, we should launch the monitor logic for the session. @@ -22,6 +23,7 @@ import { MessageHandler, ErrorLike } from './Session/agents/promisified_ipc_mana export class DashSessionAgent extends AppliedSessionAgent { private readonly signature = '-Dash Server Session Manager'; private readonly releaseDesktop = pathFromRoot('../../Desktop'); + public static notificationRecipient = 'browndashptc@gmail.com'; /** * The core method invoked when the single master thread is initialized. @@ -149,7 +151,7 @@ export class DashSessionAgent extends AppliedSessionAgent { const { _socket } = WebSocket; if (_socket) { const message = typeof reason === 'boolean' ? (reason ? 'exit' : 'temporary') : 'crash'; - Utils.Emit(_socket, MessageStore.ConnectionTerminated, message); + ServerUtils.Emit(_socket, MessageStore.ConnectionTerminated, message); } }; @@ -217,7 +219,3 @@ export class DashSessionAgent extends AppliedSessionAgent { } } } - -export namespace DashSessionAgent { - export const notificationRecipient = 'browndashptc@gmail.com'; -} diff --git a/src/server/DashSession/Session/agents/applied_session_agent.ts b/src/server/DashSession/Session/agents/applied_session_agent.ts index 2037e93e5..c42ba95cc 100644 --- a/src/server/DashSession/Session/agents/applied_session_agent.ts +++ b/src/server/DashSession/Session/agents/applied_session_agent.ts @@ -1,13 +1,13 @@ -import * as _cluster from "cluster"; -import { Monitor } from "./monitor"; -import { ServerWorker } from "./server_worker"; +import * as _cluster from 'cluster'; +import { Monitor } from './monitor'; +import { ServerWorker } from './server_worker'; + const cluster = _cluster as any; const isMaster = cluster.isPrimary; export type ExitHandler = (reason: Error | boolean) => void | Promise<void>; export abstract class AppliedSessionAgent { - // the following two methods allow the developer to create a custom // session and use the built in customization options for each thread protected abstract initializeMonitor(monitor: Monitor): Promise<string>; @@ -18,15 +18,15 @@ export abstract class AppliedSessionAgent { public killSession = (reason: string, graceful = true, errorCode = 0) => { const target = cluster.default.isPrimary ? this.sessionMonitor : this.serverWorker; target.killSession(reason, graceful, errorCode); - } + }; private sessionMonitorRef: Monitor | undefined; public get sessionMonitor(): Monitor { if (!cluster.default.isPrimary) { - this.serverWorker.emit("kill", { + this.serverWorker.emit('kill', { graceful: false, - reason: "Cannot access the session monitor directly from the server worker thread.", - errorCode: 1 + reason: 'Cannot access the session monitor directly from the server worker thread.', + errorCode: 1, }); throw new Error(); } @@ -36,7 +36,7 @@ export abstract class AppliedSessionAgent { private serverWorkerRef: ServerWorker | undefined; public get serverWorker(): ServerWorker { if (isMaster) { - throw new Error("Cannot access the server worker directly from the session monitor thread"); + throw new Error('Cannot access the server worker directly from the session monitor thread'); } return this.serverWorkerRef!; } @@ -52,8 +52,7 @@ export abstract class AppliedSessionAgent { this.serverWorkerRef = await this.initializeServerWorker(); } } else { - throw new Error("Cannot launch a session thread more than once per process."); + throw new Error('Cannot launch a session thread more than once per process.'); } } - -}
\ No newline at end of file +} diff --git a/src/server/DashSession/Session/agents/monitor.ts b/src/server/DashSession/Session/agents/monitor.ts index a6fde4356..6cdad46c2 100644 --- a/src/server/DashSession/Session/agents/monitor.ts +++ b/src/server/DashSession/Session/agents/monitor.ts @@ -1,21 +1,19 @@ -import { ExitHandler } from './applied_session_agent'; -import { Configuration, configurationSchema, defaultConfig, Identifiers, colorMapping } from '../utilities/session_config'; -import Repl, { ReplAction } from '../utilities/repl'; +import { ExecOptions, exec } from 'child_process'; import * as _cluster from 'cluster'; import { Worker } from 'cluster'; -import { manage, MessageHandler, ErrorLike } from './promisified_ipc_manager'; -import { red, cyan, white, yellow, blue } from 'colors'; -import { exec, ExecOptions } from 'child_process'; -import { validate, ValidationError } from 'jsonschema'; -import { Utilities } from '../utilities/utilities'; +import { blue, cyan, red, white, yellow } from 'colors'; import { readFileSync } from 'fs'; +import { ValidationError, validate } from 'jsonschema'; +import Repl, { ReplAction } from '../utilities/repl'; +import { Configuration, Identifiers, colorMapping, configurationSchema, defaultConfig } from '../utilities/session_config'; +import { Utilities } from '../utilities/utilities'; +import { ExitHandler } from './applied_session_agent'; import IPCMessageReceiver from './process_message_router'; +import { ErrorLike, MessageHandler, manage } from './promisified_ipc_manager'; import { ServerWorker } from './server_worker'; + const cluster = _cluster as any; -const isWorker = cluster.isWorker; -const setupMaster = cluster.setupPrimary; -const on = cluster.on; -const fork = cluster.fork; +const { isWorker, setupMaster, on, fork } = cluster; /** * Validates and reads the configuration file, accordingly builds a child process factory @@ -41,9 +39,8 @@ export class Monitor extends IPCMessageReceiver { } else if (++Monitor.count > 1) { console.error(red('cannot create more than one monitor.')); process.exit(1); - } else { - return new Monitor(); } + return new Monitor(); } private constructor() { @@ -128,25 +125,25 @@ export class Monitor extends IPCMessageReceiver { this.repl.registerCommand(basename, argPatterns, action); }; - public exec = (command: string, options?: ExecOptions) => { - return new Promise<void>(resolve => { + public exec = (command: string, options?: ExecOptions) => + new Promise<void>(resolve => { exec(command, { ...options, encoding: 'utf8' }, (error, stdout, stderr) => { if (error) { this.execLog(red(`unable to execute ${white(command)}`)); error.message.split('\n').forEach(line => line.length && this.execLog(red(`(error) ${line}`))); } else { - let outLines: string[], errorLines: string[]; - if ((outLines = stdout.split('\n').filter(line => line.length)).length) { + const outLines = stdout.split('\n').filter(line => line.length); + if (outLines.length) { outLines.forEach(line => line.length && this.execLog(cyan(`(stdout) ${line}`))); } - if ((errorLines = stderr.split('\n').filter(line => line.length)).length) { + const errorLines = stderr.split('\n').filter(line => line.length); + if (errorLines.length) { errorLines.forEach(line => line.length && this.execLog(yellow(`(stderr) ${line}`))); } } resolve(); }); }); - }; /** * Generates a blue UTC string associated with the time @@ -226,12 +223,10 @@ export class Monitor extends IPCMessageReceiver { const newPollingIntervalSeconds = Math.floor(Number(args[1])); if (newPollingIntervalSeconds < 0) { this.mainLog(red('the polling interval must be a non-negative integer')); - } else { - if (newPollingIntervalSeconds !== this.config.polling.intervalSeconds) { - this.config.polling.intervalSeconds = newPollingIntervalSeconds; - if (args[2] === 'true') { - Monitor.IPCManager.emit('updatePollingInterval', { newPollingIntervalSeconds }); - } + } else if (newPollingIntervalSeconds !== this.config.polling.intervalSeconds) { + this.config.polling.intervalSeconds = newPollingIntervalSeconds; + if (args[2] === 'true') { + Monitor.IPCManager.emit('updatePollingInterval', { newPollingIntervalSeconds }); } } }); @@ -297,6 +292,7 @@ export class Monitor extends IPCMessageReceiver { }; } +// eslint-disable-next-line no-redeclare export namespace Monitor { export enum IntrinsicEvents { KeyGenerated = 'key_generated', diff --git a/src/server/DashSession/Session/agents/process_message_router.ts b/src/server/DashSession/Session/agents/process_message_router.ts index 0745ea455..3e2b7d8d0 100644 --- a/src/server/DashSession/Session/agents/process_message_router.ts +++ b/src/server/DashSession/Session/agents/process_message_router.ts @@ -1,7 +1,6 @@ -import { MessageHandler, PromisifiedIPCManager, HandlerMap } from "./promisified_ipc_manager"; +import { MessageHandler, PromisifiedIPCManager, HandlerMap } from './promisified_ipc_manager'; export default abstract class IPCMessageReceiver { - protected static IPCManager: PromisifiedIPCManager; protected handlers: HandlerMap = {}; @@ -18,7 +17,7 @@ export default abstract class IPCMessageReceiver { } else { handlers.push(handler); } - } + }; /** * Unregister a given listener at this message. @@ -31,11 +30,10 @@ export default abstract class IPCMessageReceiver { handlers.splice(index, 1); } } - } + }; - /** + /** * Unregister all listeners at this message. */ public clearMessageListeners = (...names: string[]) => names.map(name => delete this.handlers[name]); - -}
\ No newline at end of file +} diff --git a/src/server/DashSession/Session/agents/promisified_ipc_manager.ts b/src/server/DashSession/Session/agents/promisified_ipc_manager.ts index 76e218977..99b4d4de3 100644 --- a/src/server/DashSession/Session/agents/promisified_ipc_manager.ts +++ b/src/server/DashSession/Session/agents/promisified_ipc_manager.ts @@ -1,13 +1,14 @@ -import { Utilities } from '../utilities/utilities'; import { ChildProcess } from 'child_process'; +import { Utilities } from '../utilities/utilities'; /** - * Convenience constructor - * @param target the process / worker to which to attach the specialized listeners + * Specifies a general message format for this API */ -export function manage(target: IPCTarget, handlers?: HandlerMap) { - return new PromisifiedIPCManager(target, handlers); -} +export type Message<T = any> = { + name: string; + args?: T; +}; +export type MessageHandler<T = any> = (args: T) => any | Promise<any>; /** * Captures the logic to execute upon receiving a message @@ -22,15 +23,10 @@ export type HandlerMap = { [name: string]: MessageHandler[] }; */ export type IPCTarget = NodeJS.Process | ChildProcess; -/** - * Specifies a general message format for this API - */ -export type Message<T = any> = { - name: string; - args?: T; -}; -export type MessageHandler<T = any> = (args: T) => any | Promise<any>; - +interface Metadata { + isResponse: boolean; + id: string; +} /** * When a message is emitted, it is embedded with private metadata * to facilitate the resolution of promises, etc. @@ -38,10 +34,6 @@ export type MessageHandler<T = any> = (args: T) => any | Promise<any>; interface InternalMessage extends Message { metadata: Metadata; } -interface Metadata { - isResponse: boolean; - id: string; -} /** * Allows for the transmission of the error's key features over IPC. @@ -95,7 +87,7 @@ export class PromisifiedIPCManager { } return new Promise<Response<T>>(resolve => { const messageId = Utilities.guid(); - type InternalMessageHandler = (message: any /* MessageListener*/) => any | Promise<any>; + type InternalMessageHandler = (message: any /* MessageListener */) => any | Promise<any>; const responseHandler: InternalMessageHandler = ({ metadata: { id, isResponse }, args }) => { if (isResponse && id === messageId) { this.target.removeListener('message', responseHandler); @@ -118,8 +110,8 @@ export class PromisifiedIPCManager { * completion response for each of the pending messages, allowing their * promises in the caller to resolve. */ - public destroy = () => { - return new Promise<void>(async resolve => { + public destroy = () => + new Promise<void>(async resolve => { if (this.callerIsTarget) { this.destroyHelper(); } else { @@ -127,7 +119,6 @@ export class PromisifiedIPCManager { } resolve(); }); - }; /** * Dispatches the dummy responses and sets the isDestroyed flag to true. @@ -168,12 +159,20 @@ export class PromisifiedIPCManager { error = e; } if (!this.isDestroyed && this.target.send) { - const metadata = { id, isResponse: true }; + const metadataRes = { id, isResponse: true }; const response: Response = { results, error }; - const message = { name, args: response, metadata }; + const messageRes = { name, args: response, metadata: metadataRes }; delete this.pendingMessages[id]; - this.target.send(message); + this.target.send(messageRes); } } }; } + +/** + * Convenience constructor + * @param target the process / worker to which to attach the specialized listeners + */ +export function manage(target: IPCTarget, handlers?: HandlerMap) { + return new PromisifiedIPCManager(target, handlers); +} diff --git a/src/server/DashSession/Session/agents/server_worker.ts b/src/server/DashSession/Session/agents/server_worker.ts index d8b3ee80b..85e1b31d6 100644 --- a/src/server/DashSession/Session/agents/server_worker.ts +++ b/src/server/DashSession/Session/agents/server_worker.ts @@ -1,10 +1,10 @@ -import cluster from "cluster"; -import { green, red, white, yellow } from "colors"; -import { get } from "request-promise"; -import { ExitHandler } from "./applied_session_agent"; -import { Monitor } from "./monitor"; -import IPCMessageReceiver from "./process_message_router"; -import { ErrorLike, manage } from "./promisified_ipc_manager"; +import cluster from 'cluster'; +import { green, red, white, yellow } from 'colors'; +import { get } from 'request-promise'; +import { ExitHandler } from './applied_session_agent'; +import { Monitor } from './monitor'; +import IPCMessageReceiver from './process_message_router'; +import { ErrorLike, manage } from './promisified_ipc_manager'; /** * Effectively, each worker repairs the connection to the server by reintroducing a consistent state @@ -23,18 +23,17 @@ export class ServerWorker extends IPCMessageReceiver { private isInitialized = false; public static Create(work: Function) { if (cluster.isPrimary) { - console.error(red("cannot create a worker on the monitor process.")); + console.error(red('cannot create a worker on the monitor process.')); process.exit(1); } else if (++ServerWorker.count > 1) { - ServerWorker.IPCManager.emit("kill", { - reason: "cannot create more than one worker on a given worker process.", + ServerWorker.IPCManager.emit('kill', { + reason: 'cannot create more than one worker on a given worker process.', graceful: false, - errorCode: 1 + errorCode: 1, }); process.exit(1); - } else { - return new ServerWorker(work); } + return new ServerWorker(work); } /** @@ -48,7 +47,7 @@ export class ServerWorker extends IPCMessageReceiver { * server worker (child process). This will also kill * this process (child process). */ - public killSession = (reason: string, graceful = true, errorCode = 0) => this.emit<never>("kill", { reason, graceful, errorCode }); + public killSession = (reason: string, graceful = true, errorCode = 0) => this.emit<never>('kill', { reason, graceful, errorCode }); /** * A convenience wrapper to tell the session monitor (parent process) @@ -60,7 +59,7 @@ export class ServerWorker extends IPCMessageReceiver { super(); this.configureInternalHandlers(); ServerWorker.IPCManager = manage(process, this.handlers); - this.lifecycleNotification(green(`initializing process... ${white(`[${process.execPath} ${process.execArgv.join(" ")}]`)}`)); + this.lifecycleNotification(green(`initializing process... ${white(`[${process.execPath} ${process.execArgv.join(' ')}]`)}`)); const { pollingRoute, serverPort, pollingIntervalSeconds, pollingFailureTolerance } = process.env; this.serverPort = Number(serverPort); @@ -78,8 +77,10 @@ export class ServerWorker extends IPCMessageReceiver { */ protected configureInternalHandlers = () => { // updates the local values of variables to the those sent from master - this.on("updatePollingInterval", ({ newPollingIntervalSeconds }) => this.pollingIntervalSeconds = newPollingIntervalSeconds); - this.on("manualExit", async ({ isSessionEnd }) => { + this.on('updatePollingInterval', ({ newPollingIntervalSeconds }) => { + this.pollingIntervalSeconds = newPollingIntervalSeconds; + }); + this.on('manualExit', async ({ isSessionEnd }) => { await ServerWorker.IPCManager.destroy(); await this.executeExitHandlers(isSessionEnd); process.exit(0); @@ -91,7 +92,7 @@ export class ServerWorker extends IPCMessageReceiver { const appropriateError = reason instanceof Error ? reason : new Error(`unhandled rejection: ${reason}`); this.proactiveUnplannedExit(appropriateError); }); - } + }; /** * Execute the list of functions registered to be called @@ -102,7 +103,7 @@ export class ServerWorker extends IPCMessageReceiver { /** * Notify master thread (which will log update in the console) of initialization via IPC. */ - public lifecycleNotification = (event: string) => this.emit("lifecycle", { event }); + public lifecycleNotification = (event: string) => this.emit('lifecycle', { event }); /** * Called whenever the process has a reason to terminate, either through an uncaught exception @@ -120,11 +121,11 @@ export class ServerWorker extends IPCMessageReceiver { this.lifecycleNotification(red(error.message)); await ServerWorker.IPCManager.destroy(); process.exit(1); - } + }; /** * This monitors the health of the server by submitting a get request to whatever port / route specified - * by the configuration every n seconds, where n is also given by the configuration. + * by the configuration every n seconds, where n is also given by the configuration. */ private pollServer = async (): Promise<void> => { await new Promise<void>(resolve => { @@ -156,6 +157,5 @@ export class ServerWorker extends IPCMessageReceiver { }); // controlled, asynchronous infinite recursion achieves a persistent poll that does not submit a new request until the previous has completed this.pollServer(); - } - + }; } diff --git a/src/server/DashSession/Session/utilities/repl.ts b/src/server/DashSession/Session/utilities/repl.ts index 643141286..5d9f15e4c 100644 --- a/src/server/DashSession/Session/utilities/repl.ts +++ b/src/server/DashSession/Session/utilities/repl.ts @@ -1,5 +1,5 @@ -import { createInterface, Interface } from "readline"; -import { red, green, white } from "colors"; +import { createInterface, Interface } from 'readline'; +import { red, green, white } from 'colors'; export interface Configuration { identifier: () => string | string; @@ -32,76 +32,82 @@ export default class Repl { this.interface = createInterface(process.stdin, process.stdout).on('line', this.considerInput); } - private resolvedIdentifier = () => typeof this.identifier === "string" ? this.identifier : this.identifier(); + private resolvedIdentifier = () => (typeof this.identifier === 'string' ? this.identifier : this.identifier()); private usage = (command: string, validCommand: boolean) => { if (validCommand) { const formatted = white(command); - const patterns = green(this.commandMap.get(command)!.map(({ argPatterns }) => `${formatted} ${argPatterns.join(" ")}`).join('\n')); + const patterns = green( + this.commandMap + .get(command)! + .map(({ argPatterns }) => `${formatted} ${argPatterns.join(' ')}`) + .join('\n') + ); return `${this.resolvedIdentifier()}\nthe given arguments do not match any registered patterns for ${formatted}\nthe list of valid argument patterns is given by:\n${patterns}`; - } else { - const resolved = this.keys; - if (resolved) { - return resolved; - } - const members: string[] = []; - const keys = this.commandMap.keys(); - let next: IteratorResult<string>; - while (!(next = keys.next()).done) { - members.push(next.value); - } - return `${this.resolvedIdentifier()} commands: { ${members.sort().join(", ")} }`; } - } + const resolved = this.keys; + if (resolved) { + return resolved; + } + const members: string[] = []; + const keys = this.commandMap.keys(); + let next: IteratorResult<string>; + // eslint-disable-next-line no-cond-assign + while (!(next = keys.next()).done) { + members.push(next.value); + } + return `${this.resolvedIdentifier()} commands: { ${members.sort().join(', ')} }`; + }; private success = (command: string) => `${this.resolvedIdentifier()} completed local execution of ${white(command)}`; public registerCommand = (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => { const existing = this.commandMap.get(basename); - const converted = argPatterns.map(input => input instanceof RegExp ? input : new RegExp(input)); + const converted = argPatterns.map(input => (input instanceof RegExp ? input : new RegExp(input))); const registration = { argPatterns: converted, action }; if (existing) { existing.push(registration); } else { this.commandMap.set(basename, [registration]); } - } + }; private invalid = (command: string, validCommand: boolean) => { - console.log(red(typeof this.onInvalid === "string" ? this.onInvalid : this.onInvalid(command, validCommand))); + console.log(red(typeof this.onInvalid === 'string' ? this.onInvalid : this.onInvalid(command, validCommand))); this.busy = false; - } + }; private valid = (command: string) => { - console.log(green(typeof this.onValid === "string" ? this.onValid : this.onValid(command))); + console.log(green(typeof this.onValid === 'string' ? this.onValid : this.onValid(command))); this.busy = false; - } + }; - private considerInput = async (line: string) => { + private considerInput = async (lineIn: string) => { if (this.busy) { - console.log(red("Busy")); + console.log(red('Busy')); return; } this.busy = true; - line = line.trim(); + let line = lineIn.trim(); if (this.isCaseSensitive) { line = line.toLowerCase(); } const [command, ...args] = line.split(/\s+/g); if (!command) { - return this.invalid(command, false); + this.invalid(command, false); + return; } const registered = this.commandMap.get(command); if (registered) { const { length } = args; const candidates = registered.filter(({ argPatterns: { length: count } }) => count === length); - for (const { argPatterns, action } of candidates) { + candidates.forEach(({ argPatterns, action }: { argPatterns: any; action: any }) => { const parsed: string[] = []; let matched = true; if (length) { for (let i = 0; i < length; i++) { - let matches: RegExpExecArray | null; - if ((matches = argPatterns[i].exec(args[i])) === null) { + const matches = argPatterns[i].exec(args[i]); + if (matches === null) { matched = false; break; } @@ -110,19 +116,17 @@ export default class Repl { } if (!length || matched) { const result = action(parsed); - const resolve = () => this.valid(`${command} ${parsed.join(" ")}`); + const resolve = () => this.valid(`${command} ${parsed.join(' ')}`); if (result instanceof Promise) { result.then(resolve); } else { resolve(); } - return; } - } + }); this.invalid(command, true); } else { this.invalid(command, false); } - } - -}
\ No newline at end of file + }; +} diff --git a/src/server/DashSession/Session/utilities/session_config.ts b/src/server/DashSession/Session/utilities/session_config.ts index 266759929..b42c1a3c7 100644 --- a/src/server/DashSession/Session/utilities/session_config.ts +++ b/src/server/DashSession/Session/utilities/session_config.ts @@ -1,85 +1,85 @@ -import { Schema } from "jsonschema"; -import { yellow, red, cyan, green, blue, magenta, Color, grey, gray, white, black } from "colors"; +import { Schema } from 'jsonschema'; +import { yellow, red, cyan, green, blue, magenta, Color, grey, gray, white, black } from 'colors'; const colorPattern = /black|red|green|yellow|blue|magenta|cyan|white|gray|grey/; const identifierProperties: Schema = { - type: "object", + type: 'object', properties: { text: { - type: "string", - minLength: 1 + type: 'string', + minLength: 1, }, color: { - type: "string", - pattern: colorPattern - } - } + type: 'string', + pattern: colorPattern, + }, + }, }; const portProperties: Schema = { - type: "number", + type: 'number', minimum: 443, - maximum: 65535 + maximum: 65535, }; export const configurationSchema: Schema = { - id: "/configuration", - type: "object", + id: '/configuration', + type: 'object', properties: { - showServerOutput: { type: "boolean" }, + showServerOutput: { type: 'boolean' }, ports: { - type: "object", + type: 'object', properties: { server: portProperties, - socket: portProperties + socket: portProperties, }, - required: ["server"], - additionalProperties: true + required: ['server'], + additionalProperties: true, }, identifiers: { - type: "object", + type: 'object', properties: { master: identifierProperties, worker: identifierProperties, - exec: identifierProperties - } + exec: identifierProperties, + }, }, polling: { - type: "object", + type: 'object', additionalProperties: false, properties: { intervalSeconds: { - type: "number", + type: 'number', minimum: 1, - maximum: 86400 + maximum: 86400, }, route: { - type: "string", - pattern: /\/[a-zA-Z]*/g + type: 'string', + pattern: /\/[a-zA-Z]*/g, }, failureTolerance: { - type: "number", + type: 'number', minimum: 0, - } - } + }, + }, }, - } + }, }; -type ColorLabel = "yellow" | "red" | "cyan" | "green" | "blue" | "magenta" | "grey" | "gray" | "white" | "black"; +type ColorLabel = 'yellow' | 'red' | 'cyan' | 'green' | 'blue' | 'magenta' | 'grey' | 'gray' | 'white' | 'black'; export const colorMapping: Map<ColorLabel, Color> = new Map([ - ["yellow", yellow], - ["red", red], - ["cyan", cyan], - ["green", green], - ["blue", blue], - ["magenta", magenta], - ["grey", grey], - ["gray", gray], - ["white", white], - ["black", black] + ['yellow', yellow], + ['red', red], + ['cyan', cyan], + ['green', green], + ['blue', blue], + ['magenta', magenta], + ['grey', grey], + ['gray', gray], + ['white', white], + ['black', black], ]); interface Identifier { @@ -108,22 +108,22 @@ export const defaultConfig: Configuration = { showServerOutput: false, identifiers: { master: { - text: "__monitor__", - color: "yellow" + text: '__monitor__', + color: 'yellow', }, worker: { - text: "__server__", - color: "magenta" + text: '__server__', + color: 'magenta', }, exec: { - text: "__exec__", - color: "green" - } + text: '__exec__', + color: 'green', + }, }, ports: { server: 1050 }, polling: { - route: "/", + route: '/', intervalSeconds: 30, - failureTolerance: 0 - } -};
\ No newline at end of file + failureTolerance: 0, + }, +}; diff --git a/src/server/DashSession/Session/utilities/utilities.ts b/src/server/DashSession/Session/utilities/utilities.ts index eb8de9d7e..a2ba29c67 100644 --- a/src/server/DashSession/Session/utilities/utilities.ts +++ b/src/server/DashSession/Session/utilities/utilities.ts @@ -1,31 +1,16 @@ -import { v4 } from "uuid"; +import { v4 } from 'uuid'; export namespace Utilities { - export function guid() { return v4(); } - /** - * At any arbitrary layer of nesting within the configuration objects, any single value that - * is not specified by the configuration is given the default counterpart. If, within an object, - * one peer is given by configuration and two are not, the one is preserved while the two are given - * the default value. - * @returns the composition of all of the assigned objects, much like Object.assign(), but with more - * granularity in the overwriting of nested objects - */ - export function preciseAssign(target: any, ...sources: any[]): any { - for (const source of sources) { - preciseAssignHelper(target, source); - } - return target; - } - export function preciseAssignHelper(target: any, source: any) { - Array.from(new Set([...Object.keys(target), ...Object.keys(source)])).map(property => { - let targetValue: any, sourceValue: any; - if (sourceValue = source[property]) { - if (typeof sourceValue === "object" && typeof (targetValue = target[property]) === "object") { + Array.from(new Set([...Object.keys(target), ...Object.keys(source)])).forEach(property => { + const targetValue = target[property]; + const sourceValue = source[property]; + if (sourceValue) { + if (typeof sourceValue === 'object' && typeof targetValue === 'object') { preciseAssignHelper(targetValue, sourceValue); } else { target[property] = sourceValue; @@ -34,4 +19,18 @@ export namespace Utilities { }); } -}
\ No newline at end of file + /** + * At any arbitrary layer of nesting within the configuration objects, any single value that + * is not specified by the configuration is given the default counterpart. If, within an object, + * one peer is given by configuration and two are not, the one is preserved while the two are given + * the default value. + * @returns the composition of all of the assigned objects, much like Object.assign(), but with more + * granularity in the overwriting of nested objects + */ + export function preciseAssign(target: any, ...sources: any[]): any { + sources.forEach(source => { + preciseAssignHelper(target, source); + }); + return target; + } +} diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts index a9e6af67c..808d2c6f2 100644 --- a/src/server/DashStats.ts +++ b/src/server/DashStats.ts @@ -1,9 +1,7 @@ import { cyan, magenta } from 'colors'; import { Response } from 'express'; -import SocketIO from 'socket.io'; -import { timeMap } from './ApiManagers/UserManager'; -import { WebSocket } from './websocket'; import * as fs from 'fs'; +import { socketMap, timeMap, userOperations } from './SocketData'; /** * DashStats focuses on tracking user data for each session. @@ -17,7 +15,6 @@ export namespace DashStats { const statsCSVDirectory = './src/server/stats/'; const statsCSVFilename = statsCSVDirectory + 'userLoginStats.csv'; - const columns = ['USERNAME', 'ACTION', 'TIME']; /** * UserStats holds the stats associated with a particular user. @@ -78,111 +75,14 @@ export namespace DashStats { export const lastUserOperations = new Map<string, UserLastOperations>(); /** - * handleStats is called when the /stats route is called, providing a JSON - * object with relevant stats. In this case, we return the number of - * current connections and - * @param res Response object from Express - */ - export function handleStats(res: Response) { - let current = getCurrentStats(); - const results: CSVStore[] = []; - res.json({ - currentConnections: current.length, - socketMap: current, - }); - } - - /** - * getUpdatedStatesBundle() sends an updated copy of the current stats to the - * frontend /statsview route via websockets. - * - * @returns a StatsDataBundle that is sent to the frontend view on each websocket update - */ - export function getUpdatedStatsBundle(): StatsDataBundle { - let current = getCurrentStats(); - - return { - connectedUsers: current, - }; - } - - /** - * handleStatsView() is called when the /statsview route is called. This - * will use pug to render a frontend view of the current stats - * - * @param res - */ - export function handleStatsView(res: Response) { - let current = getCurrentStats(); - - let connectedUsers = current.map(socketPair => { - return socketPair.time + ' - ' + socketPair.username + ' Operations: ' + socketPair.operations; - }); - - let serverTraffic = ServerTraffic.NOT_BUSY; - if (current.length < BUSY_SERVER_BOUND) { - serverTraffic = ServerTraffic.NOT_BUSY; - } else if (current.length >= BUSY_SERVER_BOUND && current.length < VERY_BUSY_SERVER_BOUND) { - serverTraffic = ServerTraffic.BUSY; - } else { - serverTraffic = ServerTraffic.VERY_BUSY; - } - - res.render('stats.pug', { - title: 'Dash Stats', - numConnections: connectedUsers.length, - serverTraffic: serverTraffic, - serverTrafficMessage: serverTrafficMessages[serverTraffic], - connectedUsers: connectedUsers, - }); - } - - /** - * logUserLogin() writes a login event to the CSV file. - * - * @param username the username in the format of "username@domain.com logged in" - * @param socket the websocket associated with the current connection - */ - export function logUserLogin(username: string | undefined, socket: SocketIO.Socket) { - if (!(username === undefined)) { - let currentDate = new Date(); - console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); - - let toWrite: CSVStore = { - USERNAME: username, - ACTION: 'loggedIn', - TIME: currentDate.toISOString(), - }; - - if (!fs.existsSync(statsCSVDirectory)) fs.mkdirSync(statsCSVDirectory); - let statsFile = fs.createWriteStream(statsCSVFilename, { flags: 'a' }); - statsFile.write(convertToCSV(toWrite)); - statsFile.end(); - console.log(cyan(convertToCSV(toWrite))); - } - } - - /** - * logUserLogout() writes a logout event to the CSV file. - * - * @param username the username in the format of "username@domain.com logged in" - * @param socket the websocket associated with the current connection. + * convertToCSV() is a helper method that stringifies a CSVStore object + * that can be written to the CSV file later. + * @param dataObject the object to stringify + * @returns the object as a string. */ - export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) { - if (!(username === undefined)) { - let currentDate = new Date(); - - let statsFile = fs.createWriteStream(statsCSVFilename, { flags: 'a' }); - let toWrite: CSVStore = { - USERNAME: username, - ACTION: 'loggedOut', - TIME: currentDate.toISOString(), - }; - statsFile.write(convertToCSV(toWrite)); - statsFile.end(); - } + function convertToCSV(dataObject: CSVStore): string { + return `${dataObject.USERNAME},${dataObject.ACTION},${dataObject.TIME}\n`; } - /** * getLastOperationsOrDefault() is a helper method that will attempt * to query the lastUserOperations map for a specified username. If the @@ -193,7 +93,7 @@ export namespace DashStats { */ function getLastOperationsOrDefault(username: string): UserLastOperations { if (lastUserOperations.get(username) === undefined) { - let initializeOperationsQueue = []; + const initializeOperationsQueue = []; for (let i = 0; i < RATE_INTERVAL; i++) { initializeOperationsQueue.push(0); } @@ -217,7 +117,7 @@ export namespace DashStats { */ function updateLastOperations(lastOperationData: UserLastOperations, currentOperations: number): UserLastOperations { // create a copy of the UserLastOperations to modify - let newLastOperationData: UserLastOperations = { + const newLastOperationData: UserLastOperations = { sampleOperations: lastOperationData.sampleOperations, lastSampleOperations: lastOperationData.lastSampleOperations, previousOperationsQueue: lastOperationData.previousOperationsQueue.slice(), @@ -225,7 +125,7 @@ export namespace DashStats { let newSampleOperations = newLastOperationData.sampleOperations; newSampleOperations -= newLastOperationData.previousOperationsQueue.shift()!; // removes and returns the first element of the queue - let operationsThisCycle = currentOperations - lastOperationData.lastSampleOperations; + const operationsThisCycle = currentOperations - lastOperationData.lastSampleOperations; newSampleOperations += operationsThisCycle; // add the operations this cycle to find out what our count for the interval should be (e.g operations in the last 10 seconds) // update values for the copy object @@ -245,7 +145,7 @@ export namespace DashStats { * @returns the total number of operations recorded up to this sampling cycle. */ function getUserOperationsOrDefault(username: string): number { - return WebSocket.userOperations.get(username) === undefined ? 0 : WebSocket.userOperations.get(username)!; + return userOperations.get(username) === undefined ? 0 : userOperations.get(username)!; } /** @@ -255,37 +155,129 @@ export namespace DashStats { * @returns an array of UserStats storing data for each user at the current moment. */ function getCurrentStats(): UserStats[] { - let socketPairs: UserStats[] = []; - for (let [key, value] of WebSocket.socketMap) { - let username = value.split(' ')[0]; - let connectionTime = new Date(timeMap[username]); + const socketPairs: UserStats[] = []; + Array.from(socketMap.entries()).forEach(([key, value]) => { + const username = value.split(' ')[0]; + const connectionTime = new Date(timeMap[username]); - let connectionTimeString = connectionTime.toLocaleDateString() + ' ' + connectionTime.toLocaleTimeString(); + const connectionTimeString = connectionTime.toLocaleDateString() + ' ' + connectionTime.toLocaleTimeString(); if (!key.disconnected) { - let lastRecordedOperations = getLastOperationsOrDefault(username); - let currentUserOperationCount = getUserOperationsOrDefault(username); + const lastRecordedOperations = getLastOperationsOrDefault(username); + const currentUserOperationCount = getUserOperationsOrDefault(username); socketPairs.push({ socketId: key.id, username: username, time: connectionTimeString.includes('Invalid Date') ? '' : connectionTimeString, - operations: WebSocket.userOperations.get(username) ? WebSocket.userOperations.get(username)! : 0, + operations: userOperations.get(username) ? userOperations.get(username)! : 0, rate: lastRecordedOperations.sampleOperations, }); lastUserOperations.set(username, updateLastOperations(lastRecordedOperations, currentUserOperationCount)); } - } + }); return socketPairs; } /** - * convertToCSV() is a helper method that stringifies a CSVStore object - * that can be written to the CSV file later. - * @param dataObject the object to stringify - * @returns the object as a string. + * handleStats is called when the /stats route is called, providing a JSON + * object with relevant stats. In this case, we return the number of + * current connections and + * @param res Response object from Express */ - function convertToCSV(dataObject: CSVStore): string { - return `${dataObject.USERNAME},${dataObject.ACTION},${dataObject.TIME}\n`; + export function handleStats(res: Response) { + const current = getCurrentStats(); + res.json({ + currentConnections: current.length, + socketMap: current, + }); + } + + /** + * getUpdatedStatesBundle() sends an updated copy of the current stats to the + * frontend /statsview route via websockets. + * + * @returns a StatsDataBundle that is sent to the frontend view on each websocket update + */ + export function getUpdatedStatsBundle(): StatsDataBundle { + const current = getCurrentStats(); + + return { + connectedUsers: current, + }; + } + + /** + * handleStatsView() is called when the /statsview route is called. This + * will use pug to render a frontend view of the current stats + * + * @param res + */ + export function handleStatsView(res: Response) { + const current = getCurrentStats(); + const connectedUsers = current.map(({ time, username, operations }) => time + ' - ' + username + ' Operations: ' + operations); + + let serverTraffic = ServerTraffic.NOT_BUSY; + if (current.length < BUSY_SERVER_BOUND) { + serverTraffic = ServerTraffic.NOT_BUSY; + } else if (current.length >= BUSY_SERVER_BOUND && current.length < VERY_BUSY_SERVER_BOUND) { + serverTraffic = ServerTraffic.BUSY; + } else { + serverTraffic = ServerTraffic.VERY_BUSY; + } + + res.render('stats.pug', { + title: 'Dash Stats', + numConnections: connectedUsers.length, + serverTraffic: serverTraffic, + serverTrafficMessage: serverTrafficMessages[serverTraffic], + connectedUsers: connectedUsers, + }); + } + + /** + * logUserLogin() writes a login event to the CSV file. + * + * @param username the username in the format of "username@domain.com logged in" + * @param socket the websocket associated with the current connection + */ + export function logUserLogin(username: string | undefined) { + if (!(username === undefined)) { + const currentDate = new Date(); + console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`)); + + const toWrite: CSVStore = { + USERNAME: username, + ACTION: 'loggedIn', + TIME: currentDate.toISOString(), + }; + + if (!fs.existsSync(statsCSVDirectory)) fs.mkdirSync(statsCSVDirectory); + const statsFile = fs.createWriteStream(statsCSVFilename, { flags: 'a' }); + statsFile.write(convertToCSV(toWrite)); + statsFile.end(); + console.log(cyan(convertToCSV(toWrite))); + } + } + + /** + * logUserLogout() writes a logout event to the CSV file. + * + * @param username the username in the format of "username@domain.com logged in" + * @param socket the websocket associated with the current connection. + */ + export function logUserLogout(username: string | undefined) { + if (!(username === undefined)) { + const currentDate = new Date(); + + const statsFile = fs.createWriteStream(statsCSVFilename, { flags: 'a' }); + const toWrite: CSVStore = { + USERNAME: username, + ACTION: 'loggedOut', + TIME: currentDate.toISOString(), + }; + statsFile.write(convertToCSV(toWrite)); + statsFile.end(); + } } } diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 307aec6fc..08cea1de5 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -15,16 +15,15 @@ import { basename } from 'path'; import * as parse from 'pdf-parse'; import * as request from 'request-promise'; import { Duplex, Stream } from 'stream'; -import { filesDirectory, publicDirectory } from '.'; import { Utils } from '../Utils'; -import { Opt } from '../fields/Doc'; -import { ParsedPDF } from '../server/PdfTypes'; import { createIfNotExists } from './ActionUtilities'; import { AzureManager } from './ApiManagers/AzureManager'; -import { Directory, clientPathToFile, pathToDirectory, serverPathToFile } from './ApiManagers/UploadManager'; +import { ParsedPDF } from './PdfTypes'; import { AcceptableMedia, Upload } from './SharedMediaTypes'; +import { Directory, clientPathToFile, filesDirectory, pathToDirectory, publicDirectory, serverPathToFile } from './SocketData'; import { resolvedServerUrl } from './server_Initialization'; -const spawn = require('child_process').spawn; + +const { spawn } = require('child_process'); const { exec } = require('child_process'); const requestImageSize = require('../client/util/request-image-size'); @@ -42,7 +41,7 @@ export function InjectSize(filename: string, size: SizeSuffix) { } function isLocal() { - return /Dash-Web[0-9]*[\\\/]src[\\\/]server[\\\/]public[\\\/](.*)/; + return /Dash-Web[0-9]*[\\/]src[\\/]server[\\/]public[\\/](.*)/; } function usingAzure() { @@ -68,11 +67,21 @@ export namespace DashUploadUtils { const size = 'content-length'; const type = 'content-type'; - const BLOBSTORE_URL = process.env.BLOBSTORE_URL; - const RESIZE_FUNCTION_URL = process.env.RESIZE_FUNCTION_URL; + const { BLOBSTORE_URL, RESIZE_FUNCTION_URL } = process.env; - const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; //TODO:glr + const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; // TODO:glr + export function fExists(name: string, destination: Directory) { + const destinationPath = serverPathToFile(destination, name); + return existsSync(destinationPath); + } + + export function getAccessPaths(directory: Directory, fileName: string) { + return { + client: clientPathToFile(directory, fileName), + server: serverPathToFile(directory, fileName), + }; + } export async function concatVideos(filePaths: string[]): Promise<Upload.AccessPathInfo> { // make a list of paths to create the ordered text file for ffmpeg const inputListName = 'concat.txt'; @@ -80,14 +89,14 @@ export namespace DashUploadUtils { // make a list of paths to create the ordered text file for ffmpeg const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n'); // write the text file to the file system - await new Promise<void>((res, reject) => + await new Promise<void>((res, reject) => { writeFile(textFilePath, filePathsText, err => { if (err) { reject(); console.log(err); } else res(); - }) - ); + }); + }); // make output file name based on timestamp const outputFileName = `output-${Utils.GenerateGuid()}.mp4`; @@ -95,19 +104,19 @@ export namespace DashUploadUtils { const outputFilePath = path.join(pathToDirectory(Directory.videos), outputFileName); // concatenate the videos - await new Promise((resolve, reject) => + await new Promise((resolve, reject) => { ffmpeg() .input(textFilePath) .inputOptions(['-f concat', '-safe 0']) // .outputOptions('-c copy') - //.videoCodec("copy") + // .videoCodec("copy") .save(outputFilePath) .on('error', (err: any) => { console.log(err); reject(); }) - .on('end', resolve) - ); + .on('end', resolve); + }); // delete concat.txt from the file system unlinkSync(textFilePath); @@ -135,270 +144,76 @@ export namespace DashUploadUtils { }; } - export function QueryYoutubeProgress(videoId: string, user?: Express.User) { + export const uploadProgress = new Map<string, string>(); + + export function QueryYoutubeProgress(videoId: string) { // console.log(`PROGRESS:${videoId}`, (user as any)?.email); return uploadProgress.get(videoId) ?? 'pending data upload'; } - export let uploadProgress = new Map<string, string>(); - - export function uploadYoutube(videoId: string, overwriteId: string): Promise<Upload.FileResponse> { - return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => { - const name = videoId; - const filepath = name.replace(/^-/, '__') + '.mp4'; - const finalPath = serverPathToFile(Directory.videos, filepath); - if (existsSync(finalPath)) { - uploadProgress.set(overwriteId, 'computing duration'); - exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { - const time = Array.from(stdout.trim().split(':')).reverse(); - const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); - res(resolveExistingFile(name, filepath, Directory.videos, 'video/mp4', duration, undefined)); - }); - } else { - uploadProgress.set(overwriteId, 'starting download'); - const ytdlp = spawn(`yt-dlp`, ['-o', filepath, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']); - - ytdlp.stdout.on('data', (data: any) => uploadProgress.set(overwriteId, data.toString())); - - let errors = ''; - ytdlp.stderr.on('data', (data: any) => { - uploadProgress.set(overwriteId, 'error:' + data.toString()); - errors = data.toString(); - }); - - ytdlp.on('exit', function (code: any) { - if (code) { - res({ - source: { - size: 0, - filepath: name, - originalFilename: name, - newFilename: name, - mimetype: 'video', - hashAlgorithm: 'md5', - toJSON: () => ({ newFilename: name, filepath, mimetype: 'video', mtime: new Date(), size: 0, length: 0, originalFilename: name }), - }, - result: { name: 'failed youtube query', message: `Could not archive video. ${code ? errors : uploadProgress.get(videoId)}` }, - }); - } else { - uploadProgress.set(overwriteId, 'computing duration'); - exec(`yt-dlp-o ${filepath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { - const time = Array.from(stdout.trim().split(':')).reverse(); - const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); - const data = { size: 0, filepath, name, mimetype: 'video', originalFilename: name, newFilename: name, hashAlgorithm: 'md5' as 'md5', type: 'video/mp4' }; - const file = { ...data, toJSON: () => ({ ...data, length: 0, filename: data.filepath.replace(/.*\//, ''), mtime: new Date(), toJson: () => undefined as any }) }; - MoveParsedFile(file, Directory.videos).then(output => { - console.log('OUTPUT = ' + output); - res(output); - }); - }); - } + /** + * Basically just a wrapper around rename, which 'deletes' + * the file at the old path and 'moves' it to the new one. For simplicity, the + * caller just has to pass in the name of the target directory, and this function + * will resolve the actual target path from that. + * @param file The file to move + * @param destination One of the specific media asset directories into which to move it + * @param suffix If the file doesn't have a suffix and you want to provide it one + * to appear in the new location + */ + export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix?: string, text?: string, duration?: number, targetName?: string): Promise<Upload.FileResponse> { + const { filepath } = file; + let name = targetName ?? path.basename(filepath); + suffix && (name += suffix); + return new Promise(resolve => { + const destinationPath = serverPathToFile(destination, name); + rename(filepath, destinationPath, error => { + resolve({ + source: file, + result: error ?? { + accessPaths: { + agnostic: getAccessPaths(destination, name), + }, + rawText: text, + duration, + }, }); - } + }); }); } - export async function upload(file: File, overwriteGuid?: string): Promise<Upload.FileResponse> { - const isAzureOn = usingAzure(); - const { mimetype: type, filepath, originalFilename } = file; - const types = type?.split('/') ?? []; - // uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name. - - const category = types[0]; - let format = `.${types[1]}`; - console.log(green(`Processing upload of file (${originalFilename}) and format (${format}) with upload type (${type}) in category (${category}).`)); - - switch (category) { - case 'image': - if (imageFormats.includes(format)) { - const result = await UploadImage(filepath, basename(filepath)); - return { source: file, result }; - } - fs.unlink(filepath, () => {}); - return { source: file, result: { name: 'Unsupported image format', message: `Could not upload unsupported file (${originalFilename}). Please convert to an .jpg` } }; - case 'video': - if (format.includes('x-matroska')) { - console.log('case video'); - await new Promise(res => - ffmpeg(file.filepath) - .videoCodec('copy') // this will copy the data instead of reencode it - .save(file.filepath.replace('.mkv', '.mp4')) - .on('end', res) - .on('error', (e: any) => console.log(e)) - ); - file.filepath = file.filepath.replace('.mkv', '.mp4'); - format = '.mp4'; - } - if (format.includes('quicktime')) { - let abort = false; - await new Promise<void>(res => - ffmpeg.ffprobe(file.filepath, (err: any, metadata: any) => { - if (metadata.streams.some((stream: any) => stream.codec_name === 'hevc')) { - abort = true; - } - res(); - }) - ); - if (abort) { - // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server. - // await new Promise(res => - // ffmpeg(file.path) - // .videoCodec('libx264') // this will copy the data instead of reencode it - // .audioCodec('mp2') - // .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4')) - // .on('end', res) - // ); - // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4'); - // format = '.mp4'; - fs.unlink(filepath, () => {}); - return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${originalFilename}). Please convert to an .mp4` } }; - } - } - if (videoFormats.includes(format) || format.includes('.webm')) { - return MoveParsedFile(file, Directory.videos); - } - fs.unlink(filepath, () => {}); - return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${originalFilename}). Please convert to an .mp4` } }; - case 'application': - if (applicationFormats.includes(format)) { - const val = UploadPdf(file); - if (val) return val; - } - case 'audio': - const components = format.split(';'); - if (components.length > 1) { - format = components[0]; - } - if (audioFormats.includes(format)) { - return UploadAudio(file, format); - } - fs.unlink(filepath, () => {}); - return { source: file, result: { name: 'Unsupported audio format', message: `Could not upload unsupported file (${originalFilename}). Please convert to an .mp3` } }; - case 'text': - if (types[1] == 'csv') { - return UploadCsv(file); - } - } - - console.log(red(`Ignoring unsupported file (${originalFilename}) with upload type (${type}).`)); - fs.unlink(filepath, () => {}); - return { source: file, result: new Error(`Could not upload unsupported file (${originalFilename}) with upload type (${type}).`) }; - } - - async function UploadPdf(file: File) { - const fileKey = (await md5File(file.filepath)) + '.pdf'; - const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; - if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) { - fs.unlink(file.filepath, () => {}); - return new Promise<Upload.FileResponse>(res => { - const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; - const readStream = createReadStream(serverPathToFile(Directory.text, textFilename)); - var rawText = ''; - readStream - .on('data', chunk => (rawText += chunk.toString())) // - .on('end', () => res(resolveExistingFile(file.originalFilename ?? '', fileKey, Directory.pdfs, file.mimetype, undefined, rawText))); - }); - } - const dataBuffer = readFileSync(file.filepath); - const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e); - if (!result.code) { - await new Promise<void>((resolve, reject) => { - const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); - writeStream.write(result?.text, error => (error ? reject(error) : resolve())); + const parseExifData = async (source: string) => { + const image = await request.get(source, { encoding: null }); + const { /* data, */ error } = await new Promise<{ data: any; error: any }>(resolve => { + // eslint-disable-next-line no-new + new ExifImage({ image }, (exifError, data) => { + const reason = (exifError as any)?.code; + resolve({ data, error: reason }); }); - return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey); - } - return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.originalFilename}).${result.message}` } }; - } - - async function UploadCsv(file: File) { - const { filepath: sourcePath } = file; - // read the file as a string - const data = readFileSync(sourcePath, 'utf8'); - // split the string into an array of lines - return MoveParsedFile(file, Directory.csv, undefined, data); - // console.log(csvParser(data)); - } - - const manualSuffixes = ['.webm']; - - async function UploadAudio(file: File, format: string) { - const suffix = manualSuffixes.includes(format) ? format : undefined; - return MoveParsedFile(file, Directory.audio, suffix); - } - - /** - * Uploads an image specified by the @param source to Dash's /public/files/ - * directory, and returns information generated during that upload - * - * @param {string} source is either the absolute path of an already uploaded image or - * the url of a remote image - * @param {string} filename dictates what to call the image. If not specified, - * the name {@param prefix}_upload_{GUID} - * @param {string} prefix is a string prepended to the generated image name in the - * event that @param filename is not specified - * - * @returns {ImageUploadInformation | Error} This method returns - * 1) the paths to the uploaded images (plural due to resizing) - * 2) the exif data embedded in the image, or the error explaining why exif couldn't be parsed - * 3) the size of the image, in bytes (4432130) - * 4) the content type of the image, i.e. image/(jpeg | png | ...) - */ - export const UploadImage = async (source: string, filename?: string, prefix: string = ''): Promise<Upload.ImageInformation | Error> => { - const metadata = await InspectImage(source); - if (metadata instanceof Error) { - return { name: metadata.name, message: metadata.message }; - } - const outputFile = filename || metadata.filename || ''; - - return UploadInspectedImage(metadata, outputFile, prefix); + }); + return error ? { data: undefined, error } : { data: await exifr.parse(image), error }; }; - - export async function buildFileDirectories() { - if (!existsSync(publicDirectory)) { - console.error('\nPlease ensure that the following directory exists...\n'); - console.log(publicDirectory); - process.exit(0); - } - if (!existsSync(filesDirectory)) { - console.error('\nPlease ensure that the following directory exists...\n'); - console.log(filesDirectory); - process.exit(0); - } - const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`)); - return Promise.all(pending); - } - - export interface RequestedImageSize { - width: number; - height: number; - type: string; - } - - export interface ImageResizer { - width: number; - suffix: SizeSuffix; - } - /** * Based on the url's classification as local or remote, gleans * as much information as possible about the specified image * * @param source is the path or url to the image in question */ - export const InspectImage = async (source: string): Promise<Upload.InspectionResults | Error> => { - let rawMatches: RegExpExecArray | null; + export const InspectImage = async (sourceIn: string): Promise<Upload.InspectionResults | Error> => { + let source = sourceIn; + const rawMatches = /^data:image\/([a-z]+);base64,(.*)/.exec(source); let filename: string | undefined; /** * Just more edge case handling: this if clause handles the case where an image onto the canvas that * is represented by a base64 encoded data uri, rather than a proper file. We manually write it out * to the server and then carry on as if it had been put there by the Formidable form / file parser. */ - if ((rawMatches = /^data:image\/([a-z]+);base64,(.*)/.exec(source)) !== null) { + if (rawMatches !== null) { const [ext, data] = rawMatches.slice(1, 3); - const resolved = (filename = `upload_${Utils.GenerateGuid()}.${ext}`); + filename = `upload_${Utils.GenerateGuid()}.${ext}`; + const resolved = filename; if (usingAzure()) { - const response = await AzureManager.UploadBase64ImageBlob(resolved, data); + await AzureManager.UploadBase64ImageBlob(resolved, data); source = `${AzureManager.BASE_STRING}/${resolved}`; } else { source = `${resolvedServerUrl}${clientPathToFile(Directory.images, resolved)}`; @@ -438,7 +253,7 @@ export namespace DashUploadUtils { // Use the request library to parse out file level image information in the headers const { headers } = await new Promise<any>((resolve, reject) => { - return request.head(resolvedUrl, (error, res) => (error ? reject(error) : resolve(res))); + request.head(resolvedUrl, (error, res) => (error ? reject(error) : resolve(res))); }).catch(e => { console.log('Error processing headers: ', e); }); @@ -449,6 +264,7 @@ export namespace DashUploadUtils { // Bundle up the information into an object return { source, + // eslint-disable-next-line radix contentSize: parseInt(headers[size]), contentType: headers[type], nativeWidth, @@ -462,49 +278,71 @@ export namespace DashUploadUtils { } }; + async function correctRotation(imgSourcePath: string) { + const buffer = fs.readFileSync(imgSourcePath); + try { + return (await autorotate.rotate(buffer, { quality: 30 })).buffer; + } catch (e) { + return buffer; + } + } + /** - * Basically just a wrapper around rename, which 'deletes' - * the file at the old path and 'moves' it to the new one. For simplicity, the - * caller just has to pass in the name of the target directory, and this function - * will resolve the actual target path from that. - * @param file The file to move - * @param destination One of the specific media asset directories into which to move it - * @param suffix If the file doesn't have a suffix and you want to provide it one - * to appear in the new location + * define the resizers to use + * @param ext the extension + * @returns an array of resize descriptions */ - export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number, targetName?: string): Promise<Upload.FileResponse> { - const { filepath } = file; - let name = targetName ?? path.basename(filepath); - suffix && (name += suffix); - return new Promise(resolve => { - const destinationPath = serverPathToFile(destination, name); - rename(filepath, destinationPath, error => { - resolve({ - source: file, - result: error - ? error - : { - accessPaths: { - agnostic: getAccessPaths(destination, name), - }, - rawText: text, - duration, - }, - }); - }); - }); + export function imageResampleSizes(ext: string): DashUploadUtils.ImageResizer[] { + return [ + { suffix: SizeSuffix.Original, width: 0 }, + ...[...(AcceptableMedia.imageFormats.includes(ext.toLowerCase()) ? Object.values(DashUploadUtils.Sizes) : [])].map(({ suffix, width }) => ({ + width, + suffix, + })), + ]; } - export function fExists(name: string, destination: Directory) { - const destinationPath = serverPathToFile(destination, name); - return existsSync(destinationPath); - } + /** + * outputResizedImages takes in a readable stream and resizes the images according to the sizes defined at the top of this file. + * + * The new images will be saved to the server with the corresponding prefixes. + * @param imgSourcePath file path for image being resized + * @param outputFileName the basename (No suffix) of the outputted file. + * @param outputDirectory the directory to output to, usually Directory.Images + * @returns a map with suffixes as keys and resized filenames as values. + */ + export async function outputResizedImages(imgSourcePath: string, outputFileName: string, outputDirectory: string) { + const writtenFiles: { [suffix: string]: string } = {}; + const sizes = imageResampleSizes(path.extname(outputFileName)); - export function getAccessPaths(directory: Directory, fileName: string) { - return { - client: clientPathToFile(directory, fileName), - server: serverPathToFile(directory, fileName), + const imgBuffer = await correctRotation(imgSourcePath); + const imgReadStream = new Duplex(); + imgReadStream.push(imgBuffer); + imgReadStream.push(null); + const outputPath = (suffix: SizeSuffix) => { + writtenFiles[suffix] = InjectSize(outputFileName, suffix); + return path.resolve(outputDirectory, writtenFiles[suffix]); }; + await Promise.all( + sizes.filter(({ width }) => !width).map(({ suffix }) => + new Promise<void>(res => { + imgReadStream.pipe(createWriteStream(outputPath(suffix))).on('close', res); + }) + )); // prettier-ignore + + return Jimp.read(imgBuffer) + .then(async (imgIn: any) => { + let img = imgIn; + await Promise.all( sizes.filter(({ width }) => width).map(({ width, suffix }) => { + img = img.resize(width, Jimp.AUTO).write(outputPath(suffix)); + return img; + } )); // prettier-ignore + return writtenFiles; + }) + .catch((e: any) => { + console.log('ERROR' + e); + return writtenFiles; + }); } /** @@ -555,119 +393,265 @@ export namespace DashUploadUtils { } catch (e) { // input is a blob or other, try reading it to create a metadata source file. const reqSource = request(metadata.source); - let readStream: Stream = reqSource instanceof Promise ? await reqSource : reqSource; + const readStream: Stream = reqSource instanceof Promise ? await reqSource : reqSource; const readSource = `${prefix}upload_${Utils.GenerateGuid()}.${metadata.contentType.split('/')[1].toLowerCase()}`; - await new Promise<void>((res, rej) => + await new Promise<void>((res, rej) => { readStream .pipe(createWriteStream(readSource)) .on('close', () => res()) - .on('error', () => rej()) - ); + .on('error', () => rej()); + }); writtenFiles = await outputResizedImages(readSource, resolved, pathToDirectory(Directory.images)); - fs.unlink(readSource, err => console.log("Couldn't unlink temporary image file:" + readSource)); + fs.unlink(readSource, err => console.log("Couldn't unlink temporary image file:" + readSource, err)); } } - for (const suffix of Object.keys(writtenFiles)) { + Array.from(Object.keys(writtenFiles)).forEach(suffix => { information.accessPaths[suffix] = getAccessPaths(images, writtenFiles[suffix]); - } + }); if (isLocal().test(source) && cleanUp) { unlinkSync(source); } return information; }; - const bufferConverterRec = (layer: any) => { - for (const key of Object.keys(layer)) { - const val: any = layer[key]; - if (val instanceof Buffer) { - layer[key] = val.toString(); - } else if (Array.isArray(val) && typeof val[0] === 'number') { - layer[key] = Buffer.from(val).toString(); - } else if (typeof val === 'object') { - bufferConverterRec(val); - } + /** + * Uploads an image specified by the @param source to Dash's /public/files/ + * directory, and returns information generated during that upload + * + * @param {string} source is either the absolute path of an already uploaded image or + * the url of a remote image + * @param {string} filename dictates what to call the image. If not specified, + * the name {@param prefix}_upload_{GUID} + * @param {string} prefix is a string prepended to the generated image name in the + * event that @param filename is not specified + * + * @returns {ImageUploadInformation | Error} This method returns + * 1) the paths to the uploaded images (plural due to resizing) + * 2) the exif data embedded in the image, or the error explaining why exif couldn't be parsed + * 3) the size of the image, in bytes (4432130) + * 4) the content type of the image, i.e. image/(jpeg | png | ...) + */ + export const UploadImage = async (source: string, filename?: string, prefix: string = ''): Promise<Upload.ImageInformation | Error> => { + const metadata = await InspectImage(source); + if (metadata instanceof Error) { + return { name: metadata.name, message: metadata.message }; } + const outputFile = filename || metadata.filename || ''; + + return UploadInspectedImage(metadata, outputFile, prefix); }; - const parseExifData = async (source: string) => { - const image = await request.get(source, { encoding: null }); - const { data, error } = await new Promise<{ data: any; error: any }>(resolve => { - new ExifImage({ image }, (error, data) => { - let reason: Opt<string> = undefined; - if (error) { - reason = (error as any).code; - } - resolve({ data, error: reason }); - }); + export function uploadYoutube(videoId: string, overwriteId: string): Promise<Upload.FileResponse> { + return new Promise<Upload.FileResponse<Upload.FileInformation>>(res => { + const name = videoId; + const filepath = name.replace(/^-/, '__') + '.mp4'; + const finalPath = serverPathToFile(Directory.videos, filepath); + if (existsSync(finalPath)) { + uploadProgress.set(overwriteId, 'computing duration'); + exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any /* , stderr: any */) => { + const time = Array.from(stdout.trim().split(':')).reverse(); + const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); + res(resolveExistingFile(name, filepath, Directory.videos, 'video/mp4', duration, undefined)); + }); + } else { + uploadProgress.set(overwriteId, 'starting download'); + const ytdlp = spawn(`yt-dlp`, ['-o', filepath, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']); + + ytdlp.stdout.on('data', (data: any) => uploadProgress.set(overwriteId, data.toString())); + + let errors = ''; + ytdlp.stderr.on('data', (data: any) => { + uploadProgress.set(overwriteId, 'error:' + data.toString()); + errors = data.toString(); + }); + + ytdlp.on('exit', (code: any) => { + if (code) { + res({ + source: { + size: 0, + filepath: name, + originalFilename: name, + newFilename: name, + mimetype: 'video', + hashAlgorithm: 'md5', + toJSON: () => ({ newFilename: name, filepath, mimetype: 'video', mtime: new Date(), size: 0, length: 0, originalFilename: name }), + }, + result: { name: 'failed youtube query', message: `Could not archive video. ${code ? errors : uploadProgress.get(videoId)}` }, + }); + } else { + uploadProgress.set(overwriteId, 'computing duration'); + exec(`yt-dlp-o ${filepath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (/* error: any, stdout: any, stderr: any */) => { + // const time = Array.from(stdout.trim().split(':')).reverse(); + // const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); + const data = { size: 0, filepath, name, mimetype: 'video', originalFilename: name, newFilename: name, hashAlgorithm: 'md5' as 'md5', type: 'video/mp4' }; + const file = { ...data, toJSON: () => ({ ...data, length: 0, filename: data.filepath.replace(/.*\//, ''), mtime: new Date(), toJson: () => undefined as any }) }; + MoveParsedFile(file, Directory.videos).then(output => res(output)); + }); + } + }); + } }); - //data && bufferConverterRec(data); - return error ? { data: undefined, error } : { data: await exifr.parse(image), error }; - }; + } + const manualSuffixes = ['.webm']; - const { pngs, jpgs, webps, tiffs } = AcceptableMedia; - const pngOptions = { - compressionLevel: 9, - adaptiveFiltering: true, - force: true, - }; + async function UploadAudio(file: File, format: string) { + const suffix = manualSuffixes.includes(format) ? format : undefined; + return MoveParsedFile(file, Directory.audio, suffix); + } - async function correctRotation(imgSourcePath: string) { - const buffer = fs.readFileSync(imgSourcePath); - try { - return (await autorotate.rotate(buffer, { quality: 30 })).buffer; - } catch (e) { - return buffer; + async function UploadPdf(file: File) { + const fileKey = (await md5File(file.filepath)) + '.pdf'; + const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; + if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) { + fs.unlink(file.filepath, () => {}); + return new Promise<Upload.FileResponse>(res => { + const pdfTextFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; + const readStream = createReadStream(serverPathToFile(Directory.text, pdfTextFilename)); + let rawText = ''; + readStream + .on('data', chunk => { + rawText += chunk.toString(); + }) + .on('end', () => res(resolveExistingFile(file.originalFilename ?? '', fileKey, Directory.pdfs, file.mimetype, undefined, rawText))); + }); } + const dataBuffer = readFileSync(file.filepath); + const result: ParsedPDF | any = await parse(dataBuffer).catch((e: any) => e); + if (!result.code) { + await new Promise<void>((resolve, reject) => { + const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); + writeStream.write(result?.text, error => (error ? reject(error) : resolve())); + }); + return MoveParsedFile(file, Directory.pdfs, undefined, result?.text, undefined, fileKey); + } + return { source: file, result: { name: 'faile pdf pupload', message: `Could not upload (${file.originalFilename}).${result.message}` } }; } - /** - * outputResizedImages takes in a readable stream and resizes the images according to the sizes defined at the top of this file. - * - * The new images will be saved to the server with the corresponding prefixes. - * @param imgSourcePath file path for image being resized - * @param outputFileName the basename (No suffix) of the outputted file. - * @param outputDirectory the directory to output to, usually Directory.Images - * @returns a map with suffixes as keys and resized filenames as values. - */ - export async function outputResizedImages(imgSourcePath: string, outputFileName: string, outputDirectory: string) { - const writtenFiles: { [suffix: string]: string } = {}; - const sizes = imageResampleSizes(path.extname(outputFileName)); + async function UploadCsv(file: File) { + const { filepath: sourcePath } = file; + // read the file as a string + const data = readFileSync(sourcePath, 'utf8'); + // split the string into an array of lines + return MoveParsedFile(file, Directory.csv, undefined, data); + // console.log(csvParser(data)); + } - const imgBuffer = await correctRotation(imgSourcePath); - const imgReadStream = new Duplex(); - imgReadStream.push(imgBuffer); - imgReadStream.push(null); - const outputPath = (suffix: SizeSuffix) => path.resolve(outputDirectory, (writtenFiles[suffix] = InjectSize(outputFileName, suffix))); - await Promise.all( - sizes.filter(({ width }) => !width).map(({ suffix }) => - new Promise<void>(res => imgReadStream.pipe(createWriteStream(outputPath(suffix))).on('close', res)) - )); // prettier-ignore + export async function upload(file: File /* , overwriteGuid?: string */): Promise<Upload.FileResponse> { + // const isAzureOn = usingAzure(); + const { mimetype, filepath, originalFilename } = file; + const types = mimetype?.split('/') ?? []; + // uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name. - return Jimp.read(imgBuffer) - .then(async (img: any) => { - await Promise.all( sizes.filter(({ width }) => width).map(({ width, suffix }) => - img = img.resize(width, Jimp.AUTO).write(outputPath(suffix)) - )); // prettier-ignore - return writtenFiles; - }) - .catch((e: any) => { - console.log('ERROR' + e); - return writtenFiles; - }); + const category = types[0]; + let format = `.${types[1]}`; + console.log(green(`Processing upload of file (${originalFilename}) and format (${format}) with upload type (${mimetype}) in category (${category}).`)); + + switch (category) { + case 'image': + if (imageFormats.includes(format)) { + const result = await UploadImage(filepath, basename(filepath)); + return { source: file, result }; + } + fs.unlink(filepath, () => {}); + return { source: file, result: { name: 'Unsupported image format', message: `Could not upload unsupported file (${originalFilename}). Please convert to an .jpg` } }; + case 'video': { + const vidFile = file; + if (format.includes('x-matroska')) { + await new Promise(res => { + ffmpeg(vidFile.filepath) + .videoCodec('copy') // this will copy the data instead of reencode it + .save(vidFile.filepath.replace('.mkv', '.mp4')) + .on('end', res) + .on('error', (e: any) => console.log(e)); + }); + vidFile.filepath = vidFile.filepath.replace('.mkv', '.mp4'); + format = '.mp4'; + } + if (format.includes('quicktime')) { + let abort = false; + await new Promise<void>(res => { + ffmpeg.ffprobe(vidFile.filepath, (err: any, metadata: any) => { + if (metadata.streams.some((stream: any) => stream.codec_name === 'hevc')) { + abort = true; + } + res(); + }); + }); + if (abort) { + // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server. + // await new Promise(res => + // ffmpeg(file.path) + // .videoCodec('libx264') // this will copy the data instead of reencode it + // .audioCodec('mp2') + // .save(vidFile.path.replace('.MOV', '.mp4').replace('.mov', '.mp4')) + // .on('end', res) + // ); + // vidFile.path = vidFile.path.replace('.mov', '.mp4').replace('.MOV', '.mp4'); + // format = '.mp4'; + fs.unlink(filepath, () => {}); + return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${originalFilename}). Please convert to an .mp4` } }; + } + } + if (videoFormats.includes(format) || format.includes('.webm')) { + return MoveParsedFile(vidFile, Directory.videos); + } + fs.unlink(filepath, () => {}); + return { source: vidFile, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${originalFilename}). Please convert to an .mp4` } }; + } + case 'application': + if (applicationFormats.includes(format)) { + const val = UploadPdf(file); + if (val) return val; + } + break; + case 'audio': { + const components = format.split(';'); + if (components.length > 1) { + [format] = components; + } + if (audioFormats.includes(format)) { + return UploadAudio(file, format); + } + fs.unlink(filepath, () => {}); + return { source: file, result: { name: 'Unsupported audio format', message: `Could not upload unsupported file (${originalFilename}). Please convert to an .mp3` } }; + } + case 'text': + if (types[1] === 'csv') { + return UploadCsv(file); + } + break; + default: + } + + console.log(red(`Ignoring unsupported file (${originalFilename}) with upload type (${mimetype}).`)); + fs.unlink(filepath, () => {}); + return { source: file, result: new Error(`Could not upload unsupported file (${originalFilename}) with upload type (${mimetype}).`) }; } - /** - * define the resizers to use - * @param ext the extension - * @returns an array of resize descriptions - */ - export function imageResampleSizes(ext: string): DashUploadUtils.ImageResizer[] { - return [ - { suffix: SizeSuffix.Original, width: 0 }, - ...[...(AcceptableMedia.imageFormats.includes(ext.toLowerCase()) ? Object.values(DashUploadUtils.Sizes) : [])].map(({ suffix, width }) => ({ - width, - suffix, - })), - ]; + export async function buildFileDirectories() { + if (!existsSync(publicDirectory)) { + console.error('\nPlease ensure that the following directory exists...\n'); + console.log(publicDirectory); + process.exit(0); + } + if (!existsSync(filesDirectory)) { + console.error('\nPlease ensure that the following directory exists...\n'); + console.log(filesDirectory); + process.exit(0); + } + const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`)); + return Promise.all(pending); + } + + export interface RequestedImageSize { + width: number; + height: number; + type: string; + } + + export interface ImageResizer { + width: number; + suffix: SizeSuffix; } } diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts index 423c719c2..041f65592 100644 --- a/src/server/GarbageCollector.ts +++ b/src/server/GarbageCollector.ts @@ -1,11 +1,15 @@ +/* eslint-disable no-await-in-loop */ +/* eslint-disable no-continue */ +/* eslint-disable no-cond-assign */ +/* eslint-disable no-restricted-syntax */ import * as fs from 'fs'; import * as path from 'path'; import { Database } from './database'; import { Search } from './Search'; - function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { for (const key in doc) { + // eslint-disable-next-line no-prototype-builtins if (!doc.hasOwnProperty(key)) { continue; } @@ -13,22 +17,22 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { if (field === undefined || field === null) { continue; } - if (field.__type === "proxy" || field.__type === "prefetch_proxy") { + if (field.__type === 'proxy' || field.__type === 'prefetch_proxy') { ids.push(field.fieldId); - } else if (field.__type === "list") { + } else if (field.__type === 'list') { addDoc(field.fields, ids, files); - } else if (typeof field === "string") { - const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g; + } else if (typeof field === 'string') { + const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w-]*)"/g; let match: string[] | null; while ((match = re.exec(field)) !== null) { ids.push(match[1]); } - } else if (field.__type === "RichTextField") { + } else if (field.__type === 'RichTextField') { const re = /"href"\s*:\s*"(.*?)"/g; let match: string[] | null; while ((match = re.exec(field.Data)) !== null) { const urlString = match[1]; - const split = new URL(urlString).pathname.split("doc/"); + const split = new URL(urlString).pathname.split('doc/'); if (split.length > 1) { ids.push(split[split.length - 1]); } @@ -36,7 +40,7 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { const re2 = /"src"\s*:\s*"(.*?)"/g; while ((match = re2.exec(field.Data)) !== null) { const urlString = match[1]; - const pathname = new URL(urlString).pathname; + const { pathname } = new URL(urlString); const ext = path.extname(pathname); const fileName = path.basename(pathname, ext); let exts = files[fileName]; @@ -45,9 +49,9 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { } exts.push(ext); } - } else if (["audio", "image", "video", "pdf", "web", "map"].includes(field.__type)) { + } else if (['audio', 'image', 'video', 'pdf', 'web', 'map'].includes(field.__type)) { const url = new URL(field.url); - const pathname = url.pathname; + const { pathname } = url; const ext = path.extname(pathname); const fileName = path.basename(pathname, ext); let exts = files[fileName]; @@ -60,12 +64,12 @@ function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) { } async function GarbageCollect(full: boolean = true) { - console.log("start GC"); + console.log('start GC'); const start = Date.now(); // await new Promise(res => setTimeout(res, 3000)); const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users'); const users = await cursor.toArray(); - const ids: string[] = [...users.map((user:any) => user.userDocumentId), ...users.map((user:any) => user.sharingDocumentId), ...users.map((user:any) => user.linkDatabaseId)]; + const ids: string[] = [...users.map((user: any) => user.userDocumentId), ...users.map((user: any) => user.sharingDocumentId), ...users.map((user: any) => user.linkDatabaseId)]; const visited = new Set<string>(); const files: { [name: string]: string[] } = {}; @@ -76,9 +80,11 @@ async function GarbageCollect(full: boolean = true) { if (!fetchIds.length) { continue; } - const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res)); + const docs = await new Promise<{ [key: string]: any }[]>(res => { + Database.Instance.getDocuments(fetchIds, res); + }); for (const doc of docs) { - const id = doc.id; + const { id } = doc; if (doc === undefined) { console.log(`Couldn't find field with Id ${id}`); continue; @@ -95,19 +101,27 @@ async function GarbageCollect(full: boolean = true) { const notToDelete = Array.from(visited); const toDeleteCursor = await Database.Instance.query({ _id: { $nin: notToDelete } }, { _id: 1 }); - const toDelete: string[] = (await toDeleteCursor.toArray()).map((doc:any) => doc._id); + const toDelete: string[] = (await toDeleteCursor.toArray()).map((doc: any) => doc._id); toDeleteCursor.close(); if (!full) { - await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { "deleted": true } }); - await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { "deleted": true } }); - console.log(await Search.updateDocuments( - notToDelete.map<any>(id => ({ - id, deleted: { set: null } - })) - .concat(toDelete.map(id => ({ - id, deleted: { set: true } - }))))); - console.log("Done with partial GC"); + await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { deleted: true } }); + await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { deleted: true } }); + console.log( + await Search.updateDocuments( + notToDelete + .map<any>(id => ({ + id, + deleted: { set: null }, + })) + .concat( + toDelete.map(id => ({ + id, + deleted: { set: true }, + })) + ) + ) + ); + console.log('Done with partial GC'); console.log(`Took ${(Date.now() - start) / 1000} seconds`); } else { let i = 0; @@ -123,15 +137,15 @@ async function GarbageCollect(full: boolean = true) { console.log(`${deleted} documents deleted`); await Search.deleteDocuments(toDelete); - console.log("Cleared search documents"); + console.log('Cleared search documents'); - const folder = "./src/server/public/files/"; + const folder = './src/server/public/files/'; fs.readdir(folder, (_, fileList) => { const filesToDelete = fileList.filter(file => { const ext = path.extname(file); let base = path.basename(file, ext); const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext); - return file !== ".gitignore" && !existsInDb; + return file !== '.gitignore' && !existsInDb; }); console.log(`Deleting ${filesToDelete.length} files`); filesToDelete.forEach(file => { diff --git a/src/server/MemoryDatabase.ts b/src/server/MemoryDatabase.ts index b74332bf5..1432d91c4 100644 --- a/src/server/MemoryDatabase.ts +++ b/src/server/MemoryDatabase.ts @@ -3,16 +3,15 @@ import { DocumentsCollection, IDatabase } from './IDatabase'; import { Transferable } from './Message'; export class MemoryDatabase implements IDatabase { - private db: { [collectionName: string]: { [id: string]: any } } = {}; private getCollection(collectionName: string) { const collection = this.db[collectionName]; if (collection) { return collection; - } else { - return this.db[collectionName] = {}; } + this.db[collectionName] = {}; + return {}; } public getCollectionNames() { @@ -21,15 +20,15 @@ export class MemoryDatabase implements IDatabase { public update(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateResult) => void, _upsert?: boolean, collectionName = DocumentsCollection): Promise<void> { const collection = this.getCollection(collectionName); - const set = "$set"; + const set = '$set'; if (set in value) { let currentVal = collection[id] ?? (collection[id] = {}); const val = value[set]; for (const key in val) { - const keys = key.split("."); + const keys = key.split('.'); for (let i = 0; i < keys.length - 1; i++) { const k = keys[i]; - if (typeof currentVal[k] === "object") { + if (typeof currentVal[k] === 'object') { currentVal = currentVal[k]; } else { currentVal[k] = {}; @@ -45,7 +44,7 @@ export class MemoryDatabase implements IDatabase { return Promise.resolve(undefined); } - public updateMany(query: any, update: any, collectionName = DocumentsCollection): Promise<mongodb.UpdateResult> { + public updateMany(/* query: any, update: any, collectionName = DocumentsCollection */): Promise<mongodb.UpdateResult> { throw new Error("Can't updateMany a MemoryDatabase"); } @@ -54,7 +53,9 @@ export class MemoryDatabase implements IDatabase { } public delete(query: any, collectionName?: string): Promise<mongodb.DeleteResult>; + // eslint-disable-next-line no-dupe-class-members public delete(id: string, collectionName?: string): Promise<mongodb.DeleteResult>; + // eslint-disable-next-line no-dupe-class-members public delete(id: any, collectionName = DocumentsCollection): Promise<mongodb.DeleteResult> { const i = id.id ?? id; delete this.getCollection(collectionName)[i]; @@ -75,7 +76,7 @@ export class MemoryDatabase implements IDatabase { } public insert(value: any, collectionName = DocumentsCollection): Promise<void> { - const id = value.id; + const { id } = value; this.getCollection(collectionName)[id] = value; return Promise.resolve(); } @@ -93,14 +94,18 @@ export class MemoryDatabase implements IDatabase { const count = Math.min(ids.length, 1000); const index = ids.length - count; const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); - if (!fetchIds.length) { - continue; - } - const docs = await new Promise<{ [key: string]: any }[]>(res => this.getDocuments(fetchIds, res, collectionName)); - for (const doc of docs) { - const id = doc.id; - visited.add(id); - ids.push(...(await fn(doc))); + if (fetchIds.length) { + // eslint-disable-next-line no-await-in-loop + const docs = await new Promise<{ [key: string]: any }[]>(res => { + this.getDocuments(fetchIds, res, collectionName); + }); + // eslint-disable-next-line no-restricted-syntax + for (const doc of docs) { + const { id } = doc; + visited.add(id); + // eslint-disable-next-line no-await-in-loop + ids.push(...(await fn(doc))); + } } } } diff --git a/src/server/Message.ts b/src/server/Message.ts index 8f0af08bc..03150c841 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -1,22 +1,47 @@ -import { Point } from "../pen-gestures/ndollar"; -import { Utils } from "../Utils"; +import * as uuid from 'uuid'; +import { Point } from '../pen-gestures/ndollar'; +function GenerateDeterministicGuid(seed: string): string { + return uuid.v5(seed, uuid.v5.URL); +} +// eslint-disable-next-line @typescript-eslint/no-unused-vars export class Message<T> { private _name: string; private _guid: string; constructor(name: string) { this._name = name; - this._guid = Utils.GenerateDeterministicGuid(name); + this._guid = GenerateDeterministicGuid(name); } - get Name(): string { return this._name; } - get Message(): string { return this._guid; } + get Name(): string { + return this._name; + } + get Message(): string { + return this._guid; + } } export enum Types { - Number, List, Key, Image, Web, Document, Text, Icon, RichText, DocumentReference, - Html, Video, Audio, Ink, PDF, Tuple, Boolean, Script, Templates + Number, + List, + Key, + Image, + Web, + Document, + Text, + Icon, + RichText, + DocumentReference, + Html, + Video, + Audio, + Ink, + PDF, + Tuple, + Boolean, + Script, + Templates, } export interface Transferable { @@ -26,7 +51,9 @@ export interface Transferable { } export enum YoutubeQueryTypes { - Channels, SearchVideo, VideoDetails + Channels, + SearchVideo, + VideoDetails, } export interface YoutubeQueryInput { @@ -45,7 +72,7 @@ export interface Diff extends Reference { export interface GestureContent { readonly points: Array<Point>; - readonly bounds: { right: number, left: number, bottom: number, top: number, width: number, height: number }; + readonly bounds: { right: number; left: number; bottom: number; top: number; width: number; height: number }; readonly width?: string; readonly color?: string; } @@ -73,27 +100,27 @@ export interface RoomMessage { } export namespace MessageStore { - export const Foo = new Message<string>("Foo"); - export const Bar = new Message<string>("Bar"); - export const SetField = new Message<Transferable>("Set Field"); // send Transferable (no reply) - export const GetField = new Message<string>("Get Field"); // send string 'id' get Transferable back - export const GetFields = new Message<string[]>("Get Fields"); // send string[] of 'id' get Transferable[] back - export const GetDocument = new Message<string>("Get Document"); - export const DeleteAll = new Message<any>("Delete All"); - export const ConnectionTerminated = new Message<string>("Connection Terminated"); - - export const GesturePoints = new Message<GestureContent>("Gesture Points"); - export const MobileInkOverlayTrigger = new Message<MobileInkOverlayContent>("Trigger Mobile Ink Overlay"); - export const UpdateMobileInkOverlayPosition = new Message<UpdateMobileInkOverlayPositionContent>("Update Mobile Ink Overlay Position"); - export const MobileDocumentUpload = new Message<MobileDocumentUploadContent>("Upload Document From Mobile"); - - export const GetRefField = new Message<string>("Get Ref Field"); - export const GetRefFields = new Message<string[]>("Get Ref Fields"); - export const UpdateField = new Message<Diff>("Update Ref Field"); - export const CreateField = new Message<Reference>("Create Ref Field"); - export const YoutubeApiQuery = new Message<YoutubeQueryInput>("Youtube Api Query"); - export const DeleteField = new Message<string>("Delete field"); - export const DeleteFields = new Message<string[]>("Delete fields"); - - export const UpdateStats = new Message<string>("updatestats"); + export const Foo = new Message<string>('Foo'); + export const Bar = new Message<string>('Bar'); + export const SetField = new Message<Transferable>('Set Field'); // send Transferable (no reply) + export const GetField = new Message<string>('Get Field'); // send string 'id' get Transferable back + export const GetFields = new Message<string[]>('Get Fields'); // send string[] of 'id' get Transferable[] back + export const GetDocument = new Message<string>('Get Document'); + export const DeleteAll = new Message<any>('Delete All'); + export const ConnectionTerminated = new Message<string>('Connection Terminated'); + + export const GesturePoints = new Message<GestureContent>('Gesture Points'); + export const MobileInkOverlayTrigger = new Message<MobileInkOverlayContent>('Trigger Mobile Ink Overlay'); + export const UpdateMobileInkOverlayPosition = new Message<UpdateMobileInkOverlayPositionContent>('Update Mobile Ink Overlay Position'); + export const MobileDocumentUpload = new Message<MobileDocumentUploadContent>('Upload Document From Mobile'); + + export const GetRefField = new Message<string>('Get Ref Field'); + export const GetRefFields = new Message<string[]>('Get Ref Fields'); + export const UpdateField = new Message<Diff>('Update Ref Field'); + export const CreateField = new Message<Reference>('Create Ref Field'); + export const YoutubeApiQuery = new Message<YoutubeQueryInput>('Youtube Api Query'); + export const DeleteField = new Message<string>('Delete field'); + export const DeleteFields = new Message<string[]>('Delete fields'); + + export const UpdateStats = new Message<string>('updatestats'); } diff --git a/src/server/PdfTypes.ts b/src/server/PdfTypes.ts index e87f08e1d..fb435a677 100644 --- a/src/server/PdfTypes.ts +++ b/src/server/PdfTypes.ts @@ -1,21 +1,19 @@ -export interface ParsedPDF { - numpages: number; - numrender: number; - info: PDFInfo; - metadata: PDFMetadata; - version: string; //https://mozilla.github.io/pdf.js/getting_started/ - text: string; -} - export interface PDFInfo { PDFFormatVersion: string; IsAcroFormPresent: boolean; IsXFAPresent: boolean; [key: string]: any; } - export interface PDFMetadata { parse(): void; get(name: string): string; has(name: string): boolean; -}
\ No newline at end of file +} +export interface ParsedPDF { + numpages: number; + numrender: number; + info: PDFInfo; + metadata: PDFMetadata; + version: string; // https://mozilla.github.io/pdf.js/getting_started/ + text: string; +} diff --git a/src/server/ProcessFactory.ts b/src/server/ProcessFactory.ts index f69eda4c3..3791b0e1e 100644 --- a/src/server/ProcessFactory.ts +++ b/src/server/ProcessFactory.ts @@ -1,44 +1,42 @@ -import { ChildProcess, spawn, StdioOptions } from "child_process"; -import { existsSync, mkdirSync } from "fs"; -import { Stream } from "stream"; +import { ChildProcess, spawn, StdioOptions } from 'child_process'; +import { existsSync, mkdirSync } from 'fs'; +import { rimraf } from 'rimraf'; +import { Stream } from 'stream'; import { fileDescriptorFromStream, pathFromRoot } from './ActionUtilities'; -import { rimraf } from "rimraf"; - -export namespace ProcessFactory { - - export type Sink = "pipe" | "ipc" | "ignore" | "inherit" | Stream | number | null | undefined; - - export async function createWorker(command: string, args?: readonly string[], stdio?: StdioOptions | "logfile", detached = true): Promise<ChildProcess> { - if (stdio === "logfile") { - const log_fd = await Logger.create(command, args); - stdio = ["ignore", log_fd, log_fd]; - } - const child = spawn(command, args ?? [], { detached, stdio }); - child.unref(); - return child; - } - -} export namespace Logger { - - const logPath = pathFromRoot("./logs"); + const logPath = pathFromRoot('./logs'); export async function initialize() { if (existsSync(logPath)) { if (!process.env.SPAWNED) { - await new Promise<any>(resolve => rimraf(logPath).then(resolve)); + await new Promise<any>(resolve => { + rimraf(logPath).then(resolve); + }); } } mkdirSync(logPath); } - export async function create(command: string, args?: readonly string[]): Promise<number> { - return fileDescriptorFromStream(generate_log_path(command, args)); + function generateLogPath(command: string, args?: readonly string[]) { + return pathFromRoot(`./logs/${command}-${args?.length}-${new Date().toUTCString()}.log`); } - function generate_log_path(command: string, args?: readonly string[]) { - return pathFromRoot(`./logs/${command}-${args?.length}-${new Date().toUTCString()}.log`); + export async function create(command: string, args?: readonly string[]): Promise<number> { + return fileDescriptorFromStream(generateLogPath(command, args)); } +} +export namespace ProcessFactory { + export type Sink = 'pipe' | 'ipc' | 'ignore' | 'inherit' | Stream | number | null | undefined; -}
\ No newline at end of file + export async function createWorker(command: string, args?: readonly string[], stdio?: StdioOptions | 'logfile', detached = true): Promise<ChildProcess> { + if (stdio === 'logfile') { + const logFd = await Logger.create(command, args); + // eslint-disable-next-line no-param-reassign + stdio = ['ignore', logFd, logFd]; + } + const child = spawn(command, args ?? [], { detached, stdio }); + child.unref(); + return child; + } +} diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 540bca776..d8e0455f6 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -1,9 +1,9 @@ import { cyan, green, red } from 'colors'; import { Express, Request, Response } from 'express'; -import { AdminPriviliges } from '.'; import { Utils } from '../Utils'; -import { DashUserModel } from './authentication/DashUserModel'; import RouteSubscriber from './RouteSubscriber'; +import { AdminPrivileges } from './SocketData'; +import { DashUserModel } from './authentication/DashUserModel'; export enum Method { GET, @@ -21,6 +21,34 @@ export type SecureHandler = (core: AuthorizedCore) => any | Promise<any>; export type PublicHandler = (core: CoreArguments) => any | Promise<any>; export type ErrorHandler = (core: CoreArguments & { error: any }) => any | Promise<any>; +export const STATUS = { + OK: 200, + BAD_REQUEST: 400, + EXECUTION_ERROR: 500, + PERMISSION_DENIED: 403, +}; + +export function _error(res: Response, message: string, error?: any) { + console.error(message, error); + res.statusMessage = message; + res.status(STATUS.EXECUTION_ERROR).send(error); +} + +export function _success(res: Response, body: any) { + res.status(STATUS.OK).send(body); +} + +export function _invalid(res: Response, message: string) { + res.statusMessage = message; + res.status(STATUS.BAD_REQUEST).send(); +} + +export function _permissionDenied(res: Response, message?: string) { + if (message) { + res.statusMessage = message; + } + res.status(STATUS.PERMISSION_DENIED).send(`Permission Denied! ${message}`); +} export interface RouteInitializer { method: Method; subscription: string | RouteSubscriber | (string | RouteSubscriber)[]; @@ -71,7 +99,7 @@ export default class RouteManager { console.log('please remove all duplicate routes before continuing'); } if (malformedCount) { - console.log(`please ensure all routes adhere to ^\/$|^\/[A-Za-z]+(\/\:[A-Za-z?_]+)*$`); + console.log(`please ensure all routes adhere to ^/$|^/[A-Za-z]+(/:[A-Za-z?_]+)*$`); } process.exit(1); } else { @@ -94,7 +122,7 @@ export default class RouteManager { typeof initializer.subscription === 'string' && RouteManager.routes.push(initializer.subscription); initializer.subscription instanceof RouteSubscriber && RouteManager.routes.push(initializer.subscription.root); initializer.subscription instanceof Array && - initializer.subscription.map(sub => { + initializer.subscription.forEach(sub => { typeof sub === 'string' && RouteManager.routes.push(sub); sub instanceof RouteSubscriber && RouteManager.routes.push(sub.root); }); @@ -120,23 +148,23 @@ export default class RouteManager { }; if (user) { if (requireAdmin && isRelease && process.env.PASSWORD) { - if (AdminPriviliges.get(user.id)) { - AdminPriviliges.delete(user.id); + if (AdminPrivileges.get(user.id)) { + AdminPrivileges.delete(user.id); } else { - return res.redirect(`/admin/${req.originalUrl.substring(1).replace('/', ':')}`); + res.redirect(`/admin/${req.originalUrl.substring(1).replace('/', ':')}`); + return; } } await tryExecute(secureHandler, { ...core, user }); - } else { - //req.session!.target = target; - if (publicHandler) { - await tryExecute(publicHandler, core); - if (!res.headersSent) { - // res.redirect("/login"); - } - } else { - res.redirect('/login'); + } + // req.session!.target = target; + else if (publicHandler) { + await tryExecute(publicHandler, core); + if (!res.headersSent) { + // res.redirect("/login"); } + } else { + res.redirect('/login'); } setTimeout(() => { if (!res.headersSent) { @@ -153,7 +181,7 @@ export default class RouteManager { } else { route = subscriber.build; } - if (!/^\/$|^\/[A-Za-z\*]+(\/\:[A-Za-z?_\*]+)*$/g.test(route)) { + if (!/^\/$|^\/[A-Za-z*]+(\/:[A-Za-z?_*]+)*$/g.test(route)) { this.failedRegistrations.push({ reason: RegistrationError.Malformed, route, @@ -180,6 +208,7 @@ export default class RouteManager { case Method.POST: this.server.post(route, supervised); break; + default: } } }; @@ -190,32 +219,3 @@ export default class RouteManager { } }; } - -export const STATUS = { - OK: 200, - BAD_REQUEST: 400, - EXECUTION_ERROR: 500, - PERMISSION_DENIED: 403, -}; - -export function _error(res: Response, message: string, error?: any) { - console.error(message, error); - res.statusMessage = message; - res.status(STATUS.EXECUTION_ERROR).send(error); -} - -export function _success(res: Response, body: any) { - res.status(STATUS.OK).send(body); -} - -export function _invalid(res: Response, message: string) { - res.statusMessage = message; - res.status(STATUS.BAD_REQUEST).send(); -} - -export function _permission_denied(res: Response, message?: string) { - if (message) { - res.statusMessage = message; - } - res.status(STATUS.PERMISSION_DENIED).send(`Permission Denied! ${message}`); -} diff --git a/src/server/RouteSubscriber.ts b/src/server/RouteSubscriber.ts index a1cf7c1c4..b923805a8 100644 --- a/src/server/RouteSubscriber.ts +++ b/src/server/RouteSubscriber.ts @@ -18,9 +18,8 @@ export default class RouteSubscriber { public get build() { let output = this._root; if (this.requestParameters.length) { - output = `${output}/:${this.requestParameters.join("/:")}`; + output = `${output}/:${this.requestParameters.join('/:')}`; } return output; } - -}
\ No newline at end of file +} diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts index 7db1c2dae..0a961f570 100644 --- a/src/server/SharedMediaTypes.ts +++ b/src/server/SharedMediaTypes.ts @@ -22,24 +22,17 @@ export namespace Upload { return 'duration' in uploadResponse; } + export interface AccessPathInfo { + [suffix: string]: { client: string; server: string }; + } export interface FileInformation { accessPaths: AccessPathInfo; rawText?: string; duration?: number; } - - export type FileResponse<T extends FileInformation = FileInformation> = { source: File; result: T | Error }; - - export type ImageInformation = FileInformation & InspectionResults; - - export type VideoInformation = FileInformation & VideoResults; - - export interface AccessPathInfo { - [suffix: string]: { client: string; server: string }; - } - - export interface VideoResults { - duration: number; + export interface EnrichedExifData { + data: ExifData & ExifData['gps']; + error?: string; } export interface InspectionResults { source: string; @@ -51,9 +44,13 @@ export namespace Upload { nativeHeight: number; filename?: string; } - - export interface EnrichedExifData { - data: ExifData & ExifData['gps']; - error?: string; + export interface VideoResults { + duration: number; } + + export type FileResponse<T extends FileInformation = FileInformation> = { source: File; result: T | Error }; + + export type ImageInformation = FileInformation & InspectionResults; + + export type VideoInformation = FileInformation & VideoResults; } diff --git a/src/server/SocketData.ts b/src/server/SocketData.ts new file mode 100644 index 000000000..e857996e5 --- /dev/null +++ b/src/server/SocketData.ts @@ -0,0 +1,35 @@ +import { Socket } from 'socket.io'; +import * as path from 'path'; + +export const timeMap: { [id: string]: number } = {}; +export const userOperations = new Map<string, number>(); +export const socketMap = new Map<Socket, string>(); + +export const publicDirectory = path.resolve(__dirname, 'public'); +export const filesDirectory = path.resolve(publicDirectory, 'files'); + +export const AdminPrivileges: Map<string, boolean> = new Map(); + +export const resolvedPorts: { server: number; socket: number } = { server: 1050, socket: 4321 }; + +export enum Directory { + parsed_files = 'parsed_files', + images = 'images', + videos = 'videos', + pdfs = 'pdfs', + text = 'text', + audio = 'audio', + csv = 'csv', +} + +export function serverPathToFile(directory: Directory, filename: string) { + return path.normalize(`${filesDirectory}/${directory}/${filename}`); +} + +export function pathToDirectory(directory: Directory) { + return path.normalize(`${filesDirectory}/${directory}`); +} + +export function clientPathToFile(directory: Directory, filename: string) { + return `/files/${directory}/${filename}`; +} diff --git a/src/server/apis/google/CredentialsLoader.ts b/src/server/apis/google/CredentialsLoader.ts index ef1f9a91e..46dc00b8a 100644 --- a/src/server/apis/google/CredentialsLoader.ts +++ b/src/server/apis/google/CredentialsLoader.ts @@ -1,10 +1,9 @@ -import { readFile, readFileSync } from "fs"; -import { pathFromRoot } from "../../ActionUtilities"; -import { SecureContextOptions } from "tls"; -import { blue, red } from "colors"; +import { readFile, readFileSync } from 'fs'; +import { SecureContextOptions } from 'tls'; +import { blue, red } from 'colors'; +import { pathFromRoot } from '../../ActionUtilities'; export namespace GoogleCredentialsLoader { - export interface InstalledCredentials { client_id: string; project_id: string; @@ -28,18 +27,16 @@ export namespace GoogleCredentialsLoader { }); }); } - } export namespace SSL { - export let Credentials: SecureContextOptions = {}; export let Loaded = false; const suffixes = { - privateKey: ".key", - certificate: ".crt", - caBundle: "-ca.crt" + privateKey: '.key', + certificate: '.crt', + caBundle: '-ca.crt', }; export async function loadCredentials() { @@ -57,11 +54,10 @@ export namespace SSL { } export function exit() { - console.log(red("Running this server in release mode requires the following SSL credentials in the project root:")); - const serverName = process.env.serverName ? process.env.serverName : "{process.env.serverName}"; + console.log(red('Running this server in release mode requires the following SSL credentials in the project root:')); + const serverName = process.env.serverName ? process.env.serverName : '{process.env.serverName}'; Object.values(suffixes).forEach(suffix => console.log(blue(`${serverName}${suffix}`))); - console.log(red("Please ensure these files exist and restart, or run this in development mode.")); + console.log(red('Please ensure these files exist and restart, or run this in development mode.')); process.exit(0); } - } diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 3940b7a3d..d3acc968b 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -1,9 +1,9 @@ -import { google } from 'googleapis'; -import { OAuth2Client, Credentials, OAuth2ClientOptions } from 'google-auth-library'; -import { Opt } from '../../../fields/Doc'; import { GaxiosResponse } from 'gaxios'; -import * as request from 'request-promise'; +import { Credentials, OAuth2Client, OAuth2ClientOptions } from 'google-auth-library'; +import { google } from 'googleapis'; import * as qs from 'query-string'; +import * as request from 'request-promise'; +import { Opt } from '../../../fields/Doc'; import { Database } from '../../database'; import { GoogleCredentialsLoader } from './CredentialsLoader'; @@ -57,12 +57,12 @@ export namespace GoogleApiServerUtils { * global, intentionally unauthenticated worker OAuth2 client instance. */ export function processProjectCredentials(): void { - const { client_secret, client_id, redirect_uris } = GoogleCredentialsLoader.ProjectCredentials; + const { client_secret: clientSecret, client_id: clientId, redirect_uris: redirectUris } = GoogleCredentialsLoader.ProjectCredentials; // initialize the global authorization client oAuthOptions = { - clientId: client_id, - clientSecret: client_secret, - redirectUri: redirect_uris[0], + clientId, + clientSecret, + redirectUri: redirectUris[0], }; worker = generateClient(); } diff --git a/src/server/apis/google/SharedTypes.ts b/src/server/apis/google/SharedTypes.ts index 9ad6130b6..f5e0e2e2b 100644 --- a/src/server/apis/google/SharedTypes.ts +++ b/src/server/apis/google/SharedTypes.ts @@ -1,9 +1,3 @@ -export interface NewMediaItemResult { - uploadToken: string; - status: { code: number, message: string }; - mediaItem: MediaItem; -} - export interface MediaItem { id: string; description: string; @@ -17,5 +11,10 @@ export interface MediaItem { }; filename: string; } +export interface NewMediaItemResult { + uploadToken: string; + status: { code: number; message: string }; + mediaItem: MediaItem; +} -export type MediaItemCreationResult = { newMediaItemResults: NewMediaItemResult[] };
\ No newline at end of file +export type MediaItemCreationResult = { newMediaItemResults: NewMediaItemResult[] }; diff --git a/src/server/apis/youtube/youtubeApiSample.d.ts b/src/server/apis/youtube/youtubeApiSample.d.ts index 427f54608..97c3f0b54 100644 --- a/src/server/apis/youtube/youtubeApiSample.d.ts +++ b/src/server/apis/youtube/youtubeApiSample.d.ts @@ -1,2 +1,2 @@ declare const YoutubeApi: any; -export = YoutubeApi;
\ No newline at end of file +export = YoutubeApi; diff --git a/src/server/authentication/AuthenticationManager.ts b/src/server/authentication/AuthenticationManager.ts index 9c1525df0..0cc1553c0 100644 --- a/src/server/authentication/AuthenticationManager.ts +++ b/src/server/authentication/AuthenticationManager.ts @@ -1,21 +1,21 @@ -import { default as User, DashUserModel, initializeGuest } from './DashUserModel'; -import { Request, Response, NextFunction } from 'express'; -import * as passport from 'passport'; -import { IVerifyOptions } from 'passport-local'; -import './Passport'; import * as async from 'async'; -import * as nodemailer from 'nodemailer'; import * as c from 'crypto'; -import { emptyFunction, Utils } from '../../Utils'; -import { MailOptions } from 'nodemailer/lib/stream-transport'; +import { NextFunction, Request, Response } from 'express'; import { check, validationResult } from 'express-validator'; +import * as nodemailer from 'nodemailer'; +import { MailOptions } from 'nodemailer/lib/stream-transport'; +import * as passport from 'passport'; +import { Utils } from '../../Utils'; +import User, { DashUserModel, initializeGuest } from './DashUserModel'; +import './Passport'; +// import { IVerifyOptions } from 'passport-local'; /** * GET /signup * Directs user to the signup page * modeled by signup.pug in views */ -export let getSignup = (req: Request, res: Response) => { +export const getSignup = (req: Request, res: Response) => { if (req.user) { return res.redirect('/home'); } @@ -23,13 +23,23 @@ export let getSignup = (req: Request, res: Response) => { title: 'Sign Up', user: req.user, }); + return undefined; +}; + +const tryRedirectToTarget = (req: Request, res: Response) => { + const target = (req.session as any)?.target; + if (req.session && target) { + res.redirect(target); + } else { + res.redirect('/home'); + } }; /** * POST /signup * Create a new local account. */ -export let postSignup = (req: Request, res: Response, next: NextFunction) => { +export const postSignup = (req: Request, res: Response, next: NextFunction) => { const email = req.body.email as String; check('email', 'Email is not valid').isEmail().run(req); check('password', 'Password must be at least 4 characters long').isLength({ min: 4 }).run(req); @@ -42,7 +52,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { return res.redirect('/signup'); } - const password = req.body.password; + const { password } = req.body; const model = { email, @@ -65,35 +75,29 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => { req.logIn(user, err => { if (err) return next(err); tryRedirectToTarget(req, res); + return undefined; }); }) .catch((err: any) => next(err)); + return undefined; }) .catch((err: any) => next(err)); + return undefined; }; - -const tryRedirectToTarget = (req: Request, res: Response) => { - const target = (req.session as any)?.target; - if (req.session && target) { - res.redirect(target); - } else { - res.redirect('/home'); - } -}; - /** * GET /login * Login page. */ -export let getLogin = (req: Request, res: Response) => { +export const getLogin = (req: Request, res: Response) => { if (req.user) { - //req.session.target = undefined; + // req.session.target = undefined; return res.redirect('/home'); } res.render('login.pug', { title: 'Log In', user: req.user, }); + return undefined; }; /** @@ -101,7 +105,7 @@ export let getLogin = (req: Request, res: Response) => { * Sign in using email and password. * On failure, redirect to signup page */ -export let postLogin = (req: Request, res: Response, next: NextFunction) => { +export const postLogin = (req: Request, res: Response, next: NextFunction) => { if (req.body.email === '') { User.findOne({ email: 'guest' }) .then((user: any) => !user && initializeGuest()) @@ -119,23 +123,21 @@ export let postLogin = (req: Request, res: Response, next: NextFunction) => { return res.redirect('/signup'); } - const callback = (err: Error, user: DashUserModel, _info: IVerifyOptions) => { + const callback = (err: Error, user: DashUserModel /* , _info: IVerifyOptions */) => { if (err) { next(err); - return; - } - if (!user) { + } else if (!user) { return res.redirect('/signup'); - } - req.logIn(user, err => { - if (err) { - next(err); - return; - } - tryRedirectToTarget(req, res); - }); + } else + req.logIn(user, loginErr => { + if (loginErr) { + next(loginErr); + } else tryRedirectToTarget(req, res); + }); + return undefined; }; setTimeout(() => passport.authenticate('local', callback)(req, res, next), 500); + return undefined; }; /** @@ -143,31 +145,29 @@ export let postLogin = (req: Request, res: Response, next: NextFunction) => { * Invokes the logout function on the request * and destroys the user's current session. */ -export let getLogout = (req: Request, res: Response) => { +export const getLogout = (req: Request, res: Response) => { req.logout(err => { if (err) console.log(err); else res.redirect('/login'); }); }; -export let getForgot = function (req: Request, res: Response) { +export const getForgot = function (req: Request, res: Response) { res.render('forgot.pug', { title: 'Recover Password', user: req.user, }); }; -export let postForgot = function (req: Request, res: Response, next: NextFunction) { - const email = req.body.email; +export const postForgot = function (req: Request, res: Response, next: NextFunction) { + const { email } = req.body; async.waterfall( [ function (done: any) { - c.randomBytes(20, function (err: any, buffer: Buffer) { + c.randomBytes(20, (err: any, buffer: Buffer) => { if (err) { done(null); - return; - } - done(null, buffer.toString('hex')); + } else done(null, buffer.toString('hex')); }); }, function (token: string, done: any) { @@ -204,20 +204,21 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio '\n\n' + 'If you did not request this, please ignore this email and your password will remain unchanged.\n', } as MailOptions; - smtpTransport.sendMail(mailOptions, function (err: Error | null) { + smtpTransport.sendMail(mailOptions, (err: Error | null) => { // req.flash('info', 'An e-mail has been sent to ' + user.email + ' with further instructions.'); done(null, err, 'done'); }); }, ], - function (err) { + err => { if (err) return next(err); res.redirect('/forgotPassword'); + return undefined; } ); }; -export let getReset = function (req: Request, res: Response) { +export const getReset = function (req: Request, res: Response) { User.findOne({ passwordResetToken: req.params.token, passwordResetExpires: { $gt: Date.now() } }) .then((user: any) => { if (!user) return res.redirect('/forgotPassword'); @@ -225,11 +226,12 @@ export let getReset = function (req: Request, res: Response) { title: 'Reset Password', user: req.user, }); + return undefined; }) - .catch((err: any) => res.redirect('/forgotPassword')); + .catch(() => res.redirect('/forgotPassword')); }; -export let postReset = function (req: Request, res: Response) { +export const postReset = function (req: Request, res: Response) { async.waterfall( [ function (done: any) { @@ -251,10 +253,11 @@ export let postReset = function (req: Request, res: Response) { () => (req as any).logIn(user), (err: any) => err ) - .catch((err: any) => res.redirect('/login')); + .catch(() => res.redirect('/login')); done(null, user); + return undefined; }) - .catch((err: any) => res.redirect('back')); + .catch(() => res.redirect('back')); }, function (user: DashUserModel, done: any) { const smtpTransport = nodemailer.createTransport({ @@ -268,13 +271,13 @@ export let postReset = function (req: Request, res: Response) { to: user.email, from: 'browndashptc@gmail.com', subject: 'Your password has been changed', - text: 'Hello,\n\n' + 'This is a confirmation that the password for your account ' + user.email + ' has just been changed.\n', + text: 'Hello,\n\nThis is a confirmation that the password for your account ' + user.email + ' has just been changed.\n', } as MailOptions; smtpTransport.sendMail(mailOptions, err => done(null, err)); }, ], - function (err) { + () => { res.redirect('/login'); } ); diff --git a/src/server/authentication/DashUserModel.ts b/src/server/authentication/DashUserModel.ts index dbb7a79ed..a288bfeab 100644 --- a/src/server/authentication/DashUserModel.ts +++ b/src/server/authentication/DashUserModel.ts @@ -1,9 +1,8 @@ -//@ts-ignore import * as bcrypt from 'bcrypt-nodejs'; -//@ts-ignore import * as mongoose from 'mongoose'; import { Utils } from '../../Utils'; +type comparePasswordFunction = (candidatePassword: string, cb: (err: any, isMatch: any) => void) => void; export type DashUserModel = mongoose.Document & { email: String; password: string; @@ -26,8 +25,6 @@ export type DashUserModel = mongoose.Document & { comparePassword: comparePasswordFunction; }; -type comparePasswordFunction = (candidatePassword: string, cb: (err: any, isMatch: any) => void) => void; - export type AuthToken = { accessToken: string; kind: string; @@ -75,16 +72,19 @@ userSchema.pre('save', function save(next) { bcrypt.hash( user.password, salt, - () => void {}, - (err: mongoose.Error, hash: string) => { - if (err) { - return next(err); + () => {}, + (cryptErr: mongoose.Error, hash: string) => { + if (cryptErr) { + return next(cryptErr); } user.password = hash; next(); + return undefined; } ); + return undefined; }); + return undefined; }); const comparePassword: comparePasswordFunction = function (this: DashUserModel, candidatePassword, cb) { @@ -97,7 +97,7 @@ const comparePassword: comparePasswordFunction = function (this: DashUserModel, userSchema.methods.comparePassword = comparePassword; -const User = mongoose.model('User', userSchema); +const User: any = mongoose.model('User', userSchema); export function initializeGuest() { new User({ email: 'guest', diff --git a/src/server/authentication/Passport.ts b/src/server/authentication/Passport.ts index a9cf6698b..a5222e531 100644 --- a/src/server/authentication/Passport.ts +++ b/src/server/authentication/Passport.ts @@ -1,6 +1,6 @@ import * as passport from 'passport'; import * as passportLocal from 'passport-local'; -import { DashUserModel, default as User } from './DashUserModel'; +import User, { DashUserModel } from './DashUserModel'; const LocalStrategy = passportLocal.Strategy; @@ -11,22 +11,25 @@ passport.serializeUser<any, any>((req, user, done) => { passport.deserializeUser<any, any>((id, done) => { User.findById(id) .exec() - .then(user => done(undefined, user)); + .then((user: any) => done(undefined, user)); }); // AUTHENTICATE JUST WITH EMAIL AND PASSWORD passport.use( new LocalStrategy({ usernameField: 'email', passReqToCallback: true }, (req, email, password, done) => { User.findOne({ email: email.toLowerCase() }) - .then(user => { - if (!user) return done(undefined, false, { message: 'Invalid email or password' }); // invalid email - (user as any as DashUserModel).comparePassword(password, (error: Error, isMatch: boolean) => { - if (error) return done(error); - if (!isMatch) return done(undefined, false, { message: 'Invalid email or password' }); // invalid password - // valid authentication HERE - return done(undefined, user); - }); + .then((user: any) => { + if (!user) { + done(undefined, false, { message: 'Invalid email or password' }); // invalid email + } else { + (user as any as DashUserModel).comparePassword(password, (error: Error, isMatch: boolean) => { + if (error) return done(error); + if (!isMatch) return done(undefined, false, { message: 'Invalid email or password' }); // invalid password + // valid authentication HERE + return done(undefined, user); + }); + } }) - .catch(error => done(error)); + .catch((error: any) => done(error)); }) ); diff --git a/src/server/database.ts b/src/server/database.ts index 3a087ce38..ff5635b2c 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -9,6 +9,7 @@ import { Transferable } from './Message'; import { Upload } from './SharedMediaTypes'; export namespace Database { + // eslint-disable-next-line import/no-mutable-exports export let disconnect: Function; class DocSchema implements mongodb.BSON.Document { @@ -30,7 +31,10 @@ export namespace Database { export async function tryInitializeConnection() { try { const { connection } = mongoose; - disconnect = async () => new Promise<any>(resolve => connection.close().then(resolve)); + disconnect = async () => + new Promise<any>(resolve => { + connection.close().then(resolve); + }); if (connection.readyState === ConnectionStates.disconnected) { await new Promise<void>((resolve, reject) => { connection.on('error', reject); @@ -39,7 +43,7 @@ export namespace Database { resolve(); }); mongoose.connect(url, { - //useNewUrlParser: true, + // useNewUrlParser: true, dbName: schema, // reconnectTries: Number.MAX_VALUE, // reconnectInterval: 1000, @@ -54,6 +58,7 @@ export namespace Database { } } + // eslint-disable-next-line @typescript-eslint/no-shadow export class Database implements IDatabase { private MongoClient = mongodb.MongoClient; private currentWrites: { [id: string]: Promise<void> } = {}; @@ -81,8 +86,8 @@ export namespace Database { const collection = this.db.collection<DocSchema>(collectionName); const prom = this.currentWrites[id]; let newProm: Promise<void>; - const run = (): Promise<void> => { - return new Promise<void>(resolve => { + const run = (): Promise<void> => + new Promise<void>(resolve => { collection .updateOne({ _id: id }, value, { upsert }) .then(res => { @@ -96,13 +101,12 @@ export namespace Database { console.log('MOngo UPDATE ONE ERROR:', error); }); }); - }; newProm = prom ? prom.then(run) : run(); this.currentWrites[id] = newProm; return newProm; - } else { - this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName)); } + this.onConnect.push(() => this.update(id, value, callback, upsert, collectionName)); + return undefined; } public replace(id: string, value: any, callback: (err: mongodb.MongoError, res: mongodb.UpdateResult<mongodb.Document>) => void, upsert = true, collectionName = DocumentsCollection) { @@ -110,8 +114,8 @@ export namespace Database { const collection = this.db.collection<DocSchema>(collectionName); const prom = this.currentWrites[id]; let newProm: Promise<void>; - const run = (): Promise<void> => { - return new Promise<void>(resolve => { + const run = (): Promise<void> => + new Promise<void>(resolve => { collection.replaceOne({ _id: id }, value, { upsert }).then(res => { if (this.currentWrites[id] === newProm) { delete this.currentWrites[id]; @@ -120,7 +124,6 @@ export namespace Database { callback(undefined as any, res as any); }); }); - }; newProm = prom ? prom.then(run) : run(); this.currentWrites[id] = newProm; } else { @@ -132,8 +135,10 @@ export namespace Database { const cursor = this.db?.listCollections(); const collectionNames: string[] = []; if (cursor) { + // eslint-disable-next-line no-await-in-loop while (await cursor.hasNext()) { - const collection: any = await cursor.next(); + // eslint-disable-next-line no-await-in-loop + const collection = await cursor.next(); collection && collectionNames.push(collection.name); } } @@ -141,26 +146,30 @@ export namespace Database { } public delete(query: any, collectionName?: string): Promise<mongodb.DeleteResult>; + // eslint-disable-next-line no-dupe-class-members public delete(id: string, collectionName?: string): Promise<mongodb.DeleteResult>; - public delete(id: any, collectionName = DocumentsCollection) { + // eslint-disable-next-line no-dupe-class-members + public delete(idIn: any, collectionName = DocumentsCollection) { + let id = idIn; if (typeof id === 'string') { id = { _id: id }; } if (this.db) { - const db = this.db; - return new Promise(res => - db - .collection(collectionName) + const { db } = this; + return new Promise(res => { + db.collection(collectionName) .deleteMany(id) - .then(result => res(result)) - ); - } else { - return new Promise(res => this.onConnect.push(() => res(this.delete(id, collectionName)))); + .then(result => res(result)); + }); } + return new Promise(res => { + this.onConnect.push(() => res(this.delete(id, collectionName))); + }); } public async dropSchema(...targetSchemas: string[]): Promise<any> { const executor = async (database: mongodb.Db) => { + // eslint-disable-next-line no-use-before-define const existing = await Instance.getCollectionNames(); let valid: string[]; if (targetSchemas.length) { @@ -173,12 +182,13 @@ export namespace Database { }; if (this.db) { return executor(this.db); - } else { - this.onConnect.push(() => this.db && executor(this.db)); } + this.onConnect.push(() => this.db && executor(this.db)); + return undefined; } - public async insert(value: any, collectionName = DocumentsCollection) { + public async insert(valueIn: any, collectionName = DocumentsCollection) { + const value = valueIn; if (this.db && value !== null) { if ('id' in value) { value._id = value.id; @@ -188,36 +198,36 @@ export namespace Database { const collection = this.db.collection<DocSchema>(collectionName); const prom = this.currentWrites[id]; let newProm: Promise<void>; - const run = (): Promise<void> => { - return new Promise<void>(resolve => { + const run = (): Promise<void> => + new Promise<void>(resolve => { collection .insertOne(value) - .then(res => { + .then(() => { if (this.currentWrites[id] === newProm) { delete this.currentWrites[id]; } resolve(); }) - .catch(err => { - console.log('Mongo INSERT ERROR: ', err); - }); + .catch(err => console.log('Mongo INSERT ERROR: ', err)); }); - }; newProm = prom ? prom.then(run) : run(); this.currentWrites[id] = newProm; return newProm; - } else if (value !== null) { + } + if (value !== null) { this.onConnect.push(() => this.insert(value, collectionName)); } + return undefined; } public getDocument(id: string, fn: (result?: Transferable) => void, collectionName = DocumentsCollection) { if (this.db) { const collection = this.db.collection<DocSchema>(collectionName); - collection.findOne({ _id: id }).then(result => { + collection.findOne({ _id: id }).then(resultIn => { + const result = resultIn; if (result) { result.id = result._id; - //delete result._id; + // delete result._id; fn(result as any); } else { fn(undefined); @@ -235,7 +245,8 @@ export namespace Database { .find({ _id: { $in: ids } }) .toArray(); fn( - found.map((doc: any) => { + found.map((docIn: any) => { + const doc = docIn; doc.id = doc._id; delete doc._id; return doc; @@ -253,24 +264,26 @@ export namespace Database { const count = Math.min(ids.length, 1000); const index = ids.length - count; const fetchIds = ids.splice(index, count).filter(id => !visited.has(id)); - if (!fetchIds.length) { - continue; - } - const docs = await new Promise<{ [key: string]: any }[]>(res => this.getDocuments(fetchIds, res, collectionName)); - for (const doc of docs) { - const id = doc.id; - visited.add(id); - ids.push(...(await fn(doc))); + if (fetchIds.length) { + // eslint-disable-next-line no-await-in-loop + const docs = await new Promise<{ [key: string]: any }[]>(res => { + this.getDocuments(fetchIds, res, collectionName); + }); + docs.forEach(async doc => { + const { id } = doc; + visited.add(id); + ids.push(...(await fn(doc))); + }); } } - } else { - return new Promise(res => { - this.onConnect.push(() => { - this.visit(ids, fn, collectionName); - res(); - }); - }); + return undefined; } + return new Promise(res => { + this.onConnect.push(() => { + this.visit(ids, fn, collectionName); + res(); + }); + }); } public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = DocumentsCollection): Promise<mongodb.FindCursor> { @@ -280,36 +293,31 @@ export namespace Database { cursor = cursor.project(projection); } return Promise.resolve<mongodb.FindCursor>(cursor); - } else { - return new Promise<mongodb.FindCursor>(res => { - this.onConnect.push(() => res(this.query(query, projection, collectionName))); - }); } + return new Promise<mongodb.FindCursor>(res => { + this.onConnect.push(() => { + res(this.query(query, projection, collectionName)); + }); + }); } public updateMany(query: any, update: any, collectionName = DocumentsCollection) { if (this.db) { - const db = this.db; - return new Promise<mongodb.UpdateResult>(res => - db - .collection(collectionName) + const { db } = this; + return new Promise<mongodb.UpdateResult>(res => { + db.collection(collectionName) .updateMany(query, update) .then(result => res(result)) - .catch(error => { - console.log('Mongo INSERT MANY ERROR:', error); - }) - ); - } else { - return new Promise<mongodb.UpdateResult>(res => { - this.onConnect.push(() => - this.updateMany(query, update, collectionName) - .then(res) - .catch(error => { - console.log('Mongo UPDATAE MANY ERROR: ', error); - }) - ); + .catch(error => console.log('Mongo INSERT MANY ERROR:', error)); }); } + return new Promise<mongodb.UpdateResult>(res => { + this.onConnect.push(() => + this.updateMany(query, update, collectionName) + .then(res) + .catch(error => console.log('Mongo UPDATAE MANY ERROR: ', error)) + ); + }); } public print() { @@ -375,9 +383,7 @@ export namespace Database { * Checks to see if an image with the given @param contentSize * already exists in the aux database, i.e. has already been downloaded from Google Photos. */ - export const QueryUploadHistory = async (contentSize: number) => { - return SanitizedSingletonQuery<Upload.ImageInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); - }; + export const QueryUploadHistory = async (contentSize: number) => SanitizedSingletonQuery<Upload.ImageInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); /** * Records the uploading of the image with the given @param information, @@ -405,28 +411,25 @@ export namespace Database { * Retrieves the credentials associaed with @param userId * and optionally removes their database id according to @param removeId. */ - export const Fetch = async (userId: string, removeId = true): Promise<Opt<StoredCredentials>> => { - return SanitizedSingletonQuery<StoredCredentials>({ userId }, AuxiliaryCollections.GoogleAccess, removeId); - }; + export const Fetch = async (userId: string, removeId = true): Promise<Opt<StoredCredentials>> => SanitizedSingletonQuery<StoredCredentials>({ userId }, AuxiliaryCollections.GoogleAccess, removeId); /** * Writes the @param enrichedCredentials to the database, associated * with @param userId for later retrieval and updating. */ - export const Write = async (userId: string, enrichedCredentials: GoogleApiServerUtils.EnrichedCredentials) => { - return Instance.insert({ userId, canAccess: [], ...enrichedCredentials }, AuxiliaryCollections.GoogleAccess); - }; + export const Write = async (userId: string, enrichedCredentials: GoogleApiServerUtils.EnrichedCredentials) => Instance.insert({ userId, canAccess: [], ...enrichedCredentials }, AuxiliaryCollections.GoogleAccess); /** - * Updates the @param access_token and @param expiry_date fields + * Updates the @param accessToken and @param expiryDate fields * in the stored credentials associated with @param userId. */ - export const Update = async (userId: string, access_token: string, expiry_date: number) => { + export const Update = async (userId: string, accessToken: string, expiryDate: number) => { const entry = await Fetch(userId, false); if (entry) { - const parameters = { $set: { access_token, expiry_date } }; + const parameters = { $set: { access_token: accessToken, expiry_date: expiryDate } }; return Instance.update(entry._id, parameters, emptyFunction, true, AuxiliaryCollections.GoogleAccess); } + return undefined; }; /** diff --git a/src/server/index.ts b/src/server/index.ts index 47c37c9f0..4374a72b7 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,14 +1,14 @@ -import * as dotenv from 'dotenv'; import { yellow } from 'colors'; +// eslint-disable-next-line import/no-extraneous-dependencies +import * as dotenv from 'dotenv'; import * as mobileDetect from 'mobile-detect'; import * as path from 'path'; -import * as qs from 'query-string'; -import { log_execution } from './ActionUtilities'; +import { logExecution } from './ActionUtilities'; +import { AdminPrivileges, resolvedPorts } from './SocketData'; import DataVizManager from './ApiManagers/DataVizManager'; import DeleteManager from './ApiManagers/DeleteManager'; import DownloadManager from './ApiManagers/DownloadManager'; import GeneralGoogleManager from './ApiManagers/GeneralGoogleManager'; -//import GooglePhotosManager from './ApiManagers/GooglePhotosManager'; import { SearchManager } from './ApiManagers/SearchManager'; import SessionManager from './ApiManagers/SessionManager'; import UploadManager from './ApiManagers/UploadManager'; @@ -24,13 +24,13 @@ import { Database } from './database'; import { Logger } from './ProcessFactory'; import RouteManager, { Method, PublicHandler } from './RouteManager'; import RouteSubscriber from './RouteSubscriber'; -import initializeServer, { resolvedPorts } from './server_Initialization'; +import initializeServer from './server_Initialization'; +// import GooglePhotosManager from './ApiManagers/GooglePhotosManager'; + dotenv.config(); -export const AdminPriviliges: Map<string, boolean> = new Map(); export const onWindows = process.platform === 'win32'; +// eslint-disable-next-line import/no-mutable-exports export let sessionAgent: AppliedSessionAgent; -export const publicDirectory = path.resolve(__dirname, 'public'); -export const filesDirectory = path.resolve(publicDirectory, 'files'); /** * These are the functions run before the server starts @@ -45,7 +45,7 @@ async function preliminaryFunctions() { SSL.loadCredentials(); GoogleApiServerUtils.processProjectCredentials(); if (process.env.DB !== 'MEM') { - await log_execution({ + await logExecution({ startMessage: 'attempting to initialize mongodb connection', endMessage: 'connection outcome determined', action: Database.tryInitializeConnection, @@ -62,8 +62,18 @@ async function preliminaryFunctions() { * that will manage the registration of new routes * with the server */ -function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: RouteManager) { - const managers = [new SessionManager(), new UserManager(), new UploadManager(), new DownloadManager(), new SearchManager(), new DeleteManager(), new UtilManager(), new GeneralGoogleManager(), /* new GooglePhotosManager(),*/ new DataVizManager()]; +function routeSetter({ addSupervisedRoute, logRegistrationOutcome }: RouteManager) { + const managers = [ + new SessionManager(), + new UserManager(), + new UploadManager(), + new DownloadManager(), + new SearchManager(), + new DeleteManager(), + new UtilManager(), + new GeneralGoogleManager(), + /* new GooglePhotosManager(), */ new DataVizManager(), + ]; // initialize API Managers console.log(yellow('\nregistering server routes...')); @@ -104,6 +114,7 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: }); const serve: PublicHandler = ({ req, res }) => { + // eslint-disable-next-line new-cap const detector = new mobileDetect(req.headers['user-agent'] || ''); const filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; res.sendFile(path.join(__dirname, '../../deploy/' + filename)); @@ -118,9 +129,8 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: ({ res, isRelease }) => { const { PASSWORD } = process.env; if (!(isRelease && PASSWORD)) { - return res.redirect('/home'); - } - res.render('admin.pug', { title: 'Enter Administrator Password' }); + res.redirect('/home'); + } else res.render('admin.pug', { title: 'Enter Administrator Password' }); }, }); @@ -130,18 +140,19 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: async ({ req, res, isRelease, user: { id } }) => { const { PASSWORD } = process.env; if (!(isRelease && PASSWORD)) { - return res.redirect('/home'); - } - const { password } = req.body; - const { previous_target } = req.params; - let redirect: string; - if (password === PASSWORD) { - AdminPriviliges.set(id, true); - redirect = `/${previous_target.replace(':', '/')}`; + res.redirect('/home'); } else { - redirect = `/admin/${previous_target}`; + const { password } = req.body; + const { previous_target: previousTarget } = req.params; + let redirect: string; + if (password === PASSWORD) { + AdminPrivileges.set(id, true); + redirect = `/${previousTarget.replace(':', '/')}`; + } else { + redirect = `/admin/${previousTarget}`; + } + res.redirect(redirect); } - res.redirect(redirect); }, }); @@ -151,7 +162,6 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: serve, publicHandler: ({ req, res, ...remaining }) => { const { originalUrl: target } = req; - const sharing = qs.parse(qs.extract(req.originalUrl), { sort: false }).sharing === 'true'; const docAccess = target.startsWith('/doc/'); // since this is the public handler, there's no meaning of '/home' to speak of // since there's no user logged in, so the only viable operation @@ -174,7 +184,7 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: * the main monitor (master) thread. */ export async function launchServer() { - await log_execution({ + await logExecution({ startMessage: '\nstarting execution of preliminary functions', endMessage: 'completed preliminary functions\n', action: preliminaryFunctions, diff --git a/src/server/remapUrl.ts b/src/server/remapUrl.ts index b8e17ec66..ca7ca241f 100644 --- a/src/server/remapUrl.ts +++ b/src/server/remapUrl.ts @@ -1,58 +1,69 @@ -import { Database } from "./database"; -import { resolvedPorts } from "./server_Initialization"; +import { URL } from 'url'; +import { Database } from './database'; +import { resolvedPorts } from './SocketData'; -//npx ts-node src/server/remapUrl.ts +// npx ts-node src/server/remapUrl.ts const suffixMap: { [type: string]: true } = { - "video": true, - "pdf": true, - "audio": true, - "web": true, - "image": true, - "map": true, + video: true, + pdf: true, + audio: true, + web: true, + image: true, + map: true, }; async function update() { - await new Promise(res => setTimeout(res, 10)); - console.log("update"); + await new Promise(res => { + setTimeout(res, 10); + }); + console.log('update'); const cursor = await Database.Instance.query({}); - console.log("Cleared"); + console.log('Cleared'); const updates: [string, any][] = []; function updateDoc(doc: any) { - if (doc.__type !== "Doc") { + if (doc.__type !== 'Doc') { return; } - const fields = doc.fields; + const { fields } = doc; if (!fields) { return; } - const update: any = { - }; + const updated: any = {}; let dynfield = false; - for (const key in fields) { + Array.from(Object.keys(fields)).forEach(key => { const value = fields[key]; if (value && value.__type && suffixMap[value.__type]) { const url = new URL(value.url); - if (url.href.includes("localhost") && url.href.includes("Bill")) { + if (url.href.includes('localhost') && url.href.includes('Bill')) { dynfield = true; - update.$set = { ["fields." + key + ".url"]: `${url.protocol}//dash-web.eastus2.cloudapp.azure.com:${resolvedPorts.server}${url.pathname}` }; + updated.$set = { ['fields.' + key + '.url']: `${url.protocol}//dash-web.eastus2.cloudapp.azure.com:${resolvedPorts.server}${url.pathname}` }; } } - } + }); if (dynfield) { - updates.push([doc._id, update]); + updates.push([doc._id, updated]); } } await cursor.forEach(updateDoc); - await Promise.all(updates.map(doc => { - console.log(doc[0], doc[1]); - return new Promise<void>(res => Database.Instance.update(doc[0], doc[1], () => { - console.log("wrote " + JSON.stringify(doc[1])); - res(); - }, false)); - })); - console.log("Done"); + await Promise.all( + updates.map(doc => { + console.log(doc[0], doc[1]); + return new Promise<void>(res => { + Database.Instance.update( + doc[0], + doc[1], + () => { + console.log('wrote ' + JSON.stringify(doc[1])); + res(); + }, + false + ); + }); + }) + ); + console.log('Done'); // await Promise.all(updates.map(update => { // return limit(() => Search.updateDocument(update)); // })); diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index 2d52ea906..8db7e9933 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -1,73 +1,59 @@ import * as bodyParser from 'body-parser'; +import * as brotli from 'brotli'; import { blue, yellow } from 'colors'; +import * as flash from 'connect-flash'; +import * as MongoStoreConnect from 'connect-mongo'; import * as cors from 'cors'; import * as express from 'express'; +import * as expressFlash from 'express-flash'; import * as session from 'express-session'; import { createServer } from 'https'; import * as passport from 'passport'; import * as request from 'request'; import * as webpack from 'webpack'; import * as wdm from 'webpack-dev-middleware'; +// eslint-disable-next-line import/no-extraneous-dependencies import * as whm from 'webpack-hot-middleware'; import * as zlib from 'zlib'; -import { publicDirectory } from '.'; +import * as config from '../../webpack.config.js'; import { logPort } from './ActionUtilities'; +import RouteManager from './RouteManager'; +import RouteSubscriber from './RouteSubscriber'; +import { publicDirectory, resolvedPorts } from './SocketData'; import { SSL } from './apis/google/CredentialsLoader'; import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/AuthenticationManager'; import { Database } from './database'; -import RouteManager from './RouteManager'; -import RouteSubscriber from './RouteSubscriber'; import { WebSocket } from './websocket'; -import * as expressFlash from 'express-flash'; -import * as flash from 'connect-flash'; -import * as brotli from 'brotli'; -import * as MongoStoreConnect from 'connect-mongo'; -import * as config from '../../webpack.config'; /* RouteSetter is a wrapper around the server that prevents the server from being exposed. */ export type RouteSetter = (server: RouteManager) => void; -//export let disconnect: Function; +// export let disconnect: Function; -export let resolvedPorts: { server: number; socket: number } = { server: 1050, socket: 4321 }; +// eslint-disable-next-line import/no-mutable-exports export let resolvedServerUrl: string; -export default async function InitializeServer(routeSetter: RouteSetter) { - const isRelease = determineEnvironment(); - const app = buildWithMiddleware(express()); - const compiler = webpack(config as any); - - // route table managed by express. routes are tested sequentially against each of these map rules. when a match is found, the handler is called to process the request - app.use(wdm(compiler, { publicPath: config.output.publicPath })); - app.use(whm(compiler)); - app.get(new RegExp(/^\/+$/), (req, res) => res.redirect(req.user ? '/home' : '/login')); // target urls that consist of one or more '/'s with nothing in between - app.use(express.static(publicDirectory, { setHeaders: res => res.setHeader('Access-Control-Allow-Origin', '*') })); //all urls that start with dash's public directory: /files/ (e.g., /files/images, /files/audio, etc) - app.use(cors({ origin: (_origin: any, callback: any) => callback(null, true) })); - registerAuthenticationRoutes(app); // this adds routes to authenticate a user (login, etc) - registerCorsProxy(app); // this adds a /corsProxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies - isRelease && !SSL.Loaded && SSL.exit(); - routeSetter(new RouteManager(app, isRelease)); // this sets up all the regular supervised routes (things like /home, download/upload api's, pdf, search, session, etc) - registerEmbeddedBrowseRelativePathHandler(app); // this allows renered web pages which internally have relative paths to find their content +const week = 7 * 24 * 60 * 60 * 1000; +const secret = '64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc'; +const store = process.env.DB === 'MEM' ? new session.MemoryStore() : MongoStoreConnect.create({ mongoUrl: Database.url }); - isRelease && process.env.serverPort && (resolvedPorts.server = Number(process.env.serverPort)); - const server = isRelease ? createServer(SSL.Credentials, app) : app; - await new Promise<void>(resolve => server.listen(resolvedPorts.server, resolve)); - logPort('server', resolvedPorts.server); +/* Determine if the enviroment is dev mode or release mode. */ +function determineEnvironment() { + const isRelease = process.env.RELEASE === 'true'; - resolvedServerUrl = `${isRelease && process.env.serverName ? `https://${process.env.serverName}.com` : 'http://localhost'}:${resolvedPorts.server}`; + const color = isRelease ? blue : yellow; + const label = isRelease ? 'release' : 'development'; + console.log(`\nrunning server in ${color(label)} mode`); - // initialize the web socket (bidirectional communication: if a user changes - // a field on one client, that change must be broadcast to all other clients) - await WebSocket.initialize(isRelease, SSL.Credentials); + // // swilkins: I don't think we need to read from ClientUtils.RELEASE anymore. Should be able to invoke process.env.RELEASE + // // on the client side, thanks to dotenv in webpack.config.js + // let clientUtils = fs.readFileSync('./src/client/util/ClientUtils.ts.temp', 'utf8'); + // clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; + // fs.writeFileSync('./src/client/util/ClientUtils.ts', clientUtils, 'utf8'); - //disconnect = async () => new Promise<Error>(resolve => server.close(resolve)); return isRelease; } -const week = 7 * 24 * 60 * 60 * 1000; -const secret = '64d6866242d3b5a5503c675b32c9605e4e90478e9b77bcf2bc'; -const store = process.env.DB === 'MEM' ? new session.MemoryStore() : MongoStoreConnect.create({ mongoUrl: Database.url }); - function buildWithMiddleware(server: express.Express) { [ session({ @@ -100,72 +86,43 @@ function buildWithMiddleware(server: express.Express) { return server; } -/* Determine if the enviroment is dev mode or release mode. */ -function determineEnvironment() { - const isRelease = process.env.RELEASE === 'true'; - - const color = isRelease ? blue : yellow; - const label = isRelease ? 'release' : 'development'; - console.log(`\nrunning server in ${color(label)} mode`); - - // // swilkins: I don't think we need to read from ClientUtils.RELEASE anymore. Should be able to invoke process.env.RELEASE - // // on the client side, thanks to dotenv in webpack.config.js - // let clientUtils = fs.readFileSync('./src/client/util/ClientUtils.ts.temp', 'utf8'); - // clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; - // fs.writeFileSync('./src/client/util/ClientUtils.ts', clientUtils, 'utf8'); - - return isRelease; -} - -function registerAuthenticationRoutes(server: express.Express) { - server.get('/signup', getSignup); - server.post('/signup', postSignup); - - server.get('/login', getLogin); - server.post('/login', postLogin); - - server.get('/logout', getLogout); - - server.get('/forgotPassword', getForgot); - server.post('/forgotPassword', postForgot); - - const reset = new RouteSubscriber('resetPassword').add('token').build; - server.get(reset, getReset); - server.post(reset, postReset); -} - -function registerCorsProxy(server: express.Express) { - server.use('/corsProxy', async (req, res) => { - res.setHeader('Access-Control-Allow-Origin', '*'); - res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE'); - res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers')); - const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : ''; - let requrlraw = decodeURIComponent(req.url.substring(1)); - const qsplit = requrlraw.split('?q='); - const newqsplit = requrlraw.split('&q='); - if (qsplit.length > 1 && newqsplit.length > 1) { - const lastq = newqsplit[newqsplit.length - 1]; - requrlraw = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1]; - } - const requrl = requrlraw.startsWith('/') ? referer + requrlraw : requrlraw; - // cors weirdness here... - // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative, - // then we redirect again to the cors referer and just add the relative path. - if (!requrl.startsWith('http') && req.originalUrl.startsWith('/corsProxy') && referer?.includes('corsProxy')) { - res.redirect(referer + (referer.endsWith('/') ? '' : '/') + requrl); +function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { + server.use('*', (req, res) => { + // res.setHeader('Access-Control-Allow-Origin', '*'); + // res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE'); + // res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers')); + const relativeUrl = req.originalUrl; + if (!res.headersSent && req.headers.referer?.includes('corsProxy')) { + if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home + // a request for something by a proxied referrer means it must be a relative reference. So construct a proxied absolute reference here. + try { + const proxiedRefererUrl = decodeURIComponent(req.headers.referer); // (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/wiki/Engelbart) + const dashServerUrl = proxiedRefererUrl.match(/.*corsProxy\//)![0]; // the dash server url (e.g.: http://localhost:<port>/corsProxy/ ) + const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ''); // the url of the referer without the proxy (e.g., : https://en.wikipedia.org/wiki/Engelbart) + const absoluteTargetBaseUrl = actualReferUrl.match(/https?:\/\/[^/]*/)![0]; // the base of the original url (e.g., https://en.wikipedia.org) + const redirectedProxiedUrl = dashServerUrl + encodeURIComponent(absoluteTargetBaseUrl + relativeUrl); // the new proxied full url (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/<somethingelse>) + const redirectUrl = relativeUrl.startsWith('//') ? 'http:' + relativeUrl : redirectedProxiedUrl; + res.redirect(redirectUrl); + } catch (e) { + console.log('Error embed: ', e); + } + } else if (relativeUrl.startsWith('/search') && !req.headers.referer?.includes('corsProxy')) { + // detect search query and use default search engine + res.redirect(req.headers.referer + 'corsProxy/' + encodeURIComponent('http://www.google.com' + relativeUrl)); } else { - proxyServe(req, requrl, res); + res.end(); } }); } function proxyServe(req: any, requrl: string, response: any) { + // eslint-disable-next-line global-require const htmlBodyMemoryStream = new (require('memorystream'))(); - var wasinBrFormat = false; + let wasinBrFormat = false; const sendModifiedBody = () => { const header = response.headers['content-encoding']; - const refToCors = (match: any, tag: string, sym: string, href: string, offset: any, string: any) => `${tag}=${sym + resolvedServerUrl}/corsProxy/${href + sym}`; - const relpathToCors = (match: any, href: string, offset: any, string: any) => `="${resolvedServerUrl + '/corsProxy/' + decodeURIComponent(req.originalUrl.split('/corsProxy/')[1].match(/https?:\/\/[^\/]*/)?.[0] ?? '') + '/' + href}"`; + const refToCors = (match: any, tag: string, sym: string, href: string) => `${tag}=${sym + resolvedServerUrl}/corsProxy/${href + sym}`; + // const relpathToCors = (match: any, href: string, offset: any, string: any) => `="${resolvedServerUrl + '/corsProxy/' + decodeURIComponent(req.originalUrl.split('/corsProxy/')[1].match(/https?:\/\/[^\/]*/)?.[0] ?? '') + '/' + href}"`; if (header) { try { const bodyStream = htmlBodyMemoryStream.read(); @@ -174,8 +131,8 @@ function proxyServe(req: any, requrl: string, response: any) { const htmlText = htmlInputText .toString('utf8') .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>') - .replace(/(src|href)=([\'\"])(https?[^\2\n]*)\1/g, refToCors) // replace src or href='http(s)://...' or href="http(s)://.." - //.replace(/= *"\/([^"]*)"/g, relpathToCors) + .replace(/(src|href)=(['"])(https?[^\2\n]*)\1/g, refToCors) // replace src or href='http(s)://...' or href="http(s)://.." + // .replace(/= *"\/([^"]*)"/g, relpathToCors) .replace(/data-srcset="[^"]*"/g, '') .replace(/srcset="[^"]*"/g, '') .replace(/target="_blank"/g, ''); @@ -198,7 +155,7 @@ function proxyServe(req: any, requrl: string, response: any) { } }; const retrieveHTTPBody = () => { - //req.headers.cookie = ''; + // req.headers.cookie = ''; req.pipe(request(requrl)) .on('error', (e: any) => { console.log(`CORS url error: ${requrl}`, e); @@ -227,6 +184,7 @@ function proxyServe(req: any, requrl: string, response: any) { res.headers['x-permitted-cross-domain-policies'] = 'all'; res.headers['x-frame-options'] = ''; res.headers['content-security-policy'] = ''; + // eslint-disable-next-line no-multi-assign response.headers = response._headers = res.headers; }) .on('end', sendModifiedBody) @@ -236,31 +194,78 @@ function proxyServe(req: any, requrl: string, response: any) { retrieveHTTPBody(); } -function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { - server.use('*', (req, res) => { - // res.setHeader('Access-Control-Allow-Origin', '*'); - // res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE'); - // res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers')); - const relativeUrl = req.originalUrl; - if (!res.headersSent && req.headers.referer?.includes('corsProxy')) { - if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home - // a request for something by a proxied referrer means it must be a relative reference. So construct a proxied absolute reference here. - try { - const proxiedRefererUrl = decodeURIComponent(req.headers.referer); // (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/wiki/Engelbart) - const dashServerUrl = proxiedRefererUrl.match(/.*corsProxy\//)![0]; // the dash server url (e.g.: http://localhost:<port>/corsProxy/ ) - const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ''); // the url of the referer without the proxy (e.g., : https://en.wikipedia.org/wiki/Engelbart) - const absoluteTargetBaseUrl = actualReferUrl.match(/https?:\/\/[^\/]*/)![0]; // the base of the original url (e.g., https://en.wikipedia.org) - const redirectedProxiedUrl = dashServerUrl + encodeURIComponent(absoluteTargetBaseUrl + relativeUrl); // the new proxied full url (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/<somethingelse>) - const redirectUrl = relativeUrl.startsWith('//') ? 'http:' + relativeUrl : redirectedProxiedUrl; - res.redirect(redirectUrl); - } catch (e) { - console.log('Error embed: ', e); - } - } else if (relativeUrl.startsWith('/search') && !req.headers.referer?.includes('corsProxy')) { - // detect search query and use default search engine - res.redirect(req.headers.referer + 'corsProxy/' + encodeURIComponent('http://www.google.com' + relativeUrl)); +function registerCorsProxy(server: express.Express) { + server.use('/corsProxy', async (req, res) => { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE'); + res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers')); + const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : ''; + let requrlraw = decodeURIComponent(req.url.substring(1)); + const qsplit = requrlraw.split('?q='); + const newqsplit = requrlraw.split('&q='); + if (qsplit.length > 1 && newqsplit.length > 1) { + const lastq = newqsplit[newqsplit.length - 1]; + requrlraw = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1]; + } + const requrl = requrlraw.startsWith('/') ? referer + requrlraw : requrlraw; + // cors weirdness here... + // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative, + // then we redirect again to the cors referer and just add the relative path. + if (!requrl.startsWith('http') && req.originalUrl.startsWith('/corsProxy') && referer?.includes('corsProxy')) { + res.redirect(referer + (referer.endsWith('/') ? '' : '/') + requrl); } else { - res.end(); + proxyServe(req, requrl, res); } }); } + +function registerAuthenticationRoutes(server: express.Express) { + server.get('/signup', getSignup); + server.post('/signup', postSignup); + + server.get('/login', getLogin); + server.post('/login', postLogin); + + server.get('/logout', getLogout); + + server.get('/forgotPassword', getForgot); + server.post('/forgotPassword', postForgot); + + const reset = new RouteSubscriber('resetPassword').add('token').build; + server.get(reset, getReset); + server.post(reset, postReset); +} + +export default async function InitializeServer(routeSetter: RouteSetter) { + const isRelease = determineEnvironment(); + const app = buildWithMiddleware(express()); + const compiler = webpack(config as any); + + // route table managed by express. routes are tested sequentially against each of these map rules. when a match is found, the handler is called to process the request + app.use(wdm(compiler, { publicPath: config.output.publicPath })); + app.use(whm(compiler)); + app.get(/^\/+$/, (req, res) => res.redirect(req.user ? '/home' : '/login')); // target urls that consist of one or more '/'s with nothing in between + app.use(express.static(publicDirectory, { setHeaders: res => res.setHeader('Access-Control-Allow-Origin', '*') })); // all urls that start with dash's public directory: /files/ (e.g., /files/images, /files/audio, etc) + app.use(cors({ origin: (_origin: any, callback: any) => callback(null, true) })); + registerAuthenticationRoutes(app); // this adds routes to authenticate a user (login, etc) + registerCorsProxy(app); // this adds a /corsProxy/ route to allow clients to get to urls that would otherwise be blocked by cors policies + isRelease && !SSL.Loaded && SSL.exit(); + routeSetter(new RouteManager(app, isRelease)); // this sets up all the regular supervised routes (things like /home, download/upload api's, pdf, search, session, etc) + registerEmbeddedBrowseRelativePathHandler(app); // this allows renered web pages which internally have relative paths to find their content + + isRelease && process.env.serverPort && (resolvedPorts.server = Number(process.env.serverPort)); + const server = isRelease ? createServer(SSL.Credentials, app) : app; + await new Promise<void>(resolve => { + server.listen(resolvedPorts.server, resolve); + }); + logPort('server', resolvedPorts.server); + + resolvedServerUrl = `${isRelease && process.env.serverName ? `https://${process.env.serverName}.com` : 'http://localhost'}:${resolvedPorts.server}`; + + // initialize the web socket (bidirectional communication: if a user changes + // a field on one client, that change must be broadcast to all other clients) + await WebSocket.initialize(isRelease, SSL.Credentials); + + // disconnect = async () => new Promise<Error>(resolve => server.close(resolve)); + return isRelease; +} diff --git a/src/server/updateProtos.ts b/src/server/updateProtos.ts index 2f3772a77..72a44ebf4 100644 --- a/src/server/updateProtos.ts +++ b/src/server/updateProtos.ts @@ -6,15 +6,15 @@ const protos = ['text', 'image', 'web', 'collection', 'kvp', 'video', 'audio', ' await Promise.all( protos.map( protoId => - new Promise(res => + new Promise(res => { Database.Instance.update( protoId, { $set: { 'fields.isBaseProto': true }, }, res - ) - ) + ); + }) ) ); diff --git a/src/server/websocket.ts b/src/server/websocket.ts index 38134f2c1..cece8a1b7 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -3,152 +3,26 @@ import { createServer } from 'https'; import * as _ from 'lodash'; import { networkInterfaces } from 'os'; import { Server, Socket } from 'socket.io'; -import { Utils } from '../Utils'; +import { ServerUtils } from '../ServerUtils'; import { logPort } from './ActionUtilities'; -import { timeMap } from './ApiManagers/UserManager'; import { Client } from './Client'; import { DashStats } from './DashStats'; import { DocumentsCollection } from './IDatabase'; import { Diff, GestureContent, MessageStore, MobileDocumentUploadContent, MobileInkOverlayContent, Transferable, Types, UpdateMobileInkOverlayPositionContent, YoutubeQueryInput, YoutubeQueryTypes } from './Message'; import { Search } from './Search'; +import { resolvedPorts, socketMap, timeMap, userOperations } from './SocketData'; import { GoogleCredentialsLoader } from './apis/google/CredentialsLoader'; import YoutubeApi from './apis/youtube/youtubeApiSample'; import { initializeGuest } from './authentication/DashUserModel'; import { Database } from './database'; -import { resolvedPorts } from './server_Initialization'; export namespace WebSocket { + let CurUser: string | undefined; + // eslint-disable-next-line import/no-mutable-exports export let _socket: Socket; + // eslint-disable-next-line import/no-mutable-exports + export let _disconnect: Function; export const clients: { [key: string]: Client } = {}; - export const socketMap = new Map<Socket, string>(); - export const userOperations = new Map<string, number>(); - export let disconnect: Function; - - export async function initialize(isRelease: boolean, credentials:any) { - let io: Server; - if (isRelease) { - const { socketPort } = process.env; - if (socketPort) { - resolvedPorts.socket = Number(socketPort); - } - const httpsServer = createServer(credentials); - io = new Server(httpsServer, {}) - httpsServer.listen(resolvedPorts.socket); - } else { - io = new Server(); - io.listen(resolvedPorts.socket); - } - logPort('websocket', resolvedPorts.socket); - - io.on('connection', socket => { - _socket = socket; - socket.use((_packet, next) => { - const userEmail = socketMap.get(socket); - if (userEmail) { - timeMap[userEmail] = Date.now(); - } - next(); - }); - - socket.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()); - - // convenience function to log server messages on the client - function log(message?: any, ...optionalParams: any[]) { - socket.emit('log', ['Message from server:', message, ...optionalParams]); - } - - socket.on('message', function (message, room) { - console.log('Client said: ', message); - socket.in(room).emit('message', message); - }); - - socket.on('create or join', function (room) { - console.log('Received request to create or join room ' + room); - - const clientsInRoom = socket.rooms.has(room); - const numClients = clientsInRoom ? Object.keys(room.sockets).length : 0; - console.log('Room ' + room + ' now has ' + numClients + ' client(s)'); - - if (numClients === 0) { - socket.join(room); - console.log('Client ID ' + socket.id + ' created room ' + room); - socket.emit('created', room, socket.id); - } else if (numClients === 1) { - console.log('Client ID ' + socket.id + ' joined room ' + room); - socket.in(room).emit('join', room); - socket.join(room); - socket.emit('joined', room, socket.id); - socket.in(room).emit('ready'); - } else { - // max two clients - socket.emit('full', room); - } - }); - - socket.on('ipaddr', function () { - const ifaces = networkInterfaces(); - for (const dev in ifaces) { - ifaces[dev]?.forEach(function (details) { - if (details.family === 'IPv4' && details.address !== '127.0.0.1') { - socket.emit('ipaddr', details.address); - } - }); - } - }); - - socket.on('bye', function () { - console.log('received bye'); - }); - - socket.on('disconnect', function () { - let currentUser = socketMap.get(socket); - if (!(currentUser === undefined)) { - let currentUsername = currentUser.split(' ')[0]; - DashStats.logUserLogout(currentUsername, socket); - delete timeMap[currentUsername]; - } - }); - - Utils.Emit(socket, MessageStore.Foo, 'handshooken'); - - Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); - Utils.AddServerHandler(socket, MessageStore.SetField, args => setField(socket, args)); - Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); - Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); - if (isRelease) { - Utils.AddServerHandler(socket, MessageStore.DeleteAll, () => doDelete(false)); - } - - Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); - Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); - Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); - Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); - Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); - Utils.AddServerHandler(socket, MessageStore.GesturePoints, content => processGesturePoints(socket, content)); - Utils.AddServerHandler(socket, MessageStore.MobileInkOverlayTrigger, content => processOverlayTrigger(socket, content)); - Utils.AddServerHandler(socket, MessageStore.UpdateMobileInkOverlayPosition, content => processUpdateOverlayPosition(socket, content)); - Utils.AddServerHandler(socket, MessageStore.MobileDocumentUpload, content => processMobileDocumentUpload(socket, content)); - Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); - Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); - - /** - * Whenever we receive the go-ahead, invoke the import script and pass in - * as an emitter and a terminator the functions that simply broadcast a result - * or indicate termination to the client via the web socket - */ - - disconnect = () => { - socket.broadcast.emit('connection_terminated', Date.now()); - socket.disconnect(true); - }; - }); - - setInterval(function () { - // Utils.Emit(socket, MessageStore.UpdateStats, DashStats.getUpdatedStatsBundle()); - - io.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()); - }, DashStats.SAMPLING_INTERVAL); - } function processGesturePoints(socket: Socket, content: GestureContent) { socket.broadcast.emit('receiveGesturePoints', content); @@ -174,8 +48,11 @@ export namespace WebSocket { break; case YoutubeQueryTypes.SearchVideo: YoutubeApi.authorizedGetVideos(ProjectCredentials, query.userInput, callback); + break; case YoutubeQueryTypes.VideoDetails: YoutubeApi.authorizedGetVideoDetails(ProjectCredentials, query.videoIds, callback); + break; + default: } } @@ -189,6 +66,9 @@ export namespace WebSocket { initializeGuest(); } + function printActiveUsers() { + socketMap.forEach((user, socket) => !socket.disconnected && console.log(user)); + } function barReceived(socket: Socket, userEmail: string) { clients[userEmail] = new Client(userEmail.toString()); const currentdate = new Date(); @@ -199,11 +79,11 @@ export namespace WebSocket { timeMap[userEmail] = Date.now(); socketMap.set(socket, userEmail + ' at ' + datetime); userOperations.set(userEmail, 0); - DashStats.logUserLogin(userEmail, socket); + DashStats.logUserLogin(userEmail); } function getField([id, callback]: [string, (result?: Transferable) => void]) { - Database.Instance.getDocument(id, (result?: Transferable) => callback(result ? result : undefined)); + Database.Instance.getDocument(id, (result?: Transferable) => callback(result)); } function getFields([ids, callback]: [string[], (result: Transferable[]) => void]) { @@ -248,27 +128,24 @@ export namespace WebSocket { list: [ '_l', list => { - const results = []; - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); - } - } + const results: any[] = []; + // eslint-disable-next-line no-use-before-define + list.fields.forEach((value: any) => ToSearchTerm(value) && results.push(ToSearchTerm(value)!.value)); return results.length ? results : null; }, ], }; - function ToSearchTerm(val: any): { suffix: string; value: any } | undefined { + function ToSearchTerm(valIn: any): { suffix: string; value: any } | undefined { + let val = valIn; if (val === null || val === undefined) { - return; + return undefined; } const type = val.__type || typeof val; let suffix = suffixMap[type]; if (!suffix) { - return; + return undefined; } if (Array.isArray(suffix)) { const accessor = suffix[1]; @@ -277,7 +154,7 @@ export namespace WebSocket { } else { val = val[accessor]; } - suffix = suffix[0]; + [suffix] = suffix; } return { suffix, value: val }; } @@ -285,8 +162,28 @@ export namespace WebSocket { function getSuffix(value: string | [string, any]): string { return typeof value === 'string' ? value : value[0]; } + const pendingOps = new Map<string, { diff: Diff; socket: Socket }[]>(); + + function dispatchNextOp(id: string) { + const next = pendingOps.get(id)!.shift(); + if (next) { + const { diff, socket } = next; + if (diff.diff.$addToSet) { + // eslint-disable-next-line no-use-before-define + return GetRefFieldLocal([diff.id, (result?: Transferable) => addToListField(socket, diff, result)]); // would prefer to have Mongo handle list additions direclty, but for now handle it on our own + } + if (diff.diff.$remFromSet) { + // eslint-disable-next-line no-use-before-define + return GetRefFieldLocal([diff.id, (result?: Transferable) => remFromListField(socket, diff, result)]); // would prefer to have Mongo handle list additions direclty, but for now handle it on our own + } + // eslint-disable-next-line no-use-before-define + return SetField(socket, diff); + } + return !pendingOps.get(id)!.length && pendingOps.delete(id); + } - function addToListField(socket: Socket, diff: Diff, curListItems?: Transferable): void { + function addToListField(socket: Socket, diffIn: Diff, curListItems?: Transferable): void { + const diff = diffIn; diff.diff.$set = diff.diff.$addToSet; delete diff.diff.$addToSet; // convert add to set to a query of the current fields, and then a set of the composition of the new fields with the old ones const updatefield = Array.from(Object.keys(diff.diff.$set))[0]; @@ -296,7 +193,7 @@ export namespace WebSocket { return; } const curList = (curListItems as any)?.fields?.[updatefield.replace('fields.', '')]?.fields.filter((item: any) => item !== undefined) || []; - diff.diff.$set[updatefield].fields = [...curList, ...newListItems]; //, ...newListItems.filter((newItem: any) => newItem === null || !curList.some((curItem: any) => curItem.fieldId ? curItem.fieldId === newItem.fieldId : curItem.heading ? curItem.heading === newItem.heading : curItem === newItem))]; + diff.diff.$set[updatefield].fields = [...curList, ...newListItems]; // , ...newListItems.filter((newItem: any) => newItem === null || !curList.some((curItem: any) => curItem.fieldId ? curItem.fieldId === newItem.fieldId : curItem.heading ? curItem.heading === newItem.heading : curItem === newItem))]; const sendBack = diff.diff.length !== diff.diff.$set[updatefield].fields.length; delete diff.diff.length; Database.Instance.update( @@ -305,11 +202,13 @@ export namespace WebSocket { () => { if (sendBack) { console.log('Warning: list modified during update. Composite list is being returned.'); - const id = socket.id; - (socket as any).id = ''; + const { id } = socket; + (socket as any).id = ''; // bcz: HACK. this prevents the update message from going back to the client that made the change. socket.broadcast.emit(MessageStore.UpdateField.Message, diff); (socket as any).id = id; - } else socket.broadcast.emit(MessageStore.UpdateField.Message, diff); + } else { + socket.broadcast.emit(MessageStore.UpdateField.Message, diff); + } dispatchNextOp(diff.id); }, false @@ -352,28 +251,28 @@ export namespace WebSocket { * items to delete) * @param curListItems the server's current copy of the data */ - function remFromListField(socket: Socket, diff: Diff, curListItems?: Transferable): void { + function remFromListField(socket: Socket, diffIn: Diff, curListItems?: Transferable): void { + const diff = diffIn; diff.diff.$set = diff.diff.$remFromSet; delete diff.diff.$remFromSet; const updatefield = Array.from(Object.keys(diff.diff.$set))[0]; const remListItems = diff.diff.$set[updatefield].fields; const curList = (curListItems as any)?.fields?.[updatefield.replace('fields.', '')]?.fields.filter((f: any) => f !== null) || []; - const hint = diff.diff.$set.hint; + const { hint } = diff.diff.$set; if (hint) { // indexesToRemove stores the indexes that we mark for deletion, which is later used to filter the list (delete the elements) - let indexesToRemove: number[] = []; + const indexesToRemove: number[] = []; for (let i = 0; i < hint.deleteCount; i++) { if (curList.length > i + hint.start && _.isEqual(curList[i + hint.start], remListItems[i])) { indexesToRemove.push(i + hint.start); - continue; - } - - let closestIndex = findClosestIndex(curList, indexesToRemove, remListItems[i], i + hint.start); - if (closestIndex !== -1) { - indexesToRemove.push(closestIndex); } else { - console.log('Item to delete was not found - index = -1'); + const closestIndex = findClosestIndex(curList, indexesToRemove, remListItems[i], i + hint.start); + if (closestIndex !== -1) { + indexesToRemove.push(closestIndex); + } else { + console.log('Item to delete was not found - index = -1'); + } } } @@ -398,45 +297,23 @@ export namespace WebSocket { if (sendBack) { // the two copies are different, so the server sends its copy. console.log('SEND BACK'); - const id = socket.id; - (socket as any).id = ''; + const { id } = socket; + (socket as any).id = ''; // bcz: HACK. this prevents the update message from going back to the client that made the change. socket.broadcast.emit(MessageStore.UpdateField.Message, diff); (socket as any).id = id; - } else socket.broadcast.emit(MessageStore.UpdateField.Message, diff); + } else { + socket.broadcast.emit(MessageStore.UpdateField.Message, diff); + } dispatchNextOp(diff.id); }, false ); } - const pendingOps = new Map<string, { diff: Diff; socket: Socket }[]>(); - - function dispatchNextOp(id: string) { - const next = pendingOps.get(id)!.shift(); - if (next) { - const { diff, socket } = next; - if (diff.diff.$addToSet) { - return GetRefFieldLocal([diff.id, (result?: Transferable) => addToListField(socket, diff, result)]); // would prefer to have Mongo handle list additions direclty, but for now handle it on our own - } - if (diff.diff.$remFromSet) { - return GetRefFieldLocal([diff.id, (result?: Transferable) => remFromListField(socket, diff, result)]); // would prefer to have Mongo handle list additions direclty, but for now handle it on our own - } - return GetRefFieldLocal([diff.id, (result?: Transferable) => SetField(socket, diff, result)]); - } - if (!pendingOps.get(id)!.length) pendingOps.delete(id); - } - - function printActiveUsers() { - socketMap.forEach((user, socket) => { - !socket.disconnected && console.log(user); - }); - } - var CurUser: string | undefined = undefined; - function UpdateField(socket: Socket, diff: Diff) { const curUser = socketMap.get(socket); - if (!curUser) return; - let currentUsername = curUser.split(' ')[0]; + if (!curUser) return false; + const currentUsername = curUser.split(' ')[0]; userOperations.set(currentUsername, userOperations.get(currentUsername) !== undefined ? userOperations.get(currentUsername)! + 1 : 0); if (CurUser !== socketMap.get(socket)) { @@ -454,15 +331,18 @@ export namespace WebSocket { if (diff.diff.$remFromSet) { return GetRefFieldLocal([diff.id, (result?: Transferable) => remFromListField(socket, diff, result)]); // would prefer to have Mongo handle list additions direclty, but for now handle it on our own } - return GetRefFieldLocal([diff.id, (result?: Transferable) => SetField(socket, diff, result)]); + // eslint-disable-next-line no-use-before-define + return SetField(socket, diff); } - function SetField(socket: Socket, diff: Diff, curListItems?: Transferable) { + function SetField(socket: Socket, diff: Diff /* , curListItems?: Transferable */) { Database.Instance.update(diff.id, diff.diff, () => socket.broadcast.emit(MessageStore.UpdateField.Message, diff), false); const docfield = diff.diff.$set || diff.diff.$unset; if (docfield) { const update: any = { id: diff.id }; let dynfield = false; + // eslint-disable-next-line no-restricted-syntax for (let key in docfield) { + // eslint-disable-next-line no-continue if (!key.startsWith('fields.')) continue; dynfield = true; const val = docfield[key]; @@ -504,4 +384,98 @@ export namespace WebSocket { function CreateField(newValue: any) { Database.Instance.insert(newValue); } + export async function initialize(isRelease: boolean, credentials: any) { + let io: Server; + if (isRelease) { + const { socketPort } = process.env; + if (socketPort) { + resolvedPorts.socket = Number(socketPort); + } + const httpsServer = createServer(credentials); + io = new Server(httpsServer, {}); + httpsServer.listen(resolvedPorts.socket); + } else { + io = new Server(); + io.listen(resolvedPorts.socket); + } + logPort('websocket', resolvedPorts.socket); + + io.on('connection', socket => { + _socket = socket; + socket.use((_packet, next) => { + const userEmail = socketMap.get(socket); + if (userEmail) { + timeMap[userEmail] = Date.now(); + } + next(); + }); + + socket.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()); + + socket.on('message', (message, room) => { + console.log('Client said: ', message); + socket.in(room).emit('message', message); + }); + + socket.on('ipaddr', () => { + networkInterfaces().keys?.forEach(dev => { + if (dev.family === 'IPv4' && dev.address !== '127.0.0.1') { + socket.emit('ipaddr', dev.address); + } + }); + }); + + socket.on('bye', () => { + console.log('received bye'); + }); + + socket.on('disconnect', () => { + const currentUser = socketMap.get(socket); + if (!(currentUser === undefined)) { + const currentUsername = currentUser.split(' ')[0]; + DashStats.logUserLogout(currentUsername); + delete timeMap[currentUsername]; + } + }); + + ServerUtils.Emit(socket, MessageStore.Foo, 'handshooken'); + + ServerUtils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); + ServerUtils.AddServerHandler(socket, MessageStore.SetField, args => setField(socket, args)); + ServerUtils.AddServerHandlerCallback(socket, MessageStore.GetField, getField); + ServerUtils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields); + if (isRelease) { + ServerUtils.AddServerHandler(socket, MessageStore.DeleteAll, () => doDelete(false)); + } + + ServerUtils.AddServerHandler(socket, MessageStore.CreateField, CreateField); + ServerUtils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); + ServerUtils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); + ServerUtils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); + ServerUtils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); + ServerUtils.AddServerHandler(socket, MessageStore.GesturePoints, content => processGesturePoints(socket, content)); + ServerUtils.AddServerHandler(socket, MessageStore.MobileInkOverlayTrigger, content => processOverlayTrigger(socket, content)); + ServerUtils.AddServerHandler(socket, MessageStore.UpdateMobileInkOverlayPosition, content => processUpdateOverlayPosition(socket, content)); + ServerUtils.AddServerHandler(socket, MessageStore.MobileDocumentUpload, content => processMobileDocumentUpload(socket, content)); + ServerUtils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); + ServerUtils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); + + /** + * Whenever we receive the go-ahead, invoke the import script and pass in + * as an emitter and a terminator the functions that simply broadcast a result + * or indicate termination to the client via the web socket + */ + + _disconnect = () => { + socket.broadcast.emit('connection_terminated', Date.now()); + socket.disconnect(true); + }; + }); + + setInterval(() => { + // Utils.Emit(socket, MessageStore.UpdateStats, DashStats.getUpdatedStatsBundle()); + + io.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()); + }, DashStats.SAMPLING_INTERVAL); + } } |