diff options
Diffstat (limited to 'src/server')
27 files changed, 1647 insertions, 377 deletions
diff --git a/src/server/ApiManagers/DeleteManager.ts b/src/server/ApiManagers/DeleteManager.ts index be452c0ff..9e70af2eb 100644 --- a/src/server/ApiManagers/DeleteManager.ts +++ b/src/server/ApiManagers/DeleteManager.ts @@ -2,6 +2,11 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method, _permission_denied, PublicHandler } from "../RouteManager"; import { WebSocket } from "../Websocket/Websocket"; import { Database } from "../database"; +import rimraf = require("rimraf"); +import { pathToDirectory, Directory } from "./UploadManager"; +import { filesDirectory } from ".."; +import { DashUploadUtils } from "../DashUploadUtils"; +import { mkdirSync } from "fs"; export default class DeleteManager extends ApiManager { @@ -31,21 +36,19 @@ export default class DeleteManager extends ApiManager { } }); - const hi: PublicHandler = async ({ res, isRelease }) => { - if (isRelease) { - return _permission_denied(res, deletionPermissionError); + register({ + method: Method.GET, + subscription: "/deleteAssets", + secureHandler: async ({ res, isRelease }) => { + if (isRelease) { + return _permission_denied(res, deletionPermissionError); + } + rimraf.sync(filesDirectory); + mkdirSync(filesDirectory); + await DashUploadUtils.buildFileDirectories(); + res.redirect("/delete"); } - await Database.Instance.deleteAll('users'); - res.redirect("/home"); - }; - - // register({ - // method: Method.GET, - // subscription: "/deleteUsers", - // onValidation: hi, - // onUnauthenticated: hi - // }); - + }); register({ method: Method.GET, diff --git a/src/server/ApiManagers/DownloadManager.ts b/src/server/ApiManagers/DownloadManager.ts index 1bb84f374..01d2dfcad 100644 --- a/src/server/ApiManagers/DownloadManager.ts +++ b/src/server/ApiManagers/DownloadManager.ts @@ -254,11 +254,13 @@ async function writeHierarchyRecursive(file: Archiver.Archiver, hierarchy: Hiera // and dropped in the browser and thus hosted remotely) so we upload it // to our server and point the zip file to it, so it can bundle up the bytes const information = await DashUploadUtils.UploadImage(result); - path = information.serverAccessPaths[SizeSuffix.Original]; + path = information instanceof Error ? "" : information.accessPaths[SizeSuffix.Original].server; } // write the file specified by the path to the directory in the // zip file given by the prefix. - file.file(path, { name: documentTitle, prefix }); + if (path) { + file.file(path, { name: documentTitle, prefix }); + } } else { // we've hit a collection, so we have to recurse await writeHierarchyRecursive(file, result, `${prefix}/${documentTitle}`); diff --git a/src/server/ApiManagers/GooglePhotosManager.ts b/src/server/ApiManagers/GooglePhotosManager.ts index 107542ce2..25c54ee2e 100644 --- a/src/server/ApiManagers/GooglePhotosManager.ts +++ b/src/server/ApiManagers/GooglePhotosManager.ts @@ -7,29 +7,34 @@ import { GooglePhotosUploadUtils } from "../apis/google/GooglePhotosUploadUtils" import { Opt } from "../../new_fields/Doc"; import { DashUploadUtils, InjectSize, SizeSuffix } from "../DashUploadUtils"; import { Database } from "../database"; +import { red } from "colors"; +import { Upload } from "../SharedMediaTypes"; +const prefix = "google_photos_"; +const remoteUploadError = "None of the preliminary uploads to Google's servers was successful."; const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; -const UploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; +const localUploadError = (count: number) => `Unable to upload ${count} images to Dash's server`; const requestError = "Unable to execute download: the body's media items were malformed."; const downloadError = "Encountered an error while executing downloads."; + interface GooglePhotosUploadFailure { batch: number; index: number; url: string; reason: string; } + interface MediaItem { baseUrl: string; - filename: string; } + interface NewMediaItem { description: string; simpleMediaItem: { uploadToken: string; }; } -const prefix = "google_photos_"; /** * This manager handles the creation of routes for google photos functionality. @@ -38,27 +43,47 @@ export default class GooglePhotosManager extends ApiManager { protected initialize(register: Registration): void { + /** + * This route receives a list of urls that point to images stored + * on Dash's file system, and, in a two step process, uploads them to Google's servers and + * returns the information Google generates about the associated uploaded remote images. + */ register({ method: Method.POST, - subscription: "/googlePhotosMediaUpload", + subscription: "/googlePhotosMediaPost", secureHandler: async ({ user, req, res }) => { const { media } = req.body; + + // first we need to ensure that we know the google account to which these photos will be uploaded const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); if (!token) { return _error(res, authenticationError); } + + // next, having one large list or even synchronously looping over things trips a threshold + // set on Google's servers, and would instantly return an error. So, we ease things out and send the photos to upload in + // batches of 25, where the next batch is sent 100 millieconds after we receive a response from Google's servers. const failed: GooglePhotosUploadFailure[] = []; const batched = BatchedArray.from<GooglePhotosUploadUtils.UploadSource>(media, { batchSize: 25 }); + const interval = { magnitude: 100, unit: TimeUnit.Milliseconds }; const newMediaItems = await batched.batchedMapPatientInterval<NewMediaItem>( - { magnitude: 100, unit: TimeUnit.Milliseconds }, + interval, async (batch: any, collector: any, { completedBatches }: any) => { for (let index = 0; index < batch.length; index++) { const { url, description } = batch[index]; + // a local function used to record failure of an upload const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, InjectSize(url, SizeSuffix.Original)).catch(fail); + // see image resizing - we store the size-agnostic url in our logic, but write out size-suffixed images to the file system + // so here, given a size agnostic url, we're just making that conversion so that the file system knows which bytes to actually upload + const imageToUpload = InjectSize(url, SizeSuffix.Original); + // STEP 1/2: send the raw bytes of the image from our server to Google's servers. We'll get back an upload token + // which acts as a pointer to those bytes that we can use to locate them later on + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, imageToUpload).catch(fail); if (!uploadToken) { fail(`${path.extname(url)} is not an accepted extension`); } else { + // gather the upload token return from Google (a pointer they give us to the raw, currently useless bytes + // we've uploaded to their servers) and put in the JSON format that the API accepts for image creation (used soon, below) collector.push({ description, simpleMediaItem: { uploadToken } @@ -67,11 +92,24 @@ export default class GooglePhotosManager extends ApiManager { } } ); - const failedCount = failed.length; - if (failedCount) { - console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + + // inform the developer / server console of any failed upload attempts + // does not abort the operation, since some subset of the uploads may have been successful + const { length } = failed; + if (length) { + console.error(`Unable to upload ${length} image${length === 1 ? "" : "s"} to Google's servers`); console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); } + + // if none of the preliminary uploads was successful, no need to try and create images + // report the failure to the client and return + if (!newMediaItems.length) { + console.error(red(`${remoteUploadError} Thus, aborting image creation. Please try again.`)); + _error(res, remoteUploadError); + return; + } + + // STEP 2/2: create the media items and return the API's response to the client, along with any failures return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( results => _success(res, { results, failed }), error => _error(res, mediaError, error) @@ -79,35 +117,68 @@ export default class GooglePhotosManager extends ApiManager { } }); + /** + * This route receives a list of urls that point to images + * stored on Google's servers and (following a *rough* heuristic) + * uploads each image to Dash's server if it hasn't already been uploaded. + * Unfortunately, since Google has so many of these images on its servers, + * these user content urls expire every 6 hours. So we can't store the url of a locally uploaded + * Google image and compare the candidate url to it to figure out if we already have it, + * since the same bytes on their server might now be associated with a new, random url. + * So, we do the next best thing and try to use an intrinsic attribute of those bytes as + * an identifier: the precise content size. This works in small cases, but has the obvious flaw of failing to upload + * an image locally if we already have uploaded another Google user content image with the exact same content size. + */ register({ method: Method.POST, - subscription: "/googlePhotosMediaDownload", + subscription: "/googlePhotosMediaGet", secureHandler: async ({ req, res }) => { - const contents: { mediaItems: MediaItem[] } = req.body; + const { mediaItems } = req.body as { mediaItems: MediaItem[] }; + if (!mediaItems) { + // non-starter, since the input was in an invalid format + _invalid(res, requestError); + return; + } let failed = 0; - if (contents) { - const completed: Opt<DashUploadUtils.ImageUploadInformation>[] = []; - for (const item of contents.mediaItems) { - const { contentSize, ...attributes } = await DashUploadUtils.InspectImage(item.baseUrl); - const found: Opt<DashUploadUtils.ImageUploadInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize!); - if (!found) { - const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, item.filename, prefix).catch(error => _error(res, downloadError, error)); - if (upload) { - completed.push(upload); - await Database.Auxiliary.LogUpload(upload); - } else { - failed++; - } + const completed: Opt<Upload.ImageInformation>[] = []; + for (const { baseUrl } of mediaItems) { + // start by getting the content size of the remote image + const results = await DashUploadUtils.InspectImage(baseUrl); + if (results instanceof Error) { + // if something went wrong here, we can't hope to upload it, so just move on to the next + failed++; + continue; + } + const { contentSize, ...attributes } = results; + // check to see if we have uploaded a Google user content image *specifically via this route* already + // that has this exact content size + const found: Opt<Upload.ImageInformation> = await Database.Auxiliary.QueryUploadHistory(contentSize); + if (!found) { + // if we haven't, then upload it locally to Dash's server + const upload = await DashUploadUtils.UploadInspectedImage({ contentSize, ...attributes }, undefined, prefix, false).catch(error => _error(res, downloadError, error)); + if (upload) { + completed.push(upload); + // inform the heuristic that we've encountered an image with this content size, + // to be later checked against in future uploads + await Database.Auxiliary.LogUpload(upload); } else { - completed.push(found); + // make note of a failure to upload locallys + failed++; } + } else { + // if we have, the variable 'found' is handily the upload information of the + // existing image, so we add it to the list as if we had just uploaded it now without actually + // making a duplicate write + completed.push(found); } - if (failed) { - return _error(res, UploadError(failed)); - } - return _success(res, completed); } - _invalid(res, requestError); + // if there are any failures, report a general failure to the client + if (failed) { + return _error(res, localUploadError(failed)); + } + // otherwise, return the image upload information list corresponding to the newly (or previously) + // uploaded images + _success(res, completed); } }); diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts index 4ce12f9f3..be17c3105 100644 --- a/src/server/ApiManagers/SearchManager.ts +++ b/src/server/ApiManagers/SearchManager.ts @@ -4,11 +4,15 @@ import { Search } from "../Search"; const findInFiles = require('find-in-files'); import * as path from 'path'; import { pathToDirectory, Directory } from "./UploadManager"; -import { red, cyan, yellow } from "colors"; +import { red, cyan, yellow, green } from "colors"; import RouteSubscriber from "../RouteSubscriber"; -import { exec } from "child_process"; +import { exec, execSync } from "child_process"; import { onWindows } from ".."; import { get } from "request-promise"; +import { log_execution } from "../ActionUtilities"; +import { Database } from "../database"; +import rimraf = require("rimraf"); +import { mkdirSync, chmod, chmodSync } from "fs"; export class SearchManager extends ApiManager { @@ -19,10 +23,17 @@ export class SearchManager extends ApiManager { subscription: new RouteSubscriber("solr").add("action"), secureHandler: async ({ req, res }) => { const { action } = req.params; - if (["start", "stop"].includes(action)) { - const status = req.params.action === "start"; - const success = await SolrManager.SetRunning(status); - console.log(success ? `Successfully ${status ? "started" : "stopped"} Solr!` : `Uh oh! Check the console for the error that occurred while ${status ? "starting" : "stopping"} Solr`); + switch (action) { + case "start": + case "stop": + const status = req.params.action === "start"; + SolrManager.SetRunning(status); + break; + case "update": + await SolrManager.update(); + break; + default: + console.log(yellow(`${action} is an unknown solr operation.`)); } res.redirect("/home"); } @@ -69,12 +80,10 @@ export class SearchManager extends ApiManager { export namespace SolrManager { - const command = onWindows ? "solr.cmd" : "solr"; - - export async function SetRunning(status: boolean): Promise<boolean> { + export function SetRunning(status: boolean) { const args = status ? "start" : "stop -p 8983"; console.log(`solr management: trying to ${args}`); - exec(`${command} ${args}`, { cwd: "./solr-8.3.1/bin" }, (error, stdout, stderr) => { + exec(`solr ${args}`, { cwd: "./solr-8.3.1/bin" }, (error, stdout, stderr) => { if (error) { console.log(red(`solr management error: unable to ${args} server`)); console.log(red(error.message)); @@ -82,12 +91,127 @@ export namespace SolrManager { console.log(cyan(stdout)); console.log(yellow(stderr)); }); + if (status) { + console.log(cyan("Start script is executing: please allow 15 seconds for solr to start on port 8983.")); + } + } + + export async function update() { + console.log(green("Beginning update...")); + await log_execution<void>({ + startMessage: "Clearing existing Solr information...", + endMessage: "Solr information successfully cleared", + action: Search.clear, + color: cyan + }); + const cursor = await log_execution({ + startMessage: "Connecting to and querying for all documents from database...", + endMessage: ({ result, error }) => { + const success = error === null && result !== undefined; + if (!success) { + console.log(red("Unable to connect to the database.")); + process.exit(0); + } + return "Connection successful and query complete"; + }, + action: () => Database.Instance.query({}), + color: yellow + }); + const updates: any[] = []; + let numDocs = 0; + function updateDoc(doc: any) { + numDocs++; + if ((numDocs % 50) === 0) { + console.log(`Batch of 50 complete, total of ${numDocs}`); + } + if (doc.__type !== "Doc") { + return; + } + const fields = doc.fields; + if (!fields) { + return; + } + const update: any = { id: doc._id }; + let dynfield = false; + for (const key in fields) { + const value = fields[key]; + const term = ToSearchTerm(value); + if (term !== undefined) { + const { suffix, value } = term; + update[key + suffix] = value; + dynfield = true; + } + } + if (dynfield) { + updates.push(update); + } + } + await cursor?.forEach(updateDoc); + const result = await log_execution({ + startMessage: `Dispatching updates for ${updates.length} documents`, + endMessage: "Dispatched updates complete", + action: () => Search.updateDocuments(updates), + color: cyan + }); try { - await get("http://localhost:8983"); - return true; - } catch { - return false; + if (result) { + const { status } = JSON.parse(result).responseHeader; + console.log(status ? red(`Failed with status code (${status})`) : green("Success!")); + } else { + console.log(red("Solr is likely not running!")); + } + } catch (e) { + console.log(red("Error:")); + console.log(e); + console.log("\n"); } + await cursor?.close(); + } + + const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { + "number": "_n", + "string": "_t", + "boolean": "_b", + "image": ["_t", "url"], + "video": ["_t", "url"], + "pdf": ["_t", "url"], + "audio": ["_t", "url"], + "web": ["_t", "url"], + "date": ["_d", value => new Date(value.date).toISOString()], + "proxy": ["_i", "fieldId"], + "list": ["_l", list => { + const results = []; + for (const value of list.fields) { + const term = ToSearchTerm(value); + if (term) { + results.push(term.value); + } + } + return results.length ? results : null; + }] + }; + + function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { + if (val === null || val === undefined) { + return; + } + const type = val.__type || typeof val; + let suffix = suffixMap[type]; + if (!suffix) { + return; + } + + if (Array.isArray(suffix)) { + const accessor = suffix[1]; + if (typeof accessor === "function") { + val = accessor(val); + } else { + val = val[accessor]; + } + suffix = suffix[0]; + } + + return { suffix, value: val }; } }
\ No newline at end of file diff --git a/src/server/ApiManagers/SessionManager.ts b/src/server/ApiManagers/SessionManager.ts index f1629b8f0..bcaa6598f 100644 --- a/src/server/ApiManagers/SessionManager.ts +++ b/src/server/ApiManagers/SessionManager.ts @@ -53,6 +53,15 @@ export default class SessionManager extends ApiManager { }) }); + register({ + method: Method.GET, + subscription: this.secureSubscriber("delete"), + secureHandler: this.authorizedAction(async ({ res }) => { + const { error } = await sessionAgent.serverWorker.emit("delete"); + res.send(error ? error.message : "Your request was successful: the server successfully deleted the database. Return to /home."); + }) + }); + } }
\ No newline at end of file diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index e18b6826e..f872bdf94 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -4,12 +4,12 @@ import * as formidable from 'formidable'; import v4 = require('uuid/v4'); const AdmZip = require('adm-zip'); import { extname, basename, dirname } from 'path'; -import { createReadStream, createWriteStream, unlink, readFileSync } from "fs"; +import { createReadStream, createWriteStream, unlink } from "fs"; import { publicDirectory, filesDirectory } from ".."; import { Database } from "../database"; -import { DashUploadUtils, SizeSuffix } from "../DashUploadUtils"; +import { DashUploadUtils } from "../DashUploadUtils"; import * as sharp from 'sharp'; -import { AcceptibleMedia } from "../SharedMediaTypes"; +import { AcceptibleMedia, Upload } from "../SharedMediaTypes"; import { normalize } from "path"; const imageDataUri = require('image-data-uri'); @@ -48,7 +48,7 @@ export default class UploadManager extends ApiManager { form.keepExtensions = true; return new Promise<void>(resolve => { form.parse(req, async (_err, _fields, files) => { - const results: any[] = []; + const results: Upload.FileResponse[] = []; for (const key in files) { const result = await DashUploadUtils.upload(files[key]); result && results.push(result); @@ -66,7 +66,8 @@ export default class UploadManager extends ApiManager { secureHandler: async ({ req, res }) => { const { sources } = req.body; if (Array.isArray(sources)) { - return res.send(await Promise.all(sources.map(url => DashUploadUtils.UploadImage(url)))); + const results = await Promise.all(sources.map(source => DashUploadUtils.UploadImage(source))); + return res.send(results); } res.send(); } diff --git a/src/server/ApiManagers/UserManager.ts b/src/server/ApiManagers/UserManager.ts index b0d868918..d9d346cc1 100644 --- a/src/server/ApiManagers/UserManager.ts +++ b/src/server/ApiManagers/UserManager.ts @@ -34,7 +34,7 @@ export default class UserManager extends ApiManager { register({ method: Method.GET, subscription: "/getCurrentUser", - secureHandler: ({ res, user }) => res.send(JSON.stringify(user)), + secureHandler: ({ res, user: { _id, email } }) => res.send(JSON.stringify({ id: _id, email })), publicHandler: ({ res }) => res.send(JSON.stringify({ id: "__guest__", email: "" })) }); diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts index 32aecd3c6..d18529cf2 100644 --- a/src/server/ApiManagers/UtilManager.ts +++ b/src/server/ApiManagers/UtilManager.ts @@ -1,7 +1,6 @@ import ApiManager, { Registration } from "./ApiManager"; import { Method } from "../RouteManager"; import { exec } from 'child_process'; -import { command_line } from "../ActionUtilities"; import RouteSubscriber from "../RouteSubscriber"; import { red } from "colors"; import { IBM_Recommender } from "../../client/apis/IBM_Recommender"; @@ -9,6 +8,7 @@ import { Recommender } from "../Recommender"; const recommender = new Recommender(); recommender.testModel(); +import executeImport from "../../scraping/buxton/final/BuxtonImporter"; export default class UtilManager extends ApiManager { @@ -67,20 +67,6 @@ export default class UtilManager extends ApiManager { register({ method: Method.GET, - subscription: "/buxton", - secureHandler: async ({ res }) => { - const cwd = './src/scraping/buxton'; - - const onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); }; - const onRejected = (err: any) => { console.error(err.message); res.send(err); }; - const tryPython3 = () => command_line('python3 scraper.py', cwd).then(onResolved, onRejected); - - return command_line('python scraper.py', cwd).then(onResolved, tryPython3); - }, - }); - - register({ - method: Method.GET, subscription: "/version", secureHandler: ({ res }) => { return new Promise<void>(resolve => { diff --git a/src/server/DashSession/DashSessionAgent.ts b/src/server/DashSession/DashSessionAgent.ts index c74b50555..1ed98cdbe 100644 --- a/src/server/DashSession/DashSessionAgent.ts +++ b/src/server/DashSession/DashSessionAgent.ts @@ -8,8 +8,11 @@ import { launchServer, onWindows } from ".."; import { readdirSync, statSync, createWriteStream, readFileSync, unlinkSync } from "fs"; import * as Archiver from "archiver"; import { resolve } from "path"; -import { AppliedSessionAgent, MessageHandler, ExitHandler, Monitor, ServerWorker } from "resilient-server-session"; import rimraf = require("rimraf"); +import { AppliedSessionAgent, ExitHandler } from "./Session/agents/applied_session_agent"; +import { ServerWorker } from "./Session/agents/server_worker"; +import { Monitor } from "./Session/agents/monitor"; +import { MessageHandler } from "./Session/agents/promisified_ipc_manager"; /** * If we're the monitor (master) thread, we should launch the monitor logic for the session. @@ -25,18 +28,18 @@ export class DashSessionAgent extends AppliedSessionAgent { * The core method invoked when the single master thread is initialized. * Installs event hooks, repl commands and additional IPC listeners. */ - // protected async initializeMonitor(monitor: Monitor, sessionKey: string): Promise<void> { - protected async initializeMonitor(monitor: Monitor): Promise<void> { - - // await this.dispatchSessionPassword(sessionKey); - // monitor.addReplCommand("pull", [], () => monitor.exec("git pull")); - // monitor.addReplCommand("solr", [/start|stop|index/], this.executeSolrCommand); - // monitor.addReplCommand("backup", [], this.backup); - // monitor.addReplCommand("debug", [/\S+\@\S+/], async ([to]) => this.dispatchZippedDebugBackup(to)); - // monitor.on("backup", this.backup); - // monitor.on("debug", async ({ to }) => this.dispatchZippedDebugBackup(to)); - // monitor.coreHooks.onCrashDetected(this.dispatchCrashReport); - return; + protected async initializeMonitor(monitor: Monitor): Promise<string> { + const sessionKey = Utils.GenerateGuid(); + await this.dispatchSessionPassword(sessionKey); + monitor.addReplCommand("pull", [], () => monitor.exec("git pull")); + monitor.addReplCommand("solr", [/start|stop|index/], this.executeSolrCommand); + monitor.addReplCommand("backup", [], this.backup); + monitor.addReplCommand("debug", [/\S+\@\S+/], async ([to]) => this.dispatchZippedDebugBackup(to)); + monitor.on("backup", this.backup); + monitor.on("debug", async ({ to }) => this.dispatchZippedDebugBackup(to)); + monitor.on("delete", WebSocket.deleteFields); + monitor.coreHooks.onCrashDetected(this.dispatchCrashReport); + return sessionKey; } /** diff --git a/src/server/DashSession/Session/agents/applied_session_agent.ts b/src/server/DashSession/Session/agents/applied_session_agent.ts new file mode 100644 index 000000000..46c9e22ed --- /dev/null +++ b/src/server/DashSession/Session/agents/applied_session_agent.ts @@ -0,0 +1,58 @@ +import { isMaster } from "cluster"; +import { Monitor } from "./monitor"; +import { ServerWorker } from "./server_worker"; +import { Utilities } from "../utilities/utilities"; + +export type ExitHandler = (reason: Error | boolean) => void | Promise<void>; + +export abstract class AppliedSessionAgent { + + // the following two methods allow the developer to create a custom + // session and use the built in customization options for each thread + protected abstract async initializeMonitor(monitor: Monitor): Promise<string>; + protected abstract async initializeServerWorker(): Promise<ServerWorker>; + + private launched = false; + + public killSession = (reason: string, graceful = true, errorCode = 0) => { + const target = isMaster ? this.sessionMonitor : this.serverWorker; + target.killSession(reason, graceful, errorCode); + } + + private sessionMonitorRef: Monitor | undefined; + public get sessionMonitor(): Monitor { + if (!isMaster) { + this.serverWorker.emit("kill", { + graceful: false, + reason: "Cannot access the session monitor directly from the server worker thread.", + errorCode: 1 + }); + throw new Error(); + } + return this.sessionMonitorRef!; + } + + private serverWorkerRef: ServerWorker | undefined; + public get serverWorker(): ServerWorker { + if (isMaster) { + throw new Error("Cannot access the server worker directly from the session monitor thread"); + } + return this.serverWorkerRef!; + } + + public async launch(): Promise<void> { + if (!this.launched) { + this.launched = true; + if (isMaster) { + this.sessionMonitorRef = Monitor.Create() + const sessionKey = await this.initializeMonitor(this.sessionMonitorRef); + this.sessionMonitorRef.finalize(sessionKey); + } else { + this.serverWorkerRef = await this.initializeServerWorker(); + } + } else { + throw new Error("Cannot launch a session thread more than once per process."); + } + } + +}
\ No newline at end of file diff --git a/src/server/DashSession/Session/agents/monitor.ts b/src/server/DashSession/Session/agents/monitor.ts new file mode 100644 index 000000000..6f8d25614 --- /dev/null +++ b/src/server/DashSession/Session/agents/monitor.ts @@ -0,0 +1,298 @@ +import { ExitHandler } from "./applied_session_agent"; +import { Configuration, configurationSchema, defaultConfig, Identifiers, colorMapping } from "../utilities/session_config"; +import Repl, { ReplAction } from "../utilities/repl"; +import { isWorker, setupMaster, on, Worker, fork } from "cluster"; +import { manage, MessageHandler } from "./promisified_ipc_manager"; +import { red, cyan, white, yellow, blue } from "colors"; +import { exec, ExecOptions } from "child_process"; +import { validate, ValidationError } from "jsonschema"; +import { Utilities } from "../utilities/utilities"; +import { readFileSync } from "fs"; +import IPCMessageReceiver from "./process_message_router"; +import { ServerWorker } from "./server_worker"; + +/** + * Validates and reads the configuration file, accordingly builds a child process factory + * and spawns off an initial process that will respawn as predecessors die. + */ +export class Monitor extends IPCMessageReceiver { + private static count = 0; + private finalized = false; + private exitHandlers: ExitHandler[] = []; + private readonly config: Configuration; + private activeWorker: Worker | undefined; + private key: string | undefined; + // private repl: Repl; + + public static Create() { + if (isWorker) { + ServerWorker.IPCManager.emit("kill", { + reason: "cannot create a monitor on the worker process.", + graceful: false, + errorCode: 1 + }); + process.exit(1); + } else if (++Monitor.count > 1) { + console.error(red("cannot create more than one monitor.")); + process.exit(1); + } else { + return new Monitor(); + } + } + + private constructor() { + super(); + console.log(this.timestamp(), cyan("initializing session...")); + this.configureInternalHandlers(); + this.config = this.loadAndValidateConfiguration(); + this.initializeClusterFunctions(); + // this.repl = this.initializeRepl(); + } + + protected configureInternalHandlers = () => { + // handle exceptions in the master thread - there shouldn't be many of these + // the IPC (inter process communication) channel closed exception can't seem + // to be caught in a try catch, and is inconsequential, so it is ignored + process.on("uncaughtException", ({ message, stack }): void => { + if (message !== "Channel closed") { + this.mainLog(red(message)); + if (stack) { + this.mainLog(`uncaught exception\n${red(stack)}`); + } + } + }); + + this.on("kill", ({ reason, graceful, errorCode }) => this.killSession(reason, graceful, errorCode)); + this.on("lifecycle", ({ event }) => console.log(this.timestamp(), `${this.config.identifiers.worker.text} lifecycle phase (${event})`)); + } + + private initializeClusterFunctions = () => { + // determines whether or not we see the compilation / initialization / runtime output of each child server process + const output = this.config.showServerOutput ? "inherit" : "ignore"; + setupMaster({ stdio: ["ignore", output, output, "ipc"] }); + + // a helpful cluster event called on the master thread each time a child process exits + on("exit", ({ process: { pid } }, code, signal) => { + const prompt = `server worker with process id ${pid} has exited with code ${code}${signal === null ? "" : `, having encountered signal ${signal}`}.`; + this.mainLog(cyan(prompt)); + // to make this a robust, continuous session, every time a child process dies, we immediately spawn a new one + this.spawn(); + }); + } + + public finalize = (sessionKey: string): void => { + if (this.finalized) { + throw new Error("Session monitor is already finalized"); + } + this.finalized = true; + this.key = sessionKey; + this.spawn(); + } + + public readonly coreHooks = Object.freeze({ + onCrashDetected: (listener: MessageHandler<{ error: Error }>) => this.on(Monitor.IntrinsicEvents.CrashDetected, listener), + onServerRunning: (listener: MessageHandler<{ isFirstTime: boolean }>) => this.on(Monitor.IntrinsicEvents.ServerRunning, listener) + }); + + /** + * Kill this session and its active child + * server process, either gracefully (may wait + * indefinitely, but at least allows active networking + * requests to complete) or immediately. + */ + public killSession = async (reason: string, graceful = true, errorCode = 0) => { + this.mainLog(cyan(`exiting session ${graceful ? "clean" : "immediate"}ly`)); + this.mainLog(`session exit reason: ${(red(reason))}`); + await this.executeExitHandlers(true); + await this.killActiveWorker(graceful, true); + process.exit(errorCode); + } + + /** + * Execute the list of functions registered to be called + * whenever the process exits. + */ + public addExitHandler = (handler: ExitHandler) => this.exitHandlers.push(handler); + + /** + * Extend the default repl by adding in custom commands + * that can invoke application logic external to this module + */ + public addReplCommand = (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => { + // this.repl.registerCommand(basename, argPatterns, action); + } + + public exec = (command: string, options?: ExecOptions) => { + return new Promise<void>(resolve => { + exec(command, { ...options, encoding: "utf8" }, (error, stdout, stderr) => { + if (error) { + this.execLog(red(`unable to execute ${white(command)}`)); + error.message.split("\n").forEach(line => line.length && this.execLog(red(`(error) ${line}`))); + } else { + let outLines: string[], errorLines: string[]; + if ((outLines = stdout.split("\n").filter(line => line.length)).length) { + outLines.forEach(line => line.length && this.execLog(cyan(`(stdout) ${line}`))); + } + if ((errorLines = stderr.split("\n").filter(line => line.length)).length) { + errorLines.forEach(line => line.length && this.execLog(yellow(`(stderr) ${line}`))); + } + } + resolve(); + }); + }); + } + + /** + * Generates a blue UTC string associated with the time + * of invocation. + */ + private timestamp = () => blue(`[${new Date().toUTCString()}]`); + + /** + * A formatted, identified and timestamped log in color + */ + public mainLog = (...optionalParams: any[]) => { + console.log(this.timestamp(), this.config.identifiers.master.text, ...optionalParams); + } + + /** + * A formatted, identified and timestamped log in color for non- + */ + private execLog = (...optionalParams: any[]) => { + console.log(this.timestamp(), this.config.identifiers.exec.text, ...optionalParams); + } + + /** + * Reads in configuration .json file only once, in the master thread + * and pass down any variables the pertinent to the child processes as environment variables. + */ + private loadAndValidateConfiguration = (): Configuration => { + let config: Configuration; + try { + console.log(this.timestamp(), cyan("validating configuration...")); + config = JSON.parse(readFileSync('./session.config.json', 'utf8')); + const options = { + throwError: true, + allowUnknownAttributes: false + }; + // ensure all necessary and no excess information is specified by the configuration file + validate(config, configurationSchema, options); + config = Utilities.preciseAssign({}, defaultConfig, config); + } catch (error) { + if (error instanceof ValidationError) { + console.log(red("\nSession configuration failed.")); + console.log("The given session.config.json configuration file is invalid."); + console.log(`${error.instance}: ${error.stack}`); + process.exit(0); + } else if (error.code === "ENOENT" && error.path === "./session.config.json") { + console.log(cyan("Loading default session parameters...")); + console.log("Consider including a session.config.json configuration file in your project root for customization."); + config = Utilities.preciseAssign({}, defaultConfig); + } else { + console.log(red("\nSession configuration failed.")); + console.log("The following unknown error occurred during configuration."); + console.log(error.stack); + process.exit(0); + } + } finally { + const { identifiers } = config!; + Object.keys(identifiers).forEach(key => { + const resolved = key as keyof Identifiers; + const { text, color } = identifiers[resolved]; + identifiers[resolved].text = (colorMapping.get(color) || white)(`${text}:`); + }); + return config!; + } + } + + /** + * Builds the repl that allows the following commands to be typed into stdin of the master thread. + */ + private initializeRepl = (): Repl => { + const repl = new Repl({ identifier: () => `${this.timestamp()} ${this.config.identifiers.master.text}` }); + const boolean = /true|false/; + const number = /\d+/; + const letters = /[a-zA-Z]+/; + repl.registerCommand("exit", [/clean|force/], args => this.killSession("manual exit requested by repl", args[0] === "clean", 0)); + repl.registerCommand("restart", [/clean|force/], args => this.killActiveWorker(args[0] === "clean")); + repl.registerCommand("set", [letters, "port", number, boolean], args => this.setPort(args[0], Number(args[2]), args[3] === "true")); + repl.registerCommand("set", [/polling/, number, boolean], args => { + const newPollingIntervalSeconds = Math.floor(Number(args[1])); + if (newPollingIntervalSeconds < 0) { + this.mainLog(red("the polling interval must be a non-negative integer")); + } else { + if (newPollingIntervalSeconds !== this.config.polling.intervalSeconds) { + this.config.polling.intervalSeconds = newPollingIntervalSeconds; + if (args[2] === "true") { + Monitor.IPCManager.emit("updatePollingInterval", { newPollingIntervalSeconds }); + } + } + } + }); + return repl; + } + + private executeExitHandlers = async (reason: Error | boolean) => Promise.all(this.exitHandlers.map(handler => handler(reason))); + + /** + * Attempts to kill the active worker gracefully, unless otherwise specified. + */ + private killActiveWorker = async (graceful = true, isSessionEnd = false): Promise<void> => { + if (this.activeWorker && !this.activeWorker.isDead()) { + if (graceful) { + Monitor.IPCManager.emit("manualExit", { isSessionEnd }); + } else { + await ServerWorker.IPCManager.destroy(); + this.activeWorker.process.kill(); + } + } + } + + /** + * Allows the caller to set the port at which the target (be it the server, + * the websocket, some other custom port) is listening. If an immediate restart + * is specified, this monitor will kill the active child and re-launch the server + * at the port. Otherwise, the updated port won't be used until / unless the child + * dies on its own and triggers a restart. + */ + private setPort = (port: "server" | "socket" | string, value: number, immediateRestart: boolean): void => { + if (value > 1023 && value < 65536) { + this.config.ports[port] = value; + if (immediateRestart) { + this.killActiveWorker(); + } + } else { + this.mainLog(red(`${port} is an invalid port number`)); + } + } + + /** + * Kills the current active worker and proceeds to spawn a new worker, + * feeding in configuration information as environment variables. + */ + private spawn = async (): Promise<void> => { + await this.killActiveWorker(); + const { config: { polling, ports }, key } = this; + this.activeWorker = fork({ + pollingRoute: polling.route, + pollingFailureTolerance: polling.failureTolerance, + serverPort: ports.server, + socketPort: ports.socket, + pollingIntervalSeconds: polling.intervalSeconds, + session_key: key + }); + Monitor.IPCManager = manage(this.activeWorker.process, this.handlers); + this.mainLog(cyan(`spawned new server worker with process id ${this.activeWorker?.process.pid}`)); + } + +} + +export namespace Monitor { + + export enum IntrinsicEvents { + KeyGenerated = "key_generated", + CrashDetected = "crash_detected", + ServerRunning = "server_running" + } + +}
\ No newline at end of file diff --git a/src/server/DashSession/Session/agents/process_message_router.ts b/src/server/DashSession/Session/agents/process_message_router.ts new file mode 100644 index 000000000..6cc8aa941 --- /dev/null +++ b/src/server/DashSession/Session/agents/process_message_router.ts @@ -0,0 +1,41 @@ +import { MessageHandler, PromisifiedIPCManager, HandlerMap } from "./promisified_ipc_manager"; + +export default abstract class IPCMessageReceiver { + + protected static IPCManager: PromisifiedIPCManager; + protected handlers: HandlerMap = {}; + + protected abstract configureInternalHandlers: () => void; + + /** + * Add a listener at this message. When the monitor process + * receives a message, it will invoke all registered functions. + */ + public on = (name: string, handler: MessageHandler) => { + const handlers = this.handlers[name]; + if (!handlers) { + this.handlers[name] = [handler]; + } else { + handlers.push(handler); + } + } + + /** + * Unregister a given listener at this message. + */ + public off = (name: string, handler: MessageHandler) => { + const handlers = this.handlers[name]; + if (handlers) { + const index = handlers.indexOf(handler); + if (index > -1) { + handlers.splice(index, 1); + } + } + } + + /** + * Unregister all listeners at this message. + */ + public clearMessageListeners = (...names: string[]) => names.map(name => delete this.handlers[name]); + +}
\ No newline at end of file diff --git a/src/server/DashSession/Session/agents/promisified_ipc_manager.ts b/src/server/DashSession/Session/agents/promisified_ipc_manager.ts new file mode 100644 index 000000000..9f0db8330 --- /dev/null +++ b/src/server/DashSession/Session/agents/promisified_ipc_manager.ts @@ -0,0 +1,173 @@ +import { Utilities } from "../utilities/utilities"; +import { ChildProcess } from "child_process"; + +/** + * Convenience constructor + * @param target the process / worker to which to attach the specialized listeners + */ +export function manage(target: IPCTarget, handlers?: HandlerMap) { + return new PromisifiedIPCManager(target, handlers); +} + +/** + * Captures the logic to execute upon receiving a message + * of a certain name. + */ +export type HandlerMap = { [name: string]: MessageHandler[] }; + +/** + * This will always literally be a child process. But, though setting + * up a manager in the parent will indeed see the target as the ChildProcess, + * setting up a manager in the child will just see itself as a regular NodeJS.Process. + */ +export type IPCTarget = NodeJS.Process | ChildProcess; + +/** + * Specifies a general message format for this API + */ +export type Message<T = any> = { + name: string; + args?: T; +}; +export type MessageHandler<T = any> = (args: T) => (any | Promise<any>); + +/** + * When a message is emitted, it is embedded with private metadata + * to facilitate the resolution of promises, etc. + */ +interface InternalMessage extends Message { metadata: Metadata } +interface Metadata { isResponse: boolean; id: string } +type InternalMessageHandler = (message: InternalMessage) => (any | Promise<any>); + +/** + * Allows for the transmission of the error's key features over IPC. + */ +export interface ErrorLike { + name?: string; + message?: string; + stack?: string; +} + +/** + * The arguments returned in a message sent from the target upon completion. + */ +export interface Response<T = any> { + results?: T[]; + error?: ErrorLike; +} + +const destroyEvent = "__destroy__"; + +/** + * This is a wrapper utility class that allows the caller process + * to emit an event and return a promise that resolves when it and all + * other processes listening to its emission of this event have completed. + */ +export class PromisifiedIPCManager { + private readonly target: IPCTarget; + private pendingMessages: { [id: string]: string } = {}; + private isDestroyed = false; + private get callerIsTarget() { + return process.pid === this.target.pid; + } + + constructor(target: IPCTarget, handlers?: HandlerMap) { + this.target = target; + if (handlers) { + handlers[destroyEvent] = [this.destroyHelper]; + this.target.addListener("message", this.generateInternalHandler(handlers)); + } + } + + /** + * This routine uniquely identifies each message, then adds a general + * message listener that waits for a response with the same id before resolving + * the promise. + */ + public emit = async <T = any>(name: string, args?: any): Promise<Response<T>> => { + if (this.isDestroyed) { + const error = { name: "FailedDispatch", message: "Cannot use a destroyed IPC manager to emit a message." }; + return { error }; + } + return new Promise<Response<T>>(resolve => { + const messageId = Utilities.guid(); + const responseHandler: InternalMessageHandler = ({ metadata: { id, isResponse }, args }) => { + if (isResponse && id === messageId) { + this.target.removeListener("message", responseHandler); + resolve(args); + } + }; + this.target.addListener("message", responseHandler); + const message = { name, args, metadata: { id: messageId, isResponse: false } }; + if (!(this.target.send && this.target.send(message))) { + const error: ErrorLike = { name: "FailedDispatch", message: "Either the target's send method was undefined or the act of sending failed." }; + resolve({ error }); + this.target.removeListener("message", responseHandler); + } + }); + } + + /** + * Invoked from either the parent or the child process, this allows + * any unresolved promises to continue in the target process, but dispatches a dummy + * completion response for each of the pending messages, allowing their + * promises in the caller to resolve. + */ + public destroy = () => { + return new Promise<void>(async resolve => { + if (this.callerIsTarget) { + this.destroyHelper(); + } else { + await this.emit(destroyEvent); + } + resolve(); + }); + } + + /** + * Dispatches the dummy responses and sets the isDestroyed flag to true. + */ + private destroyHelper = () => { + const { pendingMessages } = this; + this.isDestroyed = true; + Object.keys(pendingMessages).forEach(id => { + const error: ErrorLike = { name: "ManagerDestroyed", message: "The IPC manager was destroyed before the response could be returned." }; + const message: InternalMessage = { name: pendingMessages[id], args: { error }, metadata: { id, isResponse: true } }; + this.target.send?.(message) + }); + this.pendingMessages = {}; + } + + /** + * This routine receives a uniquely identified message. If the message is itself a response, + * it is ignored to avoid infinite mutual responses. Otherwise, the routine awaits its completion using whatever + * router the caller has installed, and then sends a response containing the original message id, + * which will ultimately invoke the responseHandler of the original emission and resolve the + * sender's promise. + */ + private generateInternalHandler = (handlers: HandlerMap): MessageHandler => async (message: InternalMessage) => { + const { name, args, metadata } = message; + if (name && metadata && !metadata.isResponse) { + const { id } = metadata; + this.pendingMessages[id] = name; + let error: Error | undefined; + let results: any[] | undefined; + try { + const registered = handlers[name]; + if (registered) { + results = await Promise.all(registered.map(handler => handler(args))); + } + } catch (e) { + error = e; + } + if (!this.isDestroyed && this.target.send) { + const metadata = { id, isResponse: true }; + const response: Response = { results , error }; + const message = { name, args: response , metadata }; + delete this.pendingMessages[id]; + this.target.send(message); + } + } + } + +}
\ No newline at end of file diff --git a/src/server/DashSession/Session/agents/server_worker.ts b/src/server/DashSession/Session/agents/server_worker.ts new file mode 100644 index 000000000..976d27226 --- /dev/null +++ b/src/server/DashSession/Session/agents/server_worker.ts @@ -0,0 +1,160 @@ +import { ExitHandler } from "./applied_session_agent"; +import { isMaster } from "cluster"; +import { manage } from "./promisified_ipc_manager"; +import IPCMessageReceiver from "./process_message_router"; +import { red, green, white, yellow } from "colors"; +import { get } from "request-promise"; +import { Monitor } from "./monitor"; + +/** + * Effectively, each worker repairs the connection to the server by reintroducing a consistent state + * if its predecessor has died. It itself also polls the server heartbeat, and exits with a notification + * email if the server encounters an uncaught exception or if the server cannot be reached. + */ +export class ServerWorker extends IPCMessageReceiver { + private static count = 0; + private shouldServerBeResponsive = false; + private exitHandlers: ExitHandler[] = []; + private pollingFailureCount = 0; + private pollingIntervalSeconds: number; + private pollingFailureTolerance: number; + private pollTarget: string; + private serverPort: number; + private isInitialized = false; + + public static Create(work: Function) { + if (isMaster) { + console.error(red("cannot create a worker on the monitor process.")); + process.exit(1); + } else if (++ServerWorker.count > 1) { + ServerWorker.IPCManager.emit("kill", { + reason: "cannot create more than one worker on a given worker process.", + graceful: false, + errorCode: 1 + }); + process.exit(1); + } else { + return new ServerWorker(work); + } + } + + /** + * Allows developers to invoke application specific logic + * by hooking into the exiting of the server process. + */ + public addExitHandler = (handler: ExitHandler) => this.exitHandlers.push(handler); + + /** + * Kill the session monitor (parent process) from this + * server worker (child process). This will also kill + * this process (child process). + */ + public killSession = (reason: string, graceful = true, errorCode = 0) => this.emit<never>("kill", { reason, graceful, errorCode }); + + /** + * A convenience wrapper to tell the session monitor (parent process) + * to carry out the action with the specified message and arguments. + */ + public emit = async <T = any>(name: string, args?: any) => ServerWorker.IPCManager.emit<T>(name, args); + + private constructor(work: Function) { + super(); + this.configureInternalHandlers(); + ServerWorker.IPCManager = manage(process, this.handlers); + this.lifecycleNotification(green(`initializing process... ${white(`[${process.execPath} ${process.execArgv.join(" ")}]`)}`)); + + const { pollingRoute, serverPort, pollingIntervalSeconds, pollingFailureTolerance } = process.env; + this.serverPort = Number(serverPort); + this.pollingIntervalSeconds = Number(pollingIntervalSeconds); + this.pollingFailureTolerance = Number(pollingFailureTolerance); + this.pollTarget = `http://localhost:${serverPort}${pollingRoute}`; + + work(); + this.pollServer(); + } + + /** + * Set up message and uncaught exception handlers for this + * server process. + */ + protected configureInternalHandlers = () => { + // updates the local values of variables to the those sent from master + this.on("updatePollingInterval", ({ newPollingIntervalSeconds }) => this.pollingIntervalSeconds = newPollingIntervalSeconds); + this.on("manualExit", async ({ isSessionEnd }) => { + await ServerWorker.IPCManager.destroy(); + await this.executeExitHandlers(isSessionEnd); + process.exit(0); + }); + + // one reason to exit, as the process might be in an inconsistent state after such an exception + process.on('uncaughtException', this.proactiveUnplannedExit); + process.on('unhandledRejection', reason => { + const appropriateError = reason instanceof Error ? reason : new Error(`unhandled rejection: ${reason}`); + this.proactiveUnplannedExit(appropriateError); + }); + } + + /** + * Execute the list of functions registered to be called + * whenever the process exits. + */ + private executeExitHandlers = async (reason: Error | boolean) => Promise.all(this.exitHandlers.map(handler => handler(reason))); + + /** + * Notify master thread (which will log update in the console) of initialization via IPC. + */ + public lifecycleNotification = (event: string) => this.emit("lifecycle", { event }); + + /** + * Called whenever the process has a reason to terminate, either through an uncaught exception + * in the process (potentially inconsistent state) or the server cannot be reached. + */ + private proactiveUnplannedExit = async (error: Error): Promise<void> => { + this.shouldServerBeResponsive = false; + // communicates via IPC to the master thread that it should dispatch a crash notification email + this.emit(Monitor.IntrinsicEvents.CrashDetected, { error }); + await this.executeExitHandlers(error); + // notify master thread (which will log update in the console) of crash event via IPC + this.lifecycleNotification(red(`crash event detected @ ${new Date().toUTCString()}`)); + this.lifecycleNotification(red(error.message)); + await ServerWorker.IPCManager.destroy(); + process.exit(1); + } + + /** + * This monitors the health of the server by submitting a get request to whatever port / route specified + * by the configuration every n seconds, where n is also given by the configuration. + */ + private pollServer = async (): Promise<void> => { + await new Promise<void>(resolve => { + setTimeout(async () => { + try { + await get(this.pollTarget); + if (!this.shouldServerBeResponsive) { + // notify monitor thread that the server is up and running + this.lifecycleNotification(green(`listening on ${this.serverPort}...`)); + this.emit(Monitor.IntrinsicEvents.ServerRunning, { isFirstTime: !this.isInitialized }); + this.isInitialized = true; + } + this.shouldServerBeResponsive = true; + } catch (error) { + // if we expect the server to be unavailable, i.e. during compilation, + // the listening variable is false, activeExit will return early and the child + // process will continue + if (this.shouldServerBeResponsive) { + if (++this.pollingFailureCount > this.pollingFailureTolerance) { + this.proactiveUnplannedExit(error); + } else { + this.lifecycleNotification(yellow(`the server has encountered ${this.pollingFailureCount} of ${this.pollingFailureTolerance} tolerable failures`)); + } + } + } finally { + resolve(); + } + }, 1000 * this.pollingIntervalSeconds); + }); + // controlled, asynchronous infinite recursion achieves a persistent poll that does not submit a new request until the previous has completed + this.pollServer(); + } + +}
\ No newline at end of file diff --git a/src/server/DashSession/Session/utilities/repl.ts b/src/server/DashSession/Session/utilities/repl.ts new file mode 100644 index 000000000..643141286 --- /dev/null +++ b/src/server/DashSession/Session/utilities/repl.ts @@ -0,0 +1,128 @@ +import { createInterface, Interface } from "readline"; +import { red, green, white } from "colors"; + +export interface Configuration { + identifier: () => string | string; + onInvalid?: (command: string, validCommand: boolean) => string | string; + onValid?: (success?: string) => string | string; + isCaseSensitive?: boolean; +} + +export type ReplAction = (parsedArgs: Array<string>) => any | Promise<any>; +export interface Registration { + argPatterns: RegExp[]; + action: ReplAction; +} + +export default class Repl { + private identifier: () => string | string; + private onInvalid: ((command: string, validCommand: boolean) => string) | string; + private onValid: ((success: string) => string) | string; + private isCaseSensitive: boolean; + private commandMap = new Map<string, Registration[]>(); + public interface: Interface; + private busy = false; + private keys: string | undefined; + + constructor({ identifier: prompt, onInvalid, onValid, isCaseSensitive }: Configuration) { + this.identifier = prompt; + this.onInvalid = onInvalid || this.usage; + this.onValid = onValid || this.success; + this.isCaseSensitive = isCaseSensitive ?? true; + this.interface = createInterface(process.stdin, process.stdout).on('line', this.considerInput); + } + + private resolvedIdentifier = () => typeof this.identifier === "string" ? this.identifier : this.identifier(); + + private usage = (command: string, validCommand: boolean) => { + if (validCommand) { + const formatted = white(command); + const patterns = green(this.commandMap.get(command)!.map(({ argPatterns }) => `${formatted} ${argPatterns.join(" ")}`).join('\n')); + return `${this.resolvedIdentifier()}\nthe given arguments do not match any registered patterns for ${formatted}\nthe list of valid argument patterns is given by:\n${patterns}`; + } else { + const resolved = this.keys; + if (resolved) { + return resolved; + } + const members: string[] = []; + const keys = this.commandMap.keys(); + let next: IteratorResult<string>; + while (!(next = keys.next()).done) { + members.push(next.value); + } + return `${this.resolvedIdentifier()} commands: { ${members.sort().join(", ")} }`; + } + } + + private success = (command: string) => `${this.resolvedIdentifier()} completed local execution of ${white(command)}`; + + public registerCommand = (basename: string, argPatterns: (RegExp | string)[], action: ReplAction) => { + const existing = this.commandMap.get(basename); + const converted = argPatterns.map(input => input instanceof RegExp ? input : new RegExp(input)); + const registration = { argPatterns: converted, action }; + if (existing) { + existing.push(registration); + } else { + this.commandMap.set(basename, [registration]); + } + } + + private invalid = (command: string, validCommand: boolean) => { + console.log(red(typeof this.onInvalid === "string" ? this.onInvalid : this.onInvalid(command, validCommand))); + this.busy = false; + } + + private valid = (command: string) => { + console.log(green(typeof this.onValid === "string" ? this.onValid : this.onValid(command))); + this.busy = false; + } + + private considerInput = async (line: string) => { + if (this.busy) { + console.log(red("Busy")); + return; + } + this.busy = true; + line = line.trim(); + if (this.isCaseSensitive) { + line = line.toLowerCase(); + } + const [command, ...args] = line.split(/\s+/g); + if (!command) { + return this.invalid(command, false); + } + const registered = this.commandMap.get(command); + if (registered) { + const { length } = args; + const candidates = registered.filter(({ argPatterns: { length: count } }) => count === length); + for (const { argPatterns, action } of candidates) { + const parsed: string[] = []; + let matched = true; + if (length) { + for (let i = 0; i < length; i++) { + let matches: RegExpExecArray | null; + if ((matches = argPatterns[i].exec(args[i])) === null) { + matched = false; + break; + } + parsed.push(matches[0]); + } + } + if (!length || matched) { + const result = action(parsed); + const resolve = () => this.valid(`${command} ${parsed.join(" ")}`); + if (result instanceof Promise) { + result.then(resolve); + } else { + resolve(); + } + return; + } + } + this.invalid(command, true); + } else { + this.invalid(command, false); + } + } + +}
\ No newline at end of file diff --git a/src/server/DashSession/Session/utilities/session_config.ts b/src/server/DashSession/Session/utilities/session_config.ts new file mode 100644 index 000000000..b0e65dde4 --- /dev/null +++ b/src/server/DashSession/Session/utilities/session_config.ts @@ -0,0 +1,129 @@ +import { Schema } from "jsonschema"; +import { yellow, red, cyan, green, blue, magenta, Color, grey, gray, white, black } from "colors"; + +const colorPattern = /black|red|green|yellow|blue|magenta|cyan|white|gray|grey/; + +const identifierProperties: Schema = { + type: "object", + properties: { + text: { + type: "string", + minLength: 1 + }, + color: { + type: "string", + pattern: colorPattern + } + } +}; + +const portProperties: Schema = { + type: "number", + minimum: 1024, + maximum: 65535 +}; + +export const configurationSchema: Schema = { + id: "/configuration", + type: "object", + properties: { + showServerOutput: { type: "boolean" }, + ports: { + type: "object", + properties: { + server: portProperties, + socket: portProperties + }, + required: ["server"], + additionalProperties: true + }, + identifiers: { + type: "object", + properties: { + master: identifierProperties, + worker: identifierProperties, + exec: identifierProperties + } + }, + polling: { + type: "object", + additionalProperties: false, + properties: { + intervalSeconds: { + type: "number", + minimum: 1, + maximum: 86400 + }, + route: { + type: "string", + pattern: /\/[a-zA-Z]*/g + }, + failureTolerance: { + type: "number", + minimum: 0, + } + } + }, + } +}; + +type ColorLabel = "yellow" | "red" | "cyan" | "green" | "blue" | "magenta" | "grey" | "gray" | "white" | "black"; + +export const colorMapping: Map<ColorLabel, Color> = new Map([ + ["yellow", yellow], + ["red", red], + ["cyan", cyan], + ["green", green], + ["blue", blue], + ["magenta", magenta], + ["grey", grey], + ["gray", gray], + ["white", white], + ["black", black] +]); + +interface Identifier { + text: string; + color: ColorLabel; +} + +export interface Identifiers { + master: Identifier; + worker: Identifier; + exec: Identifier; +} + +export interface Configuration { + showServerOutput: boolean; + identifiers: Identifiers; + ports: { [description: string]: number }; + polling: { + route: string; + intervalSeconds: number; + failureTolerance: number; + }; +} + +export const defaultConfig: Configuration = { + showServerOutput: false, + identifiers: { + master: { + text: "__monitor__", + color: "yellow" + }, + worker: { + text: "__server__", + color: "magenta" + }, + exec: { + text: "__exec__", + color: "green" + } + }, + ports: { server: 3000 }, + polling: { + route: "/", + intervalSeconds: 30, + failureTolerance: 0 + } +};
\ No newline at end of file diff --git a/src/server/DashSession/Session/utilities/utilities.ts b/src/server/DashSession/Session/utilities/utilities.ts new file mode 100644 index 000000000..eb8de9d7e --- /dev/null +++ b/src/server/DashSession/Session/utilities/utilities.ts @@ -0,0 +1,37 @@ +import { v4 } from "uuid"; + +export namespace Utilities { + + export function guid() { + return v4(); + } + + /** + * At any arbitrary layer of nesting within the configuration objects, any single value that + * is not specified by the configuration is given the default counterpart. If, within an object, + * one peer is given by configuration and two are not, the one is preserved while the two are given + * the default value. + * @returns the composition of all of the assigned objects, much like Object.assign(), but with more + * granularity in the overwriting of nested objects + */ + export function preciseAssign(target: any, ...sources: any[]): any { + for (const source of sources) { + preciseAssignHelper(target, source); + } + return target; + } + + export function preciseAssignHelper(target: any, source: any) { + Array.from(new Set([...Object.keys(target), ...Object.keys(source)])).map(property => { + let targetValue: any, sourceValue: any; + if (sourceValue = source[property]) { + if (typeof sourceValue === "object" && typeof (targetValue = target[property]) === "object") { + preciseAssignHelper(targetValue, sourceValue); + } else { + target[property] = sourceValue; + } + } + }); + } + +}
\ No newline at end of file diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index cb7104757..ea4c26ca2 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -1,11 +1,11 @@ -import { unlinkSync, createWriteStream, readFileSync, rename } from 'fs'; +import { unlinkSync, createWriteStream, readFileSync, rename, writeFile } from 'fs'; import { Utils } from '../Utils'; import * as path from 'path'; import * as sharp from 'sharp'; import request = require('request-promise'); -import { ExifData, ExifImage } from 'exif'; +import { ExifImage } from 'exif'; import { Opt } from '../new_fields/Doc'; -import { AcceptibleMedia } from './SharedMediaTypes'; +import { AcceptibleMedia, Upload } from './SharedMediaTypes'; import { filesDirectory } from '.'; import { File } from 'formidable'; import { basename } from "path"; @@ -14,6 +14,7 @@ import { ParsedPDF } from "../server/PdfTypes"; const parse = require('pdf-parse'); import { Directory, serverPathToFile, clientPathToFile, pathToDirectory } from './ApiManagers/UploadManager'; import { red } from 'colors'; +import { Stream } from 'stream'; const requestImageSize = require("../client/util/request-image-size"); export enum SizeSuffix { @@ -39,13 +40,6 @@ export namespace DashUploadUtils { suffix: SizeSuffix; } - export interface ImageFileResponse { - name: string; - path: string; - type: string; - exif: Opt<DashUploadUtils.EnrichedExifData>; - } - export const Sizes: { [size: string]: Size } = { SMALL: { width: 100, suffix: SizeSuffix.Small }, MEDIUM: { width: 400, suffix: SizeSuffix.Medium }, @@ -59,17 +53,9 @@ export namespace DashUploadUtils { const size = "content-length"; const type = "content-type"; - export interface ImageUploadInformation { - clientAccessPath: string; - serverAccessPaths: { [key: string]: string }; - exifData: EnrichedExifData; - contentSize?: number; - contentType?: string; - } - const { imageFormats, videoFormats, applicationFormats } = AcceptibleMedia; - export async function upload(file: File): Promise<any> { + export async function upload(file: File): Promise<Upload.FileResponse> { const { type, path, name } = file; const types = type.split("/"); @@ -79,37 +65,36 @@ export namespace DashUploadUtils { switch (category) { case "image": if (imageFormats.includes(format)) { - const results = await UploadImage(path, basename(path), format); - return { ...results, name, type }; + const result = await UploadImage(path, basename(path)); + return { source: file, result }; } case "video": if (videoFormats.includes(format)) { - return MoveParsedFile(path, Directory.videos); + return MoveParsedFile(file, Directory.videos); } case "application": if (applicationFormats.includes(format)) { - return UploadPdf(path); + return UploadPdf(file); } } console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`)); - return { clientAccessPath: undefined }; + return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) }; } - async function UploadPdf(absolutePath: string) { - const dataBuffer = readFileSync(absolutePath); + async function UploadPdf(file: File) { + const { path: sourcePath } = file; + const dataBuffer = readFileSync(sourcePath); const result: ParsedPDF = await parse(dataBuffer); - const parsedName = basename(absolutePath); await new Promise<void>((resolve, reject) => { - const textFilename = `${parsedName.substring(0, parsedName.length - 4)}.txt`; + const name = path.basename(sourcePath); + const textFilename = `${name.substring(0, name.length - 4)}.txt`; const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename)); writeStream.write(result.text, error => error ? reject(error) : resolve()); }); - return MoveParsedFile(absolutePath, Directory.pdfs); + return MoveParsedFile(file, Directory.pdfs); } - const generate = (prefix: string, extension: string) => `${prefix}upload_${Utils.GenerateGuid()}.${extension}`; - /** * Uploads an image specified by the @param source to Dash's /public/files/ * directory, and returns information generated during that upload @@ -121,32 +106,20 @@ export namespace DashUploadUtils { * @param {string} prefix is a string prepended to the generated image name in the * event that @param filename is not specified * - * @returns {ImageUploadInformation} This method returns + * @returns {ImageUploadInformation | Error} This method returns * 1) the paths to the uploaded images (plural due to resizing) - * 2) the file name of each of the resized images + * 2) the exif data embedded in the image, or the error explaining why exif couldn't be parsed * 3) the size of the image, in bytes (4432130) * 4) the content type of the image, i.e. image/(jpeg | png | ...) */ - export const UploadImage = async (source: string, filename?: string, format?: string, prefix: string = ""): Promise<ImageUploadInformation> => { + export const UploadImage = async (source: string, filename?: string, prefix: string = ""): Promise<Upload.ImageInformation | Error> => { const metadata = await InspectImage(source); - return UploadInspectedImage(metadata, filename, format, prefix); + if (metadata instanceof Error) { + return metadata; + } + return UploadInspectedImage(metadata, filename || metadata.filename, prefix); }; - export interface InspectionResults { - source: string; - requestable: string; - exifData: EnrichedExifData; - contentSize: number; - contentType: string; - nativeWidth: number; - nativeHeight: number; - } - - export interface EnrichedExifData { - data: ExifData; - error?: string; - } - export async function buildFileDirectories() { const pending = Object.keys(Directory).map(sub => createIfNotExists(`${filesDirectory}/${sub}`)); return Promise.all(pending); @@ -158,13 +131,31 @@ export namespace DashUploadUtils { type: string; } + export interface ImageResizer { + resizer?: sharp.Sharp; + suffix: SizeSuffix; + } + /** * Based on the url's classification as local or remote, gleans * as much information as possible about the specified image * * @param source is the path or url to the image in question */ - export const InspectImage = async (source: string): Promise<InspectionResults> => { + export const InspectImage = async (source: string): Promise<Upload.InspectionResults | Error> => { + let rawMatches: RegExpExecArray | null; + let filename: string | undefined; + if ((rawMatches = /^data:image\/([a-z]+);base64,(.*)/.exec(source)) !== null) { + const [ext, data] = rawMatches.slice(1, 3); + const resolved = filename = `upload_${Utils.GenerateGuid()}.${ext}`; + const error = await new Promise<Error | null>(resolve => { + writeFile(serverPathToFile(Directory.images, resolved), data, "base64", resolve); + }); + if (error !== null) { + return error; + } + source = `http://localhost:1050${clientPathToFile(Directory.images, resolved)}`; + } let resolvedUrl: string; const matches = isLocal().exec(source); if (matches === null) { @@ -187,62 +178,59 @@ export namespace DashUploadUtils { contentType: headers[type], nativeWidth, nativeHeight, + filename, ...results }; }; - export async function MoveParsedFile(absolutePath: string, destination: Directory): Promise<{ clientAccessPath: Opt<string> }> { - return new Promise<{ clientAccessPath: Opt<string> }>(resolve => { - const filename = basename(absolutePath); - const destinationPath = serverPathToFile(destination, filename); - rename(absolutePath, destinationPath, error => { - resolve({ clientAccessPath: error ? undefined : clientPathToFile(destination, filename) }); + export async function MoveParsedFile(file: File, destination: Directory): Promise<Upload.FileResponse> { + const { path: sourcePath } = file; + const name = path.basename(sourcePath); + return new Promise(resolve => { + const destinationPath = serverPathToFile(destination, name); + rename(sourcePath, destinationPath, error => { + resolve({ + source: file, + result: error ? error : { + accessPaths: { + agnostic: getAccessPaths(destination, name) + } + } + }); }); }); } - export const UploadInspectedImage = async (metadata: InspectionResults, filename?: string, format?: string, prefix = ""): Promise<ImageUploadInformation> => { + function getAccessPaths(directory: Directory, fileName: string) { + return { + client: clientPathToFile(directory, fileName), + server: serverPathToFile(directory, fileName) + }; + } + + export const UploadInspectedImage = async (metadata: Upload.InspectionResults, filename?: string, prefix = "", cleanUp = true): Promise<Upload.ImageInformation> => { const { requestable, source, ...remaining } = metadata; - const extension = remaining.contentType.toLowerCase().split("/")[1]; //format || sanitizeExtension(requestable || resolved); - const resolved = filename || generate(prefix, extension); - const information: ImageUploadInformation = { - clientAccessPath: clientPathToFile(Directory.images, resolved), - serverAccessPaths: {}, - ...remaining + const resolved = filename || `${prefix}upload_${Utils.GenerateGuid()}.${remaining.contentType.split("/")[1].toLowerCase()}`; + const { images } = Directory; + const information: Upload.ImageInformation = { + accessPaths: { + agnostic: getAccessPaths(images, resolved) + }, + ...metadata }; - const { pngs, jpgs } = AcceptibleMedia; - return new Promise<ImageUploadInformation>(async (resolve, reject) => { - const resizers = [ - { resizer: sharp().rotate(), suffix: SizeSuffix.Original }, - ...Object.values(Sizes).map(size => ({ - resizer: sharp().resize(size.width, undefined, { withoutEnlargement: true }).rotate(), - suffix: size.suffix - })) - ]; - if (pngs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.png()); - } else if (jpgs.includes(extension)) { - resizers.forEach(element => element.resizer = element.resizer.jpeg()); - } - for (const { resizer, suffix } of resizers) { - await new Promise<void>(resolve => { - const filename = InjectSize(resolved, suffix); - information.serverAccessPaths[suffix] = serverPathToFile(Directory.images, filename); - request(requestable).pipe(resizer).pipe(createWriteStream(serverPathToFile(Directory.images, filename))) - .on('close', resolve) - .on('error', reject); - }); - } - if (isLocal().test(source)) { - unlinkSync(source); - } - resolve(information); - }); + const writtenFiles = await outputResizedImages(() => request(requestable), resolved, pathToDirectory(Directory.images)); + for (const suffix of Object.keys(writtenFiles)) { + information.accessPaths[suffix] = getAccessPaths(images, writtenFiles[suffix]); + } + if (isLocal().test(source) && cleanUp) { + unlinkSync(source); + } + return information; }; - const parseExifData = async (source: string): Promise<EnrichedExifData> => { + const parseExifData = async (source: string): Promise<Upload.EnrichedExifData> => { const image = await request.get(source, { encoding: null }); - return new Promise<EnrichedExifData>(resolve => { + return new Promise(resolve => { new ExifImage({ image }, (error, data) => { let reason: Opt<string> = undefined; if (error) { @@ -253,4 +241,56 @@ export namespace DashUploadUtils { }); }; + const { pngs, jpgs, webps, tiffs } = AcceptibleMedia; + const pngOptions = { + compressionLevel: 9, + adaptiveFiltering: true, + force: true + }; + + export async function outputResizedImages(streamProvider: () => Stream | Promise<Stream>, outputFileName: string, outputDirectory: string) { + const writtenFiles: { [suffix: string]: string } = {}; + for (const { resizer, suffix } of resizers(path.extname(outputFileName))) { + const outputPath = path.resolve(outputDirectory, writtenFiles[suffix] = InjectSize(outputFileName, suffix)); + await new Promise<void>(async (resolve, reject) => { + const source = streamProvider(); + let readStream: Stream; + if (source instanceof Promise) { + readStream = await source; + } else { + readStream = source; + } + if (resizer) { + readStream = readStream.pipe(resizer.withMetadata()); + } + readStream.pipe(createWriteStream(outputPath)).on("close", resolve).on("error", reject); + }); + } + return writtenFiles; + } + + function resizers(ext: string): DashUploadUtils.ImageResizer[] { + return [ + { suffix: SizeSuffix.Original }, + ...Object.values(DashUploadUtils.Sizes).map(({ suffix, width }) => { + let initial: sharp.Sharp | undefined = sharp().resize(width, undefined, { withoutEnlargement: true }); + if (pngs.includes(ext)) { + initial = initial.png(pngOptions); + } else if (jpgs.includes(ext)) { + initial = initial.jpeg(); + } else if (webps.includes(ext)) { + initial = initial.webp(); + } else if (tiffs.includes(ext)) { + initial = initial.tiff(); + } else if (ext === ".gif") { + initial = undefined; + } + return { + resizer: initial, + suffix + }; + }) + ]; + } + }
\ No newline at end of file diff --git a/src/server/Message.ts b/src/server/Message.ts index 02ca2ceda..81f63656b 100644 --- a/src/server/Message.ts +++ b/src/server/Message.ts @@ -2,6 +2,7 @@ import { Utils } from "../Utils"; import { Point } from "../pen-gestures/ndollar"; import { Doc } from "../new_fields/Doc"; import { Image } from "canvas"; +import { AnalysisResult, ImportResults } from "../scraping/buxton/final/BuxtonImporter"; export class Message<T> { private _name: string; @@ -69,6 +70,11 @@ export interface MobileDocumentUploadContent { readonly docId: string; } +export interface RoomMessage { + readonly message: string; + readonly room: string; +} + export namespace MessageStore { export const Foo = new Message<string>("Foo"); export const Bar = new Message<string>("Bar"); @@ -78,6 +84,9 @@ export namespace MessageStore { export const GetDocument = new Message<string>("Get Document"); export const DeleteAll = new Message<any>("Delete All"); export const ConnectionTerminated = new Message<string>("Connection Terminated"); + export const BeginBuxtonImport = new Message<string>("Begin Buxton Import"); + export const BuxtonDocumentResult = new Message<AnalysisResult>("Buxton Document Result"); + export const BuxtonImportComplete = new Message<ImportResults>("Buxton Import Complete"); export const GesturePoints = new Message<GestureContent>("Gesture Points"); export const MobileInkOverlayTrigger = new Message<MobileInkOverlayContent>("Trigger Mobile Ink Overlay"); @@ -91,5 +100,4 @@ export namespace MessageStore { export const YoutubeApiQuery = new Message<YoutubeQueryInput>("Youtube Api Query"); export const DeleteField = new Message<string>("Delete field"); export const DeleteFields = new Message<string[]>("Delete fields"); - export const AnalyzeInk = new Message<string>("Analyze Ink"); } diff --git a/src/server/SharedMediaTypes.ts b/src/server/SharedMediaTypes.ts index 8d0f441f0..185e787cc 100644 --- a/src/server/SharedMediaTypes.ts +++ b/src/server/SharedMediaTypes.ts @@ -1,8 +1,49 @@ +import { ExifData } from 'exif'; +import { File } from 'formidable'; + export namespace AcceptibleMedia { export const gifs = [".gif"]; export const pngs = [".png"]; export const jpgs = [".jpg", ".jpeg"]; - export const imageFormats = [...pngs, ...jpgs, ...gifs]; + export const webps = [".webp"]; + export const tiffs = [".tiff"]; + export const imageFormats = [...pngs, ...jpgs, ...gifs, ...webps, ...tiffs]; export const videoFormats = [".mov", ".mp4"]; export const applicationFormats = [".pdf"]; +} + +export namespace Upload { + + export function isImageInformation(uploadResponse: Upload.FileInformation): uploadResponse is Upload.ImageInformation { + return "nativeWidth" in uploadResponse; + } + + export interface FileInformation { + accessPaths: AccessPathInfo; + } + + export type FileResponse<T extends FileInformation = FileInformation> = { source: File, result: T | Error }; + + export type ImageInformation = FileInformation & InspectionResults; + + export interface AccessPathInfo { + [suffix: string]: { client: string, server: string }; + } + + export interface InspectionResults { + source: string; + requestable: string; + exifData: EnrichedExifData; + contentSize: number; + contentType: string; + nativeWidth: number; + nativeHeight: number; + filename?: string; + } + + export interface EnrichedExifData { + data: ExifData; + error?: string; + } + }
\ No newline at end of file diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts index 66f7019a4..c5dc22912 100644 --- a/src/server/Websocket/Websocket.ts +++ b/src/server/Websocket/Websocket.ts @@ -1,5 +1,5 @@ import { Utils } from "../../Utils"; -import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent, MobileDocumentUploadContent } from "../Message"; +import { MessageStore, Transferable, Types, Diff, YoutubeQueryInput, YoutubeQueryTypes, GestureContent, MobileInkOverlayContent, UpdateMobileInkOverlayPositionContent, MobileDocumentUploadContent, RoomMessage } from "../Message"; import { Client } from "../Client"; import { Socket } from "socket.io"; import { Database } from "../database"; @@ -10,16 +10,9 @@ import { GoogleCredentialsLoader } from "../credentials/CredentialsLoader"; import { logPort } from "../ActionUtilities"; import { timeMap } from "../ApiManagers/UserManager"; import { green } from "colors"; -import { Image } from "canvas"; -import { write, createWriteStream } from "fs"; import { serverPathToFile, Directory } from "../ApiManagers/UploadManager"; -const tesseract = require("node-tesseract-ocr"); -const config = { - lang: "eng", - oem: 1, - psm: 8 -}; -const imageDataUri = require('image-data-uri'); +import { networkInterfaces } from "os"; +import executeImport from "../../scraping/buxton/final/BuxtonImporter"; export namespace WebSocket { @@ -28,6 +21,7 @@ export namespace WebSocket { export const socketMap = new Map<SocketIO.Socket, string>(); export let disconnect: Function; + export async function start(isRelease: boolean) { await preliminaryFunctions(); initialize(isRelease); @@ -35,7 +29,6 @@ export namespace WebSocket { async function preliminaryFunctions() { } - function initialize(isRelease: boolean) { const endpoint = io(); endpoint.on("connection", function (socket: Socket) { @@ -49,6 +42,54 @@ export namespace WebSocket { next(); }); + // convenience function to log server messages on the client + function log(message?: any, ...optionalParams: any[]) { + socket.emit('log', ['Message from server:', message, ...optionalParams]); + } + + socket.on('message', function (message, room) { + console.log('Client said: ', message); + socket.in(room).emit('message', message); + }); + + socket.on('create or join', function (room) { + console.log('Received request to create or join room ' + room); + + var clientsInRoom = socket.adapter.rooms[room]; + var numClients = clientsInRoom ? Object.keys(clientsInRoom.sockets).length : 0; + console.log('Room ' + room + ' now has ' + numClients + ' client(s)'); + + if (numClients === 0) { + socket.join(room); + console.log('Client ID ' + socket.id + ' created room ' + room); + socket.emit('created', room, socket.id); + + } else if (numClients === 1) { + console.log('Client ID ' + socket.id + ' joined room ' + room); + socket.in(room).emit('join', room); + socket.join(room); + socket.emit('joined', room, socket.id); + socket.in(room).emit('ready'); + } else { // max two clients + socket.emit('full', room); + } + }); + + socket.on('ipaddr', function () { + var ifaces = networkInterfaces(); + for (var dev in ifaces) { + ifaces[dev].forEach(function (details) { + if (details.family === 'IPv4' && details.address !== '127.0.0.1') { + socket.emit('ipaddr', details.address); + } + }); + } + }); + + socket.on('bye', function () { + console.log('received bye'); + }); + Utils.Emit(socket, MessageStore.Foo, "handshooken"); Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); @@ -61,7 +102,6 @@ export namespace WebSocket { Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField); Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery); - Utils.AddServerHandlerCallback(socket, MessageStore.AnalyzeInk, RecognizeImage); Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff)); Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id)); Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids)); @@ -71,6 +111,12 @@ export namespace WebSocket { Utils.AddServerHandler(socket, MessageStore.MobileDocumentUpload, content => processMobileDocumentUpload(socket, content)); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField); Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields); + Utils.AddServerHandler(socket, MessageStore.BeginBuxtonImport, () => { + executeImport( + deviceOrError => Utils.Emit(socket, MessageStore.BuxtonDocumentResult, deviceOrError), + results => Utils.Emit(socket, MessageStore.BuxtonImportComplete, results) + ); + }); disconnect = () => { socket.broadcast.emit("connection_terminated", Date.now()); @@ -99,17 +145,6 @@ export namespace WebSocket { socket.broadcast.emit("receiveMobileDocumentUpload", content); } - async function RecognizeImage([query, callback]: [string, (result: any) => any]) { - const path = serverPathToFile(Directory.images, "handwriting.jpg"); - imageDataUri.outputFile(query, path).then((savedName: string) => { - console.log("saved " + savedName); - const remadePath = path.split("\\").join("\\\\"); - tesseract.recognize(remadePath, config) - .then(callback) - .catch(console.log); - }); - } - function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any[]) => void]) { const { ProjectCredentials } = GoogleCredentialsLoader; switch (query.type) { diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index 329107a71..0f75833ee 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -318,13 +318,14 @@ export namespace GoogleApiServerUtils { */ async function retrieveCredentials(userId: string): Promise<{ credentials: Opt<Credentials>, refreshed: boolean }> { let credentials: Opt<Credentials> = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); - const refreshed = false; + let refreshed = false; if (!credentials) { return { credentials: undefined, refreshed }; } // check for token expiry if (credentials.expiry_date! <= new Date().getTime()) { credentials = await refreshAccessToken(credentials, userId); + refreshed = true; } return { credentials, refreshed }; } diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 8ae63caa3..d305eed0a 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -84,6 +84,7 @@ export namespace GooglePhotosUploadUtils { if (!DashUploadUtils.validateExtension(url)) { return undefined; } + const body = await request(url, { encoding: null }); // returns a readable stream with the unencoded binary image data const parameters = { method: 'POST', uri: prepend('uploads'), @@ -92,7 +93,7 @@ export namespace GooglePhotosUploadUtils { 'X-Goog-Upload-File-Name': filename || path.basename(url), 'X-Goog-Upload-Protocol': 'raw' }, - body: await request(url, { encoding: null }) // returns a readable stream with the unencoded binary image data + body }; return new Promise((resolve, reject) => request(parameters, (error, _response, body) => { if (error) { diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts index a7cc6d3e9..dc63f8a89 100644 --- a/src/server/authentication/models/current_user_utils.ts +++ b/src/server/authentication/models/current_user_utils.ts @@ -16,6 +16,10 @@ import { DragManager } from "../../../client/util/DragManager"; import { InkingControl } from "../../../client/views/InkingControl"; import { Scripting } from "../../../client/util/Scripting"; import { CollectionViewType } from "../../../client/views/collections/CollectionView"; +import { makeTemplate } from "../../../client/util/DropConverter"; +import { RichTextField } from "../../../new_fields/RichTextField"; +import { PrefetchProxy } from "../../../new_fields/Proxy"; +import { FormattedTextBox } from "../../../client/views/nodes/FormattedTextBox"; export class CurrentUserUtils { private static curr_id: string; @@ -32,31 +36,39 @@ export class CurrentUserUtils { @observable public static GuestWorkspace: Doc | undefined; @observable public static GuestMobile: Doc | undefined; - // a default set of note types .. not being used yet... - static setupNoteTypes(doc: Doc) { - return [ - Docs.Create.TextDocument("", { title: "Note", backgroundColor: "yellow", isTemplateDoc: true }), - Docs.Create.TextDocument("", { title: "Idea", backgroundColor: "pink", isTemplateDoc: true }), - Docs.Create.TextDocument("", { title: "Topic", backgroundColor: "lightBlue", isTemplateDoc: true }), - Docs.Create.TextDocument("", { title: "Person", backgroundColor: "lightGreen", isTemplateDoc: true }), - Docs.Create.TextDocument("", { title: "Todo", backgroundColor: "orange", isTemplateDoc: true }) + static setupDefaultDocTemplates(doc: Doc, buttons?: string[]) { + const taskStatusValues = [ + Docs.Create.TextDocument("todo", { title: "todo", _backgroundColor: "blue", color: "white" }), + Docs.Create.TextDocument("in progress", { title: "in progress", _backgroundColor: "yellow", color: "black" }), + Docs.Create.TextDocument("completed", { title: "completed", _backgroundColor: "green", color: "white" }) ]; + const noteTemplates = [ + Docs.Create.TextDocument("", { title: "Note", isTemplateDoc: true, backgroundColor: "yellow" }), + Docs.Create.TextDocument("", { title: "Idea", isTemplateDoc: true, backgroundColor: "pink" }), + Docs.Create.TextDocument("", { title: "Topic", isTemplateDoc: true, backgroundColor: "lightBlue" }), + Docs.Create.TextDocument("", { title: "Person", isTemplateDoc: true, backgroundColor: "lightGreen" }), + Docs.Create.TextDocument("", { title: "Todo", isTemplateDoc: true, backgroundColor: "orange", _autoHeight: false, _height: 100, _showCaption: "caption" }) + ]; + doc.fieldTypes = Docs.Create.TreeDocument([], { title: "field enumerations" }); + Doc.enumeratedTextTemplate(Doc.GetProto(noteTemplates[4]), FormattedTextBox.LayoutString("Todo"), "taskStatus", taskStatusValues); + doc.noteTypes = new PrefetchProxy(Docs.Create.TreeDocument(noteTemplates.map(nt => makeTemplate(nt) ? nt : nt), { title: "Note Types", _height: 75 })); } // setup the "creator" buttons for the sidebar-- eg. the default set of draggable document creation tools static setupCreatorButtons(doc: Doc, buttons?: string[]) { - const notes = CurrentUserUtils.setupNoteTypes(doc); - doc.noteTypes = Docs.Create.TreeDocument(notes, { title: "Note Types", _height: 75 }); + const emptyPresentation = Docs.Create.PresDocument(new List<Doc>(), { title: "Presentation", _viewType: CollectionViewType.Stacking, _LODdisable: true, _chromeStatus: "replaced", _showTitle: "title", boxShadow: "0 0" }); + const emptyCollection = Docs.Create.FreeformDocument([], { _nativeWidth: undefined, _nativeHeight: undefined, _LODdisable: true, _width: 150, _height: 100, title: "freeform" }); doc.activePen = doc; const docProtoData: { title: string, icon: string, drag?: string, ignoreClick?: boolean, click?: string, ischecked?: string, activePen?: Doc, backgroundColor?: string, dragFactory?: Doc }[] = [ - { title: "collection", icon: "folder", ignoreClick: true, drag: 'Docs.Create.FreeformDocument([], { _nativeWidth: undefined, _nativeHeight: undefined, _LODdisable: true, _width: 150, _height: 100, title: "freeform" })' }, + { title: "collection", icon: "folder", click: 'openOnRight(getCopy(this.dragFactory, true))', drag: 'getCopy(this.dragFactory, true)', dragFactory: emptyCollection }, { title: "preview", icon: "expand", ignoreClick: true, drag: 'Docs.Create.DocumentDocument(ComputedField.MakeFunction("selectedDocs(this,true,[_last_])?.[0]"), { _width: 250, _height: 250, title: "container" })' }, - { title: "todo item", icon: "check", ignoreClick: true, drag: 'getCopy(this.dragFactory, true)', dragFactory: notes[notes.length - 1] }, { title: "web page", icon: "globe-asia", ignoreClick: true, drag: 'Docs.Create.WebDocument("https://en.wikipedia.org/wiki/Hedgehog", {_width: 300, _height: 300, title: "New Webpage" })' }, { title: "cat image", icon: "cat", ignoreClick: true, drag: 'Docs.Create.ImageDocument("https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg", { _width: 200, title: "an image of a cat" })' }, + { title: "buxton", icon: "cloud-upload-alt", ignoreClick: true, drag: "Docs.Create.Buxton()" }, + { title: "webcam", icon: "video", ignoreClick: true, drag: 'Docs.Create.WebCamDocument("", { width: 400, height: 400, title: "a test cam" })' }, { title: "record", icon: "microphone", ignoreClick: true, drag: `Docs.Create.AudioDocument("${nullAudio}", { _width: 200, title: "ready to record audio" })` }, { title: "clickable button", icon: "bolt", ignoreClick: true, drag: 'Docs.Create.ButtonDocument({ _width: 150, _height: 50, title: "Button" })' }, - { title: "presentation", icon: "tv", ignoreClick: true, drag: `Doc.UserDoc().curPresentation = Docs.Create.PresDocument(new List<Doc>(), { _width: 200, _height: 500, _viewType: ${CollectionViewType.Stacking}, title: "a presentation trail" })` }, + { title: "presentation", icon: "tv", click: 'openOnRight(Doc.UserDoc().curPresentation = getCopy(this.dragFactory, true))', drag: `Doc.UserDoc().curPresentation = getCopy(this.dragFactory,true)`, dragFactory: emptyPresentation }, { title: "import folder", icon: "cloud-upload-alt", ignoreClick: true, drag: 'Docs.Create.DirectoryImportDocument({ title: "Directory Import", _width: 400, _height: 400 })' }, { title: "mobile view", icon: "phone", ignoreClick: true, drag: 'Doc.UserDoc().activeMobile' }, { title: "use pen", icon: "pen-nib", click: 'activatePen(this.activePen.pen = sameDocs(this.activePen.pen, this) ? undefined : this,2, this.backgroundColor)', backgroundColor: "blue", ischecked: `sameDocs(this.activePen.pen, this)`, activePen: doc }, @@ -67,7 +79,7 @@ export class CurrentUserUtils { { title: "use drag", icon: "mouse-pointer", click: 'deactivateInk();this.activePen.pen = this;', ischecked: `sameDocs(this.activePen.pen, this)`, backgroundColor: "white", activePen: doc }, ]; return docProtoData.filter(d => !buttons || !buttons.includes(d.title)).map(data => Docs.Create.FontIconDocument({ - _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, _dropAction: data.click ? "copy" : undefined, title: data.title, icon: data.icon, ignoreClick: data.ignoreClick, + _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, dropAction: data.click ? "copy" : undefined, title: data.title, icon: data.icon, ignoreClick: data.ignoreClick, onDragStart: data.drag ? ScriptField.MakeFunction(data.drag) : undefined, onClick: data.click ? ScriptField.MakeScript(data.click) : undefined, ischecked: data.ischecked ? ComputedField.MakeFunction(data.ischecked) : undefined, activePen: data.activePen, backgroundColor: data.backgroundColor, removeDropProperties: new List<string>(["dropAction"]), dragFactory: data.dragFactory, @@ -108,7 +120,7 @@ export class CurrentUserUtils { // { title: "upload", icon: "upload", click: 'uploadImageMobile();', backgroundColor: "cyan" }, ]; return docProtoData.filter(d => !buttons || !buttons.includes(d.title)).map(data => Docs.Create.FontIconDocument({ - _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, _dropAction: data.click ? "copy" : undefined, title: data.title, icon: data.icon, ignoreClick: data.ignoreClick, + _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, dropAction: data.click ? "copy" : undefined, title: data.title, icon: data.icon, ignoreClick: data.ignoreClick, onDragStart: data.drag ? ScriptField.MakeFunction(data.drag) : undefined, onClick: data.click ? ScriptField.MakeScript(data.click) : undefined, ischecked: data.ischecked ? ComputedField.MakeFunction(data.ischecked) : undefined, activePen: data.activePen, backgroundColor: data.backgroundColor, removeDropProperties: new List<string>(["dropAction"]), dragFactory: data.dragFactory, @@ -124,7 +136,7 @@ export class CurrentUserUtils { { title: "ignore gestures", icon: "signature", pointerUp: "setToolglass('none')", pointerDown: "setToolglass('ignoregesture')", backgroundColor: "green", ischecked: `sameDocs(this.activePen.pen, this)`, activePen: doc }, ]; return docProtoData.map(data => Docs.Create.FontIconDocument({ - _nativeWidth: 10, _nativeHeight: 10, _width: 10, _height: 10, _dropAction: data.pointerDown ? "copy" : undefined, title: data.title, icon: data.icon, ignoreClick: data.ignoreClick, + _nativeWidth: 10, _nativeHeight: 10, _width: 10, _height: 10, dropAction: data.pointerDown ? "copy" : undefined, title: data.title, icon: data.icon, ignoreClick: data.ignoreClick, onDragStart: data.drag ? ScriptField.MakeFunction(data.drag) : undefined, clipboard: data.clipboard, onPointerUp: data.pointerUp ? ScriptField.MakeScript(data.pointerUp) : undefined, onPointerDown: data.pointerDown ? ScriptField.MakeScript(data.pointerDown) : undefined, @@ -136,9 +148,9 @@ export class CurrentUserUtils { static setupThumbDoc(userDoc: Doc) { if (!userDoc.thumbDoc) { const thumbDoc = Docs.Create.LinearDocument(CurrentUserUtils.setupThumbButtons(userDoc), { - _width: 100, _height: 50, ignoreClick: true, lockedPosition: true, _chromeStatus: "disabled", title: "buttons", _autoHeight: true, _yMargin: 5, isExpanded: true, backgroundColor: "white" + _width: 100, _height: 50, ignoreClick: true, lockedPosition: true, _chromeStatus: "disabled", title: "buttons", _autoHeight: true, _yMargin: 5, linearViewIsExpanded: true, backgroundColor: "white" }); - thumbDoc.inkToTextDoc = Docs.Create.LinearDocument([], { _width: 300, _height: 25, _autoHeight: true, _chromeStatus: "disabled", isExpanded: true, flexDirection: "column" }); + thumbDoc.inkToTextDoc = Docs.Create.LinearDocument([], { _width: 300, _height: 25, _autoHeight: true, _chromeStatus: "disabled", linearViewIsExpanded: true, flexDirection: "column" }); userDoc.thumbDoc = thumbDoc; } return Cast(userDoc.thumbDoc, Doc); @@ -176,11 +188,11 @@ export class CurrentUserUtils { }); // setup a color picker const color = Docs.Create.ColorDocument({ - title: "color picker", _width: 300, _dropAction: "alias", forceActive: true, removeDropProperties: new List<string>(["dropAction", "forceActive"]) + title: "color picker", _width: 300, dropAction: "alias", forceActive: true, removeDropProperties: new List<string>(["dropAction", "forceActive"]) }); return Docs.Create.ButtonDocument({ - _width: 35, _height: 25, backgroundColor: "lightgrey", color: "rgb(34, 34, 34)", title: "Tools", fontSize: 10, targetContainer: sidebarContainer, + _width: 35, _height: 25, title: "Tools", fontSize: 10, targetContainer: sidebarContainer, letterSpacing: "0px", textTransform: "unset", borderRounding: "5px 5px 0px 0px", boxShadow: "3px 3px 0px rgb(34, 34, 34)", sourcePanel: Docs.Create.StackingDocument([dragCreators, color], { _width: 500, lockedPosition: true, _chromeStatus: "disabled", title: "tools stack" @@ -193,23 +205,23 @@ export class CurrentUserUtils { static setupLibraryPanel(sidebarContainer: Doc, doc: Doc) { // setup workspaces library item doc.workspaces = Docs.Create.TreeDocument([], { - title: "WORKSPACES", _height: 100, forceActive: true, boxShadow: "0 0", lockedPosition: true, backgroundColor: "#eeeeee" + title: "WORKSPACES", _height: 100, forceActive: true, boxShadow: "0 0", lockedPosition: true, }); doc.documents = Docs.Create.TreeDocument([], { - title: "DOCUMENTS", _height: 42, forceActive: true, boxShadow: "0 0", preventTreeViewOpen: true, lockedPosition: true, backgroundColor: "#eeeeee" + title: "DOCUMENTS", _height: 42, forceActive: true, boxShadow: "0 0", treeViewPreventOpen: true, lockedPosition: true, }); // setup Recently Closed library item doc.recentlyClosed = Docs.Create.TreeDocument([], { - title: "RECENTLY CLOSED", _height: 75, forceActive: true, boxShadow: "0 0", preventTreeViewOpen: true, lockedPosition: true, backgroundColor: "#eeeeee" + title: "RECENTLY CLOSED", _height: 75, forceActive: true, boxShadow: "0 0", treeViewPreventOpen: true, lockedPosition: true, }); return Docs.Create.ButtonDocument({ - _width: 50, _height: 25, backgroundColor: "lightgrey", color: "rgb(34, 34, 34)", title: "Library", fontSize: 10, + _width: 50, _height: 25, title: "Library", fontSize: 10, letterSpacing: "0px", textTransform: "unset", borderRounding: "5px 5px 0px 0px", boxShadow: "3px 3px 0px rgb(34, 34, 34)", - sourcePanel: Docs.Create.TreeDocument([doc.workspaces as Doc, doc.documents as Doc, doc.recentlyClosed as Doc], { - title: "Library", _xMargin: 5, _yMargin: 5, _gridGap: 5, forceActive: true, _dropAction: "alias", lockedPosition: true, boxShadow: "0 0", + sourcePanel: Docs.Create.TreeDocument([doc.workspaces as Doc, doc.documents as Doc, Docs.Prototypes.MainLinkDocument(), doc, doc.recentlyClosed as Doc], { + title: "Library", _xMargin: 5, _yMargin: 5, _gridGap: 5, forceActive: true, dropAction: "alias", lockedPosition: true, boxShadow: "0 0", }), targetContainer: sidebarContainer, onClick: ScriptField.MakeScript("this.targetContainer.proto = this.sourcePanel;") @@ -219,7 +231,7 @@ export class CurrentUserUtils { // setup the Search button which will display the search panel. static setupSearchPanel(sidebarContainer: Doc) { return Docs.Create.ButtonDocument({ - _width: 50, _height: 25, backgroundColor: "lightgrey", color: "rgb(34, 34, 34)", title: "Search", fontSize: 10, + _width: 50, _height: 25, title: "Search", fontSize: 10, letterSpacing: "0px", textTransform: "unset", borderRounding: "5px 5px 0px 0px", boxShadow: "3px 3px 0px rgb(34, 34, 34)", sourcePanel: Docs.Create.QueryDocument({ title: "search stack", ignoreClick: true @@ -243,21 +255,39 @@ export class CurrentUserUtils { // Finally, setup the list of buttons to display in the sidebar doc.sidebarButtons = Docs.Create.StackingDocument([doc.SearchBtn as Doc, doc.LibraryBtn as Doc, doc.ToolsBtn as Doc], { _width: 500, _height: 80, boxShadow: "0 0", sectionFilter: "title", hideHeadings: true, ignoreClick: true, - backgroundColor: "rgb(100, 100, 100)", _chromeStatus: "disabled", title: "library stack", - _yMargin: 10, + _chromeStatus: "disabled", title: "library stack", backgroundColor: "dimGray", }); } /// sets up the default list of buttons to be shown in the expanding button menu at the bottom of the Dash window static setupExpandingButtons(doc: Doc) { + const slideTemplate = Docs.Create.MultirowDocument( + [ + Docs.Create.MulticolumnDocument([], { title: "data", _height: 200 }), + Docs.Create.TextDocument("", { title: "contents", _height: 100 }) + ], + { _width: 400, _height: 300, title: "slideView", _chromeStatus: "disabled", _xMargin: 3, _yMargin: 3, _autoHeight: false }); + slideTemplate.isTemplateDoc = makeTemplate(slideTemplate); + const descriptionTemplate = Docs.Create.TextDocument("", { title: "descriptionView", _height: 100, _showTitle: "title" }); + Doc.GetProto(descriptionTemplate).layout = FormattedTextBox.LayoutString("description"); + descriptionTemplate.isTemplateDoc = makeTemplate(descriptionTemplate); + + const iconDoc = Docs.Create.TextDocument("", { title: "icon", _width: 150, _height: 30, isTemplateDoc: true, onClick: ScriptField.MakeScript("setNativeView(this)") }); + Doc.GetProto(iconDoc).data = new RichTextField('{"doc":{"type":"doc","content":[{"type":"paragraph","attrs":{"align":null,"color":null,"id":null,"indent":null,"inset":null,"lineSpacing":null,"paddingBottom":null,"paddingTop":null},"content":[{"type":"dashField","attrs":{"fieldKey":"title","docid":""}}]}]},"selection":{"type":"text","anchor":2,"head":2},"storedMarks":[]}', ""); + doc.isTemplateDoc = makeTemplate(iconDoc); + doc.iconView = new PrefetchProxy(iconDoc); + doc.undoBtn = Docs.Create.FontIconDocument( - { _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, _dropAction: "alias", onClick: ScriptField.MakeScript("undo()"), removeDropProperties: new List<string>(["dropAction"]), title: "undo button", icon: "undo-alt" }); + { _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, dropAction: "alias", onClick: ScriptField.MakeScript("undo()"), removeDropProperties: new List<string>(["dropAction"]), title: "undo button", icon: "undo-alt" }); doc.redoBtn = Docs.Create.FontIconDocument( - { _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, _dropAction: "alias", onClick: ScriptField.MakeScript("redo()"), removeDropProperties: new List<string>(["dropAction"]), title: "redo button", icon: "redo-alt" }); - - doc.expandingButtons = Docs.Create.LinearDocument([doc.undoBtn as Doc, doc.redoBtn as Doc], { + { _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, dropAction: "alias", onClick: ScriptField.MakeScript("redo()"), removeDropProperties: new List<string>(["dropAction"]), title: "redo button", icon: "redo-alt" }); + doc.slidesBtn = Docs.Create.FontIconDocument( + { _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, dropAction: "alias", onDragStart: ScriptField.MakeFunction('getCopy(this.dragFactory, true)'), dragFactory: slideTemplate, removeDropProperties: new List<string>(["dropAction"]), title: "presentation slide", icon: "sticky-note" }); + doc.descriptionBtn = Docs.Create.FontIconDocument( + { _nativeWidth: 100, _nativeHeight: 100, _width: 100, _height: 100, dropAction: "alias", onDragStart: ScriptField.MakeFunction('getCopy(this.dragFactory, true)'), dragFactory: descriptionTemplate, removeDropProperties: new List<string>(["dropAction"]), title: "description view", icon: "sticky-note" }); + doc.expandingButtons = Docs.Create.LinearDocument([doc.undoBtn as Doc, doc.redoBtn as Doc, doc.slidesBtn as Doc, doc.descriptionBtn as Doc], { title: "expanding buttons", _gridGap: 5, _xMargin: 5, _yMargin: 5, _height: 42, _width: 100, boxShadow: "0 0", - backgroundColor: "black", preventTreeViewOpen: true, forceActive: true, lockedPosition: true, + backgroundColor: "black", treeViewPreventOpen: true, forceActive: true, lockedPosition: true, dropConverter: ScriptField.MakeScript("convertToButtons(dragData)", { dragData: DragManager.DocumentDragData.name }) }); } @@ -265,13 +295,12 @@ export class CurrentUserUtils { // sets up the default set of documents to be shown in the Overlay layer static setupOverlays(doc: Doc) { doc.overlays = Docs.Create.FreeformDocument([], { title: "Overlays", backgroundColor: "#aca3a6" }); - doc.linkFollowBox = Docs.Create.LinkFollowBoxDocument({ x: 250, y: 20, _width: 500, _height: 370, title: "Link Follower" }); - Doc.AddDocToList(doc.overlays as Doc, "data", doc.linkFollowBox as Doc); } // the initial presentation Doc to use static setupDefaultPresentation(doc: Doc) { - doc.curPresentation = Docs.Create.PresDocument(new List<Doc>(), { title: "Presentation", _viewType: CollectionViewType.Stacking, boxShadow: "0 0" }); + doc.presentationTemplate = new PrefetchProxy(Docs.Create.PresElementBoxDocument({ backgroundColor: "transparent", _xMargin: 5, _height: 46, isTemplateDoc: true, isTemplateForField: "data" })); + doc.curPresentation = Docs.Create.PresDocument(new List<Doc>(), { title: "Presentation", _viewType: CollectionViewType.Stacking, _LODdisable: true, _chromeStatus: "replaced", _showTitle: "title", boxShadow: "0 0" }); } static setupMobileUploads(doc: Doc) { @@ -281,6 +310,7 @@ export class CurrentUserUtils { static updateUserDocument(doc: Doc) { doc.title = Doc.CurrentUserEmail; new InkingControl(); + (doc.noteTypes === undefined) && CurrentUserUtils.setupDefaultDocTemplates(doc); (doc.optionalRightCollection === undefined) && CurrentUserUtils.setupMobileUploads(doc); (doc.overlays === undefined) && CurrentUserUtils.setupOverlays(doc); (doc.expandingButtons === undefined) && CurrentUserUtils.setupExpandingButtons(doc); @@ -309,6 +339,15 @@ export class CurrentUserUtils { return doc; } + public static IsDocPinned(doc: Doc) { + //add this new doc to props.Document + const curPres = Cast(CurrentUserUtils.UserDocument.curPresentation, Doc) as Doc; + if (curPres) { + return DocListCast(curPres.data).findIndex((val) => Doc.AreProtosEqual(val, doc)) !== -1; + } + return false; + } + public static async loadCurrentUser() { return rp.get(Utils.prepend("/getCurrentUser")).then(response => { if (response) { diff --git a/src/server/database.ts b/src/server/database.ts index 83ce865c6..055f04c49 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -2,12 +2,12 @@ import * as mongodb from 'mongodb'; import { Transferable } from './Message'; import { Opt } from '../new_fields/Doc'; import { Utils, emptyFunction } from '../Utils'; -import { DashUploadUtils } from './DashUploadUtils'; import { Credentials } from 'google-auth-library'; import { GoogleApiServerUtils } from './apis/google/GoogleApiServerUtils'; import { IDatabase } from './IDatabase'; import { MemoryDatabase } from './MemoryDatabase'; import * as mongoose from 'mongoose'; +import { Upload } from './SharedMediaTypes'; export namespace Database { @@ -297,7 +297,7 @@ export namespace Database { }; export const QueryUploadHistory = async (contentSize: number) => { - return SanitizedSingletonQuery<DashUploadUtils.ImageUploadInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); + return SanitizedSingletonQuery<Upload.ImageInformation>({ contentSize }, AuxiliaryCollections.GooglePhotosUploadHistory); }; export namespace GoogleAuthenticationToken { @@ -326,7 +326,7 @@ export namespace Database { } - export const LogUpload = async (information: DashUploadUtils.ImageUploadInformation) => { + export const LogUpload = async (information: Upload.ImageInformation) => { const bundle = { _id: Utils.GenerateDeterministicGuid(String(information.contentSize!)), ...information diff --git a/src/server/index.ts b/src/server/index.ts index 55ba71dba..10205314a 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -24,7 +24,7 @@ import { Logger } from "./ProcessFactory"; import { yellow } from "colors"; import { DashSessionAgent } from "./DashSession/DashSessionAgent"; import SessionManager from "./ApiManagers/SessionManager"; -import { AppliedSessionAgent } from "resilient-server-session"; +import { AppliedSessionAgent } from "./DashSession/Session/agents/applied_session_agent"; export const onWindows = process.platform === "win32"; export let sessionAgent: AppliedSessionAgent; @@ -86,12 +86,14 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: secureHandler: ({ res }) => res.redirect("/home") }); + addSupervisedRoute({ method: Method.GET, subscription: "/serverHeartbeat", secureHandler: ({ res }) => res.send(true) }); + const serve: PublicHandler = ({ req, res }) => { const detector = new mobileDetect(req.headers['user-agent'] || ""); const filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html'; @@ -119,6 +121,7 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }: WebSocket.start(isRelease); } + /** * This function can be used in two different ways. If not in release mode, * this is simply the logic that is invoked to start the server. In release mode, diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts deleted file mode 100644 index 83094d36a..000000000 --- a/src/server/updateSearch.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { Database } from "./database"; -import { Search } from "./Search"; -import { log_execution } from "./ActionUtilities"; -import { cyan, green, yellow, red } from "colors"; - -const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = { - "number": "_n", - "string": "_t", - "boolean": "_b", - "image": ["_t", "url"], - "video": ["_t", "url"], - "pdf": ["_t", "url"], - "audio": ["_t", "url"], - "web": ["_t", "url"], - "date": ["_d", value => new Date(value.date).toISOString()], - "proxy": ["_i", "fieldId"], - "list": ["_l", list => { - const results = []; - for (const value of list.fields) { - const term = ToSearchTerm(value); - if (term) { - results.push(term.value); - } - } - return results.length ? results : null; - }] -}; - -function ToSearchTerm(val: any): { suffix: string, value: any } | undefined { - if (val === null || val === undefined) { - return; - } - const type = val.__type || typeof val; - let suffix = suffixMap[type]; - if (!suffix) { - return; - } - - if (Array.isArray(suffix)) { - const accessor = suffix[1]; - if (typeof accessor === "function") { - val = accessor(val); - } else { - val = val[accessor]; - } - suffix = suffix[0]; - } - - return { suffix, value: val }; -} - -async function update() { - console.log(green("Beginning update...")); - await log_execution<void>({ - startMessage: "Clearing existing Solr information...", - endMessage: "Solr information successfully cleared", - action: Search.clear, - color: cyan - }); - const cursor = await log_execution({ - startMessage: "Connecting to and querying for all documents from database...", - endMessage: ({ result, error }) => { - const success = error === null && result !== undefined; - if (!success) { - console.log(red("Unable to connect to the database.")); - process.exit(0); - } - return "Connection successful and query complete"; - }, - action: () => Database.Instance.query({}), - color: yellow - }); - const updates: any[] = []; - let numDocs = 0; - function updateDoc(doc: any) { - numDocs++; - if ((numDocs % 50) === 0) { - console.log(`Batch of 50 complete, total of ${numDocs}`); - } - if (doc.__type !== "Doc") { - return; - } - const fields = doc.fields; - if (!fields) { - return; - } - const update: any = { id: doc._id }; - let dynfield = false; - for (const key in fields) { - const value = fields[key]; - const term = ToSearchTerm(value); - if (term !== undefined) { - const { suffix, value } = term; - update[key + suffix] = value; - dynfield = true; - } - } - if (dynfield) { - updates.push(update); - } - } - await cursor?.forEach(updateDoc); - const result = await log_execution({ - startMessage: `Dispatching updates for ${updates.length} documents`, - endMessage: "Dispatched updates complete", - action: () => Search.updateDocuments(updates), - color: cyan - }); - try { - const { status } = JSON.parse(result).responseHeader; - console.log(status ? red(`Failed with status code (${status})`) : green("Success!")); - } catch { - console.log(red("Error:")); - console.log(result); - console.log("\n"); - } - await cursor?.close(); - process.exit(0); -} - -update();
\ No newline at end of file |