9 files changed, 291 insertions, 121 deletions

diff --git a/src/server/ActionUtilities.ts b/src/server/ActionUtilities.ts
index 53ddea2fc..053576a92 100644
--- a/src/server/ActionUtilities.ts
+++ b/src/server/ActionUtilities.ts
@@ -1,4 +1,4 @@
-import * as fs from 'fs';
+import { readFile, writeFile, exists, mkdir, unlink, createWriteStream } from 'fs';
 import { ExecOptions } from 'shelljs';
 import { exec } from 'child_process';
 import * as path from 'path';
@@ -6,6 +6,17 @@ import * as rimraf from "rimraf";
 import { yellow, Color } from 'colors';
 
 const projectRoot = path.resolve(__dirname, "../../");
+export function pathFromRoot(relative?: string) {
+    if (!relative) {
+        return projectRoot;
+    }
+    return path.resolve(projectRoot, relative);
+}
+
+export async function fileDescriptorFromStream(path: string) {
+    const logStream = createWriteStream(path);
+    return new Promise<number>(resolve => logStream.on("open", resolve));
+}
 
 export const command_line = (command: string, fromDirectory?: string) => {
     return new Promise<string>((resolve, reject) => {
@@ -20,14 +31,14 @@ export const command_line = (command: string, fromDirectory?: string) => {
 export const read_text_file = (relativePath: string) => {
     const target = path.resolve(__dirname, relativePath);
     return new Promise<string>((resolve, reject) => {
-        fs.readFile(target, (err, data) => err ? reject(err) : resolve(data.toString()));
+        readFile(target, (err, data) => err ? reject(err) : resolve(data.toString()));
     });
 };
 
 export const write_text_file = (relativePath: string, contents: any) => {
     const target = path.resolve(__dirname, relativePath);
     return new Promise<void>((resolve, reject) => {
-        fs.writeFile(target, contents, (err) => err ? reject(err) : resolve());
+        writeFile(target, contents, (err) => err ? reject(err) : resolve());
     });
 };
 
@@ -51,11 +62,7 @@ export async function log_execution<T>({ startMessage, endMessage, action, color
     } catch (e) {
         error = e;
     } finally {
-        if (typeof endMessage === "string") {
-            log_helper(`${endMessage}.`, resolvedColor);
-        } else {
-            log_helper(`${endMessage({ result, error })}.`, resolvedColor);
-        }
+        log_helper(typeof endMessage === "string" ? endMessage : endMessage({ result, error }), resolvedColor);
     }
     return result;
 }
@@ -86,10 +93,10 @@ export function msToTime(duration: number) {
 }
 
 export const createIfNotExists = async (path: string) => {
-    if (await new Promise<boolean>(resolve => fs.exists(path, resolve))) {
+    if (await new Promise<boolean>(resolve => exists(path, resolve))) {
         return true;
     }
-    return new Promise<boolean>(resolve => fs.mkdir(path, error => resolve(error === null)));
+    return new Promise<boolean>(resolve => mkdir(path, error => resolve(error === null)));
 };
 
 export async function Prune(rootDirectory: string): Promise<boolean> {
@@ -97,4 +104,4 @@ export async function Prune(rootDirectory: string): Promise<boolean> {
     return error === null;
 }
 
-export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => fs.unlink(mediaPath, error => resolve(error === null)));
+export const Destroy = (mediaPath: string) => new Promise<boolean>(resolve => unlink(mediaPath, error => resolve(error === null)));
diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts
index ccfd570b8..37d66666b 100644
--- a/src/server/ApiManagers/SearchManager.ts
+++ b/src/server/ApiManagers/SearchManager.ts
@@ -72,7 +72,7 @@ export namespace SolrManager {
         const args = status ? "start" : "stop -p 8983";
         try {
             console.log(`Solr management: trying to ${args}`);
-            console.log(await command_line(`solr.cmd ${args}`, "../../solr-8.1.1/bin"));
+            console.log(await command_line(`solr.cmd ${args}`, "./solr-8.3.1/bin"));
             return true;
         } catch (e) {
             console.log(red(`Solr management error: unable to ${args}`));
diff --git a/src/server/ApiManagers/UtilManager.ts b/src/server/ApiManagers/UtilManager.ts
index e959645e0..2f1bd956f 100644
--- a/src/server/ApiManagers/UtilManager.ts
+++ b/src/server/ApiManagers/UtilManager.ts
@@ -43,7 +43,7 @@ export default class UtilManager extends ApiManager {
             method: Method.GET,
             subscription: "/buxton",
             onValidation: async ({ res }) => {
-                const cwd = '../scraping/buxton';
+                const cwd = './src/scraping/buxton';
 
                 const onResolved = (stdout: string) => { console.log(stdout); res.redirect("/"); };
                 const onRejected = (err: any) => { console.error(err.message); res.send(err); };
diff --git a/src/server/ChildProcessUtilities/ProcessFactory.ts b/src/server/ChildProcessUtilities/ProcessFactory.ts
new file mode 100644
index 000000000..745b1479a
--- /dev/null
+++ b/src/server/ChildProcessUtilities/ProcessFactory.ts
@@ -0,0 +1,67 @@
+import { existsSync, mkdirSync } from "fs";
+import { pathFromRoot, log_execution, fileDescriptorFromStream } from '../ActionUtilities';
+import { red, green } from "colors";
+import rimraf = require("rimraf");
+import { ChildProcess, spawn, StdioOptions } from "child_process";
+import { Stream } from "stream";
+import { resolve } from "path";
+
+export namespace ProcessFactory {
+
+    export type Sink = "pipe" | "ipc" | "ignore" | "inherit" | Stream | number | null | undefined;
+
+    export async function createWorker(command: string, args?: readonly string[], stdio?: StdioOptions | "logfile", detached = true): Promise<ChildProcess> {
+        if (stdio === "logfile") {
+            const log_fd = await Logger.create(command, args);
+            stdio = ["ignore", log_fd, log_fd];
+        }
+        const child = spawn(command, args, { detached, stdio });
+        child.unref();
+        return child;
+    }
+
+    export namespace NamedAgents {
+
+        export async function persistenceDaemon() {
+            await log_execution({
+                startMessage: "\ninitializing persistence daemon",
+                endMessage: ({ result, error }) => {
+                    const success = error === null && result !== undefined;
+                    if (!success) {
+                        console.log(red("failed to initialize the persistance daemon"));
+                        console.log(error);
+                        process.exit(0);
+                    }
+                    return "failsafe daemon process successfully spawned";
+                },
+                action: () => createWorker('npx', ['ts-node', resolve(__dirname, "./daemon/persistence_daemon.ts")], ["ignore", "inherit", "inherit"]),
+                color: green
+            });
+            console.log();
+        }
+    }
+
+}
+
+export namespace Logger {
+
+    const logPath = pathFromRoot("./logs");
+
+    export async function initialize() {
+        if (existsSync(logPath)) {
+            if (!process.env.SPAWNED) {
+                await new Promise<any>(resolve => rimraf(logPath, resolve));
+            }
+        }
+        mkdirSync(logPath);
+    }
+
+    export async function create(command: string, args?: readonly string[]): Promise<number> {
+        return fileDescriptorFromStream(generate_log_path(command, args));
+    }
+
+    function generate_log_path(command: string, args?: readonly string[]) {
+        return pathFromRoot(`./logs/${command}-${args?.length}-${new Date().toUTCString()}.log`);
+    }
+
+}
\ No newline at end of file
diff --git a/src/server/ChildProcessUtilities/daemon/session.ts b/src/server/ChildProcessUtilities/daemon/session.ts
new file mode 100644
index 000000000..fb2b3551e
--- /dev/null
+++ b/src/server/ChildProcessUtilities/daemon/session.ts
@@ -0,0 +1,190 @@
+import * as request from "request-promise";
+import { log_execution, pathFromRoot } from "../../ActionUtilities";
+import { red, yellow, cyan, green, Color } from "colors";
+import * as nodemailer from "nodemailer";
+import { MailOptions } from "nodemailer/lib/json-transport";
+import { writeFileSync, existsSync, mkdirSync } from "fs";
+import { resolve } from 'path';
+import { ChildProcess, exec, execSync } from "child_process";
+import { createInterface } from "readline";
+const killport = require("kill-port");
+
+process.on('SIGINT', endPrevious);
+const identifier = yellow("__session_manager__:");
+
+let manualRestartActive = false;
+createInterface(process.stdin, process.stdout).on('line', async line => {
+    const prompt = line.trim().toLowerCase();
+    switch (prompt) {
+        case "restart":
+            manualRestartActive = true;
+            identifiedLog(cyan("Initializing manual restart..."));
+            await endPrevious();
+            break;
+        case "exit":
+            identifiedLog(cyan("Initializing session end"));
+            await endPrevious();
+            identifiedLog("Cleanup complete. Exiting session...\n");
+            execSync(killAllCommand());
+            break;
+        default:
+            identifiedLog(red("commands: { exit, restart }"));
+            return;
+    }
+});
+
+const logPath = resolve(__dirname, "./logs");
+const crashPath = resolve(logPath, "./crashes");
+if (!existsSync(logPath)) {
+    mkdirSync(logPath);
+}
+if (!existsSync(crashPath)) {
+    mkdirSync(crashPath);
+}
+
+const crashLogPath = resolve(crashPath, `./session_crashes_${new Date().toISOString()}.log`);
+function addLogEntry(message: string, color: Color) {
+    const formatted = color(`${message} ${timestamp()}.`);
+    identifiedLog(formatted);
+    // appendFileSync(crashLogPath, `${formatted}\n`);
+}
+
+function identifiedLog(message?: any, ...optionalParams: any[]) {
+    console.log(identifier, message, ...optionalParams);
+}
+
+if (!["win32", "darwin"].includes(process.platform)) {
+    identifiedLog(red("Invalid operating system: this script is supported only on Mac and Windows."));
+    process.exit(1);
+}
+
+const latency = 10;
+const ports = [1050, 4321];
+const onWindows = process.platform === "win32";
+const LOCATION = "http://localhost";
+const heartbeat = `${LOCATION}:1050/serverHeartbeat`;
+const recipient = "samuel_wilkins@brown.edu";
+const { pid } = process;
+let restarting = false;
+let count = 0;
+
+function startServerCommand() {
+    if (onWindows) {
+        return '"C:\\Program Files\\Git\\git-bash.exe" -c "npm run start-release"';
+    }
+    return `osascript -e 'tell app "Terminal"\ndo script "cd ${pathFromRoot()} && npm run start-release"\nend tell'`;
+}
+
+function killAllCommand() {
+    if (onWindows) {
+        return "taskkill /f /im node.exe";
+    }
+    return "killall -9 node";
+}
+
+identifiedLog("Initializing session...");
+
+writeLocalPidLog("session_manager", pid);
+
+function writeLocalPidLog(filename: string, contents: any) {
+    const path = `./logs/current_${filename}_pid.log`;
+    identifiedLog(cyan(`${contents} written to ${path}`));
+    writeFileSync(resolve(__dirname, path), `${contents}\n`);
+}
+
+function timestamp() {
+    return `@ ${new Date().toISOString()}`;
+}
+
+async function endPrevious() {
+    identifiedLog(yellow("Cleaning up previous connections..."));
+    current_backup?.kill("SIGTERM");
+    await Promise.all(ports.map(port => {
+        const task = killport(port, 'tcp');
+        return task.catch((error: any) => identifiedLog(red(error)));
+    }));
+    identifiedLog(yellow("Done. Any failures will be printed in red immediately above."));
+}
+
+let current_backup: ChildProcess | undefined = undefined;
+
+async function checkHeartbeat() {
+    let error: any;
+    try {
+        count && !restarting && process.stdout.write(`${identifier} 👂 `);
+        await request.get(heartbeat);
+        count && !restarting && console.log('⇠ 💚');
+        if (restarting || manualRestartActive) {
+            addLogEntry(count++ ? "Backup server successfully restarted" : "Server successfully started", green);
+            restarting = false;
+        }
+    } catch (e) {
+        count && !restarting && console.log("⇠ 💔");
+        error = e;
+    } finally {
+        if (error) {
+            if (!restarting || manualRestartActive) {
+                restarting = true;
+                if (count && !manualRestartActive) {
+                    console.log();
+                    addLogEntry("Detected a server crash", red);
+                    identifiedLog(red(error.message));
+                    await endPrevious();
+                    await log_execution({
+                        startMessage: identifier + " Sending crash notification email",
+                        endMessage: ({ error, result }) => {
+                            const success = error === null && result === true;
+                            return identifier + ` ${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`;
+                        },
+                        action: async () => notify(error || "Hmm, no error to report..."),
+                        color: cyan
+                    });
+                    identifiedLog(green("Initiating server restart..."));
+                }
+                manualRestartActive = false;
+                current_backup = exec(startServerCommand(), err => identifiedLog(err?.message || count ? "Previous server process exited." : "Spawned initial server."));
+                writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`);
+            }
+        }
+        setTimeout(checkHeartbeat, 1000 * latency);
+    }
+}
+
+async function startListening() {
+    identifiedLog(yellow(`After initialization, will poll server heartbeat repeatedly...\n`));
+    if (!LOCATION) {
+        identifiedLog(red("No location specified for session manager. Please include as a command line environment variable or in a .env file."));
+        process.exit(0);
+    }
+    await checkHeartbeat();
+}
+
+function emailText(error: any) {
+    return [
+        `Hey ${recipient.split("@")[0]},`,
+        "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:",
+        `Location: ${LOCATION}\nError: ${error}`,
+        "The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress."
+    ].join("\n\n");
+}
+
+async function notify(error: any) {
+    const smtpTransport = nodemailer.createTransport({
+        service: 'Gmail',
+        auth: {
+            user: 'brownptcdash@gmail.com',
+            pass: 'browngfx1'
+        }
+    });
+    const mailOptions = {
+        to: recipient,
+        from: 'brownptcdash@gmail.com',
+        subject: 'Dash Server Crash',
+        text: emailText(error)
+    } as MailOptions;
+    return new Promise<boolean>(resolve => {
+        smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null));
+    });
+}
+
+startListening();
\ No newline at end of file
diff --git a/src/server/Websocket/Websocket.ts b/src/server/Websocket/Websocket.ts
index 76e02122b..e1e157fc4 100644
--- a/src/server/Websocket/Websocket.ts
+++ b/src/server/Websocket/Websocket.ts
@@ -133,6 +133,7 @@ export namespace WebSocket {
         "pdf": ["_t", "url"],
         "audio": ["_t", "url"],
         "web": ["_t", "url"],
+        "RichTextField": ["_t", value => value.Text],
         "date": ["_d", value => new Date(value.date).toISOString()],
         "proxy": ["_i", "fieldId"],
         "list": ["_l", list => {
diff --git a/src/server/index.ts b/src/server/index.ts
index 3764eaabb..bc481e579 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -18,11 +18,12 @@ import { GoogleCredentialsLoader } from './credentials/CredentialsLoader';
 import DeleteManager from "./ApiManagers/DeleteManager";
 import PDFManager from "./ApiManagers/PDFManager";
 import UploadManager from "./ApiManagers/UploadManager";
-import { log_execution, command_line } from "./ActionUtilities";
+import { log_execution } from "./ActionUtilities";
 import GeneralGoogleManager from "./ApiManagers/GeneralGoogleManager";
 import GooglePhotosManager from "./ApiManagers/GooglePhotosManager";
 import { yellow, red } from "colors";
 import { disconnect } from "../server/Initialization";
+import { ProcessFactory, Logger } from "./ChildProcessUtilities/ProcessFactory";
 
 export const publicDirectory = path.resolve(__dirname, "public");
 export const filesDirectory = path.resolve(publicDirectory, "files");
@@ -35,6 +36,7 @@ export const ExitHandlers = new Array<() => void>();
  * before clients can access the server should be run or awaited here.
  */
 async function preliminaryFunctions() {
+    await Logger.initialize();
     await GoogleCredentialsLoader.loadCredentials();
     GoogleApiServerUtils.processProjectCredentials();
     await DashUploadUtils.buildFileDirectories();
@@ -119,31 +121,6 @@ function routeSetter({ isRelease, addSupervisedRoute, logRegistrationOutcome }:
         }
     });
 
-    let daemonInitialized = false;
-    addSupervisedRoute({
-        method: Method.GET,
-        subscription: "/persist",
-        onValidation: async ({ res }) => {
-            if (!daemonInitialized) {
-                daemonInitialized = true;
-                log_execution({
-                    startMessage: "\ninitializing persistence daemon",
-                    endMessage: ({ result, error }) => {
-                        const success = error === null && result !== undefined;
-                        if (!success) {
-                            console.log(red("failed to initialize the persistance daemon"));
-                            process.exit(0);
-                        }
-                        return "persistence daemon process closed";
-                    },
-                    action: async () => command_line("npx ts-node ./persistence_daemon.ts", "./src/server"),
-                    color: yellow
-                });
-            }
-            res.redirect("/home");
-        }
-    });
-
     logRegistrationOutcome();
 
     // initialize the web socket (bidirectional communication: if a user changes
diff --git a/src/server/persistence_daemon.ts b/src/server/persistence_daemon.ts
deleted file mode 100644
index 388440b49..000000000
--- a/src/server/persistence_daemon.ts
+++ /dev/null
@@ -1,79 +0,0 @@
-import * as request from "request-promise";
-import { command_line, log_execution } from "./ActionUtilities";
-import { red, yellow, cyan, green } from "colors";
-import * as nodemailer from "nodemailer";
-import { MailOptions } from "nodemailer/lib/json-transport";
-
-const { LOCATION } = process.env;
-const recipient = "samuel_wilkins@brown.edu";
-let restarting = false;
-
-async function listen() {
-    if (!LOCATION) {
-        console.log(red("No location specified for persistence daemon. Please include as a command line environment variable or in a .env file."));
-        process.exit(0);
-    }
-    const heartbeat = `${LOCATION}:1050/serverHeartbeat`;
-    // if this is on our remote server, the server must be run in release mode
-    const suffix = LOCATION.includes("localhost") ? "" : "-release";
-    setInterval(async () => {
-        let response: any;
-        let error: any;
-        try {
-            response = await request.get(heartbeat);
-        } catch (e) {
-            error = e;
-        } finally {
-            if (!response && !restarting) {
-                restarting = true;
-                console.log(yellow("Detected a server crash!"));
-                await log_execution({
-                    startMessage: "Sending crash notification email",
-                    endMessage: ({ error, result }) => {
-                        const success = error === null && result === true;
-                        return (success ? `Notification successfully sent to ` : `Failed to notify `) + recipient;
-                    },
-                    action: async () => notify(error || "Hmm, no error to report..."),
-                    color: cyan
-                });
-                console.log(await log_execution({
-                    startMessage: "Initiating server restart",
-                    endMessage: "Server successfully restarted",
-                    action: async () => command_line(`npm run start${suffix}`, "../../"),
-                    color: green
-                }));
-                restarting = false;
-            }
-        }
-    }, 1000 * 90);
-}
-
-function emailText(error: any) {
-    return [
-        `Hey ${recipient.split("@")[0]},`,
-        "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:",
-        `Location: ${LOCATION}\nError: ${error}`,
-        "The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress."
-    ].join("\n\n");
-}
-
-async function notify(error: any) {
-    const smtpTransport = nodemailer.createTransport({
-        service: 'Gmail',
-        auth: {
-            user: 'brownptcdash@gmail.com',
-            pass: 'browngfx1'
-        }
-    });
-    const mailOptions = {
-        to: recipient,
-        from: 'brownptcdash@gmail.com',
-        subject: 'Dash Server Crash',
-        text: emailText(error)
-    } as MailOptions;
-    return new Promise<boolean>(resolve => {
-        smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null));
-    });
-}
-
-listen();
\ No newline at end of file
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
index 5ae6885c5..83094d36a 100644
--- a/src/server/updateSearch.ts
+++ b/src/server/updateSearch.ts
@@ -59,7 +59,14 @@ async function update() {
     });
     const cursor = await log_execution({
         startMessage: "Connecting to and querying for all documents from database...",
-        endMessage: "Connection successful and query complete",
+        endMessage: ({ result, error }) => {
+            const success = error === null && result !== undefined;
+            if (!success) {
+                console.log(red("Unable to connect to the database."));
+                process.exit(0);
+            }
+            return "Connection successful and query complete";
+        },
         action: () => Database.Instance.query({}),
         color: yellow
     });
@@ -92,7 +99,7 @@ async function update() {
             updates.push(update);
         }
     }
-    await cursor.forEach(updateDoc);
+    await cursor?.forEach(updateDoc);
     const result = await log_execution({
         startMessage: `Dispatching updates for ${updates.length} documents`,
         endMessage: "Dispatched updates complete",
@@ -107,7 +114,7 @@ async function update() {
         console.log(result);
         console.log("\n");
     }
-    await cursor.close();
+    await cursor?.close();
     process.exit(0);
 }