diff options
Diffstat (limited to 'src/server/session_manager')
9 files changed, 315 insertions, 0 deletions
diff --git a/src/server/session_manager/config.ts b/src/server/session_manager/config.ts new file mode 100644 index 000000000..47d3375e0 --- /dev/null +++ b/src/server/session_manager/config.ts @@ -0,0 +1,32 @@ +import { resolve } from 'path'; +import { yellow } from "colors"; + +export const latency = 10; +export const ports = [1050, 4321]; +export const onWindows = process.platform === "win32"; +export const LOCATION = "http://localhost"; +export const heartbeat = `${LOCATION}:1050/serverHeartbeat`; +export const recipient = "samuel_wilkins@brown.edu"; +export const { pid, platform } = process; + +/** + * Logging + */ +export const identifier = yellow("__session_manager__:"); + +/** + * Paths + */ +export const logPath = resolve(__dirname, "./logs"); +export const crashPath = resolve(logPath, "./crashes"); + +/** + * State + */ +export enum SessionState { + INITIALIZING, + LISTENING, + AUTOMATIC_RESTART, + MANUAL_RESTART, + EXITING +}
\ No newline at end of file diff --git a/src/server/session_manager/input_manager.ts b/src/server/session_manager/input_manager.ts new file mode 100644 index 000000000..a95e6baae --- /dev/null +++ b/src/server/session_manager/input_manager.ts @@ -0,0 +1,101 @@ +import { createInterface, Interface } from "readline"; +import { red } from "colors"; + +export interface Configuration { + identifier: string; + onInvalid?: (culprit?: string) => string | string; + isCaseSensitive?: boolean; +} + +export interface Registration { + argPattern: RegExp[]; + action: (parsedArgs: IterableIterator<string>) => any | Promise<any>; +} + +export default class InputManager { + private identifier: string; + private onInvalid: ((culprit?: string) => string) | string; + private isCaseSensitive: boolean; + private commandMap = new Map<string, Registration[]>(); + public interface: Interface; + private busy = false; + private keys: string | undefined; + + constructor({ identifier: prompt, onInvalid, isCaseSensitive }: Configuration) { + this.identifier = prompt; + this.onInvalid = onInvalid || this.usage; + this.isCaseSensitive = isCaseSensitive ?? true; + this.interface = createInterface(process.stdin, process.stdout).on('line', this.considerInput); + } + + private usage = () => { + const resolved = this.keys; + if (resolved) { + return resolved; + } + const members: string[] = []; + const keys = this.commandMap.keys(); + let next: IteratorResult<string>; + while (!(next = keys.next()).done) { + members.push(next.value); + } + return `${this.identifier} commands: { ${members.sort().join(", ")} }`; + } + + public registerCommand = (basename: string, argPattern: RegExp[], action: any | Promise<any>) => { + const existing = this.commandMap.get(basename); + const registration = { argPattern, action }; + if (existing) { + existing.push(registration); + } else { + this.commandMap.set(basename, [registration]); + } + } + + private invalid = (culprit?: string) => { + console.log(red(typeof this.onInvalid === "string" ? this.onInvalid : this.onInvalid(culprit))); + this.busy = false; + } + + private considerInput = async (line: string) => { + if (this.busy) { + console.log(red("Busy")); + return; + } + this.busy = true; + line = line.trim(); + if (this.isCaseSensitive) { + line = line.toLowerCase(); + } + const [command, ...args] = line.split(/\s+/g); + if (!command) { + return this.invalid(); + } + const registered = this.commandMap.get(command); + if (registered) { + const { length } = args; + const candidates = registered.filter(({ argPattern: { length: count } }) => count === length); + for (const { argPattern, action } of candidates) { + const parsed: string[] = []; + let matched = false; + if (length) { + for (let i = 0; i < length; i++) { + let matches: RegExpExecArray | null; + if ((matches = argPattern[i].exec(args[i])) === null) { + break; + } + parsed.push(matches[0]); + } + matched = true; + } + if (!length || matched) { + await action(parsed[Symbol.iterator]()); + this.busy = false; + return; + } + } + } + this.invalid(command); + } + +}
\ No newline at end of file diff --git a/src/server/session_manager/logs/crashes/session_crashes_@ 2019-12-11T18:52:10.359Z.log b/src/server/session_manager/logs/crashes/session_crashes_@ 2019-12-11T18:52:10.359Z.log new file mode 100644 index 000000000..4cab8a6a3 --- /dev/null +++ b/src/server/session_manager/logs/crashes/session_crashes_@ 2019-12-11T18:52:10.359Z.log @@ -0,0 +1 @@ +[31mDetected a server crash @ 2019-12-11T18:52:20.416Z.[39m diff --git a/src/server/session_manager/logs/crashes/session_crashes_@ 2019-12-12T00:38:44.803Z.log b/src/server/session_manager/logs/crashes/session_crashes_@ 2019-12-12T00:38:44.803Z.log new file mode 100644 index 000000000..3c226445d --- /dev/null +++ b/src/server/session_manager/logs/crashes/session_crashes_@ 2019-12-12T00:38:44.803Z.log @@ -0,0 +1 @@ +[31mDetected a server crash @ 2019-12-12T00:38:54.828Z.[39m diff --git a/src/server/session_manager/logs/crashes/session_crashes_@ 2019-12-12T00:42:11.945Z.log b/src/server/session_manager/logs/crashes/session_crashes_@ 2019-12-12T00:42:11.945Z.log new file mode 100644 index 000000000..59f14f288 --- /dev/null +++ b/src/server/session_manager/logs/crashes/session_crashes_@ 2019-12-12T00:42:11.945Z.log @@ -0,0 +1 @@ +[31mDetected a server crash @ 2019-12-12T00:42:21.970Z.[39m diff --git a/src/server/session_manager/logs/current_daemon_pid.log b/src/server/session_manager/logs/current_daemon_pid.log new file mode 100644 index 000000000..557e3d7c3 --- /dev/null +++ b/src/server/session_manager/logs/current_daemon_pid.log @@ -0,0 +1 @@ +26860 diff --git a/src/server/session_manager/logs/current_server_pid.log b/src/server/session_manager/logs/current_server_pid.log new file mode 100644 index 000000000..6a2f267b3 --- /dev/null +++ b/src/server/session_manager/logs/current_server_pid.log @@ -0,0 +1 @@ +50888 created @ 2019-12-14T06:59:59.767Z diff --git a/src/server/session_manager/logs/current_session_manager_pid.log b/src/server/session_manager/logs/current_session_manager_pid.log new file mode 100644 index 000000000..ab19403b4 --- /dev/null +++ b/src/server/session_manager/logs/current_session_manager_pid.log @@ -0,0 +1 @@ +50846 diff --git a/src/server/session_manager/session_manager.ts b/src/server/session_manager/session_manager.ts new file mode 100644 index 000000000..6e9b03c79 --- /dev/null +++ b/src/server/session_manager/session_manager.ts @@ -0,0 +1,176 @@ +import * as request from "request-promise"; +import { log_execution, pathFromRoot } from "../ActionUtilities"; +import { red, yellow, cyan, green, Color } from "colors"; +import * as nodemailer from "nodemailer"; +import { MailOptions } from "nodemailer/lib/json-transport"; +import { writeFileSync, existsSync, mkdirSync } from "fs"; +import { resolve } from 'path'; +import { ChildProcess, exec, execSync } from "child_process"; +import InputManager from "./input_manager"; +import { identifier, logPath, crashPath, onWindows, pid, ports, heartbeat, recipient, LOCATION, latency } from "./config"; +const killport = require("kill-port"); + +process.on('SIGINT', endPrevious); + +const { registerCommand } = new InputManager({ identifier }); + +let manualRestartActive = false; +registerCommand("restart", [], async () => { + manualRestartActive = true; + identifiedLog(cyan("Initializing manual restart...")); + await endPrevious(); +}); + +registerCommand("exit", [], async () => { + identifiedLog(cyan("Initializing session end")); + await endPrevious(); + identifiedLog("Cleanup complete. Exiting session...\n"); + execSync(killAllCommand()); +}); + +if (!existsSync(logPath)) { + mkdirSync(logPath); +} +if (!existsSync(crashPath)) { + mkdirSync(crashPath); +} + +const crashLogPath = resolve(crashPath, `./session_crashes_${new Date().toISOString()}.log`); +function addLogEntry(message: string, color: Color) { + const formatted = color(`${message} ${timestamp()}.`); + identifiedLog(formatted); + // appendFileSync(crashLogPath, `${formatted}\n`); +} + +function identifiedLog(message?: any, ...optionalParams: any[]) { + console.log(identifier, message, ...optionalParams); +} + +if (!["win32", "darwin"].includes(process.platform)) { + identifiedLog(red("Invalid operating system: this script is supported only on Mac and Windows.")); + process.exit(1); +} + +let restarting = false; +let count = 0; + +function startServerCommand() { + if (onWindows) { + return '"C:\\Program Files\\Git\\git-bash.exe" -c "npm run start-release"'; + } + return `osascript -e 'tell app "Terminal"\ndo script "cd ${pathFromRoot()} && npm run start-release"\nend tell'`; +} + +function killAllCommand() { + if (onWindows) { + return "taskkill /f /im node.exe"; + } + return "killall -9 node"; +} + +identifiedLog("Initializing session..."); + +writeLocalPidLog("session_manager", pid); + +function writeLocalPidLog(filename: string, contents: any) { + const path = `./logs/current_${filename}_pid.log`; + identifiedLog(cyan(`${contents} written to ${path}`)); + writeFileSync(resolve(__dirname, path), `${contents}\n`); +} + +function timestamp() { + return `@ ${new Date().toISOString()}`; +} + +async function endPrevious() { + identifiedLog(yellow("Cleaning up previous connections...")); + current_backup?.kill("SIGKILL"); + await Promise.all(ports.map(port => { + const task = killport(port, 'tcp'); + return task.catch((error: any) => identifiedLog(red(error))); + })); + identifiedLog(yellow("Done. Any failures will be printed in red immediately above.")); +} + +let current_backup: ChildProcess | undefined = undefined; + +async function checkHeartbeat() { + let error: any; + try { + count && !restarting && process.stdout.write(`${identifier} 👂 `); + await request.get(heartbeat); + count && !restarting && console.log('⇠💚'); + if (restarting || manualRestartActive) { + addLogEntry(count++ ? "Backup server successfully restarted" : "Server successfully started", green); + restarting = false; + } + } catch (e) { + count && !restarting && console.log("⇠💔"); + error = e; + } finally { + if (error) { + if (!restarting || manualRestartActive) { + restarting = true; + if (count && !manualRestartActive) { + console.log(); + addLogEntry("Detected a server crash", red); + identifiedLog(red(error.message)); + await endPrevious(); + await log_execution({ + startMessage: identifier + " Sending crash notification email", + endMessage: ({ error, result }) => { + const success = error === null && result === true; + return identifier + ` ${(success ? `Notification successfully sent to` : `Failed to notify`)} ${recipient} ${timestamp()}`; + }, + action: async () => notify(error || "Hmm, no error to report..."), + color: cyan + }); + identifiedLog(green("Initiating server restart...")); + } + manualRestartActive = false; + current_backup = exec(startServerCommand(), err => identifiedLog(err?.message || count ? "Previous server process exited." : "Spawned initial server.")); + writeLocalPidLog("server", `${(current_backup?.pid ?? -2) + 1} created ${timestamp()}`); + } + } + setTimeout(checkHeartbeat, 1000 * latency); + } +} + +async function startListening() { + identifiedLog(yellow(`After initialization, will poll server heartbeat repeatedly...\n`)); + if (!LOCATION) { + identifiedLog(red("No location specified for session manager. Please include as a command line environment variable or in a .env file.")); + process.exit(0); + } + await checkHeartbeat(); +} + +function emailText(error: any) { + return [ + `Hey ${recipient.split("@")[0]},`, + "You, as a Dash Administrator, are being notified of a server crash event. Here's what we know:", + `Location: ${LOCATION}\nError: ${error}`, + "The server should already be restarting itself, but if you're concerned, use the Remote Desktop Connection to monitor progress." + ].join("\n\n"); +} + +async function notify(error: any) { + const smtpTransport = nodemailer.createTransport({ + service: 'Gmail', + auth: { + user: 'brownptcdash@gmail.com', + pass: 'browngfx1' + } + }); + const mailOptions = { + to: recipient, + from: 'brownptcdash@gmail.com', + subject: 'Dash Server Crash', + text: emailText(error) + } as MailOptions; + return new Promise<boolean>(resolve => { + smtpTransport.sendMail(mailOptions, (dispatchError: Error | null) => resolve(dispatchError === null)); + }); +} + +startListening();
\ No newline at end of file |