aboutsummaryrefslogtreecommitdiff
path: root/src/server
diff options
context:
space:
mode:
Diffstat (limited to 'src/server')
-rw-r--r--src/server/ApiManagers/SearchManager.ts4
-rw-r--r--src/server/ApiManagers/UploadManager.ts50
-rw-r--r--src/server/DashStats.ts150
-rw-r--r--src/server/DashUploadUtils.ts31
4 files changed, 118 insertions, 117 deletions
diff --git a/src/server/ApiManagers/SearchManager.ts b/src/server/ApiManagers/SearchManager.ts
index 186f0bcd3..72c01def7 100644
--- a/src/server/ApiManagers/SearchManager.ts
+++ b/src/server/ApiManagers/SearchManager.ts
@@ -8,7 +8,7 @@ import RouteSubscriber from '../RouteSubscriber';
import { Search } from '../Search';
import ApiManager, { Registration } from './ApiManager';
import { Directory, pathToDirectory } from './UploadManager';
-const findInFiles = require('find-in-files');
+import { find } from 'find-in-files';
export class SearchManager extends ApiManager {
protected initialize(register: Registration): void {
@@ -47,7 +47,7 @@ export class SearchManager extends ApiManager {
const dir = pathToDirectory(Directory.text);
try {
const regex = new RegExp(q.toString());
- results = await findInFiles.find({ term: q, flags: 'ig' }, dir, '.txt$');
+ results = await find({ term: q, flags: 'ig' }, dir, '.txt$');
for (const result in results) {
resObj.ids.push(path.basename(result, '.txt').replace(/upload_/, ''));
resObj.lines.push(results[result].line);
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 06f4f1a9d..8a2fe1389 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -14,8 +14,8 @@ import { SolrManager } from './SearchManager';
import * as uuid from 'uuid';
import { DashVersion } from '../../fields/DocSymbols';
import * as AdmZip from 'adm-zip';
-const imageDataUri = require('image-data-uri');
-const fs = require('fs');
+import * as imageDataUri from 'image-data-uri';
+import * as fs from 'fs';
export enum Directory {
parsed_files = 'parsed_files',
@@ -265,7 +265,7 @@ export default class UploadManager extends ApiManager {
try {
zip.extractEntryTo(entry.entryName, publicDirectory, true, false);
createReadStream(pathname).pipe(createWriteStream(targetname));
- Jimp.read(pathname).then((img:any) => {
+ Jimp.read(pathname).then(img => {
DashUploadUtils.imageResampleSizes(extension).forEach(({ width, suffix }) => {
const outputPath = InjectSize(targetname, suffix);
if (!width) createReadStream(pathname).pipe(createWriteStream(outputPath));
@@ -278,26 +278,28 @@ export default class UploadManager extends ApiManager {
}
});
const json = zip.getEntry('docs.json');
- try {
- const data = JSON.parse(json.getData().toString('utf8'), retrocycle());
- const { docs, links } = data;
- id = getId(data.id);
- const rdocs = Object.keys(docs).map(key => docs[key]);
- const ldocs = Object.keys(links).map(key => links[key]);
- [...rdocs, ...ldocs].forEach(mapFn);
- docids = rdocs.map(doc => doc.id);
- linkids = ldocs.map(link => link.id);
- await Promise.all(
- [...rdocs, ...ldocs].map(
- doc =>
- new Promise<void>(res => {
- // overwrite mongo doc with json doc contents
- Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true);
- })
- )
- );
- } catch (e) {
- console.log(e);
+ if (json) {
+ try {
+ const data = JSON.parse(json.getData().toString('utf8'), retrocycle());
+ const { docs, links } = data;
+ id = getId(data.id);
+ const rdocs = Object.keys(docs).map(key => docs[key]);
+ const ldocs = Object.keys(links).map(key => links[key]);
+ [...rdocs, ...ldocs].forEach(mapFn);
+ docids = rdocs.map(doc => doc.id);
+ linkids = ldocs.map(link => link.id);
+ await Promise.all(
+ [...rdocs, ...ldocs].map(
+ doc =>
+ new Promise<void>(res => {
+ // overwrite mongo doc with json doc contents
+ Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true);
+ })
+ )
+ );
+ } catch (e) {
+ console.log(e);
+ }
}
unlink(path_2.filepath, () => {});
}
@@ -346,7 +348,7 @@ export default class UploadManager extends ApiManager {
return imageDataUri.outputFile(uri, serverPathToFile(Directory.images, InjectSize(filename, origSuffix))).then((savedName: string) => {
const ext = path.extname(savedName).toLowerCase();
if (AcceptableMedia.imageFormats.includes(ext)) {
- Jimp.read(savedName).then((img:any) =>
+ Jimp.read(savedName).then(img =>
(!origSuffix ? [{ width: 400, suffix: SizeSuffix.Medium }] : Object.values(DashUploadUtils.Sizes)) //
.forEach(({ width, suffix }) => {
const outputPath = serverPathToFile(Directory.images, InjectSize(filename, suffix) + ext);
diff --git a/src/server/DashStats.ts b/src/server/DashStats.ts
index 8d341db63..b6164832f 100644
--- a/src/server/DashStats.ts
+++ b/src/server/DashStats.ts
@@ -3,23 +3,23 @@ import { Response } from 'express';
import SocketIO from 'socket.io';
import { timeMap } from './ApiManagers/UserManager';
import { WebSocket } from './websocket';
-const fs = require('fs');
+import * as fs from 'fs';
/**
* DashStats focuses on tracking user data for each session.
- *
+ *
* This includes time connected, number of operations, and
* the rate of their operations
*/
export namespace DashStats {
- export const SAMPLING_INTERVAL = 1000; // in milliseconds (ms) - Time interval to update the frontend.
+ export const SAMPLING_INTERVAL = 1000; // in milliseconds (ms) - Time interval to update the frontend.
export const RATE_INTERVAL = 10; // in seconds (s) - Used to calculate rate
const statsCSVFilename = './src/server/stats/userLoginStats.csv';
const columns = ['USERNAME', 'ACTION', 'TIME'];
/**
- * UserStats holds the stats associated with a particular user.
+ * UserStats holds the stats associated with a particular user.
*/
interface UserStats {
socketId: string;
@@ -30,18 +30,18 @@ export namespace DashStats {
}
/**
- * UserLastOperations is the queue object for each user
- * storing their past operations.
+ * UserLastOperations is the queue object for each user
+ * storing their past operations.
*/
interface UserLastOperations {
sampleOperations: number; // stores how many operations total are in this rate section (10 sec, for example)
lastSampleOperations: number; // stores how many total operations were recorded at the last sample
- previousOperationsQueue: number[]; // stores the operations to calculate rate.
+ previousOperationsQueue: number[]; // stores the operations to calculate rate.
}
/**
* StatsDataBundle represents an object that will be sent to the frontend view
- * on each websocket update.
+ * on each websocket update.
*/
interface StatsDataBundle {
connectedUsers: UserStats[];
@@ -57,48 +57,44 @@ export namespace DashStats {
}
/**
- * ServerTraffic describes the current traffic going to the backend.
+ * ServerTraffic describes the current traffic going to the backend.
*/
enum ServerTraffic {
NOT_BUSY,
BUSY,
- VERY_BUSY
+ VERY_BUSY,
}
// These values can be changed after further testing how many
- // users correspond to each traffic level in Dash.
+ // users correspond to each traffic level in Dash.
const BUSY_SERVER_BOUND = 2;
const VERY_BUSY_SERVER_BOUND = 3;
- const serverTrafficMessages = [
- "Not Busy",
- "Busy",
- "Very Busy"
- ]
+ const serverTrafficMessages = ['Not Busy', 'Busy', 'Very Busy'];
// lastUserOperations maps each username to a UserLastOperations
- // structure
+ // structure
export const lastUserOperations = new Map<string, UserLastOperations>();
/**
* handleStats is called when the /stats route is called, providing a JSON
* object with relevant stats. In this case, we return the number of
- * current connections and
+ * current connections and
* @param res Response object from Express
*/
export function handleStats(res: Response) {
let current = getCurrentStats();
const results: CSVStore[] = [];
res.json({
- currentConnections: current.length,
- socketMap: current,
- });
+ currentConnections: current.length,
+ socketMap: current,
+ });
}
/**
- * getUpdatedStatesBundle() sends an updated copy of the current stats to the
- * frontend /statsview route via websockets.
- *
+ * getUpdatedStatesBundle() sends an updated copy of the current stats to the
+ * frontend /statsview route via websockets.
+ *
* @returns a StatsDataBundle that is sent to the frontend view on each websocket update
*/
export function getUpdatedStatsBundle(): StatsDataBundle {
@@ -106,43 +102,43 @@ export namespace DashStats {
return {
connectedUsers: current,
- }
+ };
}
/**
- * handleStatsView() is called when the /statsview route is called. This
+ * handleStatsView() is called when the /statsview route is called. This
* will use pug to render a frontend view of the current stats
- *
- * @param res
+ *
+ * @param res
*/
export function handleStatsView(res: Response) {
let current = getCurrentStats();
- let connectedUsers = current.map((socketPair) => {
- return socketPair.time + " - " + socketPair.username + " Operations: " + socketPair.operations;
- })
+ let connectedUsers = current.map(socketPair => {
+ return socketPair.time + ' - ' + socketPair.username + ' Operations: ' + socketPair.operations;
+ });
let serverTraffic = ServerTraffic.NOT_BUSY;
- if(current.length < BUSY_SERVER_BOUND) {
+ if (current.length < BUSY_SERVER_BOUND) {
serverTraffic = ServerTraffic.NOT_BUSY;
- } else if(current.length >= BUSY_SERVER_BOUND && current.length < VERY_BUSY_SERVER_BOUND) {
+ } else if (current.length >= BUSY_SERVER_BOUND && current.length < VERY_BUSY_SERVER_BOUND) {
serverTraffic = ServerTraffic.BUSY;
} else {
serverTraffic = ServerTraffic.VERY_BUSY;
}
-
- res.render("stats.pug", {
- title: "Dash Stats",
+
+ res.render('stats.pug', {
+ title: 'Dash Stats',
numConnections: connectedUsers.length,
serverTraffic: serverTraffic,
- serverTrafficMessage : serverTrafficMessages[serverTraffic],
- connectedUsers: connectedUsers
+ serverTrafficMessage: serverTrafficMessages[serverTraffic],
+ connectedUsers: connectedUsers,
});
}
/**
- * logUserLogin() writes a login event to the CSV file.
- *
+ * logUserLogin() writes a login event to the CSV file.
+ *
* @param username the username in the format of "username@domain.com logged in"
* @param socket the websocket associated with the current connection
*/
@@ -152,12 +148,12 @@ export namespace DashStats {
console.log(magenta(`User ${username.split(' ')[0]} logged in at: ${currentDate.toISOString()}`));
let toWrite: CSVStore = {
- USERNAME : username,
- ACTION : "loggedIn",
- TIME : currentDate.toISOString()
- }
+ USERNAME: username,
+ ACTION: 'loggedIn',
+ TIME: currentDate.toISOString(),
+ };
- let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"});
+ let statsFile = fs.createWriteStream(statsCSVFilename, { flags: 'a' });
statsFile.write(convertToCSV(toWrite));
statsFile.end();
console.log(cyan(convertToCSV(toWrite)));
@@ -165,21 +161,21 @@ export namespace DashStats {
}
/**
- * logUserLogout() writes a logout event to the CSV file.
- *
+ * logUserLogout() writes a logout event to the CSV file.
+ *
* @param username the username in the format of "username@domain.com logged in"
- * @param socket the websocket associated with the current connection.
+ * @param socket the websocket associated with the current connection.
*/
export function logUserLogout(username: string | undefined, socket: SocketIO.Socket) {
if (!(username === undefined)) {
let currentDate = new Date();
- let statsFile = fs.createWriteStream(statsCSVFilename, { flags: "a"});
+ let statsFile = fs.createWriteStream(statsCSVFilename, { flags: 'a' });
let toWrite: CSVStore = {
- USERNAME : username,
- ACTION : "loggedOut",
- TIME : currentDate.toISOString()
- }
+ USERNAME: username,
+ ACTION: 'loggedOut',
+ TIME: currentDate.toISOString(),
+ };
statsFile.write(convertToCSV(toWrite));
statsFile.end();
}
@@ -188,22 +184,22 @@ export namespace DashStats {
/**
* getLastOperationsOrDefault() is a helper method that will attempt
* to query the lastUserOperations map for a specified username. If the
- * username is not in the map, an empty UserLastOperations object is returned.
- * @param username
+ * username is not in the map, an empty UserLastOperations object is returned.
+ * @param username
* @returns the user's UserLastOperations structure or an empty
* UserLastOperations object (All values set to 0) if the username is not found.
*/
function getLastOperationsOrDefault(username: string): UserLastOperations {
- if(lastUserOperations.get(username) === undefined) {
+ if (lastUserOperations.get(username) === undefined) {
let initializeOperationsQueue = [];
- for(let i = 0; i < RATE_INTERVAL; i++) {
+ for (let i = 0; i < RATE_INTERVAL; i++) {
initializeOperationsQueue.push(0);
}
return {
sampleOperations: 0,
lastSampleOperations: 0,
- previousOperationsQueue: initializeOperationsQueue
- }
+ previousOperationsQueue: initializeOperationsQueue,
+ };
}
return lastUserOperations.get(username)!;
}
@@ -211,19 +207,19 @@ export namespace DashStats {
/**
* updateLastOperations updates a specific user's UserLastOperations information
* for the current sampling cycle. The method removes old/outdated counts for
- * operations from the queue and adds new data for the current sampling
- * cycle to the queue, updating the total count as it goes.
+ * operations from the queue and adds new data for the current sampling
+ * cycle to the queue, updating the total count as it goes.
* @param lastOperationData the old UserLastOperations data that must be updated
* @param currentOperations the total number of operations measured for this sampling cycle.
- * @returns the udpated UserLastOperations structure.
+ * @returns the udpated UserLastOperations structure.
*/
function updateLastOperations(lastOperationData: UserLastOperations, currentOperations: number): UserLastOperations {
// create a copy of the UserLastOperations to modify
let newLastOperationData: UserLastOperations = {
sampleOperations: lastOperationData.sampleOperations,
lastSampleOperations: lastOperationData.lastSampleOperations,
- previousOperationsQueue: lastOperationData.previousOperationsQueue.slice()
- }
+ previousOperationsQueue: lastOperationData.previousOperationsQueue.slice(),
+ };
let newSampleOperations = newLastOperationData.sampleOperations;
newSampleOperations -= newLastOperationData.previousOperationsQueue.shift()!; // removes and returns the first element of the queue
@@ -241,20 +237,20 @@ export namespace DashStats {
/**
* getUserOperationsOrDefault() is a helper method to get the user's total
- * operations for the CURRENT sampling interval. The method will return 0
+ * operations for the CURRENT sampling interval. The method will return 0
* if the username is not in the userOperations map.
* @param username the username to search the map for
- * @returns the total number of operations recorded up to this sampling cycle.
+ * @returns the total number of operations recorded up to this sampling cycle.
*/
function getUserOperationsOrDefault(username: string): number {
- return WebSocket.userOperations.get(username) === undefined ? 0 : WebSocket.userOperations.get(username)!
+ return WebSocket.userOperations.get(username) === undefined ? 0 : WebSocket.userOperations.get(username)!;
}
/**
* getCurrentStats() calculates the total stats for this cycle. In this case,
- * getCurrentStats() returns an Array of UserStats[] objects describing
+ * getCurrentStats() returns an Array of UserStats[] objects describing
* the stats for each user
- * @returns an array of UserStats storing data for each user at the current moment.
+ * @returns an array of UserStats storing data for each user at the current moment.
*/
function getCurrentStats(): UserStats[] {
let socketPairs: UserStats[] = [];
@@ -262,20 +258,20 @@ export namespace DashStats {
let username = value.split(' ')[0];
let connectionTime = new Date(timeMap[username]);
- let connectionTimeString = connectionTime.toLocaleDateString() + " " + connectionTime.toLocaleTimeString();
+ let connectionTimeString = connectionTime.toLocaleDateString() + ' ' + connectionTime.toLocaleTimeString();
if (!key.disconnected) {
let lastRecordedOperations = getLastOperationsOrDefault(username);
let currentUserOperationCount = getUserOperationsOrDefault(username);
- socketPairs.push({
+ socketPairs.push({
socketId: key.id,
username: username,
- time: connectionTimeString.includes("Invalid Date") ? "" : connectionTimeString,
- operations : WebSocket.userOperations.get(username) ? WebSocket.userOperations.get(username)! : 0,
- rate: lastRecordedOperations.sampleOperations
+ time: connectionTimeString.includes('Invalid Date') ? '' : connectionTimeString,
+ operations: WebSocket.userOperations.get(username) ? WebSocket.userOperations.get(username)! : 0,
+ rate: lastRecordedOperations.sampleOperations,
});
- lastUserOperations.set(username, updateLastOperations(lastRecordedOperations,currentUserOperationCount));
+ lastUserOperations.set(username, updateLastOperations(lastRecordedOperations, currentUserOperationCount));
}
}
return socketPairs;
@@ -283,9 +279,9 @@ export namespace DashStats {
/**
* convertToCSV() is a helper method that stringifies a CSVStore object
- * that can be written to the CSV file later.
+ * that can be written to the CSV file later.
* @param dataObject the object to stringify
- * @returns the object as a string.
+ * @returns the object as a string.
*/
function convertToCSV(dataObject: CSVStore): string {
return `${dataObject.USERNAME},${dataObject.ACTION},${dataObject.TIME}\n`;
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 0161a8d38..a8e09818e 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -20,12 +20,12 @@ import { AzureManager } from './ApiManagers/AzureManager';
import axios from 'axios';
const spawn = require('child_process').spawn;
const { exec } = require('child_process');
-const parse = require('pdf-parse');
-const ffmpeg = require('fluent-ffmpeg');
-const fs = require('fs');
const requestImageSize = require('../client/util/request-image-size');
-const md5File = require('md5-file');
-const autorotate = require('jpeg-autorotate');
+import * as parse from 'pdf-parse';
+import { ffprobe, FfmpegCommand } from 'fluent-ffmpeg';
+import * as fs from 'fs';
+import * as md5File from 'md5-file';
+import * as autorotate from 'jpeg-autorotate';
export enum SizeSuffix {
Small = '_s',
@@ -95,7 +95,7 @@ export namespace DashUploadUtils {
// concatenate the videos
await new Promise((resolve, reject) => {
- var merge = ffmpeg();
+ var merge = new FfmpegCommand();
merge
.input(textFilePath)
.inputOptions(['-f concat', '-safe 0'])
@@ -152,7 +152,7 @@ export namespace DashUploadUtils {
exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
const time = Array.from(stdout.trim().split(':')).reverse();
const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
- res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined));
+ res(resolveExistingFile(name, filepath, Directory.videos, 'video/mp4', duration, undefined));
});
} else {
uploadProgress.set(overwriteId, 'starting download');
@@ -185,9 +185,12 @@ export namespace DashUploadUtils {
exec(`yt-dlp-o ${filepath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
const time = Array.from(stdout.trim().split(':')).reverse();
const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
- const data = { size: 0, filepath, name, mimetype: '', originalFilename: name, newFilename: name, hashAlgorithm: 'md5' as 'md5', type: 'video/mp4' };
- const file = { ...data, toJSON: () => ({ ...data, length: 0, filename: data.filepath.replace(/.*\//, ''), mtime: new Date(), mime: '', hashAlgorithm: 'md5' as 'md5', toJson: () => undefined as any }) };
- res(MoveParsedFile(file, Directory.videos));
+ const data = { size: 0, filepath, name, mimetype: 'video', originalFilename: name, newFilename: name, hashAlgorithm: 'md5' as 'md5', type: 'video/mp4' };
+ const file = { ...data, toJSON: () => ({ ...data, length: 0, filename: data.filepath.replace(/.*\//, ''), mtime: new Date(), toJson: () => undefined as any }) };
+ MoveParsedFile(file, Directory.videos).then(output => {
+ console.log('OUTPUT = ' + output);
+ res(output);
+ });
});
}
});
@@ -217,7 +220,7 @@ export namespace DashUploadUtils {
if (format.includes('x-matroska')) {
console.log('case video');
await new Promise(res =>
- ffmpeg(file.filepath)
+ new FfmpegCommand(file.filepath)
.videoCodec('copy') // this will copy the data instead of reencode it
.save(file.filepath.replace('.mkv', '.mp4'))
.on('end', res)
@@ -229,7 +232,7 @@ export namespace DashUploadUtils {
if (format.includes('quicktime')) {
let abort = false;
await new Promise<void>(res =>
- ffmpeg.ffprobe(file.filepath, (err: any, metadata: any) => {
+ ffprobe(file.filepath, (err: any, metadata: any) => {
if (metadata.streams.some((stream: any) => stream.codec_name === 'hevc')) {
abort = true;
}
@@ -615,13 +618,13 @@ export namespace DashUploadUtils {
buffer = buffer2;
} catch (e) {}
return Jimp.read(buffer ?? fileIn)
- .then(async (img:any) => {
+ .then(async (img: any) => {
await Promise.all( sizes.filter(({ width }) => width) .map(({ width, suffix }) =>
img = img.resize(width, Jimp.AUTO).write(outputPath(suffix))
)); // prettier-ignore
return writtenFiles;
})
- .catch((e:any) => {
+ .catch((e: any) => {
console.log('ERROR' + e);
return writtenFiles;
});
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-12-14 00:15:15 +0000