aboutsummaryrefslogtreecommitdiff
path: root/src/server
diff options
context:
space:
mode:
Diffstat (limited to 'src/server')
-rw-r--r--src/server/GarbageCollector.ts150
-rw-r--r--src/server/Message.ts2
-rw-r--r--src/server/RouteStore.ts1
-rw-r--r--src/server/Search.ts48
-rw-r--r--src/server/authentication/controllers/user_controller.ts41
-rw-r--r--src/server/authentication/models/current_user_utils.ts71
-rw-r--r--src/server/authentication/models/user_model.ts4
-rw-r--r--src/server/database.ts22
-rw-r--r--src/server/index.ts258
-rw-r--r--src/server/updateSearch.ts34
10 files changed, 558 insertions, 73 deletions
diff --git a/src/server/GarbageCollector.ts b/src/server/GarbageCollector.ts
new file mode 100644
index 000000000..ea5388004
--- /dev/null
+++ b/src/server/GarbageCollector.ts
@@ -0,0 +1,150 @@
+import { Database } from './database';
+
+import * as path from 'path';
+import * as fs from 'fs';
+import { Search } from './Search';
+
+function addDoc(doc: any, ids: string[], files: { [name: string]: string[] }) {
+ for (const key in doc) {
+ if (!doc.hasOwnProperty(key)) {
+ continue;
+ }
+ const field = doc[key];
+ if (field === undefined || field === null) {
+ continue;
+ }
+ if (field.__type === "proxy") {
+ ids.push(field.fieldId);
+ } else if (field.__type === "list") {
+ addDoc(field.fields, ids, files);
+ } else if (typeof field === "string") {
+ const re = /"(?:dataD|d)ocumentId"\s*:\s*"([\w\-]*)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field)) !== null) {
+ ids.push(match[1]);
+ }
+ } else if (field.__type === "RichTextField") {
+ const re = /"href"\s*:\s*"(.*?)"/g;
+ let match: string[] | null;
+ while ((match = re.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const split = new URL(urlString).pathname.split("doc/");
+ if (split.length > 1) {
+ ids.push(split[split.length - 1]);
+ }
+ }
+ const re2 = /"src"\s*:\s*"(.*?)"/g;
+ while ((match = re2.exec(field.Data)) !== null) {
+ const urlString = match[1];
+ const pathname = new URL(urlString).pathname;
+ const ext = path.extname(pathname);
+ const fileName = path.basename(pathname, ext);
+ let exts = files[fileName];
+ if (!exts) {
+ files[fileName] = exts = [];
+ }
+ exts.push(ext);
+ }
+ } else if (["audio", "image", "video", "pdf", "web"].includes(field.__type)) {
+ const url = new URL(field.url);
+ const pathname = url.pathname;
+ const ext = path.extname(pathname);
+ const fileName = path.basename(pathname, ext);
+ let exts = files[fileName];
+ if (!exts) {
+ files[fileName] = exts = [];
+ }
+ exts.push(ext);
+ }
+ }
+}
+
+async function GarbageCollect(full: boolean = true) {
+ console.log("start GC");
+ const start = Date.now();
+ // await new Promise(res => setTimeout(res, 3000));
+ const cursor = await Database.Instance.query({}, { userDocumentId: 1 }, 'users');
+ const users = await cursor.toArray();
+ const ids: string[] = users.map(user => user.userDocumentId);
+ const visited = new Set<string>();
+ const files: { [name: string]: string[] } = {};
+
+ while (ids.length) {
+ const count = Math.min(ids.length, 1000);
+ const index = ids.length - count;
+ const fetchIds = ids.splice(index, count).filter(id => !visited.has(id));
+ if (!fetchIds.length) {
+ continue;
+ }
+ const docs = await new Promise<{ [key: string]: any }[]>(res => Database.Instance.getDocuments(fetchIds, res, "newDocuments"));
+ for (const doc of docs) {
+ const id = doc.id;
+ if (doc === undefined) {
+ console.log(`Couldn't find field with Id ${id}`);
+ continue;
+ }
+ visited.add(id);
+ addDoc(doc.fields, ids, files);
+ }
+ console.log(`To Go: ${ids.length}, visited: ${visited.size}`);
+ }
+
+ console.log(`Done: ${visited.size}`);
+
+ cursor.close();
+
+ const notToDelete = Array.from(visited);
+ const toDeleteCursor = await Database.Instance.query({ _id: { $nin: notToDelete } }, { _id: 1 });
+ const toDelete: string[] = (await toDeleteCursor.toArray()).map(doc => doc._id);
+ toDeleteCursor.close();
+ if (!full) {
+ await Database.Instance.updateMany({ _id: { $nin: notToDelete } }, { $set: { "deleted": true } });
+ await Database.Instance.updateMany({ _id: { $in: notToDelete } }, { $unset: { "deleted": true } });
+ console.log(await Search.Instance.updateDocuments(
+ notToDelete.map<any>(id => ({
+ id, deleted: { set: null }
+ }))
+ .concat(toDelete.map(id => ({
+ id, deleted: { set: true }
+ })))));
+ console.log("Done with partial GC");
+ console.log(`Took ${(Date.now() - start) / 1000} seconds`);
+ } else {
+ let i = 0;
+ let deleted = 0;
+ while (i < toDelete.length) {
+ const count = Math.min(toDelete.length, 5000);
+ const toDeleteDocs = toDelete.slice(i, i + count);
+ i += count;
+ const result = await Database.Instance.delete({ _id: { $in: toDeleteDocs } }, "newDocuments");
+ deleted += result.deletedCount || 0;
+ }
+ // const result = await Database.Instance.delete({ _id: { $in: toDelete } }, "newDocuments");
+ console.log(`${deleted} documents deleted`);
+
+ await Search.Instance.deleteDocuments(toDelete);
+ console.log("Cleared search documents");
+
+ const folder = "./src/server/public/files/";
+ fs.readdir(folder, (_, fileList) => {
+ const filesToDelete = fileList.filter(file => {
+ const ext = path.extname(file);
+ let base = path.basename(file, ext);
+ const existsInDb = (base in files || (base = base.substring(0, base.length - 2)) in files) && files[base].includes(ext);
+ return file !== ".gitignore" && !existsInDb;
+ });
+ console.log(`Deleting ${filesToDelete.length} files`);
+ filesToDelete.forEach(file => {
+ console.log(`Deleting file ${file}`);
+ try {
+ fs.unlinkSync(folder + file);
+ } catch {
+ console.warn(`Couldn't delete file ${file}`);
+ }
+ });
+ console.log(`Deleted ${filesToDelete.length} files`);
+ });
+ }
+}
+
+GarbageCollect(false);
diff --git a/src/server/Message.ts b/src/server/Message.ts
index 7b208dee9..1e29aef0b 100644
--- a/src/server/Message.ts
+++ b/src/server/Message.ts
@@ -55,4 +55,6 @@ export namespace MessageStore {
export const UpdateField = new Message<Diff>("Update Ref Field");
export const CreateField = new Message<Reference>("Create Ref Field");
export const YoutubeApiQuery = new Message<YoutubeQueryInput>("Youtube Api Query");
+ export const DeleteField = new Message<string>("Delete field");
+ export const DeleteFields = new Message<string[]>("Delete fields");
}
diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts
index c4af5cdaa..5c13495ff 100644
--- a/src/server/RouteStore.ts
+++ b/src/server/RouteStore.ts
@@ -16,6 +16,7 @@ export enum RouteStore {
// USER AND WORKSPACES
getCurrUser = "/getCurrentUser",
+ getUsers = "/getUsers",
getUserDocumentId = "/getUserDocumentId",
updateCursor = "/updateCursor",
diff --git a/src/server/Search.ts b/src/server/Search.ts
index d776480c6..69e327d2d 100644
--- a/src/server/Search.ts
+++ b/src/server/Search.ts
@@ -18,19 +18,34 @@ export class Search {
}
}
- public async search(query: string) {
+ public async updateDocuments(documents: any[]) {
+ try {
+ const res = await rp.post(this.url + "dash/update", {
+ headers: { 'content-type': 'application/json' },
+ body: JSON.stringify(documents)
+ });
+ return res;
+ } catch (e) {
+ // console.warn("Search error: ", e, documents);
+ }
+ }
+
+ public async search(query: string, filterQuery: string = "", start: number = 0, rows: number = 10) {
try {
const searchResults = JSON.parse(await rp.get(this.url + "dash/select", {
qs: {
q: query,
- fl: "id"
+ fq: filterQuery,
+ fl: "id",
+ start,
+ rows,
}
}));
- const fields = searchResults.response.docs;
- const ids = fields.map((field: any) => field.id);
- return ids;
+ const { docs, numFound } = searchResults.response;
+ const ids = docs.map((field: any) => field.id);
+ return { ids, numFound };
} catch {
- return [];
+ return { ids: [], numFound: -1 };
}
}
@@ -46,4 +61,25 @@ export class Search {
});
} catch { }
}
+
+ public deleteDocuments(docs: string[]) {
+ const promises: rp.RequestPromise[] = [];
+ const nToDelete = 1000;
+ let index = 0;
+ while (index < docs.length) {
+ const count = Math.min(docs.length - index, nToDelete);
+ const deleteIds = docs.slice(index, index + count);
+ index += count;
+ promises.push(rp.post(this.url + "dash/update", {
+ body: {
+ delete: {
+ query: deleteIds.map(id => `id:"${id}"`).join(" ")
+ }
+ },
+ json: true
+ }));
+ }
+
+ return Promise.all(promises);
+ }
} \ No newline at end of file
diff --git a/src/server/authentication/controllers/user_controller.ts b/src/server/authentication/controllers/user_controller.ts
index 1dacdf3fa..0e431f1e6 100644
--- a/src/server/authentication/controllers/user_controller.ts
+++ b/src/server/authentication/controllers/user_controller.ts
@@ -12,6 +12,9 @@ import * as nodemailer from 'nodemailer';
import c = require("crypto");
import { RouteStore } from "../../RouteStore";
import { Utils } from "../../../Utils";
+import { Schema } from "mongoose";
+import { Opt } from "../../../new_fields/Doc";
+import { MailOptions } from "nodemailer/lib/stream-transport";
/**
* GET /signup
@@ -42,40 +45,45 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => {
const errors = req.validationErrors();
if (errors) {
- res.render("signup.pug", {
- title: "Sign Up",
- user: req.user,
- });
return res.redirect(RouteStore.signup);
}
- const email = req.body.email;
+ const email = req.body.email as String;
const password = req.body.password;
- const user = new User({
+ const model = {
email,
password,
userDocumentId: Utils.GenerateGuid()
- });
+ } as Partial<DashUserModel>;
+
+ const user = new User(model);
User.findOne({ email }, (err, existingUser) => {
if (err) { return next(err); }
if (existingUser) {
return res.redirect(RouteStore.login);
}
- user.save((err) => {
+ user.save((err: any) => {
if (err) { return next(err); }
req.logIn(user, (err) => {
- if (err) {
- return next(err);
- }
- res.redirect(RouteStore.home);
+ if (err) { return next(err); }
+ tryRedirectToTarget(req, res);
});
});
});
};
+let tryRedirectToTarget = (req: Request, res: Response) => {
+ if (req.session && req.session.target) {
+ res.redirect(req.session.target);
+ req.session.target = undefined;
+ } else {
+ res.redirect(RouteStore.home);
+ }
+};
+
/**
* GET /login
@@ -83,6 +91,7 @@ export let postSignup = (req: Request, res: Response, next: NextFunction) => {
*/
export let getLogin = (req: Request, res: Response) => {
if (req.user) {
+ req.session!.target = undefined;
return res.redirect(RouteStore.home);
}
res.render("login.pug", {
@@ -115,7 +124,7 @@ export let postLogin = (req: Request, res: Response, next: NextFunction) => {
}
req.logIn(user, (err) => {
if (err) { next(err); return; }
- res.redirect(RouteStore.home);
+ tryRedirectToTarget(req, res);
});
})(req, res, next);
};
@@ -184,8 +193,8 @@ export let postForgot = function (req: Request, res: Response, next: NextFunctio
'Please click on the following link, or paste this into your browser to complete the process:\n\n' +
'http://' + req.headers.host + '/reset/' + token + '\n\n' +
'If you did not request this, please ignore this email and your password will remain unchanged.\n'
- };
- smtpTransport.sendMail(mailOptions, function (err) {
+ } as MailOptions;
+ smtpTransport.sendMail(mailOptions, function (err: Error | null) {
// req.flash('info', 'An e-mail has been sent to ' + user.email + ' with further instructions.');
done(null, err, 'done');
});
@@ -255,7 +264,7 @@ export let postReset = function (req: Request, res: Response) {
subject: 'Your password has been changed',
text: 'Hello,\n\n' +
'This is a confirmation that the password for your account ' + user.email + ' has just been changed.\n'
- };
+ } as MailOptions;
smtpTransport.sendMail(mailOptions, function (err) {
done(null, err);
});
diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts
index e5b7a025b..e796ccb43 100644
--- a/src/server/authentication/models/current_user_utils.ts
+++ b/src/server/authentication/models/current_user_utils.ts
@@ -10,7 +10,7 @@ import { CollectionView } from "../../../client/views/collections/CollectionView
import { Doc } from "../../../new_fields/Doc";
import { List } from "../../../new_fields/List";
import { listSpec } from "../../../new_fields/Schema";
-import { Cast } from "../../../new_fields/Types";
+import { Cast, FieldValue, StrCast } from "../../../new_fields/Types";
import { RouteStore } from "../../RouteStore";
export class CurrentUserUtils {
@@ -29,30 +29,74 @@ export class CurrentUserUtils {
private static createUserDocument(id: string): Doc {
let doc = new Doc(id, true);
doc.viewType = CollectionViewType.Tree;
+ doc.dropAction = "alias";
doc.layout = CollectionView.LayoutString();
doc.title = this.email;
+ this.updateUserDocument(doc);
doc.data = new List<Doc>();
+ doc.gridGap = 5;
+ doc.xMargin = 5;
+ doc.yMargin = 5;
+ doc.boxShadow = "0 0";
doc.excludeFromLibrary = true;
- doc.optionalRightCollection = Docs.StackingDocument([], { title: "New mobile uploads" });
- // doc.library = Docs.TreeDocument([doc], { title: `Library: ${CurrentUserUtils.email}` });
+ doc.optionalRightCollection = Docs.Create.StackingDocument([], { title: "New mobile uploads" });
+ // doc.library = Docs.Create.TreeDocument([doc], { title: `Library: ${CurrentUserUtils.email}` });
// (doc.library as Doc).excludeFromLibrary = true;
return doc;
}
- public static async loadCurrentUser(): Promise<any> {
- let userPromise = rp.get(DocServer.prepend(RouteStore.getCurrUser)).then(response => {
+ static updateUserDocument(doc: Doc) {
+ if (doc.workspaces === undefined) {
+ const workspaces = Docs.Create.TreeDocument([], { title: "Workspaces", height: 100 });
+ workspaces.excludeFromLibrary = true;
+ workspaces.workspaceLibrary = true;
+ workspaces.boxShadow = "0 0";
+ doc.workspaces = workspaces;
+ }
+ if (doc.recentlyClosed === undefined) {
+ const recentlyClosed = Docs.Create.TreeDocument([], { title: "Recently Closed", height: 75 });
+ recentlyClosed.excludeFromLibrary = true;
+ recentlyClosed.boxShadow = "0 0";
+ doc.recentlyClosed = recentlyClosed;
+ }
+ if (doc.sidebar === undefined) {
+ const sidebar = Docs.Create.StackingDocument([doc.workspaces as Doc, doc, doc.recentlyClosed as Doc], { title: "Sidebar" });
+ sidebar.excludeFromLibrary = true;
+ sidebar.gridGap = 5;
+ sidebar.xMargin = 5;
+ sidebar.yMargin = 5;
+ Doc.GetProto(sidebar).backgroundColor = "#aca3a6";
+ sidebar.boxShadow = "1 1 3";
+ doc.sidebar = sidebar;
+ }
+ StrCast(doc.title).indexOf("@") !== -1 && (doc.title = StrCast(doc.title).split("@")[0] + "'s Library");
+
+ }
+
+ public static loadCurrentUser() {
+ return rp.get(DocServer.prepend(RouteStore.getCurrUser)).then(response => {
if (response) {
- let obj = JSON.parse(response);
- CurrentUserUtils.curr_id = obj.id as string;
- CurrentUserUtils.curr_email = obj.email as string;
+ const result: { id: string, email: string } = JSON.parse(response);
+ return result;
} else {
throw new Error("There should be a user! Why does Dash think there isn't one?");
}
});
- let userDocPromise = await rp.get(DocServer.prepend(RouteStore.getUserDocumentId)).then(id => {
+ }
+
+ public static async loadUserDocument({ id, email }: { id: string, email: string }) {
+ this.curr_id = id;
+ this.curr_email = email;
+ await rp.get(DocServer.prepend(RouteStore.getUserDocumentId)).then(id => {
if (id) {
- return DocServer.GetRefField(id).then(field =>
- runInAction(() => this.user_document = field instanceof Doc ? field : this.createUserDocument(id)));
+ return DocServer.GetRefField(id).then(async field => {
+ if (field instanceof Doc) {
+ await this.updateUserDocument(field);
+ runInAction(() => this.user_document = field);
+ } else {
+ runInAction(() => this.user_document = this.createUserDocument(id));
+ }
+ });
} else {
throw new Error("There should be a user id! Why does Dash think there isn't one?");
}
@@ -68,7 +112,6 @@ export class CurrentUserUtils {
} catch (e) {
}
- return Promise.all([userPromise, userDocPromise]);
}
/* Northstar catalog ... really just for testing so this should eventually go away */
@@ -94,12 +137,12 @@ export class CurrentUserUtils {
// new AttributeTransformationModel(atmod, AggregateFunction.None),
// new AttributeTransformationModel(atmod, AggregateFunction.Count),
// new AttributeTransformationModel(atmod, AggregateFunction.Count));
- // schemaDocuments.push(Docs.HistogramDocument(histoOp, { width: 200, height: 200, title: attr.displayName! }));
+ // schemaDocuments.push(Docs.Create.HistogramDocument(histoOp, { width: 200, height: 200, title: attr.displayName! }));
// }
// })));
// return promises;
// }, [] as Promise<void>[]));
- // return CurrentUserUtils._northstarSchemas.push(Docs.TreeDocument(schemaDocuments, { width: 50, height: 100, title: schema.displayName! }));
+ // return CurrentUserUtils._northstarSchemas.push(Docs.Create.TreeDocument(schemaDocuments, { width: 50, height: 100, title: schema.displayName! }));
// });
// }
}
diff --git a/src/server/authentication/models/user_model.ts b/src/server/authentication/models/user_model.ts
index ee85e1c05..45fbf23b1 100644
--- a/src/server/authentication/models/user_model.ts
+++ b/src/server/authentication/models/user_model.ts
@@ -16,7 +16,7 @@ mongoose.connection.on('disconnected', function () {
console.log('connection closed');
});
export type DashUserModel = mongoose.Document & {
- email: string,
+ email: String,
password: string,
passwordResetToken?: string,
passwordResetExpires?: Date,
@@ -42,7 +42,7 @@ export type AuthToken = {
};
const userSchema = new mongoose.Schema({
- email: { type: String, unique: true },
+ email: String,
password: String,
passwordResetToken: String,
passwordResetExpires: Date,
diff --git a/src/server/database.ts b/src/server/database.ts
index 70b3efced..29a8ffafa 100644
--- a/src/server/database.ts
+++ b/src/server/database.ts
@@ -41,11 +41,17 @@ export class Database {
}
}
- public delete(id: string, collectionName = Database.DocumentsCollection) {
+ public delete(query: any, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
+ public delete(id: string, collectionName?: string): Promise<mongodb.DeleteWriteOpResultObject>;
+ public delete(id: any, collectionName = Database.DocumentsCollection) {
+ if (typeof id === "string") {
+ id = { _id: id };
+ }
if (this.db) {
- this.db.collection(collectionName).remove({ id: id });
+ const db = this.db;
+ return new Promise(res => db.collection(collectionName).deleteMany(id, (err, result) => res(result)));
} else {
- this.onConnect.push(() => this.delete(id, collectionName));
+ return new Promise(res => this.onConnect.push(() => res(this.delete(id, collectionName))));
}
}
@@ -120,12 +126,16 @@ export class Database {
}
}
- public query(query: any): Promise<mongodb.Cursor> {
+ public query(query: { [key: string]: any }, projection?: { [key: string]: 0 | 1 }, collectionName = "newDocuments"): Promise<mongodb.Cursor> {
if (this.db) {
- return Promise.resolve<mongodb.Cursor>(this.db.collection('newDocuments').find(query));
+ let cursor = this.db.collection(collectionName).find(query);
+ if (projection) {
+ cursor = cursor.project(projection);
+ }
+ return Promise.resolve<mongodb.Cursor>(cursor);
} else {
return new Promise<mongodb.Cursor>(res => {
- this.onConnect.push(() => res(this.query(query)));
+ this.onConnect.push(() => res(this.query(query, projection, collectionName)));
});
}
}
diff --git a/src/server/index.ts b/src/server/index.ts
index 9e4ad9d86..026950812 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -1,3 +1,4 @@
+require('dotenv').config();
import * as bodyParser from 'body-parser';
import { exec } from 'child_process';
import * as cookieParser from 'cookie-parser';
@@ -7,9 +8,9 @@ import * as expressValidator from 'express-validator';
import * as formidable from 'formidable';
import * as fs from 'fs';
import * as sharp from 'sharp';
+import * as Pdfjs from 'pdfjs-dist';
const imageDataUri = require('image-data-uri');
import * as mobileDetect from 'mobile-detect';
-import { ObservableMap } from 'mobx';
import * as passport from 'passport';
import * as path from 'path';
import * as request from 'request';
@@ -28,6 +29,7 @@ import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQu
import { RouteStore } from './RouteStore';
const app = express();
const config = require('../../webpack.config');
+import { createCanvas, loadImage, Canvas } from "canvas";
const compiler = webpack(config);
const port = 1050; // default port to listen
const serverPort = 4321;
@@ -38,18 +40,28 @@ import { Search } from './Search';
import { debug } from 'util';
import _ = require('lodash');
import * as YoutubeApi from './youtubeApi/youtubeApiSample.js';
+import { Response } from 'express-serve-static-core';
const MongoStore = require('connect-mongo')(session);
const mongoose = require('mongoose');
-// let { google } = require('googleapis');
-// let OAuth2 = google.auth.OAuth2;
-
+const probe = require("probe-image-size");
const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest));
let youtubeApiKey: string;
YoutubeApi.readApiKey((apiKey: string) => youtubeApiKey = apiKey);
+const release = process.env.RELEASE === "true";
+if (process.env.RELEASE === "true") {
+ console.log("Running server in release mode");
+} else {
+ console.log("Running server in debug mode");
+}
+console.log(process.env.PWD);
+let clientUtils = fs.readFileSync("./src/client/util/ClientUtils.ts.temp", "utf8");
+clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(release))}`;
+fs.writeFileSync("./src/client/util/ClientUtils.ts", clientUtils, "utf8");
+
const mongoUrl = 'mongodb://localhost:27017/Dash';
-mongoose.connect(mongoUrl);
+mongoose.connection.readyState === 0 && mongoose.connect(mongoUrl);
mongoose.connection.on('connected', () => console.log("connected"));
// SESSION MANAGEMENT AND AUTHENTICATION MIDDLEWARE
@@ -94,14 +106,15 @@ enum Method {
*/
function addSecureRoute(method: Method,
handler: (user: DashUserModel, res: express.Response, req: express.Request) => void,
- onRejection: (res: express.Response) => any = (res) => res.redirect(RouteStore.logout),
+ onRejection: (res: express.Response, req: express.Request) => any = res => res.redirect(RouteStore.login),
...subscribers: string[]
) {
let abstracted = (req: express.Request, res: express.Response) => {
if (req.user) {
handler(req.user, res, req);
} else {
- onRejection(res);
+ req.session!.target = `${req.headers.host}${req.originalUrl}`;
+ onRejection(res, req);
}
};
subscribers.forEach(route => {
@@ -134,11 +147,100 @@ app.get("/pull", (req, res) =>
// GETTERS
app.get("/search", async (req, res) => {
- let query = req.query.query || "hello";
- let results = await Search.Instance.search(query);
+ const { query, filterQuery, start, rows } = req.query;
+ if (query === undefined) {
+ res.send([]);
+ return;
+ }
+ let results = await Search.Instance.search(query, filterQuery, start, rows);
res.send(results);
});
+function msToTime(duration: number) {
+ let milliseconds = Math.floor((duration % 1000) / 100),
+ seconds = Math.floor((duration / 1000) % 60),
+ minutes = Math.floor((duration / (1000 * 60)) % 60),
+ hours = Math.floor((duration / (1000 * 60 * 60)) % 24);
+
+ let hoursS = (hours < 10) ? "0" + hours : hours;
+ let minutesS = (minutes < 10) ? "0" + minutes : minutes;
+ let secondsS = (seconds < 10) ? "0" + seconds : seconds;
+
+ return hoursS + ":" + minutesS + ":" + secondsS + "." + milliseconds;
+}
+
+app.get("/whosOnline", (req, res) => {
+ let users: any = { active: {}, inactive: {} };
+ const now = Date.now();
+
+ for (const user in timeMap) {
+ const time = timeMap[user];
+ const key = ((now - time) / 1000) < (60 * 5) ? "active" : "inactive";
+ users[key][user] = `Last active ${msToTime(now - time)} ago`;
+ }
+
+ res.send(users);
+});
+
+app.get("/thumbnail/:filename", (req, res) => {
+ let filename = req.params.filename;
+ let noExt = filename.substring(0, filename.length - ".png".length);
+ let pagenumber = parseInt(noExt[noExt.length - 1]);
+ fs.exists(uploadDir + filename, (exists: boolean) => {
+ console.log(`${uploadDir + filename} ${exists ? "exists" : "does not exist"}`);
+ if (exists) {
+ let input = fs.createReadStream(uploadDir + filename);
+ probe(input, (err: any, result: any) => {
+ if (err) {
+ console.log(err);
+ console.log(filename);
+ return;
+ }
+ res.send({ path: "/files/" + filename, width: result.width, height: result.height });
+ });
+ }
+ else {
+ LoadPage(uploadDir + filename.substring(0, filename.length - "-n.png".length) + ".pdf", pagenumber, res);
+ }
+ });
+});
+
+function LoadPage(file: string, pageNumber: number, res: Response) {
+ console.log(file);
+ Pdfjs.getDocument(file).promise
+ .then((pdf: Pdfjs.PDFDocumentProxy) => {
+ let factory = new NodeCanvasFactory();
+ console.log(pageNumber);
+ pdf.getPage(pageNumber).then((page: Pdfjs.PDFPageProxy) => {
+ console.log("reading " + page);
+ let viewport = page.getViewport(1);
+ let canvasAndContext = factory.create(viewport.width, viewport.height);
+ let renderContext = {
+ canvasContext: canvasAndContext.context,
+ viewport: viewport,
+ canvasFactory: factory
+ };
+ console.log("read " + pageNumber);
+
+ page.render(renderContext).promise
+ .then(() => {
+ console.log("saving " + pageNumber);
+ let stream = canvasAndContext.canvas.createPNGStream();
+ let pngFile = `${file.substring(0, file.length - ".pdf".length)}-${pageNumber}.PNG`;
+ let out = fs.createWriteStream(pngFile);
+ stream.pipe(out);
+ out.on("finish", () => {
+ console.log(`Success! Saved to ${pngFile}`);
+ let name = path.basename(pngFile);
+ res.send({ path: "/files/" + name, width: viewport.width, height: viewport.height });
+ });
+ }, (reason: string) => {
+ console.error(reason + ` ${pageNumber}`);
+ });
+ });
+ });
+}
+
// anyone attempting to navigate to localhost at this port will
// first have to login
addSecureRoute(
@@ -150,6 +252,17 @@ addSecureRoute(
addSecureRoute(
Method.GET,
+ async (_, res) => {
+ const cursor = await Database.Instance.query({}, { email: 1, userDocumentId: 1 }, "users");
+ const results = await cursor.toArray();
+ res.send(results.map(user => ({ email: user.email, userDocumentId: user.userDocumentId })));
+ },
+ undefined,
+ RouteStore.getUsers
+);
+
+addSecureRoute(
+ Method.GET,
(user, res, req) => {
let detector = new mobileDetect(req.headers['user-agent'] || "");
let filename = detector.mobile() !== null ? 'mobile/image.html' : 'index.html';
@@ -174,7 +287,31 @@ addSecureRoute(
RouteStore.getCurrUser
);
+class NodeCanvasFactory {
+ create = (width: number, height: number) => {
+ var canvas = createCanvas(width, height);
+ var context = canvas.getContext('2d');
+ return {
+ canvas: canvas,
+ context: context,
+ };
+ }
+
+ reset = (canvasAndContext: any, width: number, height: number) => {
+ canvasAndContext.canvas.width = width;
+ canvasAndContext.canvas.height = height;
+ }
+
+ destroy = (canvasAndContext: any) => {
+ canvasAndContext.canvas.width = 0;
+ canvasAndContext.canvas.height = 0;
+ canvasAndContext.canvas = null;
+ canvasAndContext.context = null;
+ }
+}
+
const pngTypes = [".png", ".PNG"];
+const pdfTypes = [".pdf", ".PDF"];
const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"];
const uploadDir = __dirname + "/public/files/";
// SETTERS
@@ -193,9 +330,10 @@ app.post(
const file = path.basename(files[name].path);
const ext = path.extname(file);
let resizers = [
- { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" },
- { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: "_m" },
- { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: "_l" },
+ { resizer: sharp().rotate(), suffix: "_o" },
+ { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }).rotate(), suffix: "_s" },
+ { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }).rotate(), suffix: "_m" },
+ { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }).rotate(), suffix: "_l" },
];
let isImage = false;
if (pngTypes.includes(ext)) {
@@ -209,6 +347,38 @@ app.post(
});
isImage = true;
}
+ else if (pdfTypes.includes(ext)) {
+ // Pdfjs.getDocument(uploadDir + file).promise
+ // .then((pdf: Pdfjs.PDFDocumentProxy) => {
+ // let numPages = pdf.numPages;
+ // let factory = new NodeCanvasFactory();
+ // for (let pageNum = 0; pageNum < numPages; pageNum++) {
+ // console.log(pageNum);
+ // pdf.getPage(pageNum + 1).then((page: Pdfjs.PDFPageProxy) => {
+ // console.log("reading " + pageNum);
+ // let viewport = page.getViewport(1);
+ // let canvasAndContext = factory.create(viewport.width, viewport.height);
+ // let renderContext = {
+ // canvasContext: canvasAndContext.context,
+ // viewport: viewport,
+ // canvasFactory: factory
+ // }
+ // console.log("read " + pageNum);
+
+ // page.render(renderContext).promise
+ // .then(() => {
+ // console.log("saving " + pageNum);
+ // let stream = canvasAndContext.canvas.createPNGStream();
+ // let out = fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`);
+ // stream.pipe(out);
+ // out.on("finish", () => console.log(`Success! Saved to ${uploadDir + file.substring(0, file.length - ext.length) + `-${pageNum + 1}.PNG`}`));
+ // }, (reason: string) => {
+ // console.error(reason + ` ${pageNum}`);
+ // });
+ // });
+ // }
+ // });
+ }
if (isImage) {
resizers.forEach(resizer => {
fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext));
@@ -284,11 +454,21 @@ app.post(RouteStore.reset, postReset);
app.use(RouteStore.corsProxy, (req, res) =>
req.pipe(request(req.url.substring(1))).pipe(res));
-app.get(RouteStore.delete, (req, res) =>
- deleteFields().then(() => res.redirect(RouteStore.home)));
+app.get(RouteStore.delete, (req, res) => {
+ if (release) {
+ res.send("no");
+ return;
+ }
+ deleteFields().then(() => res.redirect(RouteStore.home));
+});
-app.get(RouteStore.deleteAll, (req, res) =>
- deleteAll().then(() => res.redirect(RouteStore.home)));
+app.get(RouteStore.deleteAll, (req, res) => {
+ if (release) {
+ res.send("no");
+ return;
+ }
+ deleteAll().then(() => res.redirect(RouteStore.home));
+});
app.use(wdm(compiler, { publicPath: config.output.publicPath }));
@@ -304,20 +484,33 @@ interface Map {
}
let clients: Map = {};
+let socketMap = new Map<SocketIO.Socket, string>();
+let timeMap: { [id: string]: number } = {};
+
server.on("connection", function (socket: Socket) {
- console.log("a user has connected");
+ socket.use((packet, next) => {
+ let id = socketMap.get(socket);
+ if (id) {
+ timeMap[id] = Date.now();
+ }
+ next();
+ });
Utils.Emit(socket, MessageStore.Foo, "handshooken");
- Utils.AddServerHandler(socket, MessageStore.Bar, barReceived);
+ Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid));
Utils.AddServerHandler(socket, MessageStore.SetField, (args) => setField(socket, args));
Utils.AddServerHandlerCallback(socket, MessageStore.GetField, getField);
Utils.AddServerHandlerCallback(socket, MessageStore.GetFields, getFields);
- Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields);
+ if (!release) {
+ Utils.AddServerHandler(socket, MessageStore.DeleteAll, deleteFields);
+ }
Utils.AddServerHandler(socket, MessageStore.CreateField, CreateField);
Utils.AddServerHandlerCallback(socket, MessageStore.YoutubeApiQuery, HandleYoutubeQuery);
Utils.AddServerHandler(socket, MessageStore.UpdateField, diff => UpdateField(socket, diff));
+ Utils.AddServerHandler(socket, MessageStore.DeleteField, id => DeleteField(socket, id));
+ Utils.AddServerHandler(socket, MessageStore.DeleteFields, ids => DeleteFields(socket, ids));
Utils.AddServerHandlerCallback(socket, MessageStore.GetRefField, GetRefField);
Utils.AddServerHandlerCallback(socket, MessageStore.GetRefFields, GetRefFields);
});
@@ -336,8 +529,10 @@ async function deleteAll() {
await Search.Instance.clear();
}
-function barReceived(guid: String) {
- clients[guid.toString()] = new Client(guid.toString());
+function barReceived(socket: SocketIO.Socket, guid: string) {
+ clients[guid] = new Client(guid.toString());
+ console.log(`User ${guid} has connected`);
+ socketMap.set(socket, guid);
}
function getField([id, callback]: [string, (result?: Transferable) => void]) {
@@ -380,8 +575,8 @@ function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any
const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
"number": "_n",
"string": "_t",
- // "boolean": "_b",
- // "image": ["_t", "url"],
+ "boolean": "_b",
+ "image": ["_t", "url"],
"video": ["_t", "url"],
"pdf": ["_t", "url"],
"audio": ["_t", "url"],
@@ -453,6 +648,23 @@ function UpdateField(socket: Socket, diff: Diff) {
}
}
+function DeleteField(socket: Socket, id: string) {
+ Database.Instance.delete({ _id: id }, "newDocuments").then(() => {
+ socket.broadcast.emit(MessageStore.DeleteField.Message, id);
+ });
+
+ Search.Instance.deleteDocuments([id]);
+}
+
+function DeleteFields(socket: Socket, ids: string[]) {
+ Database.Instance.delete({ _id: { $in: ids } }, "newDocuments").then(() => {
+ socket.broadcast.emit(MessageStore.DeleteFields.Message, ids);
+ });
+
+ Search.Instance.deleteDocuments(ids);
+
+}
+
function CreateField(newValue: any) {
Database.Instance.insert(newValue, "newDocuments");
}
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
index de1fd25e1..906b795f1 100644
--- a/src/server/updateSearch.ts
+++ b/src/server/updateSearch.ts
@@ -6,7 +6,7 @@ import pLimit from 'p-limit';
const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
"number": "_n",
"string": "_t",
- // "boolean": "_b",
+ "boolean": "_b",
// "image": ["_t", "url"],
"video": ["_t", "url"],
"pdf": ["_t", "url"],
@@ -67,7 +67,7 @@ async function update() {
if ((numDocs % 50) === 0) {
console.log("updateDoc " + numDocs);
}
- console.log("doc " + numDocs);
+ // console.log("doc " + numDocs);
if (doc.__type !== "Doc") {
return;
}
@@ -88,13 +88,35 @@ async function update() {
}
if (dynfield) {
updates.push(update);
- console.log(updates.length);
+ // console.log(updates.length);
}
}
await cursor.forEach(updateDoc);
- await Promise.all(updates.map(update => {
- return limit(() => Search.Instance.updateDocument(update));
- }));
+ console.log(`Updating ${updates.length} documents`);
+ const result = await Search.Instance.updateDocuments(updates);
+ try {
+ console.log(JSON.parse(result).responseHeader.status);
+ } catch {
+ console.log("Error:");
+ // console.log(updates[i]);
+ console.log(result);
+ console.log("\n");
+ }
+ // for (let i = 0; i < updates.length; i++) {
+ // console.log(i);
+ // const result = await Search.Instance.updateDocument(updates[i]);
+ // try {
+ // console.log(JSON.parse(result).responseHeader.status);
+ // } catch {
+ // console.log("Error:");
+ // console.log(updates[i]);
+ // console.log(result);
+ // console.log("\n");
+ // }
+ // }
+ // await Promise.all(updates.map(update => {
+ // return limit(() => Search.Instance.updateDocument(update));
+ // }));
cursor.close();
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-17 22:09:19 +0000