aboutsummaryrefslogtreecommitdiff
path: root/src/server
diff options
context:
space:
mode:
Diffstat (limited to 'src/server')
-rw-r--r--src/server/Message.ts7
-rw-r--r--src/server/RouteStore.ts6
-rw-r--r--src/server/apis/google/CustomizedWrapper/filters.js46
-rw-r--r--src/server/apis/google/GoogleApiServerUtils.ts119
-rw-r--r--src/server/apis/google/GooglePhotosUploadUtils.ts84
-rw-r--r--src/server/apis/google/SharedTypes.ts21
-rw-r--r--src/server/authentication/config/passport.ts3
-rw-r--r--src/server/authentication/models/current_user_utils.ts8
-rw-r--r--src/server/credentials/google_docs_credentials.json12
-rw-r--r--src/server/credentials/google_docs_token.json2
-rw-r--r--src/server/index.ts215
-rw-r--r--src/server/updateSearch.ts123
12 files changed, 379 insertions, 267 deletions
diff --git a/src/server/Message.ts b/src/server/Message.ts
index 4ec390ade..a5679797f 100644
--- a/src/server/Message.ts
+++ b/src/server/Message.ts
@@ -23,6 +23,7 @@ export interface Transferable {
readonly id: string;
readonly type: Types;
readonly data?: any;
+ readonly mongoCollection?: string;
}
export enum YoutubeQueryTypes {
@@ -43,6 +44,10 @@ export interface Diff extends Reference {
readonly diff: any;
}
+export interface SourceSpecified extends Reference {
+ readonly mongoCollection?: string;
+}
+
export namespace MessageStore {
export const Foo = new Message<string>("Foo");
export const Bar = new Message<string>("Bar");
@@ -52,7 +57,7 @@ export namespace MessageStore {
export const GetDocument = new Message<string>("Get Document");
export const DeleteAll = new Message<any>("Delete All");
- export const GetRefField = new Message<string>("Get Ref Field");
+ export const GetRefField = new Message<SourceSpecified>("Get Ref Field");
export const GetRefFields = new Message<string[]>("Get Ref Fields");
export const UpdateField = new Message<Diff>("Update Ref Field");
export const CreateField = new Message<Reference>("Create Ref Field");
diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts
index 014906054..ee9cd8a0e 100644
--- a/src/server/RouteStore.ts
+++ b/src/server/RouteStore.ts
@@ -31,6 +31,10 @@ export enum RouteStore {
// APIS
cognitiveServices = "/cognitiveservices",
- googleDocs = "/googleDocs"
+ googleDocs = "/googleDocs",
+ googlePhotosAccessToken = "/googlePhotosAccessToken",
+ googlePhotosMediaUpload = "/googlePhotosMediaUpload",
+ googlePhotosMediaDownload = "/googlePhotosMediaDownload",
+ googleDocsGet = "/googleDocsGet"
} \ No newline at end of file
diff --git a/src/server/apis/google/CustomizedWrapper/filters.js b/src/server/apis/google/CustomizedWrapper/filters.js
new file mode 100644
index 000000000..576a90b75
--- /dev/null
+++ b/src/server/apis/google/CustomizedWrapper/filters.js
@@ -0,0 +1,46 @@
+'use strict';
+
+const DateFilter = require('../common/date_filter');
+const MediaTypeFilter = require('./media_type_filter');
+const ContentFilter = require('./content_filter');
+
+class Filters {
+ constructor(includeArchivedMedia = false) {
+ this.includeArchivedMedia = includeArchivedMedia;
+ }
+
+ setDateFilter(dateFilter) {
+ this.dateFilter = dateFilter;
+ return this;
+ }
+
+ setContentFilter(contentFilter) {
+ this.contentFilter = contentFilter;
+ return this;
+ }
+
+ setMediaTypeFilter(mediaTypeFilter) {
+ this.mediaTypeFilter = mediaTypeFilter;
+ return this;
+ }
+
+ setIncludeArchivedMedia(includeArchivedMedia) {
+ this.includeArchivedMedia = includeArchivedMedia;
+ return this;
+ }
+
+ toJSON() {
+ return {
+ dateFilter: this.dateFilter instanceof DateFilter ? this.dateFilter.toJSON() : this.dateFilter,
+ mediaTypeFilter: this.mediaTypeFilter instanceof MediaTypeFilter ?
+ this.mediaTypeFilter.toJSON() :
+ this.mediaTypeFilter,
+ contentFilter: this.contentFilter instanceof ContentFilter ?
+ this.contentFilter.toJSON() :
+ this.contentFilter,
+ includeArchivedMedia: this.includeArchivedMedia
+ };
+ }
+}
+
+module.exports = Filters; \ No newline at end of file
diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts
index 8785cd974..e0bd8a800 100644
--- a/src/server/apis/google/GoogleApiServerUtils.ts
+++ b/src/server/apis/google/GoogleApiServerUtils.ts
@@ -1,10 +1,13 @@
-import { google, docs_v1, slides_v1 } from "googleapis";
+import { google } from "googleapis";
import { createInterface } from "readline";
import { readFile, writeFile } from "fs";
-import { OAuth2Client } from "google-auth-library";
+import { OAuth2Client, Credentials } from "google-auth-library";
import { Opt } from "../../../new_fields/Doc";
import { GlobalOptions } from "googleapis-common";
import { GaxiosResponse } from "gaxios";
+import request = require('request-promise');
+import * as qs from 'query-string';
+import Photos = require('googlephotos');
/**
* Server side authentication for Google Api queries.
@@ -20,6 +23,9 @@ export namespace GoogleApiServerUtils {
'presentations.readonly',
'drive',
'drive.file',
+ 'photoslibrary',
+ 'photoslibrary.appendonly',
+ 'photoslibrary.sharing'
];
export const parseBuffer = (data: Buffer) => JSON.parse(data.toString());
@@ -29,66 +35,121 @@ export namespace GoogleApiServerUtils {
Slides = "Slides"
}
-
export interface CredentialPaths {
- credentials: string;
- token: string;
+ credentialsPath: string;
+ tokenPath: string;
}
export type ApiResponse = Promise<GaxiosResponse>;
- export type ApiRouter = (endpoint: Endpoint, paramters: any) => ApiResponse;
- export type ApiHandler = (parameters: any) => ApiResponse;
+ export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse;
+ export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse;
export type Action = "create" | "retrieve" | "update";
export type Endpoint = { get: ApiHandler, create: ApiHandler, batchUpdate: ApiHandler };
export type EndpointParameters = GlobalOptions & { version: "v1" };
- export const GetEndpoint = async (sector: string, paths: CredentialPaths) => {
- return new Promise<Opt<Endpoint>>((resolve, reject) => {
- readFile(paths.credentials, (err, credentials) => {
+ export const GetEndpoint = (sector: string, paths: CredentialPaths) => {
+ return new Promise<Opt<Endpoint>>(resolve => {
+ RetrieveCredentials(paths).then(authentication => {
+ let routed: Opt<Endpoint>;
+ let parameters: EndpointParameters = { auth: authentication.client, version: "v1" };
+ switch (sector) {
+ case Service.Documents:
+ routed = google.docs(parameters).documents;
+ break;
+ case Service.Slides:
+ routed = google.slides(parameters).presentations;
+ break;
+ }
+ resolve(routed);
+ });
+ });
+ };
+
+ export const RetrieveCredentials = (paths: CredentialPaths) => {
+ return new Promise<TokenResult>((resolve, reject) => {
+ readFile(paths.credentialsPath, async (err, credentials) => {
if (err) {
reject(err);
return console.log('Error loading client secret file:', err);
}
- return authorize(parseBuffer(credentials), paths.token).then(auth => {
- let routed: Opt<Endpoint>;
- let parameters: EndpointParameters = { auth, version: "v1" };
- switch (sector) {
- case Service.Documents:
- routed = google.docs(parameters).documents;
- break;
- case Service.Slides:
- routed = google.slides(parameters).presentations;
- break;
- }
- resolve(routed);
- });
+ authorize(parseBuffer(credentials), paths.tokenPath).then(resolve, reject);
});
});
};
+ export const RetrieveAccessToken = (paths: CredentialPaths) => {
+ return new Promise<string>((resolve, reject) => {
+ RetrieveCredentials(paths).then(
+ credentials => resolve(credentials.token.access_token!),
+ error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`)
+ );
+ });
+ };
+ export const RetrievePhotosEndpoint = (paths: CredentialPaths) => {
+ return new Promise<any>((resolve, reject) => {
+ RetrieveAccessToken(paths).then(
+ token => resolve(new Photos(token)),
+ reject
+ );
+ });
+ };
+
+ type TokenResult = { token: Credentials, client: OAuth2Client };
/**
* Create an OAuth2 client with the given credentials, and returns the promise resolving to the authenticated client
* @param {Object} credentials The authorization client credentials.
*/
- export function authorize(credentials: any, token_path: string): Promise<OAuth2Client> {
+ export function authorize(credentials: any, token_path: string): Promise<TokenResult> {
const { client_secret, client_id, redirect_uris } = credentials.installed;
const oAuth2Client = new google.auth.OAuth2(
client_id, client_secret, redirect_uris[0]);
- return new Promise<OAuth2Client>((resolve, reject) => {
+ return new Promise<TokenResult>((resolve, reject) => {
readFile(token_path, (err, token) => {
// Check if we have previously stored a token.
if (err) {
return getNewToken(oAuth2Client, token_path).then(resolve, reject);
}
- oAuth2Client.setCredentials(parseBuffer(token));
- resolve(oAuth2Client);
+ let parsed: Credentials = parseBuffer(token);
+ if (parsed.expiry_date! < new Date().getTime()) {
+ return refreshToken(parsed, client_id, client_secret, oAuth2Client, token_path).then(resolve, reject);
+ }
+ oAuth2Client.setCredentials(parsed);
+ resolve({ token: parsed, client: oAuth2Client });
});
});
}
+ const refreshEndpoint = "https://oauth2.googleapis.com/token";
+ const refreshToken = (credentials: Credentials, client_id: string, client_secret: string, oAuth2Client: OAuth2Client, token_path: string) => {
+ return new Promise<TokenResult>((resolve, reject) => {
+ let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } };
+ let queryParameters = {
+ refreshToken: credentials.refresh_token,
+ client_id,
+ client_secret,
+ grant_type: "refresh_token"
+ };
+ let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`;
+ request.post(url, headerParameters).then(response => {
+ let parsed = JSON.parse(response);
+ credentials.access_token = parsed.access_token;
+ credentials.expiry_date = new Date().getTime() + (parsed.expires_in * 1000);
+ writeFile(token_path, JSON.stringify(credentials), (err) => {
+ if (err) {
+ console.error(err);
+ reject(err);
+ }
+ console.log('Refreshed token stored to', token_path);
+ oAuth2Client.setCredentials(credentials);
+ resolve({ token: credentials, client: oAuth2Client });
+ });
+ });
+ });
+ };
+
/**
* Get and store new token after prompting for user authorization, and then
* execute the given callback with the authorized OAuth2 client.
@@ -96,7 +157,7 @@ export namespace GoogleApiServerUtils {
* @param {getEventsCallback} callback The callback for the authorized client.
*/
function getNewToken(oAuth2Client: OAuth2Client, token_path: string) {
- return new Promise<OAuth2Client>((resolve, reject) => {
+ return new Promise<TokenResult>((resolve, reject) => {
const authUrl = oAuth2Client.generateAuthUrl({
access_type: 'offline',
scope: SCOPES.map(relative => prefix + relative),
@@ -122,7 +183,7 @@ export namespace GoogleApiServerUtils {
}
console.log('Token stored to', token_path);
});
- resolve(oAuth2Client);
+ resolve({ token, client: oAuth2Client });
});
});
});
diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts
index 35f986250..e91f8352b 100644
--- a/src/server/apis/google/GooglePhotosUploadUtils.ts
+++ b/src/server/apis/google/GooglePhotosUploadUtils.ts
@@ -5,6 +5,7 @@ import { Utils } from '../../../Utils';
import * as path from 'path';
import { Opt } from '../../../new_fields/Doc';
import * as sharp from 'sharp';
+import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes';
const uploadDirectory = path.join(__dirname, "../../public/files/");
@@ -49,27 +50,43 @@ export namespace GooglePhotosUploadUtils {
uri: prepend('uploads'),
body
};
- return new Promise<any>(resolve => request(parameters, (error, _response, body) => resolve(error ? undefined : body)));
+ return new Promise<any>((resolve, reject) => request(parameters, (error, _response, body) => {
+ if (error) {
+ console.log(error);
+ return reject(error);
+ }
+ resolve(body);
+ }));
};
- export const CreateMediaItems = (newMediaItems: any[], album?: { id: string }) => {
- return new Promise<any>((resolve, reject) => {
+ export const CreateMediaItems = async (newMediaItems: any[], album?: { id: string }): Promise<MediaItemCreationResult> => {
+ const quota = newMediaItems.length;
+ let handled = 0;
+ const newMediaItemResults: NewMediaItemResult[] = [];
+ while (handled < quota) {
+ const cap = Math.min(newMediaItems.length, handled + 50);
+ const batch = newMediaItems.slice(handled, cap);
+ console.log(batch.length);
const parameters = {
method: 'POST',
headers: headers('json'),
uri: prepend('mediaItems:batchCreate'),
- body: { newMediaItems } as any,
+ body: { newMediaItems: batch } as any,
json: true
};
album && (parameters.body.albumId = album.id);
- request(parameters, (error, _response, body) => {
- if (error) {
- reject(error);
- } else {
- resolve(body);
- }
- });
- });
+ newMediaItemResults.push(...(await new Promise<MediaItemCreationResult>((resolve, reject) => {
+ request(parameters, (error, _response, body) => {
+ if (error) {
+ reject(error);
+ } else {
+ resolve(body);
+ }
+ });
+ })).newMediaItemResults);
+ handled = cap;
+ }
+ return { newMediaItemResults };
};
}
@@ -87,10 +104,12 @@ export namespace DownloadUtils {
LARGE: { width: 900, suffix: "_l" },
};
- const png = ".png";
- const pngs = [".png", ".PNG"];
- const jpgs = [".jpg", ".JPG", ".jpeg", ".JPEG"];
- const formats = [".jpg", ".png", ".gif"];
+ const gifs = [".gif"];
+ const pngs = [".png"];
+ const jpgs = [".jpg", ".jpeg"];
+ const imageFormats = [...pngs, ...jpgs, ...gifs];
+ const videoFormats = [".mov", ".mp4"];
+
const size = "content-length";
const type = "content-type";
@@ -106,7 +125,8 @@ export namespace DownloadUtils {
export const UploadImage = async (url: string, filename?: string, prefix = ""): Promise<Opt<UploadInformation>> => {
const resolved = filename ? sanitize(filename) : generate(prefix, url);
- const extension = path.extname(url) || path.extname(resolved) || png;
+ let extension = path.extname(url) || path.extname(resolved);
+ extension && (extension = extension.toLowerCase());
let information: UploadInformation = {
mediaPaths: [],
fileNames: { clean: resolved }
@@ -133,23 +153,31 @@ export namespace DownloadUtils {
suffix: size.suffix
}))
];
+ let nonVisual = false;
if (pngs.includes(extension)) {
resizers.forEach(element => element.resizer = element.resizer.png());
} else if (jpgs.includes(extension)) {
resizers.forEach(element => element.resizer = element.resizer.jpeg());
- } else if (!formats.includes(extension.toLowerCase())) {
- return reject();
+ } else if (![...imageFormats, ...videoFormats].includes(extension.toLowerCase())) {
+ nonVisual = true;
+ }
+ if (imageFormats.includes(extension)) {
+ for (let resizer of resizers) {
+ const suffix = resizer.suffix;
+ let mediaPath: string;
+ await new Promise<void>(resolve => {
+ const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension;
+ information.mediaPaths.push(mediaPath = uploadDirectory + filename);
+ information.fileNames[suffix] = filename;
+ stream(url).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath))
+ .on('close', resolve)
+ .on('error', reject);
+ });
+ }
}
- for (let resizer of resizers) {
- const suffix = resizer.suffix;
- let mediaPath: string;
+ if (!isLocal || nonVisual) {
await new Promise<void>(resolve => {
- const filename = resolved.substring(0, resolved.length - extension.length) + suffix + extension;
- information.mediaPaths.push(mediaPath = uploadDirectory + filename);
- information.fileNames[suffix] = filename;
- stream(url).pipe(resizer.resizer).pipe(fs.createWriteStream(mediaPath))
- .on('close', resolve)
- .on('error', reject);
+ stream(url).pipe(fs.createWriteStream(uploadDirectory + resolved)).on('close', resolve);
});
}
resolve(information);
diff --git a/src/server/apis/google/SharedTypes.ts b/src/server/apis/google/SharedTypes.ts
new file mode 100644
index 000000000..9ad6130b6
--- /dev/null
+++ b/src/server/apis/google/SharedTypes.ts
@@ -0,0 +1,21 @@
+export interface NewMediaItemResult {
+ uploadToken: string;
+ status: { code: number, message: string };
+ mediaItem: MediaItem;
+}
+
+export interface MediaItem {
+ id: string;
+ description: string;
+ productUrl: string;
+ baseUrl: string;
+ mimeType: string;
+ mediaMetadata: {
+ creationTime: string;
+ width: string;
+ height: string;
+ };
+ filename: string;
+}
+
+export type MediaItemCreationResult = { newMediaItemResults: NewMediaItemResult[] }; \ No newline at end of file
diff --git a/src/server/authentication/config/passport.ts b/src/server/authentication/config/passport.ts
index 10b17de71..8b20d29b1 100644
--- a/src/server/authentication/config/passport.ts
+++ b/src/server/authentication/config/passport.ts
@@ -1,7 +1,6 @@
import * as passport from 'passport';
import * as passportLocal from 'passport-local';
-import * as mongodb from 'mongodb';
-import * as _ from "lodash";
+import _ from "lodash";
import { default as User } from '../models/user_model';
import { Request, Response, NextFunction } from "express";
import { RouteStore } from '../../RouteStore';
diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts
index af5774ebe..050a71eb4 100644
--- a/src/server/authentication/models/current_user_utils.ts
+++ b/src/server/authentication/models/current_user_utils.ts
@@ -2,7 +2,6 @@ import { action, computed, observable, runInAction } from "mobx";
import * as rp from 'request-promise';
import { DocServer } from "../../../client/DocServer";
import { Docs } from "../../../client/documents/Documents";
-import { Gateway, NorthstarSettings } from "../../../client/northstar/manager/Gateway";
import { Attribute, AttributeGroup, Catalog, Schema } from "../../../client/northstar/model/idea/idea";
import { ArrayUtil } from "../../../client/northstar/utils/ArrayUtil";
import { CollectionViewType } from "../../../client/views/collections/CollectionBaseView";
@@ -24,6 +23,9 @@ export class CurrentUserUtils {
public static get MainDocId() { return this.mainDocId; }
public static set MainDocId(id: string | undefined) { this.mainDocId = id; }
+ @observable public static GuestTarget: Doc | undefined;
+ @observable public static GuestWorkspace: Doc | undefined;
+
private static createUserDocument(id: string): Doc {
let doc = new Doc(id, true);
doc.viewType = CollectionViewType.Tree;
@@ -59,7 +61,7 @@ export class CurrentUserUtils {
noteTypes.excludeFromLibrary = true;
doc.noteTypes = noteTypes;
}
- PromiseValue(Cast(doc.noteTypes, Doc)).then(noteTypes => noteTypes && PromiseValue(noteTypes.data).then(vals => DocListCast(vals)));
+ PromiseValue(Cast(doc.noteTypes, Doc)).then(noteTypes => noteTypes && PromiseValue(noteTypes.data).then(DocListCast));
if (doc.recentlyClosed === undefined) {
const recentlyClosed = Docs.Create.TreeDocument([], { title: "Recently Closed", height: 75 });
recentlyClosed.excludeFromLibrary = true;
@@ -112,7 +114,7 @@ export class CurrentUserUtils {
this.curr_id = id;
Doc.CurrentUserEmail = email;
await rp.get(Utils.prepend(RouteStore.getUserDocumentId)).then(id => {
- if (id) {
+ if (id && id !== "guest") {
return DocServer.GetRefField(id).then(async field => {
if (field instanceof Doc) {
await this.updateUserDocument(field);
diff --git a/src/server/credentials/google_docs_credentials.json b/src/server/credentials/google_docs_credentials.json
index 8d097d363..955c5a3c1 100644
--- a/src/server/credentials/google_docs_credentials.json
+++ b/src/server/credentials/google_docs_credentials.json
@@ -1 +1,11 @@
-{"installed":{"client_id":"343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com","project_id":"quickstart-1565056383187","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"w8KIFSc0MQpmUYHed4qEzn8b","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} \ No newline at end of file
+{
+ "installed": {
+ "client_id": "343179513178-ud6tvmh275r2fq93u9eesrnc66t6akh9.apps.googleusercontent.com",
+ "project_id": "quickstart-1565056383187",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://oauth2.googleapis.com/token",
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+ "client_secret": "w8KIFSc0MQpmUYHed4qEzn8b",
+ "redirect_uris": ["urn:ietf:wg:oauth:2.0:oob", "http://localhost"]
+ }
+} \ No newline at end of file
diff --git a/src/server/credentials/google_docs_token.json b/src/server/credentials/google_docs_token.json
index 07c02d56c..bb313f136 100644
--- a/src/server/credentials/google_docs_token.json
+++ b/src/server/credentials/google_docs_token.json
@@ -1 +1 @@
-{"access_token":"ya29.GltjB4-x03xFpd2NY2555cxg1xlT_ajqRi78M9osOfdOF2jTIjlPkn_UZL8cUwVP0DPC8rH3vhhg8RpspFe8Vewx92shAO3RPos_uMH0CUqEiCiZlaaB5I3Jq3Mv","refresh_token":"1/teUKUqGKMLjVqs-eed0L8omI02pzSxMUYaxGc2QxBw0","scope":"https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents.readonly","token_type":"Bearer","expiry_date":1565654175862} \ No newline at end of file
+{"access_token":"ya29.GlyBB9xs77CFscdtWApHKMcsd6eS9NW3tO0FEvZlfO87HTl7zc1nIVhvtB7MLxadXvxVg4VUAvl6eFjVFsbdmA7TmURhIygYsZbds87ybMuLH5W68mRAVd3HDYyCzg","refresh_token":"1/HTv_xFHszu2Nf3iiFrUTaeKzC_Vp2-6bpIB06xW_WHI","scope":"https://www.googleapis.com/auth/presentations.readonly https://www.googleapis.com/auth/documents.readonly https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.appendonly https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/presentations https://www.googleapis.com/auth/photoslibrary.sharing","token_type":"Bearer","expiry_date":1568310697477} \ No newline at end of file
diff --git a/src/server/index.ts b/src/server/index.ts
index 50ce2b14e..d7273bd88 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -21,10 +21,10 @@ import * as wdm from 'webpack-dev-middleware';
import * as whm from 'webpack-hot-middleware';
import { Utils } from '../Utils';
import { getForgot, getLogin, getLogout, getReset, getSignup, postForgot, postLogin, postReset, postSignup } from './authentication/controllers/user_controller';
-import { DashUserModel } from './authentication/models/user_model';
+import User, { DashUserModel } from './authentication/models/user_model';
import { Client } from './Client';
import { Database } from './database';
-import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput } from "./Message";
+import { MessageStore, Transferable, Types, Diff, YoutubeQueryTypes as YoutubeQueryType, YoutubeQueryInput, SourceSpecified } from "./Message";
import { RouteStore } from './RouteStore';
import v4 = require('uuid/v4');
const app = express();
@@ -36,19 +36,16 @@ const serverPort = 4321;
import expressFlash = require('express-flash');
import flash = require('connect-flash');
import { Search } from './Search';
-import _ = require('lodash');
import * as Archiver from 'archiver';
var AdmZip = require('adm-zip');
import * as YoutubeApi from "./apis/youtube/youtubeApiSample";
import { Response } from 'express-serve-static-core';
import { GoogleApiServerUtils } from "./apis/google/GoogleApiServerUtils";
-import { GaxiosResponse } from 'gaxios';
-import { Opt } from '../new_fields/Doc';
-import { docs_v1 } from 'googleapis';
-import { Endpoint } from 'googleapis-common';
+import { GooglePhotosUploadUtils, DownloadUtils as UploadUtils } from './apis/google/GooglePhotosUploadUtils';
const MongoStore = require('connect-mongo')(session);
const mongoose = require('mongoose');
const probe = require("probe-image-size");
+import * as qs from 'query-string';
const download = (url: string, dest: fs.PathLike) => request.get(url).pipe(fs.createWriteStream(dest));
let youtubeApiKey: string;
@@ -89,6 +86,7 @@ app.use(expressValidator());
app.use(passport.initialize());
app.use(passport.session());
app.use((req, res, next) => {
+ console.log(req.originalUrl);
res.locals.user = req.user;
next();
});
@@ -115,8 +113,10 @@ function addSecureRoute(method: Method,
...subscribers: string[]
) {
let abstracted = (req: express.Request, res: express.Response) => {
- if (req.user) {
- handler(req.user as any, res, req);
+ let sharing = qs.parse(qs.extract(req.originalUrl), { sort: false }).sharing === "true";
+ sharing = sharing && req.originalUrl.startsWith("/doc/");
+ if (req.user || sharing) {
+ handler(req.user, res, req);
} else {
req.session!.target = req.originalUrl;
onRejection(res, req);
@@ -157,6 +157,12 @@ app.get("/buxton", (req, res) => {
command_line('python scraper.py', cwd).then(onResolved, tryPython3);
});
+const STATUS = {
+ OK: 200,
+ BAD_REQUEST: 400,
+ EXECUTION_ERROR: 500
+};
+
const command_line = (command: string, fromDirectory?: string) => {
return new Promise<string>((resolve, reject) => {
let options: ExecOptions = {};
@@ -194,7 +200,7 @@ app.get("/version", (req, res) => {
// SEARCH
const solrURL = "http://localhost:8983/solr/#/dash";
-// GETTERS
+// GETTERSå
app.get("/search", async (req, res) => {
const solrQuery: any = {};
@@ -420,10 +426,10 @@ app.get("/thumbnail/:filename", (req, res) => {
let filename = req.params.filename;
let noExt = filename.substring(0, filename.length - ".png".length);
let pagenumber = parseInt(noExt.split('-')[1]);
- fs.exists(uploadDir + filename, (exists: boolean) => {
- console.log(`${uploadDir + filename} ${exists ? "exists" : "does not exist"}`);
+ fs.exists(uploadDirectory + filename, (exists: boolean) => {
+ console.log(`${uploadDirectory + filename} ${exists ? "exists" : "does not exist"}`);
if (exists) {
- let input = fs.createReadStream(uploadDir + filename);
+ let input = fs.createReadStream(uploadDirectory + filename);
probe(input, (err: any, result: any) => {
if (err) {
console.log(err);
@@ -434,7 +440,7 @@ app.get("/thumbnail/:filename", (req, res) => {
});
}
else {
- LoadPage(uploadDir + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res);
+ LoadPage(uploadDirectory + filename.substring(0, filename.length - noExt.split('-')[1].length - ".PNG".length - 1) + ".pdf", pagenumber, res);
}
});
});
@@ -503,21 +509,20 @@ addSecureRoute(
res.sendFile(path.join(__dirname, '../../deploy/' + filename));
},
undefined,
- RouteStore.home,
- RouteStore.openDocumentWithId
+ RouteStore.home, RouteStore.openDocumentWithId
);
addSecureRoute(
Method.GET,
- (user, res) => res.send(user.userDocumentId || ""),
- undefined,
+ (user, res) => res.send(user.userDocumentId),
+ (res) => res.send(undefined),
RouteStore.getUserDocumentId,
);
addSecureRoute(
Method.GET,
- (user, res) => res.send(JSON.stringify({ id: user.id, email: user.email })),
- undefined,
+ (user, res) => { res.send(JSON.stringify({ id: user.id, email: user.email })); },
+ (res) => res.send(JSON.stringify({ id: "__guest__", email: "" })),
RouteStore.getCurrUser
);
@@ -556,50 +561,31 @@ class NodeCanvasFactory {
}
const pngTypes = [".png", ".PNG"];
-const pdfTypes = [".pdf", ".PDF"];
const jpgTypes = [".jpg", ".JPG", ".jpeg", ".JPEG"];
-const uploadDir = __dirname + "/public/files/";
+const uploadDirectory = __dirname + "/public/files/";
+interface FileResponse {
+ name: string;
+ path: string;
+ type: string;
+}
// SETTERS
app.post(
RouteStore.upload,
(req, res) => {
let form = new formidable.IncomingForm();
- form.uploadDir = uploadDir;
+ form.uploadDir = uploadDirectory;
form.keepExtensions = true;
- // let path = req.body.path;
- console.log("upload");
- form.parse(req, (err, fields, files) => {
- console.log("parsing");
- let names: string[] = [];
- for (const name in files) {
- const file = path.basename(files[name].path);
- const ext = path.extname(file);
- let resizers = [
- { resizer: sharp().rotate(), suffix: "_o" },
- { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }).rotate(), suffix: "_s" },
- { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }).rotate(), suffix: "_m" },
- { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }).rotate(), suffix: "_l" },
- ];
- let isImage = false;
- if (pngTypes.includes(ext)) {
- resizers.forEach(element => {
- element.resizer = element.resizer.png();
- });
- isImage = true;
- } else if (jpgTypes.includes(ext)) {
- resizers.forEach(element => {
- element.resizer = element.resizer.jpeg();
- });
- isImage = true;
- }
- if (isImage) {
- resizers.forEach(resizer => {
- fs.createReadStream(uploadDir + file).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + file.substring(0, file.length - ext.length) + resizer.suffix + ext));
- });
- }
- names.push(`/files/` + file);
+ form.parse(req, async (_err, _fields, files) => {
+ let results: FileResponse[] = [];
+ for (const key in files) {
+ const { type, path: location, name } = files[key];
+ const filename = path.basename(location);
+ await UploadUtils.UploadImage(uploadDirectory + filename, filename).catch(() => console.log(`Unable to process ${filename}`));
+ results.push({ name, type, path: `/files/${filename}` });
+ console.log(path.basename(name));
}
- res.send(names);
+ console.log("All files traversed!");
+ _success(res, results);
});
}
);
@@ -613,7 +599,7 @@ addSecureRoute(
res.status(401).send("incorrect parameters specified");
return;
}
- imageDataUri.outputFile(uri, uploadDir + filename).then((savedName: string) => {
+ imageDataUri.outputFile(uri, uploadDirectory + filename).then((savedName: string) => {
const ext = path.extname(savedName);
let resizers = [
{ resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: "_s" },
@@ -634,7 +620,7 @@ addSecureRoute(
}
if (isImage) {
resizers.forEach(resizer => {
- fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDir + filename + resizer.suffix + ext));
+ fs.createReadStream(savedName).pipe(resizer.resizer).pipe(fs.createWriteStream(uploadDirectory + filename + resizer.suffix + ext));
});
}
res.send("/files/" + filename + ext);
@@ -681,21 +667,31 @@ app.use(RouteStore.corsProxy, (req, res) => {
}).pipe(res);
});
-app.get(RouteStore.delete, (req, res) => {
- if (release) {
- res.send("no");
- return;
- }
- deleteFields().then(() => res.redirect(RouteStore.home));
-});
+addSecureRoute(
+ Method.GET,
+ (user, res, req) => {
+ if (release) {
+ res.send("no");
+ return;
+ }
+ deleteFields().then(() => res.redirect(RouteStore.home));
+ },
+ undefined,
+ RouteStore.delete
+);
-app.get(RouteStore.deleteAll, (req, res) => {
- if (release) {
- res.send("no");
- return;
- }
- deleteAll().then(() => res.redirect(RouteStore.home));
-});
+addSecureRoute(
+ Method.GET,
+ (user, res, req) => {
+ if (release) {
+ res.send("no");
+ return;
+ }
+ deleteAll().then(() => res.redirect(RouteStore.home));
+ },
+ undefined,
+ RouteStore.deleteAll
+);
app.use(wdm(compiler, { publicPath: config.output.publicPath }));
@@ -781,8 +777,8 @@ function setField(socket: Socket, newValue: Transferable) {
}
}
-function GetRefField([id, callback]: [string, (result?: Transferable) => void]) {
- Database.Instance.getDocument(id, callback, "newDocuments");
+function GetRefField([args, callback]: [SourceSpecified, (result?: Transferable) => void]) {
+ Database.Instance.getDocument(args.id, callback, args.mongoCollection || "newDocuments");
}
function GetRefFields([ids, callback]: [string[], (result?: Transferable[]) => void]) {
@@ -801,19 +797,19 @@ function HandleYoutubeQuery([query, callback]: [YoutubeQueryInput, (result?: any
}
}
-const credentials = path.join(__dirname, "./credentials/google_docs_credentials.json");
-const token = path.join(__dirname, "./credentials/google_docs_token.json");
+const credentialsPath = path.join(__dirname, "./credentials/google_docs_credentials.json");
+const tokenPath = path.join(__dirname, "./credentials/google_docs_token.json");
const EndpointHandlerMap = new Map<GoogleApiServerUtils.Action, GoogleApiServerUtils.ApiRouter>([
["create", (api, params) => api.create(params)],
- ["retrieve", (api, params) => api.get(params)],
+ ["retrieve", (api, params) => api.get(params, { params: "fields=inlineObjects" })],
["update", (api, params) => api.batchUpdate(params)],
]);
app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => {
let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service;
let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action;
- GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentials, token }).then(endpoint => {
+ GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], { credentialsPath, tokenPath }).then(endpoint => {
let handler = EndpointHandlerMap.get(action);
if (endpoint && handler) {
let execute = handler(endpoint, req.body).then(
@@ -827,6 +823,69 @@ app.post(RouteStore.googleDocs + "/:sector/:action", (req, res) => {
});
});
+app.get(RouteStore.googlePhotosAccessToken, (req, res) => GoogleApiServerUtils.RetrieveAccessToken({ credentialsPath, tokenPath }).then(token => res.send(token)));
+
+const tokenError = "Unable to successfully upload bytes for all images!";
+const mediaError = "Unable to convert all uploaded bytes to media items!";
+
+app.post(RouteStore.googlePhotosMediaUpload, async (req, res) => {
+ const media: GooglePhotosUploadUtils.MediaInput[] = req.body.media;
+ await GooglePhotosUploadUtils.initialize({ uploadDirectory, credentialsPath, tokenPath });
+ const newMediaItems = await Promise.all(media.map(async element => {
+ const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(element.url).catch(error => {
+ console.log("Dispatching upload error!");
+ console.log(error);
+ });
+ return !uploadToken ? undefined : {
+ description: element.description,
+ simpleMediaItem: { uploadToken }
+ };
+ }));
+ if (!newMediaItems.every(item => item)) {
+ return _error(res, tokenError);
+ }
+ GooglePhotosUploadUtils.CreateMediaItems(newMediaItems, req.body.album).then(
+ result => _success(res, result.newMediaItemResults),
+ error => _error(res, mediaError, error)
+ );
+});
+
+interface MediaItem {
+ baseUrl: string;
+ filename: string;
+}
+const prefix = "google_photos_";
+
+const downloadError = "Encountered an error while executing downloads.";
+const requestError = "Unable to execute download: the body's media items were malformed.";
+
+app.post(RouteStore.googlePhotosMediaDownload, async (req, res) => {
+ const contents: { mediaItems: MediaItem[] } = req.body;
+ if (contents) {
+ const pending = contents.mediaItems.map(item =>
+ UploadUtils.UploadImage(item.baseUrl, item.filename, prefix)
+ );
+ const completed = await Promise.all(pending).catch(error => _error(res, downloadError, error));
+ Array.isArray(completed) && _success(res, completed);
+ return;
+ }
+ _invalid(res, requestError);
+});
+
+const _error = (res: Response, message: string, error?: any) => {
+ res.statusMessage = message;
+ res.status(STATUS.EXECUTION_ERROR).send(error);
+};
+
+const _success = (res: Response, body: any) => {
+ res.status(STATUS.OK).send(body);
+};
+
+const _invalid = (res: Response, message: string) => {
+ res.statusMessage = message;
+ res.status(STATUS.BAD_REQUEST).send();
+};
+
const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
"number": "_n",
"string": "_t",
diff --git a/src/server/updateSearch.ts b/src/server/updateSearch.ts
deleted file mode 100644
index 906b795f1..000000000
--- a/src/server/updateSearch.ts
+++ /dev/null
@@ -1,123 +0,0 @@
-import { Database } from "./database";
-import { Cursor } from "mongodb";
-import { Search } from "./Search";
-import pLimit from 'p-limit';
-
-const suffixMap: { [type: string]: (string | [string, string | ((json: any) => any)]) } = {
- "number": "_n",
- "string": "_t",
- "boolean": "_b",
- // "image": ["_t", "url"],
- "video": ["_t", "url"],
- "pdf": ["_t", "url"],
- "audio": ["_t", "url"],
- "web": ["_t", "url"],
- "date": ["_d", value => new Date(value.date).toISOString()],
- "proxy": ["_i", "fieldId"],
- "list": ["_l", list => {
- const results = [];
- for (const value of list.fields) {
- const term = ToSearchTerm(value);
- if (term) {
- results.push(term.value);
- }
- }
- return results.length ? results : null;
- }]
-};
-
-function ToSearchTerm(val: any): { suffix: string, value: any } | undefined {
- if (val === null || val === undefined) {
- return;
- }
- const type = val.__type || typeof val;
- let suffix = suffixMap[type];
- if (!suffix) {
- return;
- }
-
- if (Array.isArray(suffix)) {
- const accessor = suffix[1];
- if (typeof accessor === "function") {
- val = accessor(val);
- } else {
- val = val[accessor];
- }
- suffix = suffix[0];
- }
-
- return { suffix, value: val };
-}
-
-function getSuffix(value: string | [string, any]): string {
- return typeof value === "string" ? value : value[0];
-}
-
-const limit = pLimit(5);
-async function update() {
- // await new Promise(res => setTimeout(res, 5));
- console.log("update");
- await Search.Instance.clear();
- const cursor = await Database.Instance.query({});
- console.log("Cleared");
- const updates: any[] = [];
- let numDocs = 0;
- function updateDoc(doc: any) {
- numDocs++;
- if ((numDocs % 50) === 0) {
- console.log("updateDoc " + numDocs);
- }
- // console.log("doc " + numDocs);
- if (doc.__type !== "Doc") {
- return;
- }
- const fields = doc.fields;
- if (!fields) {
- return;
- }
- const update: any = { id: doc._id };
- let dynfield = false;
- for (const key in fields) {
- const value = fields[key];
- const term = ToSearchTerm(value);
- if (term !== undefined) {
- let { suffix, value } = term;
- update[key + suffix] = value;
- dynfield = true;
- }
- }
- if (dynfield) {
- updates.push(update);
- // console.log(updates.length);
- }
- }
- await cursor.forEach(updateDoc);
- console.log(`Updating ${updates.length} documents`);
- const result = await Search.Instance.updateDocuments(updates);
- try {
- console.log(JSON.parse(result).responseHeader.status);
- } catch {
- console.log("Error:");
- // console.log(updates[i]);
- console.log(result);
- console.log("\n");
- }
- // for (let i = 0; i < updates.length; i++) {
- // console.log(i);
- // const result = await Search.Instance.updateDocument(updates[i]);
- // try {
- // console.log(JSON.parse(result).responseHeader.status);
- // } catch {
- // console.log("Error:");
- // console.log(updates[i]);
- // console.log(result);
- // console.log("\n");
- // }
- // }
- // await Promise.all(updates.map(update => {
- // return limit(() => Search.Instance.updateDocument(update));
- // }));
- cursor.close();
-}
-
-update(); \ No newline at end of file
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-29 21:56:05 +0000