From af25eaf2a848278a58f0993cba2e68c05da0760c Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 29 Oct 2019 20:00:54 -0400 Subject: comments and fixes for google photos server sid --- src/client/Network.ts | 12 +- src/client/apis/GoogleAuthenticationManager.tsx | 39 ++--- .../apis/google_docs/GoogleApiClientUtils.ts | 8 +- .../apis/google_docs/GooglePhotosClientUtils.ts | 10 +- .../util/Import & Export/DirectoryImportBox.tsx | 5 +- src/client/util/Import & Export/ImageUtils.ts | 4 +- src/new_fields/RichTextUtils.ts | 4 +- src/server/RouteManager.ts | 4 +- src/server/RouteStore.ts | 3 +- src/server/apis/google/GoogleApiServerUtils.ts | 171 +++++++++++---------- src/server/index.ts | 14 +- 11 files changed, 147 insertions(+), 127 deletions(-) (limited to 'src') diff --git a/src/client/Network.ts b/src/client/Network.ts index 75ccb5e99..f9ef27267 100644 --- a/src/client/Network.ts +++ b/src/client/Network.ts @@ -1,18 +1,16 @@ import { Utils } from "../Utils"; -import { CurrentUserUtils } from "../server/authentication/models/current_user_utils"; import requestPromise = require('request-promise'); -export namespace Identified { +export namespace Networking { export async function FetchFromServer(relativeRoute: string) { - return (await fetch(relativeRoute, { headers: { userId: CurrentUserUtils.id } })).text(); + return (await fetch(relativeRoute)).text(); } export async function PostToServer(relativeRoute: string, body?: any) { let options = { uri: Utils.prepend(relativeRoute), method: "POST", - headers: { userId: CurrentUserUtils.id }, body, json: true }; @@ -22,12 +20,10 @@ export namespace Identified { export async function PostFormDataToServer(relativeRoute: string, formData: FormData) { const parameters = { method: 'POST', - headers: { userId: CurrentUserUtils.id }, - body: formData, + body: formData }; const response = await fetch(relativeRoute, parameters); - const text = await response.json(); - return text; + return response.json(); } } \ No newline at end of file diff --git a/src/client/apis/GoogleAuthenticationManager.tsx b/src/client/apis/GoogleAuthenticationManager.tsx index 01dac3996..1ec9d8412 100644 --- a/src/client/apis/GoogleAuthenticationManager.tsx +++ b/src/client/apis/GoogleAuthenticationManager.tsx @@ -3,7 +3,7 @@ import { observer } from "mobx-react"; import * as React from "react"; import MainViewModal from "../views/MainViewModal"; import { Opt } from "../../new_fields/Doc"; -import { Identified } from "../Network"; +import { Networking } from "../Network"; import { RouteStore } from "../../server/RouteStore"; import "./GoogleAuthenticationManager.scss"; @@ -31,7 +31,7 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { } public fetchOrGenerateAccessToken = async () => { - let response = await Identified.FetchFromServer(RouteStore.readGoogleAccessToken); + let response = await Networking.FetchFromServer(RouteStore.readGoogleAccessToken); // if this is an authentication url, activate the UI to register the new access token if (new RegExp(AuthenticationUrl).test(response)) { this.isOpen = true; @@ -39,24 +39,25 @@ export default class GoogleAuthenticationManager extends React.Component<{}> { return new Promise(async resolve => { const disposer = reaction( () => this.authenticationCode, - authenticationCode => { - if (authenticationCode) { - Identified.PostToServer(RouteStore.writeGoogleAccessToken, { authenticationCode }).then( - ({ access_token, avatar, name }) => { - runInAction(() => { - this.avatar = avatar; - this.username = name; - }); - this.beginFadeout(); - disposer(); - resolve(access_token); - }, - action(() => { - this.hasBeenClicked = false; - this.success = false; - }) - ); + async authenticationCode => { + if (!authenticationCode) { + return; } + const { access_token, avatar, name } = await Networking.PostToServer( + RouteStore.writeGoogleAccessToken, + { authenticationCode } + ); + runInAction(() => { + this.avatar = avatar; + this.username = name; + }); + this.beginFadeout(); + disposer(); + resolve(access_token); + action(() => { + this.hasBeenClicked = false; + this.success = false; + }); } ); }); diff --git a/src/client/apis/google_docs/GoogleApiClientUtils.ts b/src/client/apis/google_docs/GoogleApiClientUtils.ts index 1cf01fc3d..183679317 100644 --- a/src/client/apis/google_docs/GoogleApiClientUtils.ts +++ b/src/client/apis/google_docs/GoogleApiClientUtils.ts @@ -3,7 +3,7 @@ import { RouteStore } from "../../../server/RouteStore"; import { Opt } from "../../../new_fields/Doc"; import { isArray } from "util"; import { EditorState } from "prosemirror-state"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; export const Pulls = "googleDocsPullCount"; export const Pushes = "googleDocsPushCount"; @@ -84,7 +84,7 @@ export namespace GoogleApiClientUtils { } }; try { - const schema: docs_v1.Schema$Document = await Identified.PostToServer(path, parameters); + const schema: docs_v1.Schema$Document = await Networking.PostToServer(path, parameters); return schema.documentId; } catch { return undefined; @@ -157,7 +157,7 @@ export namespace GoogleApiClientUtils { const path = `${RouteStore.googleDocs}/Documents/${Actions.Retrieve}`; try { const parameters = { documentId: options.documentId }; - const schema: RetrievalResult = await Identified.PostToServer(path, parameters); + const schema: RetrievalResult = await Networking.PostToServer(path, parameters); return schema; } catch { return undefined; @@ -173,7 +173,7 @@ export namespace GoogleApiClientUtils { } }; try { - const replies: UpdateResult = await Identified.PostToServer(path, parameters); + const replies: UpdateResult = await Networking.PostToServer(path, parameters); return replies; } catch { return undefined; diff --git a/src/client/apis/google_docs/GooglePhotosClientUtils.ts b/src/client/apis/google_docs/GooglePhotosClientUtils.ts index e93fa6eb4..402fc64b5 100644 --- a/src/client/apis/google_docs/GooglePhotosClientUtils.ts +++ b/src/client/apis/google_docs/GooglePhotosClientUtils.ts @@ -13,7 +13,7 @@ import { Docs, DocumentOptions } from "../../documents/Documents"; import { NewMediaItemResult, MediaItem } from "../../../server/apis/google/SharedTypes"; import { AssertionError } from "assert"; import { DocumentView } from "../../views/nodes/DocumentView"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; import GoogleAuthenticationManager from "../GoogleAuthenticationManager"; export namespace GooglePhotos { @@ -78,6 +78,7 @@ export namespace GooglePhotos { } export const CollectionToAlbum = async (options: AlbumCreationOptions): Promise> => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); const { collection, title, descriptionKey, tag } = options; const dataDocument = Doc.GetProto(collection); const images = ((await DocListCastAsync(dataDocument.data)) || []).filter(doc => Cast(doc.data, ImageField)); @@ -127,6 +128,7 @@ export namespace GooglePhotos { export type CollectionConstructor = (data: Array, options: DocumentOptions, ...args: any) => Doc; export const CollectionFromSearch = async (constructor: CollectionConstructor, requested: Opt>): Promise => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); let response = await Query.ContentSearch(requested); let uploads = await Transactions.WriteMediaItemsToServer(response); const children = uploads.map((upload: Transactions.UploadInformation) => { @@ -147,6 +149,7 @@ export namespace GooglePhotos { const comparator = (a: string, b: string) => (a < b) ? -1 : (a > b ? 1 : 0); export const TagChildImages = async (collection: Doc) => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); const idMapping = await Cast(collection.googlePhotosIdMapping, Doc); if (!idMapping) { throw new Error("Appending image metadata requires that the targeted collection have already been mapped to an album!"); @@ -304,7 +307,7 @@ export namespace GooglePhotos { }; export const WriteMediaItemsToServer = async (body: { mediaItems: any[] }): Promise => { - const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, body); + const uploads = await Networking.PostToServer(RouteStore.googlePhotosMediaDownload, body); return uploads; }; @@ -325,6 +328,7 @@ export namespace GooglePhotos { } export const UploadImages = async (sources: Doc[], album?: AlbumReference, descriptionKey = "caption"): Promise> => { + await GoogleAuthenticationManager.Instance.fetchOrGenerateAccessToken(); if (album && "title" in album) { album = await Create.Album(album.title); } @@ -341,7 +345,7 @@ export namespace GooglePhotos { media.push({ url, description }); } if (media.length) { - const results = await Identified.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); + const results = await Networking.PostToServer(RouteStore.googlePhotosMediaUpload, { media, album }); return results; } }; diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index d74b51993..2d1b6fe20 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -20,9 +20,8 @@ import { listSpec } from "../../../new_fields/Schema"; import { GooglePhotos } from "../../apis/google_docs/GooglePhotosClientUtils"; import { SchemaHeaderField } from "../../../new_fields/SchemaHeaderField"; import "./DirectoryImportBox.scss"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; import { BatchedArray } from "array-batcher"; -import { ExifData } from "exif"; const unsupported = ["text/html", "text/plain"]; @@ -117,7 +116,7 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = await Identified.PostFormDataToServer(RouteStore.upload, formData); + const responses = await Networking.PostFormDataToServer(RouteStore.upload, formData); runInAction(() => this.completed += batch.length); return responses as ImageUploadResponse[]; }); diff --git a/src/client/util/Import & Export/ImageUtils.ts b/src/client/util/Import & Export/ImageUtils.ts index c9abf38fa..914f4870a 100644 --- a/src/client/util/Import & Export/ImageUtils.ts +++ b/src/client/util/Import & Export/ImageUtils.ts @@ -3,7 +3,7 @@ import { ImageField } from "../../../new_fields/URLField"; import { Cast, StrCast } from "../../../new_fields/Types"; import { RouteStore } from "../../../server/RouteStore"; import { Docs } from "../../documents/Documents"; -import { Identified } from "../../Network"; +import { Networking } from "../../Network"; import { Id } from "../../../new_fields/FieldSymbols"; import { Utils } from "../../../Utils"; @@ -15,7 +15,7 @@ export namespace ImageUtils { return false; } const source = field.url.href; - const response = await Identified.PostToServer(RouteStore.inspectImage, { source }); + const response = await Networking.PostToServer(RouteStore.inspectImage, { source }); const { error, data } = response.exifData; document.exif = error || Docs.Get.DocumentHierarchyFromJson(data); return data !== undefined; diff --git a/src/new_fields/RichTextUtils.ts b/src/new_fields/RichTextUtils.ts index 601939ed2..63d718ce8 100644 --- a/src/new_fields/RichTextUtils.ts +++ b/src/new_fields/RichTextUtils.ts @@ -17,7 +17,7 @@ import { Cast, StrCast } from "./Types"; import { Id } from "./FieldSymbols"; import { DocumentView } from "../client/views/nodes/DocumentView"; import { AssertionError } from "assert"; -import { Identified } from "../client/Network"; +import { Networking } from "../client/Network"; export namespace RichTextUtils { @@ -129,7 +129,7 @@ export namespace RichTextUtils { return { baseUrl, filename }; }); - const uploads = await Identified.PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems }); + const uploads = await Networking.PostToServer(RouteStore.googlePhotosMediaDownload, { mediaItems }); if (uploads.length !== mediaItems.length) { throw new AssertionError({ expected: mediaItems.length, actual: uploads.length, message: "Error with internally uploading inlineObjects!" }); diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index 21ce9c9e4..eda2a49d2 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -49,9 +49,9 @@ export default class RouteManager { let supervised = async (req: express.Request, res: express.Response) => { const { user, originalUrl: target } = req; const core = { req, res, isRelease }; - const tryExecute = async (target: (args: any) => any | Promise, args: any) => { + const tryExecute = async (toExecute: (args: any) => any | Promise, args: any) => { try { - await target(args); + await toExecute(args); } catch (e) { if (onError) { onError({ ...core, error: e }); diff --git a/src/server/RouteStore.ts b/src/server/RouteStore.ts index de2553b2f..a310d0c95 100644 --- a/src/server/RouteStore.ts +++ b/src/server/RouteStore.ts @@ -39,6 +39,7 @@ export enum RouteStore { writeGoogleAccessToken = "/writeGoogleAccessToken", googlePhotosMediaUpload = "/googlePhotosMediaUpload", googlePhotosMediaDownload = "/googlePhotosMediaDownload", - googleDocsGet = "/googleDocsGet" + googleDocsGet = "/googleDocsGet", + checkGoogle = "/checkGoogleAuthentication" } \ No newline at end of file diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index b9984649e..ff5dc7081 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -7,7 +7,7 @@ import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; import { Database } from "../../database"; -import path from "path"; +import * as path from "path"; /** * @@ -45,7 +45,7 @@ export namespace GoogleApiServerUtils { * */ export interface CredentialsResult { - credentials: Credentials; + credentials: Opt; refreshed: boolean; } @@ -135,8 +135,8 @@ export namespace GoogleApiServerUtils { /** * */ - export const loadClientSecret = async () => { - return new Promise((resolve, reject) => { + export async function loadClientSecret(): Promise { + return new Promise((resolve, reject) => { readFile(path.join(__dirname, "../../credentials/google_docs_credentials.json"), async (err, projectCredentials) => { if (err) { reject(err); @@ -153,7 +153,7 @@ export namespace GoogleApiServerUtils { resolve(); }); }); - }; + } /** * @@ -165,9 +165,12 @@ export namespace GoogleApiServerUtils { * @param sector * @param userId */ - export const GetEndpoint = (sector: string, userId: string) => { - return new Promise>(resolve => { + export async function GetEndpoint(sector: string, userId: string): Promise> { + return new Promise(resolve => { retrieveOAuthClient(userId).then(auth => { + if (!auth) { + return resolve(); + } let routed: Opt; let parameters: EndpointParameters = { auth, version: "v1" }; switch (sector) { @@ -181,29 +184,38 @@ export namespace GoogleApiServerUtils { resolve(routed); }); }); - }; + } /** * * @param userId */ - export const retrieveAccessToken = (userId: string): Promise => { - return new Promise((resolve, reject) => { + export async function retrieveAccessToken(userId: string): Promise { + return new Promise(resolve => { retrieveCredentials(userId).then( - ({ credentials }) => resolve(credentials.access_token!), - error => reject(`Error: unable to authenticate Google Photos API request.\n${error}`) + ({ credentials }) => { + if (credentials) { + return resolve(credentials.access_token!); + } + resolve(); + } ); }); - }; + } /** - * - * @param userId + * Returns an initialized OAuth2 client instance, likely to be passed into Google's + * npm-installed API wrappers that use authenticated client instances rather than access codes for + * security. + * @param userId the Dash user id of the user requesting account integration */ - export const retrieveOAuthClient = (userId: string): Promise => { - return new Promise((resolve, reject) => { + export async function retrieveOAuthClient(userId: string): Promise { + return new Promise((resolve, reject) => { retrieveCredentials(userId).then( ({ credentials, refreshed }) => { + if (!credentials) { + return resolve(); + } let client = authenticationClients.get(userId); if (!client) { authenticationClients.set(userId, client = generateClient(credentials)); @@ -211,31 +223,34 @@ export namespace GoogleApiServerUtils { client.setCredentials(credentials); } resolve(client); - }, - error => reject(`Error: unable to instantiate and certify a new OAuth2 client.\n${error}`) + } ); }); - }; + } /** - * - * @param credentials + * Creates a new OAuth2Client instance, and if provided, sets + * the specific credentials on the client + * @param credentials if you have access to the credentials that you'll eventually set on + * the client, just pass them in at initialization */ - function generateClient(credentials?: Credentials) { + function generateClient(credentials?: Credentials): OAuth2Client { const client = new google.auth.OAuth2(installed); credentials && client.setCredentials(credentials); return client; } /** - * + * Calls on the worker (which does not have and does not need + * any credentials) to produce a url to which the user can + * navigate to give Dash the necessary Google permissions. */ - export const generateAuthenticationUrl = async () => { + export function generateAuthenticationUrl(): string { return worker.generateAuthUrl({ access_type: 'offline', scope: SCOPES.map(relative => prefix + relative), }); - }; + } /** * This is what we return to the server in processNewUser(), after the @@ -255,7 +270,7 @@ export namespace GoogleApiServerUtils { * and the authentication code to fetch the full set of credentials that * we'll store in the database for each user. This is called once per * new account integration. - * @param userId The Dash user id of the user requesting account integration, used to associate the new credentials + * @param userId the Dash user id of the user requesting account integration, used to associate the new credentials * with a Dash user in the googleAuthentication table of the database. * @param authenticationCode the Google-provided authentication code that the user copied * from Google's permissions UI and pasted into the overlay. @@ -263,24 +278,25 @@ export namespace GoogleApiServerUtils { * and display basic user information in the overlay on successful authentication. * This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult. */ - export const processNewUser = async (userId: string, authenticationCode: string): Promise => { - return new Promise((resolve, reject) => { + export async function processNewUser(userId: string, authenticationCode: string): Promise { + const credentials = await new Promise((resolve, reject) => { worker.getToken(authenticationCode, async (err, credentials) => { if (err || !credentials) { reject(err); - return console.error('Error retrieving access token', err); + return; } - const enriched = injectUserInfo(credentials); - await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); - const { given_name, picture } = enriched.userInfo; - resolve({ - access_token: enriched.access_token!, - avatar: picture, - name: given_name - }); + resolve(credentials); }); }); - }; + const enriched = injectUserInfo(credentials); + await Database.Auxiliary.GoogleAuthenticationToken.Write(userId, enriched); + const { given_name, picture } = enriched.userInfo; + return { + access_token: enriched.access_token!, + avatar: picture, + name: given_name + }; + } /** * This type represents the union of the full set of OAuth2 credentials @@ -299,34 +315,31 @@ export namespace GoogleApiServerUtils { * @returns the full set of credentials in the structure in which they'll be stored * in the database. */ - const injectUserInfo = (credentials: Credentials): EnrichedCredentials => { + function injectUserInfo(credentials: Credentials): EnrichedCredentials { const userInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); return { ...credentials, userInfo }; - }; + } /** * Looks in the database for any credentials object with the given user id, * and returns them. If the credentials are found but expired, the function will * automatically refresh the credentials and then resolve with the updated values. - * @param userId the id of the Dash user requesting his/her credentials. Eventually - * might have multiple. - * @returns the credentials and whether or not they were updated in the process + * @param userId the id of the Dash user requesting his/her credentials. Eventually, each user might + * be associated with multiple different sets of Google credentials. + * @returns the credentials and a flag indicating whether or not they were refreshed during retrieval */ - const retrieveCredentials = async (userId: string): Promise => { - return new Promise((resolve, reject) => { - Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId).then(credentials => { - if (!credentials) { - return reject(); - } - if (credentials.expiry_date! < new Date().getTime()) { - // Token has expired, so submitting a request for a refreshed access token - return refreshAccessToken(credentials, userId).then(resolve, reject); - } - // Authentication successful! - resolve({ credentials, refreshed: false }); - }); - }); - }; + async function retrieveCredentials(userId: string): Promise { + let credentials: Opt = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + let refreshed = false; + if (!credentials) { + return { credentials: undefined, refreshed }; + } + // if the token has expired, submit a request for a refreshed access token + if (credentials.expiry_date! <= new Date().getTime()) { + credentials = await refreshAccessToken(credentials, userId); + } + return { credentials, refreshed }; + } /** * This function submits a request to OAuth with the local refresh token @@ -334,26 +347,28 @@ export namespace GoogleApiServerUtils { * the Dash user id passed in. In addition to returning the credentials, it * writes the diff to the database. * @param credentials the credentials - * @param userId + * @param userId the id of the Dash user implicitly requesting that + * his/her credentials be refreshed + * @returns the updated credentials */ - const refreshAccessToken = (credentials: Credentials, userId: string) => { - return new Promise(resolve => { - let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; - let queryParameters = { - refreshToken: credentials.refresh_token, - grant_type: "refresh_token", - ...installed - }; - let url = `${refreshEndpoint}?${qs.stringify(queryParameters)}`; - request.post(url, headerParameters).then(async response => { - let { access_token, expires_in } = JSON.parse(response); - const expiry_date = new Date().getTime() + (expires_in * 1000); - await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); - credentials.access_token = access_token; - credentials.expiry_date = expiry_date; - resolve({ credentials, refreshed: true }); - }); + async function refreshAccessToken(credentials: Credentials, userId: string): Promise { + let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; + let url = `${refreshEndpoint}?${qs.stringify({ + refreshToken: credentials.refresh_token, + grant_type: "refresh_token", + ...installed + })}`; + const { access_token, expires_in } = await new Promise(async resolve => { + const response = await request.post(url, headerParameters); + resolve(JSON.parse(response)); }); - }; + // expires_in is in seconds, but we're building the new expiry date in milliseconds + const expiry_date = new Date().getTime() + (expires_in * 1000); + await Database.Auxiliary.GoogleAuthenticationToken.Update(userId, access_token, expiry_date); + // update the relevant properties + credentials.access_token = access_token; + credentials.expiry_date = expiry_date; + return credentials; + } } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index eb19c71a9..860cde3b5 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -593,11 +593,11 @@ function routeSetter(router: RouteManager) { subscription: RouteStore.readGoogleAccessToken, onValidation: async ({ user, res }) => { const userId = user.id; - const token = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); + const token = await GoogleApiServerUtils.retrieveAccessToken(userId); if (!token) { - return res.send(await GoogleApiServerUtils.generateAuthenticationUrl()); + return res.send(GoogleApiServerUtils.generateAuthenticationUrl()); } - return GoogleApiServerUtils.retrieveAccessToken(userId).then(token => res.send(token)); + return res.send(token); } }); @@ -609,7 +609,7 @@ function routeSetter(router: RouteManager) { } }); - const tokenError = "Unable to successfully upload bytes for all images!"; + const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; router.addSupervisedRoute({ @@ -618,8 +618,12 @@ function routeSetter(router: RouteManager) { onValidation: async ({ user, req, res }) => { const { media } = req.body; - let failed: number[] = []; const token = await GoogleApiServerUtils.retrieveAccessToken(user.id); + if (!token) { + return _error(res, authenticationError); + } + + let failed: number[] = []; const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: GooglePhotosUploadUtils.MediaInput[]) => { -- cgit v1.2.3-70-g09d2 From d4d8c2835c8e1e943f77a14e2b87df05f5848dbd Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 29 Oct 2019 23:26:20 -0400 Subject: finished cleaning and commenting GoogleApiServerUtils --- src/server/apis/google/GoogleApiServerUtils.ts | 233 ++++++++++++------------- 1 file changed, 114 insertions(+), 119 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ff5dc7081..ec7c2cfe1 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -2,7 +2,6 @@ import { google } from "googleapis"; import { readFile } from "fs"; import { OAuth2Client, Credentials, OAuth2ClientOptions } from "google-auth-library"; import { Opt } from "../../../new_fields/Doc"; -import { GlobalOptions } from "googleapis-common"; import { GaxiosResponse } from "gaxios"; import request = require('request-promise'); import * as qs from 'query-string'; @@ -10,19 +9,12 @@ import { Database } from "../../database"; import * as path from "path"; /** - * + * Scopes give Google users fine granularity of control + * over the information they make accessible via the API. + * This is the somewhat overkill list of what Dash requests + * from the user. */ -const prefix = 'https://www.googleapis.com/auth/'; - -/** - * - */ -const refreshEndpoint = "https://oauth2.googleapis.com/token"; - -/** - * - */ -const SCOPES = [ +const scope = [ 'documents.readonly', 'documents', 'presentations', @@ -33,7 +25,7 @@ const SCOPES = [ 'photoslibrary.appendonly', 'photoslibrary.sharing', 'userinfo.profile' -]; +].map(relative => `https://www.googleapis.com/auth/${relative}`); /** * This namespace manages server side authentication for Google API queries, either @@ -42,33 +34,9 @@ const SCOPES = [ export namespace GoogleApiServerUtils { /** - * - */ - export interface CredentialsResult { - credentials: Opt; - refreshed: boolean; - } - - /** - * - */ - export interface UserInfo { - at_hash: string; - aud: string; - azp: string; - exp: number; - family_name: string; - given_name: string; - iat: number; - iss: string; - locale: string; - name: string; - picture: string; - sub: string; - } - - /** - * + * As we expand out to more Google APIs that are accessible from + * the 'googleapis' module imported above, this enum will record + * the list and provide a unified string representation of each API. */ export enum Service { Documents = "Documents", @@ -76,15 +44,10 @@ export namespace GoogleApiServerUtils { } /** - * - */ - export interface CredentialInformation { - credentialsPath: string; - userId: string; - } - - /** - * + * Global credentials read once from a JSON file + * before the server is started that + * allow us to build OAuth2 clients with Dash's + * application specific credentials. */ let installed: OAuth2ClientOptions; @@ -99,27 +62,33 @@ export namespace GoogleApiServerUtils { let worker: OAuth2Client; /** - * + * A briefer format for the response from a 'googleapis' API request */ export type ApiResponse = Promise; /** - * + * A generic form for a handler that executes some request on the endpoint */ export type ApiRouter = (endpoint: Endpoint, parameters: any) => ApiResponse; /** - * + * A generic form for the asynchronous function that actually submits the + * request to the API and returns the corresporing response. Helpful when + * making an extensible endpoint definition. */ export type ApiHandler = (parameters: any, methodOptions?: any) => ApiResponse; /** - * + * A literal union type indicating the valid actions for these 'googleapis' + * requestions */ export type Action = "create" | "retrieve" | "update"; /** - * + * An interface defining any entity on which one can invoke + * anuy of the following handlers. All 'googleapis' wrappers + * such as google.docs().documents and google.slides().presentations + * satisfy this interface. */ export interface Endpoint { get: ApiHandler; @@ -128,12 +97,10 @@ export namespace GoogleApiServerUtils { } /** - * - */ - export type EndpointParameters = GlobalOptions & { version: "v1" }; - - /** - * + * This function is called once before the server is started, + * reading in Dash's project-specific credentials (client secret + * and client id) for later repeated access. It also sets up the + * global, intentionally unauthenticated worker OAuth2 client instance. */ export async function loadClientSecret(): Promise { return new Promise((resolve, reject) => { @@ -156,75 +123,83 @@ export namespace GoogleApiServerUtils { } /** - * + * Maps the Dash user id of a given user to their single + * associated OAuth2 client, mitigating the creation + * of needless duplicate clients that would arise from + * making one new client instance per request. */ const authenticationClients = new Map(); /** - * - * @param sector - * @param userId + * This function receives the target sector ("which G-Suite app's API am I interested in?") + * and the id of the Dash user making the request to the API. With this information, it generates + * an authenticated OAuth2 client and passes it into the relevant 'googleapis' wrapper. + * @param sector the particular desired G-Suite 'googleapis' API (docs, slides, etc.) + * @param userId the id of the Dash user making the request to the API + * @returns the relevant 'googleapis' wrapper, if any */ export async function GetEndpoint(sector: string, userId: string): Promise> { - return new Promise(resolve => { - retrieveOAuthClient(userId).then(auth => { - if (!auth) { - return resolve(); - } - let routed: Opt; - let parameters: EndpointParameters = { auth, version: "v1" }; - switch (sector) { - case Service.Documents: - routed = google.docs(parameters).documents; - break; - case Service.Slides: - routed = google.slides(parameters).presentations; - break; - } - resolve(routed); - }); + return new Promise(async resolve => { + const auth = await retrieveOAuthClient(userId); + if (!auth) { + return resolve(); + } + let routed: Opt; + let parameters: any = { auth, version: "v1" }; + switch (sector) { + case Service.Documents: + routed = google.docs(parameters).documents; + break; + case Service.Slides: + routed = google.slides(parameters).presentations; + break; + } + resolve(routed); }); } /** - * - * @param userId + * Returns the lengthy string or access token that can be passed into + * the headers of an API request or into the constructor of the Photos + * client API wrapper. + * @param userId the Dash user id of the user requesting his/her associated + * access_token + * @returns the current access_token associated with the requesting + * Dash user. The access_token is valid for only an hour, and + * is then refreshed. */ export async function retrieveAccessToken(userId: string): Promise { - return new Promise(resolve => { - retrieveCredentials(userId).then( - ({ credentials }) => { - if (credentials) { - return resolve(credentials.access_token!); - } - resolve(); - } - ); + return new Promise(async resolve => { + const { credentials } = await retrieveCredentials(userId); + if (!credentials) { + return resolve(); + } + resolve(credentials.access_token!); }); } /** - * Returns an initialized OAuth2 client instance, likely to be passed into Google's + * Manipulates a mapping such that, in the limit, each Dash user has + * an associated authenticated OAuth2 client at their disposal. This + * function ensures that the client's credentials always remain up to date + * @param userId the Dash user id of the user requesting account integration + * @returns returns an initialized OAuth2 client instance, likely to be passed into Google's * npm-installed API wrappers that use authenticated client instances rather than access codes for * security. - * @param userId the Dash user id of the user requesting account integration */ export async function retrieveOAuthClient(userId: string): Promise { - return new Promise((resolve, reject) => { - retrieveCredentials(userId).then( - ({ credentials, refreshed }) => { - if (!credentials) { - return resolve(); - } - let client = authenticationClients.get(userId); - if (!client) { - authenticationClients.set(userId, client = generateClient(credentials)); - } else if (refreshed) { - client.setCredentials(credentials); - } - resolve(client); - } - ); + return new Promise(async resolve => { + const { credentials, refreshed } = await retrieveCredentials(userId); + if (!credentials) { + return resolve(); + } + let client = authenticationClients.get(userId); + if (!client) { + authenticationClients.set(userId, client = generateClient(credentials)); + } else if (refreshed) { + client.setCredentials(credentials); + } + resolve(client); }); } @@ -233,6 +208,7 @@ export namespace GoogleApiServerUtils { * the specific credentials on the client * @param credentials if you have access to the credentials that you'll eventually set on * the client, just pass them in at initialization + * @returns the newly created, potentially certified, OAuth2 client instance */ function generateClient(credentials?: Credentials): OAuth2Client { const client = new google.auth.OAuth2(installed); @@ -244,12 +220,10 @@ export namespace GoogleApiServerUtils { * Calls on the worker (which does not have and does not need * any credentials) to produce a url to which the user can * navigate to give Dash the necessary Google permissions. + * @returns the newly generated url to the authentication landing page */ export function generateAuthenticationUrl(): string { - return worker.generateAuthUrl({ - access_type: 'offline', - scope: SCOPES.map(relative => prefix + relative), - }); + return worker.generateAuthUrl({ scope, access_type: 'offline' }); } /** @@ -305,6 +279,26 @@ export namespace GoogleApiServerUtils { */ export type EnrichedCredentials = Credentials & { userInfo: UserInfo }; + /** + * This interface defines all of the information we + * receive from parsing the base64 encoded info-token + * for a Google user. + */ + export interface UserInfo { + at_hash: string; + aud: string; + azp: string; + exp: number; + family_name: string; + given_name: string; + iat: number; + iss: string; + locale: string; + name: string; + picture: string; + sub: string; + } + /** * It's pretty cool: the credentials id_token is split into thirds by periods. * The middle third contains a base64-encoded JSON string with all the @@ -316,7 +310,7 @@ export namespace GoogleApiServerUtils { * in the database. */ function injectUserInfo(credentials: Credentials): EnrichedCredentials { - const userInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); + const userInfo: UserInfo = JSON.parse(atob(credentials.id_token!.split(".")[1])); return { ...credentials, userInfo }; } @@ -326,15 +320,16 @@ export namespace GoogleApiServerUtils { * automatically refresh the credentials and then resolve with the updated values. * @param userId the id of the Dash user requesting his/her credentials. Eventually, each user might * be associated with multiple different sets of Google credentials. - * @returns the credentials and a flag indicating whether or not they were refreshed during retrieval + * @returns the credentials, or undefined if the user has no stored associated credentials, + * and a flag indicating whether or not they were refreshed during retrieval */ - async function retrieveCredentials(userId: string): Promise { + async function retrieveCredentials(userId: string): Promise<{ credentials: Opt, refreshed: boolean }> { let credentials: Opt = await Database.Auxiliary.GoogleAuthenticationToken.Fetch(userId); let refreshed = false; if (!credentials) { return { credentials: undefined, refreshed }; } - // if the token has expired, submit a request for a refreshed access token + // check for token expiry if (credentials.expiry_date! <= new Date().getTime()) { credentials = await refreshAccessToken(credentials, userId); } @@ -353,7 +348,7 @@ export namespace GoogleApiServerUtils { */ async function refreshAccessToken(credentials: Credentials, userId: string): Promise { let headerParameters = { headers: { 'Content-Type': 'application/x-www-form-urlencoded' } }; - let url = `${refreshEndpoint}?${qs.stringify({ + let url = `https://oauth2.googleapis.com/token?${qs.stringify({ refreshToken: credentials.refresh_token, grant_type: "refresh_token", ...installed -- cgit v1.2.3-70-g09d2 From 109be54065038392b19d9dbafbccc9205f198766 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Tue, 29 Oct 2019 23:37:42 -0400 Subject: db error handling and example code --- src/server/apis/google/GoogleApiServerUtils.ts | 3 +++ src/server/database.ts | 4 ++++ 2 files changed, 7 insertions(+) (limited to 'src') diff --git a/src/server/apis/google/GoogleApiServerUtils.ts b/src/server/apis/google/GoogleApiServerUtils.ts index ec7c2cfe1..35a2541a9 100644 --- a/src/server/apis/google/GoogleApiServerUtils.ts +++ b/src/server/apis/google/GoogleApiServerUtils.ts @@ -248,6 +248,9 @@ export namespace GoogleApiServerUtils { * with a Dash user in the googleAuthentication table of the database. * @param authenticationCode the Google-provided authentication code that the user copied * from Google's permissions UI and pasted into the overlay. + * + * EXAMPLE CODE: 4/sgF2A5uGg4xASHf7VQDnLtdqo3mUlfQqLSce_HYz5qf1nFtHj9YTeGs + * * @returns the information necessary to authenticate a client side google photos request * and display basic user information in the overlay on successful authentication. * This can be expanded as needed by adding properties to the interface GoogleAuthenticationResult. diff --git a/src/server/database.ts b/src/server/database.ts index 79dd26b7d..b81fc03a4 100644 --- a/src/server/database.ts +++ b/src/server/database.ts @@ -60,6 +60,10 @@ export namespace Database { constructor() { this.MongoClient.connect(url, (_err, client) => { + if (!client) { + console.error("\nPlease start MongoDB by running 'mongod' in a terminal before continuing...\n"); + process.exit(0); + } this.db = client.db(); this.onConnect.forEach(fn => fn()); }); -- cgit v1.2.3-70-g09d2 From 9c7e619fb9d3116649ec3779bd528b947235d5a4 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Wed, 30 Oct 2019 15:51:29 -0400 Subject: updated array batcher --- package.json | 2 +- .../util/Import & Export/DirectoryImportBox.tsx | 5 +-- src/server/RouteManager.ts | 1 + src/server/apis/google/GooglePhotosUploadUtils.ts | 11 ++--- src/server/index.ts | 48 ++++++++++++---------- 5 files changed, 36 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/package.json b/package.json index 8cbbb84af..4572a3f73 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,7 @@ "@types/youtube": "0.0.38", "adm-zip": "^0.4.13", "archiver": "^3.0.3", - "array-batcher": "^1.1.3", + "array-batcher": "^1.2.3", "async": "^2.6.2", "babel-runtime": "^6.26.0", "bcrypt-nodejs": "0.0.3", diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx index 2d1b6fe20..bdd59cb16 100644 --- a/src/client/util/Import & Export/DirectoryImportBox.tsx +++ b/src/client/util/Import & Export/DirectoryImportBox.tsx @@ -107,7 +107,7 @@ export default class DirectoryImportBox extends React.Component runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`); - const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async batch => { + const uploads = await BatchedArray.from(validated, { batchSize: 15 }).batchedMapAsync(async (batch, collector) => { const formData = new FormData(); batch.forEach(file => { @@ -116,9 +116,8 @@ export default class DirectoryImportBox extends React.Component formData.append(Utils.GenerateGuid(), file); }); - const responses = await Networking.PostFormDataToServer(RouteStore.upload, formData); + collector.push(...(await Networking.PostFormDataToServer(RouteStore.upload, formData))); runInAction(() => this.completed += batch.length); - return responses as ImageUploadResponse[]; }); await Promise.all(uploads.map(async upload => { diff --git a/src/server/RouteManager.ts b/src/server/RouteManager.ts index eda2a49d2..c1d38327f 100644 --- a/src/server/RouteManager.ts +++ b/src/server/RouteManager.ts @@ -114,6 +114,7 @@ export const STATUS = { }; export function _error(res: express.Response, message: string, error?: any) { + console.error(message); res.statusMessage = message; res.status(STATUS.EXECUTION_ERROR).send(error); } diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index 172fa8d46..d3442338b 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,7 +1,7 @@ import request = require('request-promise'); import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; -import { MediaItemCreationResult } from './SharedTypes'; +import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; import { BatchedArray, TimeUnit } from 'array-batcher'; import { DashUploadUtils } from '../../DashUploadUtils'; @@ -50,9 +50,9 @@ export namespace GooglePhotosUploadUtils { }; export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: NewMediaItem[]) => { + async (batch: NewMediaItem[], collector) => { const parameters = { method: 'POST', headers: headers('json', bearerToken), @@ -61,7 +61,7 @@ export namespace GooglePhotosUploadUtils { json: true }; album && (parameters.body.albumId = album.id); - return (await new Promise((resolve, reject) => { + const { newMediaItemResults } = await new Promise((resolve, reject) => { request(parameters, (error, _response, body) => { if (error) { reject(error); @@ -69,7 +69,8 @@ export namespace GooglePhotosUploadUtils { resolve(body); } }); - })).newMediaItemResults; + }); + collector.push(...newMediaItemResults); } ); return { newMediaItemResults }; diff --git a/src/server/index.ts b/src/server/index.ts index 860cde3b5..05c866eae 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -573,18 +573,15 @@ function routeSetter(router: RouteManager) { onValidation: async ({ req, res, user }) => { let sector: GoogleApiServerUtils.Service = req.params.sector as GoogleApiServerUtils.Service; let action: GoogleApiServerUtils.Action = req.params.action as GoogleApiServerUtils.Action; - return GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id).then(endpoint => { - let handler = EndpointHandlerMap.get(action); - if (endpoint && handler) { - let execute = handler(endpoint, req.body).then( - response => res.send(response.data), - rejection => res.send(rejection) - ); - execute.catch(exception => res.send(exception)); - return; - } - res.send(undefined); - }); + const endpoint = await GoogleApiServerUtils.GetEndpoint(GoogleApiServerUtils.Service[sector], user.id); + let handler = EndpointHandlerMap.get(action); + if (endpoint && handler) { + handler(endpoint, req.body) + .then(response => res.send(response.data)) + .catch(exception => res.send(exception)); + return; + } + res.send(undefined); } }); @@ -611,6 +608,12 @@ function routeSetter(router: RouteManager) { const authenticationError = "Unable to authenticate Google credentials before uploading to Google Photos!"; const mediaError = "Unable to convert all uploaded bytes to media items!"; + interface GooglePhotosUploadFailure { + batch: number; + index: number; + url: string; + reason: string; + } router.addSupervisedRoute({ method: Method.POST, @@ -623,30 +626,31 @@ function routeSetter(router: RouteManager) { return _error(res, authenticationError); } - let failed: number[] = []; - const newMediaItems = await BatchedArray.from(media, { batchSize: 25 }).batchedMapPatientInterval( + let failed: GooglePhotosUploadFailure[] = []; + const batched = BatchedArray.from(media, { batchSize: 25 }); + const newMediaItems = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, - async (batch: GooglePhotosUploadUtils.MediaInput[]) => { - const newMediaItems: NewMediaItem[] = []; + async (batch, collector, { completedBatches }) => { for (let index = 0; index < batch.length; index++) { - const element = batch[index]; - const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, element.url); + const { url, description } = batch[index]; + const fail = (reason: string) => failed.push({ reason, batch: completedBatches + 1, index, url }); + const uploadToken = await GooglePhotosUploadUtils.DispatchGooglePhotosUpload(token, url).catch(fail); if (!uploadToken) { - failed.push(index); + fail(`${path.extname(url)} is not an accepted extension`); } else { - newMediaItems.push({ - description: element.description, + collector.push({ + description, simpleMediaItem: { uploadToken } }); } } - return newMediaItems; } ); const failedCount = failed.length; if (failedCount) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); + console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed: ${reason}`).join('\n')); } return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( -- cgit v1.2.3-70-g09d2 From f48b2729b294d08da0c99a242f9ebb4d7aab4407 Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 31 Oct 2019 01:58:42 -0400 Subject: commented and cleaned google photos upload utils --- src/server/DashUploadUtils.ts | 6 +- src/server/apis/google/GooglePhotosUploadUtils.ts | 100 +++++++++++++++++----- src/server/index.ts | 2 +- 3 files changed, 84 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index 46d897339..9fddb466c 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -24,9 +24,13 @@ export namespace DashUploadUtils { const gifs = [".gif"]; const pngs = [".png"]; const jpgs = [".jpg", ".jpeg"]; - export const imageFormats = [...pngs, ...jpgs, ...gifs]; + const imageFormats = [...pngs, ...jpgs, ...gifs]; const videoFormats = [".mov", ".mp4"]; + export function validateExtension(url: string) { + return imageFormats.includes(path.extname(url).toLowerCase()); + } + const size = "content-length"; const type = "content-type"; diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index d3442338b..a98399621 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -1,56 +1,111 @@ import request = require('request-promise'); -import { GoogleApiServerUtils } from './GoogleApiServerUtils'; import * as path from 'path'; import { MediaItemCreationResult, NewMediaItemResult } from './SharedTypes'; import { NewMediaItem } from "../../index"; import { BatchedArray, TimeUnit } from 'array-batcher'; import { DashUploadUtils } from '../../DashUploadUtils'; +/** + * This namespace encompasses the logic + * necessary to upload images to Google's server, + * and then initialize / create those images in the Photos + * API given the upload tokens returned from the initial + * uploading process. + * + * https://developers.google.com/photos/library/reference/rest/v1/mediaItems/batchCreate + */ export namespace GooglePhotosUploadUtils { - export interface Paths { - uploadDirectory: string; - credentialsPath: string; - tokenPath: string; - } - - export interface MediaInput { + /** + * Specifies the structure of the object + * necessary to upload bytes to Google's servers. + * The url is streamed to access the image's bytes, + * and the description is what appears in Google Photos' + * description field. + */ + export interface UploadSource { url: string; description: string; } - const prepend = (extension: string) => `https://photoslibrary.googleapis.com/v1/${extension}`; - const headers = (type: string, token: string) => ({ - 'Content-Type': `application/${type}`, - 'Authorization': `Bearer ${token}`, - }); + /** + * A utility function to streamline making + * calls to the API's url - accentuates + * the relative path in the caller. + * @param extension the desired + * subset of the API + */ + function prepend(extension: string): string { + return `https://photoslibrary.googleapis.com/v1/${extension}`; + } + + /** + * Factors out the creation of the API request's + * authentication elements stored in the header. + * @param type the contents of the request + * @param token the user-specific Google access token + */ + function headers(type: string, token: string) { + return { + 'Content-Type': `application/${type}`, + 'Authorization': `Bearer ${token}`, + }; + } - export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string) => { - if (!DashUploadUtils.imageFormats.includes(path.extname(url))) { + /** + * This is the first step in the remote image creation process. + * Here we upload the raw bytes of the image to Google's servers by + * setting authentication and other required header properties and including + * the raw bytes to the image, to be uploaded, in the body of the request. + * @param bearerToken the user-specific Google access token, specifies the account associated + * with the eventual image creation + * @param url the url of the image to upload + * @param filename an optional name associated with the uploaded image - if not specified + * defaults to the filename (basename) in the url + */ + export const DispatchGooglePhotosUpload = async (bearerToken: string, url: string, filename?: string): Promise => { + // check if the url points to a non-image or an unsupported format + if (!DashUploadUtils.validateExtension(url)) { return undefined; } - const body = await request(url, { encoding: null }); const parameters = { method: 'POST', + uri: prepend('uploads'), headers: { ...headers('octet-stream', bearerToken), - 'X-Goog-Upload-File-Name': path.basename(url), + 'X-Goog-Upload-File-Name': filename || path.basename(url), 'X-Goog-Upload-Protocol': 'raw' }, - uri: prepend('uploads'), - body + body: await request(url, { encoding: null }) // returns a readable stream with the unencoded binary image data }; - return new Promise((resolve, reject) => request(parameters, (error, _response, body) => { + return new Promise((resolve, reject) => request(parameters, (error, _response, body) => { if (error) { - console.log(error); + // on rejection, the server logs the error and the offending image return reject(error); } resolve(body); })); }; + /** + * This is the second step in the remote image creation process: having uploaded + * the raw bytes of the image and received / stored pointers (upload tokens) to those + * bytes, we can now instruct the API to finalize the creation of those images by + * submitting a batch create request with the list of upload tokens and the description + * to be associated with reach resulting new image. + * @param bearerToken the user-specific Google access token, specifies the account associated + * with the eventual image creation + * @param newMediaItems a list of objects containing a description and, effectively, the + * pointer to the uploaded bytes + * @param album if included, will add all of the newly created remote images to the album + * with the specified id + */ export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { - const newMediaItemResults = await BatchedArray.from(newMediaItems, { batchSize: 50 }).batchedMapPatientInterval( + // it's important to note that the API can't handle more than 50 items in each request and + // seems to need at least some latency between requests (spamming it synchronously has led to the server returning errors)... + const batched = BatchedArray.from(newMediaItems, { batchSize: 50 }); + // ...so we execute them in delayed batches and await the entire execution + const newMediaItemResults = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: NewMediaItem[], collector) => { const parameters = { @@ -60,6 +115,7 @@ export namespace GooglePhotosUploadUtils { body: { newMediaItems: batch } as any, json: true }; + // register the target album, if provided album && (parameters.body.albumId = album.id); const { newMediaItemResults } = await new Promise((resolve, reject) => { request(parameters, (error, _response, body) => { diff --git a/src/server/index.ts b/src/server/index.ts index 05c866eae..9f3e34761 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -627,7 +627,7 @@ function routeSetter(router: RouteManager) { } let failed: GooglePhotosUploadFailure[] = []; - const batched = BatchedArray.from(media, { batchSize: 25 }); + const batched = BatchedArray.from(media, { batchSize: 25 }); const newMediaItems = await batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch, collector, { completedBatches }) => { -- cgit v1.2.3-70-g09d2 From c53d599f8ecffe173d8df06777721658f065674a Mon Sep 17 00:00:00 2001 From: Sam Wilkins Date: Thu, 31 Oct 2019 14:32:39 -0400 Subject: cleanup --- src/server/apis/google/GooglePhotosUploadUtils.ts | 12 +++++------- src/server/index.ts | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/server/apis/google/GooglePhotosUploadUtils.ts b/src/server/apis/google/GooglePhotosUploadUtils.ts index a98399621..d8cf795b5 100644 --- a/src/server/apis/google/GooglePhotosUploadUtils.ts +++ b/src/server/apis/google/GooglePhotosUploadUtils.ts @@ -100,12 +100,12 @@ export namespace GooglePhotosUploadUtils { * @param album if included, will add all of the newly created remote images to the album * with the specified id */ - export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { + export const CreateMediaItems = async (bearerToken: string, newMediaItems: NewMediaItem[], album?: { id: string }): Promise => { // it's important to note that the API can't handle more than 50 items in each request and // seems to need at least some latency between requests (spamming it synchronously has led to the server returning errors)... const batched = BatchedArray.from(newMediaItems, { batchSize: 50 }); // ...so we execute them in delayed batches and await the entire execution - const newMediaItemResults = await batched.batchedMapPatientInterval( + return batched.batchedMapPatientInterval( { magnitude: 100, unit: TimeUnit.Milliseconds }, async (batch: NewMediaItem[], collector) => { const parameters = { @@ -117,19 +117,17 @@ export namespace GooglePhotosUploadUtils { }; // register the target album, if provided album && (parameters.body.albumId = album.id); - const { newMediaItemResults } = await new Promise((resolve, reject) => { + collector.push(...(await new Promise((resolve, reject) => { request(parameters, (error, _response, body) => { if (error) { reject(error); } else { - resolve(body); + resolve(body.newMediaItemResults); } }); - }); - collector.push(...newMediaItemResults); + }))); } ); - return { newMediaItemResults }; }; } \ No newline at end of file diff --git a/src/server/index.ts b/src/server/index.ts index 9f3e34761..25697e71f 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -650,11 +650,11 @@ function routeSetter(router: RouteManager) { const failedCount = failed.length; if (failedCount) { console.error(`Unable to upload ${failedCount} image${failedCount === 1 ? "" : "s"} to Google's servers`); - console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed: ${reason}`).join('\n')); + console.log(failed.map(({ reason, batch, index, url }) => `@${batch}.${index}: ${url} failed:\n${reason}`).join('\n\n')); } return GooglePhotosUploadUtils.CreateMediaItems(token, newMediaItems, req.body.album).then( - result => _success(res, { results: result.newMediaItemResults, failed }), + results => _success(res, { results, failed }), error => _error(res, mediaError, error) ); } -- cgit v1.2.3-70-g09d2