diff options
Diffstat (limited to 'src/server')
| -rw-r--r-- | src/server/ApiManagers/UploadManager.ts | 48 | ||||
| -rw-r--r-- | src/server/DashUploadUtils.ts | 44 | ||||
| -rw-r--r-- | src/server/server_Initialization.ts | 83 |
3 files changed, 124 insertions, 51 deletions
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index ebc9deab7..ea5d8cb33 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -64,6 +64,17 @@ export default class UploadManager extends ApiManager { subscription: '/uploadFormData', secureHandler: async ({ req, res }) => { const form = new formidable.IncomingForm(); + let fileguids = ''; + let filesize = ''; + form.on('field', (e: string, value: string) => { + if (e === 'fileguids') { + (fileguids = value).split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, 'reading file')); + } + if (e === 'filesize') { + filesize = value; + } + }); + form.on('progress', e => fileguids.split(';').map(guid => DashUploadUtils.uploadProgress.set(guid, `read:(${Math.round((100 * +e) / +filesize)}%) ${e} of ${filesize}`))); form.keepExtensions = true; form.uploadDir = pathToDirectory(Directory.parsed_files); return new Promise<void>(resolve => { @@ -102,11 +113,10 @@ export default class UploadManager extends ApiManager { //req.readableBuffer.head.data return new Promise<void>(async resolve => { req.addListener('data', async args => { - console.log(args); const payload = String.fromCharCode.apply(String, args); - const videoId = JSON.parse(payload).videoId; + const { videoId, overwriteId } = JSON.parse(payload); const results: Upload.FileResponse[] = []; - const result = await DashUploadUtils.uploadYoutube(videoId); + const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId); result && results.push(result); _success(res, results); resolve(); @@ -123,7 +133,7 @@ export default class UploadManager extends ApiManager { req.addListener('data', args => { const payload = String.fromCharCode.apply(String, args); const videoId = JSON.parse(payload).videoId; - _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) }); + _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId, req.user) }); resolve(); }); }); @@ -252,9 +262,33 @@ export default class UploadManager extends ApiManager { zip.extractEntryTo(entry.entryName, publicDirectory, true, false); createReadStream(pathname).pipe(createWriteStream(targetname)); if (extension !== '.pdf') { - createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_s' + extension))); - createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_m' + extension))); - createReadStream(pathname).pipe(createWriteStream(targetname.replace('_o' + extension, '_l' + extension))); + const { pngs, jpgs } = AcceptableMedia; + const resizers = [ + { resizer: sharp().resize(100, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Small }, + { resizer: sharp().resize(400, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Medium }, + { resizer: sharp().resize(900, undefined, { withoutEnlargement: true }), suffix: SizeSuffix.Large }, + ]; + let isImage = false; + if (pngs.includes(extension)) { + resizers.forEach(element => { + element.resizer = element.resizer.png(); + }); + isImage = true; + } else if (jpgs.includes(extension)) { + resizers.forEach(element => { + element.resizer = element.resizer.jpeg(); + }); + isImage = true; + } + if (isImage) { + resizers.forEach(resizer => { + createReadStream(pathname) + .on('error', e => console.log('Resizing read:' + e)) + .pipe(resizer.resizer) + .on('error', e => console.log('Resizing write: ' + e)) + .pipe(createWriteStream(targetname.replace('_o' + extension, resizer.suffix + extension)).on('error', e => console.log('Resizing write: ' + e))); + }); + } } unlink(pathname, () => {}); } catch (e) { diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index e5e15ce99..19cb3f240 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -135,35 +135,39 @@ export namespace DashUploadUtils { }; } - export function QueryYoutubeProgress(videoId: string) { - return uploadProgress.get(videoId) ?? 'failed'; + export function QueryYoutubeProgress(videoId: string, user?: Express.User) { + // console.log(`PROGRESS:${videoId}`, (user as any)?.email); + return uploadProgress.get(videoId) ?? 'pending data upload'; } - let uploadProgress = new Map<string, string>(); + export let uploadProgress = new Map<string, string>(); - export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> { + export function uploadYoutube(videoId: string, overwriteId: string): Promise<Upload.FileResponse> { return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => { const name = videoId; const path = name.replace(/^-/, '__') + '.mp4'; const finalPath = serverPathToFile(Directory.videos, path); if (existsSync(finalPath)) { - uploadProgress.set(videoId, 'computing duration'); + uploadProgress.set(overwriteId, 'computing duration'); exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { const time = Array.from(stdout.trim().split(':')).reverse(); const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined)); }); } else { - uploadProgress.set(videoId, 'starting download'); + uploadProgress.set(overwriteId, 'starting download'); const ytdlp = spawn(`yt-dlp`, ['-o', path, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']); - ytdlp.stdout.on('data', (data: any) => !uploadProgress.get(videoId)?.includes('Aborting.') && uploadProgress.set(videoId, data.toString())); + ytdlp.stdout.on('data', (data: any) => uploadProgress.set(overwriteId, data.toString())); let errors = ''; - ytdlp.stderr.on('data', (data: any) => (errors = data.toString())); + ytdlp.stderr.on('data', (data: any) => { + uploadProgress.set(overwriteId, 'error:' + data.toString()); + errors = data.toString(); + }); ytdlp.on('exit', function (code: any) { - if (code || uploadProgress.get(videoId)?.includes('Aborting.')) { + if (code) { res({ source: { size: 0, @@ -175,7 +179,7 @@ export namespace DashUploadUtils { result: { name: 'failed youtube query', message: `Could not archive video. ${code ? errors : uploadProgress.get(videoId)}` }, }); } else { - uploadProgress.set(videoId, 'computing duration'); + uploadProgress.set(overwriteId, 'computing duration'); exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => { const time = Array.from(stdout.trim().split(':')).reverse(); const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0); @@ -280,6 +284,7 @@ export namespace DashUploadUtils { const fileKey = (await md5File(file.path)) + '.pdf'; const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) { + fs.unlink(file.path, () => {}); return new Promise<Upload.FileResponse>(res => { const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`; const readStream = createReadStream(serverPathToFile(Directory.text, textFilename)); @@ -385,7 +390,7 @@ export namespace DashUploadUtils { const resolved = (filename = `upload_${Utils.GenerateGuid()}.${ext}`); if (usingAzure()) { const response = await AzureManager.UploadBase64ImageBlob(resolved, data); - source = `${AzureManager.BASE_STRING}/${resolved}`; + source = `${AzureManager.BASE_STRING}/${resolved}`; } else { const error = await new Promise<Error | null>(resolve => { writeFile(serverPathToFile(Directory.images, resolved), data, 'base64', resolve); @@ -596,11 +601,20 @@ export namespace DashUploadUtils { const outputPath = path.resolve(outputDirectory, (writtenFiles[suffix] = InjectSize(outputFileName, suffix))); await new Promise<void>(async (resolve, reject) => { const source = streamProvider(); - let readStream: Stream = source instanceof Promise ? await source : source; + let readStream = source instanceof Promise ? await source : source; + let error = false; if (resizer) { - readStream = readStream.pipe(resizer.withMetadata()); + readStream = readStream.pipe(resizer.withMetadata()).on('error', async args => { + error = true; + if (error) { + const source2 = streamProvider(); + let readStream2: Stream | undefined; + readStream2 = source2 instanceof Promise ? await source2 : source2; + readStream2?.pipe(createWriteStream(outputPath)).on('error', resolve).on('close', resolve); + } + }); } - readStream.pipe(createWriteStream(outputPath)).on('close', resolve).on('error', reject); + !error && readStream?.pipe(createWriteStream(outputPath)).on('error', resolve).on('close', resolve); }); } return writtenFiles; @@ -628,7 +642,7 @@ export namespace DashUploadUtils { initial = undefined; } return { - resizer: initial, + resizer: suffix === '_o' ? undefined : initial, suffix, }; }), diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts index ee32de152..ccb709453 100644 --- a/src/server/server_Initialization.ts +++ b/src/server/server_Initialization.ts @@ -121,11 +121,11 @@ function determineEnvironment() { const label = isRelease ? 'release' : 'development'; console.log(`\nrunning server in ${color(label)} mode`); - // swilkins: I don't think we need to read from ClientUtils.RELEASE anymore. Should be able to invoke process.env.RELEASE - // on the client side, thanks to dotenv in webpack.config.js - let clientUtils = fs.readFileSync('./src/client/util/ClientUtils.ts.temp', 'utf8'); - clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; - fs.writeFileSync('./src/client/util/ClientUtils.ts', clientUtils, 'utf8'); + // // swilkins: I don't think we need to read from ClientUtils.RELEASE anymore. Should be able to invoke process.env.RELEASE + // // on the client side, thanks to dotenv in webpack.config.js + // let clientUtils = fs.readFileSync('./src/client/util/ClientUtils.ts.temp', 'utf8'); + // clientUtils = `//AUTO-GENERATED FILE: DO NOT EDIT\n${clientUtils.replace('"mode"', String(isRelease))}`; + // fs.writeFileSync('./src/client/util/ClientUtils.ts', clientUtils, 'utf8'); return isRelease; } @@ -149,55 +149,78 @@ function registerAuthenticationRoutes(server: express.Express) { function registerCorsProxy(server: express.Express) { server.use('/corsProxy', async (req, res) => { - //const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : ''; - let requrl = decodeURIComponent(req.url.substring(1)); - const qsplit = requrl.split('?q='); - const newqsplit = requrl.split('&q='); + res.setHeader('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE'); + res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers')); + const referer = req.headers.referer ? decodeURIComponent(req.headers.referer) : ''; + let requrlraw = decodeURIComponent(req.url.substring(1)); + const qsplit = requrlraw.split('?q='); + const newqsplit = requrlraw.split('&q='); if (qsplit.length > 1 && newqsplit.length > 1) { const lastq = newqsplit[newqsplit.length - 1]; - requrl = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1]; + requrlraw = qsplit[0] + '?q=' + lastq.split('&')[0] + '&' + qsplit[1].split('&')[1]; + } + const requrl = requrlraw.startsWith('/') ? referer + requrlraw : requrlraw; + // cors weirdness here... + // if the referer is a cors page and the cors() route (I think) redirected to /corsProxy/<path> and the requested url path was relative, + // then we redirect again to the cors referer and just add the relative path. + if (!requrl.startsWith('http') && req.originalUrl.startsWith('/corsProxy') && referer?.includes('corsProxy')) { + res.redirect(referer + (referer.endsWith('/') ? '' : '/') + requrl); + } else { + proxyServe(req, requrl, res); } - proxyServe(req, requrl, res); }); } function proxyServe(req: any, requrl: string, response: any) { const htmlBodyMemoryStream = new (require('memorystream'))(); - var retrieveHTTPBody: any; var wasinBrFormat = false; const sendModifiedBody = () => { const header = response.headers['content-encoding']; - const httpsToCors = (match: any, href: string, offset: any, string: any) => `href="${resolvedServerUrl + '/corsProxy/http' + href}"`; - if (header?.includes('gzip')) { + const refToCors = (match: any, tag: string, sym: string, href: string, offset: any, string: any) => `${tag}=${sym + resolvedServerUrl}/corsProxy/${href + sym}`; + const relpathToCors = (match: any, href: string, offset: any, string: any) => `="${resolvedServerUrl + '/corsProxy/' + decodeURIComponent(req.originalUrl.split('/corsProxy/')[1].match(/https?:\/\/[^\/]*/)?.[0] ?? '') + '/' + href}"`; + if (header) { try { const bodyStream = htmlBodyMemoryStream.read(); if (bodyStream) { - const htmlInputText = wasinBrFormat ? Buffer.from(brotli.decompress(bodyStream)) : zlib.gunzipSync(bodyStream); + const htmlInputText = wasinBrFormat ? Buffer.from(brotli.decompress(bodyStream)) : header.includes('gzip') ? zlib.gunzipSync(bodyStream) : bodyStream; const htmlText = htmlInputText .toString('utf8') .replace('<head>', '<head> <style>[id ^= "google"] { display: none; } </style>') - // .replace('<script', '<noscript') - // .replace('</script', '</noscript') - // .replace(/href="https?([^"]*)"/g, httpsToCors) + .replace(/(src|href)=([\'\"])(https?[^\2\n]*)\1/g, refToCors) // replace src or href='http(s)://...' or href="http(s)://.." + //.replace(/= *"\/([^"]*)"/g, relpathToCors) .replace(/data-srcset="[^"]*"/g, '') .replace(/srcset="[^"]*"/g, '') .replace(/target="_blank"/g, ''); - response.send(zlib.gzipSync(htmlText)); + response.send(header?.includes('gzip') ? zlib.gzipSync(htmlText) : htmlText); } else { - req.pipe(request(requrl)).pipe(response); + req.pipe(request(requrl)) + .on('error', (e: any) => console.log('requrl ', e)) + .pipe(response) + .on('error', (e: any) => console.log('response pipe error', e)); console.log('EMPTY body:' + req.url); } } catch (e) { console.log('ERROR?: ', e); } } else { - req.pipe(htmlBodyMemoryStream).pipe(response); + req.pipe(htmlBodyMemoryStream) + .on('error', (e: any) => console.log('html body memorystream error', e)) + .pipe(response) + .on('error', (e: any) => console.log('html body memory stream response error', e)); } }; - retrieveHTTPBody = () => { - req.headers.cookie = ''; + const retrieveHTTPBody = () => { + //req.headers.cookie = ''; req.pipe(request(requrl)) - .on('error', (e: any) => console.log(`Malformed CORS url: ${requrl}`, e)) + .on('error', (e: any) => { + console.log(`CORS url error: ${requrl}`, e); + response.send(`<html><body bgcolor="red" link="006666" alink="8B4513" vlink="006666"> + <title>Error</title> + <div align="center"><h1>Failed to load: ${requrl} </h1></div> + <p>${e}</p> + </body></html>`); + }) .on('response', (res: any) => { res.headers; const headers = Object.keys(res.headers); @@ -220,16 +243,18 @@ function proxyServe(req: any, requrl: string, response: any) { response.headers = response._headers = res.headers; }) .on('end', sendModifiedBody) - .pipe(htmlBodyMemoryStream); + .pipe(htmlBodyMemoryStream) + .on('error', (e: any) => console.log('http body pipe error', e)); }; retrieveHTTPBody(); } function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { server.use('*', (req, res) => { + // res.setHeader('Access-Control-Allow-Origin', '*'); + // res.header('Access-Control-Allow-Methods', 'GET, PUT, PATCH, POST, DELETE'); + // res.header('Access-Control-Allow-Headers', req.header('access-control-request-headers')); const relativeUrl = req.originalUrl; - // if (req.originalUrl === '/css/main.css' || req.originalUrl === '/favicon.ico') res.end(); - // else if (!res.headersSent && req.headers.referer?.includes('corsProxy')) { if (!req.user) res.redirect('/home'); // When no user is logged in, we interpret a relative URL as being a reference to something they don't have access to and redirect to /home // a request for something by a proxied referrer means it must be a relative reference. So construct a proxied absolute reference here. @@ -239,8 +264,8 @@ function registerEmbeddedBrowseRelativePathHandler(server: express.Express) { const actualReferUrl = proxiedRefererUrl.replace(dashServerUrl, ''); // the url of the referer without the proxy (e.g., : https://en.wikipedia.org/wiki/Engelbart) const absoluteTargetBaseUrl = actualReferUrl.match(/https?:\/\/[^\/]*/)![0]; // the base of the original url (e.g., https://en.wikipedia.org) const redirectedProxiedUrl = dashServerUrl + encodeURIComponent(absoluteTargetBaseUrl + relativeUrl); // the new proxied full url (e.g., http://localhost:<port>/corsProxy/https://en.wikipedia.org/<somethingelse>) - if (relativeUrl.startsWith('//')) res.redirect('http:' + relativeUrl); - else res.redirect(redirectedProxiedUrl); + const redirectUrl = relativeUrl.startsWith('//') ? 'http:' + relativeUrl : redirectedProxiedUrl; + res.redirect(redirectUrl); } catch (e) { console.log('Error embed: ', e); } |