fixes for file uploads: restored progress for youtube videos, fixed info display on loadingBoxes, stop client from asking for progress on failed uploads. attempt to catch stream errors on webpages. fixed dataFieldView in text to print out layout document's field, not field of document that contains the field definition.

3 files changed, 27 insertions, 17 deletions

diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index ebc9deab7..c264803bd 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -102,11 +102,10 @@ export default class UploadManager extends ApiManager {
                 //req.readableBuffer.head.data
                 return new Promise<void>(async resolve => {
                     req.addListener('data', async args => {
-                        console.log(args);
                         const payload = String.fromCharCode.apply(String, args);
-                        const videoId = JSON.parse(payload).videoId;
+                        const { videoId, overwriteId } = JSON.parse(payload);
                         const results: Upload.FileResponse[] = [];
-                        const result = await DashUploadUtils.uploadYoutube(videoId);
+                        const result = await DashUploadUtils.uploadYoutube(videoId, overwriteId ?? videoId);
                         result && results.push(result);
                         _success(res, results);
                         resolve();
@@ -123,7 +122,7 @@ export default class UploadManager extends ApiManager {
                     req.addListener('data', args => {
                         const payload = String.fromCharCode.apply(String, args);
                         const videoId = JSON.parse(payload).videoId;
-                        _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId) });
+                        _success(res, { progress: DashUploadUtils.QueryYoutubeProgress(videoId, req.user) });
                         resolve();
                     });
                 });
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index ed2430e3a..1f23ae8c1 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -135,36 +135,39 @@ export namespace DashUploadUtils {
         };
     }
 
-    export function QueryYoutubeProgress(videoId: string) {
-        console.log("PROGRESS:" + videoId)
+    export function QueryYoutubeProgress(videoId: string, user?: Express.User) {
+        console.log(`PROGRESS:${videoId}`, (user as any)?.email);
         return uploadProgress.get(videoId) ?? 'pending data upload';
     }
 
     let uploadProgress = new Map<string, string>();
 
-    export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
+    export function uploadYoutube(videoId: string, overwriteId: string): Promise<Upload.FileResponse> {
         return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
             const name = videoId;
             const path = name.replace(/^-/, '__') + '.mp4';
             const finalPath = serverPathToFile(Directory.videos, path);
             if (existsSync(finalPath)) {
-                uploadProgress.set(videoId, 'computing duration');
+                uploadProgress.set(overwriteId, 'computing duration');
                 exec(`yt-dlp -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                     const time = Array.from(stdout.trim().split(':')).reverse();
                     const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
                     res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined));
                 });
             } else {
-                uploadProgress.set(videoId, 'starting download');
+                uploadProgress.set(overwriteId, 'starting download');
                 const ytdlp = spawn(`yt-dlp`, ['-o', path, `https://www.youtube.com/watch?v=${videoId}`, '--max-filesize', '100M', '-f', 'mp4']);
 
-                ytdlp.stdout.on('data', (data: any) => !uploadProgress.get(videoId)?.includes('Aborting.') && uploadProgress.set(videoId, data.toString()));
+                ytdlp.stdout.on('data', (data: any) => uploadProgress.set(overwriteId, data.toString()));
 
                 let errors = '';
-                ytdlp.stderr.on('data', (data: any) => (errors = data.toString()));
+                ytdlp.stderr.on('data', (data: any) => {
+                    uploadProgress.set(overwriteId, 'error:' + data.toString());
+                    errors = data.toString();
+                });
 
                 ytdlp.on('exit', function (code: any) {
-                    if (code || uploadProgress.get(videoId)?.includes('Aborting.')) {
+                    if (code) {
                         res({
                             source: {
                                 size: 0,
@@ -176,7 +179,7 @@ export namespace DashUploadUtils {
                             result: { name: 'failed youtube query', message: `Could not archive video.  ${code ? errors : uploadProgress.get(videoId)}` },
                         });
                     } else {
-                        uploadProgress.set(videoId, 'computing duration');
+                        uploadProgress.set(overwriteId, 'computing duration');
                         exec(`yt-dlp-o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
                             const time = Array.from(stdout.trim().split(':')).reverse();
                             const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
@@ -194,7 +197,7 @@ export namespace DashUploadUtils {
         const isAzureOn = usingAzure();
         const { type, path, name } = file;
         const types = type?.split('/') ?? [];
-        console.log("UPLOADING:"+ (overwriteGuid ?? name));
+        console.log('UPLOADING:' + (overwriteGuid ?? name));
         uploadProgress.set(overwriteGuid ?? name, 'uploading'); // If the client sent a guid it uses to track upload progress, use that guid. Otherwise, use the file's name.
 
         const category = types[0];
@@ -282,6 +285,7 @@ export namespace DashUploadUtils {
         const fileKey = (await md5File(file.path)) + '.pdf';
         const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
         if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) {
+            fs.unlink(file.path, () => {});
             return new Promise<Upload.FileResponse>(res => {
                 const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
                 const readStream = createReadStream(serverPathToFile(Directory.text, textFilename));
diff --git a/src/server/server_Initialization.ts b/src/server/server_Initialization.ts
index 64f90d76c..839091194 100644
--- a/src/server/server_Initialization.ts
+++ b/src/server/server_Initialization.ts
@@ -194,14 +194,20 @@ function proxyServe(req: any, requrl: string, response: any) {
                         .replace(/target="_blank"/g, '');
                     response.send(header?.includes('gzip') ? zlib.gzipSync(htmlText) : htmlText);
                 } else {
-                    req.pipe(request(requrl)).pipe(response);
+                    req.pipe(request(requrl))
+                        .on('error', (e: any) => console.log('requrl ', e))
+                        .pipe(response)
+                        .on('error', (e: any) => console.log('response pipe error', e));
                     console.log('EMPTY body:' + req.url);
                 }
             } catch (e) {
                 console.log('ERROR?: ', e);
             }
         } else {
-            req.pipe(htmlBodyMemoryStream).pipe(response);
+            req.pipe(htmlBodyMemoryStream)
+                .on('error', (e: any) => console.log('html body memorystream error', e))
+                .pipe(response)
+                .on('error', (e: any) => console.log('html body memory stream response error', e));
         }
     };
     const retrieveHTTPBody = () => {
@@ -237,7 +243,8 @@ function proxyServe(req: any, requrl: string, response: any) {
                 response.headers = response._headers = res.headers;
             })
             .on('end', sendModifiedBody)
-            .pipe(htmlBodyMemoryStream);
+            .pipe(htmlBodyMemoryStream)
+            .on('error', (e: any) => console.log('http body pipe error', e));
     };
     retrieveHTTPBody();
 }