aboutsummaryrefslogtreecommitdiff
path: root/src/server/DashUploadUtils.ts
diff options
context:
space:
mode:
Diffstat (limited to 'src/server/DashUploadUtils.ts')
-rw-r--r--src/server/DashUploadUtils.ts128
1 files changed, 86 insertions, 42 deletions
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index c38f603a4..c28ae686b 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -2,7 +2,7 @@ import { green, red } from 'colors';
import { ExifImage } from 'exif';
import * as exifr from 'exifr';
import { File } from 'formidable';
-import { createWriteStream, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs';
+import { createReadStream, createWriteStream, exists, existsSync, readFileSync, rename, unlinkSync, writeFile } from 'fs';
import * as path from 'path';
import { basename } from 'path';
import * as sharp from 'sharp';
@@ -22,6 +22,7 @@ const parse = require('pdf-parse');
const ffmpeg = require('fluent-ffmpeg');
const fs = require('fs');
const requestImageSize = require('../client/util/request-image-size');
+const md5File = require('md5-file');
export enum SizeSuffix {
Small = '_s',
@@ -99,34 +100,58 @@ export namespace DashUploadUtils {
};
}
+ function resolveExistingFile(name: string, path: string, directory: Directory, type?: string, duration?: number, rawText?: string) {
+ const data = { size: 0, path, name, type: type ?? '' };
+ const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration?.toString(), mime: '', toJson: () => undefined as any }) };
+ return {
+ source: file,
+ result: {
+ accessPaths: {
+ agnostic: getAccessPaths(directory, path),
+ },
+ rawText,
+ duration,
+ },
+ };
+ }
+
export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
return new Promise<Upload.FileResponse<Upload.FileInformation>>((res, rej) => {
console.log('Uploading YouTube video: ' + videoId);
const name = videoId;
const path = name.replace(/^-/, '__') + '.mp4';
- exec(`youtube-dl -o ${path} "https://www.youtube.com/watch?v=${videoId}" -f "mp4[filesize<5M]/bestvideo[filesize<5M]+bestaudio/bestvideo+bestaudio"`, (error: any, stdout: any, stderr: any) => {
- if (error) {
- console.log(`error: Error: ${error.message}`);
- res({
- source: {
- size: 0,
- path,
- name,
- type: '',
- toJSON: () => ({ name, path }),
- },
- result: { name: 'failed youtube query', message: `Could not upload YouTube video (${videoId}). Error: ${error.message}` },
- });
- } else {
- exec(`youtube-dl -o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
- const time = Array.from(stdout.trim().split(':')).reverse();
- const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
- const data = { size: 0, path, name, type: 'video/mp4' };
- const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) };
- res(MoveParsedFile(file, Directory.videos));
- });
- }
- });
+ const finalPath = serverPathToFile(Directory.videos, path);
+ if (existsSync(finalPath)) {
+ exec(`youtube-dl -o ${finalPath} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
+ const time = Array.from(stdout.trim().split(':')).reverse();
+ const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
+ res(resolveExistingFile(name, finalPath, Directory.videos, 'video/mp4', duration, undefined));
+ });
+ } else {
+ exec(`youtube-dl -o ${path} "https://www.youtube.com/watch?v=${videoId}" -f "mp4[filesize<5M]/bestvideo[filesize<5M]+bestaudio/bestvideo+bestaudio"`, (error: any, stdout: any, stderr: any) => {
+ if (error) {
+ console.log(`error: Error: ${error.message}`);
+ res({
+ source: {
+ size: 0,
+ path,
+ name,
+ type: '',
+ toJSON: () => ({ name, path }),
+ },
+ result: { name: 'failed youtube query', message: `Could not upload YouTube video (${videoId}). Error: ${error.message}` },
+ });
+ } else {
+ exec(`youtube-dl -o ${path} "https://www.youtube.com/watch?v=${videoId}" --get-duration`, (error: any, stdout: any, stderr: any) => {
+ const time = Array.from(stdout.trim().split(':')).reverse();
+ const duration = (time.length > 2 ? Number(time[2]) * 1000 * 60 : 0) + (time.length > 1 ? Number(time[1]) * 60 : 0) + (time.length > 0 ? Number(time[0]) : 0);
+ const data = { size: 0, path, name, type: 'video/mp4' };
+ const file = { ...data, toJSON: () => ({ ...data, filename: data.path.replace(/.*\//, ''), mtime: duration.toString(), mime: '', toJson: () => undefined as any }) };
+ res(MoveParsedFile(file, Directory.videos));
+ });
+ }
+ });
+ }
});
}
@@ -144,6 +169,7 @@ export namespace DashUploadUtils {
const result = await UploadImage(path, basename(path));
return { source: file, result };
}
+ fs.unlink(path, () => {});
return { source: file, result: { name: 'Unsupported image format', message: `Could not upload unsupported file (${name}). Please convert to an .jpg` } };
case 'video':
if (format.includes('x-matroska')) {
@@ -167,21 +193,25 @@ export namespace DashUploadUtils {
res();
})
);
- if (abort) return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } };
- // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server.
- // await new Promise(res =>
- // ffmpeg(file.path)
- // .videoCodec('libx264') // this will copy the data instead of reencode it
- // .audioCodec('mp2')
- // .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4'))
- // .on('end', res)
- // );
- // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4');
- // format = '.mp4';
+ if (abort) {
+ // bcz: instead of aborting, we could convert the file using the code below to an mp4. Problem is that this takes a long time and will clog up the server.
+ // await new Promise(res =>
+ // ffmpeg(file.path)
+ // .videoCodec('libx264') // this will copy the data instead of reencode it
+ // .audioCodec('mp2')
+ // .save(file.path.replace('.MOV', '.mp4').replace('.mov', '.mp4'))
+ // .on('end', res)
+ // );
+ // file.path = file.path.replace('.mov', '.mp4').replace('.MOV', '.mp4');
+ // format = '.mp4';
+ fs.unlink(path, () => {});
+ return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } };
+ }
}
if (videoFormats.includes(format)) {
return MoveParsedFile(file, Directory.videos);
}
+ fs.unlink(path, () => {});
return { source: file, result: { name: 'Unsupported video format', message: `Could not upload unsupported file (${name}). Please convert to an .mp4` } };
case 'application':
if (applicationFormats.includes(format)) {
@@ -195,6 +225,7 @@ export namespace DashUploadUtils {
if (audioFormats.includes(format)) {
return UploadAudio(file, format);
}
+ fs.unlink(path, () => {});
return { source: file, result: { name: 'Unsupported audio format', message: `Could not upload unsupported file (${name}). Please convert to an .mp3` } };
case 'text':
if (types[1] == 'csv') {
@@ -203,20 +234,28 @@ export namespace DashUploadUtils {
}
console.log(red(`Ignoring unsupported file (${name}) with upload type (${type}).`));
+ fs.unlink(path, () => {});
return { source: file, result: new Error(`Could not upload unsupported file (${name}) with upload type (${type}).`) };
}
async function UploadPdf(file: File) {
- const { path: sourcePath } = file;
- const dataBuffer = readFileSync(sourcePath);
+ const fileKey = (await md5File(file.path)) + '.pdf';
+ const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
+ if (fExists(fileKey, Directory.pdfs) && fExists(textFilename, Directory.text)) {
+ return new Promise<Upload.FileResponse>(res => {
+ const textFilename = `${fileKey.substring(0, fileKey.length - 4)}.txt`;
+ const readStream = createReadStream(serverPathToFile(Directory.text, textFilename));
+ var rawText = '';
+ readStream.on('data', chunk => (rawText += chunk.toString())).on('end', () => res(resolveExistingFile(file.name, fileKey, Directory.pdfs, file.type, undefined, rawText)));
+ });
+ }
+ const dataBuffer = readFileSync(file.path);
const result: ParsedPDF = await parse(dataBuffer);
await new Promise<void>((resolve, reject) => {
- const name = path.basename(sourcePath);
- const textFilename = `${name.substring(0, name.length - 4)}.txt`;
const writeStream = createWriteStream(serverPathToFile(Directory.text, textFilename));
writeStream.write(result.text, error => (error ? reject(error) : resolve()));
});
- return MoveParsedFile(file, Directory.pdfs, undefined, result.text);
+ return MoveParsedFile(file, Directory.pdfs, undefined, result.text, undefined, fileKey);
}
async function UploadCsv(file: File) {
@@ -368,9 +407,9 @@ export namespace DashUploadUtils {
* @param suffix If the file doesn't have a suffix and you want to provide it one
* to appear in the new location
*/
- export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number): Promise<Upload.FileResponse> {
+ export async function MoveParsedFile(file: formidable.File, destination: Directory, suffix: string | undefined = undefined, text?: string, duration?: number, targetName?: string): Promise<Upload.FileResponse> {
const { path: sourcePath } = file;
- let name = path.basename(sourcePath);
+ let name = targetName ?? path.basename(sourcePath);
suffix && (name += suffix);
return new Promise(resolve => {
const destinationPath = serverPathToFile(destination, name);
@@ -391,6 +430,11 @@ export namespace DashUploadUtils {
});
}
+ export function fExists(name: string, destination: Directory) {
+ const destinationPath = serverPathToFile(destination, name);
+ return existsSync(destinationPath);
+ }
+
export function getAccessPaths(directory: Directory, fileName: string) {
return {
client: clientPathToFile(directory, fileName),
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-10-15 08:59:02 +0000