From f83e5d34794ef675d4627ecef2ed7042b17b1b06 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:05:01 -0400 Subject: cleaning up zip/unzip of files --- src/server/ApiManagers/UploadManager.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src/server') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 6e28268a9..2cc2ac46c 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -184,9 +184,8 @@ export default class UploadManager extends ApiManager { if (id.endsWith('Proto')) return id; if (id in ids) { return ids[id]; - } else { - return (ids[id] = v4()); } + return (ids[id] = v4()); }; const mapFn = (doc: any) => { if (doc.id) { @@ -266,7 +265,7 @@ export default class UploadManager extends ApiManager { await Promise.all( docs.map( (doc: any) => - new Promise(res => { + new Promise(res => Database.Instance.replace( doc.id, doc, @@ -275,8 +274,8 @@ export default class UploadManager extends ApiManager { res(); }, true - ); - }) + ) + ) ) ); } catch (e) { -- cgit v1.2.3-70-g09d2 From 237e39029201aba23249de1ab5c405197acffe5a Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:50:28 -0400 Subject: more cleanup of import/export --- src/fields/Doc.ts | 15 +++++++++------ src/server/ApiManagers/UploadManager.ts | 34 +++++++++++++++------------------ 2 files changed, 24 insertions(+), 25 deletions(-) (limited to 'src/server') diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index c5af45262..35f9a1032 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -822,7 +822,6 @@ export namespace Doc { export async function Zip(doc: Doc, zipFilename = 'dashExport.zip') { const { clone, map, linkMap } = await Doc.MakeClone(doc); - clone.LINKS = new List(Array.from(linkMap.values())); const proms = [] as string[]; function replacer(key: any, value: any) { if (key && ['branchOf', 'cloneOf', 'cursors'].includes(key)) return undefined; @@ -856,8 +855,10 @@ export namespace Doc { } const docs: { [id: string]: any } = {}; + const links: { [id: string]: any } = {}; Array.from(map.entries()).forEach(f => (docs[f[0]] = f[1])); - const jsonDocs = JSON.stringify({ id: clone[Id], docs }, decycle(replacer)); + Array.from(linkMap.entries()).forEach(l => (links[l[0]] = l[1])); + const jsonDocs = JSON.stringify({ id: clone[Id], docs, links }, decycle(replacer)); const zip = new JSZip(); var count = 0; @@ -873,7 +874,7 @@ export namespace Doc { const assetPathOnServer = proms[i].replace(window.location.origin + '/', '').replace(/\//g, '%%%'); zip.file(assetPathOnServer, data, { binary: true }); if (++count == proms.length) { - zip.file('doc.json', jsonDocs); + zip.file('docs.json', jsonDocs); zip.generateAsync({ type: 'blob' }).then(content => saveAs(content, zipFilename)); // const a = document.createElement("a"); // const url = Utils.prepend(`/downloadId/${this.props.Document[Id]}`); @@ -1543,10 +1544,12 @@ export namespace Doc { const response = await fetch(upload, { method: 'POST', body: formData }); const json = await response.json(); if (json !== 'error') { - await DocServer.GetRefFields(json.docids as string[]); + const docs = await DocServer.GetRefFields(json.docids as string[]); const doc = DocCast(await DocServer.GetRefField(json.id)); - (await DocListCastAsync(doc?.LINKS))?.forEach(link => LinkManager.Instance.addLink(link)); - doc.LINKS = undefined; + const links = await DocServer.GetRefFields(json.linkids as string[]); + Array.from(Object.keys(links)) + .map(key => links[key]) + .forEach(link => link instanceof Doc && LinkManager.Instance.addLink(link)); return doc; } } diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 2cc2ac46c..1f72d72dc 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -228,6 +228,7 @@ export default class UploadManager extends ApiManager { remap = fields.remap !== 'false'; let id: string = ''; let docids: string[] = []; + let linkids: string[] = []; try { for (const name in files) { const f = files[name]; @@ -254,28 +255,23 @@ export default class UploadManager extends ApiManager { console.log(e); } }); - const json = zip.getEntry('doc.json'); + const json = zip.getEntry('docs.json'); try { const data = JSON.parse(json.getData().toString('utf8'), retrocycle()); - const datadocs = data.docs; + const { docs, links } = data; id = getId(data.id); - const docs = Object.keys(datadocs).map(key => datadocs[key]); - docs.forEach(mapFn); - docids = docs.map(doc => doc.id); + const rdocs = Object.keys(docs).map(key => docs[key]); + const ldocs = Object.keys(links).map(key => links[key]); + [...rdocs, ...ldocs].forEach(mapFn); + docids = rdocs.map(doc => doc.id); + linkids = ldocs.map(link => link.id); await Promise.all( - docs.map( - (doc: any) => - new Promise(res => - Database.Instance.replace( - doc.id, - doc, - (err, r) => { - err && console.log(err); - res(); - }, - true - ) - ) + [...rdocs, ...ldocs].map( + doc => + new Promise(res => { + // overwrite mongo doc with json doc contents + Database.Instance.replace(doc.id, doc, (err, r) => res(err && console.log(err)), true); + }) ) ); } catch (e) { @@ -284,7 +280,7 @@ export default class UploadManager extends ApiManager { unlink(path_2, () => {}); } SolrManager.update(); - res.send(JSON.stringify({ id, docids } || 'error')); + res.send(JSON.stringify({ id, docids, linkids } || 'error')); } catch (e) { console.log(e); } -- cgit v1.2.3-70-g09d2 From 16a33f53c2a031f791ed8c98d539409e1a7c6bb4 Mon Sep 17 00:00:00 2001 From: bobzel Date: Tue, 18 Apr 2023 10:52:43 -0400 Subject: from last --- src/server/ApiManagers/UploadManager.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'src/server') diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts index 1f72d72dc..74c06b4a6 100644 --- a/src/server/ApiManagers/UploadManager.ts +++ b/src/server/ApiManagers/UploadManager.ts @@ -180,11 +180,8 @@ export default class UploadManager extends ApiManager { const ids: { [id: string]: string } = {}; let remap = true; const getId = (id: string): string => { - if (!remap) return id; - if (id.endsWith('Proto')) return id; - if (id in ids) { - return ids[id]; - } + if (!remap || id.endsWith('Proto')) return id; + if (id in ids) return ids[id]; return (ids[id] = v4()); }; const mapFn = (doc: any) => { -- cgit v1.2.3-70-g09d2 From 2a584d4827c9ece87f3bd618201f356237ba7fc7 Mon Sep 17 00:00:00 2001 From: bobzel Date: Mon, 1 May 2023 12:49:06 -0400 Subject: fixed RecordingBox/View serialization to save docid, not Doc. --- src/client/util/ReplayMovements.ts | 27 +++--- src/client/util/TrackMovements.ts | 2 +- .../views/nodes/RecordingBox/RecordingBox.tsx | 106 +++++++++++---------- .../views/nodes/RecordingBox/RecordingView.tsx | 10 +- src/fields/Doc.ts | 5 +- src/server/DashUploadUtils.ts | 14 ++- src/server/websocket.ts | 23 +++-- 7 files changed, 102 insertions(+), 85 deletions(-) (limited to 'src/server') diff --git a/src/client/util/ReplayMovements.ts b/src/client/util/ReplayMovements.ts index 40261985a..22cca4a2e 100644 --- a/src/client/util/ReplayMovements.ts +++ b/src/client/util/ReplayMovements.ts @@ -1,13 +1,11 @@ +import { IReactionDisposer, observable, reaction } from 'mobx'; +import { Doc, IdToDoc } from '../../fields/Doc'; +import { CollectionDockingView } from '../views/collections/CollectionDockingView'; import { CollectionFreeFormView } from '../views/collections/collectionFreeForm'; -import { IReactionDisposer, observable, observe, reaction } from 'mobx'; -import { Doc } from '../../fields/Doc'; +import { OpenWhereMod } from '../views/nodes/DocumentView'; import { VideoBox } from '../views/nodes/VideoBox'; import { DocumentManager } from './DocumentManager'; -import { CollectionDockingView } from '../views/collections/CollectionDockingView'; -import { DocServer } from '../DocServer'; import { Movement, Presentation } from './TrackMovements'; -import { OpenWhereMod } from '../views/nodes/DocumentView'; -import { returnTransparent } from '../../Utils'; export class ReplayMovements { private timers: NodeJS.Timeout[] | null; @@ -61,7 +59,7 @@ export class ReplayMovements { return; } - const docIdtoDoc = this.loadPresentation(presentation); + this.loadPresentation(presentation); this.videoBoxDisposeFunc = reaction( () => ({ playing: videoBox._playing, timeViewed: videoBox.player?.currentTime || 0 }), @@ -94,13 +92,14 @@ export class ReplayMovements { throw '[recordingApi.ts] followMovements() failed: no presentation data'; } - // generate a set of all unique docIds - const docs = new Set(); - for (const { doc } of movements) { - if (!docs.has(doc)) docs.add(doc); - } - - return docs; + movements.forEach((movement, i) => { + if (typeof movement.doc === 'string') { + movements[i].doc = IdToDoc(movement.doc); + if (!movements[i].doc) { + console.log('ERROR: tracked doc not found'); + } + } + }); }; // returns undefined if the docView isn't open on the screen diff --git a/src/client/util/TrackMovements.ts b/src/client/util/TrackMovements.ts index 2f16307b3..cb8225643 100644 --- a/src/client/util/TrackMovements.ts +++ b/src/client/util/TrackMovements.ts @@ -234,7 +234,7 @@ export class TrackMovements { const movement: Movement = { time, panX, panY, scale, doc }; // add that movement to the current presentation data's movement array - this.currentPresentation.movements && this.currentPresentation.movements.push(movement); + this.currentPresentation.movements?.push(movement); }; // method that concatenates an array of presentatations into one diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx index 0ff7c4292..f406ffbea 100644 --- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx @@ -1,58 +1,64 @@ -import { action, observable } from "mobx"; -import { observer } from "mobx-react"; -import * as React from "react"; -import { VideoField } from "../../../../fields/URLField"; -import { Upload } from "../../../../server/SharedMediaTypes"; -import { ViewBoxBaseComponent } from "../../DocComponent"; -import { FieldView } from "../FieldView"; -import { VideoBox } from "../VideoBox"; +import { action, observable } from 'mobx'; +import { observer } from 'mobx-react'; +import * as React from 'react'; +import { VideoField } from '../../../../fields/URLField'; +import { Upload } from '../../../../server/SharedMediaTypes'; +import { ViewBoxBaseComponent } from '../../DocComponent'; +import { FieldView } from '../FieldView'; +import { VideoBox } from '../VideoBox'; import { RecordingView } from './RecordingView'; -import { DocumentType } from "../../../documents/DocumentTypes"; -import { Presentation } from "../../../util/TrackMovements"; -import { Doc } from "../../../../fields/Doc"; -import { Id } from "../../../../fields/FieldSymbols"; - +import { DocumentType } from '../../../documents/DocumentTypes'; +import { Presentation } from '../../../util/TrackMovements'; +import { Doc } from '../../../../fields/Doc'; +import { Id } from '../../../../fields/FieldSymbols'; @observer export class RecordingBox extends ViewBoxBaseComponent() { + public static LayoutString(fieldKey: string) { + return FieldView.LayoutString(RecordingBox, fieldKey); + } - public static LayoutString(fieldKey: string) { return FieldView.LayoutString(RecordingBox, fieldKey); } - - private _ref: React.RefObject = React.createRef(); + private _ref: React.RefObject = React.createRef(); - constructor(props: any) { + constructor(props: any) { super(props); - } - - componentDidMount() { - Doc.SetNativeWidth(this.dataDoc, 1280); - Doc.SetNativeHeight(this.dataDoc, 720); - } - - @observable result: Upload.AccessPathInfo | undefined = undefined - @observable videoDuration: number | undefined = undefined - - @action - setVideoDuration = (duration: number) => { - this.videoDuration = duration - } - - @action - setResult = (info: Upload.AccessPathInfo, presentation?: Presentation) => { - this.result = info - this.dataDoc.type = DocumentType.VID; - this.dataDoc[this.fieldKey + "-duration"] = this.videoDuration; - - this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey); - this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client); - this.dataDoc[this.fieldKey + "-recorded"] = true; - // stringify the presentation and store it - presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation)); - } - - render() { - return
- {!this.result && } -
; - } + } + + componentDidMount() { + Doc.SetNativeWidth(this.dataDoc, 1280); + Doc.SetNativeHeight(this.dataDoc, 720); + } + + @observable result: Upload.AccessPathInfo | undefined = undefined; + @observable videoDuration: number | undefined = undefined; + + @action + setVideoDuration = (duration: number) => { + this.videoDuration = duration; + }; + + @action + setResult = (info: Upload.AccessPathInfo, presentation?: Presentation) => { + this.result = info; + this.dataDoc.type = DocumentType.VID; + this.dataDoc[this.fieldKey + '-duration'] = this.videoDuration; + + this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey); + this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client); + this.dataDoc[this.fieldKey + '-recorded'] = true; + // stringify the presentation and store it + if (presentation?.movements) { + const presCopy = { ...presentation }; + presCopy.movements = presentation.movements.map(movement => ({ ...movement, doc: movement.doc[Id] })) as any; + this.dataDoc[this.fieldKey + '-presentation'] = JSON.stringify(presCopy); + } + }; + + render() { + return ( +
+ {!this.result && } +
+ ); + } } diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx index 6efe62e0b..424ebc384 100644 --- a/src/client/views/nodes/RecordingBox/RecordingView.tsx +++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx @@ -1,14 +1,14 @@ import * as React from 'react'; -import './RecordingView.scss'; import { useEffect, useRef, useState } from 'react'; -import { ProgressBar } from './ProgressBar'; -import { MdBackspace } from 'react-icons/md'; -import { FaCheckCircle } from 'react-icons/fa'; import { IconContext } from 'react-icons'; -import { Networking } from '../../../Network'; +import { FaCheckCircle } from 'react-icons/fa'; +import { MdBackspace } from 'react-icons/md'; import { Upload } from '../../../../server/SharedMediaTypes'; import { returnFalse, returnTrue, setupMoveUpEvents } from '../../../../Utils'; +import { Networking } from '../../../Network'; import { Presentation, TrackMovements } from '../../../util/TrackMovements'; +import { ProgressBar } from './ProgressBar'; +import './RecordingView.scss'; export interface MediaSegment { videoChunks: any[]; diff --git a/src/fields/Doc.ts b/src/fields/Doc.ts index 93c28cf08..0ec881c48 100644 --- a/src/fields/Doc.ts +++ b/src/fields/Doc.ts @@ -1703,8 +1703,11 @@ export namespace Doc { } } +export function IdToDoc(id: string) { + return DocCast(DocServer.GetCachedRefField(id)); +} ScriptingGlobals.add(function idToDoc(id: string): any { - return DocServer.GetCachedRefField(id); + return IdToDoc(id); }); ScriptingGlobals.add(function renameAlias(doc: any) { return StrCast(Doc.GetProto(doc).title).replace(/\([0-9]*\)/, '') + `(${doc.aliasNumber})`; diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts index f461cf3fa..070d49ec3 100644 --- a/src/server/DashUploadUtils.ts +++ b/src/server/DashUploadUtils.ts @@ -70,7 +70,14 @@ export namespace DashUploadUtils { // make a list of paths to create the ordered text file for ffmpeg const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n'); // write the text file to the file system - writeFile(textFilePath, filePathsText, err => console.log(err)); + await new Promise((res, reject) => + writeFile(textFilePath, filePathsText, err => { + if (err) { + reject(); + console.log(err); + } else res(); + }) + ); // make output file name based on timestamp const outputFileName = `output-${Utils.GenerateGuid()}.mp4`; @@ -86,7 +93,10 @@ export namespace DashUploadUtils { .outputOptions('-c copy') //.videoCodec("copy") .save(outputFilePath) - .on('error', reject) + .on('error', (err: any) => { + console.log(err); + reject(); + }) .on('end', resolve); }); diff --git a/src/server/websocket.ts b/src/server/websocket.ts index a11d20cfa..2acdaa5a3 100644 --- a/src/server/websocket.ts +++ b/src/server/websocket.ts @@ -51,7 +51,7 @@ export namespace WebSocket { next(); }); - socket.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()) + socket.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()); // convenience function to log server messages on the client function log(message?: any, ...optionalParams: any[]) { @@ -104,14 +104,12 @@ export namespace WebSocket { socket.on('disconnect', function () { let currentUser = socketMap.get(socket); if (!(currentUser === undefined)) { - let currentUsername = currentUser.split(' ')[0] + let currentUsername = currentUser.split(' ')[0]; DashStats.logUserLogout(currentUsername, socket); - delete timeMap[currentUsername] + delete timeMap[currentUsername]; } }); - - Utils.Emit(socket, MessageStore.Foo, 'handshooken'); Utils.AddServerHandler(socket, MessageStore.Bar, guid => barReceived(socket, guid)); @@ -146,10 +144,10 @@ export namespace WebSocket { }; }); - setInterval(function() { + setInterval(function () { // Utils.Emit(socket, MessageStore.UpdateStats, DashStats.getUpdatedStatsBundle()); - io.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()) + io.emit(MessageStore.UpdateStats.Message, DashStats.getUpdatedStatsBundle()); }, DashStats.SAMPLING_INTERVAL); } @@ -193,7 +191,7 @@ export namespace WebSocket { } function barReceived(socket: SocketIO.Socket, userEmail: string) { - clients[userEmail] = new Client(userEmail.toString()); + clients[userEmail] = new Client(userEmail.toString()); const currentdate = new Date(); const datetime = currentdate.getDate() + '/' + (currentdate.getMonth() + 1) + '/' + currentdate.getFullYear() + ' @ ' + currentdate.getHours() + ':' + currentdate.getMinutes() + ':' + currentdate.getSeconds(); console.log(blue(`user ${userEmail} has connected to the web socket at: ${datetime}`)); @@ -326,8 +324,7 @@ export namespace WebSocket { const remListItems = diff.diff.$set[updatefield].fields; const curList = (curListItems as any)?.fields?.[updatefield.replace('fields.', '')]?.fields.filter((f: any) => f !== null) || []; diff.diff.$set[updatefield].fields = curList?.filter( - (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId : - remItem.heading ? remItem.heading === curItem.heading : remItem === curItem)) + (curItem: any) => !remListItems.some((remItem: any) => (remItem.fieldId ? remItem.fieldId === curItem.fieldId : remItem.heading ? remItem.heading === curItem.heading : remItem === curItem)) ); const sendBack = diff.diff.length !== diff.diff.$set[updatefield].fields.length; delete diff.diff.length; @@ -373,9 +370,11 @@ export namespace WebSocket { var CurUser: string | undefined = undefined; function UpdateField(socket: Socket, diff: Diff) { - let currentUsername = socketMap.get(socket)!.split(' ')[0]; + const curUser = socketMap.get(socket); + if (!curUser) return; + let currentUsername = curUser.split(' ')[0]; userOperations.set(currentUsername, userOperations.get(currentUsername) !== undefined ? userOperations.get(currentUsername)! + 1 : 0); - + if (CurUser !== socketMap.get(socket)) { CurUser = socketMap.get(socket); console.log('Switch User: ' + CurUser); -- cgit v1.2.3-70-g09d2