From c656abad84ac56c3067a48ca6d34d567e3fcdd88 Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@Michaels-MacBook-Pro-5.local>
Date: Wed, 8 Jun 2022 00:05:34 -0400
Subject: Took stab at ffmpeg in server. did some reworking of recordingview
 css. fix smoothness bug for the moving segment that follow cursor.

---
 src/client/views/nodes/RecordingBox/RecordingBox.tsx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx')

diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
index 159271223..68f3b3ad4 100644
--- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
@@ -54,9 +54,9 @@ export class RecordingBox extends ViewBoxBaseComponent() {
       }
 
       render() {
-            // console.log("Proto[Is]: ", this.rootDoc.proto?.[Id])
+            console.log("Proto[Is]: ", this.rootDoc.proto?.[Id])
             return <div className="recordingBox" ref={this._ref}>
-                  {!this.result && <RecordingView setResult={this.setResult} setDuration={this.setVideoDuration} id={this.rootDoc.proto?.[Id]} />}
+                  {!this.result && <RecordingView setResult={this.setResult} setDuration={this.setVideoDuration} id={this.rootDoc.proto?.[Id] || ''} />}
             </div>;
       }
 }
-- 
cgit v1.2.3-70-g09d2


From 48c60bd982734676f972514f7074be6121e7c5df Mon Sep 17 00:00:00 2001
From: Michael <michael.foiani@gmail.com>
Date: Wed, 8 Jun 2022 18:24:53 -0400
Subject: big commit. FINALLY got the combining segments to work using ffmpeg
 using a different workflow. small ui changes as well.

---
 src/client/Network.ts                              |  15 +--
 .../views/nodes/RecordingBox/ProgressBar.tsx       |  20 +++-
 .../views/nodes/RecordingBox/RecordingBox.tsx      |   7 +-
 .../views/nodes/RecordingBox/RecordingView.tsx     | 126 ++++-----------------
 src/server/ApiManagers/UploadManager.ts            |  23 +---
 src/server/DashUploadUtils.ts                      |  59 +++-------
 6 files changed, 60 insertions(+), 190 deletions(-)

(limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx')

diff --git a/src/client/Network.ts b/src/client/Network.ts
index 8c1f31488..b26f2458d 100644
--- a/src/client/Network.ts
+++ b/src/client/Network.ts
@@ -35,20 +35,7 @@ export namespace Networking {
             const response = await fetch("/uploadFormData", parameters);
             return response.json();
       }
-    
-    export async function UploadSegmentsAndConcatenate<T extends Upload.FileInformation = Upload.FileInformation>(files: File[]): Promise<Upload.FileResponse<T>[]> {
-            console.log("network.ts : uploading segments and concatenating", files);
-            const formData = new FormData();
-            if (!Array.isArray(files) || !files.length) return [];
-            files.forEach(file => formData.append(Utils.GenerateGuid(), file));
-            const parameters = {
-                  method: 'POST',
-                  body: formData
-            };
-            const response = await fetch("/uploadVideosandConcatenate", parameters);
-            return response.json();
-      }
-
+  
       export async function UploadYoutubeToServer<T extends Upload.FileInformation = Upload.FileInformation>(videoId: string): Promise<Upload.FileResponse<T>[]> {
             const parameters = {
                   method: 'POST',
diff --git a/src/client/views/nodes/RecordingBox/ProgressBar.tsx b/src/client/views/nodes/RecordingBox/ProgressBar.tsx
index a91656cbc..effc3d8a8 100644
--- a/src/client/views/nodes/RecordingBox/ProgressBar.tsx
+++ b/src/client/views/nodes/RecordingBox/ProgressBar.tsx
@@ -53,11 +53,6 @@ export function ProgressBar(props: ProgressBarProps) {
         return [...prevOrdered, { endTime, startTime , order }];
       });
     }
-    // }props.videos.map((vid, order) => {
-    // //const { endTime, startTime } = vid
-    //   // TODO: not tranfer the blobs around
-    //   return { ...vid, order };
-    // }))
   }, [props.videos]);
 
   useEffect(() => {
@@ -96,7 +91,6 @@ export function ProgressBar(props: ProgressBarProps) {
   }
   
   const onPointerDown = (e: React.PointerEvent<HTMLDivElement>) => {
-    console.log('pointer down')
     // don't move the videobox element
     e.stopPropagation()
 
@@ -107,6 +101,20 @@ export function ProgressBar(props: ProgressBarProps) {
     // don't do anything if null
     const progressBar = progressBarRef.current
     if (progressBar == null || clickedSegment.id === progressBar.id) return
+
+    console.log('pointer down')
+    // if holding shift key, let's remove that segment
+    if (e.shiftKey) {
+      // TODO: fix this
+      const segId = parseInt(clickedSegment.id.split('-')[1])
+      const o = ordered[segId].order
+      setOrdered(prevOrdered => {
+        return prevOrdered.filter((seg, i) => i !== segId)
+      })
+
+      return
+    }
+
     const ptrId = e.pointerId;
     progressBar.setPointerCapture(ptrId)
 
diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
index 68f3b3ad4..bd4af0a75 100644
--- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
@@ -30,7 +30,7 @@ export class RecordingBox extends ViewBoxBaseComponent() {
             Doc.SetNativeHeight(this.dataDoc, 720);
       }
 
-      @observable result: Upload.FileInformation | undefined = undefined
+      @observable result: Upload.AccessPathInfo | undefined = undefined
       @observable videoDuration: number | undefined = undefined
 
       @action
@@ -39,13 +39,14 @@ export class RecordingBox extends ViewBoxBaseComponent() {
       }
 
       @action
-      setResult = (info: Upload.FileInformation, trackScreen: boolean) => {
+      setResult = (info: Upload.AccessPathInfo, trackScreen: boolean) => {
+        if (info == null) return
             this.result = info
             this.dataDoc.type = DocumentType.VID;
             this.dataDoc[this.fieldKey + "-duration"] = this.videoDuration;
 
             this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey);
-            this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.agnostic.client);
+            this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client);
             this.dataDoc[this.fieldKey + "-recorded"] = true;
             // stringify the presenation and store it
             if (trackScreen) {
diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx
index a5c2dc85c..8c8728fc3 100644
--- a/src/client/views/nodes/RecordingBox/RecordingView.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx
@@ -18,7 +18,7 @@ export interface MediaSegment {
 }
 
 interface IRecordingViewProps {
-      setResult: (info: Upload.FileInformation, trackScreen: boolean) => void
+      setResult: (info: Upload.AccessPathInfo, trackScreen: boolean) => void
       setDuration: (seconds: number) => void
       id: string
 }
@@ -57,7 +57,7 @@ export function RecordingView(props: IRecordingViewProps) {
 
     useEffect(() => {
           
-        console.log('in videos useEffect')
+        // console.log('in videos useEffect', finished)
 
           if (finished) {
               (async () => {
@@ -71,89 +71,18 @@ export function RecordingView(props: IRecordingViewProps) {
                     const { name } = videoFile;
                     inputPaths.push(name)
                   })
-                
-                  console.log(inputPaths)
                   
-                  const data = await Networking.UploadSegmentsAndConcatenate(videoFiles)
-                  console.log('data', data)
-                  const result = data[0].result
-                  if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox
-                      props.setResult(result, trackScreen)
-                  } else {
-                      alert("video conversion failed");
-                  }
-                
-                
+                const serverPaths: string[] = (await Networking.UploadFilesToServer(videoFiles))
+                  .map(res => (res.result instanceof Error) ? '' : res.result.accessPaths.agnostic.server)
                 
-                // const inputListName = 'order.txt';
-                // fs.writeFileSync(inputListName, inputPaths.join('\n'));
-                // var merge = ffmpeg();
-                // merge.input(inputListName)
-                //   .inputOptions(['-f concat', '-safe 0'])
-                //   .outputOptions('-c copy')
-                //   .save('output.mp4')
-                
-                // fs.unlinkSync(inputListName);
-
-                // const combined = await DashUploadUtils.combineSegments(videoFiles, inputPaths)
-                // console.log('combined', combined)
-                
-                // const outputFile = new File(['output.mp4'], 'output.mp4', { type: 'video/mp4', lastModified: Date.now() });
-
-                // const data = await Networking.UploadFilesToServer(combined)
-                //   const result = data[0].result
-                //   if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox
-                //       props.setResult(result, trackScreen)
-                //   } else {
-                //       alert("video conversion failed");
-                //   }
-                  
-
-                //   if (format.includes("x-matroska")) {
-                //     await new Promise(res => ffmpeg(file.path)
-                //         .videoCodec("copy") // this will copy the data instead of reencode it
-                //         .save(file.path.replace(".mkv", ".mp4"))
-                //         .on('end', res));
-                //     file.path = file.path.replace(".mkv", ".mp4");
-                //     format = ".mp4";
-                // }
-                //   console.log('crossOriginIsolated', crossOriginIsolated)
-                //   props.setDuration(recordingTimer * 100)
-
-                //   console.log('Loading ffmpeg-core.js');
-                //   const ffmpeg = createFFmpeg({ log: true });
-                //   await ffmpeg.load();
-                //   console.log('ffmpeg-core.js loaded');
-                  
-                //   let allVideoChunks: any = [];
-                //   const inputPaths: string[] = [];
-                //   // write each segment into it's indexed file
-                //   videos.forEach(async (vid, i) => {
-                //       const vidName = `segvideo${i}.mkv`
-                //       inputPaths.push(vidName)
-                //       const videoFile = new File(vid.videoChunks, vidName, { type: allVideoChunks[0].type, lastModified: Date.now() });
-                //       ffmpeg.FS('writeFile', vidName, await fetchFile(videoFile));
-                // //   })
-                //   ffmpeg.FS('writeFile', 'order.txt', inputPaths.join('\n'));
-                  
-                //   console.log('concat')
-                //   await ffmpeg.run('-f', 'concat', '-safe', '0', '-i', 'order.txt', 'ouput.mp4');
-
-                //   const { buffer } = ffmpeg.FS('readFile', 'output.mp4');
-                //   const concatVideo = new File([buffer], 'concat.mp4', { type: "video/mp4" });
-
-                //   const data = await Networking.UploadFilesToServer(concatVideo)
-                //   const result = data[0].result
-                //   if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox
-                //       props.setResult(result, trackScreen)
-                //   } else {
-                //       alert("video conversion failed");
-                //   }
-                  
-                //   // delete all files in MEMFS
-                //   inputPaths.forEach(path => ffmpeg.FS('unlink', path));
-                //   ffmpeg.FS('unlink', 'order.txt');
-                //   ffmpeg.FS('unlink', 'output.mp4');
+                const result: Upload.AccessPathInfo | Error = await Networking.PostToServer('/concatVideos', serverPaths)
+                console.log(result)
+                if (!(result instanceof Error)) { // convert this screenshotBox into normal videoBox
+                    props.setResult(result, trackScreen)
+                } else {
+                    alert("video conversion failed");
+                }
+              
               })();
               
             }
@@ -162,8 +91,6 @@ export function RecordingView(props: IRecordingViewProps) {
     }, [videos])
   
   useEffect(() => {
-          
-        console.log('in finish useEffect')
 
     if (finished) {
       setOrderVideos(true);
@@ -177,7 +104,7 @@ export function RecordingView(props: IRecordingViewProps) {
             if (!navigator.mediaDevices) {
                   console.log('This browser does not support getUserMedia.')
             }
-          console.log('This device has the correct media devices.')
+          // console.log('This device has the correct media devices.')
       }, [])
 
       useEffect(() => {
@@ -244,7 +171,6 @@ export function RecordingView(props: IRecordingViewProps) {
                   // reset the temporary chunks
                   videoChunks = []
                   setRecording(false);
-                  setFinished(true);
                   trackScreen && RecordingApi.Instance.pause();
             }
 
@@ -269,8 +195,10 @@ export function RecordingView(props: IRecordingViewProps) {
       }
 
 
-      const stop = () => {
-            if (videoRecorder.current) {
+  const stop = (e: React.MouseEvent) => {
+        e.stopPropagation()
+        if (videoRecorder.current) {
+                                      setFinished(true);
                   if (videoRecorder.current.state !== "inactive") {
                         videoRecorder.current.stop();
                         // recorder.current.stream.getTracks().forEach((track: any) => track.stop())
@@ -278,19 +206,21 @@ export function RecordingView(props: IRecordingViewProps) {
             }
       }
 
-      const pause = () => {
+      const pause = (e: React.MouseEvent) => {
+        e.stopPropagation()
             if (videoRecorder.current) {
                   if (videoRecorder.current.state === "recording") {
-                        videoRecorder.current.pause();
+                    videoRecorder.current.stop();
                   }
             }
       }
 
-      const startOrResume = () => {
+      const startOrResume =  (e: React.MouseEvent) => {
+        e.stopPropagation()
             if (!videoRecorder.current || videoRecorder.current.state === "inactive") {
                   record();
             } else if (videoRecorder.current.state === "paused") {
-                  videoRecorder.current.resume();
+              videoRecorder.current.start();
             }
       }
 
@@ -315,16 +245,6 @@ export function RecordingView(props: IRecordingViewProps) {
             const seconds = Math.floor((milliseconds % (1000 * 60)) / 1000);
             return toTwoDigit(minutes) + " : " + toTwoDigit(seconds);
       }
-    
-      const doTranscode = async () => {
-          
-        // console.log('Start transcoding');
-        // ffmpeg.FS('writeFile', 'test.avi', await fetchFile('/flame.avi'));
-        // await ffmpeg.run('-i', 'test.avi', 'test.mp4');
-        // console.log('Complete transcoding');
-        // const data = ffmpeg.FS('readFile', 'test.mp4');
-        // console.log(URL.createObjectURL(new Blob([data.buffer], { type: 'video/mp4' })));
-    };
 
       return (
             <div className="recording-container">
diff --git a/src/server/ApiManagers/UploadManager.ts b/src/server/ApiManagers/UploadManager.ts
index 398b007b5..0ee0a34df 100644
--- a/src/server/ApiManagers/UploadManager.ts
+++ b/src/server/ApiManagers/UploadManager.ts
@@ -44,25 +44,10 @@ export default class UploadManager extends ApiManager {
           
             register({
                   method: Method.POST,
-                  subscription: "/uploadVideosandConcatenate",
-                  secureHandler: async ({ req, res }) => {
-                        const form = new formidable.IncomingForm();
-                        form.keepExtensions = true;
-                        form.uploadDir = pathToDirectory(Directory.parsed_files);
-                        return new Promise<void>(resolve => {
-                              form.parse(req, async (_err, _fields, files) => {
-                                  const results: Upload.FileResponse[] = [];
-
-                                  // create an array of all the file paths
-                                  const filePaths: string[] = Object.keys(files).map(key => files[key].path);
-                                    console.log("uploading files", Array.isArray(filePaths));
-                                    const result = await DashUploadUtils.concatenateVideos(filePaths);
-                                    console.log('concatenated', result);
-                                    result && !(result.result instanceof Error) && results.push(result);
-                                    _success(res, results)
-                                    resolve();
-                              });
-                        });
+                  subscription: "/concatVideos",
+              secureHandler: async ({ req, res }) => {
+                    // req.body contains the array of server paths to the videos
+                    _success(res, await DashUploadUtils.concatVideos(req.body));
                   }
             });
 
diff --git a/src/server/DashUploadUtils.ts b/src/server/DashUploadUtils.ts
index 148b0df65..be30c115d 100644
--- a/src/server/DashUploadUtils.ts
+++ b/src/server/DashUploadUtils.ts
@@ -63,21 +63,20 @@ export namespace DashUploadUtils {
 
     const { imageFormats, videoFormats, applicationFormats, audioFormats } = AcceptableMedia; //TODO:glr 
     
-    export async function concatenateVideos(filePaths: string[]): Promise<Upload.FileResponse> {
+    export async function concatVideos(filePaths: string[]): Promise<Upload.AccessPathInfo> {
         // make a list of paths to create the ordered text file for ffmpeg
-        //const filePaths = videoFiles.map(file => file.path);
-        // write the text file to the file system
         const inputListName = 'concat.txt';
         const textFilePath = path.join(filesDirectory, inputListName);
         // make a list of paths to create the ordered text file for ffmpeg
-        const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n');
+      const filePathsText = filePaths.map(filePath => `file '${filePath}'`).join('\n');
+        // write the text file to the file system
         writeFile(textFilePath, filePathsText, (err) => console.log(err));
-        console.log(filePathsText)
+        console.log('fileTextPaths', filePathsText)
 
         // make output file name based on timestamp
-        const outputFileName = `output-${Utils.GenerateGuid()}.mkv`;
-        // create the output file path in the parsed_file directory
-        const outputFilePath = path.join(filesDirectory, outputFileName);
+        const outputFileName = `output-${Utils.GenerateGuid()}.mp4`;
+        // create the output file path in the videos directory
+        const outputFilePath = path.join(pathToDirectory(Directory.videos), outputFileName);
 
         // concatenate the videos
         await new Promise((resolve, reject) => {
@@ -92,45 +91,15 @@ export namespace DashUploadUtils {
                 .on("end", resolve);
         })
 
-        // delete concat.txt from the file system
-        unlinkSync(textFilePath);
-
-        // read the output file from the file system
-        // const outputFile = fs.readFileSync(outputFilePath);
-
-        // make a new blob object with the output file buffer
-        // const outputFileBlob = new Blob([outputFile.buffer], { type: 'x-matroska/mkv' });
-
-        // TODO: make with toJSON()
+      // delete concat.txt from the file system
+      unlinkSync(textFilePath);
+      // delete the old segment videos from the server
+      filePaths.forEach(filePath => unlinkSync(filePath));
 
-        // make a data object
-        const outputFileObject: formidable.File = {
-            size: 0,
-            name: outputFileName,
-            path: outputFilePath,
-            // size: outputFileBlob.size,
-            type: 'video/x-matroska;codecs=avc1,opus',
-            lastModifiedDate: new Date(),
-
-            toJSON: () => ({ ...outputFileObject, filename: outputFilePath.replace(/.*\//, ""), mtime: null, length: 0, mime: "", toJson: () => undefined as any })
+      // return the path(s) to the output file
+      return {
+          accessPaths: getAccessPaths(Directory.videos, outputFileName)
         }
-
-       // const file = { ...outputFileObject, toJSON: () => ({ ...outputFileObject, filename: outputFilePath.replace(/.*\//, ""), mtime: null, length: 0, mime: "", toJson: () => undefined as any }) };
-
-        // this will convert it to mp4 and save it to the server
-        //return await MoveParsedFile(outputFileObject, Directory.videos);
-
-        return await upload(outputFileObject);
-
-        // // return only the output (first) file to the videos directory
-        // return {
-        //     source: file, result: {
-        //         accessPaths: {
-        //             agnostic: getAccessPaths(Directory.videos, outputFileName)
-        //         },
-        //         rawText: undefined
-        //     }
-        // }
     }
 
     export function uploadYoutube(videoId: string): Promise<Upload.FileResponse> {
-- 
cgit v1.2.3-70-g09d2


From 1688af3de54419029773fb85b78bca4500f7f0de Mon Sep 17 00:00:00 2001
From: Michael <michael.foiani@gmail.com>
Date: Fri, 10 Jun 2022 15:41:38 -0400
Subject: big bug problems with recording. some reason interacting with the
 canvas is making the recording inactive - i think :/

---
 src/client/util/RecordingApi.ts                    | 153 +++++++++++++--------
 .../views/nodes/RecordingBox/ProgressBar.tsx       |   8 +-
 .../views/nodes/RecordingBox/RecordingBox.tsx      |  12 +-
 .../views/nodes/RecordingBox/RecordingView.tsx     |  99 ++++++++-----
 4 files changed, 166 insertions(+), 106 deletions(-)

(limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx')

diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts
index 021feee9a..ab6935e3b 100644
--- a/src/client/util/RecordingApi.ts
+++ b/src/client/util/RecordingApi.ts
@@ -3,8 +3,6 @@ import { IReactionDisposer, observable, reaction } from "mobx";
 import { NumCast } from "../../fields/Types";
 import { Doc } from "../../fields/Doc";
 import { VideoBox } from "../views/nodes/VideoBox";
-import { scaleDiverging } from "d3-scale";
-import { Transform } from "./Transform";
 
 type Movement = {
     time: number,
@@ -13,17 +11,17 @@ type Movement = {
     scale: number,
 }
 
-type Presentation = {
-    movements: Array<Movement> | null
-    meta: Object,
+export type Presentation = {
+  movements: Movement[] | null,
+  totalTime: number,
+  meta: any,
 }
 
 export class RecordingApi {
 
-    private static NULL_PRESENTATION: Presentation = {
-        movements: null,
-        meta: {},
-    }
+  private static get NULL_PRESENTATION(): Presentation {
+    return { movements: null, meta: {}, totalTime: -1,}
+  }
 
     // instance variables
     private currentPresentation: Presentation;
@@ -50,6 +48,8 @@ export class RecordingApi {
         // for now, set playFFView
         this.playFFView = null;
         this.timers = null;
+      
+      // put a pointerdown event on the doucment to see what the target
     }
 
     // little helper :)
@@ -59,8 +59,9 @@ export class RecordingApi {
 
     public start = (meta?: Object): Error | undefined => {
         // check if already init a presentation
-        if (!this.isInitPresenation) {
-            console.error('[recordingApi.ts] start() failed: current presentation data exists. please call clear() first.')
+      if (!this.isInitPresenation) {
+          console.log(this.currentPresentation)
+            console.trace('[recordingApi.ts] start() failed: current presentation data exists. please call clear() first.')
             return new Error('[recordingApi.ts] start()')
         }
 
@@ -80,50 +81,61 @@ export class RecordingApi {
         this.isRecording = true
     }
 
-    public clear = (): Error | Presentation => {
+  /* stops the video and returns the presentatation; if no presentation, returns undefined */  
+  public getPresentation = (): undefined | Presentation => {
         // TODO: maybe archive the data?
-        if (this.isRecording) {
-            console.error('[recordingApi.ts] clear() failed: currently recording presentation. call pause() first')
-            return new Error('[recordingApi.ts] clear()')
-        }
-
-        // update the presentation mode
-        Doc.UserDoc().presentationMode = 'none'
-        // set the previus recording view to the play view
-        this.playFFView = this.recordingFFView
-
-        const presCopy = { ...this.currentPresentation }
-
-        // clear presenation data
-        this.currentPresentation = RecordingApi.NULL_PRESENTATION
-        // clear isRecording
-        this.isRecording = false
-        // clear absoluteStart
-        this.absoluteStart = -1
-        // clear the disposeFunc
-        this.removeRecordingFFView()
-
-        return presCopy;
+    if (this.isRecording) console.warn('[recordingApi.ts] getPresentation() : currently recording presentation.');
+
+    // update the presentation mode
+    Doc.UserDoc().presentationMode = 'none';
+    // set the previus recording view to the play view
+    this.playFFView = this.recordingFFView;
+
+    // ensure we add the endTime now that they are done recording    
+    return { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
+  }
+
+  public stop = (): void => {
+    // make is recording false
+    this.isRecording = false
+  }
+
+  public clear = (): void => {
+    // clear presenation data
+    this.currentPresentation = RecordingApi.NULL_PRESENTATION
+    // clear isRecording
+    this.isRecording = false
+    // clear absoluteStart
+    this.absoluteStart = -1
+    // clear the disposeFunc
+    this.removeRecordingFFView()
+  }
+  
+  
+    // call on dispose function to stop tracking movements
+    public removeRecordingFFView = (): void => {
+        this.disposeFunc?.();
+        this.disposeFunc = null;
     }
 
-    public pause = (): Error | undefined => {
-        if (this.isInitPresenation) {
-            console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first')
-            return new Error('[recordingApi.ts] pause(): no presentation')
-        }
-        // don't allow track movments
-        this.isRecording = false
-
-        // set adjust absoluteStart to add the time difference
-        const timestamp = new Date().getTime()
-        this.absoluteStart = timestamp - this.absoluteStart
-    }
+    // public pause = (): Error | undefined => {
+    //     if (this.isInitPresenation) {
+    //         console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first')
+    //         return new Error('[recordingApi.ts] pause(): no presentation')
+    //     }
+    //     // don't allow track movments
+    //     this.isRecording = false
+
+    //     // set adjust absoluteStart to add the time difference
+    //     const timestamp = new Date().getTime()
+    //     this.absoluteStart = timestamp - this.absoluteStart
+    // }
 
-    public resume = () => {
-        this.isRecording = true
-        // set absoluteStart to the difference in time
-        this.absoluteStart = new Date().getTime() - this.absoluteStart
-    }
+    // public resume = () => {
+    //     this.isRecording = true
+    //     // set absoluteStart to the difference in time
+    //     this.absoluteStart = new Date().getTime() - this.absoluteStart
+    // }
 
     private trackMovements = (panX: number, panY: number, scale: number = 0): Error | undefined => {
         // ensure we are recording
@@ -134,6 +146,8 @@ export class RecordingApi {
         if (this.isInitPresenation) {
             return new Error('[recordingApi.ts] trackMovements(): no presentation')
         }
+      
+      console.log('track movment')
 
         // get the time
         const time = new Date().getTime() - this.absoluteStart
@@ -164,12 +178,6 @@ export class RecordingApi {
         this.recordingFFView = view;
     }
 
-    // call on dispose function to stop tracking movements
-    public removeRecordingFFView = (): void => {
-        this.disposeFunc?.();
-        this.disposeFunc = null;
-    }
-
     // TODO: extract this into different class with pause and resume recording
     // TODO: store the FFview with the movements
     private playFFView: CollectionFreeFormView | null;
@@ -250,6 +258,37 @@ export class RecordingApi {
             }, timeDiff)
         })
     }
+  
+  // make a public method that concatenates the movements of the an array of presentations into one array
+  // TODO: consider the meta data of the presentations
+  public concatPresentations = (presentations: Presentation[]): Presentation => {
+    console.info(presentations);
+    if (presentations.length === 0) return RecordingApi.NULL_PRESENTATION;
+    const firstPresentation = presentations[0];
+
+    let sumTime = presentations[0].totalTime;
+    let combinedPresentations = { ...firstPresentation }
+    presentations.forEach((presentation, i) => {
+      // already consider the first presentation
+      if (i === 0) return;
+
+      const { movements, totalTime } = presentation;
+      if (movements === null) return;
+
+      // add the summed time to the movements
+      const addedTimeMovements = movements.map(move => { return { ...move, time: move.time + sumTime } });
+      // concat the movements already in the combined presentation with these new ones
+      const newMovements = [...combinedPresentations.movements || [], ...addedTimeMovements];
+
+      combinedPresentations = { ...combinedPresentations, movements: newMovements }
+
+      // update the totalTime
+      sumTime += totalTime;
+    });
+
+    // return the combined presentation with the updated total summed time
+    return { ...combinedPresentations, totalTime: sumTime };
+  }
 
     // Unfinished code for tracing multiple free form views
     // export let pres: Map<CollectionFreeFormView, IReactionDisposer> = new Map()
diff --git a/src/client/views/nodes/RecordingBox/ProgressBar.tsx b/src/client/views/nodes/RecordingBox/ProgressBar.tsx
index 493069394..1bb2b7c84 100644
--- a/src/client/views/nodes/RecordingBox/ProgressBar.tsx
+++ b/src/client/views/nodes/RecordingBox/ProgressBar.tsx
@@ -161,12 +161,12 @@ export function ProgressBar(props: ProgressBarProps) {
     }
 
     // pointerdown event for the progress bar
-    const onPointerDown = (e: React.PointerEvent<HTMLDivElement>) => {
-        // don't move the videobox element
-        e.stopPropagation()
+  const onPointerDown = (e: React.PointerEvent<HTMLDivElement>) => {
+           // don't move the videobox element
+    e.stopPropagation();  
 
         // if recording, do nothing
-        if (props.recording) return;
+      if (props.recording) return;
 
         // get the segment the user clicked on to be dragged
         const clickedSegment = e.target as HTMLDivElement & EventTarget
diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
index 5e97e3eb5..6fe67b6db 100644
--- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
@@ -8,8 +8,8 @@ import { FieldView } from "../FieldView";
 import { VideoBox } from "../VideoBox";
 import { RecordingView } from './RecordingView';
 import { DocumentType } from "../../../documents/DocumentTypes";
-import { RecordingApi } from "../../../util/RecordingApi";
-import { Doc, FieldsSym } from "../../../../fields/Doc";
+import { Presentation } from "../../../util/RecordingApi";
+import { Doc } from "../../../../fields/Doc";
 import { Id } from "../../../../fields/FieldSymbols";
 
 
@@ -21,7 +21,7 @@ export class RecordingBox extends ViewBoxBaseComponent() {
       private _ref: React.RefObject<HTMLDivElement> = React.createRef();
 
       constructor(props: any) {
-            super(props);
+        super(props);
       }
 
       componentDidMount() {
@@ -38,7 +38,7 @@ export class RecordingBox extends ViewBoxBaseComponent() {
       }
 
       @action
-      setResult = (info: Upload.AccessPathInfo, trackScreen: boolean) => {
+      setResult = (info: Upload.AccessPathInfo, presentation?: Presentation) => {
             this.result = info
             this.dataDoc.type = DocumentType.VID;
             this.dataDoc[this.fieldKey + "-duration"] = this.videoDuration;
@@ -47,9 +47,7 @@ export class RecordingBox extends ViewBoxBaseComponent() {
             this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client);
             this.dataDoc[this.fieldKey + "-recorded"] = true;
             // stringify the presenation and store it
-            if (trackScreen) {
-                  this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(RecordingApi.Instance.clear());
-            }
+            presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation));
       }
 
       render() {
diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx
index 138e72274..208eaf45a 100644
--- a/src/client/views/nodes/RecordingBox/RecordingView.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx
@@ -8,16 +8,17 @@ import { IconContext } from "react-icons";
 import { Networking } from '../../../Network';
 import { Upload } from '../../../../server/SharedMediaTypes';
 import { returnFalse, returnTrue, setupMoveUpEvents } from '../../../../Utils';
-import { RecordingApi } from '../../../util/RecordingApi';
+import { Presentation, RecordingApi } from '../../../util/RecordingApi';
 
 export interface MediaSegment {
     videoChunks: any[],
     endTime: number,
-    startTime: number
+  startTime: number,
+  presenation?: Presentation,
 }
 
 interface IRecordingViewProps {
-    setResult: (info: Upload.AccessPathInfo, trackScreen: boolean) => void
+    setResult: (info: Upload.AccessPathInfo, presentation?: Presentation) => void
     setDuration: (seconds: number) => void
     id: string
 }
@@ -58,13 +59,18 @@ export function RecordingView(props: IRecordingViewProps) {
             sampleRate: 44100
         }
     }
+  
+  useEffect(() => console.info('progess', progress), [progress])
 
 
     useEffect(() => {
-        if (finished) {
+      if (finished) {
+        // make the total presentation that'll match the concatted video
+        const concatPres = trackScreen ? RecordingApi.Instance.concatPresentations(videos.map(v => v.presenation as Presentation)) : undefined;
+
             // this async function uses the server to create the concatted video and then sets the result to it's accessPaths
             (async () => {
-                const videoFiles = videos.map((vid, i) => new File(vid.videoChunks, `segvideo${i}.mkv`, { type: vid.videoChunks[0].type, lastModified: Date.now() }));
+              const videoFiles = videos.map((vid, i) => new File(vid.videoChunks, `segvideo${i}.mkv`, { type: vid.videoChunks[0].type, lastModified: Date.now() }));
 
                 // upload the segments to the server and get their server access paths
                 const serverPaths: string[] = (await Networking.UploadFilesToServer(videoFiles))
@@ -72,7 +78,7 @@ export function RecordingView(props: IRecordingViewProps) {
 
                 // concat the segments together using post call
                 const result: Upload.AccessPathInfo | Error = await Networking.PostToServer('/concatVideos', serverPaths);
-                !(result instanceof Error) ? props.setResult(result, trackScreen) : console.error("video conversion failed");
+                !(result instanceof Error) ? props.setResult(result, concatPres) : console.error("video conversion failed");
             })();
         }
     }, [videos])
@@ -133,53 +139,70 @@ export function RecordingView(props: IRecordingViewProps) {
             trackScreen && RecordingApi.Instance.start();
         }
 
-        videoRecorder.current.onstop = () => {
+      videoRecorder.current.onstop = () => {
+          RecordingApi.Instance.stop();
             // if we have a last portion
-            if (videoChunks.length > 1) {
+          if (videoChunks.length > 1) {
                 // append the current portion to the video pieces
-                setVideos(videos => [...videos, { videoChunks: videoChunks, endTime: recordingTimerRef.current, startTime: videos?.lastElement()?.endTime || 0 }])
+                  setVideos(videos => [...videos, {
+                    videoChunks: videoChunks,
+                    endTime: recordingTimerRef.current,
+                    startTime: videos?.lastElement()?.endTime || 0,
+                    // RecordingApi.stop() will return undefined if no track screen
+                    presenation: RecordingApi.Instance.getPresentation()
+                  }])
+                  // now that we got the presentation data, we can clear for the next segment to be recorded
+                  RecordingApi.Instance.clear();
             }
 
-            // reset the temporary chunks
-            videoChunks = []
-            setRecording(false);
-            trackScreen && RecordingApi.Instance.pause();
+          // reset the temporary chunks
+          videoChunks = []
+          setRecording(false);
         }
 
         videoRecorder.current.start(200)
     }
 
 
-    const stop = (e: React.PointerEvent) => {
-        e.stopPropagation();
-        if (videoRecorder.current) {
-            if (videoRecorder.current.state !== "inactive") {
-                videoRecorder.current.stop();
-            }
-
-            // this will call upon progessbar to update videos to be in the correct order
-            setFinished(true);
-
-            // end the streams (audio/video) to remove recording icon
-            const stream = videoElementRef.current!.srcObject;
-            stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop());
-        }
+  const finish = (e: React.PointerEvent) => {
+    e.stopPropagation();
+    console.log('finish', videoRecorder.current)
+      // if inactive, then we're done recording all the segments
+    if (videoRecorder.current && videoRecorder.current.state !== "inactive") {
+        console.log('stopping recorder', videoRecorder.current?.state)
+        
+        
+        
+        // call stop on the video recorder
+        videoRecorder.current?.stop();
+
+        // end the streams (audio/video) to remove recording icon
+        const stream = videoElementRef.current!.srcObject;
+        stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop());
+          
+        // clear the recoringApi - this is done in the stop method
+        // RecordingApi.Instance.clear();
+
+        // this will call upon progessbar to update videos to be in the correct order
+        console.log('setFinished to true', finished)
+        setFinished(true);
+      }
     }
 
     const pause = (e: React.PointerEvent) => {
-        e.stopPropagation()
-        if (videoRecorder.current) {
-            if (videoRecorder.current.state === "recording") {
-                videoRecorder.current.stop();
-            }
-        }
+      e.stopPropagation()
+      // if recording, then this is just a new segment
+      videoRecorder.current?.state === "recording" && videoRecorder.current.stop();
     }
 
-    const startOrResume = (e: React.PointerEvent) => {
+    const start = (e: React.PointerEvent) => {
         // the code to start or resume does not get triggered if we start dragging the button
         setupMoveUpEvents({}, e, returnTrue, returnFalse, e => {
-            (!videoRecorder.current || videoRecorder.current.state === "inactive") && record();
-            return true; // cancels propagation to documentView to avoid selecting it.
+          if (!videoRecorder.current || videoRecorder.current.state === "inactive") {
+            record();
+            // trackScreen && RecordingApi.Instance.start();
+          }
+          return true; // cancels propagation to documentView to avoid selecting it.
         }, false, false);
     }
 
@@ -218,7 +241,7 @@ export function RecordingView(props: IRecordingViewProps) {
                         <div className="record-button-wrapper">
                             {recording ?
                                 <button className="stop-button" onPointerDown={pause} /> :
-                                <button className="record-button" onPointerDown={startOrResume} />
+                                <button className="record-button" onPointerDown={start} />
                             }
                         </div>
 
@@ -229,7 +252,7 @@ export function RecordingView(props: IRecordingViewProps) {
                                     <MdBackspace onPointerDown={undoPrevious} />
                                 </IconContext.Provider>
                                 <IconContext.Provider value={{ color: "#cc1c08", className: "video-edit-buttons" }}>
-                                    <FaCheckCircle onPointerDown={stop} />
+                                    <FaCheckCircle onPointerDown={finish} />
                                 </IconContext.Provider>
                             </div>
 
-- 
cgit v1.2.3-70-g09d2


From d91ff1f937e9588b88258b07738eea56bac5fcca Mon Sep 17 00:00:00 2001
From: Michael <michael.foiani@gmail.com>
Date: Mon, 13 Jun 2022 13:21:38 -0400
Subject: fix bugs with selecting tracking. very stable now.

---
 src/client/util/RecordingApi.ts                    | 408 ++++++++++-----------
 .../views/nodes/RecordingBox/RecordingBox.tsx      |   2 +-
 .../views/nodes/RecordingBox/RecordingView.tsx     |  18 +-
 3 files changed, 208 insertions(+), 220 deletions(-)

(limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx')

diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts
index 12c1654a2..d56093cee 100644
--- a/src/client/util/RecordingApi.ts
+++ b/src/client/util/RecordingApi.ts
@@ -5,10 +5,10 @@ import { Doc } from "../../fields/Doc";
 import { VideoBox } from "../views/nodes/VideoBox";
 
 type Movement = {
-    time: number,
-    panX: number,
-    panY: number,
-    scale: number,
+  time: number,
+  panX: number,
+  panY: number,
+  scale: number,
 }
 
 export type Presentation = {
@@ -20,78 +20,82 @@ export type Presentation = {
 export class RecordingApi {
 
   private static get NULL_PRESENTATION(): Presentation {
-    return { movements: null, meta: {}, totalTime: -1,}
+    return { movements: null, meta: {}, totalTime: -1, }
   }
 
-    // instance variables
-    private currentPresentation: Presentation;
-    private tracking: boolean;
-    private absoluteStart: number;
-
-
-    // create static instance and getter for global use
-    @observable static _instance: RecordingApi;
-    public static get Instance(): RecordingApi { return RecordingApi._instance }
-    public constructor() {
-        // init the global instance
-        RecordingApi._instance = this;
-
-        // init the instance variables
-        this.currentPresentation = RecordingApi.NULL_PRESENTATION
-        this.tracking = false;
-        this.absoluteStart = -1;
-
-        // used for tracking movements in the view frame
-        this.disposeFunc = null;
-        this.recordingFFView = null;
-
-        // for now, set playFFView
-        this.playFFView = null;
-        this.timers = null;
-      
-      // put a pointerdown event on the doucment to see what the target
-    }
+  // instance variables
+  private currentPresentation: Presentation;
+  private tracking: boolean;
+  private absoluteStart: number;
 
-    // little helper :)
-    private get isInitPresenation(): boolean {
-        return this.currentPresentation.movements === null
-    }
 
-    public start = (meta?: Object) => {
-      // update the presentation mode
-      Doc.UserDoc().presentationMode = 'recording';
-
-        // (1a) get start date for presenation
-      const startDate = new Date();
-        // (1b) set start timestamp to absolute timestamp
-      this.absoluteStart = startDate.getTime();
-
-        // (2) assign meta content if it exists
-        this.currentPresentation.meta = meta || {}
-        // (3) assign start date to currentPresenation
-        this.currentPresentation.movements = []
-        // (4) set tracking true to allow trackMovements
-        this.tracking = true
-    }
+  // create static instance and getter for global use
+  @observable static _instance: RecordingApi;
+  public static get Instance(): RecordingApi { return RecordingApi._instance }
+  public constructor() {
+    // init the global instance
+    RecordingApi._instance = this;
+
+    // init the instance variables
+    this.currentPresentation = RecordingApi.NULL_PRESENTATION
+    this.tracking = false;
+    this.absoluteStart = -1;
+
+    // used for tracking movements in the view frame
+    this.disposeFunc = null;
+    this.recordingFFView = null;
 
-  /* stops the video and returns the presentatation; if no presentation, returns undefined */  
-  public * yieldPresentation(clearData: boolean = true): Generator<Presentation | null> {
+    // for now, set playFFView
+    this.playFFView = null;
+    this.timers = null;
+
+    // put a pointerdown event on the doucment to see what the target
+  }
+
+  // little helper :)
+  private get nullPresentation(): boolean {
+    return this.currentPresentation.movements === null
+  }
+
+  public start = (meta?: Object) => {
+    // update the presentation mode
+    Doc.UserDoc().presentationMode = 'recording';
+
+    // (1a) get start date for presenation
+    const startDate = new Date();
+    // (1b) set start timestamp to absolute timestamp
+    this.absoluteStart = startDate.getTime();
+
+    // (2) assign meta content if it exists
+    this.currentPresentation.meta = meta || {}
+    // (3) assign start date to currentPresenation
+    this.currentPresentation.movements = []
+    // (4) set tracking true to allow trackMovements
+    this.tracking = true
+  }
+
+  /* stops the video and returns the presentatation; if no presentation, returns undefined */
+  public yieldPresentation(clearData: boolean = true): Presentation | null {
     // if no presentation or done tracking, return null
-    if (!this.isInitPresenation || !this.tracking) return null;
+    if (this.nullPresentation || !this.tracking) return null;
 
     // set the previus recording view to the play view
     this.playFFView = this.recordingFFView;
 
     // ensure we add the endTime now that they are done recording    
-    yield { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
+    const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
 
     // reset the current presentation
     clearData && this.clear();
+
+    return cpy;
   }
 
-  public stop = (): void => {
+  public finish = (): void => {
     // make is tracking false
     this.tracking = false
+    // reset the RecordingApi instance
+    this.clear();
   }
 
   public clear = (): void => {
@@ -108,155 +112,138 @@ export class RecordingApi {
     // clear absoluteStart
     this.absoluteStart = -1
   }
-  
-    // call on dispose function to stop tracking movements
-    public removeRecordingFFView = (): void => {
-        this.disposeFunc?.();
-        this.disposeFunc = null;
-    }
 
-    // public pause = (): Error | undefined => {
-    //     if (this.isInitPresenation) {
-    //         console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first')
-    //         return new Error('[recordingApi.ts] pause(): no presentation')
-    //     }
-    //     // don't allow track movments
-    //     this.tracking = false
-
-    //     // set adjust absoluteStart to add the time difference
-    //     const timestamp = new Date().getTime()
-    //     this.absoluteStart = timestamp - this.absoluteStart
-    // }
-
-    // public resume = () => {
-    //     this.tracking = true
-    //     // set absoluteStart to the difference in time
-    //     this.absoluteStart = new Date().getTime() - this.absoluteStart
-    // }
-
-    private trackMovements = (panX: number, panY: number, scale: number = 0): Error | undefined => {
-        // ensure we are recording
-        if (!this.tracking) {
-            return new Error('[recordingApi.ts] trackMovements()')
-        }
-        // check to see if the presetation is init
-        if (this.isInitPresenation) {
-            return new Error('[recordingApi.ts] trackMovements(): no presentation')
-        }
-      
-      // TO FIX: bob
-      // console.debug('track movment')
-
-        // get the time
-        const time = new Date().getTime() - this.absoluteStart
-        // make new movement object
-        const movement: Movement = { time, panX, panY, scale }
-        
-        // add that movement to the current presentation data's movement array
-        this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
-    }
+  // call on dispose function to stop tracking movements
+  public removeRecordingFFView = (): void => {
+    this.disposeFunc?.();
+    this.disposeFunc = null;
+  }
 
-    // instance variable for the FFView
-    private disposeFunc: IReactionDisposer | null;
-    private recordingFFView: CollectionFreeFormView | null;
-    
-    // set the FFView that will be used in a reaction to track the movements
-    public setRecordingFFView = (view: CollectionFreeFormView): void => {
-        // set the view to the current view
-        if (view === this.recordingFFView || view == null) return;
-        
-        // this.recordingFFView = view;
-        // set the reaction to track the movements
-        this.disposeFunc = reaction(
-            () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }),
-            (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale)
-        )
-
-        // for now, set the most recent recordingFFView to the playFFView
-        this.recordingFFView = view;
+  private trackMovements = (panX: number, panY: number, scale: number = 0) => {
+    // ensure we are recording
+    if (!this.tracking) {
+      console.error('[recordingApi.ts] trackMovements(): tracking is false')
+      return;
     }
+    // check to see if the presetation is init
+    if (this.nullPresentation) {
+      console.error('[recordingApi.ts] trackMovements(): no presentation')
+      return;
+    }
+
+    // TO FIX: bob
+    // console.debug('track movment')
+
+    // get the time
+    const time = new Date().getTime() - this.absoluteStart
+    // make new movement object
+    const movement: Movement = { time, panX, panY, scale }
+
+    // add that movement to the current presentation data's movement array
+    this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
+  }
 
-    // TODO: extract this into different class with pause and resume recording
-    // TODO: store the FFview with the movements
-    private playFFView: CollectionFreeFormView | null;
-    private timers: NodeJS.Timeout[] | null;
+  // instance variable for the FFView
+  private disposeFunc: IReactionDisposer | null;
+  private recordingFFView: CollectionFreeFormView | null;
 
-    public setPlayFFView = (view: CollectionFreeFormView): void => {
-        this.playFFView = view
+  // set the FFView that will be used in a reaction to track the movements
+  public setRecordingFFView = (view: CollectionFreeFormView): void => {
+    // set the view to the current view
+    if (view === this.recordingFFView || view == null) return;
+
+    // this.recordingFFView = view;
+    // set the reaction to track the movements
+    this.disposeFunc = reaction(
+      () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }),
+      (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale)
+    )
+
+    // for now, set the most recent recordingFFView to the playFFView
+    this.recordingFFView = view;
+  }
+
+  // TODO: extract this into different class with pause and resume recording
+  // TODO: store the FFview with the movements
+  private playFFView: CollectionFreeFormView | null;
+  private timers: NodeJS.Timeout[] | null;
+
+  public setPlayFFView = (view: CollectionFreeFormView): void => {
+    this.playFFView = view
+  }
+
+  // pausing movements will dispose all timers that are planned to replay the movements
+  // play movemvents will recreate them when the user resumes the presentation
+  public pauseMovements = (): undefined | Error => {
+    if (this.playFFView === null) {
+      return new Error('[recordingApi.ts] pauseMovements() failed: no view')
     }
 
-    // pausing movements will dispose all timers that are planned to replay the movements
-    // play movemvents will recreate them when the user resumes the presentation
-    public pauseMovements = (): undefined | Error => {
-        if (this.playFFView === null) {
-            return new Error('[recordingApi.ts] pauseMovements() failed: no view')
-        }
-
-        if (!this._isPlaying) {
-            //return new Error('[recordingApi.ts] pauseMovements() failed: not playing')
-            return
-        }
-        this._isPlaying = false
-        // TODO: set userdoc presentMode to browsing
-        this.timers?.map(timer => clearTimeout(timer))
-
-        // this.videoBox = null;
+    if (!this._isPlaying) {
+      //return new Error('[recordingApi.ts] pauseMovements() failed: not playing')
+      return
     }
-    
-    private videoBox: VideoBox | null = null;
+    this._isPlaying = false
+    // TODO: set userdoc presentMode to browsing
+    this.timers?.map(timer => clearTimeout(timer))
+
+    // this.videoBox = null;
+  }
 
-    // by calling pause on the VideoBox, the pauseMovements will be called
-    public pauseVideoAndMovements = (): boolean => {
-        this.videoBox?.Pause()
+  private videoBox: VideoBox | null = null;
+
+  // by calling pause on the VideoBox, the pauseMovements will be called
+  public pauseVideoAndMovements = (): boolean => {
+    this.videoBox?.Pause()
+
+    this.pauseMovements()
+    return this.videoBox == null
+  }
 
-        this.pauseMovements()
-        return this.videoBox == null
+  public _isPlaying = false;
+
+  public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => {
+    if (presentation.movements === null || this.playFFView === null) {
+      return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view')
     }
+    if (this._isPlaying) return;
+
+    this._isPlaying = true;
+    Doc.UserDoc().presentationMode = 'watching';
+
+    // TODO: consider this bug at the end of the clip on seek
+    this.videoBox = videoBox || null;
 
-    public _isPlaying = false;
-
-    public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => {
-        if (presentation.movements === null || this.playFFView === null) {
-            return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view')
-        }
-        if(this._isPlaying) return;
-
-        this._isPlaying = true;
-        Doc.UserDoc().presentationMode = 'watching';
-
-        // TODO: consider this bug at the end of the clip on seek
-        this.videoBox = videoBox || null;
-
-        // only get the movements that are remaining in the video time left
-        const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
-
-        // helper to replay a movement
-        const document = this.playFFView
-        let preScale = -1;
-        const zoomAndPan = (movement: Movement) => {
-            const { panX, panY, scale } = movement;
-            (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
-            document.Document._panX = panX;
-            document.Document._panY = panY;
-
-            preScale = scale;
-        }
-
-        // set the first frame to be at the start of the pres
-        zoomAndPan(filteredMovements[0]);
-
-        // make timers that will execute each movement at the correct replay time
-        this.timers = filteredMovements.map(movement => {
-            const timeDiff = movement.time - timeViewed*1000
-            return setTimeout(() => {
-                // replay the movement
-                zoomAndPan(movement)
-                // if last movement, presentation is done -> set the instance var
-                if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false;
-            }, timeDiff)
-        })
+    // only get the movements that are remaining in the video time left
+    const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
+
+    // helper to replay a movement
+    const document = this.playFFView
+    let preScale = -1;
+    const zoomAndPan = (movement: Movement) => {
+      const { panX, panY, scale } = movement;
+      (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
+      document.Document._panX = panX;
+      document.Document._panY = panY;
+
+      preScale = scale;
     }
-  
+
+    // set the first frame to be at the start of the pres
+    zoomAndPan(filteredMovements[0]);
+
+    // make timers that will execute each movement at the correct replay time
+    this.timers = filteredMovements.map(movement => {
+      const timeDiff = movement.time - timeViewed * 1000
+      return setTimeout(() => {
+        // replay the movement
+        zoomAndPan(movement)
+        // if last movement, presentation is done -> set the instance var
+        if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false;
+      }, timeDiff)
+    })
+  }
+
   // method that concatenates an array of presentatations into one
   public concatPresentations = (presentations: Presentation[]): Presentation => {
     // these three will lead to the combined presentation
@@ -266,6 +253,7 @@ export class RecordingApi {
 
     presentations.forEach((presentation) => {
       const { movements, totalTime, meta } = presentation;
+
       // update movements if they had one
       if (movements) {
         // add the summed time to the movements
@@ -273,8 +261,10 @@ export class RecordingApi {
         // concat the movements already in the combined presentation with these new ones
         combinedMovements.push(...addedTimeMovements);
       }
+
       // update the totalTime
       sumTime += totalTime;
+
       // concatenate the metas
       combinedMetas.push(...meta);
     });
@@ -283,19 +273,19 @@ export class RecordingApi {
     return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas };
   }
 
-    // Unfinished code for tracing multiple free form views
-    // export let pres: Map<CollectionFreeFormView, IReactionDisposer> = new Map()
-
-    // export function AddRecordingFFView(ffView: CollectionFreeFormView): void {
-    //     pres.set(ffView,
-    //          reaction(() => ({ x: ffView.panX, y: ffView.panY }),
-    //     (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y)))
-    //     )
-    // }
-
-    // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void {
-    //     const disposer = pres.get(ffView);
-    //     disposer?.();
-    //     pres.delete(ffView)
-    // }
+  // Unfinished code for tracing multiple free form views
+  // export let pres: Map<CollectionFreeFormView, IReactionDisposer> = new Map()
+
+  // export function AddRecordingFFView(ffView: CollectionFreeFormView): void {
+  //     pres.set(ffView,
+  //          reaction(() => ({ x: ffView.panX, y: ffView.panY }),
+  //     (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y)))
+  //     )
+  // }
+
+  // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void {
+  //     const disposer = pres.get(ffView);
+  //     disposer?.();
+  //     pres.delete(ffView)
+  // }
 }
diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
index 6fe67b6db..a28677525 100644
--- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
@@ -46,7 +46,7 @@ export class RecordingBox extends ViewBoxBaseComponent() {
             this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey);
             this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client);
             this.dataDoc[this.fieldKey + "-recorded"] = true;
-            // stringify the presenation and store it
+            // stringify the presentation and store it
             presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation));
       }
 
diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx
index 35a6aa07e..15883632a 100644
--- a/src/client/views/nodes/RecordingBox/RecordingView.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx
@@ -140,7 +140,6 @@ export function RecordingView(props: IRecordingViewProps) {
     }
 
     videoRecorder.current.onstop = () => {
-      // RecordingApi.Instance.stop();
       // if we have a last portion
       if (videoChunks.length > 1) {
         // append the current portion to the video pieces
@@ -149,9 +148,10 @@ export function RecordingView(props: IRecordingViewProps) {
           endTime: recordingTimerRef.current,
           startTime: videos?.lastElement()?.endTime || 0
         };
+
         // depending on if a presenation exists, add it to the video
-        const { done: presError, value: presentation } = RecordingApi.Instance.yieldPresentation().next();
-        setVideos(videos => [...videos, (!presError && trackScreen) ? { ...nextVideo, presentation } : nextVideo]);
+        const presentation = RecordingApi.Instance.yieldPresentation();
+        setVideos(videos => [...videos, (presentation != null && trackScreen) ? { ...nextVideo, presentation } : nextVideo]);
       }
 
       // reset the temporary chunks
@@ -174,8 +174,8 @@ export function RecordingView(props: IRecordingViewProps) {
     const stream = videoElementRef.current!.srcObject;
     stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop());
 
-    // clear the recoringApi
-    RecordingApi.Instance.clear();
+    // finish/clear the recoringApi
+    RecordingApi.Instance.finish();
 
     // this will call upon progessbar to update videos to be in the correct order
     setFinished(true);
@@ -188,12 +188,10 @@ export function RecordingView(props: IRecordingViewProps) {
   }
 
   const start = (e: React.PointerEvent) => {
-    // the code to start or resume does not get triggered if we start dragging the button
     setupMoveUpEvents({}, e, returnTrue, returnFalse, e => {
-      if (!videoRecorder.current || videoRecorder.current.state === "inactive") {
-        record();
-        // trackScreen &&
-      }
+      // start recording if not already recording
+      if (!videoRecorder.current || videoRecorder.current.state === "inactive") record();
+
       return true; // cancels propagation to documentView to avoid selecting it.
     }, false, false);
   }
-- 
cgit v1.2.3-70-g09d2


From f8bfda0b0139db5f8bf1de602ce211a091d5d9ef Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@michaels-mbp-5.devices.brown.edu>
Date: Mon, 13 Jun 2022 14:56:03 -0400
Subject: revert to previous stable commit. lmao get ratioed by a mouse

---
 src/client/util/RecordingApi.ts                    | 383 ++++++++++++---------
 .../views/nodes/RecordingBox/RecordingBox.tsx      |   2 +-
 .../views/nodes/RecordingBox/RecordingView.tsx     |  18 +-
 3 files changed, 223 insertions(+), 180 deletions(-)

(limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx')

diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts
index bcd21a48a..4c8f55ce5 100644
--- a/src/client/util/RecordingApi.ts
+++ b/src/client/util/RecordingApi.ts
@@ -5,10 +5,10 @@ import { Doc } from "../../fields/Doc";
 import { VideoBox } from "../views/nodes/VideoBox";
 
 type Movement = {
-  time: number,
-  panX: number,
-  panY: number,
-  scale: number,
+    time: number,
+    panX: number,
+    panY: number,
+    scale: number,
 }
 
 export type Presentation = {
@@ -20,9 +20,10 @@ export type Presentation = {
 export class RecordingApi {
 
   private static get NULL_PRESENTATION(): Presentation {
-    return { movements: null, meta: {}, totalTime: -1, }
+    return { movements: null, meta: {}, totalTime: -1,}
   }
 
+<<<<<<< HEAD
   // instance variables
   private currentPresentation: Presentation;
   private tracking: boolean;
@@ -49,51 +50,77 @@ export class RecordingApi {
     this.playFFView = null;
     this.timers = null;
   }
+=======
+    // instance variables
+    private currentPresentation: Presentation;
+    private tracking: boolean;
+    private absoluteStart: number;
+
+
+    // create static instance and getter for global use
+    @observable static _instance: RecordingApi;
+    public static get Instance(): RecordingApi { return RecordingApi._instance }
+    public constructor() {
+        // init the global instance
+        RecordingApi._instance = this;
+
+        // init the instance variables
+        this.currentPresentation = RecordingApi.NULL_PRESENTATION
+        this.tracking = false;
+        this.absoluteStart = -1;
+
+        // used for tracking movements in the view frame
+        this.disposeFunc = null;
+        this.recordingFFView = null;
+
+        // for now, set playFFView
+        this.playFFView = null;
+        this.timers = null;
+      
+      // put a pointerdown event on the doucment to see what the target
+    }
+>>>>>>> parent of d91ff1f93 (fix bugs with selecting tracking. very stable now.)
 
-  // little helper :)
-  private get nullPresentation(): boolean {
-    return this.currentPresentation.movements === null
-  }
+    // little helper :)
+    private get isInitPresenation(): boolean {
+        return this.currentPresentation.movements === null
+    }
 
-  public start = (meta?: Object) => {
-    // update the presentation mode
-    Doc.UserDoc().presentationMode = 'recording';
-
-    // (1a) get start date for presenation
-    const startDate = new Date();
-    // (1b) set start timestamp to absolute timestamp
-    this.absoluteStart = startDate.getTime();
-
-    // (2) assign meta content if it exists
-    this.currentPresentation.meta = meta || {}
-    // (3) assign start date to currentPresenation
-    this.currentPresentation.movements = []
-    // (4) set tracking true to allow trackMovements
-    this.tracking = true
-  }
+    public start = (meta?: Object) => {
+      // update the presentation mode
+      Doc.UserDoc().presentationMode = 'recording';
+
+        // (1a) get start date for presenation
+      const startDate = new Date();
+        // (1b) set start timestamp to absolute timestamp
+      this.absoluteStart = startDate.getTime();
+
+        // (2) assign meta content if it exists
+        this.currentPresentation.meta = meta || {}
+        // (3) assign start date to currentPresenation
+        this.currentPresentation.movements = []
+        // (4) set tracking true to allow trackMovements
+        this.tracking = true
+    }
 
-  /* stops the video and returns the presentatation; if no presentation, returns undefined */
-  public yieldPresentation(clearData: boolean = true): Presentation | null {
+  /* stops the video and returns the presentatation; if no presentation, returns undefined */  
+  public * yieldPresentation(clearData: boolean = true): Generator<Presentation | null> {
     // if no presentation or done tracking, return null
-    if (this.nullPresentation || !this.tracking) return null;
+    if (!this.isInitPresenation || !this.tracking) return null;
 
     // set the previus recording view to the play view
     this.playFFView = this.recordingFFView;
 
     // ensure we add the endTime now that they are done recording    
-    const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
+    yield { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
 
     // reset the current presentation
     clearData && this.clear();
-
-    return cpy;
   }
 
-  public finish = (): void => {
+  public stop = (): void => {
     // make is tracking false
     this.tracking = false
-    // reset the RecordingApi instance
-    this.clear();
   }
 
   public clear = (): void => {
@@ -110,138 +137,155 @@ export class RecordingApi {
     // clear absoluteStart
     this.absoluteStart = -1
   }
-
-  // call on dispose function to stop tracking movements
-  public removeRecordingFFView = (): void => {
-    this.disposeFunc?.();
-    this.disposeFunc = null;
-  }
-
-  private trackMovements = (panX: number, panY: number, scale: number = 0) => {
-    // ensure we are recording
-    if (!this.tracking) {
-      console.error('[recordingApi.ts] trackMovements(): tracking is false')
-      return;
-    }
-    // check to see if the presetation is init
-    if (this.nullPresentation) {
-      console.error('[recordingApi.ts] trackMovements(): no presentation')
-      return;
+  
+    // call on dispose function to stop tracking movements
+    public removeRecordingFFView = (): void => {
+        this.disposeFunc?.();
+        this.disposeFunc = null;
     }
 
-    // TO FIX: bob
-    // console.debug('track movment')
-
-    // get the time
-    const time = new Date().getTime() - this.absoluteStart
-    // make new movement object
-    const movement: Movement = { time, panX, panY, scale }
-
-    // add that movement to the current presentation data's movement array
-    this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
-  }
-
-  // instance variable for the FFView
-  private disposeFunc: IReactionDisposer | null;
-  private recordingFFView: CollectionFreeFormView | null;
-
-  // set the FFView that will be used in a reaction to track the movements
-  public setRecordingFFView = (view: CollectionFreeFormView): void => {
-    // set the view to the current view
-    if (view === this.recordingFFView || view == null) return;
-
-    // this.recordingFFView = view;
-    // set the reaction to track the movements
-    this.disposeFunc = reaction(
-      () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }),
-      (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale)
-    )
-
-    // for now, set the most recent recordingFFView to the playFFView
-    this.recordingFFView = view;
-  }
-
-  // TODO: extract this into different class with pause and resume recording
-  // TODO: store the FFview with the movements
-  private playFFView: CollectionFreeFormView | null;
-  private timers: NodeJS.Timeout[] | null;
-
-  public setPlayFFView = (view: CollectionFreeFormView): void => {
-    this.playFFView = view
-  }
-
-  // pausing movements will dispose all timers that are planned to replay the movements
-  // play movemvents will recreate them when the user resumes the presentation
-  public pauseMovements = (): undefined | Error => {
-    if (this.playFFView === null) {
-      return new Error('[recordingApi.ts] pauseMovements() failed: no view')
+    // public pause = (): Error | undefined => {
+    //     if (this.isInitPresenation) {
+    //         console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first')
+    //         return new Error('[recordingApi.ts] pause(): no presentation')
+    //     }
+    //     // don't allow track movments
+    //     this.tracking = false
+
+    //     // set adjust absoluteStart to add the time difference
+    //     const timestamp = new Date().getTime()
+    //     this.absoluteStart = timestamp - this.absoluteStart
+    // }
+
+    // public resume = () => {
+    //     this.tracking = true
+    //     // set absoluteStart to the difference in time
+    //     this.absoluteStart = new Date().getTime() - this.absoluteStart
+    // }
+
+    private trackMovements = (panX: number, panY: number, scale: number = 0): Error | undefined => {
+        // ensure we are recording
+        if (!this.tracking) {
+            return new Error('[recordingApi.ts] trackMovements()')
+        }
+        // check to see if the presetation is init
+        if (this.isInitPresenation) {
+            return new Error('[recordingApi.ts] trackMovements(): no presentation')
+        }
+      
+      // TO FIX: bob
+      // console.debug('track movment')
+
+        // get the time
+        const time = new Date().getTime() - this.absoluteStart
+        // make new movement object
+        const movement: Movement = { time, panX, panY, scale }
+        
+        // add that movement to the current presentation data's movement array
+        this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
     }
 
-    if (!this._isPlaying) {
-      //return new Error('[recordingApi.ts] pauseMovements() failed: not playing')
-      return
+    // instance variable for the FFView
+    private disposeFunc: IReactionDisposer | null;
+    private recordingFFView: CollectionFreeFormView | null;
+    
+    // set the FFView that will be used in a reaction to track the movements
+    public setRecordingFFView = (view: CollectionFreeFormView): void => {
+        // set the view to the current view
+        if (view === this.recordingFFView || view == null) return;
+        
+        // this.recordingFFView = view;
+        // set the reaction to track the movements
+        this.disposeFunc = reaction(
+            () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }),
+            (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale)
+        )
+
+        // for now, set the most recent recordingFFView to the playFFView
+        this.recordingFFView = view;
     }
-    this._isPlaying = false
-    // TODO: set userdoc presentMode to browsing
-    this.timers?.map(timer => clearTimeout(timer))
-
-    // this.videoBox = null;
-  }
-
-  private videoBox: VideoBox | null = null;
-
-  // by calling pause on the VideoBox, the pauseMovements will be called
-  public pauseVideoAndMovements = (): boolean => {
-    this.videoBox?.Pause()
-
-    this.pauseMovements()
-    return this.videoBox == null
-  }
 
-  public _isPlaying = false;
+    // TODO: extract this into different class with pause and resume recording
+    // TODO: store the FFview with the movements
+    private playFFView: CollectionFreeFormView | null;
+    private timers: NodeJS.Timeout[] | null;
 
-  public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => {
-    if (presentation.movements === null || this.playFFView === null) {
-      return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view')
+    public setPlayFFView = (view: CollectionFreeFormView): void => {
+        this.playFFView = view
     }
-    if (this._isPlaying) return;
 
-    this._isPlaying = true;
-    Doc.UserDoc().presentationMode = 'watching';
-
-    // TODO: consider this bug at the end of the clip on seek
-    this.videoBox = videoBox || null;
-
-    // only get the movements that are remaining in the video time left
-    const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
+    // pausing movements will dispose all timers that are planned to replay the movements
+    // play movemvents will recreate them when the user resumes the presentation
+    public pauseMovements = (): undefined | Error => {
+        if (this.playFFView === null) {
+            return new Error('[recordingApi.ts] pauseMovements() failed: no view')
+        }
+
+        if (!this._isPlaying) {
+            //return new Error('[recordingApi.ts] pauseMovements() failed: not playing')
+            return
+        }
+        this._isPlaying = false
+        // TODO: set userdoc presentMode to browsing
+        this.timers?.map(timer => clearTimeout(timer))
+
+        // this.videoBox = null;
+    }
+    
+    private videoBox: VideoBox | null = null;
 
-    // helper to replay a movement
-    const document = this.playFFView
-    let preScale = -1;
-    const zoomAndPan = (movement: Movement) => {
-      const { panX, panY, scale } = movement;
-      (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
-      document.Document._panX = panX;
-      document.Document._panY = panY;
+    // by calling pause on the VideoBox, the pauseMovements will be called
+    public pauseVideoAndMovements = (): boolean => {
+        this.videoBox?.Pause()
 
-      preScale = scale;
+        this.pauseMovements()
+        return this.videoBox == null
     }
 
-    // set the first frame to be at the start of the pres
-    zoomAndPan(filteredMovements[0]);
-
-    // make timers that will execute each movement at the correct replay time
-    this.timers = filteredMovements.map(movement => {
-      const timeDiff = movement.time - timeViewed * 1000
-      return setTimeout(() => {
-        // replay the movement
-        zoomAndPan(movement)
-        // if last movement, presentation is done -> set the instance var
-        if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false;
-      }, timeDiff)
-    })
-  }
-
+    public _isPlaying = false;
+
+    public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => {
+        if (presentation.movements === null || this.playFFView === null) {
+            return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view')
+        }
+        if(this._isPlaying) return;
+
+        this._isPlaying = true;
+        Doc.UserDoc().presentationMode = 'watching';
+
+        // TODO: consider this bug at the end of the clip on seek
+        this.videoBox = videoBox || null;
+
+        // only get the movements that are remaining in the video time left
+        const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
+
+        // helper to replay a movement
+        const document = this.playFFView
+        let preScale = -1;
+        const zoomAndPan = (movement: Movement) => {
+            const { panX, panY, scale } = movement;
+            (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
+            document.Document._panX = panX;
+            document.Document._panY = panY;
+
+            preScale = scale;
+        }
+
+        // set the first frame to be at the start of the pres
+        zoomAndPan(filteredMovements[0]);
+
+        // make timers that will execute each movement at the correct replay time
+        this.timers = filteredMovements.map(movement => {
+            const timeDiff = movement.time - timeViewed*1000
+            return setTimeout(() => {
+                // replay the movement
+                zoomAndPan(movement)
+                // if last movement, presentation is done -> set the instance var
+                if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false;
+            }, timeDiff)
+        })
+    }
+  
   // method that concatenates an array of presentatations into one
   public concatPresentations = (presentations: Presentation[]): Presentation => {
     // these three will lead to the combined presentation
@@ -251,7 +295,6 @@ export class RecordingApi {
 
     presentations.forEach((presentation) => {
       const { movements, totalTime, meta } = presentation;
-
       // update movements if they had one
       if (movements) {
         // add the summed time to the movements
@@ -259,10 +302,8 @@ export class RecordingApi {
         // concat the movements already in the combined presentation with these new ones
         combinedMovements.push(...addedTimeMovements);
       }
-
       // update the totalTime
       sumTime += totalTime;
-
       // concatenate the metas
       combinedMetas.push(...meta);
     });
@@ -271,19 +312,19 @@ export class RecordingApi {
     return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas };
   }
 
-  // Unfinished code for tracing multiple free form views
-  // export let pres: Map<CollectionFreeFormView, IReactionDisposer> = new Map()
-
-  // export function AddRecordingFFView(ffView: CollectionFreeFormView): void {
-  //     pres.set(ffView,
-  //          reaction(() => ({ x: ffView.panX, y: ffView.panY }),
-  //     (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y)))
-  //     )
-  // }
-
-  // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void {
-  //     const disposer = pres.get(ffView);
-  //     disposer?.();
-  //     pres.delete(ffView)
-  // }
+    // Unfinished code for tracing multiple free form views
+    // export let pres: Map<CollectionFreeFormView, IReactionDisposer> = new Map()
+
+    // export function AddRecordingFFView(ffView: CollectionFreeFormView): void {
+    //     pres.set(ffView,
+    //          reaction(() => ({ x: ffView.panX, y: ffView.panY }),
+    //     (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y)))
+    //     )
+    // }
+
+    // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void {
+    //     const disposer = pres.get(ffView);
+    //     disposer?.();
+    //     pres.delete(ffView)
+    // }
 }
diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
index a28677525..6fe67b6db 100644
--- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
@@ -46,7 +46,7 @@ export class RecordingBox extends ViewBoxBaseComponent() {
             this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey);
             this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client);
             this.dataDoc[this.fieldKey + "-recorded"] = true;
-            // stringify the presentation and store it
+            // stringify the presenation and store it
             presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation));
       }
 
diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx
index d4d19f3d8..ba9479f41 100644
--- a/src/client/views/nodes/RecordingBox/RecordingView.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx
@@ -141,6 +141,7 @@ export function RecordingView(props: IRecordingViewProps) {
     }
 
     videoRecorder.current.onstop = () => {
+      // RecordingApi.Instance.stop();
       // if we have a last portion
       if (videoChunks.length > 1) {
         // append the current portion to the video pieces
@@ -149,10 +150,9 @@ export function RecordingView(props: IRecordingViewProps) {
           endTime: recordingTimerRef.current,
           startTime: videos?.lastElement()?.endTime || 0
         };
-
         // depending on if a presenation exists, add it to the video
-        const presentation = RecordingApi.Instance.yieldPresentation();
-        setVideos(videos => [...videos, (presentation != null && trackScreen) ? { ...nextVideo, presentation } : nextVideo]);
+        const { done: presError, value: presentation } = RecordingApi.Instance.yieldPresentation().next();
+        setVideos(videos => [...videos, (!presError && trackScreen) ? { ...nextVideo, presentation } : nextVideo]);
       }
 
       // reset the temporary chunks
@@ -175,8 +175,8 @@ export function RecordingView(props: IRecordingViewProps) {
     const stream = videoElementRef.current!.srcObject;
     stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop());
 
-    // finish/clear the recoringApi
-    RecordingApi.Instance.finish();
+    // clear the recoringApi
+    RecordingApi.Instance.clear();
 
     // this will call upon progessbar to update videos to be in the correct order
     setFinished(true);
@@ -189,10 +189,12 @@ export function RecordingView(props: IRecordingViewProps) {
   }
 
   const start = (e: React.PointerEvent) => {
+    // the code to start or resume does not get triggered if we start dragging the button
     setupMoveUpEvents({}, e, returnTrue, returnFalse, e => {
-      // start recording if not already recording
-      if (!videoRecorder.current || videoRecorder.current.state === "inactive") record();
-
+      if (!videoRecorder.current || videoRecorder.current.state === "inactive") {
+        record();
+        // trackScreen &&
+      }
       return true; // cancels propagation to documentView to avoid selecting it.
     }, false, false);
   }
-- 
cgit v1.2.3-70-g09d2


From c7e4bf8c4e19669666cce610020969cf0d865e49 Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@michaels-mbp-5.devices.brown.edu>
Date: Mon, 13 Jun 2022 14:57:34 -0400
Subject: Revert "debug w bob"

This reverts commit 1dfb694ffa8f34a834387eb073ffb9bd3a678039.

return to stable version without printing the logs.
---
 src/client/util/RecordingApi.ts                    | 385 +++++++++------------
 .../views/nodes/RecordingBox/RecordingBox.tsx      |   2 +-
 .../views/nodes/RecordingBox/RecordingView.tsx     |  29 +-
 3 files changed, 186 insertions(+), 230 deletions(-)

(limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx')

diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts
index 4c8f55ce5..d56093cee 100644
--- a/src/client/util/RecordingApi.ts
+++ b/src/client/util/RecordingApi.ts
@@ -5,10 +5,10 @@ import { Doc } from "../../fields/Doc";
 import { VideoBox } from "../views/nodes/VideoBox";
 
 type Movement = {
-    time: number,
-    panX: number,
-    panY: number,
-    scale: number,
+  time: number,
+  panX: number,
+  panY: number,
+  scale: number,
 }
 
 export type Presentation = {
@@ -20,10 +20,9 @@ export type Presentation = {
 export class RecordingApi {
 
   private static get NULL_PRESENTATION(): Presentation {
-    return { movements: null, meta: {}, totalTime: -1,}
+    return { movements: null, meta: {}, totalTime: -1, }
   }
 
-<<<<<<< HEAD
   // instance variables
   private currentPresentation: Presentation;
   private tracking: boolean;
@@ -49,78 +48,54 @@ export class RecordingApi {
     // for now, set playFFView
     this.playFFView = null;
     this.timers = null;
+
+    // put a pointerdown event on the doucment to see what the target
   }
-=======
-    // instance variables
-    private currentPresentation: Presentation;
-    private tracking: boolean;
-    private absoluteStart: number;
-
-
-    // create static instance and getter for global use
-    @observable static _instance: RecordingApi;
-    public static get Instance(): RecordingApi { return RecordingApi._instance }
-    public constructor() {
-        // init the global instance
-        RecordingApi._instance = this;
-
-        // init the instance variables
-        this.currentPresentation = RecordingApi.NULL_PRESENTATION
-        this.tracking = false;
-        this.absoluteStart = -1;
-
-        // used for tracking movements in the view frame
-        this.disposeFunc = null;
-        this.recordingFFView = null;
-
-        // for now, set playFFView
-        this.playFFView = null;
-        this.timers = null;
-      
-      // put a pointerdown event on the doucment to see what the target
-    }
->>>>>>> parent of d91ff1f93 (fix bugs with selecting tracking. very stable now.)
 
-    // little helper :)
-    private get isInitPresenation(): boolean {
-        return this.currentPresentation.movements === null
-    }
+  // little helper :)
+  private get nullPresentation(): boolean {
+    return this.currentPresentation.movements === null
+  }
 
-    public start = (meta?: Object) => {
-      // update the presentation mode
-      Doc.UserDoc().presentationMode = 'recording';
-
-        // (1a) get start date for presenation
-      const startDate = new Date();
-        // (1b) set start timestamp to absolute timestamp
-      this.absoluteStart = startDate.getTime();
-
-        // (2) assign meta content if it exists
-        this.currentPresentation.meta = meta || {}
-        // (3) assign start date to currentPresenation
-        this.currentPresentation.movements = []
-        // (4) set tracking true to allow trackMovements
-        this.tracking = true
-    }
+  public start = (meta?: Object) => {
+    // update the presentation mode
+    Doc.UserDoc().presentationMode = 'recording';
+
+    // (1a) get start date for presenation
+    const startDate = new Date();
+    // (1b) set start timestamp to absolute timestamp
+    this.absoluteStart = startDate.getTime();
+
+    // (2) assign meta content if it exists
+    this.currentPresentation.meta = meta || {}
+    // (3) assign start date to currentPresenation
+    this.currentPresentation.movements = []
+    // (4) set tracking true to allow trackMovements
+    this.tracking = true
+  }
 
-  /* stops the video and returns the presentatation; if no presentation, returns undefined */  
-  public * yieldPresentation(clearData: boolean = true): Generator<Presentation | null> {
+  /* stops the video and returns the presentatation; if no presentation, returns undefined */
+  public yieldPresentation(clearData: boolean = true): Presentation | null {
     // if no presentation or done tracking, return null
-    if (!this.isInitPresenation || !this.tracking) return null;
+    if (this.nullPresentation || !this.tracking) return null;
 
     // set the previus recording view to the play view
     this.playFFView = this.recordingFFView;
 
     // ensure we add the endTime now that they are done recording    
-    yield { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
+    const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
 
     // reset the current presentation
     clearData && this.clear();
+
+    return cpy;
   }
 
-  public stop = (): void => {
+  public finish = (): void => {
     // make is tracking false
     this.tracking = false
+    // reset the RecordingApi instance
+    this.clear();
   }
 
   public clear = (): void => {
@@ -137,155 +112,138 @@ export class RecordingApi {
     // clear absoluteStart
     this.absoluteStart = -1
   }
-  
-    // call on dispose function to stop tracking movements
-    public removeRecordingFFView = (): void => {
-        this.disposeFunc?.();
-        this.disposeFunc = null;
-    }
 
-    // public pause = (): Error | undefined => {
-    //     if (this.isInitPresenation) {
-    //         console.error('[recordingApi.ts] pause() failed: no presentation started. try calling init() first')
-    //         return new Error('[recordingApi.ts] pause(): no presentation')
-    //     }
-    //     // don't allow track movments
-    //     this.tracking = false
-
-    //     // set adjust absoluteStart to add the time difference
-    //     const timestamp = new Date().getTime()
-    //     this.absoluteStart = timestamp - this.absoluteStart
-    // }
-
-    // public resume = () => {
-    //     this.tracking = true
-    //     // set absoluteStart to the difference in time
-    //     this.absoluteStart = new Date().getTime() - this.absoluteStart
-    // }
-
-    private trackMovements = (panX: number, panY: number, scale: number = 0): Error | undefined => {
-        // ensure we are recording
-        if (!this.tracking) {
-            return new Error('[recordingApi.ts] trackMovements()')
-        }
-        // check to see if the presetation is init
-        if (this.isInitPresenation) {
-            return new Error('[recordingApi.ts] trackMovements(): no presentation')
-        }
-      
-      // TO FIX: bob
-      // console.debug('track movment')
-
-        // get the time
-        const time = new Date().getTime() - this.absoluteStart
-        // make new movement object
-        const movement: Movement = { time, panX, panY, scale }
-        
-        // add that movement to the current presentation data's movement array
-        this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
-    }
+  // call on dispose function to stop tracking movements
+  public removeRecordingFFView = (): void => {
+    this.disposeFunc?.();
+    this.disposeFunc = null;
+  }
 
-    // instance variable for the FFView
-    private disposeFunc: IReactionDisposer | null;
-    private recordingFFView: CollectionFreeFormView | null;
-    
-    // set the FFView that will be used in a reaction to track the movements
-    public setRecordingFFView = (view: CollectionFreeFormView): void => {
-        // set the view to the current view
-        if (view === this.recordingFFView || view == null) return;
-        
-        // this.recordingFFView = view;
-        // set the reaction to track the movements
-        this.disposeFunc = reaction(
-            () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }),
-            (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale)
-        )
-
-        // for now, set the most recent recordingFFView to the playFFView
-        this.recordingFFView = view;
+  private trackMovements = (panX: number, panY: number, scale: number = 0) => {
+    // ensure we are recording
+    if (!this.tracking) {
+      console.error('[recordingApi.ts] trackMovements(): tracking is false')
+      return;
+    }
+    // check to see if the presetation is init
+    if (this.nullPresentation) {
+      console.error('[recordingApi.ts] trackMovements(): no presentation')
+      return;
     }
 
-    // TODO: extract this into different class with pause and resume recording
-    // TODO: store the FFview with the movements
-    private playFFView: CollectionFreeFormView | null;
-    private timers: NodeJS.Timeout[] | null;
+    // TO FIX: bob
+    // console.debug('track movment')
+
+    // get the time
+    const time = new Date().getTime() - this.absoluteStart
+    // make new movement object
+    const movement: Movement = { time, panX, panY, scale }
+
+    // add that movement to the current presentation data's movement array
+    this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
+  }
+
+  // instance variable for the FFView
+  private disposeFunc: IReactionDisposer | null;
+  private recordingFFView: CollectionFreeFormView | null;
+
+  // set the FFView that will be used in a reaction to track the movements
+  public setRecordingFFView = (view: CollectionFreeFormView): void => {
+    // set the view to the current view
+    if (view === this.recordingFFView || view == null) return;
+
+    // this.recordingFFView = view;
+    // set the reaction to track the movements
+    this.disposeFunc = reaction(
+      () => ({ x: NumCast(view.Document.panX, -1), y: NumCast(view.Document.panY, -1), scale: NumCast(view.Document.viewScale, -1) }),
+      (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, res.scale)
+    )
+
+    // for now, set the most recent recordingFFView to the playFFView
+    this.recordingFFView = view;
+  }
+
+  // TODO: extract this into different class with pause and resume recording
+  // TODO: store the FFview with the movements
+  private playFFView: CollectionFreeFormView | null;
+  private timers: NodeJS.Timeout[] | null;
 
-    public setPlayFFView = (view: CollectionFreeFormView): void => {
-        this.playFFView = view
+  public setPlayFFView = (view: CollectionFreeFormView): void => {
+    this.playFFView = view
+  }
+
+  // pausing movements will dispose all timers that are planned to replay the movements
+  // play movemvents will recreate them when the user resumes the presentation
+  public pauseMovements = (): undefined | Error => {
+    if (this.playFFView === null) {
+      return new Error('[recordingApi.ts] pauseMovements() failed: no view')
     }
 
-    // pausing movements will dispose all timers that are planned to replay the movements
-    // play movemvents will recreate them when the user resumes the presentation
-    public pauseMovements = (): undefined | Error => {
-        if (this.playFFView === null) {
-            return new Error('[recordingApi.ts] pauseMovements() failed: no view')
-        }
-
-        if (!this._isPlaying) {
-            //return new Error('[recordingApi.ts] pauseMovements() failed: not playing')
-            return
-        }
-        this._isPlaying = false
-        // TODO: set userdoc presentMode to browsing
-        this.timers?.map(timer => clearTimeout(timer))
-
-        // this.videoBox = null;
+    if (!this._isPlaying) {
+      //return new Error('[recordingApi.ts] pauseMovements() failed: not playing')
+      return
     }
-    
-    private videoBox: VideoBox | null = null;
+    this._isPlaying = false
+    // TODO: set userdoc presentMode to browsing
+    this.timers?.map(timer => clearTimeout(timer))
+
+    // this.videoBox = null;
+  }
+
+  private videoBox: VideoBox | null = null;
+
+  // by calling pause on the VideoBox, the pauseMovements will be called
+  public pauseVideoAndMovements = (): boolean => {
+    this.videoBox?.Pause()
+
+    this.pauseMovements()
+    return this.videoBox == null
+  }
 
-    // by calling pause on the VideoBox, the pauseMovements will be called
-    public pauseVideoAndMovements = (): boolean => {
-        this.videoBox?.Pause()
+  public _isPlaying = false;
 
-        this.pauseMovements()
-        return this.videoBox == null
+  public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => {
+    if (presentation.movements === null || this.playFFView === null) {
+      return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view')
     }
+    if (this._isPlaying) return;
 
-    public _isPlaying = false;
-
-    public playMovements = (presentation: Presentation, timeViewed: number = 0, videoBox?: VideoBox): undefined | Error => {
-        if (presentation.movements === null || this.playFFView === null) {
-            return new Error('[recordingApi.ts] followMovements() failed: no presentation data or no view')
-        }
-        if(this._isPlaying) return;
-
-        this._isPlaying = true;
-        Doc.UserDoc().presentationMode = 'watching';
-
-        // TODO: consider this bug at the end of the clip on seek
-        this.videoBox = videoBox || null;
-
-        // only get the movements that are remaining in the video time left
-        const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
-
-        // helper to replay a movement
-        const document = this.playFFView
-        let preScale = -1;
-        const zoomAndPan = (movement: Movement) => {
-            const { panX, panY, scale } = movement;
-            (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
-            document.Document._panX = panX;
-            document.Document._panY = panY;
-
-            preScale = scale;
-        }
-
-        // set the first frame to be at the start of the pres
-        zoomAndPan(filteredMovements[0]);
-
-        // make timers that will execute each movement at the correct replay time
-        this.timers = filteredMovements.map(movement => {
-            const timeDiff = movement.time - timeViewed*1000
-            return setTimeout(() => {
-                // replay the movement
-                zoomAndPan(movement)
-                // if last movement, presentation is done -> set the instance var
-                if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false;
-            }, timeDiff)
-        })
+    this._isPlaying = true;
+    Doc.UserDoc().presentationMode = 'watching';
+
+    // TODO: consider this bug at the end of the clip on seek
+    this.videoBox = videoBox || null;
+
+    // only get the movements that are remaining in the video time left
+    const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
+
+    // helper to replay a movement
+    const document = this.playFFView
+    let preScale = -1;
+    const zoomAndPan = (movement: Movement) => {
+      const { panX, panY, scale } = movement;
+      (scale !== -1 && preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
+      document.Document._panX = panX;
+      document.Document._panY = panY;
+
+      preScale = scale;
     }
-  
+
+    // set the first frame to be at the start of the pres
+    zoomAndPan(filteredMovements[0]);
+
+    // make timers that will execute each movement at the correct replay time
+    this.timers = filteredMovements.map(movement => {
+      const timeDiff = movement.time - timeViewed * 1000
+      return setTimeout(() => {
+        // replay the movement
+        zoomAndPan(movement)
+        // if last movement, presentation is done -> set the instance var
+        if (movement === filteredMovements[filteredMovements.length - 1]) RecordingApi.Instance._isPlaying = false;
+      }, timeDiff)
+    })
+  }
+
   // method that concatenates an array of presentatations into one
   public concatPresentations = (presentations: Presentation[]): Presentation => {
     // these three will lead to the combined presentation
@@ -295,6 +253,7 @@ export class RecordingApi {
 
     presentations.forEach((presentation) => {
       const { movements, totalTime, meta } = presentation;
+
       // update movements if they had one
       if (movements) {
         // add the summed time to the movements
@@ -302,8 +261,10 @@ export class RecordingApi {
         // concat the movements already in the combined presentation with these new ones
         combinedMovements.push(...addedTimeMovements);
       }
+
       // update the totalTime
       sumTime += totalTime;
+
       // concatenate the metas
       combinedMetas.push(...meta);
     });
@@ -312,19 +273,19 @@ export class RecordingApi {
     return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas };
   }
 
-    // Unfinished code for tracing multiple free form views
-    // export let pres: Map<CollectionFreeFormView, IReactionDisposer> = new Map()
-
-    // export function AddRecordingFFView(ffView: CollectionFreeFormView): void {
-    //     pres.set(ffView,
-    //          reaction(() => ({ x: ffView.panX, y: ffView.panY }),
-    //     (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y)))
-    //     )
-    // }
-
-    // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void {
-    //     const disposer = pres.get(ffView);
-    //     disposer?.();
-    //     pres.delete(ffView)
-    // }
+  // Unfinished code for tracing multiple free form views
+  // export let pres: Map<CollectionFreeFormView, IReactionDisposer> = new Map()
+
+  // export function AddRecordingFFView(ffView: CollectionFreeFormView): void {
+  //     pres.set(ffView,
+  //          reaction(() => ({ x: ffView.panX, y: ffView.panY }),
+  //     (pt) => RecordingApi.trackMovements(ffView, pt.x, pt.y)))
+  //     )
+  // }
+
+  // export function RemoveRecordingFFView(ffView: CollectionFreeFormView): void {
+  //     const disposer = pres.get(ffView);
+  //     disposer?.();
+  //     pres.delete(ffView)
+  // }
 }
diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
index 6fe67b6db..a28677525 100644
--- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
@@ -46,7 +46,7 @@ export class RecordingBox extends ViewBoxBaseComponent() {
             this.dataDoc.layout = VideoBox.LayoutString(this.fieldKey);
             this.dataDoc[this.props.fieldKey] = new VideoField(this.result.accessPaths.client);
             this.dataDoc[this.fieldKey + "-recorded"] = true;
-            // stringify the presenation and store it
+            // stringify the presentation and store it
             presentation?.movements && (this.dataDoc[this.fieldKey + "-presentation"] = JSON.stringify(presentation));
       }
 
diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx
index ba9479f41..15883632a 100644
--- a/src/client/views/nodes/RecordingBox/RecordingView.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx
@@ -60,7 +60,7 @@ export function RecordingView(props: IRecordingViewProps) {
     }
   }
 
-  useEffect(() => console.log('progress', progress), [progress])
+  // useEffect(() => console.debug('progress', progress), [progress])
 
   useEffect(() => {
     if (finished) {
@@ -93,8 +93,7 @@ export function RecordingView(props: IRecordingViewProps) {
   useEffect(() => {
     let interval: any = null;
     if (recording) {
-      interval = setTimeout(() => {
-        console.log('update interval')
+      interval = setInterval(() => {
         setRecordingTimer(unit => unit + 1);
       }, 10);
     } else if (!recording && recordingTimer !== 0) {
@@ -141,7 +140,6 @@ export function RecordingView(props: IRecordingViewProps) {
     }
 
     videoRecorder.current.onstop = () => {
-      // RecordingApi.Instance.stop();
       // if we have a last portion
       if (videoChunks.length > 1) {
         // append the current portion to the video pieces
@@ -150,9 +148,10 @@ export function RecordingView(props: IRecordingViewProps) {
           endTime: recordingTimerRef.current,
           startTime: videos?.lastElement()?.endTime || 0
         };
+
         // depending on if a presenation exists, add it to the video
-        const { done: presError, value: presentation } = RecordingApi.Instance.yieldPresentation().next();
-        setVideos(videos => [...videos, (!presError && trackScreen) ? { ...nextVideo, presentation } : nextVideo]);
+        const presentation = RecordingApi.Instance.yieldPresentation();
+        setVideos(videos => [...videos, (presentation != null && trackScreen) ? { ...nextVideo, presentation } : nextVideo]);
       }
 
       // reset the temporary chunks
@@ -175,8 +174,8 @@ export function RecordingView(props: IRecordingViewProps) {
     const stream = videoElementRef.current!.srcObject;
     stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop());
 
-    // clear the recoringApi
-    RecordingApi.Instance.clear();
+    // finish/clear the recoringApi
+    RecordingApi.Instance.finish();
 
     // this will call upon progessbar to update videos to be in the correct order
     setFinished(true);
@@ -189,12 +188,10 @@ export function RecordingView(props: IRecordingViewProps) {
   }
 
   const start = (e: React.PointerEvent) => {
-    // the code to start or resume does not get triggered if we start dragging the button
     setupMoveUpEvents({}, e, returnTrue, returnFalse, e => {
-      if (!videoRecorder.current || videoRecorder.current.state === "inactive") {
-        record();
-        // trackScreen &&
-      }
+      // start recording if not already recording
+      if (!videoRecorder.current || videoRecorder.current.state === "inactive") record();
+
       return true; // cancels propagation to documentView to avoid selecting it.
     }, false, false);
   }
@@ -204,9 +201,7 @@ export function RecordingView(props: IRecordingViewProps) {
     setDoUndo(prev => !prev);
   }
 
-  const handleOnTimeUpdate = () => {
-    playing && setVideoProgressHelper(videoElementRef.current!.currentTime);
-  };
+  const handleOnTimeUpdate = () => { playing && setVideoProgressHelper(videoElementRef.current!.currentTime); };
 
   const millisecondToMinuteSecond = (milliseconds: number) => {
     const toTwoDigit = (digit: number) => {
@@ -223,7 +218,7 @@ export function RecordingView(props: IRecordingViewProps) {
         <video id={`video-${props.id}`}
           autoPlay
           muted
-          onTimeUpdate={handleOnTimeUpdate}
+          onTimeUpdate={() => handleOnTimeUpdate()}
           ref={videoElementRef}
         />
         <div className="recording-sign">
-- 
cgit v1.2.3-70-g09d2


From 98fba8bdb0fe81d6f71d0ae6018fcaaf7d8897df Mon Sep 17 00:00:00 2001
From: Michael Foiani <sotech117@michaels-mbp-5.devices.brown.edu>
Date: Wed, 15 Jun 2022 15:39:55 -0400
Subject: Refactor RecordingApi into two main files - TrackMovements and
 ReplayMovements. TrackMovements is for recording presentations and
 ReplayMovments is for replaying them.

---
 src/client/util/RecordingApi.ts                    | 461 ---------------------
 src/client/util/ReplayMovements.ts                 | 204 +++++++++
 src/client/util/TrackMovements.ts                  | 257 ++++++++++++
 src/client/views/Main.tsx                          |   6 +-
 .../collectionFreeForm/CollectionFreeFormView.tsx  |  16 +-
 .../views/nodes/RecordingBox/RecordingBox.tsx      |   2 +-
 .../views/nodes/RecordingBox/RecordingView.tsx     |  10 +-
 src/client/views/nodes/VideoBox.tsx                |   8 +-
 8 files changed, 477 insertions(+), 487 deletions(-)
 delete mode 100644 src/client/util/RecordingApi.ts
 create mode 100644 src/client/util/ReplayMovements.ts
 create mode 100644 src/client/util/TrackMovements.ts

(limited to 'src/client/views/nodes/RecordingBox/RecordingBox.tsx')

diff --git a/src/client/util/RecordingApi.ts b/src/client/util/RecordingApi.ts
deleted file mode 100644
index 87cb85497..000000000
--- a/src/client/util/RecordingApi.ts
+++ /dev/null
@@ -1,461 +0,0 @@
-import { CollectionFreeFormView } from "../views/collections/collectionFreeForm";
-import { IReactionDisposer, observable, observe, reaction } from "mobx";
-import { NumCast } from "../../fields/Types";
-import { Doc, DocListCast } from "../../fields/Doc";
-import { VideoBox } from "../views/nodes/VideoBox";
-import { isArray } from "lodash";
-import { SelectionManager } from "./SelectionManager";
-import { DocumentDecorations } from "../views/DocumentDecorations";
-import { DocumentManager } from "./DocumentManager";
-import { CollectionDockingView } from "../views/collections/CollectionDockingView";
-import { Id } from "../../fields/FieldSymbols";
-import { returnAll } from "../../Utils";
-import { ContextExclusionPlugin } from "webpack";
-import { DocServer } from "../DocServer";
-import { DocumentView } from "../views/nodes/DocumentView";
-
-type Movement = {
-    time: number,
-    panX: number,
-    panY: number,
-    scale: number,
-    docId: string,
-}
-
-
-export type Presentation = {
-    movements: Movement[] | null,
-    totalTime: number,
-    meta: Object | Object[],
-}
-
-export class RecordingApi {
-
-    private static get NULL_PRESENTATION(): Presentation {
-        return { movements: null, meta: {}, totalTime: -1, }
-    }
-
-    // instance variables
-    private currentPresentation: Presentation;
-    private tracking: boolean;
-    private absoluteStart: number;
-    // instance variable for holding the FFViews and their disposers
-    private recordingFFViews: Map<string, IReactionDisposer> | null;
-    private tabChangeDisposeFunc: IReactionDisposer | null;
-
-
-    // create static instance and getter for global use
-    @observable static _instance: RecordingApi;
-    public static get Instance(): RecordingApi { return RecordingApi._instance }
-    public constructor() {
-        // init the global instance
-        RecordingApi._instance = this;
-
-        // init the instance variables
-        this.currentPresentation = RecordingApi.NULL_PRESENTATION
-        this.tracking = false;
-        this.absoluteStart = -1;
-
-        // used for tracking movements in the view frame
-        this.recordingFFViews = null;
-        this.tabChangeDisposeFunc = null;
-
-        // for now, set playFFView
-        // this.playFFView = null;
-        this.timers = null;
-    }
-
-    // little helper :)
-    private get nullPresentation(): boolean {
-        return this.currentPresentation.movements === null
-    }
-
-    private addRecordingFFView(doc: Doc, key: string = doc[Id]): void {
-        console.info('adding dispose func : docId', key, 'doc', doc);
-
-        if (this.recordingFFViews === null) { console.warn('addFFView on null RecordingApi'); return; }
-        if (this.recordingFFViews.has(key)) { console.warn('addFFView : key already in map'); return; }
-
-        const disposeFunc = reaction(
-            () => ({ x: NumCast(doc.panX, -1), y: NumCast(doc.panY, -1), scale: NumCast(doc.viewScale, 0)}),
-            (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovements(res.x, res.y, key, res.scale),
-        );
-        this.recordingFFViews?.set(key, disposeFunc);
-    }
-
-    private removeRecordingFFView = (key: string) => {
-        console.info('removing dispose func : docId', key);
-        if (this.recordingFFViews === null) { console.warn('removeFFView on null RecordingApi'); return; }
-        this.recordingFFViews.get(key)?.();
-        this.recordingFFViews.delete(key);
-    }
-
-    // in the case where only one tab was changed (updates not across dashboards), set only one to true
-    private updateRecordingFFViewsFromTabs = (tabbedDocs: Doc[], onlyOne = false) => {
-        if (this.recordingFFViews === null) return;
-
-        // so that the size comparisons are correct, we must filter to only the FFViews
-        const isFFView = (doc: Doc) => doc && 'viewType' in doc && doc.viewType === 'freeform';
-        const tabbedFFViews = new Set<string>();
-        for (const DashDoc of tabbedDocs) {
-            if (isFFView(DashDoc)) tabbedFFViews.add(DashDoc[Id]);
-        }
-
-
-        // new tab was added - need to add it
-        if (tabbedFFViews.size > this.recordingFFViews.size) {
-            for (const DashDoc of tabbedDocs) {
-                if (!this.recordingFFViews.has(DashDoc[Id])) {
-                    if (isFFView(DashDoc)) {
-                        this.addRecordingFFView(DashDoc);
-
-                        // only one max change, so return
-                        if (onlyOne) return;
-                    }
-                }
-            }
-        }
-        // tab was removed - need to remove it from recordingFFViews
-        else if (tabbedFFViews.size < this.recordingFFViews.size) {
-            for (const [key] of this.recordingFFViews) {
-                if (!tabbedFFViews.has(key)) { 
-                    this.removeRecordingFFView(key);
-                    if (onlyOne) return;
-                } 
-            }
-        }
-    }
-
-    public initTabTracker = () => {
-        if (this.recordingFFViews === null) {
-            this.recordingFFViews = new Map();
-        }
-
-        // init the dispose funcs on the page
-        const docList = DocListCast(CollectionDockingView.Instance.props.Document.data);
-        this.updateRecordingFFViewsFromTabs(docList);
-
-        // create a reaction to monitor changes in tabs
-        this.tabChangeDisposeFunc =
-            reaction(() => CollectionDockingView.Instance.props.Document.data,
-                (change) => {
-                    // TODO: consider changing between dashboards
-                    console.log('change in tabs', change);
-                    this.updateRecordingFFViewsFromTabs(DocListCast(change), true);
-                });
-    }
-
-    public start = (meta?: Object) => {
-        this.initTabTracker();
-
-        // update the presentation mode
-        Doc.UserDoc().presentationMode = 'recording';
-
-        // (1a) get start date for presenation
-        const startDate = new Date();
-        // (1b) set start timestamp to absolute timestamp
-        this.absoluteStart = startDate.getTime();
-
-        // (2) assign meta content if it exists
-        this.currentPresentation.meta = meta || {}
-        // (3) assign start date to currentPresenation
-        this.currentPresentation.movements = []
-        // (4) set tracking true to allow trackMovements
-        this.tracking = true
-    }
-
-    /* stops the video and returns the presentatation; if no presentation, returns undefined */
-    public yieldPresentation(clearData: boolean = true): Presentation | null {
-        // if no presentation or done tracking, return null
-        if (this.nullPresentation || !this.tracking) return null;
-
-        // set the previus recording view to the play view
-        // this.playFFView = this.recordingFFView;
-
-        // ensure we add the endTime now that they are done recording    
-        const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
-
-        // reset the current presentation
-        clearData && this.clear();
-
-        console.log('yieldPresentation', cpy);
-        return cpy;
-    }
-
-    public finish = (): void => {
-        // make is tracking false
-        this.tracking = false
-        // reset the RecordingApi instance
-        this.clear();
-    }
-
-    public clear = (): void => {
-        // clear the disposeFunc if we are done (not tracking)
-        if (!this.tracking) {
-            this.removeAllRecordingFFViews();
-            this.tabChangeDisposeFunc?.();
-            // update the presentation mode now that we are done tracking
-            Doc.UserDoc().presentationMode = 'none';
-
-            this.recordingFFViews = null;
-            this.tabChangeDisposeFunc = null;
-        }
-
-        // clear presenation data
-        this.currentPresentation = RecordingApi.NULL_PRESENTATION
-        // clear absoluteStart
-        this.absoluteStart = -1
-    }
-
-    private removeAllRecordingFFViews = () => {
-        if (this.recordingFFViews === null) { console.warn('removeAllFFViews on null RecordingApi'); return; }
-
-        for (const [id, disposeFunc] of this.recordingFFViews) {
-            console.log('calling dispose func : docId', id);
-            disposeFunc();
-            this.recordingFFViews.delete(id);
-        }
-    }
-
-    private trackMovements = (panX: number, panY: number, docId: string, scale: number = 0) => {
-        // ensure we are recording to track
-        if (!this.tracking) {
-            console.error('[recordingApi.ts] trackMovements(): tracking is false')
-            return;
-        }
-        // check to see if the presetation is init - if not, we are between segments
-        // TODO: make this more clear - tracking should be "live tracking", not always true when the recording api being used (between start and yieldPres)
-        // bacuse tracking should be false inbetween segments high key 
-        if (this.nullPresentation) {
-            console.warn('[recordingApi.ts] trackMovements(): trying to store movemetns between segments')
-            return;
-        }
-
-        // get the time
-        const time = new Date().getTime() - this.absoluteStart
-        // make new movement object
-        const movement: Movement = { time, panX, panY, scale, docId }
-
-        // add that movement to the current presentation data's movement array
-        this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
-    }
-
-    // TODO: extract this into different class with pause and resume recording
-    // TODO: store the FFview with the movements
-    private timers: NodeJS.Timeout[] | null; 
-
-    // pausing movements will dispose all timers that are planned to replay the movements
-    // play movemvents will recreate them when the user resumes the presentation
-    public pauseMovements = (): undefined | Error => {
-
-        if (!this._isPlaying) { console.warn('[recordingApi.ts] pauseMovements(): already on paused'); return;}
-        this._isPlaying = false
-        // TODO: set userdoc presentMode to browsing
-        this.timers?.map(timer => clearTimeout(timer))
-
-        // this.videoBox = null;
-    }
-
-    private videoBoxDisposeFunc: IReactionDisposer | null = null;
-    private videoBox: VideoBox | null = null;
-
-    setVideoBox = async (videoBox: VideoBox) => {
-        console.log('setVideoBox', videoBox);
-        if (videoBox !== null) { console.warn('setVideoBox on already videoBox'); }
-        if (this.videoBoxDisposeFunc !== null) { console.warn('setVideoBox on already videoBox dispose func'); this.videoBoxDisposeFunc(); }
-
-
-        const { presentation } = videoBox;
-        if (presentation == null) { console.warn('setVideoBox on null videoBox presentation'); return; }
-        
-        let docIdtoDoc: Map<string, Doc> = new Map();
-        try {
-            docIdtoDoc = await this.loadPresentation(presentation);
-        } catch {
-            console.error('[recordingApi.ts] setVideoBox(): error loading presentation - no replay movements');
-            throw 'error loading docs from server';
-        }
-
-        
-        this.videoBoxDisposeFunc =
-            reaction(() => ({ playing: videoBox._playing, timeViewed: videoBox.player?.currentTime || 0 }),
-                ({ playing, timeViewed }) => 
-                    playing ? this.playMovements(presentation, docIdtoDoc, timeViewed) : this.pauseMovements()
-            );
-        this.videoBox = videoBox;
-    }
-
-    removeVideoBox = () => {
-        if (this.videoBoxDisposeFunc == null) { console.warn('removeVideoBox on null videoBox'); return; }
-        this.videoBoxDisposeFunc();
-
-        this.videoBox = null;
-        this.videoBoxDisposeFunc = null;
-    }
-
-
-    // by calling pause on the VideoBox, the pauseMovements will be called
-    public pauseFromInteraction = () => {
-        Doc.UserDoc().presentationMode = 'none';
-        this.videoBox?.Pause();
-
-        this.pauseMovements();
-        // return this.videoBox == null
-    }
-
-
-
-    public _isPlaying = false;
-
-    loadPresentation = async (presentation: Presentation) => {
-        const { movements } = presentation;
-        if (movements === null) {
-            throw '[recordingApi.ts] followMovements() failed: no presentation data';
-        }
-
-        // generate a set of all unique docIds
-        const docIds = new Set<string>();
-        for (const {docId} of movements) {
-            if (!docIds.has(docId)) docIds.add(docId);
-        }
-
-        const docIdtoDoc = new Map<string, Doc>();
-
-        let refFields = await DocServer.GetRefFields([...docIds.keys()]);
-        for (const docId in refFields) {
-            if (!refFields[docId]) {
-                throw `one field was undefined`;
-            }
-            docIdtoDoc.set(docId, refFields[docId] as Doc);
-        }
-        console.log('loadPresentation refFields', refFields, docIdtoDoc);
-
-        return docIdtoDoc;
-    }
-
-    // returns undefined if the docView isn't open on the screen
-    getCollectionFFView = (docId: string) => {
-        const isInView = DocumentManager.Instance.getDocumentViewById(docId);
-        if (isInView) { return isInView.ComponentView as CollectionFreeFormView; }
-    }
-
-    // will open the doc in a tab then return the CollectionFFView that holds it
-    openTab = (docId: string, docIdtoDoc: Map<string, Doc>) => {
-        const doc = docIdtoDoc.get(docId);
-        if (doc == undefined) {
-            console.error(`docIdtoDoc did not contain docId ${docId}`)
-            return undefined;
-        }
-        CollectionDockingView.AddSplit(doc, 'right');
-        const docView = DocumentManager.Instance.getDocumentViewById(docId);
-        return docView?.ComponentView as CollectionFreeFormView;
-    }
-
-    // helper to replay a movement
-    private preScale = -1;
-    zoomAndPan = (movement: Movement, document: CollectionFreeFormView) => {
-        const { panX, panY, scale } = movement;
-        (scale !== 0 && this.preScale !== scale) && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
-        document.Document._panX = panX;
-        document.Document._panY = panY;
-
-        this.preScale = scale;
-    }
-
-    getFirstMovements = (movements: Movement[], timeViewed: number): Map<string, Movement> => {
-        if (movements === null) return new Map();
-        // generate a set of all unique docIds
-        const docIdtoFirstMove = new Map();
-        for (const move of movements) {
-            const { docId } = move;
-            if (!docIdtoFirstMove.has(docId)) docIdtoFirstMove.set(docId, move);
-        }
-        return docIdtoFirstMove;
-    }
-
-    endPlayingPresentation = () => {
-        this.preScale = -1;
-        RecordingApi.Instance._isPlaying = false;
-    }
-
-    public playMovements = (presentation: Presentation, docIdtoDoc: Map<string, Doc>, timeViewed: number = 0) => {
-        console.log('playMovements', presentation, timeViewed, docIdtoDoc);
-
-        if (presentation.movements === null || presentation.movements.length === 0) { //|| this.playFFView === null) {
-            return new Error('[recordingApi.ts] followMovements() failed: no presentation data')
-        }
-        if (this._isPlaying) return;
-
-        this._isPlaying = true;
-        Doc.UserDoc().presentationMode = 'watching';
-
-        // only get the movements that are remaining in the video time left
-        const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
-
-        const handleFirstMovements = () => {
-            // if the first movement is a closed tab, open it
-            const firstMovement = filteredMovements[0];
-            const isClosed = this.getCollectionFFView(firstMovement.docId) === undefined;
-            if (isClosed) this.openTab(firstMovement.docId, docIdtoDoc);
-
-            // for the open tabs, set it to the first move
-            const docIdtoFirstMove = this.getFirstMovements(filteredMovements, timeViewed);
-            for (const [docId, firstMove] of docIdtoFirstMove) {
-                const colFFView = this.getCollectionFFView(docId);
-                if (colFFView) this.zoomAndPan(firstMove, colFFView);
-            }
-        }
-        handleFirstMovements();
-        
-
-        // make timers that will execute each movement at the correct replay time
-        this.timers = filteredMovements.map(movement => {
-            const timeDiff = movement.time - timeViewed * 1000
-
-            return setTimeout(() => {
-                const collectionFFView = this.getCollectionFFView(movement.docId);
-                if (collectionFFView) {
-                    this.zoomAndPan(movement, collectionFFView);
-                } else {
-                    // tab wasn't open - open it and play the movement
-                    const openedColFFView = this.openTab(movement.docId, docIdtoDoc);
-                    openedColFFView && this.zoomAndPan(movement, openedColFFView);
-                }
-
-                // if last movement, presentation is done -> cleanup :)
-                if (movement === filteredMovements[filteredMovements.length - 1]) {
-                    this.endPlayingPresentation();
-                }
-            }, timeDiff);
-        });
-    }
-
-    // method that concatenates an array of presentatations into one
-    public concatPresentations = (presentations: Presentation[]): Presentation => {
-        // these three will lead to the combined presentation
-        let combinedMovements: Movement[] = [];
-        let sumTime = 0;
-        let combinedMetas: any[] = [];
-
-        presentations.forEach((presentation) => {
-            const { movements, totalTime, meta } = presentation;
-
-            // update movements if they had one
-            if (movements) {
-                // add the summed time to the movements
-                const addedTimeMovements = movements.map(move => { return { ...move, time: move.time + sumTime } });
-                // concat the movements already in the combined presentation with these new ones
-                combinedMovements.push(...addedTimeMovements);
-            }
-
-            // update the totalTime
-            sumTime += totalTime;
-
-            // concatenate the metas
-            combinedMetas.push(meta);
-        });
-
-        // return the combined presentation with the updated total summed time
-		return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas };
-    }
-}
diff --git a/src/client/util/ReplayMovements.ts b/src/client/util/ReplayMovements.ts
new file mode 100644
index 000000000..e46810b52
--- /dev/null
+++ b/src/client/util/ReplayMovements.ts
@@ -0,0 +1,204 @@
+import { CollectionFreeFormView } from "../views/collections/collectionFreeForm";
+import { IReactionDisposer, observable, observe, reaction } from "mobx";
+import { Doc } from "../../fields/Doc";
+import { VideoBox } from "../views/nodes/VideoBox";
+import { DocumentManager } from "./DocumentManager";
+import { CollectionDockingView } from "../views/collections/CollectionDockingView";
+import { DocServer } from "../DocServer";
+import { Movement, Presentation } from "./TrackMovements";
+
+export class ReplayMovements {
+    private timers: NodeJS.Timeout[] | null; 
+    private videoBoxDisposeFunc: IReactionDisposer | null;
+    private videoBox: VideoBox | null;
+    private isPlaying: boolean;
+
+
+    // create static instance and getter for global use
+    @observable static _instance: ReplayMovements;
+    static get Instance(): ReplayMovements { return ReplayMovements._instance }
+    constructor() {
+        // init the global instance
+        ReplayMovements._instance = this;
+
+        // instance vars for replaying
+        this.timers = null;
+        this.videoBoxDisposeFunc = null;
+        this.videoBox = null;
+        this.isPlaying = false;
+    }
+
+    // pausing movements will dispose all timers that are planned to replay the movements
+    // play movemvents will recreate them when the user resumes the presentation
+    pauseMovements = (): undefined | Error => {
+        if (!this.isPlaying) {
+            console.warn('[recordingApi.ts] pauseMovements(): already on paused');
+            return;
+        }
+
+        this.isPlaying = false
+        // TODO: set userdoc presentMode to browsing
+        this.timers?.map(timer => clearTimeout(timer))
+    }
+
+    setVideoBox = async (videoBox: VideoBox) => {
+        console.log('setVideoBox', videoBox);
+        if (videoBox !== null) { console.warn('setVideoBox on already videoBox'); }
+        if (this.videoBoxDisposeFunc !== null) { console.warn('setVideoBox on already videoBox dispose func'); this.videoBoxDisposeFunc(); }
+
+
+        const { presentation } = videoBox;
+        if (presentation == null) { console.warn('setVideoBox on null videoBox presentation'); return; }
+        
+        let docIdtoDoc: Map<string, Doc> = new Map();
+        try {
+            docIdtoDoc = await this.loadPresentation(presentation);
+        } catch {
+            console.error('[recordingApi.ts] setVideoBox(): error loading presentation - no replay movements');
+            throw 'error loading docs from server';
+        }
+
+        
+        this.videoBoxDisposeFunc =
+            reaction(() => ({ playing: videoBox._playing, timeViewed: videoBox.player?.currentTime || 0 }),
+                ({ playing, timeViewed }) => 
+                    playing ? this.playMovements(presentation, docIdtoDoc, timeViewed) : this.pauseMovements()
+            );
+        this.videoBox = videoBox;
+    }
+
+    removeVideoBox = () => {
+        if (this.videoBoxDisposeFunc == null) { console.warn('removeVideoBox on null videoBox'); return; }
+        this.videoBoxDisposeFunc();
+
+        this.videoBox = null;
+        this.videoBoxDisposeFunc = null;
+    }
+
+    // should be called from interacting with the screen
+    pauseFromInteraction = () => {
+        Doc.UserDoc().presentationMode = 'none';
+        this.videoBox?.Pause();
+
+        this.pauseMovements();
+    }
+
+    loadPresentation = async (presentation: Presentation) => {
+        const { movements } = presentation;
+        if (movements === null) {
+            throw '[recordingApi.ts] followMovements() failed: no presentation data';
+        }
+
+        // generate a set of all unique docIds
+        const docIds = new Set<string>();
+        for (const {docId} of movements) {
+            if (!docIds.has(docId)) docIds.add(docId);
+        }
+
+        const docIdtoDoc = new Map<string, Doc>();
+
+        let refFields = await DocServer.GetRefFields([...docIds.keys()]);
+        for (const docId in refFields) {
+            if (!refFields[docId]) {
+                throw `one field was undefined`;
+            }
+            docIdtoDoc.set(docId, refFields[docId] as Doc);
+        }
+        console.log('loadPresentation refFields', refFields, docIdtoDoc);
+
+        return docIdtoDoc;
+    }
+
+    // returns undefined if the docView isn't open on the screen
+    getCollectionFFView = (docId: string) => {
+        const isInView = DocumentManager.Instance.getDocumentViewById(docId);
+        if (isInView) { return isInView.ComponentView as CollectionFreeFormView; }
+    }
+
+    // will open the doc in a tab then return the CollectionFFView that holds it
+    openTab = (docId: string, docIdtoDoc: Map<string, Doc>) => {
+        const doc = docIdtoDoc.get(docId);
+        if (doc == undefined) {
+            console.error(`docIdtoDoc did not contain docId ${docId}`)
+            return undefined;
+        }
+        CollectionDockingView.AddSplit(doc, 'right');
+        const docView = DocumentManager.Instance.getDocumentViewById(docId);
+        return docView?.ComponentView as CollectionFreeFormView;
+    }
+
+    // helper to replay a movement
+    zoomAndPan = (movement: Movement, document: CollectionFreeFormView) => {
+        const { panX, panY, scale } = movement;
+        scale !== 0 && document.zoomSmoothlyAboutPt([panX, panY], scale, 0);
+        document.Document._panX = panX;
+        document.Document._panY = panY;
+    }
+
+    getFirstMovements = (movements: Movement[], timeViewed: number): Map<string, Movement> => {
+        if (movements === null) return new Map();
+        // generate a set of all unique docIds
+        const docIdtoFirstMove = new Map();
+        for (const move of movements) {
+            const { docId } = move;
+            if (!docIdtoFirstMove.has(docId)) docIdtoFirstMove.set(docId, move);
+        }
+        return docIdtoFirstMove;
+    }
+
+    endPlayingPresentation = () => {
+        this.isPlaying = false;
+    }
+
+    public playMovements = (presentation: Presentation, docIdtoDoc: Map<string, Doc>, timeViewed: number = 0) => {
+        console.log('playMovements', presentation, timeViewed, docIdtoDoc);
+
+        if (presentation.movements === null || presentation.movements.length === 0) { //|| this.playFFView === null) {
+            return new Error('[recordingApi.ts] followMovements() failed: no presentation data')
+        }
+        if (this.isPlaying) return;
+
+        this.isPlaying = true;
+        Doc.UserDoc().presentationMode = 'watching';
+
+        // only get the movements that are remaining in the video time left
+        const filteredMovements = presentation.movements.filter(movement => movement.time > timeViewed * 1000)
+
+        const handleFirstMovements = () => {
+            // if the first movement is a closed tab, open it
+            const firstMovement = filteredMovements[0];
+            const isClosed = this.getCollectionFFView(firstMovement.docId) === undefined;
+            if (isClosed) this.openTab(firstMovement.docId, docIdtoDoc);
+
+            // for the open tabs, set it to the first move
+            const docIdtoFirstMove = this.getFirstMovements(filteredMovements, timeViewed);
+            for (const [docId, firstMove] of docIdtoFirstMove) {
+                const colFFView = this.getCollectionFFView(docId);
+                if (colFFView) this.zoomAndPan(firstMove, colFFView);
+            }
+        }
+        handleFirstMovements();
+        
+
+        // make timers that will execute each movement at the correct replay time
+        this.timers = filteredMovements.map(movement => {
+            const timeDiff = movement.time - timeViewed * 1000
+
+            return setTimeout(() => {
+                const collectionFFView = this.getCollectionFFView(movement.docId);
+                if (collectionFFView) {
+                    this.zoomAndPan(movement, collectionFFView);
+                } else {
+                    // tab wasn't open - open it and play the movement
+                    const openedColFFView = this.openTab(movement.docId, docIdtoDoc);
+                    openedColFFView && this.zoomAndPan(movement, openedColFFView);
+                }
+
+                // if last movement, presentation is done -> cleanup :)
+                if (movement === filteredMovements[filteredMovements.length - 1]) {
+                    this.endPlayingPresentation();
+                }
+            }, timeDiff);
+        });
+    }
+}
diff --git a/src/client/util/TrackMovements.ts b/src/client/util/TrackMovements.ts
new file mode 100644
index 000000000..342bb440e
--- /dev/null
+++ b/src/client/util/TrackMovements.ts
@@ -0,0 +1,257 @@
+import { IReactionDisposer, observable, observe, reaction } from "mobx";
+import { NumCast } from "../../fields/Types";
+import { Doc, DocListCast } from "../../fields/Doc";
+import { CollectionDockingView } from "../views/collections/CollectionDockingView";
+import { Id } from "../../fields/FieldSymbols";
+
+export type Movement = {
+    time: number,
+    panX: number,
+    panY: number,
+    scale: number,
+    docId: string,
+}
+
+export type Presentation = {
+    movements: Movement[] | null,
+    totalTime: number,
+    meta: Object | Object[],
+}
+
+export class TrackMovements {
+
+    private static get NULL_PRESENTATION(): Presentation {
+        return { movements: null, meta: {}, totalTime: -1, }
+    }
+
+    // instance variables
+    private currentPresentation: Presentation;
+    private tracking: boolean;
+    private absoluteStart: number;
+    // instance variable for holding the FFViews and their disposers
+    private recordingFFViews: Map<string, IReactionDisposer> | null;
+    private tabChangeDisposeFunc: IReactionDisposer | null;
+
+
+    // create static instance and getter for global use
+    @observable static _instance: TrackMovements;
+    static get Instance(): TrackMovements { return TrackMovements._instance }
+    constructor() {
+        // init the global instance
+        TrackMovements._instance = this;
+
+        // init the instance variables
+        this.currentPresentation = TrackMovements.NULL_PRESENTATION
+        this.tracking = false;
+        this.absoluteStart = -1;
+
+        // used for tracking movements in the view frame
+        this.recordingFFViews = null;
+        this.tabChangeDisposeFunc = null;
+    }
+
+    // little helper :)
+    private get nullPresentation(): boolean {
+        return this.currentPresentation.movements === null
+    }
+
+    private addRecordingFFView(doc: Doc, key: string = doc[Id]): void {
+        console.info('adding dispose func : docId', key, 'doc', doc);
+
+        if (this.recordingFFViews === null) { console.warn('addFFView on null RecordingApi'); return; }
+        if (this.recordingFFViews.has(key)) { console.warn('addFFView : key already in map'); return; }
+
+        const disposeFunc = reaction(
+            () => ({ x: NumCast(doc.panX, -1), y: NumCast(doc.panY, -1), scale: NumCast(doc.viewScale, 0)}),
+            (res) => (res.x !== -1 && res.y !== -1 && this.tracking) && this.trackMovement(res.x, res.y, key, res.scale),
+        );
+        this.recordingFFViews?.set(key, disposeFunc);
+    }
+
+    private removeRecordingFFView = (key: string) => {
+        console.info('removing dispose func : docId', key);
+        if (this.recordingFFViews === null) { console.warn('removeFFView on null RecordingApi'); return; }
+        this.recordingFFViews.get(key)?.();
+        this.recordingFFViews.delete(key);
+    }
+
+    // in the case where only one tab was changed (updates not across dashboards), set only one to true
+    private updateRecordingFFViewsFromTabs = (tabbedDocs: Doc[], onlyOne = false) => {
+        if (this.recordingFFViews === null) return;
+
+        // so that the size comparisons are correct, we must filter to only the FFViews
+        const isFFView = (doc: Doc) => doc && 'viewType' in doc && doc.viewType === 'freeform';
+        const tabbedFFViews = new Set<string>();
+        for (const DashDoc of tabbedDocs) {
+            if (isFFView(DashDoc)) tabbedFFViews.add(DashDoc[Id]);
+        }
+
+
+        // new tab was added - need to add it
+        if (tabbedFFViews.size > this.recordingFFViews.size) {
+            for (const DashDoc of tabbedDocs) {
+                if (!this.recordingFFViews.has(DashDoc[Id])) {
+                    if (isFFView(DashDoc)) {
+                        this.addRecordingFFView(DashDoc);
+
+                        // only one max change, so return
+                        if (onlyOne) return;
+                    }
+                }
+            }
+        }
+        // tab was removed - need to remove it from recordingFFViews
+        else if (tabbedFFViews.size < this.recordingFFViews.size) {
+            for (const [key] of this.recordingFFViews) {
+                if (!tabbedFFViews.has(key)) { 
+                    this.removeRecordingFFView(key);
+                    if (onlyOne) return;
+                } 
+            }
+        }
+    }
+
+    private initTabTracker = () => {
+        if (this.recordingFFViews === null) {
+            this.recordingFFViews = new Map();
+        }
+
+        // init the dispose funcs on the page
+        const docList = DocListCast(CollectionDockingView.Instance.props.Document.data);
+        this.updateRecordingFFViewsFromTabs(docList);
+
+        // create a reaction to monitor changes in tabs
+        this.tabChangeDisposeFunc =
+            reaction(() => CollectionDockingView.Instance.props.Document.data,
+                (change) => {
+                    // TODO: consider changing between dashboards
+                    console.log('change in tabs', change);
+                    this.updateRecordingFFViewsFromTabs(DocListCast(change), true);
+                });
+    }
+
+    start = (meta?: Object) => {
+        this.initTabTracker();
+
+        // update the presentation mode
+        Doc.UserDoc().presentationMode = 'recording';
+
+        // (1a) get start date for presenation
+        const startDate = new Date();
+        // (1b) set start timestamp to absolute timestamp
+        this.absoluteStart = startDate.getTime();
+
+        // (2) assign meta content if it exists
+        this.currentPresentation.meta = meta || {}
+        // (3) assign start date to currentPresenation
+        this.currentPresentation.movements = []
+        // (4) set tracking true to allow trackMovements
+        this.tracking = true
+    }
+
+    /* stops the video and returns the presentatation; if no presentation, returns undefined */
+    yieldPresentation(clearData: boolean = true): Presentation | null {
+        // if no presentation or done tracking, return null
+        if (this.nullPresentation || !this.tracking) return null;
+
+        // set the previus recording view to the play view
+        // this.playFFView = this.recordingFFView;
+
+        // ensure we add the endTime now that they are done recording    
+        const cpy = { ...this.currentPresentation, totalTime: new Date().getTime() - this.absoluteStart };
+
+        // reset the current presentation
+        clearData && this.clear();
+
+        console.log('yieldPresentation', cpy);
+        return cpy;
+    }
+
+    finish = (): void => {
+        // make is tracking false
+        this.tracking = false
+        // reset the RecordingApi instance
+        this.clear();
+    }
+
+    private clear = (): void => {
+        // clear the disposeFunc if we are done (not tracking)
+        if (!this.tracking) {
+            this.removeAllRecordingFFViews();
+            this.tabChangeDisposeFunc?.();
+            // update the presentation mode now that we are done tracking
+            Doc.UserDoc().presentationMode = 'none';
+
+            this.recordingFFViews = null;
+            this.tabChangeDisposeFunc = null;
+        }
+
+        // clear presenation data
+        this.currentPresentation = TrackMovements.NULL_PRESENTATION
+        // clear absoluteStart
+        this.absoluteStart = -1
+    }
+
+    removeAllRecordingFFViews = () => {
+        if (this.recordingFFViews === null) { console.warn('removeAllFFViews on null RecordingApi'); return; }
+
+        for (const [id, disposeFunc] of this.recordingFFViews) {
+            console.log('calling dispose func : docId', id);
+            disposeFunc();
+            this.recordingFFViews.delete(id);
+        }
+    }
+
+    private trackMovement = (panX: number, panY: number, docId: string, scale: number = 0) => {
+        // ensure we are recording to track
+        if (!this.tracking) {
+            console.error('[recordingApi.ts] trackMovements(): tracking is false')
+            return;
+        }
+        // check to see if the presetation is init - if not, we are between segments
+        // TODO: make this more clear - tracking should be "live tracking", not always true when the recording api being used (between start and yieldPres)
+        // bacuse tracking should be false inbetween segments high key 
+        if (this.nullPresentation) {
+            console.warn('[recordingApi.ts] trackMovements(): trying to store movemetns between segments')
+            return;
+        }
+
+        // get the time
+        const time = new Date().getTime() - this.absoluteStart
+        // make new movement object
+        const movement: Movement = { time, panX, panY, scale, docId }
+
+        // add that movement to the current presentation data's movement array
+        this.currentPresentation.movements && this.currentPresentation.movements.push(movement)
+    }
+
+
+    // method that concatenates an array of presentatations into one
+    public concatPresentations = (presentations: Presentation[]): Presentation => {
+        // these three will lead to the combined presentation
+        let combinedMovements: Movement[] = [];
+        let sumTime = 0;
+        let combinedMetas: any[] = [];
+
+        presentations.forEach((presentation) => {
+            const { movements, totalTime, meta } = presentation;
+
+            // update movements if they had one
+            if (movements) {
+                // add the summed time to the movements
+                const addedTimeMovements = movements.map(move => { return { ...move, time: move.time + sumTime } });
+                // concat the movements already in the combined presentation with these new ones
+                combinedMovements.push(...addedTimeMovements);
+            }
+
+            // update the totalTime
+            sumTime += totalTime;
+
+            // concatenate the metas
+            combinedMetas.push(meta);
+        });
+
+        // return the combined presentation with the updated total summed time
+		return { movements: combinedMovements, totalTime: sumTime, meta: combinedMetas };
+    }
+}
diff --git a/src/client/views/Main.tsx b/src/client/views/Main.tsx
index 517fe097c..c1b67ba19 100644
--- a/src/client/views/Main.tsx
+++ b/src/client/views/Main.tsx
@@ -8,7 +8,8 @@ import { AssignAllExtensions } from "../../extensions/General/Extensions";
 import { Docs } from "../documents/Documents";
 import { CurrentUserUtils } from "../util/CurrentUserUtils";
 import { LinkManager } from "../util/LinkManager";
-import { RecordingApi } from "../util/RecordingApi";
+import { ReplayMovements } from '../util/ReplayMovements';
+import { TrackMovements } from "../util/TrackMovements";
 import { CollectionView } from "./collections/CollectionView";
 import { MainView } from "./MainView";
 
@@ -37,6 +38,7 @@ AssignAllExtensions();
     const expires = "expires=" + d.toUTCString();
     document.cookie = `loadtime=${loading};${expires};path=/`;
     new LinkManager();
-    new RecordingApi;
+    new TrackMovements();
+    new ReplayMovements();
     ReactDOM.render(<MainView />, document.getElementById('root'));
 })();
\ No newline at end of file
diff --git a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
index 7db3b1482..a661cebb8 100644
--- a/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
+++ b/src/client/views/collections/collectionFreeForm/CollectionFreeFormView.tsx
@@ -58,7 +58,7 @@ import React = require("react");
 import { FieldView, FieldViewProps } from "../../nodes/FieldView";
 import { InkTranscription } from "../../InkTranscription";
 import e = require("connect-flash");
-import { RecordingApi } from "../../../util/RecordingApi";
+import { ReplayMovements } from "../../../util/ReplayMovements";
 
 export const panZoomSchema = createSchema({
     _panX: "number",
@@ -1005,20 +1005,8 @@ export class CollectionFreeFormView extends CollectionSubView<Partial<collection
 
     @action
     setPan(panX: number, panY: number, panTime: number = 0, clamp: boolean = false) {
-        // set the current respective FFview to the tab being panned.
-        // if (Doc.UserDoc()?.presentationMode === 'recording') {
-        //     // RecordingApi.Instance.setRecordingFFView(this);
-        //     console.log('setRecordingFFView', this);
-        // }
-        // TODO: make this based off the specific recording FFView
-        // (Doc.UserDoc()?.presentationMode === 'none' || Doc.UserDoc()?.presentationMode === 'watching') && RecordingApi.Instance.setPlayFFView(this);
-        // if (Doc.UserDoc()?.presentationMode === 'watching') {
-        //     RecordingApi.Instance.pauseVideoAndMovements();
-        //     Doc.UserDoc().presentationMode = 'none';
-        //     // RecordingApi.Instance.pauseMovements()
-        // }
         // this is the easiest way to do this -> will talk with Bob about using mobx to do this to remove this line of code.
-        if (Doc.UserDoc()?.presentationMode === 'watching') RecordingApi.Instance.pauseFromInteraction();
+        if (Doc.UserDoc()?.presentationMode === 'watching') ReplayMovements.Instance.pauseFromInteraction();
 
         if (!this.isAnnotationOverlay && clamp) {
             // this section wraps the pan position, horizontally and/or vertically whenever the content is panned out of the viewing bounds
diff --git a/src/client/views/nodes/RecordingBox/RecordingBox.tsx b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
index a28677525..0ff7c4292 100644
--- a/src/client/views/nodes/RecordingBox/RecordingBox.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingBox.tsx
@@ -8,7 +8,7 @@ import { FieldView } from "../FieldView";
 import { VideoBox } from "../VideoBox";
 import { RecordingView } from './RecordingView';
 import { DocumentType } from "../../../documents/DocumentTypes";
-import { Presentation } from "../../../util/RecordingApi";
+import { Presentation } from "../../../util/TrackMovements";
 import { Doc } from "../../../../fields/Doc";
 import { Id } from "../../../../fields/FieldSymbols";
 
diff --git a/src/client/views/nodes/RecordingBox/RecordingView.tsx b/src/client/views/nodes/RecordingBox/RecordingView.tsx
index ec9838bdd..83ed6914e 100644
--- a/src/client/views/nodes/RecordingBox/RecordingView.tsx
+++ b/src/client/views/nodes/RecordingBox/RecordingView.tsx
@@ -8,7 +8,7 @@ import { IconContext } from "react-icons";
 import { Networking } from '../../../Network';
 import { Upload } from '../../../../server/SharedMediaTypes';
 import { returnFalse, returnTrue, setupMoveUpEvents } from '../../../../Utils';
-import { Presentation, RecordingApi } from '../../../util/RecordingApi';
+import { Presentation, TrackMovements } from '../../../util/TrackMovements';
 
 export interface MediaSegment {
     videoChunks: any[],
@@ -64,7 +64,7 @@ export function RecordingView(props: IRecordingViewProps) {
     useEffect(() => {
         if (finished) {
             // make the total presentation that'll match the concatted video
-            let concatPres = trackScreen && RecordingApi.Instance.concatPresentations(videos.map(v => v.presentation as Presentation));
+            let concatPres = trackScreen && TrackMovements.Instance.concatPresentations(videos.map(v => v.presentation as Presentation));
 
             // this async function uses the server to create the concatted video and then sets the result to it's accessPaths
             (async () => {
@@ -135,7 +135,7 @@ export function RecordingView(props: IRecordingViewProps) {
         videoRecorder.current.onstart = (event: any) => {
             setRecording(true);
             // start the recording api when the video recorder starts
-            trackScreen && RecordingApi.Instance.start();
+            trackScreen && TrackMovements.Instance.start();
         };
 
         videoRecorder.current.onstop = () => {
@@ -149,7 +149,7 @@ export function RecordingView(props: IRecordingViewProps) {
                 };
 
                 // depending on if a presenation exists, add it to the video
-                const presentation = RecordingApi.Instance.yieldPresentation();
+                const presentation = TrackMovements.Instance.yieldPresentation();
                 setVideos(videos => [...videos, (presentation != null && trackScreen) ? { ...nextVideo, presentation } : nextVideo]);
             }
 
@@ -174,7 +174,7 @@ export function RecordingView(props: IRecordingViewProps) {
         stream instanceof MediaStream && stream.getTracks().forEach(track => track.stop());
 
         // finish/clear the recoringApi
-        RecordingApi.Instance.finish();
+        TrackMovements.Instance.finish();
 
         // this will call upon progessbar to update videos to be in the correct order
         setFinished(true);
diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx
index 34df03954..5a221fea4 100644
--- a/src/client/views/nodes/VideoBox.tsx
+++ b/src/client/views/nodes/VideoBox.tsx
@@ -28,9 +28,9 @@ import { AnchorMenu } from "../pdf/AnchorMenu";
 import { StyleProp } from "../StyleProvider";
 import { FieldView, FieldViewProps } from './FieldView';
 import "./VideoBox.scss";
-import { Presentation, RecordingApi } from "../../util/RecordingApi";
-import { List } from "../../../fields/List";
+import { Presentation } from "../../util/TrackMovements";
 import { RecordingBox } from "./RecordingBox";
+import { ReplayMovements } from "../../util/ReplayMovements";
 const path = require('path');
 
 
@@ -148,7 +148,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
           
           // if presentation data exists, pass it to the recordingPi
           if (this.presentation != null) {
-            RecordingApi.Instance.setVideoBox(this);  
+            ReplayMovements.Instance.setVideoBox(this);  
           }
       }
 
@@ -159,7 +159,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent<ViewBoxAnnotatableProp
           
           // dispose the recordingApi's observer
           if (this.presentation != null) {
-            RecordingApi.Instance.removeVideoBox();  
+            ReplayMovements.Instance.removeVideoBox();  
           }
       }
 
-- 
cgit v1.2.3-70-g09d2

