From 9c2ffb3e7319c5592b6f210273f5373319daca4b Mon Sep 17 00:00:00 2001 From: bobzel Date: Sun, 21 Mar 2021 15:52:58 -0400 Subject: added an audio track to video players to allow screen grabs to grab audio and join it with the video. --- src/client/views/nodes/ScreenshotBox.tsx | 47 ++++++++++++++++++++------------ src/client/views/nodes/VideoBox.tsx | 23 ++++++++++++++-- 2 files changed, 49 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/client/views/nodes/ScreenshotBox.tsx b/src/client/views/nodes/ScreenshotBox.tsx index 9ad93d4dd..403fb3573 100644 --- a/src/client/views/nodes/ScreenshotBox.tsx +++ b/src/client/views/nodes/ScreenshotBox.tsx @@ -8,7 +8,7 @@ import { documentSchema } from "../../../fields/documentSchemas"; import { InkTool } from "../../../fields/InkField"; import { listSpec, makeInterface } from "../../../fields/Schema"; import { Cast, NumCast } from "../../../fields/Types"; -import { VideoField } from "../../../fields/URLField"; +import { VideoField, AudioField } from "../../../fields/URLField"; import { emptyFunction, returnFalse, returnOne, returnZero, Utils, OmitKeys } from "../../../Utils"; import { Docs, DocUtils } from "../../documents/Documents"; import { CollectionFreeFormView } from "../collections/collectionFreeForm/CollectionFreeFormView"; @@ -47,7 +47,7 @@ export class ScreenshotBox extends ViewBoxAnnotatableComponent { return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow" /* audioStart */, "_timecodeToHide" /* audioEnd */, Cast(this.layoutDoc._currentTimecode, "number", null) || - (this._recorder ? (Date.now() - (this.recordingStart || 0)) / 1000 : undefined)) + (this._vrecorder ? (Date.now() - (this.recordingStart || 0)) / 1000 : undefined)) || this.rootDoc; } @@ -157,34 +157,45 @@ export class ScreenshotBox extends ViewBoxAnnotatableComponent; } - _chunks: any; - _recorder: any; + _vchunks: any; + _achunks: any; + _vrecorder: any; + _arecorder: any; toggleRecording = action(async () => { this._screenCapture = !this._screenCapture; if (this._screenCapture) { - const stream = !this._screenCapture ? undefined : await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); - this._videoRef!.srcObject = stream; - this._recorder = new MediaRecorder(stream); this.dataDoc[this.props.fieldKey + "-recordingStart"] = new DateField(new Date()); - this._chunks = []; - this._recorder.ondataavailable = (e: any) => this._chunks.push(e.data); - this._recorder.onstop = async (e: any) => { - const file = new File(this._chunks, `${this.rootDoc[Id]}.mkv`, { type: this._chunks[0].type, lastModified: Date.now() }); - const completeBlob = new Blob(this._chunks, { type: this._chunks[0].type }); - (completeBlob as any).lastModifiedDate = new Date(); - (completeBlob as any).name = `${this.rootDoc[Id]}.mkv`; - const [{ result }] = await Networking.UploadFilesToServer(file);//completeBlob as File); + this._arecorder = new MediaRecorder(await navigator.mediaDevices.getUserMedia({ audio: true })); + this._achunks = []; + this._arecorder.ondataavailable = (e: any) => this._achunks.push(e.data); + this._arecorder.onstop = async (e: any) => { + const [{ result }] = await Networking.UploadFilesToServer(this._achunks); + if (!(result instanceof Error)) { + this.dataDoc[this.props.fieldKey + "-audio"] = new AudioField(Utils.prepend(result.accessPaths.agnostic.client)); + } + }; + const vstream = await (navigator.mediaDevices as any).getDisplayMedia({ video: true }); + this._videoRef!.srcObject = vstream; + this._vrecorder = new MediaRecorder(vstream); + this._vchunks = []; + this._vrecorder.ondataavailable = (e: any) => this._vchunks.push(e.data); + this._vrecorder.onstop = async (e: any) => { + const file = new File(this._vchunks, `${this.rootDoc[Id]}.mkv`, { type: this._vchunks[0].type, lastModified: Date.now() }); + const [{ result }] = await Networking.UploadFilesToServer(file); + this.dataDoc[this.fieldKey + "-duration"] = (new Date().getTime() - this.recordingStart!) / 1000; if (!(result instanceof Error)) { this.dataDoc.type = DocumentType.VID; this.layoutDoc.layout = VideoBox.LayoutString(this.fieldKey); this.dataDoc[this.props.fieldKey] = new VideoField(Utils.prepend(result.accessPaths.agnostic.client)); } else alert("video conversion failed"); }; - this._recorder.start(); + this._arecorder.start(); + this._vrecorder.start(); DocUtils.ActiveRecordings.push(this); } else { - this._recorder.stop(); + this._arecorder.stop(); + this._vrecorder.stop(); const ind = DocUtils.ActiveRecordings.indexOf(this); ind !== -1 && (DocUtils.ActiveRecordings.splice(ind, 1)); } @@ -195,7 +206,7 @@ export class ScreenshotBox extends ViewBoxAnnotatableComponent -
+
); diff --git a/src/client/views/nodes/VideoBox.tsx b/src/client/views/nodes/VideoBox.tsx index 0bded738a..d475ab9f2 100644 --- a/src/client/views/nodes/VideoBox.tsx +++ b/src/client/views/nodes/VideoBox.tsx @@ -9,7 +9,7 @@ import { documentSchema } from "../../../fields/documentSchemas"; import { InkTool } from "../../../fields/InkField"; import { makeInterface } from "../../../fields/Schema"; import { Cast, NumCast, StrCast } from "../../../fields/Types"; -import { VideoField } from "../../../fields/URLField"; +import { VideoField, AudioField, nullAudio } from "../../../fields/URLField"; import { emptyFunction, formatTime, OmitKeys, returnOne, setupMoveUpEvents, Utils } from "../../../Utils"; import { Docs, DocUtils } from "../../documents/Documents"; import { Networking } from "../../Network"; @@ -72,8 +72,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent { const timecode = Cast(this.layoutDoc._currentTimecode, "number", null); - const anchor = CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow"/* videoStart */, "_timecodeToHide" /* videoEnd */, timecode ? timecode : undefined) || this.rootDoc; - return anchor; + return CollectionStackedTimeline.createAnchor(this.rootDoc, this.dataDoc, this.annotationKey, "_timecodeToShow"/* videoStart */, "_timecodeToHide" /* videoEnd */, timecode ? timecode : undefined) || this.rootDoc; } choosePath(url: string) { @@ -92,6 +91,7 @@ export class VideoBox extends ViewBoxAnnotatableComponent { this._playing = false; try { update && this.player?.pause(); + update && this._audioPlayer?.pause(); update && this._youtubePlayer?.pauseVideo(); this._youtubePlayer && this._playTimer && clearInterval(this._playTimer); this._youtubePlayer?.seekTo(this._youtubePlayer?.getCurrentTime(), true); @@ -292,6 +294,15 @@ export class VideoBox extends ViewBoxAnnotatableComponent this._audioPlayer = e; @computed get content() { const field = Cast(this.dataDoc[this.fieldKey], VideoField); const interactive = CurrentUserUtils.SelectedTool !== InkTool.None || !this.props.isSelected() ? "" : "-interactive"; @@ -310,6 +321,10 @@ export class VideoBox extends ViewBoxAnnotatableComponent Not supported. + ; ; } @@ -461,7 +476,9 @@ export class VideoBox extends ViewBoxAnnotatableComponent this._playing = true); if (endTime !== this.duration) { this._playRegionTimer = setTimeout(() => this.Pause(), (this._playRegionDuration) * 1000); // use setTimeout to play a specific duration -- cgit v1.2.3-70-g09d2