several fixes to audio to better support generic timecode linking

6 files changed, 67 insertions, 48 deletions

diff --git a/src/client/documents/Documents.ts b/src/client/documents/Documents.ts
index 5ae4ca82a..cad417d33 100644
--- a/src/client/documents/Documents.ts
+++ b/src/client/documents/Documents.ts
@@ -84,7 +84,7 @@ export interface DocumentOptions {
     columnWidth?: number;
     fontSize?: number;
     curPage?: number;
-    currentTimecode?: number; // the current timecode of a time-based document (e.g., current time of a video)
+    currentTimecode?: number; // the current timecode of a time-based document (e.g., current time of a video)  value is in seconds
     displayTimecode?: number; // the time that a document should be displayed (e.g., time an annotation should be displayed on a video)
     documentText?: string;
     borderRounding?: string;
@@ -104,7 +104,7 @@ export interface DocumentOptions {
     gridGap?: number; // gap between items in masonry view
     xMargin?: number; // gap between left edge of document and start of masonry/stacking layouts
     yMargin?: number; // gap between top edge of dcoument and start of masonry/stacking layouts
-    panel?: Doc; // panel to display in 'targetContainer' as the result of a button onClick script
+    sourcePanel?: Doc; // panel to display in 'targetContainer' as the result of a button onClick script
     targetContainer?: Doc; // document whose proto will be set to 'panel' as the result of a onClick click script
     dropConverter?: ScriptField; // script to run when documents are dropped on this Document.
     // [key: string]: Opt<Field>;
diff --git a/src/client/views/nodes/AudioBox.tsx b/src/client/views/nodes/AudioBox.tsx
index 62ec683da..135874400 100644
--- a/src/client/views/nodes/AudioBox.tsx
+++ b/src/client/views/nodes/AudioBox.tsx
@@ -2,7 +2,7 @@ import React = require("react");
 import { FieldViewProps, FieldView } from './FieldView';
 import { observer } from "mobx-react";
 import "./AudioBox.scss";
-import { Cast, DateCast } from "../../../new_fields/Types";
+import { Cast, DateCast, NumCast } from "../../../new_fields/Types";
 import { AudioField, nullAudio } from "../../../new_fields/URLField";
 import { DocExtendableComponent } from "../DocComponent";
 import { makeInterface, createSchema } from "../../../new_fields/Schema";
@@ -41,33 +41,33 @@ export class AudioBox extends DocExtendableComponent<FieldViewProps, AudioDocume
     _scrubbingDisposer: IReactionDisposer | undefined;
     _ele: HTMLAudioElement | null = null;
     _recorder: any;
+    _recordStart = 0;
     _lastUpdate = 0;
 
-    @observable private _audioState = 0;
+    @observable private _audioState: "unrecorded" | "recording" | "recorded" = "unrecorded";
     @observable public static ScrubTime = 0;
     public static ActiveRecordings: Doc[] = [];
 
     componentDidMount() {
-        runInAction(() => this._audioState = this.path ? 2 : 0);
+        runInAction(() => this._audioState = this.path ? "recorded" : "unrecorded");
         this._linkPlayDisposer = reaction(() => this.layoutDoc.scrollToLinkID,
             scrollLinkId => {
-                scrollLinkId && DocListCast(this.dataDoc.links).map(l => {
-                    const la1 = l.anchor1 as Doc;
-                    const la2 = l.anchor2 as Doc;
-                    if (l[Id] === scrollLinkId && la1 && la2) {
-                        let doc = Doc.AreProtosEqual(la1, this.dataDoc) ? la2 : la1;
-                        let seek = DateCast(la1.creationTime);
-                        setTimeout(() => this.playFrom(seek.date.getTime()), 250);
-                    }
+                scrollLinkId && DocListCast(this.dataDoc.links).filter(l => l[Id] === scrollLinkId).map(l => {
+                    let la1 = l.anchor1 as Doc;
+                    let linkTime = Doc.AreProtosEqual(la1, this.dataDoc) ? NumCast(l.anchor1Timecode) : NumCast(l.anchor2Timecode);
+                    setTimeout(() => this.playFrom(linkTime), 250);
                 });
-                scrollLinkId && (this.layoutDoc.scrollLinkID = undefined);
+                scrollLinkId && Doc.SetInPlace(this.layoutDoc, "scrollToLinkID", undefined, false);
             }, { fireImmediately: true });
         this._reactionDisposer = reaction(() => SelectionManager.SelectedDocuments(),
             selected => {
                 let sel = selected.length ? selected[0].props.Document : undefined;
                 this.Document.playOnSelect && sel && !Doc.AreProtosEqual(sel, this.props.Document) && this.playFrom(DateCast(sel.creationTime).date.getTime());
             });
-        this._scrubbingDisposer = reaction(() => AudioBox.ScrubTime, time => this.Document.playOnSelect && this.playFrom(time));
+        this._scrubbingDisposer = reaction(() => AudioBox.ScrubTime, timeInMillisecondsFrom1970 => {
+            let start = this.extensionDoc && DateCast(this.extensionDoc.recordingStart);
+            start && this.playFrom((timeInMillisecondsFrom1970 - start.date.getTime()) / 1000);
+        });
     }
 
     updateHighlights = () => {
@@ -76,14 +76,15 @@ export class AudioBox extends DocExtendableComponent<FieldViewProps, AudioDocume
         const start = extensionDoc && DateCast(extensionDoc.recordingStart);
         if (htmlEle && !htmlEle.paused && start) {
             setTimeout(this.updateHighlights, 30);
+            this.Document.currentTimecode = htmlEle.currentTime;
             DocListCast(this.dataDoc.links).map(l => {
                 let la1 = l.anchor1 as Doc;
+                let linkTime = NumCast(l.anchor2Timecode);
                 if (Doc.AreProtosEqual(la1, this.dataDoc)) {
                     la1 = l.anchor2 as Doc;
+                    linkTime = NumCast(l.anchor1Timecode);
                 }
-                let date = DateCast(la1.creationDate);
-                let offset = (date!.date.getTime() - start.date.getTime()) / 1000;
-                if (offset > this._lastUpdate && offset < htmlEle.currentTime) {
+                if (linkTime > this._lastUpdate && linkTime < htmlEle.currentTime) {
                     Doc.linkFollowHighlight(la1);
                 }
             });
@@ -91,16 +92,15 @@ export class AudioBox extends DocExtendableComponent<FieldViewProps, AudioDocume
         }
     }
 
-    playFrom = (seek: number) => {
+    playFrom = (seekTimeInSeconds: number) => {
         const extensionDoc = this.extensionDoc;
         let start = extensionDoc && DateCast(extensionDoc.recordingStart);
         if (this._ele && start) {
-            if (seek) {
-                let delta = (seek - start.date.getTime()) / 1000;
-                if (start && delta > 0 && delta < this._ele.duration) {
-                    this._ele.currentTime = delta;
+            if (seekTimeInSeconds) {
+                if (seekTimeInSeconds >= 0 && seekTimeInSeconds <= this._ele.duration) {
+                    this._ele.currentTime = seekTimeInSeconds;
                     this._ele.play();
-                    this._lastUpdate = delta;
+                    this._lastUpdate = seekTimeInSeconds;
                     setTimeout(this.updateHighlights, 0);
                 } else {
                     this._ele.pause();
@@ -117,6 +117,14 @@ export class AudioBox extends DocExtendableComponent<FieldViewProps, AudioDocume
         this._scrubbingDisposer && this._scrubbingDisposer();
     }
 
+
+    updateRecordTime = () => {
+        if (this._audioState === "recording") {
+            setTimeout(this.updateRecordTime, 30);
+            this.Document.currentTimecode = (new Date().getTime() - this._recordStart) / 1000;
+        }
+    }
+
     recordAudioAnnotation = () => {
         let gumStream: any;
         let self = this;
@@ -140,7 +148,9 @@ export class AudioBox extends DocExtendableComponent<FieldViewProps, AudioDocume
                 // upload to server with known URL 
                 self.props.Document[self.props.fieldKey] = new AudioField(url);
             };
-            runInAction(() => self._audioState = 1);
+            runInAction(() => self._audioState = "recording");
+            self._recordStart = new Date().getTime();
+            setTimeout(self.updateRecordTime, 0);
             self._recorder.start();
             setTimeout(() => {
                 self.stopRecording();
@@ -158,7 +168,7 @@ export class AudioBox extends DocExtendableComponent<FieldViewProps, AudioDocume
 
     stopRecording = action(() => {
         this._recorder.stop();
-        this._audioState = 2;
+        this._audioState = "recorded";
         let ind = AudioBox.ActiveRecordings.indexOf(this.props.Document);
         ind !== -1 && (AudioBox.ActiveRecordings.splice(ind, 1));
     });
@@ -174,6 +184,7 @@ export class AudioBox extends DocExtendableComponent<FieldViewProps, AudioDocume
 
     setRef = (e: HTMLAudioElement | null) => {
         e && e.addEventListener("play", this.playClick as any);
+        e && e.addEventListener("timeupdate", () => this.props.Document.currentTimecode = e!.currentTime);
         this._ele = e;
     }
 
@@ -197,8 +208,8 @@ export class AudioBox extends DocExtendableComponent<FieldViewProps, AudioDocume
             <div className={`audiobox-container`} onContextMenu={this.specificContextMenu} onClick={!this.path ? this.recordClick : undefined}>
                 <div className="audiobox-handle"></div>
                 {!this.path ?
-                    <button className={`audiobox-record${interactive}`} style={{ backgroundColor: ["black", "red", "blue"][this._audioState] }}>
-                        {this._audioState === 1 ? "STOP" : "RECORD"}
+                    <button className={`audiobox-record${interactive}`} style={{ backgroundColor: this._audioState === "recording" ? "red" : "black" }}>
+                        {this._audioState === "recording" ? "STOP" : "RECORD"}
                     </button> :
                     this.audio
                 }
diff --git a/src/client/views/nodes/DocumentView.tsx b/src/client/views/nodes/DocumentView.tsx
index 67c85e158..c82fd0f0f 100644
--- a/src/client/views/nodes/DocumentView.tsx
+++ b/src/client/views/nodes/DocumentView.tsx
@@ -165,7 +165,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                 Doc.UnBrushDoc(this.props.Document);
             } else if (this.onClickHandler && this.onClickHandler.script) {
                 this.onClickHandler.script.run({ this: this.Document.isTemplateField && this.props.DataDoc ? this.props.DataDoc : this.props.Document }, console.log);
-            } else if (this.props.Document.type === DocumentType.BUTTON) {
+            } else if (this.Document.type === DocumentType.BUTTON) {
                 ScriptBox.EditButtonScript("On Button Clicked ...", this.props.Document, "onClick", e.clientX, e.clientY);
             } else if (this.Document.isButton) {
                 SelectionManager.SelectDoc(this, e.ctrlKey); // don't think this should happen if a button action is actually triggered.
@@ -179,8 +179,8 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     }
 
     buttonClick = async (altKey: boolean, ctrlKey: boolean) => {
-        let maximizedDocs = await DocListCastAsync(this.props.Document.maximizedDocs);
-        let summarizedDocs = await DocListCastAsync(this.props.Document.summarizedDocs);
+        let maximizedDocs = await DocListCastAsync(this.Document.maximizedDocs);
+        let summarizedDocs = await DocListCastAsync(this.Document.summarizedDocs);
         let linkDocs = LinkManager.Instance.getAllRelatedLinks(this.props.Document);
         let expandedDocs: Doc[] = [];
         expandedDocs = maximizedDocs ? [...maximizedDocs, ...expandedDocs] : expandedDocs;
@@ -333,9 +333,9 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     onDrop = (e: React.DragEvent) => {
         let text = e.dataTransfer.getData("text/plain");
         if (!e.isDefaultPrevented() && text && text.startsWith("<div")) {
-            let oldLayout = StrCast(this.props.Document.layout);
+            let oldLayout = this.Document.layout || "";
             let layout = text.replace("{layout}", oldLayout);
-            this.props.Document.layout = layout;
+            this.Document.layout = layout;
             e.stopPropagation();
             e.preventDefault();
         }
@@ -355,7 +355,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
     @undoBatch
     @action
     makeIntoPortal = async () => {
-        let anchors = await Promise.all(DocListCast(this.props.Document.links).map(async (d: Doc) => Cast(d.anchor2, Doc)));
+        let anchors = await Promise.all(DocListCast(this.Document.links).map(async (d: Doc) => Cast(d.anchor2, Doc)));
         if (!anchors.find(anchor2 => anchor2 && anchor2.title === this.Document.title + ".portal" ? true : false)) {
             let portalID = (this.Document.title + ".portal").replace(/^-/, "").replace(/\([0-9]*\)$/, "");
             DocServer.GetRefField(portalID).then(existingPortal => {
@@ -590,7 +590,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
 
     render() {
         if (!this.props.Document) return (null);
-        let animDims = this.props.Document.animateToDimensions ? Array.from(Cast(this.props.Document.animateToDimensions, listSpec("number"))!) : undefined;
+        const animDims = this.Document.animateToDimensions ? Array.from(this.Document.animateToDimensions) : undefined;
         const ruleColor = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleColor_" + this.Document.heading]) : undefined;
         const ruleRounding = this.props.ruleProvider ? StrCast(this.props.ruleProvider["ruleRounding_" + this.Document.heading]) : undefined;
         const colorSet = this.setsLayoutProp("backgroundColor");
@@ -644,7 +644,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
             <div className={`documentView-node${this.topMost ? "-topmost" : ""}`}
                 ref={this._mainCont}
                 style={{
-                    transition: this.props.Document.isAnimating !== undefined ? ".5s linear" : StrCast(this.Document.transition),
+                    transition: this.Document.isAnimating !== undefined ? ".5s linear" : StrCast(this.Document.transition),
                     pointerEvents: this.Document.isBackground && !this.isSelected() ? "none" : "all",
                     color: StrCast(this.Document.color),
                     outline: highlighting && !borderRounding ? `${highlightColors[fullDegree]} ${highlightStyles[fullDegree]} ${localScale}px` : "solid 0px",
@@ -658,7 +658,7 @@ export class DocumentView extends DocComponent<DocumentViewProps, Document>(Docu
                 onDrop={this.onDrop} onContextMenu={this.onContextMenu} onPointerDown={this.onPointerDown} onClick={this.onClick}
                 onPointerEnter={() => Doc.BrushDoc(this.props.Document)} onPointerLeave={() => Doc.UnBrushDoc(this.props.Document)}
             >
-                {this.props.Document.links && DocListCast(this.props.Document.links).map((d, i) =>
+                {this.Document.links && DocListCast(this.Document.links).map((d, i) =>
                     //this.linkEndpointDoc(d).type === DocumentType.PDFANNO ? (null) :
                     <div style={{ pointerEvents: "none", position: "absolute", transformOrigin: "top left", width: "100%", height: "100%", transform: `scale(${this.layoutDoc.fitWidth ? 1 : 1 / this.props.ContentScaling()})` }}>
                         <DocumentView {...this.props} backgroundColor={returnTransparent} Document={d} layoutKey={this.linkEndpoint(d)} />
diff --git a/src/client/views/nodes/FormattedTextBox.tsx b/src/client/views/nodes/FormattedTextBox.tsx
index 5c2d39d98..5588bb4c9 100644
--- a/src/client/views/nodes/FormattedTextBox.tsx
+++ b/src/client/views/nodes/FormattedTextBox.tsx
@@ -605,7 +605,7 @@ export class FormattedTextBox extends DocExtendableComponent<(FieldViewProps & F
                         setTimeout(() => editor.dispatch(editor.state.tr.addMark(selection.from, selection.to, mark)), 0);
                         setTimeout(() => this.unhighlightSearchTerms(), 2000);
                     }
-                    this.layoutDoc.scrollToLinkID = undefined;
+                    Doc.SetInPlace(this.layoutDoc, "scrollToLinkID", undefined, false);
                 }
 
             },
diff --git a/src/new_fields/documentSchemas.ts b/src/new_fields/documentSchemas.ts
index 8c3b62067..e2730914f 100644
--- a/src/new_fields/documentSchemas.ts
+++ b/src/new_fields/documentSchemas.ts
@@ -1,9 +1,10 @@
 import { makeInterface, createSchema, listSpec } from "./Schema";
 import { ScriptField } from "./ScriptField";
 import { Doc } from "./Doc";
+import { DateField } from "./DateField";
 
 export const documentSchema = createSchema({
-    // layout: "string", // this should be a "string" or Doc, but can't do that in schemas, so best to leave it out
+    layout: "string",           // this is the native layout string for the document.  templates can be added using other fields and setting layoutKey below (see layoutCustom as an example)
     layoutKey: "string",        // holds the field key for the field that actually holds the current lyoat
     layoutCustom: Doc,          // used to hold a custom layout (there's nothing special about this field .. any field could hold a custom layout that can be selected by setting 'layoutKey')
     title: "string",            // document title (can be on either data document or layout)
@@ -14,7 +15,8 @@ export const documentSchema = createSchema({
     color: "string",            // foreground color of document
     backgroundColor: "string",  // background color of document
     opacity: "number",          // opacity of document
-    //links: listSpec(Doc),       // computed (readonly) list of links associated with this document
+    creationDate: DateField,     // when the document was created
+    links: listSpec(Doc),       // computed (readonly) list of links associated with this document
     dropAction: "string",       // override specifying what should happen when this document is dropped (can be "alias" or "copy")
     removeDropProperties: listSpec("string"), // properties that should be removed from the alias/copy/etc of this document when it is dropped
     onClick: ScriptField,       // script to run when document is clicked (can be overriden by an onClick prop)
@@ -25,6 +27,9 @@ export const documentSchema = createSchema({
     isTemplateField: "boolean", // whether this document acts as a template layout for describing how other documents should be displayed
     isBackground: "boolean",    // whether document is a background element and ignores input events (can only selet with marquee)
     type: "string",             // enumerated type of document
+    currentTimecode: "number",   // current play back time of a temporal document (video / audio)
+    summarizedDocs: listSpec(Doc), // documents that are summarized by this document (and which will typically be opened by clicking this document)
+    maximizedDocs: listSpec(Doc), // documents to maximize when clicking this document (generally this document will be an icon)
     maximizeLocation: "string", // flag for where to place content when following a click interaction (e.g., onRight, inPlace, inTab) 
     lockedPosition: "boolean",  // whether the document can be spatially manipulated
     inOverlay: "boolean",       // whether the document is rendered in an OverlayView which handles selection/dragging differently
@@ -33,8 +38,11 @@ export const documentSchema = createSchema({
     heading: "number",          // the logical layout 'heading' of this document (used by rule provider to stylize h1 header elements, from h2, etc)
     showCaption: "string",      // whether editable caption text is overlayed at the bottom of the document 
     showTitle: "string",        // whether an editable title banner is displayed at tht top of the document
-    isButton: "boolean",        // whether document functions as a button (overiding native interactions of its content)      
+    isButton: "boolean",        // whether document functions as a button (overiding native interactions of its content)    
     ignoreClick: "boolean",     // whether documents ignores input clicks (but does not ignore manipulation and other events) 
+    isAnimating: "boolean",     // whether the document is in the midst of animating between two layouts (used by icons to de/iconify documents).
+    animateToDimensions: listSpec("number"), // layout information about the target rectangle a document is animating towards 
+    scrollToLinkID: "string",   // id of link being traversed. allows this doc to scroll/highlight/etc its link anchor. scrollToLinkID should be set to undefined by this doc after it sets up its scroll,etc.
 });
 
 export const positionSchema = createSchema({
diff --git a/src/server/authentication/models/current_user_utils.ts b/src/server/authentication/models/current_user_utils.ts
index 95ebe3cb6..044ec746b 100644
--- a/src/server/authentication/models/current_user_utils.ts
+++ b/src/server/authentication/models/current_user_utils.ts
@@ -83,10 +83,10 @@ export class CurrentUserUtils {
 
         return Docs.Create.ButtonDocument({
             width: 35, height: 35, borderRounding: "50%", boxShadow: "2px 2px 1px", title: "Create", targetContainer: sidebarContainer,
-            panel: Docs.Create.StackingDocument([dragCreators, color], {
+            sourcePanel: Docs.Create.StackingDocument([dragCreators, color], {
                 width: 500, height: 800, chromeStatus: "disabled", title: "creator stack"
             }),
-            onClick: ScriptField.MakeScript("this.targetContainer.proto = this.panel"),
+            onClick: ScriptField.MakeScript("this.targetContainer.proto = this.sourcePanel"),
         });
     }
 
@@ -108,11 +108,11 @@ export class CurrentUserUtils {
 
         return Docs.Create.ButtonDocument({
             width: 50, height: 35, borderRounding: "50%", boxShadow: "2px 2px 1px", title: "Library",
-            panel: Docs.Create.TreeDocument([doc.workspaces as Doc, doc.documents as Doc, doc.recentlyClosed as Doc], {
+            sourcePanel: Docs.Create.TreeDocument([doc.workspaces as Doc, doc.documents as Doc, doc.recentlyClosed as Doc], {
                 title: "Library", xMargin: 5, yMargin: 5, gridGap: 5, forceActive: true, dropAction: "alias", lockedPosition: true
             }),
             targetContainer: sidebarContainer,
-            onClick: ScriptField.MakeScript("this.targetContainer.proto = this.panel")
+            onClick: ScriptField.MakeScript("this.targetContainer.proto = this.sourcePanel")
         });
     }
 
@@ -120,11 +120,11 @@ export class CurrentUserUtils {
     static setupSearchPanel(sidebarContainer: Doc) {
         return Docs.Create.ButtonDocument({
             width: 50, height: 35, borderRounding: "50%", boxShadow: "2px 2px 1px", title: "Search",
-            panel: Docs.Create.QueryDocument({
+            sourcePanel: Docs.Create.QueryDocument({
                 title: "search stack", ignoreClick: true
             }),
             targetContainer: sidebarContainer,
-            onClick: ScriptField.MakeScript("this.targetContainer.proto = this.panel")
+            onClick: ScriptField.MakeScript("this.targetContainer.proto = this.sourcePanel")
         });
     }
 
@@ -190,7 +190,7 @@ export class CurrentUserUtils {
             stackingDoc && PromiseValue(Cast(stackingDoc.data, listSpec(Doc))).then(sidebarButtons => {
                 sidebarButtons && sidebarButtons.map((sidebarBtn, i) => {
                     sidebarBtn && PromiseValue(Cast(sidebarBtn, Doc)).then(async btn => {
-                        btn && btn.panel && btn.targetContainer && i === 1 && (btn.onClick as ScriptField).script.run({ this: btn });
+                        btn && btn.sourcePanel && btn.targetContainer && i === 1 && (btn.onClick as ScriptField).script.run({ this: btn });
                     });
                 });
             });