From 3f6f0a878b15aa5c569a8b522d1b24915749498e Mon Sep 17 00:00:00 2001
From: Sam Wilkins <samwilkins333@gmail.com>
Date: Tue, 17 Sep 2019 21:57:43 -0400
Subject: batching developments

---
 src/client/northstar/utils/Extensions.ts           | 28 +++++---
 .../util/Import & Export/DirectoryImportBox.tsx    |  2 +-
 src/client/util/UtilExtensions.ts                  | 80 +++++++++++++++++-----
 3 files changed, 79 insertions(+), 31 deletions(-)

(limited to 'src')

diff --git a/src/client/northstar/utils/Extensions.ts b/src/client/northstar/utils/Extensions.ts
index 04af36731..65af7ea87 100644
--- a/src/client/northstar/utils/Extensions.ts
+++ b/src/client/northstar/utils/Extensions.ts
@@ -20,21 +20,27 @@ String.prototype.Truncate = function (length: number, replacement: string): Stri
     return target;
 };
 
-type BatchConverterSync<I, O> = (batch: I[], isFullBatch: boolean) => O[];
-type BatchHandlerSync<I> = (batch: I[], isFullBatch: boolean) => void;
-type BatchConverterAsync<I, O> = (batch: I[], isFullBatch: boolean) => Promise<O[]>;
-type BatchHandlerAsync<I> = (batch: I[], isFullBatch: boolean) => Promise<void>;
+interface BatchContext {
+    completedBatches: number;
+    remainingBatches: number;
+    isFullBatch: boolean;
+}
+type BatchConverterSync<I, O> = (batch: I[], context: BatchContext) => O[];
+type BatchHandlerSync<I> = (batch: I[], context: BatchContext) => void;
+type BatchConverterAsync<I, O> = (batch: I[], context: BatchContext) => Promise<O[]>;
+type BatchHandlerAsync<I> = (batch: I[], context: BatchContext) => Promise<void>;
 type BatchConverter<I, O> = BatchConverterSync<I, O> | BatchConverterAsync<I, O>;
 type BatchHandler<I> = BatchHandlerSync<I> | BatchHandlerAsync<I>;
+type Batcher<I, A> = number | ((element: I, accumulator: A) => boolean | Promise<boolean>);
 
 interface Array<T> {
-    batch(batchSize: number): T[][];
-    batchedForEach(batchSize: number, handler: BatchHandlerSync<T>): void;
-    batchedMap<O>(batchSize: number, handler: BatchConverterSync<T, O>): O[];
-    batchedForEachAsync(batchSize: number, handler: BatchHandler<T>): Promise<void>;
-    batchedMapAsync<O>(batchSize: number, handler: BatchConverter<T, O>): Promise<O[]>;
-    batchedForEachInterval(batchSize: number, handler: BatchHandler<T>, interval: number): Promise<void>;
-    batchedMapInterval<O>(batchSize: number, handler: BatchConverter<T, O>, interval: number): Promise<O[]>;
+    batch(batchSize: undefined): T[][];
+    batchedForEach<A = undefined>(batcher: Batcher<T, A>, handler: BatchHandlerSync<T>): void;
+    batchedMap<O, A = undefined>(batcher: Batcher<T, A>, handler: BatchConverterSync<T, O>): O[];
+    batchedForEachAsync<A = undefined>(batcher: Batcher<T, A>, handler: BatchHandler<T>): Promise<void>;
+    batchedMapAsync<O, A = undefined>(batcher: Batcher<T, A>, handler: BatchConverter<T, O>): Promise<O[]>;
+    batchedForEachInterval<A = undefined>(batcher: Batcher<T, A>, handler: BatchHandler<T>, interval: number): Promise<void>;
+    batchedMapInterval<O, A = undefined>(batcher: Batcher<T, A>, handler: BatchConverter<T, O>, interval: number): Promise<O[]>;
     lastElement(): T;
 }
 
diff --git a/src/client/util/Import & Export/DirectoryImportBox.tsx b/src/client/util/Import & Export/DirectoryImportBox.tsx
index c9d34b594..a86ef9a31 100644
--- a/src/client/util/Import & Export/DirectoryImportBox.tsx	
+++ b/src/client/util/Import & Export/DirectoryImportBox.tsx	
@@ -103,7 +103,7 @@ export default class DirectoryImportBox extends React.Component<FieldViewProps>
 
         runInAction(() => this.phase = `Internal: uploading ${this.quota - this.completed} files to Dash...`);
 
-        const uploads = await validated.batchedMapAsync<FileResponse>(15, async (batch: File[]) => {
+        const uploads = await validated.batchedMapAsync<FileResponse>(15, async batch => {
             const formData = new FormData();
             const parameters = { method: 'POST', body: formData };
 
diff --git a/src/client/util/UtilExtensions.ts b/src/client/util/UtilExtensions.ts
index 0b325bc51..792cd00a1 100644
--- a/src/client/util/UtilExtensions.ts
+++ b/src/client/util/UtilExtensions.ts
@@ -9,11 +9,19 @@ module.exports.Batch = function <T>(batchSize: number): T[][] {
     return batches;
 };
 
-module.exports.ExecuteBatches = function <I, O>(batchSize: number, handler: BatchHandlerSync<I>): void {
+module.exports.ExecuteBatches = function <I>(batchSize: number, handler: BatchHandlerSync<I>): void {
     if (this.length) {
-        for (let batch of this.batch(batchSize)) {
-            const isFullBatch = batch.length === batchSize;
-            handler(batch, isFullBatch);
+        let completed = 0;
+        const batches = this.batch(batchSize);
+        const quota = batches.length;
+        for (let batch of batches) {
+            const context: BatchContext = {
+                completedBatches: completed,
+                remainingBatches: quota - completed,
+                isFullBatch: batch.length === batchSize
+            };
+            handler(batch, context);
+            completed++;
         }
     }
 };
@@ -23,18 +31,34 @@ module.exports.ConvertInBatches = function <I, O>(batchSize: number, handler: Ba
         return [];
     }
     let collector: O[] = [];
-    for (let batch of this.batch(batchSize)) {
-        const isFullBatch = batch.length === batchSize;
-        collector.push(...handler(batch, isFullBatch));
+    let completed = 0;
+    const batches = this.batch(batchSize);
+    const quota = batches.length;
+    for (let batch of batches) {
+        const context: BatchContext = {
+            completedBatches: completed,
+            remainingBatches: quota - completed,
+            isFullBatch: batch.length === batchSize
+        };
+        collector.push(...handler(batch, context));
+        completed++;
     }
     return collector;
 };
 
 module.exports.ExecuteInBatchesAsync = async function <I>(batchSize: number, handler: BatchHandler<I>): Promise<void> {
     if (this.length) {
-        for (let batch of this.batch(batchSize)) {
-            const isFullBatch = batch.length === batchSize;
-            await handler(batch, isFullBatch);
+        let completed = 0;
+        const batches = this.batch(batchSize);
+        const quota = batches.length;
+        for (let batch of batches) {
+            const context: BatchContext = {
+                completedBatches: completed,
+                remainingBatches: quota - completed,
+                isFullBatch: batch.length === batchSize
+            };
+            await handler(batch, context);
+            completed++;
         }
     }
 };
@@ -44,9 +68,17 @@ module.exports.ConvertInBatchesAsync = async function <I, O>(batchSize: number,
         return [];
     }
     let collector: O[] = [];
-    for (let batch of this.batch(batchSize)) {
-        const isFullBatch = batch.length === batchSize;
-        collector.push(...(await handler(batch, isFullBatch)));
+    let completed = 0;
+    const batches = this.batch(batchSize);
+    const quota = batches.length;
+    for (let batch of batches) {
+        const context: BatchContext = {
+            completedBatches: completed,
+            remainingBatches: quota - completed,
+            isFullBatch: batch.length === batchSize
+        };
+        collector.push(...(await handler(batch, context)));
+        completed++;
     }
     return collector;
 };
@@ -56,6 +88,7 @@ module.exports.ExecuteInBatchesAtInterval = async function <I>(batchSize: number
         return;
     }
     const batches = this.batch(batchSize);
+    const quota = batches.length;
     return new Promise<void>(async resolve => {
         const iterator = batches[Symbol.iterator]();
         let completed = 0;
@@ -64,12 +97,16 @@ module.exports.ExecuteInBatchesAtInterval = async function <I>(batchSize: number
             await new Promise<void>(resolve => {
                 setTimeout(async () => {
                     const batch = next.value;
-                    const isFullBatch = batch.length === batchSize;
-                    await handler(batch, isFullBatch);
+                    const context: BatchContext = {
+                        completedBatches: completed,
+                        remainingBatches: quota - completed,
+                        isFullBatch: batch.length === batchSize
+                    };
+                    await handler(batch, context);
                     resolve();
                 }, interval * 1000);
             });
-            if (++completed === batches.length) {
+            if (++completed === quota) {
                 break;
             }
         }
@@ -83,6 +120,7 @@ module.exports.ConvertInBatchesAtInterval = async function <I, O>(batchSize: num
     }
     let collector: O[] = [];
     const batches = this.batch(batchSize);
+    const quota = batches.length;
     return new Promise<O[]>(async resolve => {
         const iterator = batches[Symbol.iterator]();
         let completed = 0;
@@ -91,12 +129,16 @@ module.exports.ConvertInBatchesAtInterval = async function <I, O>(batchSize: num
             await new Promise<void>(resolve => {
                 setTimeout(async () => {
                     const batch = next.value;
-                    const isFullBatch = batch.length === batchSize;
-                    collector.push(...(await handler(batch, isFullBatch)));
+                    const context: BatchContext = {
+                        completedBatches: completed,
+                        remainingBatches: quota - completed,
+                        isFullBatch: batch.length === batchSize
+                    };
+                    collector.push(...(await handler(batch, context)));
                     resolve();
                 }, interval * 1000);
             });
-            if (++completed === batches.length) {
+            if (++completed === quota) {
                 resolve(collector);
                 break;
             }
-- 
cgit v1.2.3-70-g09d2