feat: pipeline improvements

packages/collection-utils/src/map-async-limit.js

@@ -0,0 +1,42 @@
+/**
+ * Copyright (C) 2023 StApps
+ *  This program is free software: you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the Free
+ *  Software Foundation, version 3.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ *  more details.
+ *
+ *  You should have received a copy of the GNU General Public License along with
+ *  this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Runs async tasks in parallel, with a limit
+ *
+ * Note: JavaScript is not multithreaded.
+ * This will not let you run tasks in parallel, use it only to limit how many network requests should be
+ * running at once.
+ * @template T
+ * @template U
+ * @param items {T[]} the items to iterate through
+ * @param task {(item: T, index: number) => Promise<U>} the task to be run
+ * @param [limit] {number} the maximum number of tasks that should be run asynchronously
+ * @returns {Promise<U[]>}
+ */
+export async function mapAsyncLimit(items, task, limit = 5) {
+  return Promise.all(
+    Array.from({length: limit}).map(async () => {
+      let i = 0;
+      /** @type {U[]} */
+      const results = [];
+      for (let item = items.shift(); item !== undefined; item = items.shift()) {
+        results.push(await task(item, i));
+        i++;
+      }
+      return results;
+    }),
+  ).then(it => it.flat());
+}