Process a batch of promises in parallel

promise_utils.ts

/**
 * Processes a batch of items in parallel using a handler function to generate the promises. It starts by initializing
 * the amount of Promises given by the batchSize parameter. Once a Promise finishes it immediately starts the next one.
 * The function returns the results in an array in the order in which the promises finished.
 *
 * The optional onUpdate function is called every time a new Promises starts or is finished
 */
export async function processPromisesParallel<T, K>(
  items: T[],
  batchSize: number,
  handler: (item: T) => Promise<K>,
  onUpdate?: (status: { running: number, done: number }) => void
): Promise<K[]> {
  items = [...items];
  const all: K[] = [];

  let done = 0;
  let running = 0;

  function update() {
    if (onUpdate) onUpdate({ running, done });
  }

  async function executeNext() {
    const item = items.shift();
    if (!item) {
      return;
    }
    running++;
    update();
    const result = await handler(item);
    all.push(result);
    running--;
    done++;
    update();
    if (items.length > 0) {
      await executeNext();
    }
  }

  const promises = [];
  for (let i = 0; i < batchSize; i++) {
    promises.push(executeNext());
  }

  await Promise.all(promises);
  return all;
}