Run many shell commands concurrently, up to a limit. Written before I knew about GNU parallel.

conker.sh

#!/bin/sh

#
# Run many shell commands concurrently, up to a limit.
#
# - Commands are specified in a file, one per line
# - Single-line comments allowed with # at the start
# - Ensure the last line of the file is empty
#
# Originally written to download NML episodes from Archive.org. I found
# the downloads so slow I wanted a way to download many of them at once,
# up to a concurrency-limit.
#
# Apparently curl is able to do this with the -O command-line option,
# but I couldn't get it to do more than one at a time. Perhaps that's a
# macOS thing, or I'm missing something. In either case, this script
# will try to run anything concurrently, not just curl.
#
# Download and use:
#
#   curl -s https://gist.githubusercontent.com/shuckster/9d1d5baa6cfc4065d94686c6133f24ae/raw/conker.sh -o conker.sh
#   chmod +x ./conker.sh
#   ./conker.sh 3 example.txt
#
# Written by Conan: https://gist.github.com/shuckster/9d1d5baa6cfc4065d94686c6133f24ae

concurrency_limit=$1
shift 1
list_of_commands="$*"

running_count=0
running_pids=""

run_in_sequence_with_concurrency()
{
  while IFS= read -r command
  do
    # Skip if line starts with "#" to allow comments
    if echo "$command"|grep -Eq '^#'
    then
      echo "Skipping :: $command"
      continue
    fi

    echo "Running :: $command"
    eval "$command &"
    pid=$!
    running_pids="$running_pids $pid"

    : $((running_count += 1))
    if [ $running_count = "$concurrency_limit" ]
    then
      wait_for_any_pid_to_complete
    fi
  done < "$list_of_commands"

  wait_for_remaining_pids_to_complete
}

wait_for_any_pid_to_complete()
{
  while [ $running_count = "$concurrency_limit" ]
  do
    sleep 0.1
    remaining_pids=""

    for pid in $running_pids
    do
      if ps -p "$pid" > /dev/null
      then
        remaining_pids="$remaining_pids $pid"
      else
        : $((running_count -= 1))
      fi
    done
  done

  running_pids="$remaining_pids"
}

wait_for_remaining_pids_to_complete()
{
  for pid in $running_pids
  do
    wait "$pid"
  done
}

kill_remaining_pids()
{
  for pid in $running_pids
  do
    kill -9 "$pid"
  done
}

ctrl_c_hit()
{
  kill_remaining_pids
  bail "[CTRL+C]: Killing all concurrent processes..."
}

bail()
{
  echo ""
  echo "[BAIL] :: $1"
  echo ""
  exit 1
}

if ! echo "$concurrency_limit"|grep -Eq '^[0-9]+$' || [ ! -f "$list_of_commands" ]
then
  bail "Usage: ./conker.sh <concurrency-limit> <list-of-commands.txt>"
fi

trap "ctrl_c_hit" INT

run_in_sequence_with_concurrency

example.txt

# sleep 1
# sleep 2
sleep 3
sleep 4
sleep 5
sleep 1
sleep 2

norm.txt

curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E01%20-%20Norm%20Macdonald%20with%20Guest%20Super%20Dave%20Osborne.ogv" -o "Norm Macdonald Live - S01E01 - Norm Macdonald with Guest Super Dave Osborne.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E02%20-%20Norm%20Macdonald%20with%20Guest%20Tom%20Green.ogv" -o "Norm Macdonald Live - S01E02 - Norm Macdonald with Guest Tom Green.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E03%20-%20Norm%20Macdonald%20with%20Guest%20Fred%20Stoller.ogv" -o "Norm Macdonald Live - S01E03 - Norm Macdonald with Guest Fred Stoller.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E04%20-%20Norm%20Macdonald%20with%20Guest%20Russell%20Brand.ogv" -o "Norm Macdonald Live - S01E04 - Norm Macdonald with Guest Russell Brand.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E05%20-%20Norm%20Macdonald%20with%20Guest%20Billy%20Bob%20Thornton.ogv" -o "Norm Macdonald Live - S01E05 - Norm Macdonald with Guest Billy Bob Thornton.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E06%20-%20Norm%20Macdonald%20with%20Guest%20Larry%20King.ogv" -o "Norm Macdonald Live - S01E06 - Norm Macdonald with Guest Larry King.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E07%20-%20Norm%20Macdonald%20with%20Guest%20Kevin%20Nealon.ogv" -o "Norm Macdonald Live - S01E07 - Norm Macdonald with Guest Kevin Nealon.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E08%20-%20Norm%20Macdonald%20with%20Guest%20Simon%20Helberg.ogv" -o "Norm Macdonald Live - S01E08 - Norm Macdonald with Guest Simon Helberg.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E09%20-%20Norm%20Macdonald%20with%20Guest%20Nick%20Swardson.ogv" -o "Norm Macdonald Live - S01E09 - Norm Macdonald with Guest Nick Swardson.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E10%20-%20Norm%20Macdonald%20with%20Guest%20Andy%20Dick.ogv" -o "Norm Macdonald Live - S01E10 - Norm Macdonald with Guest Andy Dick.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E11%20-%20Norm%20Macdonald%20with%20Guest%20Gilbert%20Gottfried%20(Pt%201).ogv" -o "Norm Macdonald Live - S01E11 - Norm Macdonald with Guest Gilbert Gottfried (Pt 1).ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S01E12%20-%20Norm%20Macdonald%20with%20Guest%20Gilbert%20Gottfried%20(Pt%202).ogv" -o "Norm Macdonald Live - S01E12 - Norm Macdonald with Guest Gilbert Gottfried (Pt 2).ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E01%20-%20Norm%20Macdonald%20with%20Guest%20Ray%20Romano.ogv" -o "Norm Macdonald Live - S02E01 - Norm Macdonald with Guest Ray Romano.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E02%20-%20Norm%20Macdonald%20with%20Guest%20Adam%20Sandler.ogv" -o "Norm Macdonald Live - S02E02 - Norm Macdonald with Guest Adam Sandler.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E03%20-%20Norm%20Macdonald%20with%20Guest%20David%20Spade.ogv" -o "Norm Macdonald Live - S02E03 - Norm Macdonald with Guest David Spade.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E04%20-%20Norm%20Macdonald%20with%20Guest%20Carl%20Reiner.ogv" -o "Norm Macdonald Live - S02E04 - Norm Macdonald with Guest Carl Reiner.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E05%20-%20Norm%20Macdonald%20with%20Guest%20Fred%20Willard.ogv" -o "Norm Macdonald Live - S02E05 - Norm Macdonald with Guest Fred Willard.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E06%20-%20Norm%20Macdonald%20with%20Guest%20Todd%20Glass%20(Pt%201).ogv" -o "Norm Macdonald Live - S02E06 - Norm Macdonald with Guest Todd Glass (Pt 1).ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E07%20-%20Norm%20Macdonald%20with%20Guest%20Todd%20Glass%20(Pt%202).ogv" -o "Norm Macdonald Live - S02E07 - Norm Macdonald with Guest Todd Glass (Pt 2).ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E08%20-%20Norm%20Macdonald%20with%20Guest%20Bob%20Saget.ogv" -o "Norm Macdonald Live - S02E08 - Norm Macdonald with Guest Bob Saget.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E09%20-%20Norm%20Macdonald%20with%20Guest%20David%20Koechner.ogv" -o "Norm Macdonald Live - S02E09 - Norm Macdonald with Guest David Koechner.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E10%20-%20Norm%20Macdonald%20with%20Guest%20Roseanne%20Barr.ogv" -o "Norm Macdonald Live - S02E10 - Norm Macdonald with Guest Roseanne Barr.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E11%20-%20Norm%20Macdonald%20with%20Guest%20Marc%20Maron.ogv" -o "Norm Macdonald Live - S02E11 - Norm Macdonald with Guest Marc Maron.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E12%20-%20Norm%20Macdonald%20with%20Guest%20Martin%20Mull.ogv" -o "Norm Macdonald Live - S02E12 - Norm Macdonald with Guest Martin Mull.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S02E13%20-%20Norm%20Macdonald%20with%20Guest%20Jack%20Carter.ogv" -o "Norm Macdonald Live - S02E13 - Norm Macdonald with Guest Jack Carter.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E01%20-%20Norm%20Macdonald%20with%20Guest%20Stephen%20Merchant.ogv" -o "Norm Macdonald Live - S03E01 - Norm Macdonald with Guest Stephen Merchant.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E02%20-%20Norm%20Macdonald%20with%20Guest%20Bill%20Hader.ogv" -o "Norm Macdonald Live - S03E02 - Norm Macdonald with Guest Bill Hader.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E03%20-%20Norm%20Macdonald%20with%20Guest%20David%20Letterman.ogv" -o "Norm Macdonald Live - S03E03 - Norm Macdonald with Guest David Letterman.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E04%20-%20Norm%20Macdonald%20with%20Guest%20Mike%20Tyson.ogv" -o "Norm Macdonald Live - S03E04 - Norm Macdonald with Guest Mike Tyson.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E05%20-%20Norm%20Macdonald%20with%20Guest%20Bobby%20Lee.ogv" -o "Norm Macdonald Live - S03E05 - Norm Macdonald with Guest Bobby Lee.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E06%20-%20Norm%20Macdonald%20with%20Guest%20Sarah%20Silverman%20(Pt%201).ogv" -o "Norm Macdonald Live - S03E06 - Norm Macdonald with Guest Sarah Silverman (Pt 1).ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E07%20-%20Norm%20Macdonald%20with%20Guest%20Sarah%20Silverman%20(Pt%202).ogv" -o "Norm Macdonald Live - S03E07 - Norm Macdonald with Guest Sarah Silverman (Pt 2).ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E08%20-%20Norm%20Macdonald%20with%20Guest%20Dana%20Carvey.ogv" -o "Norm Macdonald Live - S03E08 - Norm Macdonald with Guest Dana Carvey.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E09%20-%20Norm%20Macdonald%20with%20Guest%20Jerry%20Seinfeld.ogv" -o "Norm Macdonald Live - S03E09 - Norm Macdonald with Guest Jerry Seinfeld.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E10%20-%20Norm%20Macdonald%20with%20Guest%20Caitlyn%20Jenner.ogv" -o "Norm Macdonald Live - S03E10 - Norm Macdonald with Guest Caitlyn Jenner.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E11%20-%20Norm%20Macdonald%20with%20Guest%20Rich%20Little.ogv" -o "Norm Macdonald Live - S03E11 - Norm Macdonald with Guest Rich Little.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E12%20-%20Norm%20Macdonald%20with%20Guest%20Margaret%20Cho.ogv" -o "Norm Macdonald Live - S03E12 - Norm Macdonald with Guest Margaret Cho.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E13%20-%20Norm%20Macdonald%20with%20Guest%20Tim%20Allen.ogv" -o "Norm Macdonald Live - S03E13 - Norm Macdonald with Guest Tim Allen.ogv"
curl -sSLC - "https://archive.org/download/Norm_Macdonald_Live/Norm%20Macdonald%20Live%20-%20S03E14%20-%20Norm%20Macdonald%20with%20Guest%20Jim%20Carrey.ogv" -o "Norm Macdonald Live - S03E14 - Norm Macdonald with Guest Jim Carrey.ogv"