multi-format compressor (zstd, xz, gzip, lz4, snappy, brotli, zpaq)

tarz.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_NAME="$(basename "$0")"

usage() {
  cat <<EOF
$SCRIPT_NAME — multi-format compressor (zstd, xz, gzip, lz4, snappy, brotli, zpaq)

Quick guide (which algo to use?):
  • zstd    — Best all-rounder for backups/artifacts: fast & small. Try -m/-H.
  • lz4     — Ultra-fast compress/decompress for logs, caches, streams. Use -l.
  • brotli  — Web/text assets (HTML/JS/CSS). Highest text ratio. Use -H.
  • xz      — Cold archival where time doesn't matter; smallest of tar-classic.
  • gzip    — Legacy compatibility; OK ratio, fast enough.
  • snappy  — Very low CPU, huge speed, bigger files; DB/log pipelines.
  • zpaq    — Dedup & archival; very small but slow; creates .zpaq (not tar).

Usage:
  $SCRIPT_NAME [--zstd|-Z | --xz|-x | --gzip|-g | --lz4|-L | --snappy|-S | --brotli|-B | --zpaq|-Q]
               [--low|-l | --med|-m | --high|-H]
               [--cpu <num>]
               <path>
               [-- --extra-args]

Examples (using -Z for zstd):
  $SCRIPT_NAME -Z myfile.txt              # file  → myfile.txt.zst (no tar)
  $SCRIPT_NAME -Z mydir                   # dir   → mydir.tar.zst
  $SCRIPT_NAME -Z -D myfile.txt           # compress file, then delete original
  $SCRIPT_NAME -Z -D mydir                # compress dir, then delete original
  $SCRIPT_NAME -Z -H backup/              # high compression
  $SCRIPT_NAME -Z -- -C /tmp mydir        # pass extra args to tar

Options:
  --zstd,   -Z   Use Zstandard (.zst / .tar.zst)           [default]
  --xz,     -x   Use xz (.xz / .tar.xz)
  --gzip,   -g   Use gzip (.gz / .tar.gz)
  --lz4,    -L   Use LZ4 (.lz4 / .tar.lz4)
  --snappy, -S   Use Snappy via snzip (.snz / .tar.snz)
  --brotli, -B   Use Brotli (.br / .tar.br)
  --zpaq,   -Q   Use ZPAQ (.zpaq) — builds its own archive (no tar)

  --low,    -l   Faster, lower compression
  --med,    -m   Balanced (default)
  --high,   -H   Higher compression (slower)
  --cpu <num>    Use <num> CPU cores where supported; 0 means all CPUs
  --delete, -D   Delete original file/dir after successful compression

  -h, --help     Show this help

Notes:
  • FILES  → compress to <file>.<ext> (original kept).
  • DIRS   → create <dir>.tar.<ext> for tar-based algos; ZPAQ makes <dir>.zpaq.
  • After "--", args go to tar (dir-mode for tar-based algos) or the compressor (files; ZPAQ always).
  • On macOS, install GNU tar:   brew install gnu-tar   (use 'gtar').
EOF
}

die()  { echo "$SCRIPT_NAME: $*" >&2; exit 1; }
need() { command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1"; }

# Defaults
ALGO="zstd"      # zstd|xz|gzip|lz4|snappy|brotli|zpaq
LEVEL="high"      # low|med|high
DELETE_ORIGINAL=false
INPUT=""
EXTRA_ARGS=()
CPU=""

cpu_count() {
  if command -v nproc >/dev/null 2>&1; then
    nproc
    return
  fi

  if command -v getconf >/dev/null 2>&1; then
    getconf _NPROCESSORS_ONLN
    return
  fi

  if command -v sysctl >/dev/null 2>&1; then
    sysctl -n hw.ncpu
    return
  fi

  die "unable to determine CPU count"
}

# Parse args
while (( $# )); do
  case "$1" in
    --zstd|-Z)    ALGO="zstd" ;;
    --xz|-x)      ALGO="xz" ;;
    --gzip|-g)    ALGO="gzip" ;;
    --lz4|-L)     ALGO="lz4" ;;
    --snappy|-S)  ALGO="snappy" ;;
    --brotli|-B)  ALGO="brotli" ;;
    --zpaq|-Q)    ALGO="zpaq" ;;
    --low|-l)     LEVEL="low" ;;
    --med|-m)     LEVEL="med" ;;
    --high|-H)    LEVEL="high" ;;
    --cpu)
      shift
      [[ $# -gt 0 ]] || die "--cpu requires a value"
      [[ "$1" =~ ^[0-9]+$ ]] || die "--cpu must be a non-negative integer"
      CPU="$1"
      ;;
    --delete|-D)  DELETE_ORIGINAL=true ;;
    -h|--help)    usage; exit 0 ;;
    --) shift; EXTRA_ARGS=("$@"); break ;;
    -*) die "unknown option: $1" ;;
    *)  INPUT="$1" ;;
  esac
  shift
done

[[ -z "$INPUT" ]] && { usage; exit 1; }
[[ -e "$INPUT" ]] || die "path not found: $INPUT"

if [[ -n "$CPU" && "$CPU" -eq 0 ]]; then
  CPU="$(cpu_count)"
fi

# tar detection (for tar-based algorithms)
TAR_BIN="$(command -v gtar || command -v tar || true)"
[[ -z "$TAR_BIN" ]] && die "tar not found"
if "$TAR_BIN" --help 2>&1 | grep -qE -- '-I[ =]'; then
  TAR_HAS_I=true
else
  TAR_HAS_I=false
fi

# ----- Level mappings (arrays so multi-token flags work) -----
# Default to empty arrays; we'll fill per-algo
ZSTD_LVL=(); XZ_LVL=(); GZ_LVL=(); LZ4_LVL=()   # arrays
ZSTD_CPU=(); XZ_CPU=(); LZ4_CPU=()              # arrays
CPU_SUPPORTED=false
Q=9   # brotli quality default (overwritten per level)
M=3   # zpaq method default (overwritten per level)

case "$ALGO" in
  zstd)
    need zstd
    CPU_SUPPORTED=true
    case "$LEVEL" in
      low)  ZSTD_LVL=( -3 ) ;;
      med)  ZSTD_LVL=( -19 ) ;;
      high) ZSTD_LVL=( -22 --ultra ) ;;  # multi-token level supported
      *)    die "invalid level: $LEVEL" ;;
    esac
    [[ -n "$CPU" ]] && ZSTD_CPU=( "-T$CPU" )
    EXT_FILE="zst"; EXT_TAR="zst"
    ;;
  xz)
    need xz
    CPU_SUPPORTED=true
    case "$LEVEL" in
      low)  XZ_LVL=( -3 ) ;;
      med)  XZ_LVL=( -6 ) ;;
      high) XZ_LVL=( -9 ) ;;
      *)    die "invalid level: $LEVEL" ;;
    esac
    [[ -n "$CPU" ]] && XZ_CPU=( "-T$CPU" )
    EXT_FILE="xz"; EXT_TAR="xz"
    ;;
  gzip)
    need gzip
    case "$LEVEL" in
      low)  GZ_LVL=( -3 ) ;;
      med)  GZ_LVL=( -6 ) ;;
      high) GZ_LVL=( -9 ) ;;
      *)    die "invalid level: $LEVEL" ;;
    esac
    EXT_FILE="gz"; EXT_TAR="gz"
    ;;
  lz4)
    need lz4
    CPU_SUPPORTED=true
    case "$LEVEL" in
      low)  LZ4_LVL=() ;;         # fastest (no level flag)
      med)  LZ4_LVL=( -3 ) ;;
      high) LZ4_LVL=( -9 ) ;;
      *)    die "invalid level: $LEVEL" ;;
    esac
    [[ -n "$CPU" ]] && LZ4_CPU=( "-T$CPU" )
    EXT_FILE="lz4"; EXT_TAR="lz4"
    ;;
  snappy)
    need snzip
    # snappy ignores levels; accept flags but do nothing with them
    EXT_FILE="snz"; EXT_TAR="snz"
    ;;
  brotli)
    need brotli
    case "$LEVEL" in
      low)  Q=5 ;;
      med)  Q=9 ;;
      high) Q=11 ;;
      *)    die "invalid level: $LEVEL" ;;
    esac
    EXT_FILE="br"; EXT_TAR="br"
    ;;
  zpaq)
    need zpaq
    case "$LEVEL" in
      low)  M=1 ;;
      med)  M=3 ;;
      high) M=5 ;;
      *)    die "invalid level: $LEVEL" ;;
    esac
    EXT_FILE="zpaq"
    ;;
  *) die "unsupported algorithm: $ALGO" ;;
esac

if [[ -n "$CPU" && "$CPU_SUPPORTED" != true ]]; then
  echo "$SCRIPT_NAME: warning: --cpu is ignored for algorithm '$ALGO'" >&2
fi

OUTFILE=""

# ----------------- File compression -----------------
compress_file() {
  local src="$1"
  case "$ALGO" in
    zstd)
      # zstd writes file.zst and keeps input by default
      zstd "${ZSTD_LVL[@]}" "${ZSTD_CPU[@]}" "${EXTRA_ARGS[@]}" -q -f -- "$src"
      OUTFILE="${src}.zst"
      ;;
    xz)
      xz "${XZ_LVL[@]}" "${XZ_CPU[@]}" -c "${EXTRA_ARGS[@]}" -- "$src" > "${src}.xz"
      OUTFILE="${src}.xz"
      ;;
    gzip)
      gzip "${GZ_LVL[@]}" -c "${EXTRA_ARGS[@]}" -- "$src" > "${src}.gz"
      OUTFILE="${src}.gz"
      ;;
    lz4)
      lz4 "${LZ4_LVL[@]}" "${LZ4_CPU[@]}" -c "${EXTRA_ARGS[@]}" -- "$src" > "${src}.lz4"
      OUTFILE="${src}.lz4"
      ;;
    snappy)
      snzip -c "${EXTRA_ARGS[@]}" -- "$src" > "${src}.snz"
      OUTFILE="${src}.snz"
      ;;
    brotli)
      brotli -q "$Q" -c "${EXTRA_ARGS[@]}" -- "$src" > "${src}.br"
      OUTFILE="${src}.br"
      ;;
    zpaq)
      OUTFILE="${src}.zpaq"
      rm -f -- "$OUTFILE"
      zpaq a "$OUTFILE" "$src" -method "$M" "${EXTRA_ARGS[@]}"
      ;;
  esac
}

# Build compressor strings for tar -I (join arrays with spaces)
join_array() { local IFS=' '; echo "$*"; }
ZSTD_CMD_STR="zstd $(join_array "${ZSTD_LVL[@]}" "${ZSTD_CPU[@]}")"
XZ_CMD_STR="xz $(join_array "${XZ_LVL[@]}" "${XZ_CPU[@]}")"
GZ_CMD_STR="gzip $(join_array "${GZ_LVL[@]}")"
LZ4_CMD_STR="lz4 $(join_array "${LZ4_LVL[@]}" "${LZ4_CPU[@]}")"
BROTLI_CMD_STR="brotli -q $Q -c"   # brotli needs -c for stdout
SNZIP_CMD_STR="snzip -c"           # snzip writes to stdout

# --- dir mode ---
compress_dir() {
  local dir="${1%/}"
  local base="$(basename "$dir")"
  local parent="$(dirname "$dir")"

  case "$ALGO" in
    zpaq)
      OUTFILE="${base}.zpaq"
      rm -f -- "$OUTFILE"
      zpaq a "$OUTFILE" "$dir" -method "$M" "${EXTRA_ARGS[@]}"
      return
      ;;
    *)
      OUTFILE="${base}.tar.${EXT_TAR}"
      ;;
  esac

  if $TAR_HAS_I; then
    # Use GNU tar -I "<cmd>" and ensure options (EXTRA_ARGS) come BEFORE operands
    case "$ALGO" in
      zstd)   "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -I "$ZSTD_CMD_STR"   -cf "$OUTFILE" "$base" ;;
      xz)     "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -I "$XZ_CMD_STR"     -cf "$OUTFILE" "$base" ;;
      gzip)   "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -I "$GZ_CMD_STR"     -cf "$OUTFILE" "$base" ;;
      lz4)    "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -I "$LZ4_CMD_STR"    -cf "$OUTFILE" "$base" ;;
      snappy) "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -I "$SNZIP_CMD_STR"  -cf "$OUTFILE" "$base" ;;
      brotli) "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -I "$BROTLI_CMD_STR" -cf "$OUTFILE" "$base" ;;
    esac
  else
    # Portable fallback: tar stream | compressor (still uses -C; options before operand)
    case "$ALGO" in
      zstd)   "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -cf - "$base" | zstd   "${ZSTD_LVL[@]}" "${ZSTD_CPU[@]}" -q -f -o "$OUTFILE" ;;
      xz)     "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -cf - "$base" | xz     "${XZ_LVL[@]}"   "${XZ_CPU[@]}" -c > "$OUTFILE" ;;
      gzip)   "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -cf - "$base" | gzip   "${GZ_LVL[@]}"   -c > "$OUTFILE" ;;
      lz4)    "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -cf - "$base" | lz4    "${LZ4_LVL[@]}"  "${LZ4_CPU[@]}" -c > "$OUTFILE" ;;
      snappy) "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -cf - "$base" | snzip  -c > "$OUTFILE" ;;
      brotli) "$TAR_BIN" -C "$parent" "${EXTRA_ARGS[@]}" -cf - "$base" | brotli -q "$Q" -c > "$OUTFILE" ;;
    esac
  fi
}

# run: files → compress directly (no tar); dirs → tar then compress
if [[ -d "$INPUT" ]]; then
  compress_dir "$INPUT"
elif [[ -f "$INPUT" ]]; then
  compress_file "$INPUT"
else
  die "not a file or directory: $INPUT"
fi

wait # for background jobs (if any)

# Optional deletion (only after successful compression)
if [[ "$DELETE_ORIGINAL" == true ]]; then
  if [[ -d "$INPUT" ]]; then
    rm -rf -- "$INPUT"
    echo "Deleted original directory: $INPUT"
  else
    rm -f -- "$INPUT"
    echo "Deleted original file: $INPUT"
  fi
fi

echo "Algorithm: $ALGO | Level: $LEVEL"
if [[ -n "$CPU" && "$CPU_SUPPORTED" == true ]]; then
  echo "CPU cores: $CPU"
fi
echo "$OUTFILE"