Inventory PVE guests that are NOT ready for on-the-fly resource hotplug

pve-hotplug-remediate.sh

#!/usr/bin/env bash
#
# pve-hotplug-remediate.sh
#
# Inventory PVE guests that are NOT ready for on-the-fly resource hotplug
# (vCPU / RAM / disk / NIC), and optionally pre-stage the fix as *pending*
# config changes (the orange values in the web UI), applied at next cold boot.
#
# DRY RUN IS THE DEFAULT. Nothing is written without --apply.
#
# Requires: jq, awk, pvesh. Run from any cluster node.
#
set -euo pipefail

EXCLUDE_RE="${EXCLUDE_RE:-test|template|tmp|poc}"
TARGET="${TARGET:-disk,network,usb,memory,cpu}"
HEADROOM="${HEADROOM:-0}"
DRYRUN=1
SHOW_ALL=0
ONE_VMID=""
STRICT=0

usage() {
  cat <<'EOF'
Usage: pve-hotplug-remediate.sh [options]

  (no options)        cluster-wide inventory of NON-READY VMs, dry run

  --vmid N            one-shot: target this VMID only. Bypasses the name
                      exclusion and the running-only filter, and always shows
                      the row even when the VM is already ready.
  --apply             execute the staging commands (default: print only)
  --all               include already-ready VMs in the table
  --headroom N        extra cores PER SOCKET so vCPUs can grow above the
                      current boot-time count (default 0 = leave topology)
  --target LIST       hotplug flags to converge on
                      (default disk,network,usb,memory,cpu)
  --exclude RE        case-insensitive name regex to skip
                      (default test|template|tmp|poc, '' disables)
  --strict            exit 3 when non-ready VMs are found (for CI gating)
  -h, --help          this text

Examples:
  ./pve-hotplug-remediate.sh                      # what is not ready?
  ./pve-hotplug-remediate.sh --vmid 100           # inspect one VM
  ./pve-hotplug-remediate.sh --vmid 100 --apply   # one-shot test on one VM
  ./pve-hotplug-remediate.sh --apply              # stage the whole cluster
EOF
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --vmid)     ONE_VMID="${2:?--vmid needs a value}"; SHOW_ALL=1; shift 2 ;;
    --apply)    DRYRUN=0; shift ;;
    --all)      SHOW_ALL=1; shift ;;
    --headroom) HEADROOM="${2:?--headroom needs a value}"; shift 2 ;;
    --target)   TARGET="${2:?--target needs a value}"; shift 2 ;;
    --exclude)  EXCLUDE_RE="${2-}"; shift 2 ;;
    --strict)   STRICT=1; shift ;;
    -h|--help)  usage; exit 0 ;;
    *) printf 'unknown option: %s\n\n' "$1" >&2; usage >&2; exit 2 ;;
  esac
done

[[ -z "$ONE_VMID" || "$ONE_VMID" =~ ^[0-9]+$ ]] || { echo "--vmid must be numeric" >&2; exit 2; }
[[ "$HEADROOM" =~ ^[0-9]+$ ]] || { echo "--headroom must be a non-negative integer" >&2; exit 2; }

TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT

# ---------------------------------------------------------------------------
# 1. Build the candidate list -> node, vmid, name, status, template
# ---------------------------------------------------------------------------
if [[ -n "$ONE_VMID" ]]; then
  SCOPE="vmid $ONE_VMID (one-shot)"
  pvesh get /cluster/resources --type vm --output-format json \
  | jq -r --argjson v "$ONE_VMID" '.[]
      | select(.type == "qemu")
      | select(.vmid == $v)
      | [.node, .vmid, (.name // "-"), .status, ((.template // 0) | tostring)]
      | @tsv' > "$TMP/vms"
  if [[ ! -s "$TMP/vms" ]]; then
    echo "vmid $ONE_VMID is not a qemu guest in this cluster" >&2
    exit 4
  fi
else
  if [[ -z "$EXCLUDE_RE" ]]; then
    SCOPE="running VMs, no name filter"
  else
    SCOPE="running VMs, excluding /${EXCLUDE_RE}/i"
  fi
  pvesh get /cluster/resources --type vm --output-format json \
  | jq -r --arg re "$EXCLUDE_RE" '.[]
      | select(.type == "qemu")
      | select(.status == "running")
      | select((.template // 0) != 1)
      | select($re == "" or ((.name // "") | test($re; "i") | not))
      | [.node, .vmid, (.name // "-"), .status, "0"]
      | @tsv' \
  | sort -k1,1 -k2,2n > "$TMP/vms"
fi

# ---------------------------------------------------------------------------
# 2. Pull the config keys that gate hotplug (jq does no logic here)
# ---------------------------------------------------------------------------
while IFS=$'\t' read -r node vmid name status tmpl; do
  pvesh get "/nodes/$node/qemu/$vmid/config" --output-format json \
  | jq -r --arg n "$node" --arg i "$vmid" --arg m "$name" \
          --arg s "$status" --arg t "$tmpl" '
      [ $n, $i, $m, $s, $t,
        (.hotplug // 1 | tostring),
        (.numa    // 0),
        (.vcpus   // 0),
        (.sockets // 1),
        (.cores   // 1),
        (.ostype  // "")
      ] | @tsv'
done < "$TMP/vms" > "$TMP/raw"

# ---------------------------------------------------------------------------
# 3. Logic: expand the hotplug default, diff against TARGET, decide readiness.
#    MODE=table | cmd | verify. Only non-ready rows reach cmd/verify.
# ---------------------------------------------------------------------------
cat > "$TMP/logic.awk" <<'AWK'
BEGIN {
  FS = "\t"
  ncan  = split("disk,network,usb,memory,cpu", can, ",")
  nwant = split(TARGET, want, ",")
  if (MODE == "table") {
    print "| Node | VMID | Name | State | disk | net | usb | mem | cpu | numa | vcpus/max | To stage |"
    print "|---|---|---|---|:-:|:-:|:-:|:-:|:-:|:-:|:-:|---|"
  }
}
{
  node = $1; vmid = $2; name = $3; status = $4; tmpl = $5 + 0; raw = $6
  numa = $7 + 0; vcpus = $8 + 0; sockets = $9 + 0; cores = $10 + 0; ostype = $11
  if (sockets < 1) sockets = 1
  if (cores   < 1) cores   = 1
  max = sockets * cores

  # PVE default when the key is absent is network,disk,usb ("1" is its alias)
  if      (raw == "1") raw = "disk,network,usb"
  else if (raw == "0") raw = ""
  delete cur
  if (raw != "") { n = split(raw, a, ","); for (i = 1; i <= n; i++) cur[a[i]] = 1 }

  nmiss = 0
  for (i = 1; i <= nwant; i++) if (!(want[i] in cur)) nmiss++

  # union of current and target, so a narrow TARGET never strips a live flag
  union = ""
  for (i = 1; i <= ncan; i++) {
    keep = (can[i] in cur)
    for (j = 1; j <= nwant; j++) if (want[j] == can[i]) keep = 1
    if (keep) union = union (union ? "," : "") can[i]
  }

  numa_ok  = (numa == 1)
  headroom = (vcpus > 0 && vcpus < max)

  stage = ""
  if (nmiss > 0) stage = "hotplug"
  if (!numa_ok)  stage = stage (stage ? "+" : "") "numa"
  if (HEADROOM + 0 > 0 && !headroom) stage = stage (stage ? "+" : "") "vcpus"

  if (MODE == "table") {
    if (!stage && SHOW_ALL + 0 == 0) next
    label = status; if (tmpl == 1) label = label ",template"
    row = "| " node " | " vmid " | " name " | " label " |"
    for (i = 1; i <= ncan; i++) row = row " " ((can[i] in cur) ? "ok" : "**NO**") " |"
    row = row " " (numa_ok ? "ok" : "**NO**") " |"
    row = row " " (headroom ? vcpus "/" max : "**" max "/" max "**") " |"
    print row " " (stage ? stage : "ready") " |"
    next
  }

  if (!stage) next

  if (MODE == "verify") {
    if (status == "running")
      print "pvesh get /nodes/" node "/qemu/" vmid "/pending --output-format json" \
            " | jq -r " Q ".[] | select(.pending) | \"" vmid " \\(.key): \\(.value // \"-\") -> \\(.pending)\"" Q
    else
      print "pvesh get /nodes/" node "/qemu/" vmid "/config --output-format json" \
            " | jq -r " Q "\"" vmid " hotplug=\\(.hotplug) numa=\\(.numa) vcpus=\\(.vcpus // \"unset\")\"" Q \
            "  # not running: applied directly, nothing pending"
    next
  }

  cmd = "pvesh set /nodes/" node "/qemu/" vmid "/config"
  if (nmiss > 0) cmd = cmd " --hotplug " union
  if (!numa_ok)  cmd = cmd " --numa 1"
  if (HEADROOM + 0 > 0 && !headroom) cmd = cmd " --cores " (cores + HEADROOM + 0) " --vcpus " max
  note = ""
  if (status != "running") note = note "  # " toupper(status) ": writes straight to config, no pending stage"
  if (tmpl == 1)           note = note "  # TEMPLATE"
  if (ostype ~ /^w/)       note = note "  # WINDOWS: confirm edition supports cpu hotplug"
  print cmd note
}
AWK

run_awk() { awk -v MODE="$1" -v TARGET="$TARGET" -v HEADROOM="$HEADROOM" \
                -v SHOW_ALL="$SHOW_ALL" -v Q="'" -f "$TMP/logic.awk" "$TMP/raw"; }

run_awk cmd > "$TMP/cmds"
total=$(wc -l < "$TMP/raw" | tr -d ' ')
notready=$(wc -l < "$TMP/cmds" | tr -d ' ')

# ---------------------------------------------------------------------------
# 4. Report
# ---------------------------------------------------------------------------
printf '## Hotplug readiness: %s\n' "$SCOPE"
printf '   scanned %s, non-ready %s, target %s, headroom %s\n\n' \
       "$total" "$notready" "$TARGET" "$HEADROOM"

if (( notready == 0 && SHOW_ALL == 0 )); then
  echo "All scanned VMs already support on-the-fly hotplug for: $TARGET"
  exit 0
fi

run_awk table
echo

if (( notready == 0 )); then
  echo "Nothing to stage."
  exit 0
fi

if (( DRYRUN )); then
  echo "## DRY RUN, no changes made. Commands that would run:"
  echo
  cat "$TMP/cmds"
  echo
  echo "# re-run with --apply to stage them"
  (( STRICT )) && exit 3
  exit 0
fi

echo "## Applying to $notready VM(s)"
echo
rc=0
while IFS= read -r line; do
  read -ra argv <<<"${line%%  #*}"   # no arg contains spaces, so no eval needed
  printf '+ %s\n' "${argv[*]}"
  "${argv[@]}" || { rc=1; echo "  ! FAILED (continuing)" >&2; }
done < "$TMP/cmds"

echo
echo "## Verify the pending values"
echo
run_awk verify

exit "$rc"