A console mode GPU status dashboard for the DGX Spark

sparktop.sh

#!/usr/bin/env bash
# sparktop — a text-mode dashboard for the DGX Spark lab.
#
# Shows, at a glance:
#   * GB10 GPU utilisation, clocks, power, and the unified memory pool
#     (GB10 has no discrete VRAM: nvidia-smi reports memory.total/used as
#     [N/A], so the pool is read from /proc/meminfo and the GPU share is
#     summed from per-process compute allocations).
#   * Running Docker containers with CPU / memory, and GPU memory attributed
#     back to the owning container via /proc/<pid>/cgroup.
#   * Ollama models resident in memory, with footprint and context size.
#   * Claude Code usage limits (session + weekly windows, with reset
#     countdowns). Read from the cache Claude Code keeps in ~/.claude.json,
#     which a periodic 'claude -p /usage' refreshes — that is a client-side
#     command, so it costs no tokens and makes no inference call.
#
# Usage:  ./sparktop.sh [-n SECONDS]
#         (default 10s; 'q' quits, 'r' refreshes, 's' cycles the table sort)

set -uo pipefail

INTERVAL=10
OLLAMA_URL="${OLLAMA_HOST:-http://localhost:11434}"
CLAUDE_JSON="${HOME}/.claude.json"
CLAUDE_LIVE=1   # ask the claude CLI rather than reading Claude Code's cache
CLAUDE_REFRESH=300  # usage limits move slowly, and the endpoint rate-limits
CLAUDE_BACKOFF_MAX=1800
HIST_LEN=32
MIN_DISK_GB=100  # hide filesystems smaller than this (GiB) from the DISK panel

# Container table sort, cycled with 's'. Default is cpu: when the window is too
# short for the whole table the sort also decides who gets shown, and "busiest
# first" is the most useful thing to lose the bottom of.
SORT_MODES=(gpu mem cpu uptime)
SORT_IDX=2
declare -A SORT_LABEL=([gpu]='gpu mem' [mem]='mem' [cpu]='cpu %' [uptime]='uptime')

set_sort() {
	local want="$1" i
	for i in "${!SORT_MODES[@]}"; do
		[[ ${SORT_MODES[i]} == "$want" ]] && {
			SORT_IDX=$i
			return 0
		}
	done
	return 1
}

usage() {
	cat <<-EOF
		sparktop — DGX Spark system dashboard

		  usage: ${0##*/} [-n SECONDS] [SECONDS]

		  -n, --interval SECONDS   refresh interval (default: ${INTERVAL})
		      --no-live            don't shell out to claude; read Claude Code's
		                           cached utilization from ~/.claude.json instead
		      --live-interval SEC  how often to re-run /usage (default: ${CLAUDE_REFRESH})
		      --sort MODE          container table order: ${SORT_MODES[*]}
		                           (default: ${SORT_MODES[SORT_IDX]})
		      --min-disk GB        hide filesystems smaller than this in the
		                           DISK panel (default: ${MIN_DISK_GB})
		  -h, --help               this help

		  keys: q = quit    r = refresh now    s = cycle sort

		Claude usage limits come from 'claude -p /usage', which is a client-side
		command: it reports num_turns 0 and \$0.00, so it costs no tokens and
		makes no inference call. --no-session-persistence keeps it from leaving
		session files behind, and an empty strict MCP config stops it loading
		MCP servers. If the claude CLI is missing or fails, sparktop falls back
		to the cached values in ~/.claude.json and labels them as stale.
	EOF
}

while [[ $# -gt 0 ]]; do
	case "$1" in
	-n | --interval)
		INTERVAL="${2:-}"
		shift 2
		;;
	--no-live)
		CLAUDE_LIVE=0
		shift
		;;
	--live-interval)
		CLAUDE_REFRESH="${2:-300}"
		shift 2
		;;
	--sort)
		set_sort "${2:-}" || {
			printf 'unknown sort mode: %s (want one of: %s)\n' "${2:-}" "${SORT_MODES[*]}" >&2
			exit 1
		}
		shift 2
		;;
	--min-disk)
		MIN_DISK_GB="${2:-}"
		shift 2
		;;
	-h | --help)
		usage
		exit 0
		;;
	'' | *[!0-9.]*)
		printf 'unknown argument: %s\n\n' "$1" >&2
		usage
		exit 1
		;;
	*)
		INTERVAL="$1"
		shift
		;;
	esac
done
[[ "$INTERVAL" =~ ^[0-9]+([.][0-9]+)?$ ]] || {
	echo "interval must be a number" >&2
	exit 1
}
[[ "$MIN_DISK_GB" =~ ^[0-9]+([.][0-9]+)?$ ]] || {
	echo "--min-disk must be a number" >&2
	exit 1
}

# ── palette ──────────────────────────────────────────────────────────────────
E=$'\e'
R="${E}[0m" B="${E}[1m" D="${E}[2m"
RULE="${E}[38;5;66m"    # desaturated teal, for section rules
HEAD="${E}[38;5;80m"    # section titles
LBL="${E}[38;5;103m"    # field labels
VAL="${E}[38;5;252m"    # values
DIMV="${E}[38;5;242m"   # secondary / units
OK="${E}[38;5;78m"      # green
WARN="${E}[38;5;221m"   # amber
CRIT="${E}[38;5;203m"   # red
ACC="${E}[38;5;146m"    # accent (names, ids)
GPUC="${E}[38;5;114m"   # GPU-memory segment
APPC="${E}[38;5;68m"    # other-process segment
CACHEC="${E}[38;5;60m"  # page-cache segment

BLOCKS=(▁ ▂ ▃ ▄ ▅ ▆ ▇ █)

# ── terminal lifecycle ───────────────────────────────────────────────────────
# Collectors run as background jobs, and those subshells inherit this trap —
# guard on the main PID so a finishing collector doesn't tear down the screen
# and delete the scratch dir out from under the renderer.
MAINPID=$$
CLEANED=0
cleanup() {
	[[ $BASHPID -ne $MAINPID ]] && return
	((CLEANED)) && return
	CLEANED=1
	# stop collectors before removing the dir they are writing into
	local j
	for j in $(jobs -p 2>/dev/null); do kill "$j" 2>/dev/null; done
	printf '%s[?25h%s[?7h%s[?1049l' "$E" "$E" "$E"
	stty echo 2>/dev/null
	[[ -n "${TMPDIR_RUN:-}" ]] && rm -rf "$TMPDIR_RUN"
}
# INT/TERM must exit: a bare trap runs cleanup and then RESUMES the loop, which
# leaves the process spinning against a temp dir it just deleted.
trap cleanup EXIT
trap 'cleanup; exit 130' INT
trap 'cleanup; exit 143' TERM
# A resize should repaint immediately rather than waiting out the refresh
# interval. SIGWINCH interrupts the read below, so the loop wakes at once.
RESIZED=0
trap 'RESIZED=1' WINCH
# ?7l disables auto-wrap: an over-long row is truncated by the terminal instead
# of wrapping onto a second physical line, which would scroll the frame.
printf '%s[?1049h%s[?25l%s[?7l' "$E" "$E" "$E"
stty -echo 2>/dev/null

TMPDIR_RUN="$(mktemp -d)"

# ── small helpers ────────────────────────────────────────────────────────────

# pad PLAIN text to width (no ANSI inside), left- or right-aligned.
# printf's field width counts bytes, which mis-pads multi-byte glyphs like "—",
# so measure with ${#s} (characters) and emit the spaces ourselves.
pad() {
	local s="$1" w="$2"
	((${#s} > w)) && s="${s:0:w}"
	printf '%s%*s' "$s" "$((w - ${#s}))" ''
}
padr() {
	local s="$1" w="$2"
	((${#s} > w)) && s="${s:0:w}"
	printf '%*s%s' "$((w - ${#s}))" '' "$s"
}
# truncate without padding — for the final column on a row
trunc() {
	local s="$1" w="$2"
	((${#s} > w)) && s="${s:0:w}"
	printf '%s' "$s"
}

# colour by threshold: <60 green, <85 amber, else red
sev() {
	local p=${1%%.*}
	((p < 60)) && {
		printf '%s' "$OK"
		return
	}
	((p < 85)) && {
		printf '%s' "$WARN"
		return
	}
	printf '%s' "$CRIT"
}

# bar WIDTH PERCENT [COLOR] — a single-colour meter
bar() {
	local w=$1 pct=${2%%.*} col="${3:-}"
	((pct < 0)) && pct=0
	((pct > 100)) && pct=100
	[[ -z "$col" ]] && col="$(sev "$pct")"
	local fill=$((pct * w / 100)) i out=""
	for ((i = 0; i < w; i++)); do
		((i < fill)) && out+="█" || out+="─"
	done
	printf '%s%s%s%s%s' "$col" "${out:0:fill}" "${D}${RULE}" "${out:fill}" "$R"
}

# segbar WIDTH TOTAL  v1 c1  v2 c2  ...  — a stacked, multi-segment meter
segbar() {
	local w=$1 total=$2
	shift 2
	local out="" used=0
	while [[ $# -ge 2 ]]; do
		local v=$1 c=$2
		shift 2
		local n=$((total > 0 ? v * w / total : 0))
		((used + n > w)) && n=$((w - used))
		((n > 0)) && {
			out+="${c}$(printf '█%.0s' $(seq "$n"))"
			used=$((used + n))
		}
	done
	((used < w)) && out+="${D}${RULE}$(printf '─%.0s' $(seq $((w - used))))"
	printf '%s%s' "$out" "$R"
}

# spark VALUES... — unicode sparkline (values are 0-100)
spark() {
	local out="" v idx
	for v in "$@"; do
		v=${v%%.*}
		((v < 0)) && v=0
		((v > 100)) && v=100
		idx=$((v * 7 / 100))
		out+="${BLOCKS[$idx]}"
	done
	printf '%s' "$out"
}

# rule TITLE — full-width section header
rule() {
	local title=" $1 "
	local n=$((WIDTH - ${#title} - 3))
	((n < 0)) && n=0
	printf '%s╺━%s%s%s%s%s\n' "$RULE" "$B$HEAD" "$title" "$R$RULE" "$(printf '━%.0s' $(seq "$n"))" "$R"
}

# Every memory figure on the dashboard is whole MiB: no auto-scaling to GiB,
# no fractions. Keeps columns comparable at a glance instead of forcing the
# reader to rescale between rows.
# Prepended to each awk program below so "124546" reads as "124,546".
AWK_COMMIFY='function commify(n,   t, o) {
	t = sprintf("%d", n + 0.5); o = ""
	while (length(t) > 3) { o = "," substr(t, length(t)-2) o; t = substr(t, 1, length(t)-3) }
	return t o
}'

human() { # bytes -> MiB
	awk -v b="$1" "$AWK_COMMIFY"'BEGIN{ printf "%s M", commify(b/1048576) }'
}

mib_h() { # MiB -> MiB, rounded to whole
	awk -v m="$1" "$AWK_COMMIFY"'BEGIN{ printf "%s M", commify(m) }'
}

# docker stats prints memory pre-formatted ("1.234GiB", "512MiB", "9.7kB"), so
# parse it back to a bare MiB number — the table sorts on it as well as printing
# it, so mib_h does the commifying separately. Decimal MB/GB are treated as
# binary; the difference is below one rounding step at the sizes we show.
to_mib_n() {
	awk -v s="$1" 'BEGIN{
		n = s + 0
		u = s; sub(/^[0-9.]+/, "", u)
		if      (u ~ /^[Kk]/) f = 1/1024
		else if (u ~ /^[Mm]/) f = 1
		else if (u ~ /^[Gg]/) f = 1024
		else if (u ~ /^[Tt]/) f = 1048576
		else                  f = 1/1048576
		printf "%d", n*f + 0.5
	}'
}

# seconds until an ISO-8601 timestamp, formatted compactly
until_ts() {
	local target now diff
	target=$(date -d "$1" +%s 2>/dev/null) || {
		printf '?'
		return
	}
	now=$(date +%s)
	diff=$((target - now))
	((diff <= 0)) && {
		printf 'elapsed'
		return
	}
	local d=$((diff / 86400)) h=$((diff % 86400 / 3600)) m=$((diff % 3600 / 60))
	((d > 0)) && printf '%dd %dh' "$d" "$h" && return
	((h > 0)) && printf '%dh %02dm' "$h" "$m" && return
	printf '%dm' "$m"
}

# ── collectors (run in parallel into $TMPDIR_RUN) ────────────────────────────

collect_gpu() {
	nvidia-smi --query-gpu=name,utilization.gpu,temperature.gpu,power.draw,clocks.sm,clocks.max.sm,driver_version \
		--format=csv,noheader,nounits 2>/dev/null >"$TMPDIR_RUN/gpu"
	nvidia-smi --query-compute-apps=pid,used_memory,name \
		--format=csv,noheader,nounits 2>/dev/null >"$TMPDIR_RUN/apps"
}

collect_mem() { grep -E '^(MemTotal|MemAvailable|MemFree|Cached|Buffers|SReclaimable):' /proc/meminfo >"$TMPDIR_RUN/mem"; }

# real, local filesystems only — pseudo (tmpfs/overlay/proc/...) and NFS mounts
# (already called out separately per the NAS convention) would just be noise
collect_disk() {
	local min_bytes
	min_bytes=$(awk -v g="$MIN_DISK_GB" 'BEGIN{ printf "%.0f", g*1073741824 }')
	df -P -B1 -x tmpfs -x devtmpfs -x overlay -x squashfs -x proc -x sysfs \
		-x cgroup2 -x devpts -x mqueue -x tracefs -x debugfs -x efivarfs \
		-x nfs4 -x nfs 2>/dev/null |
		awk -v min="$min_bytes" 'NR>1 && $2>=min { mp=$6; for(i=7;i<=NF;i++) mp=mp" "$i; printf "%s\t%s\t%s\t%s\t%s\n", $1,$2,$3,$4,mp }' |
		sort -t$'\t' -k5,5 |
		head -n 6 >"$TMPDIR_RUN/disk"
}

collect_docker() {
	docker stats --no-stream --format '{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}' \
		2>/dev/null >"$TMPDIR_RUN/dstats"
	docker ps --format '{{.ID}}\t{{.Names}}\t{{.Status}}' 2>/dev/null >"$TMPDIR_RUN/dps"
	# StartedAt for the uptime sort. {{.Status}} coarsens to whole days once a
	# container is a day old, so most of the stack would tie on "Up 2 days".
	# One inspect covers every container, so this costs two forks, not N.
	local ids
	ids=$(docker ps -q 2>/dev/null)
	if [[ -n $ids ]]; then
		# shellcheck disable=SC2086 # deliberate word-split: one id per argument
		docker inspect --format '{{.Name}} {{.State.StartedAt}}' $ids \
			2>/dev/null >"$TMPDIR_RUN/dstart"
	else
		: >"$TMPDIR_RUN/dstart"
	fi
}

collect_ollama() {
	curl -s --max-time 3 "$OLLAMA_URL/api/ps" 2>/dev/null >"$TMPDIR_RUN/ollama"
}

# The live API response and the cached ~/.claude.json blob have the same shape,
# so one jq program formats either — the caller just picks the root.
CLAUDE_JQ='
	[ (.limits // [])[]
	  | [ .kind,
	      ( if .scope.model.display_name then "\(.kind)|\(.scope.model.display_name)" else .kind end ),
	      (.percent // 0), (.resets_at // ""), (.severity // "normal"),
	      (if .is_active then "1" else "0" end)
	    ] | "LIM\t" + join("\t") ] []
	, ( .extra_usage
	    | if (.is_enabled // false) then "EXTRA\t\(.utilization // 0)" else empty end )
'

collect_claude() {
	local out="$TMPDIR_RUN/claude" now
	now=$(date +%s)

	# Ask Claude Code itself, rate-limited to CLAUDE_REFRESH — the CLI takes
	# ~2s of CPU per call, which is not something to do on every frame.
	if ((CLAUDE_LIVE)) && command -v claude >/dev/null 2>&1; then
		local last=0 wait_s=$CLAUDE_REFRESH
		[[ -f "$TMPDIR_RUN/claude.at" ]] && last=$(<"$TMPDIR_RUN/claude.at")
		[[ -f "$TMPDIR_RUN/claude.backoff" ]] && wait_s=$(<"$TMPDIR_RUN/claude.backoff")
		if ((now - last >= wait_s)); then
			echo "$now" >"$TMPDIR_RUN/claude.at"
			if timeout 30 claude -p '/usage' \
				--output-format json \
				--no-session-persistence \
				--strict-mcp-config --mcp-config '{"mcpServers":{}}' >/dev/null 2>&1; then
				: >"$TMPDIR_RUN/usage.err"
				echo "$CLAUDE_REFRESH" >"$TMPDIR_RUN/claude.backoff"
			else
				echo "cli" >"$TMPDIR_RUN/usage.err"
				# Repeated failures back off so a broken or hanging CLI doesn't
				# get re-spawned on every cycle for the life of the dashboard.
				local nb=$((wait_s * 2))
				((nb > CLAUDE_BACKOFF_MAX)) && nb=$CLAUDE_BACKOFF_MAX
				echo "$nb" >"$TMPDIR_RUN/claude.backoff"
			fi
		fi
	fi

	# One source, always: Claude Code's own cache, which the /usage run above
	# has just refreshed. A blob young enough to be ours counts as live; older
	# than that means the run failed or is switched off, and the panel says so.
	if [[ -r "$CLAUDE_JSON" ]]; then
		local at=0 src=cache
		at=$(jq -r '((.cachedUsageUtilization.fetchedAtMs // 0) / 1000 | floor)' "$CLAUDE_JSON" 2>/dev/null) || at=0
		[[ $at =~ ^[0-9]+$ ]] || at=0
		((CLAUDE_LIVE && at > 0 && now - at <= CLAUDE_REFRESH + 60)) && src=live
		{
			printf 'SRC\t%s\t%s\t%s\n' "$src" "$at" "$(cat "$TMPDIR_RUN/usage.err" 2>/dev/null)"
			jq -r ".cachedUsageUtilization.utilization // {} | $CLAUDE_JQ" "$CLAUDE_JSON" 2>/dev/null
		} >"$out"
	else
		: >"$out"
	fi
}

# map a host PID to a container short-id via its cgroup
pid_container() {
	local cg
	cg=$(tr -d '\0' <"/proc/$1/cgroup" 2>/dev/null) || return
	[[ $cg =~ docker-([0-9a-f]{12}) ]] && printf '%s' "${BASH_REMATCH[1]}"
}

# ── panels ───────────────────────────────────────────────────────────────────

UTIL_HIST=()
MEM_HIST=()

panel_header() {
	local name="" drv="" host up now
	IFS=',' read -r name _ _ _ _ _ drv <"$TMPDIR_RUN/gpu" 2>/dev/null || :
	name="${name## }"
	drv="${drv## }"
	host=$(hostname -s)
	up=$(uptime -p 2>/dev/null | sed 's/^up //')
	now=$(date '+%a %d %b  %H:%M:%S')
	local left="${B}${HEAD}sparktop${R}  ${DIMV}·${R}  ${ACC}${host}${R}  ${DIMV}·${R}  ${VAL}${name:-GPU}${R}  ${DIMV}driver ${drv:-?}${R}"
	local right="${DIMV}up ${up:-?}  ·  ${R}${VAL}${now}${R}"
	local lp="sparktop  ·  ${host}  ·  ${name:-GPU}  driver ${drv:-?}"
	local rp="up ${up:-?}  ·  ${now}"
	local gap=$((WIDTH - ${#lp} - ${#rp}))
	((gap < 1)) && gap=1
	printf '%s%s%s\n' "$left" "$(printf ' %.0s' $(seq $gap))" "$right"
}

panel_gpu() {
	rule "GPU"
	local name="" util=0 temp=0 pwr=0 sm=0 smmax=0
	IFS=',' read -r name util temp pwr sm smmax _ <"$TMPDIR_RUN/gpu" 2>/dev/null || :
	util=$(echo "${util:-0}" | tr -dc '0-9')
	temp=$(echo "${temp:-0}" | tr -dc '0-9')
	sm=$(echo "${sm:-0}" | tr -dc '0-9')
	smmax=$(echo "${smmax:-0}" | tr -dc '0-9')
	pwr=$(echo "${pwr:-0}" | tr -dc '0-9.')
	: "${util:=0}" "${temp:=0}" "${sm:=0}"

	UTIL_HIST+=("$util")
	((${#UTIL_HIST[@]} > HIST_LEN)) && UTIL_HIST=("${UTIL_HIST[@]: -$HIST_LEN}")

	local bw=$((WIDTH - 46))
	((bw < 10)) && bw=10
	((bw > 46)) && bw=46

	printf '  %sUtil %s %s %s%4s%%%s   %shist%s %s%s%s\n' \
		"$LBL" "$R" "$(bar "$bw" "$util")" "$B$(sev "$util")" "$util" "$R" \
		"$DIMV" "$R" "$(sev "$util")" "$(spark "${UTIL_HIST[@]}")" "$R"

	local tcol
	tcol=$(sev $((temp * 100 / 90)))
	printf '  %sTemp%s %s%s°C%s   %sPower%s %s%sW%s   %sSM clock%s %s%s%s %sMHz (max %s)%s\n' \
		"$LBL" "$R" "$tcol" "${temp:-?}" "$R" \
		"$LBL" "$R" "$VAL" "${pwr:-?}" "$R" \
		"$LBL" "$R" "$VAL" "${sm:-?}" "$R" "$DIMV" "${smmax:-?}" "$R"
}

panel_memory() {
	rule "UNIFIED MEMORY  (GB10 — CPU and GPU share one pool)"
	local total avail mfree cached buffers sreclaim
	total=$(awk '/^MemTotal:/{print $2}' "$TMPDIR_RUN/mem")
	avail=$(awk '/^MemAvailable:/{print $2}' "$TMPDIR_RUN/mem")
	mfree=$(awk '/^MemFree:/{print $2}' "$TMPDIR_RUN/mem")
	# anchor "Cached:" so it doesn't also match SwapCached/SReclaimable
	cached=$(awk '/^Cached:/{print $2}' "$TMPDIR_RUN/mem")
	buffers=$(awk '/^Buffers:/{print $2}' "$TMPDIR_RUN/mem")
	sreclaim=$(awk '/^SReclaimable:/{print $2}' "$TMPDIR_RUN/mem")
	: "${total:=0}" "${avail:=0}" "${mfree:=0}" "${cached:=0}" "${buffers:=0}" "${sreclaim:=0}"

	# Mirror free(1): used = total - free - buff/cache. Do NOT subtract
	# MemAvailable — it already counts reclaimable cache, so total-avail-cache
	# double-counts and goes negative once the page cache is large.
	local free_kb=$mfree
	local cache_kb=$((cached + buffers + sreclaim))
	local used_kb=$((total - free_kb - cache_kb))
	((used_kb < 0)) && used_kb=0

	# GPU share = sum of per-process compute allocations (MiB -> KiB)
	local gpu_kb=0 m
	while IFS=',' read -r _ m _; do
		m=$(echo "$m" | tr -dc '0-9')
		[[ -n "$m" ]] && gpu_kb=$((gpu_kb + m * 1024))
	done <"$TMPDIR_RUN/apps"
	((gpu_kb > used_kb)) && gpu_kb=$used_kb
	local app_kb=$((used_kb - gpu_kb))

	local pct=$((total > 0 ? used_kb * 100 / total : 0))
	MEM_HIST+=("$pct")
	((${#MEM_HIST[@]} > HIST_LEN)) && MEM_HIST=("${MEM_HIST[@]: -$HIST_LEN}")

	local bw=$((WIDTH - 46))
	((bw < 10)) && bw=10
	((bw > 46)) && bw=46

	printf '  %sPool %s %s %s%4s%%%s   %shist%s %s%s%s\n' \
		"$LBL" "$R" \
		"$(segbar "$bw" "$total" "$gpu_kb" "$GPUC" "$app_kb" "$APPC" "$cache_kb" "$CACHEC")" \
		"$B$(sev "$pct")" "$pct" "$R" "$DIMV" "$R" "$(sev "$pct")" "$(spark "${MEM_HIST[@]}")" "$R"

	printf '  %s█ GPU%s %s%-10s%s %s█ apps%s %s%-10s%s %s█ cache%s %s%-10s%s %s· free%s %s%s%s %s(avail %s) / %s%s\n' \
		"$GPUC" "$R" "$VAL" "$(human $((gpu_kb * 1024)))" "$R" \
		"$APPC" "$R" "$VAL" "$(human $((app_kb * 1024)))" "$R" \
		"$CACHEC" "$R" "$VAL" "$(human $((cache_kb * 1024)))" "$R" \
		"$DIMV" "$R" "$OK" "$(human $((free_kb * 1024)))" "$R" \
		"$DIMV" "$(human $((avail * 1024)))" "$(human $((total * 1024)))" "$R"
}

panel_disk() {
	[[ -s "$TMPDIR_RUN/disk" ]] || return
	rule "DISK"
	local bw=$((WIDTH - 46))
	((bw < 10)) && bw=10
	((bw > 36)) && bw=36
	local namew=14

	local fs total used avail mount pct
	while IFS=$'\t' read -r fs total used avail mount; do
		[[ -z "$total" || "$total" -eq 0 ]] && continue
		pct=$((used * 100 / total))
		printf '  %s%s%s %s %s%4s%%%s   %s%s%s / %s%s%s\n' \
			"$LBL" "$(pad "$mount" "$namew")" "$R" \
			"$(bar "$bw" "$pct")" "$B$(sev "$pct")" "$pct" "$R" \
			"$VAL" "$(human "$used")" "$R" "$DIMV" "$(human "$total")" "$R"
	done <"$TMPDIR_RUN/disk"
}

panel_gpu_procs() {
	[[ -s "$TMPDIR_RUN/apps" ]] || return
	local maxrows=${1:-6}
	rule "GPU PROCESSES"
	printf '  %s%s %s %s %s%s\n' "$DIMV" \
		"$(pad PID 8)" "$(padr 'GPU MEM' 9)" "$(pad CONTAINER 18)" "$(pad COMMAND 30)" "$R"
	local pid mem cmd cid cname
	while IFS=',' read -r pid mem cmd; do
		pid=$(echo "$pid" | tr -dc '0-9')
		mem=$(echo "$mem" | tr -dc '0-9')
		[[ -z "$pid" ]] && continue
		cmd="${cmd## }"
		cid=$(pid_container "$pid")
		cname="—"
		[[ -n "$cid" ]] && cname=$(awk -v i="$cid" -F'\t' '$1==i{print $2}' "$TMPDIR_RUN/dps")
		[[ -z "$cname" ]] && cname="${cid:0:12}"
		printf '  %s%s%s %s%s%s %s%s%s %s%s%s\n' \
			"$DIMV" "$(pad "$pid" 8)" "$R" \
			"$GPUC" "$(padr "$(mib_h "${mem:-0}")" 9)" "$R" \
			"$ACC" "$(pad "$cname" 18)" "$R" \
			"$VAL" "$(trunc "$cmd" $((WIDTH - 42)))" "$R"
	done < <(sort -t',' -k2 -rn "$TMPDIR_RUN/apps" | head -n "$maxrows")
}

panel_docker() {
	local maxrows=${1:-99}
	rule "DOCKER CONTAINERS"
	if [[ ! -s "$TMPDIR_RUN/dstats" ]]; then
		printf '  %sno running containers (or docker unavailable)%s\n' "$DIMV" "$R"
		return
	fi

	# maxrows 0 = no room for the table; collapse to a single summary line
	if ((maxrows == 0)); then
		local ngpu=0 pid mem cid
		while IFS=',' read -r pid mem _; do
			pid=$(echo "$pid" | tr -dc '0-9')
			[[ -z "$pid" ]] && continue
			cid=$(pid_container "$pid")
			[[ -n "$cid" ]] && ngpu=$((ngpu + 1))
		done <"$TMPDIR_RUN/apps"
		printf '  %s%s running%s%s · %s\n' \
			"$VAL" "$(wc -l <"$TMPDIR_RUN/dstats")" "$R" \
			"$([[ $ngpu -gt 0 ]] && printf '%s, %d on GPU%s' "$GPUC" "$ngpu" "$R")" \
			"${DIMV}grow the window for the table${R}"
		return 0
	fi

	# container short-id -> GPU MiB, summed from compute processes
	declare -A GPUMEM=()
	local pid mem cid
	while IFS=',' read -r pid mem _; do
		pid=$(echo "$pid" | tr -dc '0-9')
		mem=$(echo "$mem" | tr -dc '0-9')
		[[ -z "$pid" || -z "$mem" ]] && continue
		cid=$(pid_container "$pid")
		[[ -n "$cid" ]] && GPUMEM[$cid]=$((${GPUMEM[$cid]:-0} + mem))
	done <"$TMPDIR_RUN/apps"

	declare -A NAME2ID=()
	local id nm st
	while IFS=$'\t' read -r id nm st; do
		NAME2ID[$nm]="$id"
		STATUS[$nm]="$st"
	done <"$TMPDIR_RUN/dps"

	# name -> ISO start time, for the uptime sort
	declare -A STARTED=()
	local sat
	while read -r nm sat; do
		[[ -n "$nm" ]] && STARTED["${nm#/}"]="$sat"
	done <"$TMPDIR_RUN/dstart"

	local namew=$((WIDTH - 52))
	((namew < 14)) && namew=14
	((namew > 26)) && namew=26

	# ▾ marks the column the table is ordered by
	local mode="${SORT_MODES[SORT_IDX]}"
	local h_cpu=CPU h_mem=MEM h_gpu='GPU MEM' h_up=UPTIME
	case "$mode" in
	cpu) h_cpu+=' ▾' ;;
	mem) h_mem+=' ▾' ;;
	gpu) h_gpu+=' ▾' ;;
	uptime) h_up+=' ▾' ;;
	esac
	printf '  %s%s %s %s %s %s %s%s\n' "$DIMV" \
		"$(pad NAME "$namew")" "$(padr "$h_cpu" 7)" "$(padr "$h_mem" 9)" "$(padr 'MEM%' 6)" \
		"$(padr "$h_gpu" 9)" "$(pad "$h_up" 8)" "$R"

	# reserve a row for the "N more" line when the list won't fit
	local total shown=0 limit
	total=$(wc -l <"$TMPDIR_RUN/dstats")
	limit=$maxrows
	((total > maxrows)) && limit=$((maxrows - 1))
	((limit < 1)) && limit=1

	# Format every container first and sort afterwards: the sort key for two of
	# the four modes (GPU memory, uptime) isn't in the docker stats output, so
	# it can't be a `sort` on the raw file — and the order has to be settled
	# before the row limit is applied, or a short window would hide the wrong
	# containers. Rows are keyed as SORTKEY \1 MiB \1 rendered-line; MiB is the
	# tiebreak everywhere, so equal keys (all the zero-GPU containers, say)
	# still come out in a stable, meaningful order.
	local rows="$TMPDIR_RUN/drows" key
	local name cpu memu memp cpun mempn memn cpuf gmem gtxt row
	while IFS=$'\t' read -r name cpu memu memp; do
		# docker gives "12.34%" / "1.234GiB / 119.7GiB" — drop the fractions and
		# the limit half, and restate memory in MiB. A container that is
		# restarting reports "--", which stays as-is rather than becoming a 0.
		# the sort keeps docker's fractional CPU even though the column doesn't,
		# so two containers both showing "0%" still order sensibly
		cpuf="${cpu%\%}"
		[[ $cpuf =~ ^[0-9.]+$ ]] || cpuf=0
		if [[ $cpu =~ ^([0-9]+) ]]; then
			cpun="${BASH_REMATCH[1]}"
			cpu="${cpun}%"
		else cpun=0; fi
		if [[ $memp =~ ^([0-9]+) ]]; then
			mempn="${BASH_REMATCH[1]}"
			memp="${mempn}%"
		else mempn=0; fi
		memn=$(to_mib_n "${memu%% /*}")
		memu=$(mib_h "$memn")
		id="${NAME2ID[$name]:-}"
		gmem=${GPUMEM[$id]:-0}
		if ((gmem > 0)); then
			gtxt="${GPUC}$(padr "$(mib_h "$gmem")" 9)${R}"
		else
			gtxt="${DIMV}$(padr '—' 9)${R}"
		fi
		# "Up 45 hours (healthy)" -> "45h" plus a health glyph
		st="${STATUS[$name]:-}"
		local stc="$DIMV" mark="·" dur="?"
		case "$st" in
		*"(healthy)"*)
			mark="●"
			stc="$OK"
			;;
		*"(unhealthy)"*)
			mark="●"
			stc="$CRIT"
			;;
		*starting*)
			mark="◐"
			stc="$WARN"
			;;
		Restarting*)
			mark="⟳"
			stc="$CRIT"
			;;
		Exited* | Dead*)
			mark="✕"
			stc="$CRIT"
			;;
		Up*)
			mark="·"
			stc="$OK"
			;;
		esac
		if [[ $st =~ Up[[:space:]]+([0-9]+)[[:space:]]+(second|minute|hour|day|week|month) ]]; then
			case "${BASH_REMATCH[2]}" in
			second) dur="${BASH_REMATCH[1]}s" ;;
			minute) dur="${BASH_REMATCH[1]}m" ;;
			hour) dur="${BASH_REMATCH[1]}h" ;;
			day) dur="${BASH_REMATCH[1]}d" ;;
			week) dur="${BASH_REMATCH[1]}w" ;;
			month) dur="${BASH_REMATCH[1]}mo" ;;
			esac
		elif [[ $st == Up*econd* || $st == Up*About\ a\ minute* ]]; then
			dur="<1m"
		else
			dur="${st%% *}"
		fi
		printf -v row '  %s%s%s %s%s%s %s%s%s %s%s%s %s %s%s %s' \
			"$ACC" "$(pad "$name" "$namew")" "$R" \
			"$(sev "${cpun:-0}")" "$(padr "$cpu" 7)" "$R" \
			"$VAL" "$(padr "$memu" 9)" "$R" \
			"$(sev "$mempn")" "$(padr "$memp" 6)" "$R" \
			"$gtxt" \
			"$stc" "$mark" "$(pad "$dur" 6)" "$R"
		case "$mode" in
		gpu) key="$gmem" ;;
		mem) key="$memn" ;;
		cpu) key="$cpuf" ;;
		# RFC-3339 start times are fixed-field, so they order correctly as
		# plain strings — no epoch conversion, and awk needn't have mktime.
		# Oldest first, and a container with no start time sorts last.
		uptime) key="${STARTED[$name]:-9}" ;;
		esac
		printf '%s\001%s\001%s\n' "$key" "$memn" "$row"
	done <"$TMPDIR_RUN/dstats" >"$rows"

	local -a sortopt
	case "$mode" in
	uptime) sortopt=(-t $'\001' -k1,1 -k2,2rn) ;;
	*) sortopt=(-t $'\001' -k1,1rn -k2,2rn) ;;
	esac
	while IFS= read -r row; do
		((shown++))
		printf '%s\n' "$row"
	done < <(LC_ALL=C sort "${sortopt[@]}" "$rows" | head -n "$limit" | cut -d$'\001' -f3-)

	((total > shown)) && printf '  %s… %d more (grow the window to see them)%s\n' \
		"$DIMV" "$((total - shown))" "$R"
	return 0
}

panel_ollama() {
	rule "OLLAMA MODELS"
	local json
	json=$(cat "$TMPDIR_RUN/ollama" 2>/dev/null)
	if [[ -z "$json" ]]; then
		printf '  %sollama not reachable at %s%s\n' "$DIMV" "$OLLAMA_URL" "$R"
		return
	fi
	local n
	n=$(jq -r '.models | length' <<<"$json" 2>/dev/null) || n=0
	if [[ "${n:-0}" == "0" ]]; then
		printf '  %sno models loaded%s\n' "$DIMV" "$R"
		return
	fi

	local namew=$((WIDTH - 56))
	((namew < 16)) && namew=16
	((namew > 32)) && namew=32

	printf '  %s%s %s %s %s %s %s%s\n' "$DIMV" \
		"$(pad MODEL "$namew")" "$(padr SIZE 9)" "$(padr 'IN GPU' 9)" "$(padr PLACEMENT 11)" \
		"$(padr CONTEXT 8)" "$(pad EXPIRES 9)" "$R"

	local name size vram ctx exp params quant place placepct
	while IFS=$'\t' read -r name size vram ctx exp params quant; do
		[[ -z "$name" ]] && continue
		placepct=$((size > 0 ? vram * 100 / size : 0))
		if ((placepct >= 99)); then place="100% GPU"; elif ((placepct == 0)); then place="100% CPU"; else place="${placepct}% GPU"; fi
		printf '  %s%s%s %s%s%s %s%s%s %s%s%s %s%s%s %s%s%s %s%s%s\n' \
			"$ACC" "$(pad "$name" "$namew")" "$R" \
			"$VAL" "$(padr "$(human "$size")" 9)" "$R" \
			"$GPUC" "$(padr "$(human "$vram")" 9)" "$R" \
			"$(sev $((100 - placepct)))" "$(padr "$place" 11)" "$R" \
			"$VAL" "$(padr "$(awk -v c="$ctx" 'BEGIN{ if (c>=1024) printf "%dK", c/1024; else print c }')" 8)" "$R" \
			"$DIMV" "$(pad "$(until_ts "$exp")" 9)" "$R" \
			"$DIMV" "${params:+$params }${quant:-}" "$R"
	done < <(jq -r '.models[] | [ (.name // .model), (.size // 0), (.size_vram // 0),
		(.context_length // 0), (.expires_at // ""),
		(.details.parameter_size // ""), (.details.quantization_level // "") ] | @tsv' <<<"$json" 2>/dev/null)
}

panel_claude() {
	rule "CLAUDE USAGE LIMITS"
	if [[ ! -s "$TMPDIR_RUN/claude" ]]; then
		printf '  %sno cached usage data in %s%s\n' "$DIMV" "$CLAUDE_JSON" "$R"
		return
	fi

	local src="" src_at=0 src_err="" have=0
	local bw=$((WIDTH - 48))
	((bw < 10)) && bw=10
	((bw > 40)) && bw=40

	local tag kind label pct resets severity active title
	while IFS=$'\t' read -r tag rest; do
		case "$tag" in
		SRC)
			IFS=$'\t' read -r src src_at src_err <<<"$rest"
			;;
		LIM)
			IFS=$'\t' read -r kind label pct resets severity active <<<"$rest"
			have=1
			case "${label%%|*}" in
			session) title="Session (5h)" ;;
			weekly_all) title="Weekly (all models)" ;;
			weekly_scoped) title="Weekly (${label#*|})" ;;
			*) title="$kind" ;;
			esac
			local col
			case "$severity" in
			warning) col="$WARN" ;;
			critical | exceeded) col="$CRIT" ;;
			*) col="$(sev "$pct")" ;;
			esac
			local mark="  "
			[[ "$active" == "1" ]] && mark="${ACC}▸ ${R}"
			# a reset time in the past means this window already rolled over and
			# the cached percentage is stale — say so rather than "resets in ..."
			local rt when
			rt=$(until_ts "$resets")
			if [[ "$rt" == elapsed ]]; then
				when="${DIMV}window rolled over${R}"
				col="$DIMV"
			else
				when="${DIMV}resets in ${R}${VAL}${rt}${R}"
			fi
			printf '  %s%s%s %s %s%4s%%%s   %s\n' \
				"$mark" "$LBL" "$(pad "$title" 22)" \
				"$(bar "$bw" "$pct" "$col")" "$B$col" "$pct" "$R" "$when"
			;;
		EXTRA)
			printf '  %s  %s %s %s%4s%%%s\n' "$LBL" "$(pad 'Extra usage' 22)" \
				"$(bar "$bw" "$rest")" "$B$(sev "$rest")" "$rest" "$R"
			;;
		esac
	done <"$TMPDIR_RUN/claude"

	((have == 0)) && printf '  %sno usage data — sign in with Claude Code, or check --no-live%s\n' "$DIMV" "$R"

	# provenance line: live poll vs Claude Code's cache, and how old it is
	local age_s=0 agetxt="?" col="$DIMV" note=""
	((src_at > 0)) && age_s=$(($(date +%s) - src_at))
	((age_s < 0)) && age_s=0
	if ((age_s < 90)); then agetxt="${age_s}s ago"; elif ((age_s < 5400)); then agetxt="$((age_s / 60))m ago"; else agetxt="$((age_s / 3600))h ago"; fi

	case "$src" in
	live)
		note="live · /usage refreshed ${agetxt}"
		;;
	cache)
		col="$WARN"
		note="cached by Claude Code ${agetxt}"
		if ((CLAUDE_LIVE == 0)); then
			note+=" · live polling off"
			col="$DIMV"
		elif ! command -v claude >/dev/null 2>&1; then
			note+=" · claude CLI not on PATH"
		elif [[ -n "$src_err" ]]; then
			note+=" · claude -p /usage failed"
		fi
		;;
	*) return ;;
	esac

	printf '  %s  ▸ active window · %s%s\n' "$col" "$note" "$R"
}

panel_footer() {
	local hint="q quit   r refresh   s sort:${SORT_LABEL[${SORT_MODES[SORT_IDX]}]}   ·   every ${INTERVAL}s"
	local n=$((WIDTH - ${#hint} - 2))
	((n < 0)) && n=0
	printf '%s%s%s %s%s\n' "$RULE" "$(printf '━%.0s' $(seq "$n"))" "$R" "$DIMV$hint" "$R"
}

# ── main loop ────────────────────────────────────────────────────────────────
declare -A STATUS=()

nlines() { printf '%s\n' "$1" | wc -l; }

SKIP_COLLECT=0
# refresh interval in tenths of a second, for the sliced wait below
TENTHS=$(awk -v i="$INTERVAL" 'BEGIN{ printf "%d", (i*10 < 4 ? 4 : i*10) }')

while :; do
	# one fork for both dimensions instead of two tput calls
	if ! read -r HEIGHT WIDTH < <(stty size 2>/dev/null); then
		HEIGHT=$(tput lines 2>/dev/null || echo 40)
		WIDTH=$(tput cols 2>/dev/null || echo 100)
	fi
	((WIDTH < 60)) && WIDTH=60
	((WIDTH > 160)) && WIDTH=160
	((HEIGHT < 16)) && HEIGHT=16

	# A shrink leaves debris the per-line erase can't reach (rows below the new
	# bottom, and anything the old frame wrote past the new width), so wipe once.
	if ((RESIZED)); then
		RESIZED=0
		printf '%s[2J' "$E"
	fi

	# On a resize wake, repaint from the data we already have — re-collecting
	# would stall the redraw for the ~2s docker stats takes.
	if ((SKIP_COLLECT == 0)); then
		# guarantee every panel has a file to read, even if a tool is missing
		touch "$TMPDIR_RUN"/{gpu,apps,mem,disk,dstats,dps,dstart,ollama,claude} 2>/dev/null

		collect_gpu &
		collect_mem &
		collect_disk &
		collect_docker &
		collect_ollama &
		collect_claude &
		wait
	fi
	SKIP_COLLECT=0

	# Fit the frame to the terminal. If the frame is taller than the screen the
	# terminal scrolls while drawing, so the next \e[H homes to a shifted origin
	# and the whole dashboard creeps upward on every refresh. Everything above
	# and below the container table is fixed-height, so measure those and give
	# the table whatever rows are left.
	# Layout tiers, richest first. On a short terminal shed the low-value panels
	# (GPU process list, then Ollama) rather than letting the clip eat the
	# Claude panel and footer off the bottom.
	for tier in "6 1 1 1" "3 1 1 1" "0 1 1 1" "0 0 1 1" "0 0 0 1" "0 0 0 0"; do
		read -r gp show_ollama docker_table show_footer <<<"$tier"
		local_top=$(
			panel_header
			echo
			panel_gpu
			echo
			panel_memory
			echo
			panel_disk
			((gp > 0)) && {
				echo
				panel_gpu_procs "$gp"
			}
			true
		)
		local_bot=$(
			((show_ollama)) && {
				panel_ollama
				echo
			}
			panel_claude
			((show_footer)) && {
				echo
				panel_footer
			}
			true
		)
		if ((docker_table)); then
			# 4 = blank line either side of the table + its rule + column header
			budget=$((HEIGHT - 1 - $(nlines "$local_top") - $(nlines "$local_bot") - 4))
			((budget >= 2)) && break
		else
			# summary mode: rule + one line, plus the blank line either side
			budget=0
			(($(nlines "$local_top") + $(nlines "$local_bot") + 4 <= HEIGHT - 1)) && break
		fi
	done
	((docker_table && budget < 2)) && budget=2

	frame=$(
		printf '%s\n\n' "$local_top"
		panel_docker "$budget"
		printf '\n%s' "$local_bot"
	)

	# belt and braces: never emit more rows than the screen has
	frame=$(printf '%s\n' "$frame" | head -n $((HEIGHT - 1)))

	# Repaint in one write: home the cursor, draw, then clear anything below.
	# Every line gets \e[K so a line shorter than the one previously at that row
	# doesn't leave its tail behind (very visible when the window narrows).
	# No trailing newline — on a full-height frame it would scroll by one row.
	printf '%s[H%s%s[K%s[J' "$E" "${frame//$'\n'/$'\e'[K$'\n'}" "$E" "$E"

	# rc >128 means the read timed out (the normal path); rc 1 means stdin is
	# closed or not a terminal, so fall back to a plain sleep instead of spinning
	# Wait in short slices rather than one long read. bash will not always
	# interrupt a blocked `read` to run the WINCH trap, so a single long read
	# can sit on a stale frame for the whole interval; slicing bounds the delay
	# to one slice without polling the terminal size (which would cost a fork).
	waited=0
	while ((waited < TENTHS)); do
		key=""
		read -rsn1 -t 0.4 key
		rc=$?
		if ((rc == 0)); then
			case "$key" in
			q | Q) exit 0 ;;
			# reordering is a repaint, not new data — skip the collectors so
			# the table flips the moment the key lands
			s | S)
				SORT_IDX=$(((SORT_IDX + 1) % ${#SORT_MODES[@]}))
				SKIP_COLLECT=1
				;;
			esac
			break # any other key: refresh immediately
		elif ((rc == 1)); then
			# stdin closed / not a terminal — don't spin
			sleep "$INTERVAL"
			break
		fi
		((RESIZED)) && {
			SKIP_COLLECT=1
			break
		}
		waited=$((waited + 4))
	done
done