vLLM deadlock watchdog — detect stalled inference and recover via ss -K

vllm-watchdog.sh

#!/bin/bash
# vLLM deadlock watchdog — detect stalled inference and recover by dropping stale TCP connections.
# Runs as a system service (root).

PORT="${VLLM_PORT:-8000}"
STALL_WINDOW="${VLLM_WATCHDOG_STALL:-30}"
CHECK_INTERVAL="${VLLM_WATCHDOG_INTERVAL:-5}"
RECOVERY_COOLDOWN="${VLLM_WATCHDOG_COOLDOWN:-15}"

log() { echo "$(date '+%F %T') $*"; }

metric() {
  timeout 3 curl -sf "http://localhost:${PORT}/metrics" 2>/dev/null \
    | awk -v name="${1}" '$1 ~ "^vllm:" name "\\{" { print $2; exit }'
}

delta() {
  python3 -c "import sys; print(int(round(float('${1}') - float('${2}'))))"
}

check_state() {
  local running waiting
  running="$(metric num_requests_running)" || return 2
  waiting="$(metric num_requests_waiting)" || return 2
  local busy=$(( $(printf '%.0f' "${running:-0}") + $(printf '%.0f' "${waiting:-0}") ))

  local pa pb ga gb
  pa="$(metric prompt_tokens_total)" || return 2
  ga="$(metric generation_tokens_total)" || return 2

  sleep "${STALL_WINDOW}"

  pb="$(metric prompt_tokens_total)" || return 2
  gb="$(metric generation_tokens_total)" || return 2

  local pd gd
  pd="$(delta "$pb" "$pa")"
  gd="$(delta "$gb" "$ga")"
  log "prompt(+${pd}) generation(+${gd}) | busy=${busy} (running=${running} waiting=${waiting})"

  [ "$busy" -eq 0 ] && return 0
  { [ "$pd" -gt 0 ] || [ "$gd" -gt 0 ]; } && return 0

  return 1
}

log "vLLM watchdog started (port ${PORT}, stall window ${STALL_WINDOW}s)"

while true; do
  check_state
  rc=$?
  case "$rc" in
    1)
      log "engine stalled — deadlocked, dropping TCP to :${PORT}"
      if ss -K dst ":${PORT}" >/dev/null 2>&1; then
        log "connections dropped"
      else
        log "ss -K failed"
      fi
      sleep "${RECOVERY_COOLDOWN}"
      ;;
    2)
      log "API server down — waiting"
      sleep "${CHECK_INTERVAL}"
      ;;
  esac
  sleep "${CHECK_INTERVAL}"
done