cleanup_bastion.sh

gistfile1.txt

#!/usr/bin/env bash
# Bastion cleanup before redeploying an MNO cluster (Scale Lab / Performance Lab).
#
# Implements the cleanup steps documented in:
#   docs/troubleshooting.md  (pods, registry, ingress VIP / power-off, MinIO)
#   ansible/bastion-dnsmasq-clean.yml
#   ansible/bastion-minio-clean.yml
#
# Run as root on the bastion from the jetlag repo root, for example:
#   ./scripts/mno-bastion-cleanup.sh -i ansible/inventory/cloud99.local
#   ./scripts/mno-bastion-cleanup.sh -i ansible/inventory/cloud99.local --clean-registry
#
# This script does NOT run setup-bastion or mno-deploy.

set -euo pipefail

INVENTORY=""
CLEAN_REGISTRY=false
CLEAN_CLUSTER_DIR=false
SKIP_POWEROFF=false
SKIP_DNSMASQ=false
SKIP_MINIO=false
DRY_RUN=false
MINIO_STORE_PATH="${MINIO_STORE_PATH:-/opt/minio}"
REGISTRY_DATA_PATH="${REGISTRY_DATA_PATH:-/opt/registry/data/docker}"
CLUSTER_DIR="${CLUSTER_DIR:-/root/mno}"

usage() {
  cat <<'EOF'
Usage: mno-bastion-cleanup.sh [options]

Options:
  -i, --inventory PATH   Ansible inventory (needed for BMC power-off and optional
                         ansible-based clean playbooks)
      --clean-registry   Also wipe bastion registry image data
                         (/opt/registry/data/docker)
      --clean-cluster-dir
                         Remove cluster artifacts under /root/mno
      --skip-poweroff    Do not power off controlplane/worker/hv via BMC
      --skip-dnsmasq     Do not remove Jetlag dnsmasq configuration
      --skip-minio       Do not wipe MinIO data under /opt/minio/data
      --dry-run          Print actions without changing the system
  -h, --help             Show this help

Default actions (on bastion):
  1. Power off non-bastion hosts via BMC (controlplane, worker, hv)
  2. Stop and remove all podman pods/containers
  3. Wipe MinIO data directory (if present)
  4. Remove Jetlag dnsmasq configuration

After this script, typically:
  source bootstrap.sh
  ansible-playbook -i <inventory> ansible/setup-bastion.yml
  ansible-playbook -i <inventory> ansible/mno-deploy.yml
EOF
}

log() { printf '==> %s\n' "$*"; }
warn() { printf 'WARNING: %s\n' "$*" >&2; }
run() {
  if [[ "${DRY_RUN}" == true ]]; then
    printf '[dry-run] %s\n' "$*"
  else
    eval "$@"
  fi
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    -i|--inventory) INVENTORY="${2:-}"; shift 2 ;;
    --clean-registry) CLEAN_REGISTRY=true; shift ;;
    --clean-cluster-dir) CLEAN_CLUSTER_DIR=true; shift ;;
    --skip-poweroff) SKIP_POWEROFF=true; shift ;;
    --skip-dnsmasq) SKIP_DNSMASQ=true; shift ;;
    --skip-minio) SKIP_MINIO=true; shift ;;
    --dry-run) DRY_RUN=true; shift ;;
    -h|--help) usage; exit 0 ;;
    *)
      printf 'Unknown option: %s\n\n' "$1" >&2
      usage >&2
      exit 1
      ;;
  esac
done

if [[ "${EUID}" -ne 0 && "${DRY_RUN}" != true ]]; then
  warn "This script is intended to run as root on the bastion."
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
cd "${REPO_ROOT}"

# -----------------------------------------------------------------------------
# 1. Power off cluster / hypervisor nodes (leave bastion up)
#    docs/troubleshooting.md#ingress-vip-already-in-use
# -----------------------------------------------------------------------------
poweroff_nodes() {
  if [[ "${SKIP_POWEROFF}" == true ]]; then
    log "Skipping BMC power-off (--skip-poweroff)"
    return
  fi
  if [[ -z "${INVENTORY}" ]]; then
    warn "No inventory given; skip BMC power-off. Shut down non-bastion nodes manually, then re-run with -i <inventory>."
    return
  fi
  if [[ ! -f "${INVENTORY}" ]]; then
    warn "Inventory not found: ${INVENTORY}; skipping BMC power-off"
    return
  fi
  if ! command -v ansible >/dev/null 2>&1; then
    warn "ansible not found; skipping BMC power-off (source bootstrap.sh first, or power off nodes manually)"
    return
  fi
  if ! command -v ipmitool >/dev/null 2>&1; then
    warn "ipmitool not found; skipping BMC power-off"
    return
  fi

  log "Powering off controlplane, worker, and hv hosts via BMC (bastion left alone)"
  # -c local: run ipmitool on the bastion using each host's BMC credentials
  local cmd
  cmd=$(cat <<'EOF'
ansible 'controlplane:worker:hv' -i "__INVENTORY__" -c local -m shell -a \
  'ipmitool -I lanplus -H {{ bmc_address }} -U {{ bmc_user }} -P {{ bmc_password }} chassis power off' \
  || true
EOF
)
  cmd="${cmd//__INVENTORY__/${INVENTORY}}"
  run "${cmd}"
}

# -----------------------------------------------------------------------------
# 2. Clean all podman pods and containers
#    docs/troubleshooting.md#clean-all-container-services--podman-pods
# -----------------------------------------------------------------------------
clean_podman() {
  log "Stopping and removing all podman pods and containers"
  if [[ "${DRY_RUN}" == true ]]; then
    printf '[dry-run] podman pod/container stop+rm (all)\n'
    return
  fi
  # Tolerate empty lists / already-clean state
  podman pod ps -q 2>/dev/null | xargs -r -I % podman pod stop % || true
  podman ps -q 2>/dev/null | xargs -r -I % podman stop % || true
  podman pod ps -q 2>/dev/null | xargs -r -I % podman pod rm % || true
  podman ps -aq 2>/dev/null | xargs -r -I % podman rm % || true
}

# -----------------------------------------------------------------------------
# 3. Optional: wipe bastion registry image data
#    docs/troubleshooting.md#clean-all-container-images-from-bastion-registry
# -----------------------------------------------------------------------------
clean_registry() {
  if [[ "${CLEAN_REGISTRY}" != true ]]; then
    log "Skipping registry image wipe (pass --clean-registry to enable)"
    return
  fi
  if [[ ! -d "${REGISTRY_DATA_PATH}" ]]; then
    log "Registry data path not present: ${REGISTRY_DATA_PATH}"
    return
  fi
  log "Removing bastion registry image data: ${REGISTRY_DATA_PATH}"
  run "rm -rf '${REGISTRY_DATA_PATH}'"
}

# -----------------------------------------------------------------------------
# 4. Wipe MinIO data (pods already removed; do not restart MinIO here)
#    ansible/roles/bastion-minio-clean + docs/bastion-minio.md
# -----------------------------------------------------------------------------
clean_minio() {
  if [[ "${SKIP_MINIO}" == true ]]; then
    log "Skipping MinIO data wipe (--skip-minio)"
    return
  fi
  local data_dir="${MINIO_STORE_PATH}/data"
  if [[ ! -e "${MINIO_STORE_PATH}" ]]; then
    log "MinIO store path not present: ${MINIO_STORE_PATH}"
    return
  fi
  log "Wiping MinIO data: ${data_dir}"
  run "rm -rf '${data_dir}'"
  run "mkdir -p '${data_dir}'"
}

# -----------------------------------------------------------------------------
# 5. Remove Jetlag dnsmasq configuration
#    ansible/roles/bastion-dnsmasq-clean
# -----------------------------------------------------------------------------
clean_dnsmasq() {
  if [[ "${SKIP_DNSMASQ}" == true ]]; then
    log "Skipping dnsmasq cleanup (--skip-dnsmasq)"
    return
  fi

  # Prefer the playbook when inventory + ansible are available
  if [[ -n "${INVENTORY}" && -f "${INVENTORY}" ]] && command -v ansible-playbook >/dev/null 2>&1; then
    if [[ -f "${REPO_ROOT}/ansible/vars/all.yml" ]]; then
      log "Running ansible/bastion-dnsmasq-clean.yml"
      run "ansible-playbook -i '${INVENTORY}' '${REPO_ROOT}/ansible/bastion-dnsmasq-clean.yml'"
      return
    fi
    warn "ansible/vars/all.yml missing; falling back to inline dnsmasq cleanup"
  fi

  log "Removing Jetlag dnsmasq configuration (inline)"
  if [[ "${DRY_RUN}" == true ]]; then
    printf '[dry-run] stop dnsmasq, remove jetlag dnsmasq/NM DNS files, restart NetworkManager\n'
    return
  fi
  systemctl stop dnsmasq 2>/dev/null || true
  systemctl disable dnsmasq 2>/dev/null || true
  rm -f /etc/dnsmasq.d/jetlag.conf
  rm -f /etc/NetworkManager/conf.d/99-jetlag-dns.conf
  if [[ -f /etc/dnsmasq.conf ]]; then
    # Match role: uncomment interface=lo if it was commented as #interface=lo
    sed -i 's/^#interface=lo$/interface=lo/' /etc/dnsmasq.conf || true
  fi
  rm -f /etc/resolv.conf
  systemctl restart NetworkManager || true
}

# -----------------------------------------------------------------------------
# 6. Optional: remove local MNO cluster artifacts
# -----------------------------------------------------------------------------
clean_cluster_dir() {
  if [[ "${CLEAN_CLUSTER_DIR}" != true ]]; then
    log "Skipping cluster dir cleanup (pass --clean-cluster-dir to remove ${CLUSTER_DIR})"
    return
  fi
  if [[ ! -e "${CLUSTER_DIR}" ]]; then
    log "Cluster dir not present: ${CLUSTER_DIR}"
    return
  fi
  log "Removing cluster artifacts: ${CLUSTER_DIR}"
  run "rm -rf '${CLUSTER_DIR}'"
}

log "Jetlag MNO bastion cleanup (repo: ${REPO_ROOT})"
#poweroff_nodes
clean_podman
clean_registry
clean_minio
clean_dnsmasq
clean_cluster_dir

cat <<EOF

Cleanup finished.

Next steps (run manually when ready):
  cd ${REPO_ROOT}
  source bootstrap.sh
  ansible-playbook -i ${INVENTORY:-ansible/inventory/<cloud>.local} ansible/setup-bastion.yml
  ansible-playbook -i ${INVENTORY:-ansible/inventory/<cloud>.local} ansible/mno-deploy.yml

EOF