Deterministic kind-based reproduction for cloudnative-pg#9770 (rollout stalls with plugin-barman-cloud; kubelet kubernetes#137146)

repro-cnpg-9770.sh

#!/usr/bin/env bash
# Deterministic reproduction for https://github.com/cloudnative-pg/cloudnative-pg/issues/9770
# "Postgres image update stalls when plugin-barman-cloud is installed"
#
# The stall needs a kubelet affected by https://github.com/kubernetes/kubernetes/issues/137146
# (a 1.35.0-1.35.3 regression): after a kubelet restart, a pod's main container that has a
# restartable init container (the plugin sidecar) and a startup probe is never restarted
# once it exits. CNPG instance pods with plugin sidecars match this exactly.
#
# Sequence: healthy 2-instance cluster -> restart the kubelet (arms the trap for every pod
# created before the restart) -> trigger a rolling update -> the replica is recreated fine,
# then after the switchover the demoted primary's instance manager exits (exit 0), the
# kubelet never restarts it, the barman sidecar keeps the pod phase Running -> the pod is
# stuck "1/2 Completed" and the operator loops on "Cannot extract Pod status" forever.
#
# Control experiment: run with KIND_IMAGE=kindest/node:v1.35.5 (fixed kubelet) - the same
# sequence completes cleanly.
#
# Requirements: kind, kubectl, helm, docker. Cleanup: kind delete cluster --name cnpg-9770
set -euo pipefail

KIND_IMAGE="${KIND_IMAGE:-kindest/node:v1.35.1}" # kubelet 1.35.1: affected by kubernetes#137146
CLUSTER_NAME="${CLUSTER_NAME:-cnpg-9770}"
NS=repro

step() { echo; echo "=== $*"; }

step "Creating kind cluster ($KIND_IMAGE)"
kind create cluster --name "$CLUSTER_NAME" --image "$KIND_IMAGE" --wait 120s

step "Installing cert-manager, CNPG operator 1.28.1, plugin-barman-cloud"
helm repo add jetstack https://charts.jetstack.io >/dev/null 2>&1 || true
helm repo add cnpg https://cloudnative-pg.github.io/charts >/dev/null 2>&1 || true
helm repo update >/dev/null
helm upgrade --install cert-manager jetstack/cert-manager \
  --namespace cert-manager --create-namespace --version 1.19.2 \
  --set crds.enabled=true --wait --timeout 5m
helm upgrade --install cnpg-operator cnpg/cloudnative-pg \
  --namespace cnpg-system --create-namespace --version 0.27.1 --wait --timeout 5m
helm upgrade --install plugin-barman-cloud cnpg/plugin-barman-cloud \
  --namespace cnpg-system --version 0.7.0 --wait --timeout 5m

step "Deploying minio as the S3 object store"
kubectl create ns "$NS" --dry-run=client -o yaml | kubectl apply -f -
kubectl apply -n "$NS" -f - <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata: {name: minio}
spec:
  replicas: 1
  selector: {matchLabels: {app: minio}}
  template:
    metadata: {labels: {app: minio}}
    spec:
      containers:
      - name: minio
        image: quay.io/minio/minio:latest
        args: ["server", "/data"]
        env:
        - {name: MINIO_ROOT_USER, value: reprouser}
        - {name: MINIO_ROOT_PASSWORD, value: reprosecret123}
        ports: [{containerPort: 9000}]
---
apiVersion: v1
kind: Service
metadata: {name: minio}
spec:
  selector: {app: minio}
  ports: [{port: 9000, targetPort: 9000}]
EOF
kubectl -n "$NS" rollout status deploy/minio --timeout=300s
kubectl -n "$NS" run mc --rm -i --restart=Never --image=quay.io/minio/mc:latest --command -- \
  /bin/sh -c "mc alias set m http://minio.$NS.svc:9000 reprouser reprosecret123 && mc mb -p m/repro-backups"

step "Creating a 2-instance cluster with the barman-cloud plugin"
kubectl apply -n "$NS" -f - <<'EOF'
apiVersion: v1
kind: Secret
metadata: {name: minio-creds}
type: Opaque
stringData:
  ACCESS_KEY_ID: reprouser
  ACCESS_SECRET_KEY: reprosecret123
---
apiVersion: barmancloud.cnpg.io/v1
kind: ObjectStore
metadata: {name: minio-store}
spec:
  configuration:
    destinationPath: s3://repro-backups/
    endpointURL: http://minio.repro.svc:9000
    s3Credentials:
      accessKeyId: {name: minio-creds, key: ACCESS_KEY_ID}
      secretAccessKey: {name: minio-creds, key: ACCESS_SECRET_KEY}
---
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata: {name: repro-psql-cluster}
spec:
  instances: 2
  primaryUpdateMethod: switchover
  plugins:
  - name: barman-cloud.cloudnative-pg.io
    isWALArchiver: true
    parameters:
      barmanObjectName: minio-store
  postgresql:
    parameters:
      archive_timeout: "60s"
  resources:
    requests: {cpu: 100m, memory: 256Mi}
    limits: {cpu: "1", memory: 512Mi}
  storage:
    size: 1Gi
EOF

step "Waiting for the cluster to become healthy"
until [ "$(kubectl get cluster -n "$NS" repro-psql-cluster -o jsonpath='{.status.phase}' 2>/dev/null)" = "Cluster in healthy state" ]; do
  sleep 10
done
kubectl get pods -n "$NS" -l cnpg.io/podRole=instance

step "ARMING: restarting the kubelet inside the kind node (kubernetes#137146 precondition)"
# Every instance pod created BEFORE this restart is now armed: if its postgres
# container exits, this kubelet will never restart it.
docker exec "${CLUSTER_NAME}-control-plane" systemctl restart kubelet
sleep 15

step "TRIGGER: rolling update via a resources change (an image change works the same)"
kubectl patch cluster -n "$NS" repro-psql-cluster --type merge \
  -p '{"spec":{"resources":{"limits":{"cpu":"2"}}}}'

step "Waiting for the stall (replica recreates fine, then the demoted primary sticks)"
for i in $(seq 1 60); do
  stuck=$(kubectl get pods -n "$NS" -l cnpg.io/podRole=instance --no-headers 2>/dev/null \
    | awk '$2=="1/2" && $3=="Completed" {print $1}')
  if [ -n "$stuck" ]; then
    echo
    echo "======================================================================"
    echo "REPRODUCED: pod '$stuck' is stuck (1/2 Completed)"
    echo "======================================================================"
    kubectl get pods -n "$NS" -l cnpg.io/podRole=instance
    echo
    echo "Cluster status: $(kubectl get cluster -n "$NS" repro-psql-cluster -o jsonpath='{.status.phase}')"
    echo
    echo "postgres container state (terminated exit 0, restartCount stays 0):"
    kubectl get pod -n "$NS" "$stuck" -o jsonpath='{.status.phase} restarts={.status.containerStatuses[?(@.name=="postgres")].restartCount} {.status.containerStatuses[?(@.name=="postgres")].state}' ; echo
    echo
    echo "Operator log (loops on this forever):"
    kubectl logs -n cnpg-system deploy/cnpg-operator-cloudnative-pg --since=1m \
      | grep -m2 "Cannot extract Pod status" || true
    echo
    echo "The cluster will now stay in this state indefinitely."
    echo "Workaround: kubectl delete pod -n $NS $stuck"
    exit 0
  fi
  sleep 10
done
echo "Rollout completed without a stall (fixed kubelet? re-check KIND_IMAGE)"; exit 1