CNPG Timeline WAL Bug Reproducer - cloudnative-pg/cloudnative-pg#10394

cleanup.sh

#!/bin/bash
# Cleanup: delete kind cluster
kind delete cluster --name cnpg-bug
echo "Cluster deleted."

cluster.yaml

apiVersion: v1
kind: Secret
metadata:
  name: minio-creds
  namespace: default
type: Opaque
stringData:
  ACCESS_KEY_ID: minio
  ACCESS_SECRET_KEY: minio123
---
apiVersion: v1
kind: Secret
metadata:
  name: timeline-bug-app
  namespace: default
type: kubernetes.io/basic-auth
stringData:
  username: testdb
  password: testdb123
---
apiVersion: barmancloud.cnpg.io/v1
kind: ObjectStore
metadata:
  name: minio-store
  namespace: default
spec:
  configuration:
    destinationPath: s3://cnpg-backup/
    endpointURL: http://minio-service.minio:9000
    s3Credentials:
      accessKeyId:
        name: minio-creds
        key: ACCESS_KEY_ID
      secretAccessKey:
        name: minio-creds
        key: ACCESS_SECRET_KEY
---
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
  name: timeline-bug
  namespace: default
  annotations:
    cnpg.io/skipEmptyWalArchiveCheck: enabled
spec:
  instances: 3
  imageName: "ghcr.io/cloudnative-pg/postgresql:18"
  primaryUpdateMethod: switchover
  plugins:
  - name: barman-cloud.cloudnative-pg.io
    isWALArchiver: true
    parameters:
      barmanObjectName: minio-store
  bootstrap:
    initdb:
      database: testdb
      owner: testdb
      secret:
        name: timeline-bug-app
  storage:
    size: 1Gi
  postgresql:
    synchronous:
      method: any
      number: 1
    parameters:
      archive_timeout: "5s"
      wal_receiver_timeout: "2s"
      log_replication_commands: "on"
      shared_buffers: "128MB"

minio.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: minio
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: minio
  namespace: minio
spec:
  selector:
    matchLabels:
      app: minio
  template:
    metadata:
      labels:
        app: minio
    spec:
      containers:
      - name: minio
        image: minio/minio:latest
        args: ["server", "/data"]
        env:
        - name: MINIO_ROOT_USER
          value: "minio"
        - name: MINIO_ROOT_PASSWORD
          value: "minio123"
        ports:
        - containerPort: 9000
        volumeMounts:
        - name: data
          mountPath: /data
      volumes:
      - name: data
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: minio-service
  namespace: minio
spec:
  selector:
    app: minio
  ports:
  - port: 9000

reproduce.sh

#!/bin/bash
set -euo pipefail

# =============================================================================
# CNPG Timeline WAL Bug Reproducer
#
# Bug: after rolling restart (triggered by parameter change), replicas download
# old-timeline WAL segments from S3 via restore_command, their timeline history
# becomes incompatible with the primary, and they crash with:
#   FATAL: requested timeline N is not a child of this server's history
#
# Related: https://github.com/cloudnative-pg/cloudnative-pg/issues/4990
#
# Root cause: restore_command (barman-cloud wal-restore) does not validate
# the timeline of regular WAL segments — only .history files are checked
# (validateTimelineHistoryFile in walrestore/cmd.go:443).
# =============================================================================

CLUSTER_NAME="timeline-bug"
KIND_CLUSTER="cnpg-bug"
MAX_ATTEMPTS=10

log() { echo "$(date '+%H:%M:%S') [INFO] $*"; }
sep() { echo ""; echo "========================================"; echo "$*"; echo "========================================"; }

for cmd in kind kubectl helm; do
  command -v "$cmd" &>/dev/null || { echo "$cmd not found"; exit 1; }
done

# ---- Step 1: Kind cluster ----
sep "Step 1: Create kind cluster"

if kind get clusters 2>/dev/null | grep -q $KIND_CLUSTER; then
  log "Kind cluster '$KIND_CLUSTER' already exists, reusing"
else
  log "Creating kind cluster..."
  kind create cluster --name $KIND_CLUSTER --image kindest/node:v1.35.1 --wait 120s
fi
kubectl config use-context kind-$KIND_CLUSTER

# ---- Step 2: Install operators ----
sep "Step 2: Install cert-manager + CNPG operator + barman-cloud"

helm repo add cnpg https://cloudnative-pg.github.io/charts 2>/dev/null || true
helm repo add jetstack https://charts.jetstack.io 2>/dev/null || true
helm repo update cnpg jetstack

helm list -n cert-manager 2>/dev/null | grep -q cert-manager || \
  helm install cert-manager jetstack/cert-manager -n cert-manager --create-namespace --set crds.enabled=true --wait --timeout 120s

helm list -n cnpg-system 2>/dev/null | grep -q cnpg-operator || \
  helm install cnpg-operator cnpg/cloudnative-pg -n cnpg-system --create-namespace --wait --timeout 120s

helm list -n cnpg-system 2>/dev/null | grep -q barman-cloud || \
  helm install barman-cloud cnpg/plugin-barman-cloud -n cnpg-system --wait --timeout 120s

kubectl wait --for=condition=available deployment/cnpg-operator-cloudnative-pg -n cnpg-system --timeout=120s

# ---- Step 3: MinIO ----
sep "Step 3: Deploy MinIO"

kubectl apply -f minio.yaml
kubectl wait --for=condition=available deployment/minio -n minio --timeout=120s

kubectl run mc --image=minio/mc --rm -it --restart=Never --namespace=minio -- \
  sh -c "mc alias set local http://minio-service:9000 minio minio123 && mc mb local/cnpg-backup --ignore-existing" 2>/dev/null || true

# ---- Step 4: Create cluster ----
sep "Step 4: Create CNPG cluster"

kubectl apply -f cluster.yaml
log "Waiting for cluster..."
for i in $(seq 1 60); do
  R=$(kubectl get cluster $CLUSTER_NAME -o jsonpath='{.status.readyInstances}' 2>/dev/null || echo 0)
  [ "$R" = "3" ] && break; sleep 5
done

kubectl cnpg status $CLUSTER_NAME 2>/dev/null | grep -E 'Status:|Instance|Standby|Primary'

# ---- Step 5: Fill WAL + backup ----
sep "Step 5: Generate data and backup"

PRIMARY=$(kubectl get cluster $CLUSTER_NAME -o jsonpath='{.status.currentPrimary}')
log "Primary: $PRIMARY"

kubectl exec $PRIMARY -c postgres -- psql -U postgres -d testdb -c \
  "CREATE TABLE IF NOT EXISTS wal_gen (id serial, data text); INSERT INTO wal_gen (data) SELECT repeat('x', 1000) FROM generate_series(1, 100000);"
sleep 20

kubectl apply -f - <<'EOF'
apiVersion: postgresql.cnpg.io/v1
kind: Backup
metadata:
  name: timeline-bug-backup
  namespace: default
spec:
  method: plugin
  pluginConfiguration:
    name: barman-cloud.cloudnative-pg.io
  cluster:
    name: timeline-bug
EOF

for i in $(seq 1 30); do
  P=$(kubectl get backup timeline-bug-backup -o jsonpath='{.status.phase}' 2>/dev/null)
  [ "$P" = "completed" ] && break; sleep 5
done
log "Backup completed"

kubectl exec $PRIMARY -c postgres -- psql -U postgres -c "SELECT archived_count, last_archived_wal FROM pg_stat_archiver;"

# ---- Step 6: Rolling restarts via parameter change ----
sep "Step 6: Trigger rolling restarts ($MAX_ATTEMPTS attempts)"

for attempt in $(seq 1 $MAX_ATTEMPTS); do
  echo ""
  echo "--- Attempt $attempt / $MAX_ATTEMPTS  $(date '+%H:%M:%S') ---"

  PRIMARY=$(kubectl get cluster $CLUSTER_NAME -o jsonpath='{.status.currentPrimary}')

  # Write data between restarts
  kubectl exec $PRIMARY -c postgres -- psql -U postgres -d testdb -c \
    "INSERT INTO wal_gen (data) SELECT repeat('iter$attempt', 100) FROM generate_series(1, 50000);" 2>/dev/null || true
  sleep 10

  # Toggle shared_buffers
  if [ $((attempt % 2)) -eq 1 ]; then SB="196MB"; else SB="128MB"; fi
  log "Patching shared_buffers=$SB..."
  kubectl patch cluster $CLUSTER_NAME --type merge -p \
    "{\"spec\":{\"postgresql\":{\"parameters\":{\"shared_buffers\":\"$SB\"}}}}"

  # Wait for rolling restart
  sleep 15
  for i in $(seq 1 60); do
    READY=$(kubectl get cluster $CLUSTER_NAME -o jsonpath='{.status.readyInstances}' 2>/dev/null || echo 0)
    PENDING=$(kubectl cnpg status $CLUSTER_NAME 2>/dev/null | grep "pending restart" || true)
    CRASH=$(kubectl get pods -l cnpg.io/cluster=$CLUSTER_NAME 2>/dev/null | grep -E 'CrashLoop|Error' || true)

    if [ -n "$CRASH" ]; then
      echo ""
      echo "############################################"
      echo "BUG REPRODUCED on attempt $attempt!"
      echo "############################################"
      echo ""
      echo "$CRASH"
      echo ""
      kubectl cnpg status $CLUSTER_NAME 2>/dev/null | grep -E 'timeline-bug-|Status:'
      echo ""

      for POD in $(kubectl get pods -l cnpg.io/cluster=$CLUSTER_NAME -o name 2>/dev/null | sed 's|pod/||'); do
        echo "=== $POD ==="
        kubectl logs $POD -c postgres --tail=300 2>/dev/null | python3 -c "
import json,sys
for l in sys.stdin:
  try:
    d=json.loads(l);r=d.get('record',{});m=r.get('message','');s=r.get('error_severity','')
    if any(k in m.lower() for k in ['restored log','timeline','streaming','file based','invalid record','checkpoint','forked','not a child','recovery point']) or s in ('FATAL',):
      print(f'  {r.get(\"log_time\",\"\")} {s}: {m}')
  except:pass
" | tail -15
        echo ""
      done
      exit 0
    fi

    if [ "$READY" = "3" ] && [ -z "$PENDING" ]; then
      break
    fi
    sleep 5
  done

  # Also check file-based
  FILE_BASED=$(kubectl cnpg status $CLUSTER_NAME 2>/dev/null | grep "file based" || true)
  if [ -n "$FILE_BASED" ]; then
    echo ""
    echo "############################################"
    echo "BUG REPRODUCED (file-based) on attempt $attempt!"
    echo "############################################"
    echo "$FILE_BASED"
    kubectl cnpg status $CLUSTER_NAME 2>/dev/null | grep -E 'timeline-bug-'
    exit 0
  fi

  log "OK on attempt $attempt"
done

echo ""
echo "Bug NOT reproduced in $MAX_ATTEMPTS attempts"
exit 1