Measure the memory reduction in cert-manager with WatchList + WatchListClient (enabled using KUBE_FEATURE_WatchListClient environment variable): https://github.com/cert-manager/cert-manager/pull/7315

changes.diff

diff -r -u /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/cainjector-binary.memory /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/cainjector-binary.memory
--- /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/cainjector-binary.memory	2024-10-01 14:27:42.868515393 +0100
+++ /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/cainjector-binary.memory	2024-10-01 14:28:57.217698454 +0100
@@ -1,10 +1,10 @@
-VmPeak:	 1279204 kB
-VmSize:	 1279204 kB
+VmPeak:	 1278436 kB
+VmSize:	 1278436 kB
 VmLck:	       0 kB
 VmPin:	       0 kB
-VmHWM:	   45924 kB
-VmRSS:	   45924 kB
-VmData:	   57520 kB
+VmHWM:	   44900 kB
+VmRSS:	   44900 kB
+VmData:	   56752 kB
 VmStk:	     132 kB
 VmExe:	   23476 kB
 VmLib:	       8 kB
diff -r -u /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/controller-binary.memory /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/controller-binary.memory
--- /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/controller-binary.memory	2024-10-01 14:27:42.868515393 +0100
+++ /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/controller-binary.memory	2024-10-01 14:28:57.207926332 +0100
@@ -1,12 +1,12 @@
-VmPeak:	 1889804 kB
-VmSize:	 1889804 kB
+VmPeak:	 1429808 kB
+VmSize:	 1429808 kB
 VmLck:	       0 kB
 VmPin:	       0 kB
-VmHWM:	  437200 kB
-VmRSS:	  437200 kB
-VmData:	  550912 kB
+VmHWM:	  229004 kB
+VmRSS:	  229004 kB
+VmData:	  226084 kB
 VmStk:	     132 kB
 VmExe:	   32468 kB
 VmLib:	       8 kB
-VmPTE:	     988 kB
+VmPTE:	     580 kB
 VmSwap:	       0 kB
diff -r -u /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/etcd.memory /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/etcd.memory
--- /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/etcd.memory	2024-10-01 14:27:42.858499328 +0100
+++ /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/etcd.memory	2024-10-01 14:28:57.207926332 +0100
@@ -1,10 +1,10 @@
-VmPeak:	11940856 kB
-VmSize:	11940856 kB
+VmPeak:	11941048 kB
+VmSize:	11941048 kB
 VmLck:	       0 kB
 VmPin:	       0 kB
-VmHWM:	  351952 kB
-VmRSS:	  351952 kB
-VmData:	  288960 kB
+VmHWM:	  357984 kB
+VmRSS:	  357984 kB
+VmData:	  289152 kB
 VmStk:	     132 kB
 VmExe:	   10660 kB
 VmLib:	       8 kB
diff -r -u /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/kube-apiserver.memory /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/kube-apiserver.memory
--- /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/kube-apiserver.memory	2024-10-01 14:27:42.858499328 +0100
+++ /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/kube-apiserver.memory	2024-10-01 14:28:57.198154210 +0100
@@ -1,12 +1,12 @@
-VmPeak:	 2444268 kB
-VmSize:	 2444268 kB
+VmPeak:	 2116760 kB
+VmSize:	 2116760 kB
 VmLck:	       0 kB
 VmPin:	       0 kB
-VmHWM:	 1131244 kB
-VmRSS:	 1131244 kB
-VmData:	 1110392 kB
+VmHWM:	  891216 kB
+VmRSS:	  891216 kB
+VmData:	  860708 kB
 VmStk:	     132 kB
 VmExe:	   42168 kB
 VmLib:	       8 kB
-VmPTE:	    2332 kB
+VmPTE:	    1860 kB
 VmSwap:	       0 kB
diff -r -u /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/webhook-binary.memory /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/webhook-binary.memory
--- /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/before/webhook-binary.memory	2024-10-01 14:27:42.878531456 +0100
+++ /home/richard/projects/cert-manager/cert-manager/out.test.cert-manager.7315.pII8tE0/measurements/after/webhook-binary.memory	2024-10-01 14:28:57.217698454 +0100
@@ -1,10 +1,10 @@
-VmPeak:	 1286332 kB
-VmSize:	 1286332 kB
+VmPeak:	 1286076 kB
+VmSize:	 1286076 kB
 VmLck:	       0 kB
 VmPin:	       0 kB
-VmHWM:	   48956 kB
-VmRSS:	   48956 kB
-VmData:	   52320 kB
+VmHWM:	   50296 kB
+VmRSS:	   50296 kB
+VmData:	   52064 kB
 VmStk:	     132 kB
 VmExe:	   27092 kB
 VmLib:	       8 kB

test.cert-manager-PR-7315.sh

#!/usr/bin/env bash
#
# https://github.com/cert-manager/cert-manager/pull/7175
#
# Creates a Kind cluster and installs cert-manager from master and then from the
# bug fix branch containing the fix.

set -o errexit
set -o nounset
set -o pipefail
set -o xtrace

PR=7315
export KO_REGISTRY=ttl.sh/b0c82fa3-22a7-4299-a9c1-57714b6a75e1

cluster_name="test.cert-manager.${PR}"
out_dir=$(mktemp -d "${PWD}/out.test.cert-manager.${PR}.XXXXXXX")

cat <<EOF > "${out_dir}/kind.config.yaml"
apiVersion: kind.x-k8s.io/v1alpha4
kind: Cluster
featureGates:
  # Enable the WatchList / Streaming Lists feature on the API server.
  #
  # - https://kind.sigs.k8s.io/docs/user/configuration/#feature-gates
  # - https://kubernetes.io/docs/reference/using-api/api-concepts/#streaming-lists
  WatchList: true
kubeadmConfigPatches:
  - |
    kind: ClusterConfiguration
    metadata:
      name: config
    etcd:
      local:
        extraArgs:
          unsafe-no-fsync: "true"
nodes:
  - role: control-plane
EOF

cat <<EOF > "${out_dir}/secret.yaml"
apiVersion: v1
kind: Secret
metadata:
  generateName: s-
  labels:
    # Label the Secret so that the cert-manager controller will cache the data.
    controller.cert-manager.io/fao: "true"
data:
  f1: $(dd if=/dev/zero bs=1048576 count=1 | base64 -w0)
EOF

function measure_memory() {
    name="$1"
    pid="$(pidof $name)"
    xargs -0 < "/proc/${pid}/cmdline"
    cat "/proc/${pid}/status" | grep Vm > "${name}.memory"
}

function measure() {
    results_dir="$1"
    values_file="$2"

    # Create cluster
    kind delete cluster --name "$cluster_name" || true
    kind create cluster --name "$cluster_name" --config="${out_dir}/kind.config.yaml"

    # Create ~100Mi of Secrets, 5 at a time
    echo -n {0..99} | xargs -d ' ' -P5 -I{} kubectl create -f "${out_dir}/secret.yaml"

    export "KO_HELM_VALUES_FILES=${values_file}"
    make -j4 ko-deploy-certmanager

    # Wait long enough for all caches to sync
    sleep 10

    mkdir -p "${out_dir}/measurements/${results_dir}"
    pushd "${out_dir}/measurements/${results_dir}"
    for name in kube-apiserver etcd controller-binary cainjector-binary webhook-binary; do
        measure_memory "$name"
    done
    popd

    mkdir -p "${out_dir}/logs/${results_dir}"
    pushd "${out_dir}/logs/${results_dir}"
    for deployment in cert-manager cert-manager-cainjector cert-manager-webhook; do
        kubectl logs -n cert-manager "deployments/$deployment" > "$deployment.log"
    done
    popd
}

git fetch origin pull/${PR}/head
git checkout FETCH_HEAD

measure before $PWD/values.before.yaml
measure after $PWD/values.after.yaml

diff -r -u "${out_dir}/measurements/before" "${out_dir}/measurements/after" | tee "${out_dir}/changes.diff"

values.after.yaml

# values.cert-manager.yaml
global:
  logLevel: 6
config:
  logging:
    format: json
  featureGates:
    AllAlpha: true
    AllBeta: true
extraEnv:
  - name: KUBE_FEATURE_WatchListClient
    value: "true"
cainjector:
  config:
    logging:
      format: json
    featureGates:
      AllAlpha: true
      AllBeta: true
  # extraEnv:
  #   - name: KUBE_FEATURE_ClientWatchList
  #     value: "true"
webhook:
  config:
    logging:
      format: json
    featureGates:
      AllAlpha: true
      AllBeta: true
  # extraEnv:
  #   - name: KUBE_FEATURE_ClientWatchList
  #     value: "true"

values.before.yaml

# values.cert-manager.yaml
global:
  logLevel: 6
config:
  logging:
    format: json
  featureGates:
    AllAlpha: true
    AllBeta: true
extraEnv:
  - name: KUBE_FEATURE_WatchListClient
    value: "false"
cainjector:
  config:
    logging:
      format: json
    featureGates:
      AllAlpha: true
      AllBeta: true
  # extraEnv:
  #   - name: KUBE_FEATURE_ClientWatchList
  #     value: "true"
webhook:
  config:
    logging:
      format: json
    featureGates:
      AllAlpha: true
      AllBeta: true
  # extraEnv:
  #   - name: KUBE_FEATURE_ClientWatchList
  #     value: "true"