nvidia-container-toolkit-installer

nvidia-container-toolkit-installer.yaml

# Installs nvidia-container-toolkit and configures containerd for CDI on each
# simulated GPU node. This privileged DaemonSet remains running to cover node
# churn (see README step 3).
#
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: nvidia-container-toolkit-installer
  namespace: kube-system
  labels:
    app.kubernetes.io/name: nvidia-container-toolkit-installer
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: nvidia-container-toolkit-installer
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        app.kubernetes.io/name: nvidia-container-toolkit-installer
    spec:
      # Required for nsenter to target the host's systemd (PID 1).
      hostPID: true
      tolerations:
        - operator: Exists
      nodeSelector:
        mokka.nvidia.com/type: sgpu
      containers:
        - name: installer
          image: ubuntu:22.04
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true
          volumeMounts:
            - name: host-root
              mountPath: /host
          command:
            - bash
            - -ceu
            - |
              # The host may use systemd-resolved's loopback stub. That stub is
              # unreachable from the pod network namespace, so make the pod's
              # Kubernetes DNS configuration available inside the chroot.
              HOST_RESOLV="$(readlink -f /host/etc/resolv.conf)"
              if [ ! -e "$HOST_RESOLV" ]; then
                mkdir -p "$(dirname "$HOST_RESOLV")"
                touch "$HOST_RESOLV"
              fi
              mount --bind /etc/resolv.conf "$HOST_RESOLV"
              cleanup() {
                umount "$HOST_RESOLV" || true
              }
              trap cleanup EXIT

              chroot /host bash -s <<'HOSTSCRIPT'
              set -euo pipefail
              export DEBIAN_FRONTEND=noninteractive
              # needrestart's dbus scan fails noisily in a container chroot.
              export NEEDRESTART_SUSPEND=1

              SENTINEL=/run/nvidia-container-toolkit-setup.done

              host_systemctl() {
                nsenter --target 1 --mount --pid -- systemctl "$@"
              }

              # Matches the effective config first (works whether enable_cdi
              # lands in config.toml or a conf.d/*.toml drop-in), then the raw
              # files as a fallback.
              check_cdi() {
                containerd config dump 2>/dev/null \
                  | grep -q 'enable_cdi = true' && return 0
                grep -q 'enable_cdi = true' \
                  /etc/containerd/config.toml 2>/dev/null && return 0
                grep -q 'enable_cdi = true' \
                  /etc/containerd/conf.d/99-nvidia.toml 2>/dev/null && return 0
                return 1
              }

              # The effective config must expose the nvidia runtime handler,
              # otherwise pods with runtimeClassName: nvidia fail to start with
              # "no runtime for nvidia is configured".
              check_nvidia_runtime() {
                containerd config dump 2>/dev/null \
                  | grep -q 'nvidia-container-runtime'
              }

              verify() {
                command -v nvidia-ctk >/dev/null 2>&1
                command -v nvidia-container-runtime >/dev/null 2>&1
                command -v nvidia-cdi-hook >/dev/null 2>&1
                host_systemctl is-active --quiet containerd
                check_cdi
                check_nvidia_runtime
              }

              # The host /run sentinel survives pod restarts; skip re-converging
              # a node that is already installed and verifies clean.
              if [ -f "$SENTINEL" ] && verify; then
                echo "nvidia-container-toolkit already ready on $(hostname)"
                exec sleep infinity
              fi

              rm -f "$SENTINEL"
              if command -v nvidia-ctk >/dev/null 2>&1 \
                 && command -v nvidia-container-runtime >/dev/null 2>&1; then
                echo "Toolkit already installed: $(nvidia-ctk --version 2>/dev/null | head -1)"
              else
                # Recover an apt transaction interrupted by an earlier
                # containerd restart.
                dpkg --configure -a || true

                need=""
                command -v curl >/dev/null 2>&1 || need="$need curl"
                command -v gpg >/dev/null 2>&1 || need="$need gnupg"
                if [ -n "$need" ]; then
                  apt-get update -qq
                  apt-get install -y -qq --no-install-recommends \
                    --allow-change-held-packages $need
                fi

                curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
                  | gpg --batch --yes --dearmor \
                    -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
                curl -fsSL \
                  https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
                  | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
                  > /etc/apt/sources.list.d/nvidia-container-toolkit.list

                apt-get update -qq
                apt-get install -y -qq --no-install-recommends \
                  --allow-change-held-packages nvidia-container-toolkit
                echo "Installed: $(nvidia-ctk --version 2>/dev/null | head -1)"
              fi

              # Clear a stale nvidia drop-in so its runtime definition cannot
              # shadow the config written directly to config.toml below.
              rm -f /etc/containerd/conf.d/99-nvidia.toml

              # --config forces the single-file behavior (nvidia-ctk >= 1.18
              # defaults to a conf.d/*.toml drop-in that may not be imported by
              # config.toml).
              nvidia-ctk runtime configure \
                --runtime=containerd \
                --config=/etc/containerd/config.toml \
                --cdi.enabled

              cat > /etc/nvidia-container-runtime/config.toml <<'TOML'
              [nvidia-container-runtime]
              mode = "cdi"

              [nvidia-container-runtime.modes.cdi]
              default-kind = "nvidia.com/gpu"
              spec-dirs = ["/var/run/cdi", "/etc/cdi"]
              TOML

              # Record completed configuration before the disruptive restart.
              # If this pod is lost, its replacement verifies the sentinel.
              touch "$SENTINEL"
              host_systemctl restart containerd || true

              for attempt in $(seq 1 24); do
                if verify; then
                  echo "nvidia-container-toolkit ready on $(hostname)"
                  exec sleep infinity
                fi
                echo "Waiting for containerd/CDI (${attempt}/24)"
                sleep 5
              done

              echo "containerd did not return with CDI enabled" >&2
              exit 1
              HOSTSCRIPT
          readinessProbe:
            exec:
              command:
                - chroot
                - /host
                - bash
                - -ceu
                - |
                  test -f /run/nvidia-container-toolkit-setup.done
                  command -v nvidia-ctk >/dev/null
                  command -v nvidia-container-runtime >/dev/null
                  command -v nvidia-cdi-hook >/dev/null
                  nsenter --target 1 --mount --pid -- \
                    systemctl is-active --quiet containerd
                  effective="$(containerd config dump 2>/dev/null)"
                  grep -q 'nvidia-container-runtime' <<<"$effective"
                  grep -q 'enable_cdi = true' <<<"$effective" \
                    || grep -q 'enable_cdi = true' \
                      /etc/containerd/config.toml 2>/dev/null \
                    || grep -q 'enable_cdi = true' \
                      /etc/containerd/conf.d/99-nvidia.toml 2>/dev/null
            initialDelaySeconds: 5
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 90
          resources:
            requests:
              cpu: 10m
              memory: 32Mi
            limits:
              cpu: "1"
              memory: 256Mi
      volumes:
        - name: host-root
          hostPath:
            path: /
            type: Directory