Last active
August 13, 2026 16:46
-
-
Save cicorias/814958b68e61da788ac7bc7e178eab75 to your computer and use it in GitHub Desktop.
mise local toml for phyiscal ai toolchain gpu offload automation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [env] | |
| PIP_INDEX_URL = "https://packagefeedproxy.microsoft.io/pypi/simple/" | |
| UV_INDEX_URL = "https://packagefeedproxy.microsoft.io/pypi/simple/" | |
| npm_config_registry = "https://packagefeedproxy.microsoft.io/npm/" | |
| [tasks."gpu-offload:cluster-00-list"] | |
| description = "List local Podman-backed kind clusters and node containers" | |
| run = ''' | |
| KIND_EXPERIMENTAL_PROVIDER=podman kind get clusters | |
| podman ps --all --filter label=io.x-k8s.kind.cluster | |
| ''' | |
| [tasks."gpu-offload:cluster-01-setup-host-packages"] | |
| description = "Install the Ubuntu host packages required for local kind" | |
| run = ''' | |
| sudo apt-get update | |
| sudo apt-get install --yes curl jq podman | |
| ''' | |
| [tasks."gpu-offload:cluster-02-setup-kubernetes-tools"] | |
| description = "Install the pinned kind, kubectl, and Helm versions with mise" | |
| run = "mise use --global kind@0.30.0 kubectl@1.35.1 helm@3.21.3" | |
| [tasks."gpu-offload:cluster-10-check-prerequisites"] | |
| description = "Check local tools and render the GPU offload Helm chart" | |
| run = ''' | |
| podman info --format '{{ "{{" }}.Host.Security.Rootless{{ "}}" }} {{ "{{" }}.Host.OCIRuntime.Name{{ "}}" }}' | |
| kind version | |
| kubectl version --client | |
| helm version | |
| helm template gpu-offload gpu-offload/helm/gpu-offload \ | |
| --namespace gpu-offload \ | |
| --set image.registry=localhost >/dev/null | |
| ''' | |
| [tasks."gpu-offload:cluster-20-setup-cpu"] | |
| description = "Set up the CPU-only Podman-backed kind cluster" | |
| run = ''' | |
| KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster \ | |
| --name gpu-offload \ | |
| --image kindest/node:v1.35.0 | |
| kubectl config use-context kind-gpu-offload | |
| kubectl wait \ | |
| --for=condition=Ready \ | |
| node/gpu-offload-control-plane \ | |
| --timeout=120s | |
| ''' | |
| [tasks."gpu-offload:cluster-21-run-cpu-check"] | |
| description = "Run Podman and Kubernetes CPU smoke checks" | |
| run = ''' | |
| podman run --rm docker.io/library/alpine:3.22 \ | |
| sh -c 'uname -m; echo Podman CPU container works' | |
| kubectl --context kind-gpu-offload run cpu-check \ | |
| --image=docker.io/library/alpine:3.22 \ | |
| --restart=Never \ | |
| --command -- sh -c 'uname -m; echo Kubernetes CPU pod works' | |
| kubectl --context kind-gpu-offload wait pod/cpu-check \ | |
| --for=jsonpath='{.status.phase}'=Succeeded \ | |
| --timeout=120s | |
| kubectl --context kind-gpu-offload logs pod/cpu-check | |
| kubectl --context kind-gpu-offload delete pod/cpu-check | |
| ''' | |
| [tasks."gpu-offload:cluster-30-setup-nvidia-toolkit"] | |
| description = "Install NVIDIA Container Toolkit and generate the WSL2 CDI specification" | |
| run = ''' | |
| curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \ | |
| sudo gpg --dearmor --yes \ | |
| --output /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg | |
| curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ | |
| sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#' | \ | |
| sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list >/dev/null | |
| sudo apt-get update | |
| sudo apt-get install --yes nvidia-container-toolkit | |
| sudo mkdir -p /etc/cdi | |
| sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml | |
| nvidia-ctk cdi list | |
| ''' | |
| [tasks."gpu-offload:cluster-31-run-nvidia-host-check"] | |
| description = "Verify WSL2 and rootless Podman NVIDIA access" | |
| run = ''' | |
| nvidia-smi | |
| test -c /dev/dxg | |
| podman run --rm \ | |
| --security-opt=label=disable \ | |
| --device nvidia.com/gpu=all \ | |
| docker.io/nvidia/cuda:12.8.1-base-ubuntu24.04 \ | |
| nvidia-smi | |
| ''' | |
| [tasks."gpu-offload:cluster-32-setup-nvidia"] | |
| description = "Set up the WSL2 NVIDIA Podman-backed kind cluster" | |
| run = ''' | |
| test -c /dev/dxg | |
| test -d /usr/lib/wsl | |
| config_file="$(mktemp --suffix=-gpu-offload-kind.yaml)" | |
| trap 'rm -f "$config_file"' EXIT | |
| cat >"$config_file" <<'EOF' | |
| kind: Cluster | |
| apiVersion: kind.x-k8s.io/v1alpha4 | |
| nodes: | |
| - role: control-plane | |
| extraMounts: | |
| - hostPath: /dev/dxg | |
| containerPath: /dev/dxg | |
| - hostPath: /usr/lib/wsl | |
| containerPath: /usr/lib/wsl | |
| readOnly: true | |
| EOF | |
| KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster \ | |
| --name gpu-offload-nvidia \ | |
| --image kindest/node:v1.35.0 \ | |
| --config="$config_file" | |
| kubectl config use-context kind-gpu-offload-nvidia | |
| kubectl wait \ | |
| --for=condition=Ready \ | |
| node/gpu-offload-nvidia-control-plane \ | |
| --timeout=120s | |
| ''' | |
| [tasks."gpu-offload:cluster-33-run-nvidia-node-check"] | |
| description = "Verify NVIDIA access inside the WSL2 kind node" | |
| run = ''' | |
| podman exec gpu-offload-nvidia-control-plane sh -c \ | |
| 'driver_dir=$(find /usr/lib/wsl/drivers -mindepth 1 -maxdepth 1 -type d | head -n 1); LD_LIBRARY_PATH="/usr/lib/wsl/lib:${driver_dir}" /usr/lib/wsl/lib/nvidia-smi' | |
| ''' | |
| [tasks."gpu-offload:cluster-34-setup-nvidia-runtime"] | |
| description = "Configure the WSL2 NVIDIA kind node container runtime" | |
| run = ''' | |
| test -c /dev/dxg | |
| podman exec gpu-offload-nvidia-control-plane sh -c \ | |
| 'jq '\''if any(.mounts[]; .destination == "/usr/lib/wsl") then . else .mounts += [{"destination":"/usr/lib/wsl","type":"none","source":"/usr/lib/wsl","options":["rbind","ro","nosuid","nodev"]}] end'\'' /etc/containerd/cri-base.json > /etc/containerd/cri-base.json.new && mv /etc/containerd/cri-base.json.new /etc/containerd/cri-base.json' | |
| podman exec gpu-offload-nvidia-control-plane systemctl restart containerd | |
| kubectl wait \ | |
| --context kind-gpu-offload-nvidia \ | |
| --for=condition=Ready \ | |
| node/gpu-offload-nvidia-control-plane \ | |
| --timeout=180s | |
| ''' | |
| [tasks."gpu-offload:cluster-35-setup-nvidia-device-plugin"] | |
| description = "Register the WSL2 GPU with the NVIDIA kind cluster" | |
| run = ''' | |
| archive="$(mktemp --suffix=-generic-device-plugin-0.2.0.tar)" | |
| trap 'rm -f "$archive"' EXIT | |
| podman pull docker.io/squat/generic-device-plugin:0.2.0 | |
| podman save \ | |
| --output "$archive" \ | |
| docker.io/squat/generic-device-plugin:0.2.0 | |
| KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive \ | |
| "$archive" \ | |
| --name gpu-offload-nvidia | |
| kubectl --context kind-gpu-offload-nvidia apply -f - <<'EOF' | |
| apiVersion: apps/v1 | |
| kind: DaemonSet | |
| metadata: | |
| name: wsl-gpu-device-plugin | |
| namespace: kube-system | |
| spec: | |
| selector: | |
| matchLabels: | |
| app: wsl-gpu-device-plugin | |
| template: | |
| metadata: | |
| labels: | |
| app: wsl-gpu-device-plugin | |
| spec: | |
| priorityClassName: system-node-critical | |
| tolerations: | |
| - operator: Exists | |
| containers: | |
| - name: device-plugin | |
| image: docker.io/squat/generic-device-plugin@sha256:66c8d5c270eb2b721f1064c549b9b7898152a6d2f0163380a5d37dc7636c20ff | |
| imagePullPolicy: IfNotPresent | |
| args: | |
| - --domain=nvidia.com | |
| - --device={"name":"gpu","groups":[{"paths":[{"path":"/dev/dxg"}]}]} | |
| securityContext: | |
| privileged: true | |
| volumeMounts: | |
| - name: device-plugins | |
| mountPath: /var/lib/kubelet/device-plugins | |
| - name: dxg | |
| mountPath: /dev/dxg | |
| volumes: | |
| - name: device-plugins | |
| hostPath: | |
| path: /var/lib/kubelet/device-plugins | |
| - name: dxg | |
| hostPath: | |
| path: /dev/dxg | |
| type: CharDevice | |
| EOF | |
| rollout_timeout="${GPU_PLUGIN_ROLLOUT_TIMEOUT:-600}" | |
| register_timeout="${GPU_PLUGIN_REGISTER_TIMEOUT:-600}" | |
| kubectl --context kind-gpu-offload-nvidia rollout status \ | |
| daemonset/wsl-gpu-device-plugin \ | |
| --namespace kube-system \ | |
| --timeout="${rollout_timeout}s" | |
| # The rollout completes once the pod is Running, but kubelet only publishes | |
| # nvidia.com/gpu after the plugin registers over its gRPC socket. | |
| trap 'echo; echo "Interrupted while waiting for nvidia.com/gpu." >&2; exit 130' INT TERM | |
| interval=5 | |
| elapsed=0 | |
| while :; do | |
| gpu_allocatable="$(kubectl --context kind-gpu-offload-nvidia get node gpu-offload-nvidia-control-plane \ | |
| -o jsonpath='{.status.allocatable.nvidia\.com/gpu}' 2>/dev/null || true)" | |
| if [ "$gpu_allocatable" = "1" ]; then | |
| break | |
| fi | |
| if [ "$elapsed" -ge "$register_timeout" ]; then | |
| echo "Timed out after ${register_timeout}s waiting for nvidia.com/gpu; found: ${gpu_allocatable:-none}" >&2 | |
| echo "Raise the limit with GPU_PLUGIN_REGISTER_TIMEOUT=<seconds>." >&2 | |
| kubectl --context kind-gpu-offload-nvidia logs \ | |
| --namespace kube-system \ | |
| --selector app=wsl-gpu-device-plugin \ | |
| --tail=50 >&2 || true | |
| exit 1 | |
| fi | |
| echo "Waiting for the device plugin to register nvidia.com/gpu (${elapsed}s/${register_timeout}s; Ctrl+C to abort)..." | |
| sleep "$interval" | |
| elapsed=$((elapsed + interval)) | |
| done | |
| trap - INT TERM | |
| kubectl --context kind-gpu-offload-nvidia get nodes \ | |
| -o custom-columns='NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu' | |
| ''' | |
| [tasks."gpu-offload:cluster-36-run-nvidia-kubernetes-check"] | |
| description = "Run a GPU-allocated Kubernetes smoke check" | |
| run = ''' | |
| archive="$(mktemp --suffix=-nvidia-cuda-12.8.1.tar)" | |
| cleanup() { | |
| exit_code=$? | |
| trap - EXIT | |
| if [ "$exit_code" -ne 0 ]; then | |
| kubectl --context kind-gpu-offload-nvidia get pod/wsl-gpu-check -o wide >&2 || true | |
| kubectl --context kind-gpu-offload-nvidia describe pod/wsl-gpu-check >&2 || true | |
| kubectl --context kind-gpu-offload-nvidia logs pod/wsl-gpu-check --all-containers=true >&2 || true | |
| fi | |
| rm -f "$archive" | |
| kubectl --context kind-gpu-offload-nvidia delete pod/wsl-gpu-check --ignore-not-found >/dev/null | |
| exit "$exit_code" | |
| } | |
| trap cleanup EXIT | |
| if ! podman exec gpu-offload-nvidia-control-plane \ | |
| jq -e 'any(.mounts[]; .destination == "/usr/lib/wsl")' \ | |
| /etc/containerd/cri-base.json >/dev/null; then | |
| echo "WSL driver mounts are not configured; run: mise run gpu-offload:cluster-34-setup-nvidia-runtime" >&2 | |
| exit 1 | |
| fi | |
| gpu_allocatable="$(kubectl --context kind-gpu-offload-nvidia get node gpu-offload-nvidia-control-plane \ | |
| -o jsonpath='{.status.allocatable.nvidia\.com/gpu}')" | |
| if [ "$gpu_allocatable" != "1" ]; then | |
| echo "Expected one allocatable NVIDIA GPU, found: ${gpu_allocatable:-none}; run: mise run gpu-offload:cluster-35-setup-nvidia-device-plugin" >&2 | |
| exit 1 | |
| fi | |
| podman pull docker.io/nvidia/cuda:12.8.1-base-ubuntu24.04 | |
| podman save \ | |
| --output "$archive" \ | |
| docker.io/nvidia/cuda:12.8.1-base-ubuntu24.04 | |
| KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive \ | |
| "$archive" \ | |
| --name gpu-offload-nvidia | |
| kubectl --context kind-gpu-offload-nvidia apply -f - <<'EOF' | |
| apiVersion: v1 | |
| kind: Pod | |
| metadata: | |
| name: wsl-gpu-check | |
| spec: | |
| restartPolicy: Never | |
| containers: | |
| - name: cuda | |
| image: docker.io/nvidia/cuda:12.8.1-base-ubuntu24.04 | |
| imagePullPolicy: IfNotPresent | |
| command: ["/bin/sh", "-c"] | |
| args: | |
| - | | |
| driver_dir=$(find /usr/lib/wsl/drivers -mindepth 1 -maxdepth 1 -type d | head -n 1) | |
| export LD_LIBRARY_PATH="/usr/lib/wsl/lib:${driver_dir}" | |
| test -c /dev/dxg | |
| /usr/lib/wsl/lib/nvidia-smi | |
| resources: | |
| limits: | |
| nvidia.com/gpu: "1" | |
| EOF | |
| kubectl --context kind-gpu-offload-nvidia wait pod/wsl-gpu-check \ | |
| --for=jsonpath='{.status.phase}'=Succeeded \ | |
| --timeout=120s | |
| kubectl --context kind-gpu-offload-nvidia logs pod/wsl-gpu-check | |
| ''' | |
| [tasks."gpu-offload:cluster-70-start-cpu"] | |
| description = "Start the stopped CPU-only Podman-backed kind cluster" | |
| run = ''' | |
| podman start gpu-offload-control-plane | |
| kubectl config use-context kind-gpu-offload | |
| kubectl wait \ | |
| --for=condition=Ready \ | |
| node/gpu-offload-control-plane \ | |
| --timeout=120s | |
| ''' | |
| [tasks."gpu-offload:cluster-71-start-nvidia"] | |
| description = "Start the stopped WSL2 NVIDIA Podman-backed kind cluster" | |
| run = ''' | |
| podman start gpu-offload-nvidia-control-plane | |
| kubectl config use-context kind-gpu-offload-nvidia | |
| kubectl wait \ | |
| --for=condition=Ready \ | |
| node/gpu-offload-nvidia-control-plane \ | |
| --timeout=120s | |
| ''' | |
| [tasks."gpu-offload:cluster-80-stop-cpu"] | |
| description = "Stop the CPU-only Podman-backed kind cluster without deleting it" | |
| run = "podman stop gpu-offload-control-plane" | |
| [tasks."gpu-offload:cluster-81-stop-nvidia"] | |
| description = "Stop the WSL2 NVIDIA Podman-backed kind cluster without deleting it" | |
| run = "podman stop gpu-offload-nvidia-control-plane" | |
| [tasks."gpu-offload:cluster-90-teardown-cpu"] | |
| description = "Tear down the CPU-only Podman-backed kind cluster" | |
| run = "KIND_EXPERIMENTAL_PROVIDER=podman kind delete cluster --name gpu-offload" | |
| [tasks."gpu-offload:cluster-91-teardown-nvidia"] | |
| description = "Tear down the WSL2 NVIDIA Podman-backed kind cluster" | |
| run = "KIND_EXPERIMENTAL_PROVIDER=podman kind delete cluster --name gpu-offload-nvidia" | |
| [tasks."gpu-offload:offload-40-build-images"] | |
| description = "Build the controller and first-run images with Podman" | |
| run = ''' | |
| podman build \ | |
| --build-arg "PIP_INDEX_URL=$PIP_INDEX_URL" \ | |
| --file gpu-offload/controller/Containerfile \ | |
| --tag localhost/xavier-mutate:local \ | |
| gpu-offload/controller | |
| podman build \ | |
| --build-arg "UV_INDEX_URL=$UV_INDEX_URL" \ | |
| --file gpu-offload/examples/first-run/Containerfile \ | |
| --tag localhost/gpu-offload-first-run:local \ | |
| . | |
| podman image exists localhost/xavier-mutate:local | |
| podman image exists localhost/gpu-offload-first-run:local | |
| ''' | |
| [tasks."gpu-offload:offload-41-load-images-cpu"] | |
| description = "Load the first-offload images into the CPU kind cluster" | |
| run = ''' | |
| controller_archive="$(mktemp --suffix=-xavier-mutate-local.tar)" | |
| runtime_archive="$(mktemp --suffix=-gpu-offload-first-run-local.tar)" | |
| trap 'rm -f "$controller_archive" "$runtime_archive"' EXIT | |
| podman save --output "$controller_archive" localhost/xavier-mutate:local | |
| podman save --output "$runtime_archive" localhost/gpu-offload-first-run:local | |
| KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive "$controller_archive" --name gpu-offload | |
| KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive "$runtime_archive" --name gpu-offload | |
| podman exec gpu-offload-control-plane \ | |
| ctr --namespace k8s.io images list | \ | |
| grep -E 'localhost/(xavier-mutate|gpu-offload-first-run):local' | |
| ''' | |
| [tasks."gpu-offload:offload-42-load-images-nvidia"] | |
| description = "Load the first-offload images into the NVIDIA kind cluster" | |
| run = ''' | |
| controller_archive="$(mktemp --suffix=-xavier-mutate-local.tar)" | |
| runtime_archive="$(mktemp --suffix=-gpu-offload-first-run-local.tar)" | |
| trap 'rm -f "$controller_archive" "$runtime_archive"' EXIT | |
| podman save --output "$controller_archive" localhost/xavier-mutate:local | |
| podman save --output "$runtime_archive" localhost/gpu-offload-first-run:local | |
| KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive "$controller_archive" --name gpu-offload-nvidia | |
| KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive "$runtime_archive" --name gpu-offload-nvidia | |
| podman exec gpu-offload-nvidia-control-plane \ | |
| ctr --namespace k8s.io images list | \ | |
| grep -E 'localhost/(xavier-mutate|gpu-offload-first-run):local' | |
| ''' | |
| [tasks."gpu-offload:offload-43-install-controller-cpu"] | |
| description = "Install the admission controller in the CPU kind cluster" | |
| run = ''' | |
| helm --kube-context kind-gpu-offload upgrade --install gpu-offload gpu-offload/helm/gpu-offload \ | |
| --namespace gpu-offload \ | |
| --create-namespace \ | |
| --set image.registry=localhost \ | |
| --set mutate.image.repository=xavier-mutate \ | |
| --set mutate.image.tag=local \ | |
| --set image.pullPolicy=Never | |
| kubectl --context kind-gpu-offload rollout status deployment/gpu-offload-mutate \ | |
| --namespace gpu-offload \ | |
| --timeout=120s | |
| ''' | |
| [tasks."gpu-offload:offload-44-install-controller-nvidia"] | |
| description = "Install the admission controller in the NVIDIA kind cluster" | |
| run = ''' | |
| helm --kube-context kind-gpu-offload-nvidia upgrade --install gpu-offload gpu-offload/helm/gpu-offload \ | |
| --namespace gpu-offload \ | |
| --create-namespace \ | |
| --set image.registry=localhost \ | |
| --set mutate.image.repository=xavier-mutate \ | |
| --set mutate.image.tag=local \ | |
| --set image.pullPolicy=Never | |
| kubectl --context kind-gpu-offload-nvidia rollout status deployment/gpu-offload-mutate \ | |
| --namespace gpu-offload \ | |
| --timeout=120s | |
| ''' | |
| [tasks."gpu-offload:offload-50-setup-cpu-stage"] | |
| description = "Deploy the first-offload CPU server stage" | |
| run = ''' | |
| helm --kube-context kind-gpu-offload upgrade --install first-run gpu-offload/examples/first-run \ | |
| --namespace gpu-offload-demo \ | |
| --create-namespace \ | |
| --set image.registry=localhost | |
| kubectl --context kind-gpu-offload wait --for=create \ | |
| deployment/first-run-client-remote-server-cpu \ | |
| --namespace gpu-offload-demo \ | |
| --timeout=120s | |
| kubectl --context kind-gpu-offload rollout status deployment/first-run-client-remote-server-cpu \ | |
| --namespace gpu-offload-demo \ | |
| --timeout=120s | |
| kubectl --context kind-gpu-offload rollout restart deployment/first-run-client \ | |
| --namespace gpu-offload-demo | |
| kubectl --context kind-gpu-offload rollout status deployment/first-run-client \ | |
| --namespace gpu-offload-demo \ | |
| --timeout=120s | |
| ''' | |
| [tasks."gpu-offload:offload-51-run-cpu-check"] | |
| description = "Verify remote execution in the CPU server stage" | |
| run = ''' | |
| result="$(kubectl --context kind-gpu-offload logs deployment/first-run-client \ | |
| --namespace gpu-offload-demo | grep '"executed_by"' | tail -n 1)" | |
| printf '%s\n' "$result" | |
| RESULT="$result" python3 - <<'PY' | |
| import json | |
| import os | |
| result = json.loads(os.environ["RESULT"]) | |
| assert result["executed_by"].startswith("first-run-client-remote-server-cpu-") | |
| assert result["predictions"] == [1, 4, 9, 16] | |
| PY | |
| ''' | |
| [tasks."gpu-offload:offload-60-setup-nvidia-stage"] | |
| description = "Deploy the first-offload NVIDIA server stage" | |
| run = ''' | |
| helm --kube-context kind-gpu-offload-nvidia upgrade --install first-run gpu-offload/examples/first-run \ | |
| --namespace gpu-offload-demo \ | |
| --create-namespace \ | |
| --set image.registry=localhost \ | |
| --set serverStage.name=nvidia \ | |
| --set serverStage.wslNvidia.enabled=true | |
| kubectl --context kind-gpu-offload-nvidia wait --for=create \ | |
| deployment/first-run-client-remote-server-nvidia \ | |
| --namespace gpu-offload-demo \ | |
| --timeout=120s | |
| kubectl --context kind-gpu-offload-nvidia rollout status deployment/first-run-client-remote-server-nvidia \ | |
| --namespace gpu-offload-demo \ | |
| --timeout=120s | |
| kubectl --context kind-gpu-offload-nvidia delete deployment first-run-client-remote-server-cpu \ | |
| --namespace gpu-offload-demo \ | |
| --ignore-not-found | |
| kubectl --context kind-gpu-offload-nvidia rollout restart deployment/first-run-client \ | |
| --namespace gpu-offload-demo | |
| kubectl --context kind-gpu-offload-nvidia rollout status deployment/first-run-client \ | |
| --namespace gpu-offload-demo \ | |
| --timeout=120s | |
| ''' | |
| [tasks."gpu-offload:offload-61-run-nvidia-allocation-check"] | |
| description = "Verify GPU allocation on the NVIDIA server stage" | |
| run = ''' | |
| server_gpu="$(kubectl --context kind-gpu-offload-nvidia get deployment \ | |
| first-run-client-remote-server-nvidia \ | |
| --namespace gpu-offload-demo \ | |
| --output jsonpath='{.spec.template.spec.containers[0].resources.limits.nvidia\.com/gpu}')" | |
| client_gpu="$(kubectl --context kind-gpu-offload-nvidia get deployment first-run-client \ | |
| --namespace gpu-offload-demo \ | |
| --output jsonpath='{.spec.template.spec.containers[0].resources.limits.nvidia\.com/gpu}')" | |
| test "$server_gpu" = "1" | |
| test -z "$client_gpu" | |
| server_pod="$(kubectl --context kind-gpu-offload-nvidia get pods \ | |
| --namespace gpu-offload-demo \ | |
| --selector app=first-run-client-remote-server-nvidia \ | |
| --output jsonpath='{.items[0].metadata.name}')" | |
| printf 'SERVER_POD=%s\n' "$server_pod" | |
| kubectl --context kind-gpu-offload-nvidia exec "$server_pod" \ | |
| --namespace gpu-offload-demo \ | |
| -- sh -c 'test -c /dev/dxg && echo DEVICE=/dev/dxg; /usr/lib/wsl/lib/nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader' | |
| ''' | |
| [tasks."gpu-offload:offload-62-run-nvidia-execution-check"] | |
| description = "Verify remote execution in the NVIDIA server stage" | |
| run = ''' | |
| server_pod="$(kubectl --context kind-gpu-offload-nvidia get pods \ | |
| --namespace gpu-offload-demo \ | |
| --selector app=first-run-client-remote-server-nvidia \ | |
| --output jsonpath='{.items[0].metadata.name}')" | |
| result="" | |
| for _ in $(seq 1 24); do | |
| result="$(kubectl --context kind-gpu-offload-nvidia logs deployment/first-run-client \ | |
| --namespace gpu-offload-demo | grep '"executed_by"' | tail -n 1 || true)" | |
| [ -n "$result" ] && break | |
| sleep 5 | |
| done | |
| test -n "$result" | |
| printf '%s\n' "$result" | |
| RESULT="$result" SERVER_POD="$server_pod" python3 - <<'PY' | |
| import json | |
| import os | |
| result = json.loads(os.environ["RESULT"]) | |
| assert result["executed_by"] == os.environ["SERVER_POD"] | |
| assert result["predictions"] == [1, 4, 9, 16] | |
| PY | |
| ''' | |
| [tasks."gpu-offload:offload-90-teardown-cpu"] | |
| description = "Remove the first-offload resources from the CPU cluster" | |
| run = ''' | |
| helm --kube-context kind-gpu-offload uninstall first-run --namespace gpu-offload-demo | |
| kubectl --context kind-gpu-offload delete namespace gpu-offload-demo | |
| helm --kube-context kind-gpu-offload uninstall gpu-offload --namespace gpu-offload | |
| kubectl --context kind-gpu-offload delete namespace gpu-offload | |
| ''' | |
| [tasks."gpu-offload:offload-91-teardown-nvidia"] | |
| description = "Remove the first-offload resources from the NVIDIA cluster" | |
| run = ''' | |
| helm --kube-context kind-gpu-offload-nvidia uninstall first-run --namespace gpu-offload-demo | |
| kubectl --context kind-gpu-offload-nvidia delete namespace gpu-offload-demo | |
| helm --kube-context kind-gpu-offload-nvidia uninstall gpu-offload --namespace gpu-offload | |
| kubectl --context kind-gpu-offload-nvidia delete namespace gpu-offload | |
| ''' | |
| [tasks."npm:ci-local"] | |
| description = "Install npm dependencies through the machine-local package feed" | |
| run = "npm ci" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment