Skip to content

Instantly share code, notes, and snippets.

@cicorias
Last active August 13, 2026 16:46
Show Gist options
  • Select an option

  • Save cicorias/814958b68e61da788ac7bc7e178eab75 to your computer and use it in GitHub Desktop.

Select an option

Save cicorias/814958b68e61da788ac7bc7e178eab75 to your computer and use it in GitHub Desktop.
mise local toml for phyiscal ai toolchain gpu offload automation
[env]
PIP_INDEX_URL = "https://packagefeedproxy.microsoft.io/pypi/simple/"
UV_INDEX_URL = "https://packagefeedproxy.microsoft.io/pypi/simple/"
npm_config_registry = "https://packagefeedproxy.microsoft.io/npm/"
[tasks."gpu-offload:cluster-00-list"]
description = "List local Podman-backed kind clusters and node containers"
run = '''
KIND_EXPERIMENTAL_PROVIDER=podman kind get clusters
podman ps --all --filter label=io.x-k8s.kind.cluster
'''
[tasks."gpu-offload:cluster-01-setup-host-packages"]
description = "Install the Ubuntu host packages required for local kind"
run = '''
sudo apt-get update
sudo apt-get install --yes curl jq podman
'''
[tasks."gpu-offload:cluster-02-setup-kubernetes-tools"]
description = "Install the pinned kind, kubectl, and Helm versions with mise"
run = "mise use --global kind@0.30.0 kubectl@1.35.1 helm@3.21.3"
[tasks."gpu-offload:cluster-10-check-prerequisites"]
description = "Check local tools and render the GPU offload Helm chart"
run = '''
podman info --format '{{ "{{" }}.Host.Security.Rootless{{ "}}" }} {{ "{{" }}.Host.OCIRuntime.Name{{ "}}" }}'
kind version
kubectl version --client
helm version
helm template gpu-offload gpu-offload/helm/gpu-offload \
--namespace gpu-offload \
--set image.registry=localhost >/dev/null
'''
[tasks."gpu-offload:cluster-20-setup-cpu"]
description = "Set up the CPU-only Podman-backed kind cluster"
run = '''
KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster \
--name gpu-offload \
--image kindest/node:v1.35.0
kubectl config use-context kind-gpu-offload
kubectl wait \
--for=condition=Ready \
node/gpu-offload-control-plane \
--timeout=120s
'''
[tasks."gpu-offload:cluster-21-run-cpu-check"]
description = "Run Podman and Kubernetes CPU smoke checks"
run = '''
podman run --rm docker.io/library/alpine:3.22 \
sh -c 'uname -m; echo Podman CPU container works'
kubectl --context kind-gpu-offload run cpu-check \
--image=docker.io/library/alpine:3.22 \
--restart=Never \
--command -- sh -c 'uname -m; echo Kubernetes CPU pod works'
kubectl --context kind-gpu-offload wait pod/cpu-check \
--for=jsonpath='{.status.phase}'=Succeeded \
--timeout=120s
kubectl --context kind-gpu-offload logs pod/cpu-check
kubectl --context kind-gpu-offload delete pod/cpu-check
'''
[tasks."gpu-offload:cluster-30-setup-nvidia-toolkit"]
description = "Install NVIDIA Container Toolkit and generate the WSL2 CDI specification"
run = '''
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
sudo gpg --dearmor --yes \
--output /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#' | \
sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list >/dev/null
sudo apt-get update
sudo apt-get install --yes nvidia-container-toolkit
sudo mkdir -p /etc/cdi
sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
nvidia-ctk cdi list
'''
[tasks."gpu-offload:cluster-31-run-nvidia-host-check"]
description = "Verify WSL2 and rootless Podman NVIDIA access"
run = '''
nvidia-smi
test -c /dev/dxg
podman run --rm \
--security-opt=label=disable \
--device nvidia.com/gpu=all \
docker.io/nvidia/cuda:12.8.1-base-ubuntu24.04 \
nvidia-smi
'''
[tasks."gpu-offload:cluster-32-setup-nvidia"]
description = "Set up the WSL2 NVIDIA Podman-backed kind cluster"
run = '''
test -c /dev/dxg
test -d /usr/lib/wsl
config_file="$(mktemp --suffix=-gpu-offload-kind.yaml)"
trap 'rm -f "$config_file"' EXIT
cat >"$config_file" <<'EOF'
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
extraMounts:
- hostPath: /dev/dxg
containerPath: /dev/dxg
- hostPath: /usr/lib/wsl
containerPath: /usr/lib/wsl
readOnly: true
EOF
KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster \
--name gpu-offload-nvidia \
--image kindest/node:v1.35.0 \
--config="$config_file"
kubectl config use-context kind-gpu-offload-nvidia
kubectl wait \
--for=condition=Ready \
node/gpu-offload-nvidia-control-plane \
--timeout=120s
'''
[tasks."gpu-offload:cluster-33-run-nvidia-node-check"]
description = "Verify NVIDIA access inside the WSL2 kind node"
run = '''
podman exec gpu-offload-nvidia-control-plane sh -c \
'driver_dir=$(find /usr/lib/wsl/drivers -mindepth 1 -maxdepth 1 -type d | head -n 1); LD_LIBRARY_PATH="/usr/lib/wsl/lib:${driver_dir}" /usr/lib/wsl/lib/nvidia-smi'
'''
[tasks."gpu-offload:cluster-34-setup-nvidia-runtime"]
description = "Configure the WSL2 NVIDIA kind node container runtime"
run = '''
test -c /dev/dxg
podman exec gpu-offload-nvidia-control-plane sh -c \
'jq '\''if any(.mounts[]; .destination == "/usr/lib/wsl") then . else .mounts += [{"destination":"/usr/lib/wsl","type":"none","source":"/usr/lib/wsl","options":["rbind","ro","nosuid","nodev"]}] end'\'' /etc/containerd/cri-base.json > /etc/containerd/cri-base.json.new && mv /etc/containerd/cri-base.json.new /etc/containerd/cri-base.json'
podman exec gpu-offload-nvidia-control-plane systemctl restart containerd
kubectl wait \
--context kind-gpu-offload-nvidia \
--for=condition=Ready \
node/gpu-offload-nvidia-control-plane \
--timeout=180s
'''
[tasks."gpu-offload:cluster-35-setup-nvidia-device-plugin"]
description = "Register the WSL2 GPU with the NVIDIA kind cluster"
run = '''
archive="$(mktemp --suffix=-generic-device-plugin-0.2.0.tar)"
trap 'rm -f "$archive"' EXIT
podman pull docker.io/squat/generic-device-plugin:0.2.0
podman save \
--output "$archive" \
docker.io/squat/generic-device-plugin:0.2.0
KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive \
"$archive" \
--name gpu-offload-nvidia
kubectl --context kind-gpu-offload-nvidia apply -f - <<'EOF'
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: wsl-gpu-device-plugin
namespace: kube-system
spec:
selector:
matchLabels:
app: wsl-gpu-device-plugin
template:
metadata:
labels:
app: wsl-gpu-device-plugin
spec:
priorityClassName: system-node-critical
tolerations:
- operator: Exists
containers:
- name: device-plugin
image: docker.io/squat/generic-device-plugin@sha256:66c8d5c270eb2b721f1064c549b9b7898152a6d2f0163380a5d37dc7636c20ff
imagePullPolicy: IfNotPresent
args:
- --domain=nvidia.com
- --device={"name":"gpu","groups":[{"paths":[{"path":"/dev/dxg"}]}]}
securityContext:
privileged: true
volumeMounts:
- name: device-plugins
mountPath: /var/lib/kubelet/device-plugins
- name: dxg
mountPath: /dev/dxg
volumes:
- name: device-plugins
hostPath:
path: /var/lib/kubelet/device-plugins
- name: dxg
hostPath:
path: /dev/dxg
type: CharDevice
EOF
rollout_timeout="${GPU_PLUGIN_ROLLOUT_TIMEOUT:-600}"
register_timeout="${GPU_PLUGIN_REGISTER_TIMEOUT:-600}"
kubectl --context kind-gpu-offload-nvidia rollout status \
daemonset/wsl-gpu-device-plugin \
--namespace kube-system \
--timeout="${rollout_timeout}s"
# The rollout completes once the pod is Running, but kubelet only publishes
# nvidia.com/gpu after the plugin registers over its gRPC socket.
trap 'echo; echo "Interrupted while waiting for nvidia.com/gpu." >&2; exit 130' INT TERM
interval=5
elapsed=0
while :; do
gpu_allocatable="$(kubectl --context kind-gpu-offload-nvidia get node gpu-offload-nvidia-control-plane \
-o jsonpath='{.status.allocatable.nvidia\.com/gpu}' 2>/dev/null || true)"
if [ "$gpu_allocatable" = "1" ]; then
break
fi
if [ "$elapsed" -ge "$register_timeout" ]; then
echo "Timed out after ${register_timeout}s waiting for nvidia.com/gpu; found: ${gpu_allocatable:-none}" >&2
echo "Raise the limit with GPU_PLUGIN_REGISTER_TIMEOUT=<seconds>." >&2
kubectl --context kind-gpu-offload-nvidia logs \
--namespace kube-system \
--selector app=wsl-gpu-device-plugin \
--tail=50 >&2 || true
exit 1
fi
echo "Waiting for the device plugin to register nvidia.com/gpu (${elapsed}s/${register_timeout}s; Ctrl+C to abort)..."
sleep "$interval"
elapsed=$((elapsed + interval))
done
trap - INT TERM
kubectl --context kind-gpu-offload-nvidia get nodes \
-o custom-columns='NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu'
'''
[tasks."gpu-offload:cluster-36-run-nvidia-kubernetes-check"]
description = "Run a GPU-allocated Kubernetes smoke check"
run = '''
archive="$(mktemp --suffix=-nvidia-cuda-12.8.1.tar)"
cleanup() {
exit_code=$?
trap - EXIT
if [ "$exit_code" -ne 0 ]; then
kubectl --context kind-gpu-offload-nvidia get pod/wsl-gpu-check -o wide >&2 || true
kubectl --context kind-gpu-offload-nvidia describe pod/wsl-gpu-check >&2 || true
kubectl --context kind-gpu-offload-nvidia logs pod/wsl-gpu-check --all-containers=true >&2 || true
fi
rm -f "$archive"
kubectl --context kind-gpu-offload-nvidia delete pod/wsl-gpu-check --ignore-not-found >/dev/null
exit "$exit_code"
}
trap cleanup EXIT
if ! podman exec gpu-offload-nvidia-control-plane \
jq -e 'any(.mounts[]; .destination == "/usr/lib/wsl")' \
/etc/containerd/cri-base.json >/dev/null; then
echo "WSL driver mounts are not configured; run: mise run gpu-offload:cluster-34-setup-nvidia-runtime" >&2
exit 1
fi
gpu_allocatable="$(kubectl --context kind-gpu-offload-nvidia get node gpu-offload-nvidia-control-plane \
-o jsonpath='{.status.allocatable.nvidia\.com/gpu}')"
if [ "$gpu_allocatable" != "1" ]; then
echo "Expected one allocatable NVIDIA GPU, found: ${gpu_allocatable:-none}; run: mise run gpu-offload:cluster-35-setup-nvidia-device-plugin" >&2
exit 1
fi
podman pull docker.io/nvidia/cuda:12.8.1-base-ubuntu24.04
podman save \
--output "$archive" \
docker.io/nvidia/cuda:12.8.1-base-ubuntu24.04
KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive \
"$archive" \
--name gpu-offload-nvidia
kubectl --context kind-gpu-offload-nvidia apply -f - <<'EOF'
apiVersion: v1
kind: Pod
metadata:
name: wsl-gpu-check
spec:
restartPolicy: Never
containers:
- name: cuda
image: docker.io/nvidia/cuda:12.8.1-base-ubuntu24.04
imagePullPolicy: IfNotPresent
command: ["/bin/sh", "-c"]
args:
- |
driver_dir=$(find /usr/lib/wsl/drivers -mindepth 1 -maxdepth 1 -type d | head -n 1)
export LD_LIBRARY_PATH="/usr/lib/wsl/lib:${driver_dir}"
test -c /dev/dxg
/usr/lib/wsl/lib/nvidia-smi
resources:
limits:
nvidia.com/gpu: "1"
EOF
kubectl --context kind-gpu-offload-nvidia wait pod/wsl-gpu-check \
--for=jsonpath='{.status.phase}'=Succeeded \
--timeout=120s
kubectl --context kind-gpu-offload-nvidia logs pod/wsl-gpu-check
'''
[tasks."gpu-offload:cluster-70-start-cpu"]
description = "Start the stopped CPU-only Podman-backed kind cluster"
run = '''
podman start gpu-offload-control-plane
kubectl config use-context kind-gpu-offload
kubectl wait \
--for=condition=Ready \
node/gpu-offload-control-plane \
--timeout=120s
'''
[tasks."gpu-offload:cluster-71-start-nvidia"]
description = "Start the stopped WSL2 NVIDIA Podman-backed kind cluster"
run = '''
podman start gpu-offload-nvidia-control-plane
kubectl config use-context kind-gpu-offload-nvidia
kubectl wait \
--for=condition=Ready \
node/gpu-offload-nvidia-control-plane \
--timeout=120s
'''
[tasks."gpu-offload:cluster-80-stop-cpu"]
description = "Stop the CPU-only Podman-backed kind cluster without deleting it"
run = "podman stop gpu-offload-control-plane"
[tasks."gpu-offload:cluster-81-stop-nvidia"]
description = "Stop the WSL2 NVIDIA Podman-backed kind cluster without deleting it"
run = "podman stop gpu-offload-nvidia-control-plane"
[tasks."gpu-offload:cluster-90-teardown-cpu"]
description = "Tear down the CPU-only Podman-backed kind cluster"
run = "KIND_EXPERIMENTAL_PROVIDER=podman kind delete cluster --name gpu-offload"
[tasks."gpu-offload:cluster-91-teardown-nvidia"]
description = "Tear down the WSL2 NVIDIA Podman-backed kind cluster"
run = "KIND_EXPERIMENTAL_PROVIDER=podman kind delete cluster --name gpu-offload-nvidia"
[tasks."gpu-offload:offload-40-build-images"]
description = "Build the controller and first-run images with Podman"
run = '''
podman build \
--build-arg "PIP_INDEX_URL=$PIP_INDEX_URL" \
--file gpu-offload/controller/Containerfile \
--tag localhost/xavier-mutate:local \
gpu-offload/controller
podman build \
--build-arg "UV_INDEX_URL=$UV_INDEX_URL" \
--file gpu-offload/examples/first-run/Containerfile \
--tag localhost/gpu-offload-first-run:local \
.
podman image exists localhost/xavier-mutate:local
podman image exists localhost/gpu-offload-first-run:local
'''
[tasks."gpu-offload:offload-41-load-images-cpu"]
description = "Load the first-offload images into the CPU kind cluster"
run = '''
controller_archive="$(mktemp --suffix=-xavier-mutate-local.tar)"
runtime_archive="$(mktemp --suffix=-gpu-offload-first-run-local.tar)"
trap 'rm -f "$controller_archive" "$runtime_archive"' EXIT
podman save --output "$controller_archive" localhost/xavier-mutate:local
podman save --output "$runtime_archive" localhost/gpu-offload-first-run:local
KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive "$controller_archive" --name gpu-offload
KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive "$runtime_archive" --name gpu-offload
podman exec gpu-offload-control-plane \
ctr --namespace k8s.io images list | \
grep -E 'localhost/(xavier-mutate|gpu-offload-first-run):local'
'''
[tasks."gpu-offload:offload-42-load-images-nvidia"]
description = "Load the first-offload images into the NVIDIA kind cluster"
run = '''
controller_archive="$(mktemp --suffix=-xavier-mutate-local.tar)"
runtime_archive="$(mktemp --suffix=-gpu-offload-first-run-local.tar)"
trap 'rm -f "$controller_archive" "$runtime_archive"' EXIT
podman save --output "$controller_archive" localhost/xavier-mutate:local
podman save --output "$runtime_archive" localhost/gpu-offload-first-run:local
KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive "$controller_archive" --name gpu-offload-nvidia
KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive "$runtime_archive" --name gpu-offload-nvidia
podman exec gpu-offload-nvidia-control-plane \
ctr --namespace k8s.io images list | \
grep -E 'localhost/(xavier-mutate|gpu-offload-first-run):local'
'''
[tasks."gpu-offload:offload-43-install-controller-cpu"]
description = "Install the admission controller in the CPU kind cluster"
run = '''
helm --kube-context kind-gpu-offload upgrade --install gpu-offload gpu-offload/helm/gpu-offload \
--namespace gpu-offload \
--create-namespace \
--set image.registry=localhost \
--set mutate.image.repository=xavier-mutate \
--set mutate.image.tag=local \
--set image.pullPolicy=Never
kubectl --context kind-gpu-offload rollout status deployment/gpu-offload-mutate \
--namespace gpu-offload \
--timeout=120s
'''
[tasks."gpu-offload:offload-44-install-controller-nvidia"]
description = "Install the admission controller in the NVIDIA kind cluster"
run = '''
helm --kube-context kind-gpu-offload-nvidia upgrade --install gpu-offload gpu-offload/helm/gpu-offload \
--namespace gpu-offload \
--create-namespace \
--set image.registry=localhost \
--set mutate.image.repository=xavier-mutate \
--set mutate.image.tag=local \
--set image.pullPolicy=Never
kubectl --context kind-gpu-offload-nvidia rollout status deployment/gpu-offload-mutate \
--namespace gpu-offload \
--timeout=120s
'''
[tasks."gpu-offload:offload-50-setup-cpu-stage"]
description = "Deploy the first-offload CPU server stage"
run = '''
helm --kube-context kind-gpu-offload upgrade --install first-run gpu-offload/examples/first-run \
--namespace gpu-offload-demo \
--create-namespace \
--set image.registry=localhost
kubectl --context kind-gpu-offload wait --for=create \
deployment/first-run-client-remote-server-cpu \
--namespace gpu-offload-demo \
--timeout=120s
kubectl --context kind-gpu-offload rollout status deployment/first-run-client-remote-server-cpu \
--namespace gpu-offload-demo \
--timeout=120s
kubectl --context kind-gpu-offload rollout restart deployment/first-run-client \
--namespace gpu-offload-demo
kubectl --context kind-gpu-offload rollout status deployment/first-run-client \
--namespace gpu-offload-demo \
--timeout=120s
'''
[tasks."gpu-offload:offload-51-run-cpu-check"]
description = "Verify remote execution in the CPU server stage"
run = '''
result="$(kubectl --context kind-gpu-offload logs deployment/first-run-client \
--namespace gpu-offload-demo | grep '"executed_by"' | tail -n 1)"
printf '%s\n' "$result"
RESULT="$result" python3 - <<'PY'
import json
import os
result = json.loads(os.environ["RESULT"])
assert result["executed_by"].startswith("first-run-client-remote-server-cpu-")
assert result["predictions"] == [1, 4, 9, 16]
PY
'''
[tasks."gpu-offload:offload-60-setup-nvidia-stage"]
description = "Deploy the first-offload NVIDIA server stage"
run = '''
helm --kube-context kind-gpu-offload-nvidia upgrade --install first-run gpu-offload/examples/first-run \
--namespace gpu-offload-demo \
--create-namespace \
--set image.registry=localhost \
--set serverStage.name=nvidia \
--set serverStage.wslNvidia.enabled=true
kubectl --context kind-gpu-offload-nvidia wait --for=create \
deployment/first-run-client-remote-server-nvidia \
--namespace gpu-offload-demo \
--timeout=120s
kubectl --context kind-gpu-offload-nvidia rollout status deployment/first-run-client-remote-server-nvidia \
--namespace gpu-offload-demo \
--timeout=120s
kubectl --context kind-gpu-offload-nvidia delete deployment first-run-client-remote-server-cpu \
--namespace gpu-offload-demo \
--ignore-not-found
kubectl --context kind-gpu-offload-nvidia rollout restart deployment/first-run-client \
--namespace gpu-offload-demo
kubectl --context kind-gpu-offload-nvidia rollout status deployment/first-run-client \
--namespace gpu-offload-demo \
--timeout=120s
'''
[tasks."gpu-offload:offload-61-run-nvidia-allocation-check"]
description = "Verify GPU allocation on the NVIDIA server stage"
run = '''
server_gpu="$(kubectl --context kind-gpu-offload-nvidia get deployment \
first-run-client-remote-server-nvidia \
--namespace gpu-offload-demo \
--output jsonpath='{.spec.template.spec.containers[0].resources.limits.nvidia\.com/gpu}')"
client_gpu="$(kubectl --context kind-gpu-offload-nvidia get deployment first-run-client \
--namespace gpu-offload-demo \
--output jsonpath='{.spec.template.spec.containers[0].resources.limits.nvidia\.com/gpu}')"
test "$server_gpu" = "1"
test -z "$client_gpu"
server_pod="$(kubectl --context kind-gpu-offload-nvidia get pods \
--namespace gpu-offload-demo \
--selector app=first-run-client-remote-server-nvidia \
--output jsonpath='{.items[0].metadata.name}')"
printf 'SERVER_POD=%s\n' "$server_pod"
kubectl --context kind-gpu-offload-nvidia exec "$server_pod" \
--namespace gpu-offload-demo \
-- sh -c 'test -c /dev/dxg && echo DEVICE=/dev/dxg; /usr/lib/wsl/lib/nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader'
'''
[tasks."gpu-offload:offload-62-run-nvidia-execution-check"]
description = "Verify remote execution in the NVIDIA server stage"
run = '''
server_pod="$(kubectl --context kind-gpu-offload-nvidia get pods \
--namespace gpu-offload-demo \
--selector app=first-run-client-remote-server-nvidia \
--output jsonpath='{.items[0].metadata.name}')"
result=""
for _ in $(seq 1 24); do
result="$(kubectl --context kind-gpu-offload-nvidia logs deployment/first-run-client \
--namespace gpu-offload-demo | grep '"executed_by"' | tail -n 1 || true)"
[ -n "$result" ] && break
sleep 5
done
test -n "$result"
printf '%s\n' "$result"
RESULT="$result" SERVER_POD="$server_pod" python3 - <<'PY'
import json
import os
result = json.loads(os.environ["RESULT"])
assert result["executed_by"] == os.environ["SERVER_POD"]
assert result["predictions"] == [1, 4, 9, 16]
PY
'''
[tasks."gpu-offload:offload-90-teardown-cpu"]
description = "Remove the first-offload resources from the CPU cluster"
run = '''
helm --kube-context kind-gpu-offload uninstall first-run --namespace gpu-offload-demo
kubectl --context kind-gpu-offload delete namespace gpu-offload-demo
helm --kube-context kind-gpu-offload uninstall gpu-offload --namespace gpu-offload
kubectl --context kind-gpu-offload delete namespace gpu-offload
'''
[tasks."gpu-offload:offload-91-teardown-nvidia"]
description = "Remove the first-offload resources from the NVIDIA cluster"
run = '''
helm --kube-context kind-gpu-offload-nvidia uninstall first-run --namespace gpu-offload-demo
kubectl --context kind-gpu-offload-nvidia delete namespace gpu-offload-demo
helm --kube-context kind-gpu-offload-nvidia uninstall gpu-offload --namespace gpu-offload
kubectl --context kind-gpu-offload-nvidia delete namespace gpu-offload
'''
[tasks."npm:ci-local"]
description = "Install npm dependencies through the machine-local package feed"
run = "npm ci"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment