Install the latest NVIDIA drivers on a Proxmox host and LXC container with a GPU configured for passthrough.

proxmox_lxc_install_latest_nvidia_drivers.sh

#!/bin/bash
#
# This script installs the latest NVIDIA drivers on a Proxmox host
# and LXC container guests (Debian based) with a card passed through to them.
#
# It essentially automates the repeatable steps from this guide for re-running after kernel upgrades
# https://forums.plex.tv/t/plex-hw-acceleration-in-lxc-container-anyone-with-success/219289/35
# It automatically grabs the latest version of the driver from: https://github.com/keylase/nvidia-patch
#
# This other script exists that appears to do the same thing, perhaps in a more clever way for the LXC container,
# in addition to automating the creation of the LXC with the GPU configured:
# https://github.com/Saberwolf64/Proxmox-Nvidia-LXC-/blob/proxmox-6.2-1-ubunutu-contributor-Whiskerz007/6.2-1.sh
#
# HOST: Place this script in /opt/nvidia/ and execute
# LXC Guest: Bind-mount the host's /opt/nvidia/ directory to /opt/pve-host, e.g. with:
#   mp2: /opt/nvidia,mp=/opt/pve-host,mountoptions=noatime,ro=1,replicate=0,shared=1
#
# For the guests, it assumes a configuration similar to the following for passing through the GPU:
# lxc.cgroup2.devices.allow: c 195:* rwm
# lxc.cgroup2.devices.allow: c 235:* rwm
# lxc.cgroup2.devices.allow: c 511:* rwm
# lxc.mount.entry: /dev/nvidia0 dev/nvidia0 none bind,optional,create=file
# lxc.mount.entry: /dev/nvidiactl dev/nvidiactl none bind,optional,create=file
# lxc.mount.entry: /dev/nvidia-uvm dev/nvidia-uvm none bind,optional,create=file
# lxc.mount.entry: /dev/nvidia-modeset dev/nvidia-modeset none bind,optional,create=file
# lxc.mount.entry: /dev/nvidia-uvm-tools dev/nvidia-uvm-tools none bind,optional,create=file
#
# Verified to work on Proxmox 6/7/8, with nVidia Quadro P1000/P2000 cards. Tested with Debian and Ubuntu LXC guests.
# Should work (and patch) for RTX/gaming cards as well.
#
# Need to uninstall?
# systemctl disable nvidia-persistenced.service
# systemctl stop nvidia-persistenced.service
# ./$(ls -1 ./NVIDIA-Linux-x86_64-*.run | sort -n | tail -n 1) --uninstall

# Bail on any errors
set -e

#
# functions
#

function die() {
	echo "[FATAL] $@" >&2
	exit 1
}

function ensure_prereqs() {
	required_pkgs=("build-essential" "proxmox-headers-$(uname -r | cut -d . -f 1,2)" "git")
	pkgs_to_install=()

	# Build list of missing packages
	for pkg in "${required_pkgs[@]}"; do
		if ! dpkg -s "$pkg" &>/dev/null; then
			pkgs_to_install+=("$pkg")
		fi
	done

	# Install any missing packages
	if [ "${#pkgs_to_install[@]}" -gt 0 ]; then

		# Set non-interactive frontend for apt
		export DEBIAN_FRONTEND=noninteractive
		export DEBCONF_NONINTERACTIVE_SEEN=true

		apt-get update -q

		for pkg in "${pkgs_to_install[@]}"; do
			apt-get install -y --no-install-recommends "$pkg"
		done
	fi
}

function fetch_driver_if_not_exists() {
	cd "${nv_base_dir}"

	if [[ ! -e "NVIDIA-Linux-x86_64-${driver_version}.run" ]]; then
		if ! wget "https://international.download.nvidia.com/XFree86/Linux-x86_64/${driver_version}/NVIDIA-Linux-x86_64-${driver_version}.run" ; then
			echo "Failed to download current NVIDIA Driver Installer (${driver_version})"
		exit 1
		fi
	else
		echo "Current driver installer present (${nv_installer})"
	fi
}

function install_host() {
	if [[ "$nvidia_module_uptodate" == "true" ]]; then
		die "[Host] nVidia module(s) already up to date and loaded - aborting"
	else
		echo "[Host] nVidia module(s) not up to date or currently loaded - proceeding to install"
		# Install nVidia binary driver
		cd "${nv_base_dir}"
		chmod 755 "$nv_installer"
		if ! $nv_installer --no-questions --ui=none --disable-nouveau ; then
			die "[Host] Failed to run $nv_installer"
		fi
		if ! update-initramfs -u ; then
			die "[Host] Failed to run update-initramfs"
		fi

		# Install systemd service
		if ! cd $(dirname "$nv_persistenced_installer"); then
			echo "[Host] nvidia-persistenced repository not present, cloning..."
			cd "${nv_base_dir}"
			git clone --depth 1 https://github.com/NVIDIA/nvidia-persistenced.git
			cd $(dirname "$nv_persistenced_installer")
		else
			echo "[Host] nvidia-persistenced repository present, pulling..."
			git pull
		fi
		if ! $nv_persistenced_installer ; then
			die "[Host] Failed to run $nv_persistenced_installer"
		fi

		if ! systemctl enable nvidia-persistenced.service ; then
			die '[Host] Failed to enable nvidia-persistenced.service'
		fi

		if ! systemctl start nvidia-persistenced.service ; then
			die '[Host] Failed to start nvidia-persistenced.service'
		fi

		# Patch driver to remove concurrent stream limits
		if ! cd $(dirname "$nv_patch_installer"); then
			echo "[Host] nvidia-patch repository not present, cloning..."
			cd "${nv_base_dir}"
			git clone --depth 1 https://github.com/keylase/nvidia-patch.git
			cd $(dirname "$nv_patch_installer")
		else
			echo "[Host] nvidia-patch repository present, pulling..."
			git pull
		fi
		if ! $nv_patch_installer ; then
			die "[Host] Failed to run $nv_patch_installer"
		fi

		# Run nvidia-smi to verify that everything is working
		if ! nvidia-smi ; then
			die "[LXC] Failed to run nvidia-smi"
		fi
	fi
}

function install_lxc() {
	# Install nVidia binary driver
	cd $(dirname "$nv_installer")
	if ! $nv_installer --no-questions --ui=none --no-kernel-module ; then
		die "[LXC] Failed to run $nv_installer"
	fi

	if ! nvidia-smi ; then
		die "[LXC] Failed to run nvidia-smi"
	fi
}

# Bail early if this isn't being ran as root

[[ "$EUID" -eq 0 ]] || die "$0 must be ran as root - aborting."

# Ensure pre-reqs are present
ensure_prereqs

#
# Global variables
#

driver_version=$(curl -s https://raw.githubusercontent.com/keylase/nvidia-patch/master/README.md | grep -F 'NVIDIA-Linux-' | sed -e 's/[()]/^/g' | awk -F '^' '/ \| YES \| YES \| \[Driver link\]\^/ {print $2}' | tail -n1 | cut -d'/' -f6)

if grep -q '^lxc$' /run/systemd/container 2>/dev/null ; then
	is_container="LXC"
	nv_base_dir="/opt/pve-host"
else
	is_container="Host"
	nv_base_dir="/opt/nvidia"
fi

# https://raw.githubusercontent.com/keylase/nvidia-patch/master/README.md
nv_installer="${nv_base_dir}/NVIDIA-Linux-x86_64-${driver_version}.run"
# https://github.com/NVIDIA/nvidia-persistenced
nv_persistenced_installer="${nv_base_dir}/nvidia-persistenced/init/install.sh"
# https://github.com/keylase/nvidia-patch
nv_patch_installer="${nv_base_dir}/nvidia-patch/patch.sh"


# Determine if current loaded version (if any) is greater than/equal to our install target
cur_driver_ver=$(modinfo -F version nvidia 2>/dev/null)
if ( [[ "$cur_driver_ver" == "$driver_version" ]] || [[ "$cur_driver_ver" > "$driver_version" ]] ) ; then
	nvidia_module_uptodate="true"
else
	nvidia_module_uptodate="false"
fi


#
# main
#

echo "Current system context: $is_container"

fetch_driver_if_not_exists

if [[ "$is_container" == "LXC" ]]; then
	install_lxc
elif [[ "$is_container" == "Host" ]]; then
	install_host
else
	die "Can not determine if we are a container or host"
fi