Steer host IRQ/workqueue/RPS away from SPDK reactor cores (given SPDK cpu mask)

steer-away-from-spdk.sh

#!/bin/bash
# Steer host kernel work (IRQ / workqueue / RPS) away from SPDK CPUs.
#
# Usage: steer-away-from-spdk.sh <spdk_cpu_mask>
#   <spdk_cpu_mask>  hex bitmask of CPUs used by spdk_tgt, e.g. 0x3 (CPU 0,1)
#
# It computes "all online CPUs AND NOT spdk_mask" and pins:
#   - IRQ smp_affinity (default + per-IRQ)
#   - unbound workqueue cpumask (global + per-workqueue)
#   - RPS rps_cpus on physical NICs
# to that non-SPDK set.
set -euo pipefail

if [ $# -lt 1 ]; then
    echo "Usage: $0 <spdk_cpu_mask>  (e.g. 0x3)" >&2
    exit 1
fi
spdk_mask="$1"

log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*"; }

# --- compute target mask (non-SPDK online CPUs), formatted for sysfs -----------

online_range=$(cat /sys/devices/system/cpu/online)

# Linux "comma-separated 32-bit hex groups" mask of (online & ~spdk).
target_mask=$(python3 -c "
import sys
rng, spdk_s = sys.argv[1], sys.argv[2]
online = 0
for part in rng.strip().split(','):
    if not part: continue
    if '-' in part:
        a, b = (int(x) for x in part.split('-', 1))
        for i in range(a, b + 1): online |= (1 << i)
    else:
        online |= (1 << int(part))
m = online & ~int(spdk_s, 0)
if m == 0:
    sys.exit('ERROR: spdk mask covers every online CPU; refusing empty target')
groups = []
while m > 0:
    groups.append('{:08x}'.format(m & 0xffffffff)); m >>= 32
print(','.join(reversed(groups)))
" "$online_range" "$spdk_mask")

log "SPDK mask=${spdk_mask}, online=[${online_range}] -> steering to mask ${target_mask}"

# --- print existing masks (compact) --------------------------------------------
# One value per unique mask, with a count of how many files share it, so a node
# with hundreds of IRQs/queues stays a few lines instead of hundreds.

compact() {
    # reads mask values on stdin, prints "value (xN)" groups on one line
    sort | uniq -c | awk '{printf "%s%s (x%s)", (NR>1?", ":""), $2, $1} END{print ""}'
}

# Print current IRQ / workqueue / RPS masks compactly. $1 is a label ("before"/"after").
snapshot() {
    log "Current masks (${1}):"
    printf '  IRQ default : %s\n' "$(cat /proc/irq/default_smp_affinity 2>/dev/null || echo '?')"
    printf '  IRQ per-irq : %s\n' "$(cat /proc/irq/[0-9]*/smp_affinity 2>/dev/null | compact)"
    printf '  workqueue   : %s\n' "$(cat /sys/devices/virtual/workqueue/cpumask 2>/dev/null || echo 'n/a')"
    printf '  wq per-wq   : %s\n' "$(cat /sys/devices/virtual/workqueue/*/cpumask 2>/dev/null | compact)"
    local rps_now="" path
    for path in /sys/class/net/*; do
        [ -L "$path" ] || continue
        case "$(readlink "$path")" in *virtual*) continue ;; esac
        rps_now+=$(cat "$path"/queues/rx-*/rps_cpus 2>/dev/null; echo)
    done
    printf '  RPS         : %s\n' "$(printf '%s' "$rps_now" | grep -v '^$' | compact)"
}

snapshot "before change"

# --- IRQ affinity --------------------------------------------------------------
# Preserve, don't overwrite: for each existing IRQ we compute
# (current_affinity & ~spdk_mask) so that pinning installed by other actors
# (irqbalance, tuned, NIC RSS scripts, other latency-sensitive apps) is kept
# intact and we only carve the SPDK cores out. If that intersection is empty
# (the IRQ was pinned *only* to SPDK cores), we fall back to the full non-SPDK
# mask rather than write an illegal empty set. The global default is a template
# for future IRQs with no per-device intent to preserve, so it just gets the
# full non-SPDK mask.

echo "$target_mask" > /proc/irq/default_smp_affinity

python3 - "$spdk_mask" "$target_mask" <<'PY'
import sys, glob, os

spdk = int(sys.argv[1], 0)
fallback = sys.argv[2]  # already-formatted "all non-SPDK" mask

def parse(s):
    return int(s.strip().replace(',', ''), 16)

def fmt(m):
    groups = []
    while m > 0:
        groups.append('{:08x}'.format(m & 0xffffffff)); m >>= 32
    return ','.join(reversed(groups)) or '0'

updated = skipped = preserved = 0
for f in glob.glob('/proc/irq/[0-9]*/smp_affinity'):
    if not os.access(f, os.W_OK):
        skipped += 1; continue
    try:
        cur = parse(open(f).read())
    except Exception:
        skipped += 1; continue
    new = cur & ~spdk
    if new == 0:
        out = fallback                      # pinned only to SPDK cores -> widen
    else:
        out = fmt(new)
        if new != cur:
            preserved += 1                  # kept some non-SPDK intent
    try:
        with open(f, 'w') as fh:
            fh.write(out)
        updated += 1
    except Exception:
        skipped += 1

print('IRQ: {} updated ({} preserved existing pinning), {} skipped'.format(
    updated, preserved, skipped))
PY

# --- workqueue cpumask ---------------------------------------------------------

wq_dir=/sys/devices/virtual/workqueue
if [ -w "$wq_dir/cpumask" ]; then
    echo "$target_mask" > "$wq_dir/cpumask"
    wq_ok=0 wq_skip=0
    for wq in "$wq_dir"/*/; do
        f="${wq}cpumask"
        [ -w "$f" ] || { wq_skip=$((wq_skip+1)); continue; }
        if echo "$target_mask" > "$f" 2>/dev/null; then wq_ok=$((wq_ok+1)); else wq_skip=$((wq_skip+1)); fi
    done
    log "workqueue: global set, ${wq_ok} per-wq updated, ${wq_skip} skipped"
else
    log "workqueue: $wq_dir/cpumask not writable; skipping"
fi

# --- RPS (physical NICs only) --------------------------------------------------

rps_ok=0 rps_skip=0
for path in /sys/class/net/*; do
    [ -L "$path" ] || continue
    case "$(readlink "$path")" in *virtual*) continue ;; esac
    ifname=$(basename "$path")
    for q in "$path"/queues/rx-*/rps_cpus; do
        [ -w "$q" ] || { rps_skip=$((rps_skip+1)); continue; }
        if echo "$target_mask" > "$q" 2>/dev/null; then rps_ok=$((rps_ok+1)); else rps_skip=$((rps_skip+1)); fi
    done
done
log "RPS: ${rps_ok} queues updated, ${rps_skip} skipped"

snapshot "after change"

log "Done."