llama-server launch script: dynamic VBR KV cache + inline MTP (buun-llama-cpp)

start_llama_vbr.sh

#!/bin/bash
# Launch buun-llama-cpp with dynamic VBR KV cache + inline MTP speculative decoding.
#
# Companion to: "God damnit buun, there's no binaries on your site"
# Tested on: RX 9070 XT (gfx1201, ROCm 7.2), buun-llama-cpp 3823c9eb6,
#            Qwen3.8-27B-GSQ-RCO-IQ3_XXS-mtp.gguf
#
# FLAG NOTES (comments live here, not inline -- a "# comment" after a trailing
# backslash silently breaks the line continuation and truncates the command):
#
#   -ngl 99        all layers on GPU (anything >= n_layer works)
#   -c 262144      context size. OMIT THIS LINE ENTIRELY to let --fit pick a
#                  size that fits your card. Set it only if you want a specific
#                  target and are willing to fail at startup if it won't fit.
#   -fa on         REQUIRED for turbo/VBR. There is no non-flash-attention path.
#   --kv-unified   REQUIRED for VBR (it needs n_stream == 1).
#   -np 1          VBR is INCOMPATIBLE with -np > 1. One request at a time.
#   -ctk/-ctv vbr  VBR on both cache sides.
#   --vbr-floor    How far it is ALLOWED to degrade -- not where it starts.
#                  Your cache is f16 until the memory budget actually binds.
#                  NOTE: explicit "-ctk vbr -ctv vbr" defaults the floor to
#                  1.25 bpv (the bottom rung). t4 = 4.125 bpv is far more
#                  conservative and is what this script uses.
#   -b/-ub         batch / micro-batch. Bigger = faster prefill, more VRAM.
#   --jinja        use the model's own chat template.
#   --spec-type draft-mtp
#                  Use the MTP draft head baked into the "-mtp" GGUF. Measured
#                  1.68x decode on this model (30.25 -> 50.85 tok/s, temp 0).
#                  Requires an -mtp variant of the model file; the head lives
#                  at blk.64. Without it, this flag does nothing.
#   --spec-draft-n-max 2
#                  Draft depth. Deeper is not always better -- acceptance falls.

MODEL="/path/to/Qwen3.8-27B-GSQ-RCO-IQ3_XXS-mtp.gguf"
SERVER="/path/to/buun-llama-cpp/build/bin/llama-server"

# Fork binaries link their own libggml. Without this you may silently load the
# system llama.cpp's libraries and wonder why the turbo types don't exist.
export LD_LIBRARY_PATH="$(dirname "$SERVER"):$LD_LIBRARY_PATH"

"$SERVER" -m "$MODEL" \
    -ngl 99 \
    -c 262144 \
    -fa on \
    --kv-unified \
    -np 1 \
    -ctk vbr -ctv vbr \
    --vbr-floor t4 \
    -b 2048 -ub 512 \
    --jinja \
    --host 127.0.0.1 --port 8080 \
    --spec-type draft-mtp --spec-draft-n-max 2

# For vision, add an mmproj projector (and --mmproj-gpu-swap if it and the
# speculative context won't both fit in VRAM at once):
#   --mmproj "/path/to/mmproj-F16.gguf"