Qwen 3.8 Flash - 2x DGX Spark

docker-compose.qwen-3.8-flash.yml

# Adam Conway
#
# Qwen3.8-Flash-Next (NVFP4) on 2x DGX Spark (GB10): TP=2/PP=1 over ConnectX-7
# You can use this file on both nodes, and it's identical on both head and worker; 
# role comes from the compose profile you pick:
#
#   head (spark A, fabric IP 192.168.100.1): docker compose --profile head up -d
#   worker (spark B, fabric IP 192.168.100.2): docker compose --profile worker up -d
#
# Start worker before head
# Endpoint after launch: http://<head-lan-ip>:8000/v1
# model name: qwen38-flash-next
#
# PP=2 is not an option as the N-gram embedding implementation does not support
# pipeline parallelism currently.

x-base-env: &base-env
  HF_HOME: /cache/huggingface
  VLLM_CACHE_ROOT: /cache/huggingface/vllm-cache-qwen38
  PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"

  VLLM_PLE_CPU_OFFLOAD: "0"

  # Required as RadixArk NVFP4 quant is hybrid, using NVFP4 routed experts + FP8 N-gram table
  # vLLM needs FP8 PLE declared and ModelOpt checkpoint doesn't have it
  # so it loads unquantized; we force it here instead
  QWEN38_FORCE_FP8_PLE: "1"

  TORCH_CUDA_ARCH_LIST: 12.1a
  FLASHINFER_CUDA_ARCH_LIST: 12.1a
  CUTE_DSL_ARCH: sm_121a
  FLASHINFER_DISABLE_VERSION_CHECK: "1"
  FLASHINFER_WORKSPACE_BASE: /cache/huggingface/flashinfer

  MASTER_ADDR: 192.168.100.1
  MASTER_PORT: "25000"

  VLLM_NCCL_SO_PATH: /usr/local/lib/python3.12/dist-packages/nvidia/nccl/lib/libnccl.so.2
  NCCL_NET: IB
  NCCL_IB_DISABLE: "0"
  NCCL_IB_HCA: rocep1s0f0,roceP2p1s0f0
  NCCL_SOCKET_IFNAME: enp1s0f0np0,enP2p1s0f0np0
  GLOO_SOCKET_IFNAME: enp1s0f0np0
  TP_SOCKET_IFNAME: enp1s0f0np0
  NCCL_IB_GID_INDEX: "3"
  NCCL_IB_ADDR_FAMILY: AF_INET
  NCCL_IB_ROCE_VERSION_NUM: "2"
  NCCL_CROSS_NIC: "1"
  NCCL_CUMEM_ENABLE: "0"
  NCCL_IGNORE_CPU_AFFINITY: "1"
  NCCL_NVLS_ENABLE: "0"
  NCCL_DEBUG: WARN

x-common: &common
  # Qwen4ExpForConditionalGeneration is only in this VLLM build
  image: vllm/vllm-openai:qwen38-flash-next-arm64-cu130
  restart: "no" # prevent crash loop
  network_mode: host
  ipc: host
  # Both required or NCCL fails
  shm_size: "64gb"
  ulimits:
    memlock: -1
    stack: 67108864
  deploy:
    resources:
      reservations:
        devices:
          - driver: nvidia
            count: all
            capabilities: [gpu]
  devices:
    - /dev/infiniband:/dev/infiniband
  volumes:
      # You can also download on one node and transfer to the second over ConnectX-7; that's what I did
    - ./hf-cache:/cache/huggingface
  # Entrypoint applies hybrid-FP8-PLE patch
  entrypoint:
    - bash
    - -c
    - |
      set -e
      python3 - <<'PY'
      from pathlib import Path
      import sys
      p = Path("/usr/local/lib/python3.12/dist-packages/vllm/models/" "qwen3_8_flash_next/nvidia/ple_layer.py")
      marker = "QWEN38_FORCE_FP8_PLE"
      anchor = (
          '    """Select global-scale FP8 only for quantized PLE checkpoint shards."""\n'
          "\n"
          "    if not isinstance(quant_config, Fp8Config):\n"
      )
      replacement = (
          '    """Select global-scale FP8 only for quantized PLE checkpoint shards."""\n'
          "\n"
          "    # Hybrid checkpoints (modelopt NVFP4 experts + FP8 PLE shards) have a\n"
          "    # ModelOpt config at model level, so Fp8Config never enables\n"
          "    # and PLE is built unquantised. Return FP8 method early;\n"
          "    import os as _os\n"
          "\n"
          '    if _os.environ.get("QWEN38_FORCE_FP8_PLE", "0") == "1" and not isinstance(\n'
          "        quant_config, Fp8Config\n"
          "    ):\n"
          "        return Qwen3_8FlashNextPLEFp8EmbeddingMethod()\n"
          "\n"
          "    if not isinstance(quant_config, Fp8Config):\n"
      )
      src = p.read_text()
      if marker in src:
          print("[patch] hybrid FP8 PLE: already applied")
      elif anchor not in src:
          sys.exit("[patch] FATAL: ple_layer.py has changed, can't patch")
      elif "class Qwen3_8FlashNextPLEFp8EmbeddingMethod" not in src:
          sys.exit("[patch] FATAL: FP8 PLE method class missing, didn't patch")
      else:
          out = src.replace(anchor, replacement, 1)
          compile(out, str(p), "exec")
          p.write_text(out)
          print("[patch] hybrid FP8 PLE applied")
      PY
      exec vllm serve "$$@"

services:
  ## Runs on head, serves model
  head:
    <<: *common
    container_name: qwen38-flash-next
    profiles: [head]
    environment:
      <<: *base-env
      NODE_RANK: "0"
      VLLM_HOST_IP: 192.168.100.1
    command: >
      RadixArk/Qwen3.8-Flash-Next-NVFP4
      --served-model-name qwen38-flash-next
      --host 0.0.0.0 --port 8000
      --tensor-parallel-size 2 --pipeline-parallel-size 1
      --distributed-executor-backend mp
      --gpu-memory-utilization 0.85
      --max-model-len 262144
      --max-num-seqs 256
      --max-num-batched-tokens 8192
      --kv-cache-memory 21474836480
      --max-parallel-loading-workers 1
      --enable-prefix-caching
      --no-enable-flashinfer-autotune
      --enable-auto-tool-choice
      --tool-call-parser qwen3_coder
      --reasoning-parser qwen3
      --nnodes 2 --node-rank 0 --master-addr 192.168.100.1 --master-port 25000
      --enforce-eager
      --limit-mm-per-prompt.image 1 --limit-mm-per-prompt.video 0 --mm-processor-cache-gb 0
      --speculative-config '{"method":"mtp","num_speculative_tokens":3}'
    healthcheck:
      test:
        - CMD-SHELL
        - >-
          python3 -c "import urllib.request;
          urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"
      interval: 15s
      timeout: 10s
      retries: 5
      start_period: 30m

  ## second spark, start this one first
  worker:
    <<: *common
    container_name: qwen38-flash-next
    profiles: [worker]
    environment:
      <<: *base-env
      NODE_RANK: "1"
      VLLM_HOST_IP: 192.168.100.2
    command: >
      RadixArk/Qwen3.8-Flash-Next-NVFP4
      --served-model-name qwen38-flash-next
      --host 0.0.0.0 --port 8000
      --tensor-parallel-size 2 --pipeline-parallel-size 1
      --distributed-executor-backend mp
      --gpu-memory-utilization 0.85
      --max-model-len 262144
      --max-num-seqs 256
      --max-num-batched-tokens 8192
      --kv-cache-memory 21474836480
      --max-parallel-loading-workers 1
      --enable-prefix-caching
      --no-enable-flashinfer-autotune
      --enable-auto-tool-choice
      --tool-call-parser qwen3_coder
      --reasoning-parser qwen3
      --nnodes 2 --node-rank 1 --master-addr 192.168.100.1 --master-port 25000
      --headless
      --enforce-eager
      --limit-mm-per-prompt.image 1 --limit-mm-per-prompt.video 0 --mm-processor-cache-gb 0
      --speculative-config '{"method":"mtp","num_speculative_tokens":3}'