Run vLLM in WSL2 environment with Ryzen AI Max+ 395 (gfx1151)

docker-compose.yml

services:
  vllm:
    image: rocm/vllm:rocm7.13.0_gfx1151_ubuntu24.04_py3.13_pytorch_2.10.0_vllm_0.19.1
    ports:
      - 8000:80
    ipc: host
    shm_size: 8G
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp:unconfined
    group_add:
      - video
    devices:
      - /dev/dxg
    environment:
      VLLM_LOGGING_LEVEL: null # set DEBUG to enable verbose logging
      HF_TOKEN: ${HF_TOKEN}
    volumes:
      - type: bind
        source: /usr/lib/wsl/lib/libdxcore.so
        target: /usr/lib/libdxcore.so
        read_only: true
      - type: bind
        source: /opt/rocm/lib/librocdxg.so
        target: /usr/lib/librocdxg.so
        read_only: true
      - type: bind
        source: /opt/rocm/share/rocdxg/dids.conf
        target: /usr/share/rocdxg/dids.conf
        read_only: true
      - huggingface-cache:/root/.cache/huggingface
    command:
      - bash
      - -ec
      - |
        patch -p0 -d /opt/python/lib/python3.13/site-packages/ << 'EOF'
        --- vllm/platforms/__init__.py
        +++ vllm/platforms/__init__.py
        @@ -111,20 +111,3 @@
         def rocm_platform_plugin() -> str | None:
        -    is_rocm = False
        -    logger.debug("Checking if ROCm platform is available.")
        -    try:
        -        import amdsmi
        -
        -        amdsmi.amdsmi_init()
        -        try:
        -            if len(amdsmi.amdsmi_get_processor_handles()) > 0:
        -                is_rocm = True
        -                logger.debug("Confirmed ROCm platform is available.")
        -            else:
        -                logger.debug("ROCm platform is not available because no GPU is found.")
        -        finally:
        -            amdsmi.amdsmi_shut_down()
        -    except Exception as e:
        -        logger.debug("ROCm platform is not available because: %s", str(e))
        -
        -    return "vllm.platforms.rocm.RocmPlatform" if is_rocm else None
        +    return "vllm.platforms.rocm.RocmPlatform"

        --- vllm/platforms/rocm.py
        +++ vllm/platforms/rocm.py
        @@ -125,17 +125,3 @@
         def _get_gcn_arch() -> str:
        -    """
        -    Get GCN arch via amdsmi (no CUDA init), fallback to torch.cuda.
        -    Called once at module level; result stored in _GCN_ARCH.
        -    """
        -    try:
        -        return _query_gcn_arch_from_amdsmi()
        -    except Exception as e:
        -        logger.debug("Failed to get GCN arch via amdsmi: %s", e)
        -        logger.warning_once(
        -            "Failed to get GCN arch via amdsmi, falling back to torch.cuda. "
        -            "This will initialize CUDA and may cause "
        -            "issues if CUDA_VISIBLE_DEVICES is not set yet."
        -        )
        -    # Ultimate fallback: use torch.cuda (will initialize CUDA)
        -    return torch.cuda.get_device_properties("cuda").gcnArchName
        +    return "gfx1151"

        EOF
        vllm serve "${MODEL_NAME}" --host 0.0.0.0 --port 80 --enforce-eager

volumes:
  huggingface-cache:
    external: true

~README.md

1. Setup librocdxg (777e09af)

2. Create .env file

   MODEL_NAME=Qwen/Qwen3-0.6B
   HF_TOKEN=

3. Start the vLLM server

   $ docker volume create huggingface-cache
   $ docker compose up -d

4. Try it

   $ curl http://localhost:8000/v1/responses -H "Content-Type: application/json" -d '{"model": "Qwen/Qwen3-0.6B","input": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Explain quantum computing in one short sentence."}]}'

Note

This warning message is a expected log.

/opt/python/lib/python3.13/site-packages/torch/cuda/__init__.py:971: UserWarning: Can't initialize amdsmi - Error code: 34
  raw_cnt = _raw_device_count_amdsmi()

Tip

Running without --enforce-eager flag rases an error.

/path/to/librocdxg/src/wddm/queue.cpp:841: hsa_status_t wsl::thunk::ComputeQueue::VendorSpecificAqlToPm4(char*, amd_aql_pm4_ib*): Assertion `packet->ven_hdr == AMD_AQL_FORMAT_PM4_IB' failed.
(APIServer pid=1) INFO 06-07 09:46:02 [loggers.py:259] Engine 000: Avg prompt throughput: 2.8 tokens/s, Avg generation throughput: 0.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
(APIServer pid=1) INFO 06-07 09:46:12 [loggers.py:259] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
(APIServer pid=1) ERROR 06-07 09:46:24 [core_client.py:667] Engine core proc EngineCore died unexpectedly, shutting down client.
(APIServer pid=1) ERROR 06-07 09:46:26 [async_llm.py:707] AsyncLLM output_handler failed.
(APIServer pid=1) ERROR 06-07 09:46:26 [async_llm.py:707] Traceback (most recent call last):
(APIServer pid=1) ERROR 06-07 09:46:26 [async_llm.py:707]   File "/opt/python/lib/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 663, in output_handler
(APIServer pid=1) ERROR 06-07 09:46:26 [async_llm.py:707]     outputs = await engine_core.get_output_async()
(APIServer pid=1) ERROR 06-07 09:46:26 [async_llm.py:707]               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...
(APIServer pid=1) ERROR 06-07 09:46:26 [async_llm.py:707] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause.