llama-server hosted qwen3.6-35b-a3b with pi.dev config

_start.sh

# Run these manually

##### settings.json below goes in ~/.pi/agent/

curl -OL https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf
curl -LO https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/resolve/main/mmproj-BF16.gguf

nix run github:numtide/llm-agents.nix#pi -- install npm:pi-llama-cpp

# This shell will be used for launching llama-server
NIXPKGS_ALLOW_UNFREE=1 nix-shell -p "llama-cpp.override { cudaSupport = true; }" --command zsh
nix run github:numtide/nix-gl-host -- ./run-llama-server.sh

client-opencode-settings.jsonc

## % jq <~/.config/opencode/opencode.jsonc
{
  "$schema": "https://opencode.ai/config.json",
  "provider": {
    "local": {
      "npm": "@ai-sdk/openai-compatible",
      "name": "Local Models",
      "options": {
        "baseURL": "http://localhost:8001/v1"
      },
      "models": {
        "qwen3.6": {
          "id": "qwen3.6-35b-a3b"
        }
      }
    }
  },
  "model": "local/qwen3.6",
  "compaction": {
    "auto": true,
    "prune": true,
    "reserved": 8192
  }
}

client-pi-dev-settings.jsonc

## % jq <~/.pi/agent/settings.json
{
  "theme": "light",
  "packages": [
    "npm:pi-llama-cpp",
    "npm:@gonrocca/zero-pi"
  ],
  "llamaServerUrl": "http://127.0.0.1:8001",
  "defaultProvider": "llama-server",
  "defaultModel": "qwen3.6-35b-a3b",
  "defaultThinkingLevel": "medium"
}

client-qwen-code-settings.jsonc

#### % jq <~/.qwen/settings.json
{
  "modelProviders": {
    "openai": [
      {
        "id": "qwen3.6-35b-a3b",
        "name": "Qwen3 35B",
        "baseUrl": "http://localhost:15000/v1",
        "description": "Local Qwen 3.6 35B",
        "envKey": "LOCAL_API_KEY_NOT_NEEDED",
        "generationConfig": {
          "contextWindowSize": 250000
        }
      }
    ]
  },
  "env": {
    "LOCAL_API_KEY_NOT_NEEDED": "na"
  },
  "security": {
    "auth": {
      "selectedType": "openai"
    }
  },
  "model": {
    "name": "qwen3.6-35b-a3b"
  },
}

run-llama-server.sh

#!/bin/sh
exec llama-server \
  --model ./Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf \
  --mmproj ./mmproj-BF16.gguf \
  --image-min-tokens 1024 \
  --port 8001 \
  --alias qwen3.6-35b-a3b \
  --ctx-size 0 \
  --predict 32768 \
  --no-context-shift \
  --temp 0.6 \
  --top-p 0.95 \
  --top-k 20 \
  --repeat-penalty 1.00 \
  --presence-penalty 0.00 \
  --fit on \
  --flash-attn on \
  --cache-type-k q8_0 \
  --cache-type-v q8_0 \
  --chat-template-kwargs '{"preserve_thinking": true}' \
  --metrics

ZZZnvidia-smi.txt

Takes up 30G across the two GPUs:


Sat May 23 16:19:22 2026
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 595.58.03              Driver Version: 595.58.03      CUDA Version: 13.2     |
+-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA H100 NVL                Off |   00000000:01:00.0 Off |                    0 |
| N/A   47C    P0             94W /  400W |   15719MiB /  95830MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+
|   1  NVIDIA H100 NVL                Off |   00000000:C1:00.0 Off |                    0 |
| N/A   45C    P0             93W /  400W |   14353MiB /  95830MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+

+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI              PID   Type   Process name                        GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
|    0   N/A  N/A          504118      C   llama-server                          15706MiB |
|    1   N/A  N/A          504118      C   llama-server                          14340MiB |
+-----------------------------------------------------------------------------------------+