vast-qwen.sh - spin up / tear down a 2x RTX 3090 vast.ai box serving

gistfile1.txt

#!/usr/bin/env bash
# vast-qwen.sh — spin up / tear down a 2x RTX 3090 vast.ai box serving
# Qwen3.8-27B via vLLM: a plain OpenAI-compatible
# API on localhost:8080, usable from opencode, aider, LiteLLM, curl, etc.
#
# Requires: curl, jq, ssh (all already on this machine). No vastai CLI, no pip.
#
# Auth: export VAST_API_KEY=xxxx   (from https://cloud.vast.ai/manage-keys/)
#       or put the key alone in ~/.vast_api_key
#
# Usage:
#   ./vast-qwen.sh search              # list cheapest matching 2x3090 offers
#   ./vast-qwen.sh create [offer_id]   # rent (cheapest offer if id omitted), launch vLLM
#   ./vast-qwen.sh list                # show your running instances
#   ./vast-qwen.sh wait <instance_id>  # block until ssh + vLLM model endpoint are up
#   ./vast-qwen.sh ssh <instance_id>   # open interactive ssh with -L 8080 tunnel
#   ./vast-qwen.sh tunnel <instance_id>[&] # background-only tunnel (no shell)
#   ./vast-qwen.sh logs <instance_id>  # tail vLLM startup log on the box
#   ./vast-qwen.sh opencode-config     # optional: write a vast-qwen provider into opencode.json
#   ./vast-qwen.sh destroy <instance_id> # stop billing, delete instance
#   ./vast-qwen.sh up                  # create + wait + print ssh tunnel cmd (one shot)
#
# opencode integration is opt-in (run opencode-config yourself) — up/create
# never touch your local client config, since not everyone uses opencode.
#
# Model/launch config can be overridden via env vars, see CONFIG block below.

set -euo pipefail

API="https://console.vast.ai/api/v0"
API_V1="https://console.vast.ai/api/v1"

# ---- CONFIG (override via env) --------------------------------------------
GPU_NAME="${GPU_NAME:-RTX 3090}"
NUM_GPUS="${NUM_GPUS:-2}"
MIN_DISK_GB="${MIN_DISK_GB:-60}"
MIN_CUDA="${MIN_CUDA:-12.4}"
REQUIRE_VERIFIED="${REQUIRE_VERIFIED:-true}"

IMAGE="${IMAGE:-vllm/vllm-openai:v0.27.1}"
MODEL="${MODEL:-Qwen/Qwen3.8-27B-FP8}"
SERVED_NAME="${SERVED_NAME:-qwen3.8-27b-fp8}"
PORT="${PORT:-8080}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-120000}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-160}"
TOOL_CALL_PARSER="${TOOL_CALL_PARSER:-hermes}"
REASONING_PARSER="${REASONING_PARSER:-qwen3}"
GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.92}"
DISK_GB="${DISK_GB:-70}"
LABEL="${LABEL:-qwen38-27b-fp8-dual}"
HF_TOKEN="${HF_TOKEN:-}"
SSH_IDENTITY="${SSH_IDENTITY:-}"

OPENCODE_CONFIG="${OPENCODE_CONFIG:-$HOME/.config/opencode/opencode.json}"
# -----------------------------------------------------------------------------

need() { command -v "$1" >/dev/null 2>&1 || { echo "missing dep: $1" >&2; exit 1; }; }
need curl; need jq; need ssh

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null)
[ -n "$SSH_IDENTITY" ] && SSH_OPTS+=(-i "$SSH_IDENTITY")

api_key() {
  if [ -n "${VAST_API_KEY:-}" ]; then
    printf '%s' "$VAST_API_KEY"
  elif [ -f "$SCRIPT_DIR/.vast_api_key" ]; then
    tr -d ' \n' < "$SCRIPT_DIR/.vast_api_key"
  elif [ -f "$HOME/.vast_api_key" ]; then
    tr -d ' \n' < "$HOME/.vast_api_key"
  else
    echo "No API key. export VAST_API_KEY=... or write it to $SCRIPT_DIR/.vast_api_key or ~/.vast_api_key" >&2
    exit 1
  fi
}

auth_curl() {
  curl -sS -H "Authorization: Bearer $(api_key)" -H "Content-Type: application/json" "$@"
}

cmd_search() {
  local body
  body=$(jq -n \
    --arg gpu "$GPU_NAME" \
    --argjson n "$NUM_GPUS" \
    --argjson disk "$MIN_DISK_GB" \
    --argjson cuda "$MIN_CUDA" \
    --argjson verified "$REQUIRE_VERIFIED" \
    '{
      gpu_name: {eq: $gpu},
      num_gpus: {eq: $n},
      disk_space: {gte: $disk},
      cuda_max_good: {gte: $cuda},
      rentable: {eq: true},
      verified: {eq: $verified},
      type: "on-demand"
    }')
  auth_curl -d "$body" "$API/bundles/" \
    | jq -r '(.offers // .) | sort_by(.dph_total)
        | .[:15][]
        | "\(.id)\t$\(.dph_total|tostring|.[0:6])/hr\t\(.num_gpus)x \(.gpu_name)\tdisk=\(.disk_space)GB\tcuda>=\(.cuda_max_good)\t\(.geolocation // "?")"' \
    | { echo -e "OFFER_ID\tPRICE\tGPUS\tDISK\tCUDA\tLOCATION"; cat; } | column -t -s $'\t'
}

cheapest_offer_id() {
  local body
  body=$(jq -n \
    --arg gpu "$GPU_NAME" \
    --argjson n "$NUM_GPUS" \
    --argjson disk "$MIN_DISK_GB" \
    --argjson cuda "$MIN_CUDA" \
    --argjson verified "$REQUIRE_VERIFIED" \
    '{
      gpu_name: {eq: $gpu},
      num_gpus: {eq: $n},
      disk_space: {gte: $disk},
      cuda_max_good: {gte: $cuda},
      rentable: {eq: true},
      verified: {eq: $verified},
      type: "on-demand"
    }')
  auth_curl -d "$body" "$API/bundles/" | jq -r '(.offers // .) | sort_by(.dph_total) | .[0].id'
}

onstart_script() {
  local hf_line=""
  [ -n "$HF_TOKEN" ] && hf_line="export HF_TOKEN='$HF_TOKEN'"
  cat <<EOF
$hf_line
export HF_HUB_DISABLE_XET=1
nohup bash -c '
  for i in \$(seq 1 5); do
    vllm serve $MODEL \
      --tensor-parallel-size $NUM_GPUS \
      --host 0.0.0.0 \
      --port $PORT \
      --served-model-name $SERVED_NAME \
      --max-model-len $MAX_MODEL_LEN \
      --max-num-seqs $MAX_NUM_SEQS \
      --gpu-memory-utilization $GPU_MEM_UTIL \
      --enable-auto-tool-choice \
      --tool-call-parser $TOOL_CALL_PARSER \
      --reasoning-parser $REASONING_PARSER \
      --trust-remote-code
    echo "vllm exited (attempt \$i), retrying in 15s..." >&2
    sleep 15
  done
  echo "vllm gave up after 5 attempts" >&2
' > /var/log/vllm.log 2>&1 &
EOF
}

cmd_create() {
  local offer_id="${1:-}"
  if [ -z "$offer_id" ]; then
    echo "no offer_id given, picking cheapest matching offer..." >&2
    offer_id=$(cheapest_offer_id)
    [ -n "$offer_id" ] && [ "$offer_id" != "null" ] || { echo "no matching offers found" >&2; exit 1; }
    echo "picked offer $offer_id" >&2
  fi

  local env_str="-p ${PORT}:${PORT}"
  local body
  body=$(jq -n \
    --arg image "$IMAGE" \
    --argjson disk "$DISK_GB" \
    --arg label "$LABEL" \
    --arg env "$env_str" \
    --arg onstart "$(onstart_script)" \
    '{image: $image, disk: $disk, label: $label, env: $env, onstart: $onstart, runtype: "ssh", target_state: "running"}')

  echo "renting offer $offer_id ..." >&2
  local resp instance_id
  resp=$(auth_curl -X PUT -d "$body" "$API/asks/$offer_id/")
  instance_id=$(echo "$resp" | jq -r '.new_contract // .id // empty')
  if [ -z "$instance_id" ]; then
    echo "create failed:" >&2
    echo "$resp" | jq . >&2
    exit 1
  fi
  echo "instance created: $instance_id"
  echo "$instance_id"
}

instance_json() {
  auth_curl "$API/instances/$1/" | jq '.instances // .'
}

cmd_list() {
  # /api/v0/instances/ (collection GET) is deprecated in favor of v1; single-item
  # GET/DELETE by id are still v0 and used elsewhere in this script.
  auth_curl "$API_V1/instances/" \
    | jq -r '.instances[] | "\(.id)\t\(.actual_status // .status)\t\(.label // "-")\t\(.ssh_host // "-")\t\(.ssh_port // "-")\t$\(.dph_total|tostring|.[0:6])/hr"' \
    | { echo -e "ID\tSTATUS\tLABEL\tSSH_HOST\tSSH_PORT\tPRICE"; cat; } | column -t -s $'\t'
}

cmd_wait() {
  local id="$1"
  echo "waiting for instance $id to boot ssh..." >&2
  local host="" port=""
  for i in $(seq 1 60); do
    local j; j=$(instance_json "$id")
    host=$(echo "$j" | jq -r '.ssh_host // empty')
    port=$(echo "$j" | jq -r '.ssh_port // empty')
    [ -n "$host" ] && [ "$host" != "null" ] && [ -n "$port" ] && [ "$port" != "null" ] && break
    sleep 10
  done
  if [ -z "$host" ] || [ "$host" = "null" ]; then
    echo "timed out waiting for ssh_host" >&2; exit 1
  fi
  echo "ssh up: $host:$port — waiting for vLLM model endpoint on :$PORT..." >&2
  for i in $(seq 1 60); do
    if ssh "${SSH_OPTS[@]}" -o ConnectTimeout=10 \
        -p "$port" "root@$host" "curl -sf localhost:$PORT/v1/models" >/dev/null 2>&1; then
      echo "vLLM ready."
      echo "ssh${SSH_IDENTITY:+ -i $SSH_IDENTITY} -p $port root@$host -L $PORT:localhost:$PORT"
      return 0
    fi
    sleep 10
  done
  echo "vLLM did not come up in time — check: $0 logs $id" >&2
  exit 1
}

cmd_ssh() {
  local id="$1"
  local j host port
  j=$(instance_json "$id")
  host=$(echo "$j" | jq -r '.ssh_host')
  port=$(echo "$j" | jq -r '.ssh_port')
  exec ssh "${SSH_OPTS[@]}" \
    -p "$port" "root@$host" -L "$PORT:localhost:$PORT"
}

cmd_tunnel() {
  local id="$1"
  local j host port
  j=$(instance_json "$id")
  host=$(echo "$j" | jq -r '.ssh_host')
  port=$(echo "$j" | jq -r '.ssh_port')
  exec ssh "${SSH_OPTS[@]}" \
    -p "$port" "root@$host" -N -L "$PORT:localhost:$PORT"
}

cmd_logs() {
  local id="$1"
  local j host port
  j=$(instance_json "$id")
  host=$(echo "$j" | jq -r '.ssh_host')
  port=$(echo "$j" | jq -r '.ssh_port')
  ssh "${SSH_OPTS[@]}" \
    -p "$port" "root@$host" "tail -n 100 -f /var/log/vllm.log"
}

cmd_destroy() {
  local id="$1"
  auth_curl -X DELETE "$API/instances/$id/" | jq .
  echo "destroyed $id — billing stopped."
}

cmd_opencode_config() {
  mkdir -p "$(dirname "$OPENCODE_CONFIG")"
  local tmp; tmp=$(mktemp)
  if [ -f "$OPENCODE_CONFIG" ]; then
    jq --arg port "$PORT" --arg served "$SERVED_NAME" '
      .["$schema"] = "https://opencode.ai/config.json" |
      .provider["vast-qwen"] = {
        npm: "@ai-sdk/openai-compatible",
        name: "Vast 2x3090 Qwen3.8-27B",
        options: { baseURL: ("http://localhost:" + $port + "/v1") },
        models: { ($served): { name: "Qwen3.8-27B (AutoRound INT4)" } }
      }' "$OPENCODE_CONFIG" > "$tmp"
  else
    jq -n --arg port "$PORT" --arg served "$SERVED_NAME" '
      {
        "$schema": "https://opencode.ai/config.json",
        provider: {
          "vast-qwen": {
            npm: "@ai-sdk/openai-compatible",
            name: "Vast 2x3090 Qwen3.8-27B",
            options: { baseURL: ("http://localhost:" + $port + "/v1") },
            models: { ($served): { name: "Qwen3.8-27B (AutoRound INT4)" } }
          }
        }
      }' > "$tmp"
  fi
  mv "$tmp" "$OPENCODE_CONFIG"
  echo "wrote $OPENCODE_CONFIG"
  echo "run: opencode  ->  /connect  -> Other -> id 'vast-qwen' (dummy key)  -> /models"
}

cmd_up() {
  local id; id=$(cmd_create "${1:-}" | tail -n1)
  cmd_wait "$id"
  echo "instance id: $id  (destroy with: $0 destroy $id)"
  echo "OpenAI-compatible endpoint once tunneled: http://localhost:$PORT/v1"
  echo "using opencode? run: $0 opencode-config"
}

case "${1:-}" in
  search) cmd_search ;;
  create) shift; cmd_create "${1:-}" ;;
  list) cmd_list ;;
  wait) shift; cmd_wait "$1" ;;
  ssh) shift; cmd_ssh "$1" ;;
  tunnel) shift; cmd_tunnel "$1" ;;
  logs) shift; cmd_logs "$1" ;;
  destroy) shift; cmd_destroy "$1" ;;
  opencode-config) cmd_opencode_config ;;
  up) shift; cmd_up "${1:-}" ;;
  *)
    sed -n '2,25p' "$0"
    exit 1
    ;;
esac