start_qwen3.6_35B_uncensored.sh

gistfile1.txt

#!/bin/bash

# This script starts a local llama.cpp in a rootless podman container.
# Install podman and run this script as a non-root user.

set -xeuo pipefail

# 256k context window.
CONTEXT_WINDOW="${1:-262144}" # 65534, 131072, 262144

MODEL_ID="HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive"
MODEL_URL="https://huggingface.co/HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive/resolve/main/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q5_K_P.gguf"
MMPROJ_URL="https://huggingface.co/HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive/resolve/main/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf"
MODEL_FILE="models/${MODEL_ID}/$(basename $MODEL_URL)"
MMPROJ_FILE="models/${MODEL_ID}/$(basename $MMPROJ_URL)"

SCRIPT_DIR="$(realpath "$(dirname "$0")")"
MODEL_PATH="$SCRIPT_DIR/$MODEL_FILE"
MMPROJ_PATH="$SCRIPT_DIR/$MMPROJ_FILE"

if [ ! -f "$MODEL_PATH" ]; then
    # Create the output directory if it doesn't exist
    mkdir -p "$(dirname "$MODEL_PATH")"
    # Download the file
    echo "Downloading $MODEL_URL to $MODEL_PATH..."
    wget -c "$MODEL_URL" -O "$MODEL_PATH"
fi

if [ ! -f "$MMPROJ_PATH" ]; then
    # Create the output directory if it doesn't exist
    mkdir -p "$(dirname "$MMPROJ_PATH")"
    # Download the file
    echo "Downloading $MMPROJ_URL to $MMPROJ_PATH..."
    wget -c "$MMPROJ_URL" -O "$MMPROJ_PATH"
fi

echo "Starting llama.cpp container..."

CONTAINER=ghcr.io/ggml-org/llama.cpp:server-cuda
podman pull "$CONTAINER"
POD_NAME="qwen36"
PORT="11444"
#PODMAN="podman run -p 8012:8012 -p 8080:8080 -p 11434:11434 --name ai --init -d --rm --device nvidia.com/gpu=all -v $SCRIPT_DIR/models:/models -v $SCRIPT_DIR/models:/root/.cache/llama.cpp $CONTAINER"
PODMAN="podman run -p $PORT:$PORT --name $POD_NAME --init -d --rm -v $SCRIPT_DIR/models:/models -v $SCRIPT_DIR/models:/root/.cache/llama.cpp $CONTAINER"

podman rm -f $POD_NAME || true

$PODMAN --host 0.0.0.0 --port 11444 \
    --model "/$MODEL_FILE" \
    --mmproj "/$MMPROJ_FILE" \
    --jinja \
    -c "$CONTEXT_WINDOW" \
    --parallel 1 \
    --seed 3407 \
    -ngl 0 \
    --no-kv-offload \
    --cache-type-k q4_0 \
    --cache-type-v q4_0 \
    --temp 0.6 \
    --top-p 0.95 \
    --top-k 20 \
    --min-p 0.0 \
    --repeat-penalty 1.0 \
    --presence-penalty 0.0 \
    --chat-template-kwargs '{"enable_thinking":true}'