quantization

quantization.md

https://dev.to/0xkoji/quantizing-gemma-4-on-mac-with-llamacpp-54k7

1. download a model to models via hf-cli

hf download google/gemma-4-12B-it --local-dir models/google--gemma-4-12B-it

2. run quantize.sh

#!/bin/bash

# Exit on any unexpected error
set -e

# Define relative paths based on the dev.to article structure
LLAMA_CPP_DIR="../llama.cpp"
CONVERT_SCRIPT="${LLAMA_CPP_DIR}/convert_hf_to_gguf.py"
QUANTIZE_BIN="${LLAMA_CPP_DIR}/build/bin/llama-quantize"
MODELS_DIR="models"
GGUF_DIR="gguf"

# 0. Sanity checks to ensure paths exist
if [ ! -d "$LLAMA_CPP_DIR" ]; then
    echo "❌ Error: llama.cpp directory not found at $LLAMA_CPP_DIR"
    echo "Please make sure this script is run from your workspace directory (e.g., 'quantization')."
    exit 1
fi

if [ ! -f "$CONVERT_SCRIPT" ]; then
    echo "❌ Error: Conversion script not found at $CONVERT_SCRIPT"
    exit 1
fi

if [ ! -f "$QUANTIZE_BIN" ]; then
    echo "❌ Error: Quantize binary not found at $QUANTIZE_BIN"
    echo "Please build llama.cpp first using cmake."
    exit 1
fi

# Ensure output gguf directory exists
mkdir -p "$GGUF_DIR"

# ==========================================
# 1. Show selections of models from models folder
# ==========================================
if [ ! -d "$MODELS_DIR" ] || [ -z "$(ls -A $MODELS_DIR 2>/dev/null)" ]; then
    echo "❌ Error: The '$MODELS_DIR' directory is empty or does not exist."
    echo "Please download your model into '$MODELS_DIR/' first (e.g., models/gemma-4-E4B-it)."
    exit 1
fi

echo "==========================================="
echo " Available Models in '$MODELS_DIR':"
echo "==========================================="

# Read directories into an array
mapfile -t models < <(find "$MODELS_DIR" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)

if [ ${#models[@]} -eq 0 ]; then
    echo "No model directories found inside '$MODELS_DIR'."
    exit 1
fi

for i in "${!models[@]}"; do
    echo " [$((i+1))] ${models[$i]}"
done

# ==========================================
# 2. Select a target model
# ==========================================
echo "==========================================="
read -p "Select a model number to process (1-${#models[@]}): " selection

# Validate input
if ! [[ "$selection" =~ ^[0-9]+$ ]] || [ "$selection" -lt 1 ] || [ "$selection" -gt "${#models[@]}" ]; then
    echo "❌ Invalid selection. Exiting."
    exit 1
fi

# Target model folder configuration
SELECTED_MODEL_NAME="${models[$((selection-1))]}"
TARGET_MODEL_PATH="${MODELS_DIR}/${SELECTED_MODEL_NAME}"

echo "👉 Selected: $SELECTED_MODEL_NAME"

# ==========================================
# 3. Convert model to GGUF BF16
# ==========================================
OUTPUT_BF16_GGUF="${GGUF_DIR}/${SELECTED_MODEL_NAME}-BF16.gguf"

echo ""
echo "⏳ Step 3: Converting Hugging Face model to GGUF BF16 format..."
echo "Running: python $CONVERT_SCRIPT $TARGET_MODEL_PATH --outfile $OUTPUT_BF16_GGUF --outtype bf16"
echo "----------------------------------------------------------------"

python "$CONVERT_SCRIPT" \
    "$TARGET_MODEL_PATH" \
    --outfile "$OUTPUT_BF16_GGUF" \
    --outtype bf16

echo "✅ Successfully created: $OUTPUT_BF16_GGUF"

# ==========================================
# 4. Quantize the GGUF model
# ==========================================
# Default quantization type from the article is Q4_K_M
DEFAULT_QUANT="Q4_K_M"

echo ""
echo "==========================================="
read -p "Enter quantization type [Default: $DEFAULT_QUANT]: " USER_QUANT
QUANT_TYPE="${USER_QUANT:-$DEFAULT_QUANT}"

OUTPUT_QUANT_GGUF="${GGUF_DIR}/${SELECTED_MODEL_NAME}-${QUANT_TYPE}.gguf"

echo ""
echo "⏳ Step 4: Quantizing BF16 GGUF model to $QUANT_TYPE..."
echo "Running: $QUANTIZE_BIN $OUTPUT_BF16_GGUF $OUTPUT_QUANT_GGUF $QUANT_TYPE"
echo "----------------------------------------------------------------"

"$QUANTIZE_BIN" \
    "$OUTPUT_BF16_GGUF" \
    "$OUTPUT_QUANT_GGUF" \
    "$QUANT_TYPE"

echo "----------------------------------------------------------------"
echo "🎉 Workflow Complete!"
echo "✅ Quantized model available at: $OUTPUT_QUANT_GGUF"
echo ""
echo "💡 To test this model, run the following command:"
echo "../llama.cpp/build/bin/llama-cli -m $OUTPUT_QUANT_GGUF -ngl 99 --temp 0.7 -c 4096"