Victoriametric cluster setup with alert manager. https://blackmetalz.github.io/wip-victoriametrics-from-small-business-to-large-scale.html

setup-vm-cluster.sh

#!/bin/bash

# Variables
VM_VERSION="v1.113.0"
VM_TAR="victoria-metrics-linux-amd64-${VM_VERSION}-cluster.tar.gz"
VM_UTILS_TAR="vmutils-linux-amd64-${VM_VERSION}.tar.gz"
VM_URL="https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/${VM_VERSION}/${VM_TAR}"
VM_UTILS_URL="https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/${VM_VERSION}/${VM_UTILS_TAR}"
INSTALL_DIR="/opt/victoriametrics"
STORAGE_DIR="/data/vmstorage"
CONFIG_DIR="/etc/victoriametrics"
AM_VERSION="0.28.1"
AM_TAR="alertmanager-${AM_VERSION}.linux-amd64.tar.gz"
AM_URL="https://github.com/prometheus/alertmanager/releases/download/v${AM_VERSION}/${AM_TAR}"
# AM_URL="https://github.com/prometheus/alertmanager/releases/download/v0.28.1/alertmanager-0.28.1.linux-amd64.tar.gz"

# Step 1: Create directories
echo "Creating directories..."
mkdir -p ${INSTALL_DIR}/bin
mkdir -p ${STORAGE_DIR}
mkdir -p ${CONFIG_DIR}/vmagent
mkdir -p ${CONFIG_DIR}/vmalert
mkdir -p ${STORAGE_DIR}/alertmanager

# Step 2: Download and extract VM cluster and utils
echo "Downloading and extracting VictoriaMetrics ${VM_VERSION}..."
wget ${VM_URL} -O /tmp/${VM_TAR}
tar -xzf /tmp/${VM_TAR} -C ${INSTALL_DIR}/bin
wget ${VM_UTILS_URL} -O /tmp/${VM_UTILS_TAR}
tar -xzf /tmp/${VM_UTILS_TAR} -C ${INSTALL_DIR}/bin
chmod +x ${INSTALL_DIR}/bin/*-prod
rm /tmp/${VM_TAR} /tmp/${VM_UTILS_TAR}
echo "Downloading and extracting Alertmanager ${AM_VERSION}..."
wget ${AM_URL} -O /tmp/${AM_TAR}
tar -xzf /tmp/${AM_TAR} -C ${INSTALL_DIR}/bin --strip-components=1 alertmanager-0.28.1.linux-amd64/alertmanager
chmod +x ${INSTALL_DIR}/bin/alertmanager
rm /tmp/${AM_TAR}

# Step 3: Update /etc/hosts (if not already set)
echo "Updating /etc/hosts..."
cat << EOF >> /etc/hosts
10.0.0.1 vm-node1
10.0.0.2  vm-node2
10.0.0.3 vm-node3
EOF

# Step 4: Create systemd service files
echo "Creating systemd service files..."

# vmstorage-prod
cat << EOF > /etc/systemd/system/vmstorage.service
[Unit]
Description=VictoriaMetrics vmstorage service
After=network.target

[Service]
Type=simple
ExecStart=${INSTALL_DIR}/bin/vmstorage-prod \
  -storageDataPath=${STORAGE_DIR} \
  -retentionPeriod=30d \
  -httpListenAddr=0.0.0.0:8482 \
  -vminsertAddr=0.0.0.0:8400 \
  -vmselectAddr=0.0.0.0:8401
Restart=on-failure
User=root
Group=root

[Install]
WantedBy=multi-user.target
EOF

# vminsert-prod
cat << EOF > /etc/systemd/system/vminsert.service
[Unit]
Description=VictoriaMetrics vminsert service
After=network.target

[Service]
Type=simple
ExecStart=${INSTALL_DIR}/bin/vminsert-prod \
  -httpListenAddr=0.0.0.0:8480 \
  -replicationFactor=2 \
  -storageNode=vm-node1:8400 \
  -storageNode=vm-node2:8400 \
  -storageNode=vm-node3:8400
Restart=on-failure
User=root
Group=root

[Install]
WantedBy=multi-user.target
EOF

# vmselect-prod
cat << EOF > /etc/systemd/system/vmselect.service
[Unit]
Description=VictoriaMetrics vmselect service
After=network.target

[Service]
Type=simple
ExecStart=${INSTALL_DIR}/bin/vmselect-prod \
  -httpListenAddr=0.0.0.0:8481 \
  -storageNode=vm-node1:8401 \
  -storageNode=vm-node2:8401 \
  -storageNode=vm-node3:8401
Restart=on-failure
User=root
Group=root

[Install]
WantedBy=multi-user.target
EOF

# vmagent-prod
cat << EOF > /etc/systemd/system/vmagent.service
[Unit]
Description=VictoriaMetrics vmagent service
After=network.target

[Service]
Type=simple
ExecStart=${INSTALL_DIR}/bin/vmagent-prod \
  -promscrape.config=${CONFIG_DIR}/vmagent/prometheus.yml \
  -remoteWrite.url=http://vm-node1:8480/api/v1/write \
  -httpListenAddr=0.0.0.0:8429
Restart=on-failure
User=root
Group=root

[Install]
WantedBy=multi-user.target
EOF

# vmalert-prod
cat << EOF > /etc/systemd/system/vmalert.service
[Unit]
Description=VictoriaMetrics vmalert service
After=network.target

[Service]
Type=simple
ExecStart=${INSTALL_DIR}/bin/vmalert-prod \
  -rule=${CONFIG_DIR}/vmalert/alert_rules.yml \
  -datasource.url=http://vm-node1:8481 \
  -notifier.url=http://vm-node1:9093 \
  -httpListenAddr=0.0.0.0:8880
Restart=on-failure
User=root
Group=root

[Install]
WantedBy=multi-user.target
EOF

# alertmanager
cat << EOF > /etc/systemd/system/alertmanager.service
[Unit]
Description=Prometheus Alertmanager service
After=network.target

[Service]
Type=simple
ExecStart=${INSTALL_DIR}/bin/alertmanager \
  --config.file=${CONFIG_DIR}/alertmanager.yml \
  --web.listen-address=0.0.0.0:9093 \
  --storage.path=${STORAGE_DIR}/alertmanager \
  --cluster.listen-address=0.0.0.0:9094 \
  --cluster.peer=vm-node1:9094 \
  --cluster.peer=vm-node2:9094 \
  --cluster.peer=vm-node3:9094
Restart=on-failure
User=root
Group=root

[Install]
WantedBy=multi-user.target
EOF

# Step 5: Create example config files
echo "Creating example config files..."

# prometheus.yml for vmagent
cat << EOF > ${CONFIG_DIR}/vmagent/prometheus.yml
global:
  scrape_interval: 15s
scrape_configs:
  - job_name: 'example'
    static_configs:
      - targets: ['vm-node1:9090']
EOF

# alert_rules.yml for vmalert
cat << EOF > ${CONFIG_DIR}/vmalert/alert_rules.yml
groups:
  - name: ExampleAlerts
    rules:
      - alert: HighLatency
        expr: rate(http_request_duration_seconds[5m]) > 0.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High request latency detected"
          description: "Latency exceeds 0.5s for 5 minutes"
EOF

# Create default alertmanager.yml
cat << EOF > ${CONFIG_DIR}/alertmanager.yml
global:
  resolve_timeout: 5m

route:
  receiver: 'blackhole'
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h

receivers:
- name: 'blackhole'
EOF

# Step 6: Reload systemd and start services
echo "Reloading systemd and starting services..."
systemctl daemon-reload
systemctl enable vmstorage vminsert vmselect vmagent vmalert alertmanager
systemctl start vmstorage vminsert vmselect vmagent vmalert alertmanager

sleep 5

# Step 7: Check status
echo "Checking service status..."
systemctl status vmstorage vminsert vmselect vmagent vmalert alertmanager

echo "Setup completed! Check services at:
- vmstorage: :8482 (HTTP), :8400 (insert), :8401 (select)
- vminsert: :8480
- vmselect: :8481
- vmagent: :8429
- vmalert: :8880
- alertmanager: :9093"