User Data Installation for EC2 RKE2 (CNI Calico)

rancher_rke2_base.sh

#!/bin/bash
# base.sh - Main entry point for user data configuration

# ============================================================================== #
#  Enable logging
# ============================================================================== #
exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
echo "[START] UserData Script - $(date)"

# ============================================================================== #
#  Environment Variables
# ============================================================================== #
NODE_ROLE=${node_role}
CLUSTER_NAME=${cluster_name}
HOSTNAME=${hostname}
ENVIRONMENT=${environment}
AWS_REGION=${region}
KUBERNETES_VERSION=${kubernetes_version}
RANCHER_VERSION=${rancher_version}
CERT_MANAGER_VERSION=${cert_manager_version}
DOMAIN_NAME=${domain_name}
NLB_DNS_NAME=${nlb_dns_name}
RANCHER_TOKEN=${rancher_token}
MASTER_PRIVATE_IP=${master_private_ip}
CALICO_VERSION="v3.26.3"
TIGERA_VERSION="v3.26.3"
ENABLE_BGP="true"
ENABLE_VXLAN="false"
ENABLE_WIREGUARD="false"
POD_CIDR="10.42.0.0/16"
SERVICE_CIDR="10.43.0.0/16"
CLUSTER_DOMAIN="cluster.local"

# Set hostname
hostnamectl set-hostname $HOSTNAME

# Detect OS and set package manager
if [ -f /etc/os-release ]; then
    . /etc/os-release
    if [[ "$NAME" == "Ubuntu" ]]; then
        OS="Ubuntu"
        DEFAULT_USER="ubuntu"
        NFS_BIN="nfs-common"
        PKG_MANAGER="apt-get"
        PKG_UPDATE="$PKG_MANAGER update && $PKG_MANAGER upgrade -y"

        # Additional packages for Ubuntu
        echo "[INFO] Installing Ubuntu prerequisites..."
        $PKG_MANAGER install -y apt-transport-https ca-certificates curl software-properties-common

        # Add AWS CloudWatch Agent repository for Ubuntu
        echo "[INFO] Adding AWS repositories..."
        wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
        dpkg -i amazon-cloudwatch-agent.deb
        rm amazon-cloudwatch-agent.deb

        # Install AWS CLI v2 for Ubuntu
        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
        unzip awscliv2.zip
        ./aws/install
        rm -rf aws awscliv2.zip

        # Set Ubuntu-specific packages
        COMMON_PACKAGES="$NFS_BIN curl htop jq netcat-openbsd net-tools vim tmux wget unzip"

    elif [[ "$NAME" == *"Amazon Linux"* ]]; then
        OS="Amazon Linux"
        DEFAULT_USER="ec2-user"
        NFS_BIN="nfs-utils"
        if [[ "$VERSION" == *"2023"* ]]; then
            PKG_MANAGER="dnf"
        else
            PKG_MANAGER="yum"
        fi
        PKG_UPDATE="$PKG_MANAGER update -y && $PKG_MANAGER upgrade -y"

        # Add AWS repositories for Amazon Linux
        echo "[INFO] Adding AWS repositories..."
        $PKG_MANAGER install -y amazon-cloudwatch-agent aws-cli

        # Set Amazon Linux-specific packages
        COMMON_PACKAGES="$NFS_BIN curl htop jq nc net-tools vim tmux wget unzip"
    fi
fi


# ============================================================================== #
#  Update System Packages
# ============================================================================== #
echo "[INFO] Updating system packages..."
eval $PKG_UPDATE


# ============================================================================== #
#  Install Common Packages
# ============================================================================== #
echo "[INFO] Installing common packages..."
$PKG_MANAGER install -y $COMMON_PACKAGES


# ============================================================================== #
#  Verify Installations
# ============================================================================== #
echo "[INFO] Verifying critical package installations..."
command -v aws >/dev/null 2>&1 || { echo "AWS CLI is not installed correctly"; exit 1; }
command -v amazon-cloudwatch-agent >/dev/null 2>&1 || { echo "CloudWatch agent is not installed correctly"; exit 1; }


# ============================================================================== #
#  Configure AWS CLI
# ============================================================================== #
echo "[INFO] Configuring AWS CLI..."
if [[ "$OS" == "Ubuntu" ]]; then
    # Ubuntu configuration
    mkdir -p /home/ubuntu/.aws
    cat > /home/ubuntu/.aws/config <<EOF
[default]
region = $AWS_REGION
output = json
EOF
else
    # Amazon Linux configuration
    mkdir -p /home/ec2-user/.aws
    cat > /home/ec2-user/.aws/config <<EOF
[default]
region = $AWS_REGION
output = json
EOF
fi


# ============================================================================== #
#  Install and Configure CloudWatch Agent
# ============================================================================== #
echo "[INFO] Installing CloudWatch agent..."

# Download and install CloudWatch agent if not present
if [[ "$OS" == "Ubuntu" ]]; then
    if ! dpkg -l | grep -q "amazon-cloudwatch-agent"; then
        wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
        dpkg -i amazon-cloudwatch-agent.deb
        rm amazon-cloudwatch-agent.deb
    fi
else
    if ! rpm -q amazon-cloudwatch-agent; then
        yum install -y amazon-cloudwatch-agent
    fi
fi

echo "[INFO] Configuring CloudWatch agent..."
mkdir -p /opt/aws/amazon-cloudwatch-agent/etc/

# Create CloudWatch agent configuration with proper variable handling
cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json <<EOF
{
    "agent": {
        "metrics_collection_interval": 60,
        "run_as_user": "root"
    },
    "metrics": {
        "metrics_collected": {
            "cpu": {
                "measurement": [
                    "cpu_usage_idle",
                    "cpu_usage_user",
                    "cpu_usage_system"
                ],
                "totalcpu": true
            },
            "disk": {
                "measurement": [
                    "used_percent",
                    "used",
                    "total"
                ],
                "resources": [ "*" ]
            },
            "mem": {
                "measurement": [
                    "mem_used_percent",
                    "mem_total",
                    "mem_used"
                ]
            }
        },
        "append_dimensions": {
            "AutoScalingGroupName": "$$aws:AutoScalingGroupName",
            "ImageId": "$$aws:ImageId",
            "InstanceId": "$$aws:InstanceId",
            "InstanceType": "$$aws:InstanceType",
            "NodeRole": "$NODE_ROLE"
        }
    },
    "logs": {
        "logs_collected": {
            "files": {
                "collect_list": [
                    {
                        "file_path": "/var/log/syslog",
                        "log_group_name": "/$ENVIRONMENT/system",
                        "log_stream_name": "{instance_id}",
                        "timestamp_format": "%b %d %H:%M:%S"
                    },
                    {
                        "file_path": "/var/log/auth.log",
                        "log_group_name": "/$ENVIRONMENT/security",
                        "log_stream_name": "{instance_id}",
                        "timestamp_format": "%b %d %H:%M:%S"
                    },
                    {
                        "file_path": "/var/log/user-data.log",
                        "log_group_name": "/$ENVIRONMENT/userdata",
                        "log_stream_name": "{instance_id}",
                        "timestamp_format": "%Y-%m-%d %H:%M:%S"
                    }
                ]
            }
        }
    }
}
EOF

# Start the CloudWatch agent with proper configuration
/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json

# Enable and start the service
systemctl enable amazon-cloudwatch-agent
systemctl start amazon-cloudwatch-agent

# Verify the agent status
echo "[INFO] Verifying CloudWatch agent status..."
systemctl status amazon-cloudwatch-agent --no-pager

# Additional error checking
if ! systemctl is-active amazon-cloudwatch-agent >/dev/null 2>&1; then
    echo "[WARNING] CloudWatch agent failed to start. Checking logs..."
    journalctl -u amazon-cloudwatch-agent --no-pager | tail -n 50

    # Check configuration
    /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a status
fi


# ============================================================================== #
#  System configuration
# ============================================================================== #
echo "[INFO] Configuring system settings..."

# Set timezone
timedatectl set-timezone UTC

# Configure sysctl
cat > /etc/sysctl.d/99-instance.conf <<EOF
# Network performance
net.core.somaxconn = 65535
net.core.netdev_max_backlog = 65536
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_max_syn_backlog = 65536
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 87380 16777216
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_intvl = 15

# Kubernetes networking requirements
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.ipv4.conf.all.forwarding = 1
net.ipv4.neigh.default.gc_thresh1 = 4096
net.ipv4.neigh.default.gc_thresh2 = 8192
net.ipv4.neigh.default.gc_thresh3 = 16384
fs.inotify.max_user_instances = 8192
fs.inotify.max_user_watches = 1048576

# Bridge networking for Kubernetes
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1

# Calico-specific settings
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.forwarding = 1
net.ipv6.conf.all.forwarding = 1
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 10
net.ipv4.tcp_keepalive_intvl = 60
EOF

sysctl -p /etc/sysctl.d/99-instance.conf


# ============================================================================== #

# Execute additional scripts in order
%{ for script in additional_scripts ~}
echo "[INFO] Executing ${basename(script)} configuration..."
${script}
%{ endfor ~}

echo "[END] UserData Script - $(date)"