Script for Endpoint Security manifest download and version check

download-manifest-esagent.sh

#!/bin/bash

### Script for Endpoint Security manifest download and version check
### This script is extracted from the main Elastic Stack Airgap Installation script.
### Created with Google Gemini
### Internal repo kepada Elastic Fleet.
### use with own risks
### buat directory ini bawah html /usr/share/nginx/html
### Added Manifest - 20250522
### Ubuntu / Debian - apt install nginx
### Alma Linux / Rocky Linux - dnf install nginx
### Created with Google Gemini

# Define common variables (ensure these match your main script's context)
DOWNLPATH="/usr/share/nginx/html" # Assuming Nginx root for downloads
# CRITICAL CHECK: Ensure DOWNLPATH is not empty right after definition
if [[ -z "$DOWNLPATH" ]]; then
  echo "CRITICAL ERROR: DOWNLPATH variable is empty. Cannot proceed. Please check script file for hidden characters or encoding issues."
  exit 1
fi

VERSI="8.18.1" # Use consistent versioning
ENDPOINT_VERSION="$VERSI" # Use consistent versioning
LOCAL_MIRROR_URL="http://localhost" # Base URL for your local Nginx mirror

# Initialize manifest versions to empty strings for later display
REMOTE_MANIFEST_VERSION=""
LOCAL_MANIFEST_VERSION=""

# Define manifest specific paths and URLs
ENDPOINT_DOWNLOAD_PATH="$DOWNLPATH/downloads/endpoint"
# CRITICAL CHECK: Ensure ENDPOINT_DOWNLOAD_PATH is not empty
if [[ -z "$ENDPOINT_DOWNLOAD_PATH" ]]; then
  echo "CRITICAL ERROR: ENDPOINT_DOWNLOAD_PATH variable is empty. Cannot proceed. Please check script file for hidden characters or encoding issues."
  exit 1
fi

MANIFEST_DIR="$ENDPOINT_DOWNLOAD_PATH/manifest"
# CRITICAL CHECK: Ensure MANIFEST_DIR is not empty
if [[ -z "$MANIFEST_DIR" ]]; then
  echo "CRITICAL ERROR: MANIFEST_DIR variable is empty. Cannot proceed. Please check script file for hidden characters or encoding issues."
  exit 1
fi

MANIFEST_ZIP_FILENAME="artifacts-$ENDPOINT_VERSION.zip"
REMOTE_MANIFEST_URL="https://artifacts.security.elastic.co/downloads/endpoint/manifest/$MANIFEST_ZIP_FILENAME"
LOCAL_MANIFEST_URL="$LOCAL_MIRROR_URL/downloads/endpoint/manifest/$MANIFEST_ZIP_FILENAME"
LOCAL_MANIFEST_PATH="$MANIFEST_DIR/$MANIFEST_ZIP_FILENAME" # Direct path to the local zip file

# CRITICAL CHECKS for URL variables
if [[ -z "$REMOTE_MANIFEST_URL" ]]; then
  echo "CRITICAL ERROR: REMOTE_MANIFEST_URL variable is empty. Cannot proceed. Please check script file for hidden characters or encoding issues."
  exit 1
fi
if [[ -z "$LOCAL_MIRROR_URL" ]]; then
  echo "CRITICAL ERROR: LOCAL_MIRROR_URL variable is empty. Cannot proceed. Please check script file for hidden characters or encoding issues."
  exit 1
fi
if [[ -z "$LOCAL_MANIFEST_URL" ]]; then
  echo "CRITICAL ERROR: LOCAL_MANIFEST_URL variable is empty. Cannot proceed. Please check script file for hidden characters or encoding issues."
  exit 1
fi
if [[ -z "$LOCAL_MANIFEST_PATH" ]]; then
  echo "CRITICAL ERROR: LOCAL_MANIFEST_PATH variable is empty. Cannot proceed. Please check script file for hidden characters or encoding issues."
  exit 1
fi


echo "--- Endpoint Security Manifest Debug Script ---"
echo "DOWNLPATH: $DOWNLPATH"
echo "ENDPOINT_VERSION: $ENDPOINT_VERSION"
echo "LOCAL_MIRROR_URL: $LOCAL_MIRROR_URL"
echo "REMOTE_MANIFEST_URL: $REMOTE_MANIFEST_URL"
echo "LOCAL_MANIFEST_URL (via Nginx): $LOCAL_MANIFEST_URL"
echo "LOCAL_MANIFEST_PATH (direct file): $LOCAL_MANIFEST_PATH"
echo "-----------------------------------------------"

# Check for distribution and install Nginx, jq, curl, wget, gnupg, and unzip
echo "Checking OS distribution and installing necessary packages (Nginx, jq, curl, wget, gnupg, unzip)..."
if [[ -f /etc/os-release ]]; then
  source /etc/os-release
  if [[ "$ID" == "ubuntu" || "$ID" == "debian" ]]; then
    sudo apt update
    sudo apt install -y nginx jq curl wget gnupg unzip
  elif [[ "$ID_LIKE" == *"rhel"* || "$ID" == "centos" || "$ID" == "rocky" || "$ID" == "almalinux" ]]; then
    sudo dnf install -y nginx jq curl wget gnupg unzip
  else
    echo "Warning: Unrecognized distribution. Please ensure Nginx, jq, curl, wget, gnupg, and unzip are installed manually."
  fi
else
  echo "Warning: Could not determine OS distribution. Please ensure Nginx, jq, curl, wget, gnupg, and unzip are installed manually."
fi
echo "All required packages checked/installed."
echo "-----------------------------------------------"

# --- Dependency Check (Crucial for debugging manifest processing) ---
echo "Checking required commands for manifest processing..."
REQUIRED_CMDS="curl unzip jq wget"
ALL_CMDS_EXIST=true
for cmd in $REQUIRED_CMDS; do
  if ! command -v "$cmd" &> /dev/null; then
    echo "Error: '$cmd' command not found. Please install it."
    ALL_CMDS_EXIST=false
  fi
done

if ! $ALL_CMDS_EXIST; then
  echo "Please install missing dependencies before running this debug script."
  echo "On Debian/Ubuntu: sudo apt install -y curl unzip jq wget"
  echo "On RHEL/CentOS/AlmaLinux/RockyLinux: sudo dnf install -y curl unzip jq wget"
  exit 1
fi
echo "All required commands for manifest processing found."
echo "-----------------------------------------------"

# --- Directory Setup ---
echo "Ensuring manifest directory exists: $MANIFEST_DIR"
sudo mkdir -p "$MANIFEST_DIR"
if [ $? -ne 0 ]; then
  echo "Error: Failed to create directory $MANIFEST_DIR. Exiting debug script."
  exit 1
fi
sudo chmod 755 "$ENDPOINT_DOWNLOAD_PATH" "$MANIFEST_DIR"
echo "Manifest directory setup complete."
echo "-----------------------------------------------"

# --- Get Remote Manifest Version ---
echo "Step 1: Getting remote Endpoint Security manifest version from $REMOTE_MANIFEST_URL..."
TEMP_REMOTE_ZIP="/tmp/remote_manifest_$$.zip" # Temporary file for remote zip
echo "  Downloading remote manifest to temporary file: $TEMP_REMOTE_ZIP"
curl -s "$REMOTE_MANIFEST_URL" -o "$TEMP_REMOTE_ZIP"
CURL_STATUS=$?

if [[ "$CURL_STATUS" -ne 0 ]]; then
  echo "Warning: curl failed to download remote manifest (status: $CURL_STATUS)."
  echo "  Raw content (if any): $(cat "$TEMP_REMOTE_ZIP" 2>/dev/null | head -c 200) (truncated)"
elif [ ! -f "$TEMP_REMOTE_ZIP" ]; then
  echo "Warning: Remote manifest zip file was not created."
else
  # Extract manifest.json from the downloaded zip and pipe to jq
  REMOTE_MANIFEST_VERSION=$(unzip -p "$TEMP_REMOTE_ZIP" manifest.json 2>/dev/null | jq -r .manifest_version 2>/dev/null)
  UNZIP_JQ_STATUS=$?
  if [[ "$UNZIP_JQ_STATUS" -ne 0 || -z "$REMOTE_MANIFEST_VERSION" ]]; then
    echo "Warning: Could not extract version from remote manifest zip."
    echo "  unzip/jq exit status: $UNZIP_JQ_STATUS"
    echo "  Content of manifest.json from zip (if any): $(unzip -p "$TEMP_REMOTE_ZIP" manifest.json 2>/dev/null | head -c 200) (truncated)"
  else
    echo "Remote Endpoint Security manifest version: $REMOTE_MANIFEST_VERSION"
  fi
fi
# Clean up temporary file
rm -f "$TEMP_REMOTE_ZIP"
echo "-----------------------------------------------"

# --- Get Local Manifest Version ---
echo "Step 2: Checking local Endpoint Security manifest version."
TEMP_LOCAL_CURL_ZIP="/tmp/local_curl_manifest_$$.zip" # Temporary file for curl from localhost
curl -s "$LOCAL_MIRROR_URL/downloads/endpoint/manifest/$MANIFEST_ZIP_FILENAME" -o "$TEMP_LOCAL_CURL_ZIP"
CURL_LOCAL_STATUS=$?

if [[ "$CURL_LOCAL_STATUS" -eq 0 && -s "$TEMP_LOCAL_CURL_ZIP" ]]; then # -s checks if file is not empty
  echo "  Successfully fetched content from local Nginx."
  LOCAL_MANIFEST_VERSION=$(unzip -p "$TEMP_LOCAL_CURL_ZIP" manifest.json 2>/dev/null | jq -r .manifest_version 2>/dev/null)
  if [[ -z "$LOCAL_MANIFEST_VERSION" ]]; then
    echo "  Warning: Could not extract version from content fetched via Nginx. Content might be invalid or not a zip."
    echo "  Content (truncated): $(cat "$TEMP_LOCAL_CURL_ZIP" | head -c 200)"
  fi
else
  echo "  Local manifest not yet served by Nginx or curl failed (status: $CURL_LOCAL_STATUS)."
  echo "  Attempting to read directly from local file: $LOCAL_MANIFEST_PATH."
  if [ -f "$LOCAL_MANIFEST_PATH" ]; then
    echo "  Reading directly from local file: $LOCAL_MANIFEST_PATH"
    LOCAL_MANIFEST_VERSION=$(sudo unzip -p "$LOCAL_MANIFEST_PATH" manifest.json 2>/dev/null | jq -r .manifest_version 2>/dev/null)
    if [[ -z "$LOCAL_MANIFEST_VERSION" ]]; then
      echo "  Warning: Could not extract version from local file directly. File might be invalid or not a zip."
      echo "  File content (truncated): $(sudo head -c 200 "$LOCAL_MANIFEST_PATH")"
    fi
  else
    echo "  Local manifest file not found at $LOCAL_MANIFEST_PATH."
  fi
fi
rm -f "$TEMP_LOCAL_CURL_ZIP" # Clean up temporary file

if [[ -z "$LOCAL_MANIFEST_VERSION" ]]; then
  echo "Local Endpoint Security manifest version: NOT FOUND"
else
  echo "Local Endpoint Security manifest version: $LOCAL_MANIFEST_VERSION"
fi
echo "-----------------------------------------------"

# --- Compare Versions and Determine Download ---
echo "Step 3: Comparing versions and determining if download is required."
DOWNLOAD_REQUIRED="false"
if [[ -z "$LOCAL_MANIFEST_VERSION" ]]; then
  echo "Decision: Local manifest not found. Download is required."
  DOWNLOAD_REQUIRED="true"
elif [[ -n "$REMOTE_MANIFEST_VERSION" && "$REMOTE_MANIFEST_VERSION" != "$LOCAL_MANIFEST_VERSION" ]]; then
  echo "Decision: Remote manifest version ($REMOTE_MANIFEST_VERSION) is different from local ($LOCAL_MANIFEST_VERSION). Download is required."
  DOWNLOAD_REQUIRED="true"
else
  echo "Decision: Local manifest is up-to-date ($LOCAL_MANIFEST_VERSION). Skipping download of manifest and granular artifacts."
fi
echo "DOWNLOAD_REQUIRED: $DOWNLOAD_REQUIRED"
echo "-----------------------------------------------"

# --- Perform Download if Required ---
if [[ "$DOWNLOAD_REQUIRED" == "true" ]]; then
  echo "Step 4: Performing download of latest Endpoint Security manifest."
  echo "Downloading latest Endpoint Security manifest from $REMOTE_MANIFEST_URL..."
  sudo wget -q "$REMOTE_MANIFEST_URL" -O "$LOCAL_MANIFEST_PATH"
  WGET_STATUS=$?
  if [[ "$WGET_STATUS" -ne 0 ]]; then
    echo "Error: Could not download Endpoint Security manifest from remote (wget status: $WGET_STATUS)."
    echo "Granular artifacts will not be downloaded."
  else
    echo "Manifest downloaded successfully to $LOCAL_MANIFEST_PATH."
    echo "Verifying newly downloaded manifest version:"
    NEWLY_DOWNLOADED_VERSION=$(sudo unzip -p "$LOCAL_MANIFEST_PATH" manifest.json 2>/dev/null | jq -r .manifest_version 2>/dev/null)
    if [[ -z "$NEWLY_DOWNLOADED_VERSION" ]]; then
      echo "Warning: Could not read version from newly downloaded manifest. File might be corrupt."
    else
      echo "Newly downloaded manifest version: $NEWLY_DOWNLOADED_VERSION"
    fi

    echo "Extracting and downloading granular artifacts..."
    # Using a subshell for cd to avoid affecting the main script's working directory
    ( cd "$MANIFEST_DIR" && \
      sudo unzip -p "$MANIFEST_ZIP_FILENAME" manifest.json | \
      sudo jq -r '.artifacts | to_entries[] | .value.relative_url' | \
      sudo xargs -I@ -P4 curl -s "https://artifacts.security.elastic.co@" --create-dirs -o "$ENDPOINT_DOWNLOAD_PATH/@" )
    XARGS_STATUS=$?
    if [[ "$XARGS_STATUS" -ne 0 ]]; then
        echo "Error: Failed to download some granular artifacts (xargs exit status: $XARGS_STATUS)."
    else
        echo "Finished downloading granular Endpoint Security artifacts to $ENDPOINT_DOWNLOAD_PATH"
    fi
  fi
else
  echo "No download of manifest or granular artifacts performed as local is up-to-date."
fi
echo "-----------------------------------------------"
echo "Debug script completed."

# --- Add Nginx Configuration ---
echo "Adding Nginx configuration..."

NGINX_CONF_DIR="/etc/nginx/conf.d"
ELASTIC_ASSETS_CONF_FILE="$NGINX_CONF_DIR/elastic_assets.conf"
NGINX_MAIN_CONF="/etc/nginx/nginx.conf"

ELASTIC_ASSETS_CONFIG="# set compatible etag format
map \$sent_http_etag \$elastic_etag {
  \"~(.*)-(.*)\" \"\$1\$2\";
}
server {
  listen 80 reuseport;
  server_name _ default_server; # You might want to adjust this
  root $DOWNLPATH; # Root should be /usr/share/nginx/html to serve /downloads/

  location /downloads/ { # This location block is crucial for general downloads
    alias $DOWNLPATH/downloads/; # Maps /downloads/ URL to the actual directory
    try_files \$uri \$uri/ =404;
    add_header ETag \"\$elastic_etag\";
  }

  # Add a location block for the Fleet Server PGP key
  # The Fleet Server endpoint is GET /api/agents/upgrades/{{major}}.{minor}.{{patch}}/pgp-public-key
  # This mapping serves default.pgp for any version request matching the pattern
  location ~ ^/api/agents/upgrades/([0-9]+\.[0-9]+\.[0-9]+)/pgp-public-key {
    alias $DOWNLPATH/downloads/fleet-server/elastic-agent-upgrade-keys/default.pgp;
    add_header Content-Type text/plain; # Or application/pgp-keys
    add_header ETag \"\$elastic_etag\";
  }

  # Optional: Default Nginx root for other requests if needed
  location / {
    # This can be your default Nginx welcome page or another application
    root /usr/share/nginx/html;
    index index.html index.htm;
    try_files \$uri \$uri/ =404;
  }
  
  # favicon.ico
  location = /favicon.ico {
    log_not_found off;
  }

  # robots.txt
  location = /robots.txt {
    log_not_found off;
  }

}"

# Check if the conf.d directory exists
if [[ -d "$NGINX_CONF_DIR" ]]; then
  echo "Creating Nginx configuration file: $ELASTIC_ASSETS_CONF_FILE"
  echo "$ELASTIC_ASSETS_CONFIG" | sudo tee "$ELASTIC_ASSETS_CONF_FILE" > /dev/null
else
  echo "Warning: Directory $NGINX_CONF_DIR not found. Appending to main Nginx configuration: $NGINX_MAIN_CONF"
  echo "$ELASTIC_ASSETS_CONFIG" | sudo tee -a "$NGINX_MAIN_CONF" > /dev/null
fi

# Set permissions for the Nginx configuration file
if [ -f "$ELASTIC_ASSETS_CONF_FILE" ]; then
    sudo chmod 644 "$ELASTIC_ASSETS_CONF_FILE"
    echo "Permissions set for $ELASTIC_ASSETS_CONF_FILE."
else
    echo "Warning: Nginx configuration file not found at $ELASTIC_ASSETS_CONF_FILE. Skipping permission set."
fi


# Test Nginx configuration before reloading
echo "Testing Nginx configuration..."
sudo nginx -t
if [[ "$?" -ne 0 ]]; then
  echo "Error: Nginx configuration test failed. Please check the syntax above. Exiting."
  exit 1
fi
echo "Nginx configuration test successful."

# Reload Nginx to apply changes
if command -v systemctl &> /dev/null; then
  sudo systemctl reload nginx
  if [[ "$?" -eq 0 ]]; then
    echo "Nginx reloaded successfully."
  else
    echo "Error reloading Nginx."
  fi
elif command -v service &> /dev/null; then
  sudo service nginx reload
  if [[ "$?" -eq 0 ]]; then
    echo "Nginx reloaded successfully."
  else
    echo "Error reloading Nginx."
  fi
else
  echo "Warning: Could not find systemctl or service command to reload Nginx. Please reload Nginx manually."
fi

echo ""
echo "-------------------------------------------------------------------------"
echo "Air-Gapped Installation Considerations:"
echo "-------------------------------------------------------------------------"
echo "1. Configure Elastic Agents to point to this internal artifact server."
echo "   For binaries, set Fleet's 'Agent binary download URL' in Kibana to:"
echo "   http://your_nginx_ip/downloads/beats/elastic-agent/"
echo "   For PGP key verification during upgrades from Fleet Server, ensure your Fleet Server"
echo "   is configured to check its local 'elastic-agent-upgrade-keys/default.pgp' file."
echo "   (This script handles placing 'default.pgp' correctly)."
echo "2. Configure Fleet Server to be aware of this internal artifact server."
echo "   If using server.pgp.upstream_url in Fleet Server config, ensure it's empty or points"
echo "   to your internal Nginx if you choose to serve it this way, otherwise"
echo "   the locally placed 'default.pgp' will be used."
echo "3. In Fleet, configure Endpoint Security policies to use the offline"
echo "   package hosted on this internal server for updates. This might involve"
echo "   setting appropriate URLs in Kibana's Fleet settings."
echo "4. Install the downloaded Kibana plugin for Endpoint Security in your Kibana instance."
echo "-------------------------------------------------------------------------"
echo "Artifacts will be served from: $DOWNLPATH/downloads"
echo "You can access them via your Nginx server (e.g., http://your_nginx_ip/downloads/)."
echo "The Endpoint Security manifest is served via: $LOCAL_MIRROR_URL/downloads/endpoint/manifest/artifacts-$ENDPOINT_VERSION.zip"
echo "The Fleet Server PGP key is served via: $LOCAL_MIRROR_URL/api/agents/upgrades/X.x.x/pgp-public-key"
echo "-------------------------------------------------------------------------"

# --- Final Verification Summary ---
echo ""
echo "-------------------------------------------------------------------------"
echo "Final Verification Summary:"
echo "-------------------------------------------------------------------------"
echo "Binary Version (VERSI): $VERSI"
echo "Remote Endpoint Security Manifest Version: ${REMOTE_MANIFEST_VERSION:-'N/A (check logs above)'}"
echo "Local Endpoint Security Manifest Version: ${LOCAL_MANIFEST_VERSION:-'N/A (check logs above)'}"
echo "-------------------------------------------------------------------------"

exit

Script to download files for airgap. Make sure to change version in the script

https://gist.github.com/linuxmalaysia/2d356c1548f1cd6fa5a49eed87ba1cd9