Bash script to monitor health of Airflow scheduler container deployed with docker compose, restarting it if necessary

airflow_health_monitor.sh

#!/bin/bash

# Requires the sendemail package being installed on the host machine

# Get container health status
healthy=$(docker inspect -f '{{.State.Health.Status}}' airflow-airflow-scheduler-1)

# Check if healthy
if [[ $healthy != "healthy" ]]; then
  # attempt to restart Airflow - the scheduler will stop if the Azure Postgres DB becomes unavailable due to maintenance
  cd ~/airflow  
  docker compose -f docker-compose.yaml --env-file .env down
  docker compose -f docker-compose.yaml --env-file .env up -d
  sleep 300
  # Check if healthy
  healthy=$(docker inspect -f '{{.State.Health.Status}}' airflow-airflow-scheduler-1)
  if [[ $healthy != "healthy" ]]; then
    # Send email notification
    SUBJ="Airflow scheduler container is unhealthy"
    MESSAGE="Please investigate as Airflow jobs may not be running.  Sam's health monitor bash script attempted a docker compose down & up and the container remains unhealthy."
  else
    SUBJ="Airflow scheduler container went unhealthy and was restarted successfully"
    MESSAGE="This happens when the Azure Postgres DB becomes unavailable, either due to maintenance (15 minutes no more often than monthly) or some other issue like max connections.  Airflow jobs should be running again, but confirm in the web UI and consider checking relevant logs in the scheduler container and the Azure Portal for the Flexible PostgreSQL server resource."
  fi
  SERVER="my.smtp.server"
  FROM="[email protected]"
  TO="[email protected]"
  CHARSET="utf-8"
  sendemail -f $FROM -t $TO -u $SUBJ -s $SERVER -m $MESSAGE -v -o message-charset=$CHARSET
  sleep 21600
fi

#!/bin/bash # Requires the sendemail package being installed on the host machine # Get container health status healthy=$(docker inspect -f '{{.State.Health.Status}}' airflow-airflow-scheduler-1) # Check if healthy if [[ $healthy != "healthy" ]]; then # attempt to restart Airflow - the scheduler will stop if the Azure Postgres DB becomes unavailable due to maintenance cd ~/airflow docker compose -f docker-compose.yaml --env-file .env down docker compose -f docker-compose.yaml --env-file .env up -d sleep 300 # Check if healthy healthy=$(docker inspect -f '{{.State.Health.Status}}' airflow-airflow-scheduler-1) if [[ $healthy != "healthy" ]]; then # Send email notification SUBJ="Airflow scheduler container is unhealthy" MESSAGE="Please investigate as Airflow jobs may not be running. Sam's health monitor bash script attempted a docker compose down & up and the container remains unhealthy." else SUBJ="Airflow scheduler container went unhealthy and was restarted successfully" MESSAGE="This happens when the Azure Postgres DB becomes unavailable, either due to maintenance (15 minutes no more often than monthly) or some other issue like max connections. Airflow jobs should be running again, but confirm in the web UI and consider checking relevant logs in the scheduler container and the Azure Portal for the Flexible PostgreSQL server resource." fi SERVER="my.smtp.server" FROM="[email protected]" TO="[email protected]" CHARSET="utf-8" sendemail -f $FROM -t $TO -u $SUBJ -s $SERVER -m $MESSAGE -v -o message-charset=$CHARSET sleep 21600 fi