Script to publish JMX metrics from Kafka broker to AWS Cloudwatch

publish-kafka-metrics.sh

#!/usr/bin/env bash

# Kafka utilises JMX to provide an access to some useful data
# This script uses command line JMX client to retrieve those metrics from a Kafka broker
# and publish them to AWS Cloudwatch

# JMX Client: http://crawler.archive.org/cmdline-jmxclient/downloads.html
# Runtime dependencies: Java, AWS CLI, IAM role attached to the instance with permissions
# to put Cloudwatch metrics

# Input parameters:
# - Name of your cluster that will be used as a Cloudwatch dimension for incoming metrics
# - IP address of the broker (used only if we can't retrieve EC2 instance's local IP)

# Example: ./publish-kafka-metrics.sh UAT 0.0.0.0

CLUSTER_NAME=$1
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
CLIENT=$DIR/cmdline-jmxclient-0.10.3.jar
KAFKA_JMX_PORT=7203
REGION="ap-southeast-2"
KAFKA_IP=`curl -s http://169.254.169.254/latest/meta-data/local-ipv4`
if [[ -z "${KAFKA_IP// }" ]] ; then
  KAFKA_IP=$2
fi

# get_metric <jmx-path> <jmx-property>
function get_metric {
    VALUE="$(java -jar $CLIENT - $KAFKA_IP:$KAFKA_JMX_PORT $1 $2 2>&1 | cut -d ' ' -f6)"
    echo $VALUE
}

function push_data {
    aws cloudwatch put-metric-data \
        --metric-name "$1" \
        --namespace "Kafka/JMX" \
        --unit "$2" \
        --value "$3" \
        --dimensions "ClusterName=$CLUSTER_NAME" \
        --region $REGION
}

# Here is just a list of metrics that can be useful
# Run cmdline-jmxclient (or any other JMX client) to inspect all the available metrics

UnderReplicatedPartitions=$(get_metric "kafka.server:name=UnderReplicatedPartitions,type=ReplicaManager" "Value")
ResponseQueueTimeMs=$(get_metric "kafka.network:name=ResponseQueueTimeMs,request=Fetch,type=RequestMetrics" "99thPercentile")
LeaderElectionRate=$(get_metric "kafka.controller:name=LeaderElectionRateAndTimeMs,type=ControllerStats" "FiveMinuteRate")
PartitionCount=$(get_metric "kafka.server:name=PartitionCount,type=ReplicaManager" "Value")
FailedProduceRequestsPerSec=$(get_metric "kafka.server:name=FailedProduceRequestsPerSec,type=BrokerTopicMetrics" "FiveMinuteRate")
TotalFetchRequestsPerSec=$(get_metric "kafka.server:name=TotalFetchRequestsPerSec,type=BrokerTopicMetrics" "FiveMinuteRate")
MessagesInPerSec=$(get_metric "kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics" "FiveMinuteRate")

push_data "UnderReplicatedPartitions" "Count" $UnderReplicatedPartitions
push_data "ResponseQueueTimeMs" "Count" $ResponseQueueTimeMs
push_data "LeaderElectionRate" "Count" $LeaderElectionRate
push_data "PartitionCount" "Count" $PartitionCount
push_data "FailedProduceRequestsPerSec" "Count" $FailedProduceRequestsPerSec
push_data "TotalFetchRequestsPerSec" "Count" $TotalFetchRequestsPerSec
push_data "MessagesInPerSec" "Count" $MessagesInPerSec