Load elasticsearch errors into airbrake

logstashToAirbrake.sh

#!/bin/bash

echo "Make sure queryData.json is also downloaded"

read -p "Elasticsearch username: "  esUser
read -p "Elasticsearch password: "  esPass
read -p "Elasticsearch domain: "  esAddress
read -p "Airbrake project key: "  airbrakeProjectKey
read -p "Airbrake project id: "  airbrakeProjectId
airbrakeHostname=https://airbrake.io

# Make sure queryData.json exists:
if test ! -e "queryData.json"; then
  echo "Please create a queryData.json file to indicate what data should be sent to airbrake"
  exit
fi

# Make sure jq is installed:
if jq --version > /dev/null 2>&1; then
  echo "Great, jq is installed"
else
  echo "Please install jq to use this script; on mac you can run 'brew install jq', see https://github.com/stedolan/jq/wiki/Installation for more details"
  exit
fi

# Set the date range to collect errors to the last 24 hours.
# GNU and BSD date binaries are different, so we check for this.
if date -v 1d > /dev/null 2>&1; then
  startDate=$(date -v -3d '+%Y.%m.%d')
  endDate=$(date +%Y.%m.%d)
else
  startDate=$(date --date="3 days ago" +%Y.%m.%d)
  endDate=$(date +%Y.%m.%d)
fi

# create working dir
rm -rf ./errorMessages
mkdir errorMessages

# get error messages from elasticsearch
echo "Loading error logs from elasticsearch"
curl -s -S --user $esUser:$esPass -XGET "https://$esAddress/__es/logstash-$endDate,logstash-$startDate/_search?pretty" -d @./queryData2.json > ./errorMessages/raw-dump.json \
|| (echo "Could not load error logs from elasticsearch" && exit 1)

if grep -q "401 Unauthorized" "./errorMessages/raw-dump.json"; then
  echo "Invalid user/password for elasticsearch"
  exit 1
fi

numHits=$(cat ./errorMessages/raw-dump.json | jq '.hits.total')
if [ $numHits -eq 0 ]; then
  echo "No errors found in elasticsearch"
  exit 1
fi

# strip out the error messages
cat ./errorMessages/raw-dump.json | jq '.hits.hits[]._source.message' -c > ./errorMessages/dump.json

cd ./errorMessages

# split the error messages into files
split -l 1 -a 10 ./dump.json

echo "Now sending errors to airbrake, this process is slowed to 1 error/sec to stay under rate limits"

# iterate over all error messages
i=1
for msgFile in x*
do
  # remove enclosing quotes
  cutMsg=$(cat ./$msgFile | cut -c2- | rev | cut -c2- | rev)

  # format msg as notification for airbrake
  jq -n --arg cutMsg "${cutMsg}" '{ "errors":[{"message": $cutMsg}], "session": {}, "environment": {}, "context":{ "notifier": { "name": "elsticsearch uploader", "version": "v1"}}}' > ./$msgFile-notice.json

  # send error to airbrake
  curl -s -S -X POST -H "Content-Type: application/json" -d @$msgFile-notice.json "$airbrakeHostname/api/v3/projects/$airbrakeProjectId/notices?key=$airbrakeProjectKey" > /dev/null \
  || (echo "Could not send error logs to Airbrake" && exit 1)

  if [ `echo "$i % 10" | bc` -eq 0 ]
  then
    echo "Sent $i errors to airbrake"
  fi

  ((i = i + 1))

  # wait for a second so rate limits do not get hit
  sleep 1
done

echo "Done!"