mailer-ses.rb extension for sensu to avoid fork bomb

etc_sensu_conf.d_extensions_mailer-ses.json

{
  "handlers": {
    "mailer": {
      "command": null,
      "type": "extension",
      "severities": [
        "ok",
        "warning",
        "critical",
        "unknown"
      ]
    }
  },
  "mailer": {
    "admin_gui": "http://monitoring.client.io/",
    "mail_from": "[email protected]",
    "aws_access_key": "XXXXXXXXXXXXXXX",
    "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXX",
    "aws_ses_endpoint": "email.eu-west-1.amazonaws.com",
    "contacts": [
      {
        "name": "Client",
        "mail_to": "[email protected]",
        "severities": [
          "ok",
          "warning",
          "unknown",
          "critical"
        ]
      },
      {
        "name": "Myself",
        "mail_to": "[email protected]",
        "severities": [
          "ok",
          "warning",
          "unknown",
          "critical"
        ]
      }
    ]
  }
}

etc_sensu_extensions_mailer-ses.rb

#!/opt/sensu/embedded/bin/ruby
#
# Sensu Extension: mailer-ec2
#
# This mailer extension moved to an extension so the sensu doesn't fork bomb itself if you have a lot
# of alerts
#
# Author Julien Syx @enovance
#
# Released under the same terms as Sensu (the MIT license); see LICENSE
# for details.

require 'rubygems' if RUBY_VERSION < '1.9.0'
require 'aws/ses'
require 'timeout'

module Sensu::Extension

  class Mailer < Handler

    def post_init
    end

    def definition
      {
        type: 'extension',
        name: 'mailer-ses',
        mutator: 'ruby_hash'
      }
    end

    def name
      definition[:name]
    end

    def description
      'A Mailer handler that will not fork'
    end

    def build_playbook(check)
      # If the playbook attribute exists and is a URL, "[<a href='url'>playbook</a>]" will be output.
      # To control the link name, set the playbook value to the HTML output you would like.
      if check[:playbook]
        begin
          uri = URI.parse(check[:playbook])
          if %w( http https ).include?(uri.scheme)
            "[#{build_link(check[:playbook], "Playbook")}]"
          else
            "Playbook:  #{check[:playbook]}"
          end
        rescue
          "Playbook:  #{check[:playbook]}"
        end
      else
        nil
      end
    end

    # convert the status to a string
    def status_to_string(check)
      case check[:status]
      when 0
        'OK'
      when 1
        'WARNING'
      when 2
        'CRITICAL'
      else
        'UNKNOWN'
      end
    end

    # Log something and return false.
    def bail(msg, event)
      @logger.info("Mailer handler: #{msg}: #{event[:client][:name]}/#{event[:check][:name]}")
      false
    end

    def stash_exists?(path)
      api_request(:GET, '/stash' + path).code == '200'
    end

    def event_exists?(client, check)
      api_request(:GET, '/event/' + client + '/' + check).code == '200'
    end

    # Lifted from the sensu-plugin gem, makes an api request to sensu
    def api_request(method, path, &blk)
      http = Net::HTTP.new(@settings['api']['host'], @settings['api']['port'])
      req = net_http_req_class(method).new(path)
      if @settings['api']['user'] && @settings['api']['password']
        req.basic_auth(@settings['api']['user'], @settings['api']['password'])
      end
      yield(req) if block_given?
      http.request(req)
    end

    # also lifted from the sensu-plugin gem. In fact, most of the rest was.
    def net_http_req_class(method)
      case method.to_s.upcase
      when 'GET'
        Net::HTTP::Get
      when 'POST'
        Net::HTTP::Post
      when 'DELETE'
        Net::HTTP::Delete
      when 'PUT'
        Net::HTTP::Put
      end
    end

    # Has this check been disabled from handlers?
    def filter_disabled(event)
      if event[:check].has_key?(:alert)
        if event[:check][:alert] == false
          bail 'alert disabled', event
        end
      end

      true
    end

    # Don't spam too much!
    def filter_repeated(event)
      defaults = {
        'occurrences' => 1,
        'interval'    => 60,
        'refresh'     => 1800
      }

      occurrences = event[:check][:occurrences] || defaults['occurrences']
      interval    = event[:check][:interval]    || defaults['interval']
      refresh     = event[:check][:refresh]     || defaults['refresh']

      return bail 'not enough occurrences', event if event[:occurrences] < occurrences

      if event[:occurrences] > occurrences && event[:action] == :create
        number = refresh.fdiv(interval).to_i
        unless number == 0 || event[:occurrences] % number == 0
          return bail 'only handling every ' + number.to_s + ' occurrences', event
        end
      end

      true
    end

    # Has the event been silenced through the API?
    def filter_silenced(event)
      stashes = [
        ['client', '/silence/' + event[:client][:name]],
        ['check', '/silence/' + event[:client][:name] + '/' + event[:check][:name]],
        ['check', '/silence/all/' + event[:check][:name]]
      ]
      stashes.each do |(scope, path)|
        begin
          timeout(2) do
            if stash_exists?(path)
              return bail scope + ' alerts silenced', event
            end
          end
        rescue Timeout::Error
          @logger.warn('timed out while attempting to query the sensu api for a stash')
        end
      end

      true
    end

    # Does this event have dependencies?
    def filter_dependencies(event)
      if event[:check].has_key?(:dependencies) && event[:check][:dependencies].is_a?(Array)
          event[:check][:dependencies].each do |dependency|
            begin
              timeout(2) do
                check, client = dependency.split('/').reverse
                if event_exists?(client || event[:client][:name], check)
                  return bail 'check dependency event exists', event
                end
              end
            rescue Timeout::Error
              @logger.warn('timed out while attempting to query the sensu api for an event')
            end
          end
      end

      true
    end

    # Has this check timedout?
    def filter_timeout(event)
      if event[:check][:output].include? 'Plugin timed out after'
        bail 'check timed out', event
      end

      true
    end


    # Run all the filters in some order. Only run the handler if they all return true
    def filters(event_data)
      return false unless filter_repeated(event_data)
      return false unless filter_silenced(event_data)
      return false unless filter_dependencies(event_data)
      return false unless filter_disabled(event_data)
      return false unless filter_timeout(event_data)

      @logger.info("#{event_data[:client][:name]}/#{event_data[:check][:name]} not being filtered!")

      true
    end


    def send_mail(event)
      check = event[:check]
      client = event[:client]

      admin_gui = @settings["mailer-ses"]["gui"] || 'https://monitoring.myfox.io'
      mail_from =  @settings["mailer-ses"]["mail_from"]
      contacts =  @settings["mailer-ses"]["contacts"]
      short_name = client[:name] + '/' + check[:name]
      action_to_string = event[:action].eql?(:resolve) ? "RESOLVED" : "ALERT"
      status = status_to_string(check)
      playbook = build_playbook(check)

      body = <<-BODY.gsub(/^\s+/, '')
              #{check[:output]}
              Admin GUI: #{admin_gui}/#/client/api/#{client[:name]}
              Host: #{client[:name]}
              Timestamp: #{Time.at(check[:issued])}
              Address:  #{client[:address]}
              Check Name:  #{check[:name]}
              Command:  #{check[:command]}
              Status:  #{status}
              Occurrences:  #{event[:occurrences]}
              #{playbook}
            BODY
      subject = "#{action_to_string} - #{short_name}: #{status}"

      ses = AWS::SES::Base.new(
        :access_key_id     => @settings["mailer-ses"]["aws_access_key"],
        :secret_access_key => @settings["mailer-ses"]["aws_secret_key"],
        :server            => @settings["mailer-ses"]["aws_ses_endpoint"]
      )

      begin
        contacts.each do |(contact)|
          if contact.has_key?(:severities)
            severity = Sensu::SEVERITIES[event[:check][:status]] || 'unknown'
            if contact[:severities].include?(severity)
              @logger.info("================Sending a mail to #{contact[:mail_to]} from #{settings["mailer-ses"]["mail_from"]}=============")
              timeout 10 do
                ses.send_email(
                  :to        => contact[:mail_to],
                  :source    => @settings["mailer-ses"]["mail_from"],
                  :subject   => subject,
                  :text_body => body
                )
                @logger.info('mail -- sent alert for ' + short_name + ' to ' + contact[:mail_to].to_s)
              end
            end
          end
        end
      rescue Timeout::Error
        return 'mail -- timed out while attempting to ' + event[:action] + ' an incident -- ' + short_name
      end
    end

    def run(event_data)
      event = event_data
      # Is this event a resolution?
      resolved = event[:action].eql?(:resolve)

      if (resolved || [1, 2, 3].include?(event[:check][:status])) && filters(event)
        operation = proc { send_mail(event) }
        callback = proc { |result| yield "Mail message: #{result}", 0 }

        EM.defer(operation, callback)
      else
        yield("Mailer not handling", 0)
      end
    end

    # Called when Sensu begins to shutdown.
    def stop
      true
    end
  end