Parsing iRODS log files with logstash

irods_logstash.conf

input {
  file {
    type => "rodslog"
    start_position => beginning
    debug => true
    sincedb_path => "/root/logstash/.sincedb"
    path => [ "/root/logstash/logs/eu/var/log/irods/rodsLog.*" ]
  }
}

filter {
  # Drop verbose and non-informative events
  if [message] =~ /environment variable set/ {drop { }}
  if [message] =~ /Authenticated/ {drop { }}

  grok {
        match => { "message" => "%{SYSLOGTIMESTAMP:log_timestamp}\spid:%{POSINT:pid}\s%{LOGLEVEL:loglevel}:\s+%{GREEDYDATA:rods_msg}" }
        add_field => [ "event_timestamp", "%{@timestamp}" ]
  }

  if [loglevel] == "DEBUG" {drop { }}

  # Get client IPs
  if [rods_msg] =~ /^Agent/ {
          grok {
                match => { "rods_msg" => "Agent\sprocess\s%{POSINT:agent_pid}\sstarted\sfor\spuser=%{USER:puser}\sand\scuser=%{USER:cuser}\sfrom\s%{IPV4:irods_client_addr}" }
                remove_field => "rods_msg"
          }
  }

  # Year is not present in the log date format, getting it from the filename
  grok {
        match => { "path" => "%{YEAR:log_year}" }
  }

  mutate {
     replace => [ "log_timestamp", "%{log_timestamp} %{log_year}" ]
     replace => [ "host", "ids-eu.incf.net" ]
     remove_field => "log_year"
  }

  date { match => [ "log_timestamp", "MMM dd HH:mm:ss yyyy", "MMM  d HH:mm:ss yyyy" ] }

  # GeoIP
  if [irods_client_addr] {
          geoip {
            source => "irods_client_addr"
            target => "geoip"
            add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
            add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
            add_tag => ["geoip"]
          }

          mutate {
            convert => [ "[geoip][coordinates]", "float" ]
          }
   }
}

output {
  stdout { codec => rubydebug }
  elasticsearch {
        host => "127.0.0.1"
 	      bind_host => "localhost"
        # Month-based indexing, daily is too expensive
        index => "logstash-%{+YYYY.MM}"
        workers => 4
  }
}

Launching with:

rm -rf /var/lib/elasticsearch/elasticsearch/ && service elasticsearch restart &&
rm -f .sincedb && java -jar logstash-*-flatjar.jar agent -f irods_simpler.conf