Elasticsearch Logstash Kibana Asterisk

asterisk-grok-patterns

# Inspired by https://github.com/gbirke/grok-asterisk
# See too https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns
# Save this file in /usr/share/logstash/patterns/asterisk-grok-patterns

ASTLEVEL (?:VERBOSE|ERROR|NOTICE|INFO|DEBUG|DTMF|WARNING)
ASTPID [0-9]+
ASTCHANNEL_ID C\-[A-Za-z0-9]+
ASTSRC [-a-z._0-9\/]+

filebeat.yml

# You can find the full configuration reference here:
# https://www.elastic.co/guide/en/beats/filebeat/index.html
# Save this file in /etc/filebeat/filebeat.yml

#----------------------------- Logstash prospectors ---------------------------
filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/asterisk/cdr-csv/Master.csv
  document_type: asterisk-cdr
  encoding: "utf-8"

- input_type: log
  paths:
    - /var/log/asterisk/full
  document_type: asterisk-log
  encoding: "utf-8"

  env: production

#----------------------------- Logstash output --------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["127.0.0.1:5044"]

#================================ Logging =====================================

# Sets log level. The default log level is info.
# Available log levels are: critical, error, warning, info, debug
#logging.level: debug
logging.level: info

# At debug level, you can selectively enable logging only for some components.
# To enable all selectors use ["*"]. Examples of other selectors are "beat",
# "publish", "service".
logging.selectors: ["*"]

logstash.conf

# You can find the full configuration reference here:
# https://www.elastic.co/guide/en/logstash/current/pipeline.html
# You need install some plugins
# -> logstash-plugin install logstash-input-beats
# -> logstash-plugin install logstash-filter-csv
# -> logstash-plugin install logstash-output-elasticsearch
# -> logstash-plugin install logstash-codec-rubydebug
# Save this file in /etc/logstash/conf.d/logstash.conf

input {
  beats {
    port => 5044
  }
}

filter {
  if [type] == "asterisk-log"  {
    grok {
      patterns_dir => [ "/usr/share/logstash/patterns" ]
      match => { "message" => "\[(%{TIMESTAMP_ISO8601:datetime}|%{SYSLOGTIMESTAMP:datetime})\]\s*%{ASTLEVEL:asterisk_severity}\s*\[%{ASTPID:asterisk_pid}\](\[%{ASTCHANNEL_ID:asterisk_channel}\])?\s*%{ASTSRC:asterisk_source}:\s*%{GREEDYDATA:asterisk_data}" }
    }
  }
  if [type] == "asterisk-cdr" {
    csv {
      separator => ","
      # http://www.asteriskdocs.org/en/3rd_Edition/asterisk-book-html-chunk/asterisk-SysAdmin-SECT-1.html
      # Asterisk fieds
      columns => ["accountcode","src","dst","dcontext","clid","channel","dstchannel","lastapp","lastdata","start","answer","end","duration","billsec","disposition","amaflags","userfield","uniqueid"]
      convert => {
      #  "accountcode" => "string"
      #  "src" => "string"
      #  "dst" => "string"
      #  "dcontext" => "string"
      #  "clid" => "string"
      #  "channel" => "string"
      #  "dstchannel" => "string"
      #  "lastapp" => "string"
      #  "lastdata" => "string"
         "start" => "date_time"
         "answer" => "date_time"
         "end" => "date_time"
         "duration" => "integer"
         "billsec" => "integer"
      #  "disposition" => "string"
      #  "amaflags" => "string"
      #  "userfield" => "string"
      #  "uniqueid" => "string"
      }
  } }
}

output {
  elasticsearch {
    hosts => "localhost:9200"
    user => elastic
    password => changeme
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
  # for debug purpose of pipeline with command: ./logstash -f /etc/logstash/conf.d/logstash.conf
  # stdout { codec => rubydebug  }
}

requirements.txt

# I'm using the following softwares:

Ubuntu Server 16.04
  Kibana 5.5.1
  Elastic Search 5.5.1
  Logstash 5.5.1

CentOS 7.2
  Asterisk 13.9
  Filebeat 5.5.1