ELK Filebeat for Docker

2018-03-06.log

{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"Establishing connection with Rethinkdb\" \n","stream":"stderr","time":"2017-09-22T17:50:58.428872932Z"}
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"writing out configs and watching for changes\" \n","stream":"stderr","time":"2017-09-22T17:50:58.48144991Z"}
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"watching configs\" \n","stream":"stderr","time":"2017-09-22T17:50:58.481480667Z"}
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_ca/cert.pem\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"21f500d9-8562-48d6-98db-65b3792665eb\\\"}\" \n","stream":"stderr","time":"2017-09-22T17:50:58.481520965Z"}
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_server/cert.pem/a2382528a00b\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"bb175fae-12bf-4f5d-8e86-679bbab6a1d4\\\"}\" \n","stream":"stderr","time":"2017-09-22T17:50:58.483007834Z"}
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_server/key.pem/a2382528a00b\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"3ca2965e-2251-40b6-b16e-28ee4aa23d6d\\\"}\" \n","stream":"stderr","time":"2017-09-22T17:50:58.484391451Z"}
{"log":"The files belonging to this database system will be owned by user \"postgres\".\n","stream":"stdout","time":"2017-09-22T17:50:58.510394403Z"}
{"log":"This user must also own the server process.\n","stream":"stdout","time":"2017-09-22T17:50:58.510438517Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:58.510442818Z"}
{"log":"The database cluster will be initialized with locale \"en_US.utf8\".\n","stream":"stdout","time":"2017-09-22T17:50:58.5107831Z"}
{"log":"The default database encoding has accordingly been set to \"UTF8\".\n","stream":"stdout","time":"2017-09-22T17:50:58.510792947Z"}
{"log":"The default text search configuration will be set to \"english\".\n","stream":"stdout","time":"2017-09-22T17:50:58.510796053Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:58.510798597Z"}
{"log":"Data page checksums are disabled.\n","stream":"stdout","time":"2017-09-22T17:50:58.510800711Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:58.510803129Z"}
{"log":"fixing permissions on existing directory /postgres-data ... ok\n","stream":"stdout","time":"2017-09-22T17:50:58.510805321Z"}
{"log":"creating subdirectories ... ok\n","stream":"stdout","time":"2017-09-22T17:50:58.511153395Z"}
{"log":"selecting default max_connections ... 100\n","stream":"stdout","time":"2017-09-22T17:50:58.517570064Z"}
{"log":"selecting default shared_buffers ... 128MB\n","stream":"stdout","time":"2017-09-22T17:50:58.55041664Z"}
{"log":"selecting dynamic shared memory implementation ... posix\n","stream":"stdout","time":"2017-09-22T17:50:58.550432984Z"}
{"log":"creating configuration files ... ok\n","stream":"stdout","time":"2017-09-22T17:50:58.643305227Z"}
{"log":"sh: locale: not found\n","stream":"stderr","time":"2017-09-22T17:50:59.385190916Z"}
{"log":"running bootstrap script ... ok\n","stream":"stdout","time":"2017-09-22T17:50:59.385260527Z"}
{"log":"performing post-bootstrap initialization ... No usable system locales were found.\n","stream":"stdout","time":"2017-09-22T17:50:59.385269302Z"}
{"log":"Use the option \"--debug\" to see details.\n","stream":"stdout","time":"2017-09-22T17:50:59.38527276Z"}
{"log":"ok\n","stream":"stdout","time":"2017-09-22T17:50:59.843973425Z"}
{"log":"\n","stream":"stderr","time":"2017-09-22T17:50:59.892636879Z"}
{"log":"WARNING: enabling \"trust\" authentication for local connections\n","stream":"stderr","time":"2017-09-22T17:50:59.892668026Z"}
{"log":"You can change this by editing pg_hba.conf or using the option -A, or\n","stream":"stderr","time":"2017-09-22T17:50:59.892673107Z"}
{"log":"--auth-local and --auth-host, the next time you run initdb.\n","stream":"stderr","time":"2017-09-22T17:50:59.892684617Z"}
{"log":"syncing data to disk ... ok\n","stream":"stdout","time":"2017-09-22T17:50:59.892729467Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:59.892744781Z"}
{"log":"Success.\n","stream":"stdout","time":"2017-09-22T17:50:59.892747929Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:59.892750783Z"}
{"log":"===============================\n","stream":"stdout","time":"2017-09-22T17:50:59.904870642Z"}
{"log":"!!! Use $POSTGRES_PASSWORD env var to secure your database !!!\n","stream":"stdout","time":"2017-09-22T17:50:59.904888953Z"}
{"log":"===============================\n","stream":"stdout","time":"2017-09-22T17:50:59.904892439Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:59.904895124Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:59.917770393Z"}
{"log":"PostgreSQL stand-alone backend 9.6.5\n","stream":"stdout","time":"2017-09-22T17:50:59.917796057Z"}
{"log":"backend\u003e statement: CREATE DATABASE fuzzomatic;\n","stream":"stdout","time":"2017-09-22T17:50:59.917799573Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:59.917802605Z"}
{"log":"backend\u003e \n","stream":"stdout","time":"2017-09-22T17:50:59.945551169Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:59.957150477Z"}
{"log":"PostgreSQL stand-alone backend 9.6.5\n","stream":"stdout","time":"2017-09-22T17:50:59.957165267Z"}
{"log":"backend\u003e statement: CREATE USER fuzzomatic WITH SUPERUSER ;\n","stream":"stdout","time":"2017-09-22T17:50:59.957167707Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:50:59.95716978Z"}
{"log":"backend\u003e \n","stream":"stdout","time":"2017-09-22T17:50:59.963082111Z"}
{"log":"waiting for server to start....LOG:  database system was shut down at 2017-09-22 17:50:59 UTC\n","stream":"stdout","time":"2017-09-22T17:50:59.981906599Z"}
{"log":"LOG:  MultiXact member wraparound protections are now enabled\n","stream":"stdout","time":"2017-09-22T17:50:59.982529141Z"}
{"log":"LOG:  database system is ready to accept connections\n","stream":"stdout","time":"2017-09-22T17:50:59.982980251Z"}
{"log":"LOG:  autovacuum launcher started\n","stream":"stdout","time":"2017-09-22T17:50:59.983339966Z"}
{"log":" done\n","stream":"stdout","time":"2017-09-22T17:51:01.014462845Z"}
{"log":"server started\n","stream":"stdout","time":"2017-09-22T17:51:01.014558061Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:51:01.015742478Z"}
{"log":"/start.sh: ignoring /docker-entrypoint-initdb.d/*\n","stream":"stdout","time":"2017-09-22T17:51:01.01584683Z"}
{"log":"\n","stream":"stdout","time":"2017-09-22T17:51:01.01586923Z"}
{"log":"waiting for server to shut down...LOG:  received fast shutdown request\n","stream":"stdout","time":"2017-09-22T17:51:01.032377144Z"}
{"log":"LOG:  aborting any active transactions\n","stream":"stdout","time":"2017-09-22T17:51:01.032420799Z"}
{"log":".LOG:  autovacuum launcher shutting down\n","stream":"stdout","time":"2017-09-22T17:51:01.033848686Z"}
{"log":"LOG:  shutting down\n","stream":"stdout","time":"2017-09-22T17:51:01.03671214Z"}
{"log":"LOG:  database system is shut down\n","stream":"stdout","time":"2017-09-22T17:51:01.053754061Z"}
{"log":" done\n","stream":"stdout","time":"2017-09-22T17:51:02.034062867Z"}
{"log":"server stopped\n","stream":"stdout","time":"2017-09-22T17:51:02.034095293Z"}
{"log":"LOG:  database system was shut down at 2017-09-22 17:51:01 UTC\n","stream":"stderr","time":"2017-09-22T17:51:02.065597773Z"}
{"log":"LOG:  MultiXact member wraparound protections are now enabled\n","stream":"stderr","time":"2017-09-22T17:51:02.066921167Z"}
{"log":"LOG:  database system is ready to accept connections\n","stream":"stderr","time":"2017-09-22T17:51:02.070247438Z"}
{"log":"LOG:  autovacuum launcher started\n","stream":"stderr","time":"2017-09-22T17:51:02.070288283Z"}
{"log":"time=\"2017-09-22T18:17:13Z\" level=info msg=\"writing out configs and watching for changes\" \n","stream":"stderr","time":"2017-09-22T18:17:13.778693392Z"}
{"log":"time=\"2017-09-22T18:17:13Z\" level=info msg=\"watching configs\" \n","stream":"stderr","time":"2017-09-22T18:17:13.778773427Z"}
{"log":"time=\"2017-09-22T18:17:13Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_ca/cert.pem\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"d7e7948c-d3e3-4c8f-8f05-e40455890b83\\\"}\" \n","stream":"stderr","time":"2017-09-22T18:17:13.778782108Z"}
{"log":"time=\"2017-09-22T18:17:13Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_server/cert.pem/a2382528a00b\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"d84d3998-83df-4c3c-a4c4-8a2f9ea3efbc\\\"}\" \n","stream":"stderr","time":"2017-09-22T18:17:13.781504612Z"}
{"log":"time=\"2017-09-22T18:17:13Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_server/key.pem/a2382528a00b\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"57d1ac91-6b44-41fb-9627-a33bf039c9a6\\\"}\" \n","stream":"stderr","time":"2017-09-22T18:17:13.784720537Z"}
{"log":"LOG:  database system was interrupted; last known up at 2017-09-22 18:16:19 UTC\n","stream":"stderr","time":"2017-09-22T18:17:13.950203916Z"}
{"log":"LOG:  database system was not properly shut down; automatic recovery in progress\n","stream":"stderr","time":"2017-09-22T18:17:13.960081319Z"}
{"log":"LOG:  redo starts at 0/1A1FA48\n","stream":"stderr","time":"2017-09-22T18:17:13.961865555Z"}
{"log":"LOG:  invalid magic number 0000 in log segment 000000010000000000000001, offset 12468224\n","stream":"stderr","time":"2017-09-22T18:17:13.99033137Z"}
{"log":"LOG:  redo done at 0/1BE3F90\n","stream":"stderr","time":"2017-09-22T18:17:13.990365732Z"}
{"log":"LOG:  MultiXact member wraparound protections are now enabled\n","stream":"stderr","time":"2017-09-22T18:17:14.001794335Z"}
{"log":"LOG:  database system is ready to accept connections\n","stream":"stderr","time":"2017-09-22T18:17:14.002587156Z"}
{"log":"LOG:  autovacuum launcher started\n","stream":"stderr","time":"2017-09-22T18:17:14.003038581Z"}
{"log":"time=\"2017-09-22T18:17:55Z\" level=info msg=\"Establishing connection with Rethinkdb\" \n","stream":"stderr","time":"2017-09-22T18:17:55.099603503Z"}
{"log":"time=\"2017-09-22T18:17:55Z\" level=info msg=\"writing out configs and watching for changes\" \n","stream":"stderr","time":"2017-09-22T18:17:55.170139754Z"}
{"log":"time=\"2017-09-22T18:17:55Z\" level=info msg=\"watching configs\" \n","stream":"stderr","time":"2017-09-22T18:17:55.170159254Z"}
{"log":"time=\"2017-09-22T18:17:55Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_ca/cert.pem\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"0fdce033-1af9-4e90-95b9-f7848bd025e7\\\"}\" \n","stream":"stderr","time":"2017-09-22T18:17:55.170459958Z"}
{"log":"time=\"2017-09-22T18:17:55Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_server/cert.pem/a2382528a00b\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"d3c7f77c-faa6-453e-89f8-37b5a08eb151\\\"}\" \n","stream":"stderr","time":"2017-09-22T18:17:55.171474985Z"}
{"log":"time=\"2017-09-22T18:17:55Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_server/key.pem/a2382528a00b\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"c6c12665-3999-4944-8cb9-bc4f05b1ddf4\\\"}\" \n","stream":"stderr","time":"2017-09-22T18:17:55.172106732Z"}
{"log":"LOG:  database system was interrupted; last known up at 2017-09-22 18:17:14 UTC\n","stream":"stderr","time":"2017-09-22T18:17:55.206341426Z"}
{"log":"LOG:  database system was not properly shut down; automatic recovery in progress\n","stream":"stderr","time":"2017-09-22T18:17:55.21386258Z"}
{"log":"LOG:  redo starts at 0/1BE4058\n","stream":"stderr","time":"2017-09-22T18:17:55.215192389Z"}
{"log":"LOG:  invalid record length at 0/1BE6978: wanted 24, got 0\n","stream":"stderr","time":"2017-09-22T18:17:55.215204641Z"}
{"log":"LOG:  redo done at 0/1BE6868\n","stream":"stderr","time":"2017-09-22T18:17:55.215207077Z"}
{"log":"LOG:  MultiXact member wraparound protections are now enabled\n","stream":"stderr","time":"2017-09-22T18:17:55.218016223Z"}
{"log":"LOG:  database system is ready to accept connections\n","stream":"stderr","time":"2017-09-22T18:17:55.218438204Z"}
{"log":"LOG:  autovacuum launcher started\n","stream":"stderr","time":"2017-09-22T18:17:55.218702764Z"}
{"log":"{\"user_id\":1233434,\"event_name\":\"LOGGED_IN\"}","stream":"stderr","time":"2018-03-05T17:30:55.218702764Z"}

elasticsearch.sh

#!/bin/bash

#exec docker run --rm -ti --name elasticsearch \
#  -p 9200:9200 -p 9300:9300 \
#  -v $(pwd)/esdata:/usr/share/elasticsearch/data elasticsearch

exec docker run --rm -ti --name elasticsearch \
  -p 9200:9200 -p 9300:9300 elasticsearch

filebeat.sh

#!/bin/bash

exec docker run --rm -ti --link logstash:logstash \
  -v $(pwd)/2018-03-06.log:/var/lib/docker/containers/53c836ef-822b-4246-ae2d-c14d0f78d7b8/2018-03-06.log \
  -v $(pwd)/filebeat.yml:/usr/share/filebeat/filebeat.yml \
  docker.elastic.co/beats/filebeat:6.2.2

filebeat.yml

output.logstash:
  hosts: ["logstash:5044"]
filebeat.prospectors:
  - type: docker
    paths:
      - '/var/lib/docker/containers/*/*.log'
    containers.ids: '*'
processors:
  - decode_json_fields:
      fields: ["message"]
      target: ""
      overwrite_keys: false

kibana.sh

#!/bin/bash

exec docker run --name kibana --rm -ti \
  --link elasticsearch:elasticsearch -p 5601:5601 kibana

logstash.conf

input {
  beats {
    port => 5044
  }
}
filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGLINE}" }
    }

    date {
      match => [ "timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  }

}
output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
      index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
  }
  stdout {
    codec => rubydebug
  }
}

logstash.sh

#!/bin/bash

exec docker run --rm -ti \
  --name logstash -p 5044:5044 \
  --link elasticsearch:elasticsearch \
  -v $(pwd)/logstash.conf:/logstash.conf logstash -f /logstash.conf

• Download all files into some directory
• To start ELK run relevant script in a separate window, for instance: ./elasticsearch.sh