PiHole + Elasticsearch Ingest processing

README.md

NOTE - this was specifically built for a docker instance, with the Filebeat docker module collecting the logs.

filebeat.yml

filebeat.inputs:
- type: log
  paths: 
    - "/var/log/pihole.log"
  fields:
    fileset.module: pihole
  fields_under_root: true
  max_backoff: 10s
  pipeline: "pihole-logging"

processors:
- add_cloud_metadata: ~
- add_docker_metadata: ~
- drop_event:
    when:
      contains:
        message: 'unifi'

#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
# we need to do two when.contains here
  hosts: ["10.3.0.10:9200"]
setup.ilm.enabled: false

pipeline.json

PUT _ingest/pipeline/pihole-logging
{
  "description" : "Pi-Hole Logging",
  "processors": [
    {
      "grok": {
        "field": "message",
        "patterns": ["""%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: query\[%{NOTSPACE:query_record}\] %{NOTSPACE:query_target} from %{NOTSPACE:device_ip}""",
                     """%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\:( gravity | )%{NOTSPACE:query_action} %{NOTSPACE:query_target} is %{NOTSPACE:query_answer}""",
                     """%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: %{NOTSPACE:query_action} %{NOTSPACE:query_target} to %{NOTSPACE:dns_server}""",
                     """%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: (DHCPREQUEST|DHCPACK|DHCPDISCOVER|DHCPOFFER|DHCPRELEASE)\(%{WORD:pihole_interface}\)( %{IP:device_ip})? %{COMMONMAC:device_mac}( %{NOTSPACE:device_name})?"""]
      }
    },
    {
      "date": {
        "field": "reqtimestamp",
        "target_field": "@timestamp",
        "timezone" : "Asia/Karachi",
        "formats": [
                    "MMM  d HH:mm:ss",
                    "MMM dd HH:mm:ss"
        ]
      }
    },
    {
      "remove": {
        "field": "reqtimestamp"
      }
    },
    {
      "remove": {
        "field": "message"
      }
    }
  ],
  "on_failure" : [{
    "set" : {
      "field" : "error.message",
      "value" : "{{ _ingest.on_failure_message }}"
    }
  }]
}