Created
November 15, 2018 08:00
-
-
Save abhi-nav/3a9026165d847971add965a356a50769 to your computer and use it in GitHub Desktop.
Multiple File harvesting using Filebeat and logstash in single Instance
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
filebeat.prospectors: | |
- type: log | |
paths: | |
- /path/to/log1 | |
tags: ["log1"] | |
- type: log | |
paths: | |
- /path/to/custom_geo_json_log | |
tags: ["custom-geo-json-log"] | |
output.logstash: #sending it to logstash instance | |
hosts: ["localhost:5044"] #logstash host |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
input { | |
#listening beats at port 5044 | |
beats { | |
port => "5044" | |
} | |
} | |
filter { | |
if "log1" in [tags] { | |
grok { | |
match => { "message" => "%{COMBINEDAPACHELOG}" } | |
} | |
} else if "custom-geo-json-log" in [tags] { | |
json { #parsing json | |
source => "message" | |
add_field => { | |
#declaring elastic geo point from parsed log in field latitude and longitude | |
"geoip[location][lat]" => "%{latitude}" | |
"geoip[location][lon]" => "%{longitude}" | |
} | |
} | |
date { | |
#replace timestamp from beats to custom log timestamp field | |
match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss" ] | |
target => "@timestamp" | |
} | |
} | |
} | |
output { | |
if "apache-log" in [tags] { | |
stdout { | |
id => "apache-id" | |
codec => rubydebug | |
} | |
#can be replace above with elastic output. for eg:- | |
# elasticsearch { | |
# hosts => ["https://elasticHost:elasticPort"] | |
# user => "username" | |
# password => "password" | |
# cacert => "/etc/logstash/ca.pem" | |
# ssl => true | |
# ssl_certificate_verification=> false | |
# index => "logstash-log1-%{+YYYY.MM}" | |
# } | |
} else if "custom-log" in [tags] { | |
stdout { | |
id => "custom-id" | |
codec => rubydebug | |
} | |
#similarly above stdout can be replaced by elasticsearch out at diff index or diff elastic server | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment