Skip to content

Instantly share code, notes, and snippets.

@fefore

fefore/cloudfront_filter

Last active May 23, 2018 11:09
Download ZIP

Revisions

  1. fefore renamed this gist Oct 31, 2017. 1 changed file with 0 additions and 0 deletions.
    0 cloudfront_grok → cloudfront_filter
    View file
    File renamed without changes.
  2. fefore revised this gist Oct 31, 2017. 1 changed file with 27 additions and 24 deletions.
    51 changes: 27 additions & 24 deletions cloudfront_grok
    View file
    Original file line number Diff line number Diff line change
    @@ -1,27 +1,30 @@
    filter {
    if [type] == "cloudfront" {
    if ( ("#Version: 1.0" in [message]) or ("#Fields: date" in [message])) {
    drop {}
    }

    grok {
    match => { "message" => "%{DATE_US:date}\t%{TIME:time}\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes}|-)\t%{IPORHOST:c_ip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:User_Agent}\t%{GREEDYDATA:cs_uri_stem}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes}\t%{GREEDYDATA:time_taken}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}" }
    }

    mutate {
    add_field => [ "received_at", "%{@timestamp}" ]
    add_field => [ "listener_timestamp", "%{date} %{time}" ]
    }

    date {
    match => [ "listener_timestamp", "yy-MM-dd HH:mm:ss" ]
    }

    date {
    locale => "en"
    timezone => "UCT"
    match => [ "listener_timestamp", "yy-MM-dd HH:mm:ss" ]
    target => "@timestamp"
    }
    if [cloudfront_version] {
    mutate {
    gsub => ["message", "\t", " "]
    }
    dissect {
    mapping => {"message" => "%{date} %{time} %{x_edge_location} %{sc_bytes} %{clientip} %{cs_method} %{cs_host} %{cs_uri_stem} %{sc_status} %{referrer} %{agent} %{cs_uri_query} %{cookies} %{x_edge_result_type} %{x_edge_request_id} %{x_host_header} %{cs_protocol} %{cs_bytes} %{time_taken} %{x_forwarded_for} %{ssl_protocol} %{ssl_cipher} %{x_edge_response_result_type} %{cs_protocol_version}"}
    convert_datatype => {
    sc_bytes => "int"
    sc_status => "int"
    cs_bytes => "int"
    time_taken => "float"
    }
    add_field => {"timestamp" => "%{date} %{time}"}
    }
    date {
    match => ["timestamp", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss"]
    timezone => "UTC"
    }
    geoip {
    source => "clientip"
    }
    mutate {
    replace => {
    "type" => "cloudfront"
    }
    remove_field => ["timestamp","message","cloudfront_fields"]
    }
    }
    }
  3. fefore created this gist Oct 18, 2017.
    27 changes: 27 additions & 0 deletions cloudfront_grok
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,27 @@
    filter {
    if [type] == "cloudfront" {
    if ( ("#Version: 1.0" in [message]) or ("#Fields: date" in [message])) {
    drop {}
    }

    grok {
    match => { "message" => "%{DATE_US:date}\t%{TIME:time}\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes}|-)\t%{IPORHOST:c_ip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:User_Agent}\t%{GREEDYDATA:cs_uri_stem}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes}\t%{GREEDYDATA:time_taken}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}" }
    }

    mutate {
    add_field => [ "received_at", "%{@timestamp}" ]
    add_field => [ "listener_timestamp", "%{date} %{time}" ]
    }

    date {
    match => [ "listener_timestamp", "yy-MM-dd HH:mm:ss" ]
    }

    date {
    locale => "en"
    timezone => "UCT"
    match => [ "listener_timestamp", "yy-MM-dd HH:mm:ss" ]
    target => "@timestamp"
    }
    }
    }