Forked from lanyonm/apache-and-tomcat-logs.json
Created
January 15, 2019 18:20
Revisions
-
lanyonm revised this gist
Apr 12, 2016 . No changes.There are no files selected for viewing
-
Sep 28, 2014 . 1 changed file with 3 additions and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -52,6 +52,9 @@ filter { } output { if [type] == "tomcat" and "_grokparsefailure" in [tags] { # if we didn't drop the messages above, we could send them to a special failure log here } stdout { debug => true } -
Jan 13, 2014 . 1 changed file with 0 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -9,7 +9,6 @@ jar="logstash-$version-flatjar.jar" config='logstash-java.conf' java_opts='-Xmx2048m -Xms1024m' if [ ! -f $jar ]; then echo "logstash jar not found - downloading $jar..." curl -o $jar https://download.elasticsearch.org/logstash/logstash/$jar -
Jan 13, 2014 . 2 changed files with 315 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,315 @@ { "title": "Apache and Tomcat Logs", "services": { "query": { "list": { "0": { "query": "apache !tomcat !static", "alias": "", "color": "#7EB26D", "id": 0, "pin": false, "type": "lucene", "enable": true }, "1": { "id": 1, "color": "#BF1B00", "alias": "", "pin": false, "type": "lucene", "enable": true, "query": "tomcat prod1 !apache !prod2" }, "2": { "id": 2, "color": "#EF843C", "alias": "", "pin": false, "type": "lucene", "enable": true, "query": "tomcat prod2 !apache !prod1" }, "6": { "id": 6, "color": "#1F78C1", "alias": "", "pin": false, "type": "lucene", "enable": true, "query": "static" } }, "ids": [ 0, 1, 2, 6 ] }, "filter": { "list": { "0": { "from": "2014-01-09T12:00:00.000Z", "to": "2014-01-10T12:00:00.000Z", "type": "time", "field": "@timestamp", "mandate": "must", "active": true, "alias": "", "id": 0 }, "2": { "type": "field", "field": "_type", "query": "\"apache\"", "mandate": "mustNot", "active": false, "alias": "", "id": 2 } }, "ids": [ 2, 0 ] } }, "rows": [ { "title": "Graph", "height": "500px", "editable": true, "collapse": false, "collapsable": true, "panels": [ { "span": 12, "editable": true, "group": [ "default" ], "type": "histogram", "mode": "count", "time_field": "@timestamp", "value_field": null, "auto_int": false, "resolution": 100, "interval": "5m", "fill": 3, "linewidth": 3, "timezone": "browser", "spyable": true, "zoomlinks": true, "bars": true, "stack": true, "points": false, "lines": false, "legend": true, "x-axis": true, "y-axis": true, "percentage": false, "interactive": true, "queries": { "mode": "all", "ids": [ 0, 1, 2, 6 ] }, "title": "Events over time", "intervals": [ "auto", "1s", "1m", "5m", "10m", "30m", "1h", "3h", "12h", "1d", "1w", "1M", "1y" ], "options": true, "tooltip": { "value_type": "cumulative", "query_as_alias": true }, "scale": 1, "y_format": "none", "grid": { "max": null, "min": 0 }, "annotate": { "enable": false, "query": "*", "size": 20, "field": "_type", "sort": [ "_score", "desc" ] }, "pointradius": 5, "show_query": true, "legend_counts": true, "zerofill": true, "derivative": false } ], "notice": false }, { "title": "Events", "height": "350px", "editable": true, "collapse": false, "collapsable": true, "panels": [ { "title": "All events", "error": false, "span": 12, "editable": true, "group": [ "default" ], "type": "table", "size": 250, "pages": 10, "offset": 0, "sort": [ "@timestamp", "asc" ], "style": { "font-size": "9pt" }, "overflow": "min-height", "fields": [ "@timestamp", "message", "response" ], "localTime": true, "timeField": "@timestamp", "highlight": [], "sortable": true, "header": true, "paging": true, "spyable": true, "queries": { "mode": "all", "ids": [ 0, 1, 2, 6 ] }, "field_list": true, "status": "Stable", "trimFactor": 300, "normTimes": true, "all_fields": false } ], "notice": false } ], "editable": true, "failover": false, "index": { "interval": "day", "pattern": "[logstash-]YYYY.MM.DD", "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED", "warm_fields": true }, "style": "dark", "panel_hints": true, "pulldowns": [ { "type": "query", "collapse": false, "notice": false, "query": "*", "pinned": true, "history": [ "static", "tomcat prod2 !apache !prod1", "tomcat prod1 !apache !prod2", "apache !tomcat !static", "js", "css", "jpg | png | gif", ".jpg | .png | .gif", "apache !tomcat static", "apache !tomcat !css !js !(.jpg | .png | .gif)" ], "remember": 10, "enable": true }, { "type": "filtering", "collapse": false, "notice": true, "enable": true } ], "nav": [ { "type": "timepicker", "collapse": false, "notice": false, "status": "Stable", "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ], "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "timefield": "@timestamp", "now": false, "filter_id": 0, "enable": true } ], "loader": { "save_gist": false, "save_elasticsearch": true, "save_local": true, "save_default": true, "save_temp": true, "save_temp_ttl_enable": true, "save_temp_ttl": "30d", "load_gist": true, "load_elasticsearch": true, "load_elasticsearch_size": 20, "load_local": true, "hide": false }, "refresh": false } This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -96,7 +96,6 @@ JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+ JAVAFILE (?:[A-Za-z0-9_.-]+) JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\) JAVALOGMESSAGE (.*) # MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM) # yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800 -
Jan 12, 2014 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -6,7 +6,7 @@ version='1.3.2' jar="logstash-$version-flatjar.jar" config='logstash-java.conf' java_opts='-Xmx2048m -Xms1024m' myjava=$(which java) -
Jan 12, 2014 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,106 @@ USERNAME [a-zA-Z0-9._-]+ USER %{USERNAME} INT (?:[+-]?(?:[0-9]+)) BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) NUMBER (?:%{BASE10NUM}) BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+)) BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b POSINT \b(?:[1-9][0-9]*)\b NONNEGINT \b(?:[0-9]+)\b WORD \b\w+\b NOTSPACE \S+ SPACE \s* DATA .*? GREEDYDATA .* QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} # Networking MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9]) IP (?:%{IPV6}|%{IPV4}) HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b) HOST %{HOSTNAME} IPORHOST (?:%{HOSTNAME}|%{IP}) HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT}) # paths PATH (?:%{UNIXPATH}|%{WINPATH}) UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+ TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+)) WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? URIHOST %{IPORHOST}(?::%{POSINT:port})? # uripath comes loosely from RFC1738, but mostly from what Firefox # doesn't turn into %XX URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ #URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? # Months: January, Feb, 3, 03, 12, December MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b MONTHNUM (?:0?[1-9]|1[0-2]) MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) # Days: Monday, Tue, Thu, etc... DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) # Years? YEAR (?>\d\d){1,2} HOUR (?:2[0123]|[01]?[0-9]) MINUTE (?:[0-5][0-9]) # '60' is a leap second in most time standards and thus is valid. SECOND (?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?) TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) # datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) ISO8601_SECOND (?:%{SECOND}|60) TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? DATE %{DATE_US}|%{DATE_EU} DATESTAMP %{DATE}[- ]%{TIME} TZ (?:[PMCE][SD]T|UTC) DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} # Syslog Dates: Month Day HH:MM:SS SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} PROG (?:[\w._/%-]+) SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? SYSLOGHOST %{IPORHOST} SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} # Shortcuts QS %{QUOTEDSTRING} # Log formats SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} # Log Levels LOGLEVEL ([A-a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) # Java Logs JAVATHREAD (?:[A-Z]{2}-Processor[\d]+) JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+ JAVAFILE (?:[A-Za-z0-9_.-]+) JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\) JAVALOGMESSAGE (.*) JBOSSLOG %{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} \[\s?%{JAVACLASS:class}\s?\] # MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM) # yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800 TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE} CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage} # 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened... TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage} This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,61 @@ input { tcp { type => "apache" port => 3333 add_field => { "server" => "prod1" } } tcp { type => "apache" port => 3334 add_field => { "server" => "prod2" } } tcp { type => "tomcat" port => 3335 add_field => { "server" => "prod1" } } tcp { type => "tomcat" port => 3336 add_field => { "server" => "prod2" } } } filter { if [type] == "apache" { grok { patterns_dir => "/Users/lanyonm/logstash/patterns" match => { "message" => "%{COMBINEDAPACHELOG}" } } date { match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ] } } if [type] == "tomcat" { multiline { patterns_dir => "/Users/lanyonm/logstash/patterns" pattern => "(^%{TOMCAT_DATESTAMP})|(^%{CATALINA_DATESTAMP})" negate => true what => "previous" } if "_grokparsefailure" in [tags] { drop { } } grok { patterns_dir => "/Users/lanyonm/logstash/patterns" match => [ "message", "%{TOMCATLOG}", "message", "%{CATALINALOG}" ] } date { match => [ "timestamp", "yyyy-MM-dd HH:mm:ss,SSS Z", "MMM dd, yyyy HH:mm:ss a" ] } } } output { stdout { debug => true } elasticsearch { embedded => true } } This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,20 @@ #!/bin/bash # # This script assumes you have java installed. It will also download the # Logstash jar if you don't already have it in the same folder as this script. # version='1.3.2' jar="logstash-$version-flatjar.jar" config='java-parse.conf' java_opts='-Xmx2048m -Xms1024m' myjava=$(which java) if [ ! -f $jar ]; then echo "logstash jar not found - downloading $jar..." curl -o $jar https://download.elasticsearch.org/logstash/logstash/$jar fi cmd="java $java_opts -jar $jar agent -f $config -- web" echo "running $cmd" $cmd