Revisions
-
asyd revised this gist
Jul 15, 2015 . 1 changed file with 20 additions and 6 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -17,19 +17,30 @@ local sp = l.space local timestamp = "[" * l.Cg(dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S") * dt.time_secfrac / dt.time_to_ns, "Timestamp") * "]" local log_date = l.Cg(dt.build_strftime_grammar("%b %d %H:%M:%S"), "LogDate") local host = (l.alnum^1 + l.S("-_"))^1 local fqdn = (l.alnum^1 + l.S("-_."))^1 local proc = l.P"haproxy[" * l.R("09")^1 * l.P"]:" * l.Cg(l.Cc"haproxy", "Type") local remote_addr = l.Cg(ip.v4, "remote_addr") * ":" * l.Cg(l.R("09")^1, "port") local request = l.P{'"' * l.Cg(((1 - l.P'"') + l.V(1))^0, "request") * '"'} local status = l.Cg(l.digit * l.digit * l.digit, "status") local bytes = l.Cg(l.digit^1, "bytes") -- local srv_data = host * l.P"/" * host local ha_shit = (l.digit^1 + l.P"/")^1 local sep = "- - ----" local protocol = l.Cg((l.alnum^1 + l.P("-"))^1, "protocol") local backend = l.Cg(host, "backend_name") * l.P"/" * l.Cg(host, "backend_server") local server = l.P("{")^1 * l.Cg(fqdn, "http_host") * l.P("}")^1 local pattern = log_date * sp * host * sp * proc * sp * remote_addr * sp * timestamp * sp * protocol * sp * backend * sp * ha_shit * sp * status * sp * bytes * sp * sep * sp * ha_shit * sp * ha_shit * sp * server * sp * request local msg = { Timestamp = nil, Type = nil, Payload = nil, Fields = nil } local grammar = l.Ct(pattern) function process_message () local log = read_message("Payload") @@ -46,16 +57,19 @@ function process_message () msg.Hostname = fields.hostname fields.hostname = nil local m = grammar:match(log) if m then msg.Type = m.Type msg.Payload = nil msg.Timestamp = m.Timestamp fields.remote_addr = m.remote_addr fields.request = m.request fields.status = m.status fields.bytes = m.bytes fields.protocol = m.protocol fields.backend_name = m.backend_name fields.backend_server = m.backend_server fields.http_host = m.http_host else -- Fail with return -1 or do whatever you want msg.Type = "Ignore" -
Victor Castell revised this gist
Apr 21, 2015 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -16,7 +16,7 @@ local syslog_grammar = syslog.build_rsyslog_grammar("%TIMESTAMP% %HOSTNAME% %sys local sp = l.space local timestamp = "[" * l.Cg(dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S") * dt.time_secfrac / dt.time_to_ns, "Timestamp") * "]" local log_date = l.Cg(dt.build_strftime_grammar("%b %d %H:%M:%S"), "LogDate") local host = (l.alnum^1 + l.S("-_"))^1 local proc = l.P"haproxy[" * l.R("09")^1 * l.P"]:" * l.Cg(l.Cc"haproxy", "Type") local remote_addr = l.Cg(ip.v4, "remote_addr") * ":" * l.Cg(l.R("09")^1, "port") -
Victor Castell revised this gist
Apr 21, 2015 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -25,7 +25,7 @@ local uuid = "[" * l.Cg((l.R("AZ", "09") + l.S":_")^1, "uuid") * "]" local request = l.P{'"' * l.Cg(((1 - l.P'"') + l.V(1))^0, "request") * '"'} local status = l.Cg(l.digit * l.digit * l.digit, "status") local bytes = l.Cg(l.digit^1, "bytes") local srv_data = host * sp * host * l.P"/" * host local ha_shit = (l.digit^1 + l.P"/")^1 local sep = "- - ----" -
Victor Castell revised this gist
Apr 21, 2015 . 1 changed file with 1 addition and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -20,6 +20,7 @@ local host = (l.alnum^1 + l.P("-"))^1 local proc = l.P"haproxy[" * l.R("09")^1 * l.P"]:" * l.Cg(l.Cc"haproxy", "Type") local remote_addr = l.Cg(ip.v4, "remote_addr") * ":" * l.Cg(l.R("09")^1, "port") -- Ommit this for the default haproxy log format local uuid = "[" * l.Cg((l.R("AZ", "09") + l.S":_")^1, "uuid") * "]" local request = l.P{'"' * l.Cg(((1 - l.P'"') + l.V(1))^0, "request") * '"'} local status = l.Cg(l.digit * l.digit * l.digit, "status") -
Victor Castell revised this gist
Apr 21, 2015 . 1 changed file with 1 addition and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -56,6 +56,7 @@ function process_message () fields.status = m.status fields.bytes = m.bytes else -- Fail with return -1 or do whatever you want msg.Type = "Ignore" msg.Payload = fields.msg end -
Victor Castell revised this gist
Apr 21, 2015 . 1 changed file with 11 additions and 11 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -16,19 +16,19 @@ local syslog_grammar = syslog.build_rsyslog_grammar("%TIMESTAMP% %HOSTNAME% %sys local sp = l.space local timestamp = "[" * l.Cg(dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S") * dt.time_secfrac / dt.time_to_ns, "Timestamp") * "]" local log_date = l.Cg(dt.build_strftime_grammar("%b %d %H:%M:%S"), "LogDate") local host = (l.alnum^1 + l.P("-"))^1 local proc = l.P"haproxy[" * l.R("09")^1 * l.P"]:" * l.Cg(l.Cc"haproxy", "Type") local remote_addr = l.Cg(ip.v4, "remote_addr") * ":" * l.Cg(l.R("09")^1, "port") local uuid = "[" * l.Cg((l.R("AZ", "09") + l.S":_")^1, "uuid") * "]" local request = l.P{'"' * l.Cg(((1 - l.P'"') + l.V(1))^0, "request") * '"'} local status = l.Cg(l.digit * l.digit * l.digit, "status") local bytes = l.Cg(l.digit^1, "bytes") local srv_data = l.alnum^1 * sp * l.alnum^1 * l.P"/" * l.alnum^1 local ha_shit = (l.digit^1 + l.P"/")^1 local sep = "- - ----" local msg_grammar = l.Ct(log_date * sp * host * sp * proc * sp * remote_addr * sp * timestamp * sp * uuid * sp * srv_data * sp * ha_shit * sp * status * sp * bytes * sp * sep * sp * ha_shit * sp * ha_shit * sp * request) function process_message () local log = read_message("Payload") @@ -50,14 +50,14 @@ function process_message () msg.Type = m.Type msg.Payload = nil msg.Timestamp = m.Timestamp fields.uuid = m.uuid fields.remote_addr = m.remote_addr fields.request = m.request fields.status = m.status fields.bytes = m.bytes else msg.Type = "Ignore" msg.Payload = fields.msg end fields.msg = nil -
Victor Castell revised this gist
Apr 21, 2015 . 1 changed file with 4 additions and 4 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -15,6 +15,7 @@ Fields = nil local syslog_grammar = syslog.build_rsyslog_grammar("%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n") local sp = l.space local timestamp = "[" * l.Cg(dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S") * dt.time_secfrac / dt.time_to_ns, "Timestamp") * "]" local log_date = l.Cg(dt.build_strftime_grammar("%b %d %H:%M:%S"), "LogDate") local proc = l.P"haproxy[" * l.R("09")^1 * l.P"]:" * l.Cg(l.Cc"haproxy", "Type") local host = l.Cg(ip.v4, "Ip") * ":" * l.Cg(l.R("09")^1, "port") @@ -27,7 +28,7 @@ local srv_data = l.R("az")^1 * sp * l.R("az")^1 * l.P"/" * l.R("az", "09")^1 local ha_shit = (l.digit^1 + l.P"/")^1 local sep = "- - ----" local msg_grammar = log_date * sp * proc * sp * host * sp * timestamp * sp * uuid * sp * srv_data * sp * ha_shit * sp * status * sp * bytes * sp * sep * sp * ha_shit * sp * ha_shit * sp * request function process_message () local log = read_message("Payload") @@ -52,8 +53,8 @@ function process_message () fields.Uuid = m.Uuid fields.Remoteaddr = m.Ip fields.Request = m.Request fields.status = m.status fields.bytes = m.bytes else msg.Type = "Ignore" msg.Payload = fields.msg @@ -64,4 +65,3 @@ function process_message () inject_message(msg) return 0 end -
Victor Castell revised this gist
Apr 21, 2015 . 1 changed file with 1 addition and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -64,3 +64,4 @@ function process_message () inject_message(msg) return 0 end -
Victor Castell created this gist
Apr 21, 2015 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,66 @@ local dt = require "date_time" local ip = require "ip_address" local l = require 'lpeg' local syslog = require "syslog" l.locale(l) local msg = { Timestamp = nil, Hostname = nil, Payload = nil, Pid = nil, Fields = nil } local syslog_grammar = syslog.build_rsyslog_grammar("%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n") local sp = l.space local timestamp = "[" * l.Cg(dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S") * dt.time_secfrac / dt.time_to_ns, "Timestamp") * "]" local proc = l.P"haproxy[" * l.R("09")^1 * l.P"]:" * l.Cg(l.Cc"haproxy", "Type") local host = l.Cg(ip.v4, "Ip") * ":" * l.Cg(l.R("09")^1, "port") local uuid = "[" * l.Cg((l.R("AZ", "09") + l.S":_")^1, "Uuid") * "]" local err = l.P"ERROR:" * l.Cg(l.Cc"SphinxError", "Type") local request = l.P{'"' * l.Cg(((1 - l.P'"') + l.V(1))^0, "Request") * '"'} local status = l.Cg(l.digit * l.digit * l.digit, "status") local bytes = l.Cg(l.digit^1, "bytes") local srv_data = l.R("az")^1 * sp * l.R("az")^1 * l.P"/" * l.R("az", "09")^1 local ha_shit = (l.digit^1 + l.P"/")^1 local sep = "- - ----" local msg_grammar = proc * sp * host * sp * timestamp * sp * uuid * sp * srv_data * sp * ha_shit * sp * status * sp * bytes * sp * sep * sp * ha_shit * sp * ha_shit * sp * request function process_message () local log = read_message("Payload") local fields = syslog_grammar:match(log) if not fields then return -1 end msg.Timestamp = fields.timestamp fields.timestamp = nil fields.programname = fields.syslogtag.programname msg.Pid = fields.syslogtag.pid or nil fields.syslogtag = nil msg.Hostname = fields.hostname fields.hostname = nil local m = msg_grammar:match(fields.msg) if m then msg.Type = m.Type msg.Payload = nil msg.Timestamp = m.Timestamp fields.Uuid = m.Uuid fields.Remoteaddr = m.Ip fields.Request = m.Request fields.status = m.status fields.bytes = m.bytes else msg.Type = "Ignore" msg.Payload = fields.msg end fields.msg = nil msg.Fields = fields inject_message(msg) return 0 end