Last active
April 29, 2025 23:32
-
-
Save itslukej/14302dcc005a82e20c118a14279be49b to your computer and use it in GitHub Desktop.
Emulates a syslog server to push nginx request data to StatsD. A lot of this is fit for my implementation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
log_format json escape=json | |
'{' | |
'"timestamp":"$time_iso8601",' | |
'"request_time":"$request_time",' | |
'"request_time_ms":"$msec",' | |
'"upstream_cache_status":"$upstream_cache_status",' | |
'"remote_addr":"$remote_addr",' | |
'"cf_connecting_ip":"$http_cf_connecting_ip",' | |
'"range":"$http_range",' | |
'"uri":"$uri",' | |
'"args":"$args",' | |
'"status":"$status",' | |
'"bytes_sent":"$body_bytes_sent",' | |
'"http_user_agent":"$http_user_agent",' | |
'"http_host":"$http_host", ' | |
'}'; | |
access_log syslog:server=127.0.0.1:8514 json; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import { config } from 'dotenv'; | |
import dgram from 'dgram'; | |
import StatsD from './StatsD.js'; | |
config({ path: '.env' }); | |
const server = dgram.createSocket('udp4'); | |
const statsd = new StatsD(process.env.STATSD_DSN, { | |
prefix: '_', | |
globalTags: { env: process.env.NODE_ENV || 'development' }, | |
errorHandler: (error) => console.error('StatsD error:', error) | |
}); | |
interface RequestData { | |
request_time: number; | |
request_time_ms: number; | |
bytes_sent: number; | |
status: number; | |
args: { key: string; value: string }[]; | |
timestamp: Date; | |
host_ip?: string; | |
uri: string; | |
upstream_cache_status?: string; | |
} | |
const processMessage = (msg: Buffer, hostIP: string): void => { | |
const json = msg.slice(msg.indexOf('{'), msg.lastIndexOf('}') + 1).toString(); | |
const data = JSON.parse(json) as RequestData; | |
data.request_time = parseFloat(data.request_time?.toString() || '0'); | |
data.request_time_ms = parseFloat(data.request_time_ms?.toString() || '0') * 1000; | |
data.bytes_sent = parseInt(data.bytes_sent?.toString() || '0'); | |
data.status = parseInt(data.status?.toString() || '0'); | |
data.args = (data.args as unknown as string).split('&').map(arg => { | |
const [key, value] = arg.split('='); | |
return { key, value }; | |
}); | |
data.timestamp = new Date(data.timestamp as unknown as string); | |
data.host_ip = hostIP; | |
statsd.gauge('bw.bytes', data.bytes_sent, { | |
// Add whatever else here | |
host_ip: data.host_ip, | |
cache_status: data.upstream_cache_status, | |
}); | |
}; | |
let messagesPerSec = 0; | |
setInterval(() => { | |
console.log('Messages per sec:', Math.round(messagesPerSec / 5)); | |
messagesPerSec = 0; | |
}, 5000); | |
server.on('message', (msg: Buffer, rinfo: dgram.RemoteInfo) => { | |
try { | |
processMessage(msg, rinfo.address); | |
} catch (e) { | |
console.error(e); | |
} | |
messagesPerSec++; | |
}); | |
server.bind(8514, process.env.LOG_BIND_IP || '127.0.0.1'); | |
process.on('SIGINT', () => server.close()); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment