Gists for Blog post about logging and latency

gen-csv.js

#!/usr/bin/env node

const fs = require('fs')
const path = require('path')
const { promisify, format } = require('util')

const defaultOutputPath = path.join(__dirname, 'dump.csv')
const outPath = process.argv[3] ? path.resolve(process.argv[3]) : path.resolve(defaultOutputPath)
const entryCount = process.argv[2] ? Number(process.argv[2]) : 50000

console.info("Csv  generator for contacts")
console.log("Specify number of entries, default is 50k entries")
console.log(`Specify location to write file to, default is  ${defaultOutputPath}`)
console.log("Usage gen-csv.sh 200000 ~/files/dump.csv")


console.log(`Generating ${entryCount} entries and writing to ${outPath}`)


const seed = {
    first: ['Ali', 'Simbi', 'Chuks', 'Musa', 'Hegazi', 'Raymond', 'Sule', 'Samson', '', '', '', '', '', '', '', '', '', '', '', '', '', 'Stefan', 'Kalus', 'Kitoshi'],
    second: ['Mikasa', 'Eren', 'Armin', 'Levi', '', '', '',, '', 'Luc', 'Oyebanjo', 'Dupont', 'Raymond', 'Clark', 'Stefan', '', '', 'Musatafa', 'Edmond', 'Tesfaye'],
    third: ['[email protected]', '[email protected]', '[email protected]', '', '', '', '', 'a@b', '', '', '[email protected]', '[email protected]', 'defo not an email', 'sssjsjsjs', '[email protected]', '', '[email protected]', '[email protected]', '', '[email protected]', '[email protected]', '[email protected]', '[email protected]', '[email protected]', '[email protected]', '[email protected]', '[email protected]', '[email protected]', '[email protected]', '[email protected]', 'retro'],
    fourth: ['(+01) 200020202', '(+234) 700 1111 0000', '(+234) 890 0923 3456', '019203304', '00129201092', '8192202011', '91982290101', '020928281910', '0192822939381', '727211919191', '8828229300303', '91910222', '222922=290290029', '2782882-9292828', '(01) 902 3455', '09 7283 9921', '02 345 8904', '05 783 0234', '01 234 0023', '01 234 4234', '05 3920 0293', '01 2939 039393', '01 0293 93300', '01 2334 9948', '01 299304 0', 'sssshj', 'definitely nota phone number', 'eueuueeeyeyey'],
    fifth: Array(20).fill(0).map(() => Math.random().toString(36).replace(/[^a-z]+/g, '',).substr(0, 5))
}

console.log(outPath)
/**
 * @type {fs.WriteStream}
 */
let fHandle;




truncateFile(outPath)
.then(() => { fHandle = fs.createWriteStream(outPath, { flags: 'a' }); })
.then(() => appendToFile(fHandle, "first_row, second_row, thrid_row, fourth_row, fifth_row\n"))
.then(() => Promise.all(Array(entryCount).fill(0).map(() => {
    return appendToFile(fHandle, format("%s, %s, %s, %s, %s\n", ...getRow())).catch(err => {
        console.error('Failed to write: ', err.message)
    });
})))
.finally(() => fHandle.end())

/**
 * 
 * @returns {Array<string>}
 */
function getRow() {
    return Object.keys(seed).map(key => getRandomFromArray(seed[key]))
}

function getRandomFromArray(array = []) {
    return array[~~(Math.random() * array.length)]
}


/**
 * 
 * @param {string} filePath 
 * @returns 
 */
function truncateFile(filePath) {
    return promisify(fs.writeFile)(filePath, '',  { flag: 'w' });
}

/**
 * 
 * @param {fs.WriteStream} fileHandle
 * @param {Buffer | string} data 
 * @returns 
 */
function appendToFile(fileHandle, data) {
    return promisify(fileHandle.write.bind(fileHandle))(data);
}

gen-csv.sh

#!/bin/bash
echo "Csv  generator for anything"
echo "Specify number of entries, default is 50k entries"
echo "Specify location to write file to, default is $CWD/dump.csv"
echo "Usage gen-csv.sh 200000 ~/files/dump.csv"

entry_count=${1:-50000}
write_loc=${2:-'dump.csv'}

echo "Generating $entry_count entries and writing to $write_loc"

echo "header_one, header_two, other headers " > $write_loc

one=('Ali' 'Simbi' 'Chuks' 'Musa' 'Hegazi' 'Raymond' 'Sule' 'Samson' '' '' '' '' '' '' '' '' '' '' '' '' '' 'Stefan' 'Kalus' 'Kitoshi')
second_row=('Mikasa' 'Eren' 'Armin' 'Levi' '' '' '' '' 'Luc' 'Oyebanjo' 'Dupont' 'Raymond' 'Clark' 'Stefan' '' '' 'Musatafa' 'Edmond' 'Tesfaye')

RANDOM=$$$(date +%s)

for index in $(seq 1 $entry_count)
do
  random_first=${one[$RANDOM % ${#one[@]}]}
  random_last=${second_row[$RANDOM % ${#second_row[@]}]}
  echo "$random_first, $random_last" >> $write_loc
done

loggers.js

const Winston = require('winston')

function createLogger(transport) {
    return Winston.createLogger({
        transports: [transport],
        level: process.env.LOG_LEVEL || 'silly'
    })
}



module.exports = {
    proxiedConsole: new Proxy({}, {
        get: (_, p) => {
            if(['info', 'error'].includes(p)) return console[p];
            return console.log;
        }
    }),

    bareConsole: ({
        log: console.log,
        info: console.info,
        debug: console.log,
        error: console.error
    }),

    winstonConsole: createLogger(new Winston.transports.Console({
        format: Winston.format.combine(
          Winston.format.timestamp(),
          Winston.format.colorize(),
          Winston.format.printf(info =>
            `${info.level}[${info.label || info.timestamp}]: ${info.message}`)
        )
      })),

      winstonFile: createLogger(new Winston.transports.File({
        filename: 'demo.log',
        dirname: __dirname
    }))
}

server.js

const http = require('http')
const loggers = require('./loggers');
const debug = (...args) => console.log(require('util').format(...args));

const server = http.createServer((req, res) => {
    req.on('error', (err) => handleServerErrorResponse(res, JSON.stringify({ message: err.message })));
    if (req.method === 'POST') {
        streamBody(req, () => handleServerResponse(res, JSON.stringify({ message: 'Success' })), 'application/json');
    } else {
        return handleServerResponse(res, JSON.stringify({ message: 'noop' }));
    }
});

server.listen(9000, () => {
    const address = server.address();
    const formattedAddress = `${address.family} ${address.address}:${address.port}`;
    console.log(`Server up and listening at ${formattedAddress}`);
})

/**
 * 
 * @param {import('http').IncomingMessage} res 
 */
function handleServerResponse(res, payload, type = 'text/plain', code = 200) {
    res.writeHead(code, { 'Content-Type': type });
    res.write(payload);
    res.end();
}

function handleServerErrorResponse(res, payload) {
    return handleServerResponse(res, payload, 'application/json', 500);
}

/**
 * 
 * @param {import('http').IncomingMessage} req 
 * @param {Function} successCallBack
 */
function streamBody(req, successCallBack) {
    // use worker thread???
    let processCount = 0;
    req.on('data', (chunk) => {
        const asString = chunk.toString();
        // For our server, we;d just parse all lines
        // do a little cleaning
        if (asString.startsWith('--------------------------') || asString.startsWith('Content-')) return
        // console.log(chunk.length, chunk);
        // console.log('Raw log line');
        // loggers.proxiedConsole.debug(`Chunk size processed: ${chunk.length}`)
        // loggers.bareConsole.debug(`Chunk size processed: ${chunk.length}`)
        // loggers.winstonConsole.debug(`Chunk size processed: ${chunk.length}`)
        // loggers.winstonFile.debug(`Chunk size processed: ${chunk.length}`)
        // debug("Chunk size processed: %d\n", chunk.lengh);
        // console.log(`Chunk size processed: ${chunk.length}`)
        processCount++;
    })

    req.on('end', () => {
        console.log(`Done reading request body in ${processCount} iterations`);
        successCallBack();
    });
}