FIO (Flexible I/O Tester) JSON report parser

fio-report-parser.js

/*
Supported job types:
  - read: sequential reads
  - write: sequential writes
  - randread: random reads
  - randwrite: random writes

Unsupported job types:
  - rw: sequential mixed reads and writes
  - readwrite: same as above
  - randrw: random mixed reads and writes

SSD tests. Not supported yet.
  - trim: sequential trims (Linux block devices and SCSI character devices only)
  - randtrim: random trims (Linux block devices and SCSI character devices only)
  - trimwrite: sequential trim+write sequences
  - randtrimwrite: like trimwrite, but uses random offsets rather than sequential writes
*/
const IO_PATTERN = {
  read: 'Sequential read',
  write: 'Sequential write',
  randread: 'Random read',
  randwrite: 'Random write',
  // rw: 'sequential mixed read and write',
  // readwrite: 'sequential mixed read and write',
  // randrw: 'random mixed read and write',
  // trim: sequential trims (Linux block devices and SCSI character devices only)
  // randtrim: random trims (Linux block devices and SCSI character devices only)
  // trimwrite: sequential trim+write sequences
  // randtrimwrite: like trimwrite, but uses random offsets rather than sequential writes
};
const CACHE_TITLE = {
  '0': 'Buffered I/O',
  '1': 'Non-buffered I/O (this is usually O_DIRECT)',
};
const UNSUPPORTED_TYPE = '_UNSUPPORTED_';
const FIRST_COLUMN_HEADER = 'Name';

const removeEscapeSequence = (str) => str.replace(/\x1b\[\d+m/g, '');
const drawTable = (table) => {
  const thead = table.reduce(
    (acc, row) => {
      Object.entries(row).forEach(([k, v]) => {
        const keyLength = k.length;
        const valueLength = removeEscapeSequence(v.toString()).length;
        const l = keyLength > valueLength ? keyLength : valueLength;

        if (!acc[k] || acc[k] < l) {
          acc[k] = l;
        }
      });

      return acc;
    },
    {},
  );
  const tbody = table.reduce(
    (acc, row) => {
      const tr = {};

      Object.entries(row).forEach(([k, v]) => {
        const val = v.toString();
        const targetLength = val.length - removeEscapeSequence(val).length + thead[k];

        tr[k] = val[k === FIRST_COLUMN_HEADER ? 'padEnd' : 'padStart'](targetLength);
      });
      acc.push(tr);

      return acc;
    },
    [],
  );
  const lanes = [
    ['┌'],
    ['│'],
    ['├'],
    ...Array.from(
      { length: tbody.length },
      () => ['│'], // Must be new array for each lane.
    ),
    ['└'],
  ];
  const tfootIndex = lanes.length - 1;

  // Populate table with header, rows and footer.
  Object.entries(thead).forEach(([k, v]) => {
    lanes[0].push('─'.repeat(v + 2), '┬'); // +2 = spaces before and after column label.
    lanes[1].push(` ${k.padEnd(v)} `, '│');
    lanes[2].push('─'.repeat(v + 2), '┼');
    tbody.forEach((td, index) => {
      lanes[3 + index].push(` ${td[k] ?? ''.padEnd(v)} `, '│');
    });
    lanes[tfootIndex].push('─'.repeat(v + 2), '┴');
  });

  lanes[0] = lanes[0].with(-1, '┐');
  lanes[2] = lanes[2].with(-1, '┤');
  lanes[tfootIndex] = lanes[tfootIndex].with(-1, '┘');

  lanes.forEach((line) => {
    console.log(line.join(''));
  });
};
const kBtoMb = (kb) => kb / 1024;
const ns2ms = (ns) => ns / 1000000;

if (process.argv.length === 2) {
  console.error('\x1b[31m%s\x1b[0m', 'Please provide FIO test results in JSON format.');
  console.log('\x1b[33mExample:\x1b[0m %s', `node ${__filename} ./path/to/report.json`);

  process.exit(1);
}

let filepath = process.argv[2];

if (!(filepath.startsWith('./') || filepath.startsWith('/'))) {
  filepath = `./${filepath}`;
}

const parsingMessage = `Parsing ${filepath}`;

console.time(parsingMessage);

const report = require(filepath);

console.timeEnd(parsingMessage);

const globalOptions = report['global options'];

console.info(`Tests were executed in ${globalOptions.directory}`);
console.info('\x1b[1m%s\x1b[0m', CACHE_TITLE[globalOptions?.direct ?? 0], 'was used in tests.\n');

const jobs = report.jobs.reduce(
  (
    acc,
    {
      error,
      'job options': options,
      jobname,
      read,
      write,
    },
  ) => {
    if (error > 0) {
      console.warn('\x1b[33m%s\x1b[0m', `${jobname} job has error. Skipping...`);
    } else {
      let type;

      if (options.rw === 'read' || options.rw === 'randread') {
        type = read;
      } else if (options.rw === 'write' || options.rw === 'randwrite') {
        type = write;
      } else {
        throw new Error(`\x1b[1m\x1b[41mUnsupported job found: ${jobname} of type ${options.rw}\x1b[0m`);
      }

      acc.push({
        ...globalOptions,
        ...options,
        ...type,
        jobname,
      });
    }

    return acc;
  },
  [],
);

if (jobs.length > 0) {
  console.info('\x1b[32m%s\x1b[0m', `${jobs.length} successful jobs found.`);
} else {
  console.info('\x1b[31m%s\x1b[0m', 'No successful jobs found.');
  process.exit();
}

const jobGroups = Object.groupBy(jobs, ({ rw }) => {
  if (IO_PATTERN[rw]) {
    return rw.replace('rand', '');
  } else {
    console.warn('\x1b[33m%s\x1b[0m', `Skip unsupported job type: ${rw}`);

    return UNSUPPORTED_TYPE;
  }
});
const table = [];

delete jobGroups[UNSUPPORTED_TYPE];
Object.entries(jobGroups).forEach(([k, v]) => {
  table.push({
    [FIRST_COLUMN_HEADER]: `\x1b[100m[${k.toUpperCase()}]\x1b[0m`,
  });
  v.forEach(({
    bs,
    bw,
    clat_ns: {
      max: latencyMax,
      mean: latencyMean,
      min: latencyMin,
    },
    iodepth,
    iops,
    jobname,
    numjobs,
    rw,
  }) => {
    const dim = rw.startsWith('rand') ? '\x1b[2m' : '';

    table.push({
      [FIRST_COLUMN_HEADER]: `${dim}${jobname}\x1b[0m`,
      'IO pattern': `${dim}${IO_PATTERN[rw]}\x1b[0m`,
      'Block size': `${dim}\x1b[90m${bs}\x1b[0m`,
      'Queue depth': `${dim}${iodepth}\x1b[0m`,
      'Threads': `${dim}${numjobs}\x1b[0m`,
      'MB/s': `${dim}\x1b[35m${kBtoMb(bw).toFixed(2)}\x1b[0m`,
      'IOPS': `${dim}\x1b[33m${Math.round(iops)}\x1b[0m`,
      'Min latency (ms)': `${dim}\x1b[34m${ns2ms(latencyMin).toFixed(1)}\x1b[0m`,
      'Mean latency (ms)': `${dim}\x1b[34m${ns2ms(latencyMean).toFixed(1)}\x1b[0m`,
      'Max latency (ms)': `${dim}\x1b[34m${ns2ms(latencyMax).toFixed(1)}\x1b[0m`,
    });
  });
});

drawTable(table);

Usage example

$ node fio-report-parser.js ./output/Samsung64GB-20231103200514.json
Parsing ./output/Samsung64GB-20231103200514.json
Tests were executed in /media/usb/1.42.6-8017/
Non-buffered I/O (this is usually O_DIRECT) was used in tests.

Successful jobs: 12
--------------------------------------------------
3 sequential read performance tests.
3 sequential write performance tests.
3 random read performance tests.
3 random write performance tests.

Results for sequential read tests
Seq-Read-Q32T1	QD32	Read	264.98	MB/s	265	IOPS	Latency (min/mean/max)	7.4	ms	115.7	ms	128.3	ms
Seq-Read-Q8T1	QD8	Read	268.37	MB/s	268	IOPS	Latency (min/mean/max)	7.3	ms	26.0	ms	30.5	ms
Seq-Read-Q1T1	QD1	Read	262.55	MB/s	263	IOPS	Latency (min/mean/max)	3.3	ms	3.8	ms	6.3	ms

Results for sequential write tests
Seq-Write-Q32T1	QD32	Write	34.07	MB/s	34	IOPS	Latency (min/mean/max)	25.5	ms	831.1	ms	978.8	ms
Seq-Write-Q8T1	QD8	Write	34.30	MB/s	34	IOPS	Latency (min/mean/max)	31.5	ms	200.4	ms	231.9	ms
Seq-Write-Q1T1	QD1	Write	34.67	MB/s	35	IOPS	Latency (min/mean/max)	23.6	ms	28.7	ms	36.3	ms

Results for random read tests
Rand-Read-4K-Q8T8	QD8	Read	13.63	MB/s	3490	IOPS	Latency (min/mean/max)	0.5	ms	16.0	ms	21.6	ms
Rand-Read-4K-Q32T1	QD32	Read	13.35	MB/s	3419	IOPS	Latency (min/mean/max)	0.6	ms	9.1	ms	11.6	ms
Rand-Read-4K-Q1T1	QD1	Read	12.47	MB/s	3193	IOPS	Latency (min/mean/max)	0.2	ms	0.3	ms	1.0	ms

Results for random write tests
Rand-Write-4K-Q8T8	QD8	Write	15.40	MB/s	3944	IOPS	Latency (min/mean/max)	0.1	ms	14.2	ms	152.5	ms
Rand-Write-4K-Q32T1	QD32	Write	15.66	MB/s	4010	IOPS	Latency (min/mean/max)	0.1	ms	7.7	ms	162.6	ms
Rand-Write-4K-Q1T1	QD1	Write	15.60	MB/s	3993	IOPS	Latency (min/mean/max)	0.1	ms	0.2	ms	146.9	ms


DONE

Display report results in a table.

$ node fio-report-parser.js reports-json/Samsung64GB-20231103200514.json
Parsing ./reports-json/Samsung64GB-20231103200514.json: 0.851ms
Tests were executed in /media/usb/1.42.6-8017/
Non-buffered I/O (this is usually O_DIRECT) was used in tests.

12 successful jobs found.
┌─────────────────────┬──────────────────┬────────────┬─────────────┬─────────┬────────┬──────┬──────────────────┬───────────────────┬──────────────────┐
│ Name                │ IO pattern       │ Block size │ Queue depth │ Threads │ MB/s   │ IOPS │ Min latency (ms) │ Mean latency (ms) │ Max latency (ms) │
├─────────────────────┼──────────────────┼────────────┼─────────────┼─────────┼────────┼──────┼──────────────────┼───────────────────┼──────────────────┤
│ [READ]              │                  │            │             │         │        │      │                  │                   │                  │
│ Seq-Read-Q32T1      │  Sequential read │         1m │          32 │       1 │ 264.98 │  265 │              7.4 │             115.7 │            128.3 │
│ Seq-Read-Q8T1       │  Sequential read │         1m │           8 │       1 │ 268.37 │  268 │              7.3 │              26.0 │             30.5 │
│ Seq-Read-Q1T1       │  Sequential read │         1m │           1 │       1 │ 262.55 │  263 │              3.3 │               3.8 │              6.3 │
│ Rand-Read-4K-Q8T8   │      Random read │         4k │           8 │       8 │  13.63 │ 3490 │              0.5 │              16.0 │             21.6 │
│ Rand-Read-4K-Q32T1  │      Random read │         4k │          32 │       1 │  13.35 │ 3419 │              0.6 │               9.1 │             11.6 │
│ Rand-Read-4K-Q1T1   │      Random read │         4k │           1 │       1 │  12.47 │ 3193 │              0.2 │               0.3 │              1.0 │
│ [WRITE]             │                  │            │             │         │        │      │                  │                   │                  │
│ Seq-Write-Q32T1     │ Sequential write │         1m │          32 │       1 │  34.07 │   34 │             25.5 │             831.1 │            978.8 │
│ Seq-Write-Q8T1      │ Sequential write │         1m │           8 │       1 │  34.30 │   34 │             31.5 │             200.4 │            231.9 │
│ Seq-Write-Q1T1      │ Sequential write │         1m │           1 │       1 │  34.67 │   35 │             23.6 │              28.7 │             36.3 │
│ Rand-Write-4K-Q8T8  │     Random write │         4k │           8 │       8 │  15.40 │ 3944 │              0.1 │              14.2 │            152.5 │
│ Rand-Write-4K-Q32T1 │     Random write │         4k │          32 │       1 │  15.66 │ 4010 │              0.1 │               7.7 │            162.6 │
│ Rand-Write-4K-Q1T1  │     Random write │         4k │           1 │       1 │  15.60 │ 3993 │              0.1 │               0.2 │            146.9 │
└─────────────────────┴──────────────────┴────────────┴─────────────┴─────────┴────────┴──────┴──────────────────┴───────────────────┴──────────────────┘