benchmarking sqlite implementations in deno

benchmark_results.log

Benchmarking sqlite drivers against PRAGMA journal_mode=DELETE
    CPU | Apple M1 Max
Runtime | Deno 2.2.1 (aarch64-apple-darwin)

file:///Users/andrew.kaiser/Code/scratchwork/deno-sqlite-benchmark/bench2.ts

benchmark        time/iter (avg)        iter/s      (min … max)           p75      p99     p995
---------------- ----------------------------- --------------------- --------------------------

group insert random data
NodeDatabase            310.6 µs         3,220 (226.8 µs …   2.1 ms) 331.4 µs 493.2 µs 524.2 µs
WASMDatabase             14.7 ms          68.0 ( 12.7 ms …  20.0 ms)  15.3 ms  20.0 ms  20.0 ms
FFIDatabase             299.2 µs         3,342 (239.2 µs … 621.2 µs) 315.9 µs 452.9 µs 482.6 µs
LibSqlDatabase          355.7 µs         2,811 (246.6 µs …  29.3 ms) 335.0 µs 543.5 µs   3.4 ms

summary
  FFIDatabase
     1.04x faster than NodeDatabase
     1.19x faster than LibSqlDatabase
    49.15x faster than WASMDatabase

group select random data (indexed)
NodeDatabase              6.8 µs       147,200 (  6.5 µs …   7.2 µs)   6.9 µs   7.2 µs   7.2 µs
WASMDatabase             31.1 µs        32,120 ( 25.1 µs … 279.1 µs)  31.2 µs  56.8 µs  78.8 µs
FFIDatabase               7.0 µs       142,900 (  6.7 µs …   7.7 µs)   7.1 µs   7.7 µs   7.7 µs
LibSqlDatabase           12.8 µs        78,230 (  9.2 µs … 170.8 µs)  12.8 µs  19.7 µs  26.9 µs

summary
  NodeDatabase
     1.03x faster than FFIDatabase
     1.88x faster than LibSqlDatabase
     4.58x faster than WASMDatabase

group select random data (unindexed)
NodeDatabase             76.6 µs        13,060 ( 69.8 µs … 170.9 µs)  78.0 µs  97.5 µs 105.4 µs
WASMDatabase             32.6 µs        30,650 ( 30.1 µs … 482.2 µs)  32.7 µs  48.9 µs  59.8 µs
FFIDatabase              73.5 µs        13,600 ( 65.8 µs … 510.8 µs)  74.5 µs  95.5 µs 104.3 µs
LibSqlDatabase           67.4 µs        14,840 ( 60.4 µs … 182.5 µs)  68.6 µs  88.7 µs  99.6 µs

summary
  WASMDatabase
     2.06x faster than LibSqlDatabase
     2.25x faster than FFIDatabase
     2.35x faster than NodeDatabase

group insert then select (cache busting)
NodeDatabase            355.3 µs         2,814 (241.0 µs …  31.5 ms) 325.1 µs 530.1 µs   1.6 ms
WASMDatabase             15.3 ms          65.5 ( 12.8 ms …  20.1 ms)  15.8 ms  20.1 ms  20.1 ms
FFIDatabase             323.1 µs         3,095 (249.4 µs … 988.0 µs) 345.6 µs 543.6 µs 596.9 µs
LibSqlDatabase          336.6 µs         2,971 (262.2 µs …   6.1 ms) 346.5 µs 584.4 µs 783.5 µs

summary
  FFIDatabase
     1.04x faster than LibSqlDatabase
     1.10x faster than NodeDatabase
    47.28x faster than WASMDatabase

benchmark_results_wal_mode.log

Benchmarking sqlite drivers against PRAGMA journal_mode=WAL
    CPU | Apple M1 Max
Runtime | Deno 2.2.1 (aarch64-apple-darwin)

file:///Users/andrew.kaiser/Code/scratchwork/deno-sqlite-benchmark/bench2.ts

benchmark        time/iter (avg)        iter/s      (min … max)           p75      p99     p995
---------------- ----------------------------- --------------------- --------------------------

group insert random data
NodeDatabase             60.2 µs        16,610 ( 29.9 µs … 136.3 ms)  45.1 µs  77.0 µs  91.4 µs
WASMDatabase             14.4 ms          69.5 ( 12.8 ms …  21.1 ms)  14.9 ms  21.1 ms  21.1 ms
FFIDatabase              18.2 µs        55,060 ( 12.3 µs …   2.3 ms)  16.2 µs  40.0 µs  51.0 µs
LibSqlDatabase           78.5 µs        12,740 ( 37.6 µs …  43.7 ms)  62.5 µs 164.6 µs 330.2 µs

summary
  FFIDatabase
     3.32x faster than NodeDatabase
     4.32x faster than LibSqlDatabase
   791.70x faster than WASMDatabase

group select random data (indexed)
NodeDatabase              3.1 µs       323,500 (  2.9 µs …   3.4 µs)   3.3 µs   3.4 µs   3.4 µs
WASMDatabase             30.1 µs        33,180 ( 24.7 µs … 292.8 µs)  30.3 µs  50.5 µs  67.2 µs
FFIDatabase               3.4 µs       293,000 (  3.2 µs …   3.9 µs)   3.5 µs   3.9 µs   3.9 µs
LibSqlDatabase            8.6 µs       116,800 (  8.3 µs …   8.9 µs)   8.7 µs   8.9 µs   8.9 µs

summary
  NodeDatabase
     1.10x faster than FFIDatabase
     2.77x faster than LibSqlDatabase
     9.75x faster than WASMDatabase

group select random data (unindexed)
NodeDatabase            391.4 µs         2,555 (349.8 µs … 515.8 µs) 400.1 µs 467.1 µs 476.8 µs
WASMDatabase             32.6 µs        30,650 ( 29.8 µs … 380.0 µs)  33.0 µs  52.9 µs  65.6 µs
FFIDatabase               1.1 ms         928.6 (972.5 µs …   1.4 ms)   1.1 ms   1.3 ms   1.3 ms
LibSqlDatabase          257.9 µs         3,878 (233.5 µs … 384.0 µs) 264.2 µs 310.9 µs 335.2 µs

summary
  WASMDatabase
     7.90x faster than LibSqlDatabase
    11.99x faster than NodeDatabase
    33.00x faster than FFIDatabase

group insert then select (cache busting)
NodeDatabase             83.3 µs        12,000 ( 34.2 µs …  86.2 ms)  56.4 µs 135.7 µs 312.4 µs
WASMDatabase             16.4 ms          61.0 ( 13.9 ms …  22.1 ms)  17.1 ms  22.1 ms  22.1 ms
FFIDatabase              24.3 µs        41,220 ( 16.0 µs …   7.5 ms)  21.0 µs  46.5 µs  63.5 µs
LibSqlDatabase           65.4 µs        15,300 ( 46.9 µs …   6.6 ms)  64.8 µs 127.5 µs 159.8 µs

summary
  FFIDatabase
     2.69x faster than LibSqlDatabase
     3.43x faster than NodeDatabase
   676.10x faster than WASMDatabase

deno_sqlite_bench.ts

import * as path from '@std/path'
import * as sqlite_npm from 'better-sqlite3'
import * as sqlite_node from 'node:sqlite'
import * as sqlite_ffi from '@db/sqlite'
import * as sqlite_libsql from 'libsql'
import * as sqlite_wasm from 'https://deno.land/x/[email protected]/mod.ts'

type Params = Record<string, any>
type Result = Record<string, any>

abstract class Statement {
  abstract get(params: Params): Result | undefined
  abstract all(params: Params): Result[]
  abstract exec(params?: Params): void
  abstract close(): void
}
abstract class Database {
  constructor(database: string) {}
  abstract prepare(sql: string): Statement
  abstract close(): void

  exec(sql: string) {
    const stmt = this.prepare(sql)
    stmt.exec()
    stmt.close()
  }
}

class FFIStatement extends Statement {
  constructor(private stmt: sqlite_ffi.Statement) {
    super()
  }

  get(params: Params) {
    return this.stmt.get(params) as any
  }

  all(params: Params) {
    return this.stmt.all(params) as any
  }

  exec(params?: Params) {
    this.stmt.run(params)
  }

  close() {
    this.stmt.finalize()
  }
}
class FFIDatabase extends Database {
  #db: sqlite_ffi.Database

  constructor(database: string) {
    super(database)
    this.#db = new sqlite_ffi.Database(database)
  }
  prepare(sql: string) {
    return new FFIStatement(this.#db.prepare(sql))
  }

  close() {
    this.#db.close()
  }
}

class NodeStatement extends Statement {
  constructor(public stmt: sqlite_node.StatementSync) {
    super()
  }

  get(params: Params) {
    return this.stmt.get(params) as any
  }

  all(params: Params) {
    return this.stmt.all(params) as any
  }

  exec(params?: Params) {
    this.stmt.run(params ?? {})
  }

  close() {
    // NOTE it looks like node sqlite does not have a finalize/close method on statements?
  }
}
class NodeDatabase extends Database {
  #db: sqlite_node.DatabaseSync

  constructor(database: string) {
    super(database)
    this.#db = new sqlite_node.DatabaseSync(database)
  }
  prepare(sql: string) {
    return new NodeStatement(this.#db.prepare(sql))
  }

  close() {
    this.#db.close()
  }
}

class WASMStatement extends Statement {
  constructor(private stmt: sqlite_wasm.PreparedQuery) {
    super()
  }

  get(params: Params) {
    return this.stmt.firstEntry(params) as any
  }

  all(params: Params) {
    return this.stmt.allEntries(params) as any
  }

  exec(params?: Params) {
    this.stmt.execute(params)
  }

  close() {
    this.stmt.finalize()
  }
}
class WASMDatabase extends Database {
  #db: sqlite_wasm.DB

  constructor(database: string) {
    super(database)
    this.#db = new sqlite_wasm.DB(database)
  }
  prepare(sql: string) {
    return new WASMStatement(this.#db.prepareQuery(sql))
  }

  close() {
    this.#db.close()
  }
}

class LibSqlStatement extends Statement {
  constructor(private stmt: sqlite_libsql.Statement) {
    super()
  }

  get(params: Params) {
    return this.stmt.get(params) as any
  }

  all(params: Params) {
    return this.stmt.all(params) as any
  }

  exec(params?: Params) {
    this.stmt.run(params)
  }

  close() {
    // NOTE it looks like libsql does not have a finalize/close method on statements?
  }
}
class LibSqlDatabase extends Database {
  #db: sqlite_libsql.Database

  constructor(database: string) {
    super(database)
    this.#db = new sqlite_libsql.default(database)
  }
  prepare(sql: string) {
    return new LibSqlStatement(this.#db.prepare(sql))
  }

  close() {
    this.#db.close()
  }
}

function bench(group: string, name: string, fn: Deno.BenchDefinition['fn']) {
  Deno.bench({
    name: name,
    group: group,
    fn(ctx) {
      // ctx.start()
      fn(ctx)
      // ctx.end()
    }
  })
}

async function benchmark_drivers(options: {wal_mode: boolean}) {
  for (const driver_cls of sqlite_drivers) {
    const database_folder = path.join('temp', driver_cls.name)
    await Deno.remove(database_folder, {recursive: true}).catch(e => {
      if (e instanceof Deno.errors.NotFound) {}
      else throw e
    })
    await Deno.mkdir(database_folder, {recursive: true})
    const database_path = path.join(database_folder, 'sqlite.db')
    const driver = new driver_cls(database_path)

    driver.exec(`
CREATE table foobar (
  id INTEGER PRIMARY KEY NOT NULL,
  name TEXT NOT NULL,
  number INTEGER NOT NULL
)
`)
    driver.exec(`CREATE INDEX foobar_number ON foobar (number)`)

    if (options.wal_mode) {
      driver.exec(`PRAGMA journal_mode=WAL`)
    }

    const insert_stmt = driver.prepare(`INSERT INTO foobar (name, number) VALUES (:name, :number)`)
    const select_stmt = driver.prepare(`SELECT * FROM foobar WHERE number > :number LIMIT 1`)
    const select_unindexed_stmt = driver.prepare(`SELECT * FROM foobar WHERE name = :name LIMIT 1`)

    bench('insert random data', driver_cls.name, (ctx: Deno.BenchContext) => {
      const random = Math.random()
      insert_stmt.exec({name: `name-${random}`, number: random})
    })

    bench('select random data (indexed)', driver_cls.name, (ctx: Deno.BenchContext) => {
      let i = 0
      const random = Math.random()
      const row = select_stmt.get({number: random})
      i += row?.number
    })

    bench('select random data (unindexed)', driver_cls.name, (ctx: Deno.BenchContext) => {
      const random = Math.random()
      const row = select_unindexed_stmt.get({name: `name-${random}`})
      row?.name.split('-')
    })

    bench('insert then select (cache busting)', driver_cls.name, (ctx: Deno.BenchContext) => {
      const random = Math.random()
      insert_stmt.exec({name: `name-${random}`, number: random})
      const row = select_stmt.get({number: Math.random()})
      row?.number
    })
  }
}

const sqlite_drivers = [
  NodeDatabase,
  WASMDatabase,
  FFIDatabase,
  LibSqlDatabase,
]
await benchmark_drivers({wal_mode: false})

The full benchmarking suite is one file. To run this yourself, just run: deno bench -A --unstable-ffi --check deno_sqlite_bench.ts