sqlitedict vs plyvel(LevelDB)

sqlitedict_vs_plyvel.py

# -*- coding: utf-8 -*-
import time
import hashlib
import os
from contextlib import contextmanager
import shutil

import plyvel
from sqlitedict import SqliteDict


@contextmanager
def measure(task):
    print task
    t1 = time.time()
    yield
    t2 = time.time()
    print '    -> %fsec' % (t2 - t1)


def build_value(key):
    return hashlib.md5(str(key) + 'hoge').hexdigest()


def kv_gen(length):
    for i in xrange(length):
        key = str(i)
        value = build_value(key)
        yield (key, value)

def keys_gen(length):
    for i in xrange(length):
        key = str(i)
        yield key


def main():
    sqlitedict_f = '/tmp/benchmark_test.sqlite'
    sqlitedict_f2 = '/tmp/benchmark_test2.sqlite'
    sqlitedict_f3 = '/tmp/benchmark_test3.sqlite'
    sqlitedict_f4 = '/tmp/benchmark_test4.sqlite'
    leveldb_dir = '/tmp/benchmark_test.ldb'

    # n = 1000000
    # n = 100000
    n = 60000

    def clean_up():
        if os.path.exists(leveldb_dir):
            # os.removedirs(leveldb_dir)
            shutil.rmtree(leveldb_dir)

        sqlite_files = [
            sqlitedict_f, sqlitedict_f2, sqlitedict_f3,
            sqlitedict_f4
        ]
        for f in sqlite_files:
            if os.path.exists(f):
                os.remove(f)

    clean_up()

    with measure('Write: plyvel(LevelDB) N=%d' % n):
        my_db = plyvel.DB(leveldb_dir, create_if_missing=True)
        for k, v in kv_gen(length=n):
            my_db.put(k, v)
        my_db.close()

    with measure('Read: plyvel(LevelDB) N=%d' % n):
        my_db = plyvel.DB(leveldb_dir, create_if_missing=True)
        for key in keys_gen(length=n):
            my_db.get(key)
        my_db.close()

    with measure('Write: sqlitedb(commit once, no autocommit) N=%d' % n):
        mydict = SqliteDict(sqlitedict_f3, autocommit=False)
        for k, v in kv_gen(length=n):
            mydict[k] = v
        mydict.commit()
        mydict.close()

    with measure('Write: sqlitedb(commit every time, no autocommit) N=%d' % n):
        mydict = SqliteDict(sqlitedict_f4, autocommit=False)
        for k, v in kv_gen(length=n):
            mydict[k] = v
            mydict.commit()
        mydict.close()

    with measure('Write: sqlitedb(commit once, autocommit) N=%d' % n):
        mydict = SqliteDict(sqlitedict_f2, autocommit=True)
        for k, v in kv_gen(length=n):
            mydict[k] = v
        mydict.commit()
        mydict.close()

    with measure('Write: sqlitedb(commit every time, autocommit) N=%d' % n):
        mydict = SqliteDict(sqlitedict_f, autocommit=True)
        for k, v in kv_gen(length=n):
            mydict[k] = v
            mydict.commit()
        mydict.close()

    with measure('Read: sqlitedb(enabled autocommit) N=%d' % n):
        mydict = SqliteDict(sqlitedict_f, autocommit=True)
        for key in keys_gen(length=n):
            mydict[key]
        mydict.close()

    with measure('Read: sqlitedb(no autocommit) N=%d' % n):
        mydict = SqliteDict(sqlitedict_f, autocommit=False)
        for key in keys_gen(length=n):
            mydict[key]
        mydict.close()

    clean_up()


if __name__ == '__main__':
    main()

result on my laptop:

Write: plyvel(LevelDB) N=60000
-> 0.320133sec
Read: plyvel(LevelDB) N=60000
-> 0.211151sec
Write: sqlitedb(commit once, no autocommit) N=60000
-> 3.954300sec
Write: sqlitedb(commit every time, no autocommit) N=60000
-> 33.235961sec
Write: sqlitedb(commit once, autocommit) N=60000
-> 24.637719sec
Write: sqlitedb(commit every time, autocommit) N=60000
-> 32.528606sec
Read: sqlitedb(enabled autocommit) N=60000
-> 8.303143sec
Read: sqlitedb(no autocommit) N=60000
-> 8.337658sec