graphite serialization perf

gistfile1.txt

# Test run output
# $ python test.py 
# Serializer: test_json_buffer Payload Size: 80000
# 0.0628697872162
# Serializer: test_json_stream Payload Size: 80000
# 0.0611701011658
# Serializer: test_pickle_buffer Payload Size: 90006
# 0.688886880875
# Serializer: test_pickle_stream Payload Size: 90006
# 17.879488945
# Serializer: test_umsgpack_buffer Payload Size: 90003
# 0.735457181931
# Serializer: test_umsgpack_stream Payload Size: 90003
# 4.34650301933

import umsgpack
import timeit
import random
import json
import pickle
import io

random.seed(100)
TEST_DATA = [float(random.randint(0, 10000))] * 10000
TEST_JSON_DATA = json.dumps(TEST_DATA)
TEST_PICKLE_DATA = pickle.dumps(TEST_DATA)
TEST_UMSGPACK_DATA = umsgpack.dumps(TEST_DATA)


class BufferedIO(io.IOBase):
    def __init__(self, data):
        self.data = data
        self.pos = 0

    def read(self, size=None):
        if not size:
            return self.data

        if len(self.data) == 0:
            return ''
        result = self.data[:size]
        self.data = self.data[size:]
        self.pos += len(result)
        return result


def test_pickle_buffer():
    pickle.loads(TEST_PICKLE_DATA)


def test_pickle_stream():
    pickle.load(BufferedIO(TEST_PICKLE_DATA))


def test_umsgpack_buffer():
    umsgpack.unpackb(TEST_UMSGPACK_DATA)


def test_umsgpack_stream():
    umsgpack.unpack(BufferedIO(TEST_UMSGPACK_DATA))


def test_json_buffer():
    json.loads(TEST_JSON_DATA)


def test_json_stream():
    json.load(BufferedIO(TEST_JSON_DATA))


def measure(name, f, payload_size):
    print("Serializer: " + name + " Payload Size: " + str(payload_size))
    print(timeit.timeit(f, number=100))


def main():
    measure("test_json_buffer", test_json_buffer, len(TEST_JSON_DATA))
    measure("test_json_stream", test_json_stream, len(TEST_JSON_DATA))
    measure("test_pickle_buffer", test_pickle_buffer, len(TEST_PICKLE_DATA))
    measure("test_pickle_stream", test_pickle_stream, len(TEST_PICKLE_DATA))
    measure("test_umsgpack_buffer", test_umsgpack_buffer, len(TEST_UMSGPACK_DATA))
    measure("test_umsgpack_stream", test_umsgpack_stream, len(TEST_UMSGPACK_DATA))


if __name__ == "__main__":
    main()