tabular_deltastore.py

import struct

def deltaify(list_of_tuples):
	return list_of_tuples[0], [ tuple( b-a for a,b in zip(tup1, tup2) ) for tup1, tup2 in zip(list_of_tuples[:-1], list_of_tuples[1:]) ]

smallest_format = lambda num: 'b' if num < 2**7 else 'h' if num < 2**15 else 'i' if num < 2**31 else 'q'
make_format = lambda es: ''.join(smallest_format(e) for e in es)

def pack(list_of_tuples):
	offx, deltas = deltaify(list_of_tuples)
	of, df = make_format(offx), make_format([max(ds) for ds in zip(*deltas)])
	return bytes((len(of),)) +\
			(of + df).encode() +\
			struct.pack(of, *offx) +\
			b''.join( struct.pack('!'+df, *tup) for tup in deltas )

def undeltaify(offx, deltas):
	st, rv = offx, [offx]
	for delta in deltas:
		st = tuple( a+b for a,b in zip(st, delta) )
		rv.append(st)
	return rv

def unpack(bytestring):
	# data format: [element count] {means format} {data format} {means} {data}
	n, bytestring = bytestring[0], bytestring[1:]
	of, df, bytestring = bytestring[:n], bytestring[n:2*n], bytestring[2*n:]
	offx   = struct.unpack_from(of.decode(), bytestring)
	deltas = list(struct.iter_unpack('!'+df.decode(), bytestring[ struct.calcsize(of): ]))
	return undeltaify(offx, deltas)