Serialize benchmarks BSON vs STON vs NeoJSON vs Fuel vs MCDataStream

benchmarks.md

mockOne := [ |map| 
	map := Dictionary new.
	32 timesRepeat: [ 
    map 
        at: 1000 atRandom asWords asSymbol 
        put: { 
            2 atRandom = 1. 
            1e9 atRandom. 
            1e9 atRandom negated. 
            1000 atRandom asWords.
            1e6 atRandom seconds humanReadablePrintString.  
            nil. 
            DateAndTime now printString. 
            (-1000 to: 1000) atRandom * Float pi } atRandom ].
	map ].
mocks := (1 to: 1000) collect:[:i| mockOne value].

randomGenerator := SecureRandom new.
getOne := [ mocks atRandom: randomGenerator ].
one := getOne value.
fuelSerializer := FLSerializer newDefault.
dataStream := [:oject| MCDataStream streamedRepresentationOf: object].

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			aStream := WriteStream on: (ByteArray new: 100).
			fuelSerializer serialize: one on: aStream.
			aStream contents
			 ].
	].
 "B is 65.26% SLOWER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			STON toString: one.
			 ].
	].
 "B is 33.07% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			NeoJSONWriter toString: one.
			 ].
	].
 "B is 89.40% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			dataStream value: one ] 
		b: [ 
			one := getOne value. 
			STON toString: one.
			 ].
	].
  "B is 88.26% SLOWER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			NeoJSONWriter toString: one. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	].
 "B is 476.56% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	].
 "B is 979.82% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [
			one := getOne value. 
			aStream := WriteStream on: (ByteArray new: 100).
			fuelSerializer serialize: one on: aStream. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	].
 "B is 3313.03% FASTER than A"

Thanks for taking over my sample. I will study this in detail and report back. I have little time today/tomorrow, we'll see.

This is quite interesting though, I really want to understand this better.

In my tests ((Pharo 11, macOS 13.0.1, Intel i5 MacBook Pro), NeoJSONWriter is faster than MCDataStream:

mappings := OrderedCollection new.
1000 timesRepeat: [ 
	| map |
	map := Dictionary new.
	32 timesRepeat: [ 
	map 
		at: 1000 atRandom asWords asSymbol 
		put: { 
			2 atRandom = 1. 
			1e9 atRandom. 
			1e9 atRandom negated. 
			1000 atRandom asWords.
			1e6 atRandom seconds humanReadablePrintString.  
			nil. 
			DateAndTime now printString. 
			(-1000 to: 1000) atRandom * Float pi } atRandom ].
	mappings add: map ].
 
[ FLSerializer serializeToByteArray: mappings atRandom ] bench. "'644.071 per second'"
[ STON toString: mappings atRandom ] bench.  "'13094.144 per second'"
[ NeoJSONWriter toString: mappings atRandom ] bench.  "'21755.946 per second'"
[ MCDataStream streamedRepresentationOf: mappings atRandom ] bench.  "'15461.723 per second'"

Most welcome, thanks for caring to take a look.
No hurries at all.
And good detail @svenvc , I've forgot to mention the Pharo version I've used in these benchs, it was:

Pharo 7.0.5
Build information: Pharo-7.0.5+build.177.sha.fd8c156d959653f52be6fd7f8c71edf8cfb8fe90 (64 Bit)

macOS 12.6
2,5 GHz Quad-Core Intel Core i7

I'll benchmark the same again in Pharo 10 and follow up here

@svenvc this was on:

macOS 12.6
2,5 GHz Quad-Core Intel Core i7

Pharo 10.0.0
Build information: Pharo-10.0.0+build.533.sha.1264c724db69db4b887e08173cb286010ce413fb (64 Bit)


➜  matter ./pharo-ui -version                       
Pharo 9.0.20 built on Oct  5 2022 14:57:44 Compiler: 4.2.1 Compatible Apple LLVM 11.0.3 (clang-1103.0.32.29)
Built from: v9.0.20 - Commit: 2f226ea2 - Date: 2022-10-05 14:39:39 +0200

mockOne := [ |map| 
	map := Dictionary new.
	32 timesRepeat: [ 
    map 
        at: 1000 atRandom asWords asSymbol 
        put: { 
            2 atRandom = 1. 
            1e9 atRandom. 
            1e9 atRandom negated. 
            1000 atRandom asWords.
            1e6 atRandom seconds humanReadablePrintString.  
            nil. 
            DateAndTime now printString. 
            (-1000 to: 1000) atRandom * Float pi } atRandom ].
	map ].
mocks := (1 to: 1000) collect:[:i| mockOne value].

randomGenerator := Random new.
getOne := [ mocks atRandom: randomGenerator ].
one := getOne value.
fuelSerializer := FLSerializer newDefault.
dataStream := [:oject| MCDataStream streamedRepresentationOf: object].

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			aStream := WriteStream on: (ByteArray new: 100).
			fuelSerializer serialize: one on: aStream.
			aStream contents
			 ].
	]. 
"B is 70.16% SLOWER than A" 


ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			STON toString: one.
			 ].
	]. 
 "B is 7.61% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			NeoJSONWriter toString: one.
			 ].
	].
 "B is 56.27% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			dataStream value: one ] 
		b: [ 
			one := getOne value. 
			STON toString: one.
			 ].
	].
 "B is 98.68% SLOWER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			NeoJSONWriter toString: one. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	].
 "B is 4867.22% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	].
 "B is 8121.50% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [
			one := getOne value. 
			aStream := WriteStream on: (ByteArray new: 100).
			fuelSerializer serialize: one on: aStream. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	]. 
 "B is 28184.82% FASTER than A"

I don't know what ABBench is, so I can't run it, but

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			NeoJSONWriter toString: one. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	].
 "B is 4867.22% FASTER than A"

is in total conflict with what I see, for me NeoJSON is about 50% faster than MCDataStream, not 4000% slower. Something is wrong here.

Why don't you try running my code as Yanni did it Discord (where he repeated my result more or less), it has no further dependencies.

Ah @svenvc I thought you were also using it.

Here it is for your reference:
https://github.com/emdonahue/ABBench

It might be something with the test itself?

I'll take samples again but not using ABBench, and use the closure bench method instead.

Intrigued.

Hi @sebastianconcept,

Running my blocks with ABBench gives this (which conforms to my previous measurements):

ABBench bench: [ 
	ABBench 
		a: [ MCDataStream streamedRepresentationOf: mappings atRandom ]
		b: [ NeoJSONWriter toString: mappings atRandom ] ].
 "B is 53.53% FASTER than A"	
	
ABBench bench: [ 
	ABBench 
		a: [ NeoJSONWriter toString: mappings atRandom ]
		b: [ MCDataStream streamedRepresentationOf: mappings atRandom ] ].
 "B is 31.35% SLOWER than A"

And I think I found the problem: you made a typo in the dataStream block:

dataStream := [:oject | MCDataStream streamedRepresentationOf: object].

it has to be:

dataStream := [:object | MCDataStream streamedRepresentationOf: object].

In the first case, the actual parameter is ignored and object is nil, hence that is serialised as just a single byte #[1], of course this is very fast ;-)

🤦🏻‍♂️ Oh boy... that was nasty! 😂
Great catch @svenvc

so... NeoJSONWriter will be the preferred one then! And by a lot!

[ one := getOne value. NeoJSONWriter toString: one ] bench. "'15433.940 per second'"
[ one := getOne value. dataStream value: one ] bench. "'8704.977 per second'"

And correcting previous tests:

mockOne := [ |map| 
	map := Dictionary new.
	32 timesRepeat: [ 
    map 
        at: 1000 atRandom asWords asSymbol 
        put: { 
            2 atRandom = 1. 
            1e9 atRandom. 
            1e9 atRandom negated. 
            1000 atRandom asWords.
            1e6 atRandom seconds humanReadablePrintString.  
            nil. 
            DateAndTime now printString. 
            (-1000 to: 1000) atRandom * Float pi } atRandom ].
	map ].
mocks := (1 to: 1000) collect:[:i| mockOne value].

randomGenerator := Random new.
getOne := [ mocks atRandom: randomGenerator ].
one := getOne value.
fuelSerializer := FLSerializer newDefault.
dataStream := [ :object | MCDataStream streamedRepresentationOf: object ].

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			aStream := WriteStream on: (ByteArray new: 100).
			fuelSerializer serialize: one on: aStream.
			aStream contents
			 ].
	]. 
 "B is 71.35% SLOWER than A"


ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			STON toString: one.
			 ].
	]. 
 "B is 14.45% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			NeoJSONWriter toString: one.
			 ].
	].
 "B is 63.02% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			dataStream value: one ] 
		b: [ 
			one := getOne value. 
			STON toString: one.
			 ].
	].
 "B is 5.15% SLOWER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			NeoJSONWriter toString: one. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	].
 "B is 33.39% SLOWER than A"

ABBench bench: [ 
	ABBench 
		a: [ one := getOne value. 
			serialized := BSON write: one. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	].
 "B is 12.72% FASTER than A"

ABBench bench: [ 
	ABBench 
		a: [
			one := getOne value. 
			aStream := WriteStream on: (ByteArray new: 100).
			fuelSerializer serialize: one on: aStream. ] 
		b: [ 
			one := getOne value. 
			dataStream value: one.
			 ].
	]. 
 "B is 332.32% FASTER than A"