box.execute() measurements

.gitignore

tarantool-[abc]
log.txt
*.snap
*.xlog
*.csv

README.md

Overview

Benchmark box.execute() using SELECT 1 queries to verify #4231 implementations.

Experiment set up

Clone tarantool into three directories: tarantool-a, tarantool-b, tarantool-c.

tarantool-a

Tarantool 2.5.0-16-g9f2819785.

tarantool-b

The same as previous, but with Maria's patches from eljashm/gh-4231-box.execute-immutable-function upward (9383e5683c4df9bc9d3b04a2c0bf945b9dcb3bc1 and e2648197b0bb580a1d06033e5dff853b6b4a4569).

tarantool-c

The same as previous, but with caching of the value in Lua:

diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index 3ab43b000..de70c5c63 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -556,8 +556,18 @@ setmetatable(box, {
   end
 })

+local box_is_configured_cached = false
 local function box_is_configured()
-    return ffi.C.box_is_configured()
+    -- Using of cached Lua value is faster then ffi call.
+    if box_is_configured_cached then
+        return true
+    end
+
+    local res = ffi.C.box_is_configured()
+    if res == true then
+        box_is_configured_cached = true
+    end
+    return res
 end

 local function load_cfg(cfg)

How to build

$ cd tarantool-a
$ cmake .                             \
    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
    -DENABLE_BACKTRACE=ON             \
    -DENABLE_DIST=ON                  \
    -DENABLE_BUNDLED_LIBCURL=OFF      \
    -DENABLE_WERROR=ON                \
    && make -j

How to collect results

$ ./run.sh

Results will be saved into result_*.txt files.

Then figures can be plotted:

$ ./gen_csv.sh
$ gnuplot plot_avg.txt
$ gnuplot plot.txt

bench.lua

#!/usr/bin/env tarantool

local clock = require('clock')

box.cfg{log = 'log.txt'}

local start_time = clock.monotonic()
for i = 1, 10^6 do
    box.execute('SELECT 1')
end
local duration = clock.monotonic() - start_time
print(duration)

os.exit()

gen_csv.sh

#!/bin/sh

for var in a b c; do
    for i in $(seq 1 10); do
        result_file="result_${var}_${i}.txt"
        j=0
        while read line; do
            j=$((j+1))
            n=$((j+10*(i-1)))
            printf "${n},${line}\n"
        done < "${result_file}"
    done > "result_${var}.csv"
done

for var in a b c; do
    for i in $(seq 1 10); do
        result_file="result_${var}_${i}.txt"
        ./stat.awk < "${result_file}" | grep '^avg:' | awk '{ print "'"${i}"',"$2; }'
    done > result_avg_${var}.csv
done

plot.svg

<?xml version="1.0" encoding="utf-8"  standalone="no"?>
<svg 
 width="800" height="400"
 viewBox="0 0 800 400"
 xmlns="http://www.w3.org/2000/svg"
 xmlns:xlink="http://www.w3.org/1999/xlink"
>

<title>Gnuplot</title>
<desc>Produced by GNUPLOT 5.2 patchlevel 7 (Gentoo revision r0) </desc>

<g id="gnuplot_canvas">

<rect x="0" y="0" width="800" height="400" fill="none"/>
<defs>

	<circle id='gpDot' r='0.5' stroke-width='0.5' stroke='currentColor'/>
	<path id='gpPt0' stroke-width='0.541' stroke='currentColor' d='M-1,0 h2 M0,-1 v2'/>
	<path id='gpPt1' stroke-width='0.541' stroke='currentColor' d='M-1,-1 L1,1 M1,-1 L-1,1'/>
	<path id='gpPt2' stroke-width='0.541' stroke='currentColor' d='M-1,0 L1,0 M0,-1 L0,1 M-1,-1 L1,1 M-1,1 L1,-1'/>
	<rect id='gpPt3' stroke-width='0.541' stroke='currentColor' x='-1' y='-1' width='2' height='2'/>
	<rect id='gpPt4' stroke-width='0.541' stroke='currentColor' fill='currentColor' x='-1' y='-1' width='2' height='2'/>
	<circle id='gpPt5' stroke-width='0.541' stroke='currentColor' cx='0' cy='0' r='1'/>
	<use xlink:href='#gpPt5' id='gpPt6' fill='currentColor' stroke='none'/>
	<path id='gpPt7' stroke-width='0.541' stroke='currentColor' d='M0,-1.33 L-1.33,0.67 L1.33,0.67 z'/>
	<use xlink:href='#gpPt7' id='gpPt8' fill='currentColor' stroke='none'/>
	<use xlink:href='#gpPt7' id='gpPt9' stroke='currentColor' transform='rotate(180)'/>
	<use xlink:href='#gpPt9' id='gpPt10' fill='currentColor' stroke='none'/>
	<use xlink:href='#gpPt3' id='gpPt11' stroke='currentColor' transform='rotate(45)'/>
	<use xlink:href='#gpPt11' id='gpPt12' fill='currentColor' stroke='none'/>
	<path id='gpPt13' stroke-width='0.541' stroke='currentColor' d='M0,1.330 L1.265,0.411 L0.782,-1.067 L-0.782,-1.076 L-1.265,0.411 z'/>
	<use xlink:href='#gpPt13' id='gpPt14' fill='currentColor' stroke='none'/>
	<filter id='textbox' filterUnits='objectBoundingBox' x='0' y='0' height='1' width='1'>
	  <feFlood flood-color='white' flood-opacity='1' result='bgnd'/>
	  <feComposite in='SourceGraphic' in2='bgnd' operator='atop'/>
	</filter>
	<filter id='greybox' filterUnits='objectBoundingBox' x='0' y='0' height='1' width='1'>
	  <feFlood flood-color='lightgrey' flood-opacity='1' result='grey'/>
	  <feComposite in='SourceGraphic' in2='grey' operator='atop'/>
	</filter>
</defs>
<g fill="none" color="white" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,362.5 L29.9,362.5 M789.4,362.5 L785.7,362.5  '/>	<g transform="translate(22.7,364.2)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.15</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,323.1 L29.9,323.1 M789.4,323.1 L785.7,323.1  '/>	<g transform="translate(22.7,324.8)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.2</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,283.6 L29.9,283.6 M789.4,283.6 L785.7,283.6  '/>	<g transform="translate(22.7,285.3)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.25</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,244.2 L29.9,244.2 M789.4,244.2 L785.7,244.2  '/>	<g transform="translate(22.7,245.9)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.3</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,204.8 L29.9,204.8 M789.4,204.8 L785.7,204.8  '/>	<g transform="translate(22.7,206.5)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.35</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,165.3 L29.9,165.3 M789.4,165.3 L785.7,165.3  '/>	<g transform="translate(22.7,167.0)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.4</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,125.9 L29.9,125.9 M789.4,125.9 L785.7,125.9  '/>	<g transform="translate(22.7,127.6)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.45</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,86.5 L29.9,86.5 M789.4,86.5 L785.7,86.5  '/>	<g transform="translate(22.7,88.2)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.5</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,47.0 L29.9,47.0 M789.4,47.0 L785.7,47.0  '/>	<g transform="translate(22.7,48.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.55</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,7.6 L29.9,7.6 M789.4,7.6 L785.7,7.6  '/>	<g transform="translate(22.7,9.3)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text><tspan font-family="Verdana" > 2.6</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,362.5 L26.2,358.8 M26.2,7.6 L26.2,11.3  '/>	<g transform="translate(26.2,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 0</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M102.5,362.5 L102.5,358.8 M102.5,7.6 L102.5,11.3  '/>	<g transform="translate(102.5,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 10</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M178.8,362.5 L178.8,358.8 M178.8,7.6 L178.8,11.3  '/>	<g transform="translate(178.8,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 20</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M255.2,362.5 L255.2,358.8 M255.2,7.6 L255.2,11.3  '/>	<g transform="translate(255.2,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 30</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M331.5,362.5 L331.5,358.8 M331.5,7.6 L331.5,11.3  '/>	<g transform="translate(331.5,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 40</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M407.8,362.5 L407.8,358.8 M407.8,7.6 L407.8,11.3  '/>	<g transform="translate(407.8,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 50</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M484.1,362.5 L484.1,358.8 M484.1,7.6 L484.1,11.3  '/>	<g transform="translate(484.1,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 60</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M560.4,362.5 L560.4,358.8 M560.4,7.6 L560.4,11.3  '/>	<g transform="translate(560.4,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 70</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M636.8,362.5 L636.8,358.8 M636.8,7.6 L636.8,11.3  '/>	<g transform="translate(636.8,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 80</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M713.1,362.5 L713.1,358.8 M713.1,7.6 L713.1,11.3  '/>	<g transform="translate(713.1,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 90</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M789.4,362.5 L789.4,358.8 M789.4,7.6 L789.4,11.3  '/>	<g transform="translate(789.4,371.7)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="middle">
		<text><tspan font-family="Verdana" > 100</tspan></text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,7.6 L26.2,362.5 L789.4,362.5 L789.4,7.6 L26.2,7.6 Z  '/></g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
	<g id="gnuplot_plot_1" ><title>'result_a.csv'</title>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<g transform="translate(761.2,16.8)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text>'result_a.csv'</text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='rgb(148,   0, 211)'  d='M764.7,15.1 L782.4,15.1 M33.8,335.8 L41.5,322.3 L49.1,318.4 L56.7,358.7 L64.4,324.5 L72.0,325.9
		L79.6,360.1 L87.3,320.2 L94.9,330.8 L102.5,317.7 L110.2,289.0 L117.8,303.4 L125.4,304.7 L133.0,190.1
		L140.7,167.8 L148.3,240.2 L155.9,267.9 L163.6,264.5 L171.2,234.8 L178.8,289.8 L186.5,227.1 L194.1,246.5
		L201.7,262.0 L209.4,237.5 L217.0,228.5 L224.6,255.0 L232.3,210.3 L239.9,241.2 L247.5,237.3 L255.2,232.9
		L262.8,225.8 L270.4,244.2 L278.1,247.7 L285.7,179.2 L293.3,232.5 L301.0,232.3 L308.6,262.3 L316.2,204.0
		L323.8,241.2 L331.5,217.7 L339.1,261.7 L346.7,254.4 L354.4,242.1 L362.0,222.6 L369.6,225.4 L377.3,209.1
		L384.9,253.7 L392.5,239.5 L400.2,247.7 L407.8,209.4 L415.4,233.1 L423.1,220.3 L430.7,227.7 L438.3,203.3
		L446.0,209.5 L453.6,224.2 L461.2,242.7 L468.9,222.1 L476.5,213.7 L484.1,218.6 L491.8,211.3 L499.4,182.4
		L507.0,228.7 L514.6,203.5 L522.3,61.2 L529.9,217.8 L537.5,199.5 L545.2,210.1 L552.8,204.7 L560.4,192.4
		L568.1,178.1 L575.7,196.4 L583.3,221.1 L591.0,235.6 L598.6,209.8 L606.2,186.7 L613.9,200.5 L621.5,199.7
		L629.1,165.5 L636.8,211.4 L644.4,191.9 L652.0,191.1 L659.7,184.3 L667.3,211.9 L674.9,180.2 L682.6,201.6
		L690.2,209.8 L697.8,188.9 L705.4,194.8 L713.1,194.9 L720.7,146.5 L728.3,176.6 L736.0,186.9 L743.6,199.8
		L751.2,201.0 L758.9,213.4 L766.5,188.3 L774.1,194.1 L781.8,191.3 L789.4,195.5  '/></g>
	</g>
	<g id="gnuplot_plot_2" ><title>'result_b.csv'</title>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<g transform="translate(761.2,24.3)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text>'result_b.csv'</text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='rgb(  0, 158, 115)'  d='M764.7,22.6 L782.4,22.6 M33.8,345.0 L41.5,310.7 L49.1,219.8 L56.7,227.4 L64.4,319.8 L72.0,362.3
		L79.6,311.8 L87.3,338.1 L94.9,310.5 L102.5,325.4 L110.2,291.6 L117.8,268.0 L125.4,273.8 L133.0,270.4
		L140.7,284.7 L148.3,277.2 L155.9,253.5 L163.6,270.0 L171.2,276.1 L178.8,243.7 L186.5,243.3 L194.1,234.6
		L201.7,234.3 L209.4,232.0 L217.0,229.4 L224.6,212.1 L232.3,202.0 L239.9,221.7 L247.5,186.8 L255.2,226.0
		L262.8,248.4 L270.4,211.9 L278.1,192.2 L285.7,205.6 L293.3,245.0 L301.0,250.1 L308.6,237.0 L316.2,238.1
		L323.8,220.1 L331.5,227.3 L339.1,244.0 L346.7,200.4 L354.4,247.3 L362.0,242.0 L369.6,228.8 L377.3,242.5
		L384.9,247.1 L392.5,248.3 L400.2,242.0 L407.8,252.6 L415.4,185.1 L423.1,193.5 L430.7,217.1 L438.3,201.1
		L446.0,204.4 L453.6,226.1 L461.2,193.4 L468.9,219.6 L476.5,222.3 L484.1,207.5 L491.8,221.5 L499.4,184.8
		L507.0,216.2 L514.6,193.8 L522.3,206.3 L529.9,172.5 L537.5,200.9 L545.2,232.2 L552.8,204.2 L560.4,202.3
		L568.1,186.1 L575.7,188.0 L583.3,145.1 L591.0,197.0 L598.6,194.7 L606.2,214.1 L613.9,194.8 L621.5,189.9
		L629.1,195.4 L636.8,204.7 L644.4,184.1 L652.0,193.8 L659.7,162.7 L667.3,207.2 L674.9,189.6 L682.6,198.1
		L690.2,197.3 L697.8,180.2 L705.4,203.2 L713.1,178.9 L720.7,206.6 L728.3,189.1 L736.0,104.0 L743.6,192.8
		L751.2,205.0 L758.9,172.2 L766.5,217.7 L774.1,195.5 L781.8,201.7 L789.4,182.7  '/></g>
	</g>
	<g id="gnuplot_plot_3" ><title>'result_c.csv'</title>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<g transform="translate(761.2,31.8)" stroke="none" fill="black" font-family="Verdana" font-size="5.00"  text-anchor="end">
		<text>'result_c.csv'</text>
	</g>
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='rgb( 86, 180, 233)'  d='M764.7,30.1 L782.4,30.1 M33.8,302.4 L41.5,313.9 L49.1,296.9 L56.7,303.2 L64.4,304.8 L72.0,297.8
		L79.6,285.7 L87.3,296.4 L94.9,297.5 L102.5,264.8 L110.2,141.7 L117.8,267.0 L125.4,270.3 L133.0,258.9
		L140.7,255.8 L148.3,238.1 L155.9,253.0 L163.6,252.4 L171.2,259.9 L178.8,271.1 L186.5,242.7 L194.1,238.4
		L201.7,221.4 L209.4,140.8 L217.0,222.9 L224.6,246.0 L232.3,249.8 L239.9,206.7 L247.5,231.8 L255.2,250.9
		L262.8,241.0 L270.4,211.8 L278.1,232.3 L285.7,234.3 L293.3,244.6 L301.0,233.6 L308.6,233.6 L316.2,250.0
		L323.8,240.9 L331.5,267.4 L339.1,237.5 L346.7,228.0 L354.4,239.6 L362.0,244.3 L369.6,232.5 L377.3,237.9
		L384.9,197.1 L392.5,199.0 L400.2,224.5 L407.8,227.3 L415.4,195.4 L423.1,220.9 L430.7,203.8 L438.3,218.4
		L446.0,189.7 L453.6,188.5 L461.2,213.8 L468.9,221.5 L476.5,216.8 L484.1,218.1 L491.8,186.8 L499.4,210.2
		L507.0,201.3 L514.6,215.2 L522.3,221.1 L529.9,184.4 L537.5,199.9 L545.2,236.2 L552.8,195.7 L560.4,208.8
		L568.1,158.1 L575.7,176.6 L583.3,168.8 L591.0,195.1 L598.6,200.5 L606.2,184.9 L613.9,218.2 L621.5,195.8
		L629.1,173.4 L636.8,218.7 L644.4,205.2 L652.0,220.7 L659.7,178.0 L667.3,201.5 L674.9,189.8 L682.6,190.9
		L690.2,166.4 L697.8,167.4 L705.4,195.6 L713.1,189.7 L720.7,198.5 L728.3,190.4 L736.0,190.6 L743.6,39.1
		L751.2,149.1 L758.9,199.5 L766.5,177.8 L774.1,211.1 L781.8,191.0 L789.4,202.4  '/></g>
	</g>
<g fill="none" color="white" stroke="rgb( 86, 180, 233)" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="black" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
	<path stroke='black'  d='M26.2,7.6 L26.2,362.5 L789.4,362.5 L789.4,7.6 L26.2,7.6 Z  '/></g>
<g fill="none" color="black" stroke="currentColor" stroke-width="0.50" stroke-linecap="butt" stroke-linejoin="miter">
</g>
</g>
</svg>

plot.txt

set terminal svg enhanced linewidth 0.5 font 'Verdana,5' size 800,400
set output 'plot.svg'

set datafile separator ","

set bmargin 5

plot 'result_a.csv' with lines, \
    'result_b.csv' with lines, \
    'result_c.csv' with lines

plot_avg.svg

plot_avg.txt

set terminal svg enhanced linewidth 0.5 font 'Verdana,5' size 800,400
set output 'plot_avg.svg'

set datafile separator ","

set bmargin 5

plot 'result_avg_a.csv' with lines, \
    'result_avg_b.csv' with lines, \
    'result_avg_c.csv' with lines

result_a_1.txt

2.183851675014
2.2010253299959
2.2059130580164
2.1548372660764
2.1981472680345
2.1964296519291
2.1530008020345
2.203642737004
2.1902553950204
2.2068677869393

result_a_10.txt

2.4238469790434
2.3857251110021
2.3726613880135
2.3562455849024
2.3547751719598
2.3390066369902
2.3709119899431
2.3635417009937
2.3670372019988
2.3617378970375

result_a_2.txt

2.2431526610162
2.2249788320623
2.2233266300755
2.3686036400031
2.3968613180332
2.3051012769574
2.2699106959626
2.2742626300314
2.3118613419356
2.2421325709438

result_a_3.txt

2.3216244590003
2.2970892440062
2.2774457340129
2.308553775074
2.3199673019117
2.2862964730011
2.3430129119661
2.30376385001
2.3088064539479
2.3143525889609

result_a_4.txt

2.3233553649625
2.3000423880294
2.2956064089667
2.3823762529064
2.3148005449912
2.3151135300286
2.2770598960342
2.3510061210254
2.3037679110421
2.3336005699821

result_a_5.txt

2.2778507500188
2.2870792039903
2.3026154020336
2.3273941009538
2.3238671029685
2.3444810310612
2.287918947055
2.3059889629949
2.2955966240261
2.3441700520925

result_a_6.txt

2.3140660169302
2.3302840030519
2.3209050010191
2.3518842019839
2.34396985895
2.3253920850111
2.3019648719346
2.3279942299705
2.3386879099999
2.3324496719288

result_a_7.txt

2.3416609120322
2.3784086280502
2.319693802041
2.3515759140719
2.5320774890715
2.3334857169539
2.3566832549404
2.3432298999978
2.3501163759502
2.3657067290042

result_a_8.txt

2.3837763059419
2.360636768979
2.3292783709476
2.3109147050418
2.3435845959466
2.3728822550038
2.3554537569871
2.3564220459666
2.3998506349744
2.341564183007

result_a_9.txt

2.3663268119562
2.3673711690353
2.3759915729752
2.3409229828976
2.3811626080424
2.3540755830472
2.3436625490431
2.3701225990662
2.3626037570648
2.3625621440588

result_b_1.txt

2.1721468300093
2.2157173969317
2.3309524039505
2.3213202729821
2.2041695190128
2.1503148599295
2.2143290729728
2.1809951791074
2.2159951870563
2.1970973620191

result_b_10.txt

2.3476838479983
2.3698919869494
2.4778261779575
2.3651461750269
2.3496906630462
2.3912773960037
2.3335878551006
2.3617814480094
2.3539334990783
2.3780213490827

result_b_2.txt

2.2398517410038
2.2698501150589
2.2624415040482
2.2667167090112
2.2486715570558
2.2581154530635
2.288227356039
2.2673271680251
2.2595223239623
2.3006926330272

result_b_3.txt

2.3011557710124
2.3122267080471
2.3125644159736
2.315487607033
2.3187415889697
2.3406896529486
2.3534616389079
2.3285531690344
2.3727233769605
2.323025865946

result_b_4.txt

2.2946393380407
2.3409075349336
2.3659313800745
2.3489935430698
2.2989957339596
2.2925689490512
2.3091021151049
2.3077802379848
2.3305225620279
2.321394560975

result_b_5.txt

2.3001921509858
2.3555914819008
2.2960486439988
2.3028342280304
2.319501243066
2.302100069006
2.2963371230289
2.2947819929104
2.3028026969405
2.2893871699926

result_b_6.txt

2.3749364509713
2.3642615820281
2.3343368470669
2.3546078240033
2.3504951969953
2.3229639199562
2.3644168049796
2.3312463210896
2.3277274060529
2.3465569580439

result_b_7.txt

2.3287260549841
2.3752686119406
2.3354461489944
2.3639543890022
2.3480944569455
2.390899463906
2.3549479009816
2.315207852982
2.3507516619284
2.3530765089672

result_b_8.txt

2.3736338580493
2.3712129920023
2.4255959839793
2.3598586380249
2.3627250340069
2.3381837169873
2.3626922300318
2.3688410969917
2.3619190879399
2.3500282989116

result_b_9.txt

2.3761640820885
2.3638811931014
2.4033743360778
2.3469399219612
2.3692381230649
2.3584292650921
2.3595145659056
2.381172241061
2.351987412083
2.3827444280032

result_c_1.txt

2.2262576729991
2.2116516559618
2.2331773809856
2.225227690069
2.2231981629739
2.2320844889618
2.2474297389854
2.2337648599641
2.2324356409954
2.2738552379888

result_c_10.txt

2.3579911000561
2.3682610290125
2.3679134659469
2.5600304179825
2.4205272969557
2.3566284120316
2.3842005199986
2.3420283310115
2.3674776169937
2.3530219569802

result_c_2.txt

2.4299559619976
2.2710726889782
2.2668603280326
2.2813214350026
2.2853040259797
2.307692958042
2.288833115017
2.2896101720398
2.2800726820715
2.265849368996

result_c_3.txt

2.3019344470231
2.3073219190119
2.32897284301
2.4311254660133
2.3270523620304
2.2977166129276
2.292844603071
2.3474997379817
2.3157216929831
2.291511913063

result_c_4.txt

2.3040684879525
2.3410955670988
2.3151469100267
2.3125061660539
2.2995461549144
2.3134762439877
2.3134062360041
2.2926461489405
2.3042298349319
2.2706114730099

result_c_5.txt

2.3084959780099
2.3205928140087
2.3058324039448
2.2998834209284
2.3147991379956
2.3079335809452
2.3597684049746
2.3573643629206
2.3249161699787
2.3214313089848

result_c_6.txt

2.3618965820642
2.3294959800551
2.3512242659926
2.3327103459742
2.3690591719933
2.3706771460129
2.3385903210146
2.3287489770446
2.3347806980601
2.3331263510045

result_c_7.txt

2.3727868050337
2.3430765660014
2.3544429650065
2.3367907009088
2.3293382209959
2.3758684400236
2.3561903620139
2.3101701360429
2.361468355055
2.3448743249755

result_c_8.txt

2.4091879209736
2.3856923410203
2.3955610980047
2.3622817189898
2.3553720399505
2.3751690589124
2.3330018620472
2.3613747929921
2.389731174917
2.3323583758902

result_c_9.txt

2.3494517450454
2.3297445769422
2.3839998419862
2.3541212070268
2.3690287499921
2.3676315339981
2.3986981040798
2.3974225240527
2.3616663200082
2.3690825259546

run.sh

#!/bin/sh

for i in $(seq 1 10); do
    for var in a b c; do
        result_file="result_${var}_${i}.txt"
        for j in $(seq 1 10); do
            rm -f *.{xlog,snap}
            taskset 0x2 ./tarantool-${var}/src/tarantool bench.lua >> "${result_file}"
        done
    done
done

stat.awk

#!/usr/bin/awk -f

BEGIN {
    min = "NaN"
    max = "NaN"
    cnt = 0
    var = "NaN"
    sum = "NaN"
    avg = "NaN"
}
{
    min = (NR==1 || $1<min ? $1 : min)
    max = (NR==1 || $1>max ? $1 : max)
    cnt += 1
    sum += $1
}
END {
    avg = sum / cnt
    var = max - min
    var_rel = 100 * var / avg
    printf "cnt: %d\n", cnt
    printf "min: %.6f\n", min
    printf "max: %.6f\n", max
    printf "avg: %.6f\n", avg
    printf "var: %.6f (%.6f%)\n", var, var_rel
}