tarantool memory eviction issue

test.lua

local log_level = 5

box.cfg {
    username = nil;
    work_dir = nil;
    wal_dir = "/tmp";
    snap_dir = "/tmp";
    vinyl_dir = "/tmp";
    listen = 3301;
    --pid_file = "s.pid";
    pid_file = "/run/tarantooltest.pid";
    background = false;
    slab_alloc_arena = 1;
    slab_alloc_minimal = 8;
    slab_alloc_maximal = 1048576 + 200;

    slab_alloc_factor = 1.2;
    snapshot_period = 0;
    snapshot_count = 2;
    panic_on_snap_error = true;
    panic_on_wal_error = true;
    rows_per_wal = 500000;
    snap_io_rate_limit = nil;
    wal_mode = "none";
    wal_dir_rescan_delay = 2.0;
    io_collect_interval = nil;
    readahead = 16320;
    log_level = log_level;
    logger_nonblock = true;
    too_long_threshold = 0.5;
}


box.schema.user.grant('guest', 'read,write,execute', 'universe')

local spacename = 'meta'
box.schema.space.create(spacename, { if_not_exists = true })
box.space[spacename]:create_index('primary', { type = 'HASH', parts = { 1, 'string' }, unique = true, if_not_exists = true })
box.space[spacename]:create_index('secondary', { type = 'TREE', parts = { 2, 'number' }, unique = true, if_not_exists = true })


local spacename2 = 'blob'
box.schema.space.create(spacename2, { if_not_exists = true })
box.space[spacename2]:create_index('primary', { type = 'HASH', parts = { 1, 'number' }, unique = true, if_not_exists = true })
box.space[spacename2]:create_index('weight', { type = 'TREE', parts = { 2, 'number' }, unique = false, if_not_exists = true })

local log = require('log')
local fiber = require('fiber')

spaces_len = function()
    local len = 0
    for sid, s in pairs(box.space) do
        if type(sid) == 'number' and sid >= 512 then
            len = len + s:len()
        end
    end
    return len
end

index_size_all = function()
    local res = 0 for sid, s in pairs(box.space) do
        for iid, i in pairs(s.index) do if type(sid) == 'number' and type(iid) == 'number' then
            res = res + i:bsize()
        end
        end
    end
    return res
end

free_mem = function()
    local info = box.slab.info()
    return 100 * ((info.quota_size - info.arena_size) / info.quota_size)
end

slab_stats = function()
    local s = box.slab.stats()
    local sum = 0
    for _, slab in pairs(s) do
        sum = sum + (slab.slab_size * slab.slab_count)
    end

    return { sum, box.slab.info(), s, index_size_all(), spaces_len(), free_mem() }
end


local function logwarn(text, ...)

    local x = debug.getinfo(2)
    local dbg = string.format('[%s:%d][%s]', x.source, x.currentline, x.name)
    dbg = dbg:gsub("/usr/local/share/tarantool/", "")

    local args = ''
    local st, d = pcall(function(...)
        return ' ' .. require('json').encode({ ... })
    end, ...)

    if st then
        args = d
    else
        args = ' args to json convert fail'
    end
    log.warn("%s", dbg .. ' - ' .. '::' .. ' - ' .. text .. args)
end

local function logwarnyml(text, ...)

    local x = debug.getinfo(2)
    local dbg = string.format('[%s:%d][%s]', x.source, x.currentline, x.name)
    dbg = dbg:gsub("/usr/local/share/tarantool/", "")

    local args = ''
    local st, d = pcall(function(...)
        return ' ' .. require('yaml').encode({ ... })
    end, ...)

    if st then
        args = d
    else
        args = ' args to json convert fail'
    end
    log.warn("%s", dbg .. ' - ' .. '::' .. ' - ' .. text .. args)
end


local function eviction(iter)


    if iter > 100 then
        logwarn('eviction fail:', iter)
        logwarn('box.slab.info() ', box.slab.info())
        return false
    end

    local count = box.space.blob.index.primary:count()

    local c = math.floor(count * (iter / 500))

    --    logwarn('do eviction count:', iter, c)
    --    logwarn('before eviction', count)

    for _, t in box.space.blob.index.weight:pairs({ 0 }, { iterator = box.index.GE }) do

        box.space.blob:delete(t[1])
        c = c - 1
        if c <= 0 then
            break
        end
    end
    collectgarbage('collect')
    --TODO

    return true
end


local errhandler = function(err)

--    log.error("pcalleviction:  handler error: %s", err)
--    log.error("%s", debug.traceback())
--    logwarn('errhandler: slab stats ', err, slab_stats())
    return err
end

local function pcalleviction(func, ...)
    local st
    local res
    local exit = true
    local c = 0
    local errfound = nil

    repeat
        box.begin()
        st, res = xpcall(func, errhandler, ...)
        if not st then
            box.rollback()

            local err = box.error.last()

            if err ~= nil and err.type ~= nil and err.type == 'OutOfMemory' then

                if errfound == nil then
                    errfound = slab_stats()
                    errfound[7] = err
                end
                --                logwarn('error OutOfMemory', err, 'free_mem=' .. free_mem())
                --                    haserr = true
                if not eviction(c) then
                    logwarn('Eviction refusing')

                    return false, false
                end

                --                    dd('slab_stats()', slab_stats()) -- debug

                exit = false
                c = c + 1
            else

                return false, res
            end
        else

            box.commit()
            exit = true


            if errfound ~= nil then
                local after = slab_stats()


                local free_mem_before = errfound[6]
                local free_mem_after = after[6]

                if (free_mem_after - free_mem_before > 20) then

                    logwarn('')
                    logwarn('____________')
                    logwarn('FATAL ERROR:')
                    logwarn('', errfound[7])
                    logwarn('')
                    logwarn('FREE BEFORE:', free_mem_before)
                    logwarn('TRANSACTION WAS COMMITED ONLY WNEN FREE MORE THEN :', free_mem_after)
                    logwarn('____________')
                    logwarn('-- desc:')
                    logwarn('')
                    logwarnyml('state before eviction', errfound)
                    logwarnyml('state after eviction', after)

                end
            end
        end
    until exit

    return st, res
end


local seq = 0

local nextID = function()
    seq = seq + 1
    return seq
end

local function transaction(dataid, data, dataname)

    local weight = math.random(100000)

    --any order
    box.space.blob:insert({ dataid, weight, data })
    box.space.meta:insert({ dataname, dataid })
end

local data = '12334567890'
for j = 1, 2000 do
    data = data .. '_'
end

function test1()
    local iterations = 720000
    local c = iterations
    while c > 0 do
        c = c - 1
        local dataid = nextID()
        local dataname = 'dataname' .. dataid

        local st, res = pcalleviction(transaction, dataid, data, dataname)
        if st then
        else
            if res == false then
                logwarn('writeLog OutOfMemory')
            end
            logwarn('log insert fail', res)
            return false
        end
    end

    if box.space.meta:len() == iterations then
        logwarn('meta:len = iterations')
    else
        logwarn('fail')
    end
end


test1()