MariaDB semi-sync hang reproduction scripts

README.md

Download scripts to directory and execute in this order:

1. master.sh
2. slave.sh
3. trigger.sh

The trigger script will download a smallish dump from the Wikimedia dataset which is large enough to trigger tha hang reliably with the smaller binlog settings.

master.sh

#!/bin/bash

set -e

PORT="3306"
DATA_DIR="$PWD/master"

exec_root() {
    /usr/bin/mysql --host 127.0.0.1 --port $PORT --user root -e "$1"
}

# reset data dir
rm -rf $DATA_DIR
mkdir $DATA_DIR

# write config
cat > master.cnf << EOF
[client]
port = $PORT

[mysqld]
user = ""
port = $PORT
bind_address = 127.0.0.1
server_id = 1
socket = ""
pid_file = $DATA_DIR/mysql.pid
report_host = 127.0.0.1

datadir = $DATA_DIR
log_bin = $DATA_DIR/mariadb-bin
log_bin_index = $DATA_DIR/mariadb-bin.index
log_slave_updates = 1
relay_log = $DATA_DIR/relay-bin
relay_log_index = $DATA_DIR/relay-bin.index
max_binlog_size = 1048576
EOF

# install db
echo "running mysql_install_db"
/usr/bin/mysql_install_db --defaults-extra-file=master.cnf > /dev/null 2>&1

# start server
echo "starting server"
/usr/sbin/mysqld --defaults-extra-file=master.cnf > master.log 2>&1 &

# wait for server to start
echo "waiting for server to start"
sleep 5

# create repl user
echo "creating repl user"
exec_root "CREATE USER 'repl_user'@'%' IDENTIFIED BY 'repl_password'"
exec_root "GRANT ALL ON *.* TO 'repl_user'@'%' WITH GRANT OPTION"

# install semi-sync master plugin
echo "installing semi-sync master plugin"
exec_root "INSTALL PLUGIN rpl_semi_sync_master SONAME 'semisync_master.so'"

# flush privledges
echo "flush privleges"
exec_root "FLUSH PRIVILEGES"

# set semi-sync variables
echo "setup semi-sync settings"
exec_root "SET GLOBAL rpl_semi_sync_master_wait_point = AFTER_SYNC"
exec_root "SET GLOBAL rpl_semi_sync_master_timeout = 18446744073709551615"
exec_root "SET GLOBAL rpl_semi_sync_master_enabled = 1"
exec_root "SET GLOBAL rpl_semi_sync_master_wait_no_slave = 1"
exec_root "SET GLOBAL rpl_semi_sync_master_trace_level = 17"

slave.sh

#!/bin/bash

set -e

PORT="3307"
DATA_DIR="$PWD/slave"

exec_repl() {
    MYSQL_PWD="repl_password" /usr/bin/mysql --host 127.0.0.1 --port $PORT --user repl_user -e "$1"
}

# reset data dir
rm -rf $DATA_DIR
mkdir $DATA_DIR

# write config
cat > slave.cnf << EOF
[client]
port = $PORT

[mysqld]
user = ""
port = $PORT
bind_address = 127.0.0.1
server_id = 2
socket = ""
pid_file = $DATA_DIR/mysql.pid
report_host = 127.0.0.1

datadir = $DATA_DIR
log_bin = $DATA_DIR/mariadb-bin
log_bin_index = $DATA_DIR/mariadb-bin.index
log_slave_updates = 1
relay_log = $DATA_DIR/relay-bin
relay_log_index = $DATA_DIR/relay-bin.index
max_binlog_size = 1048576

read_only = 1
EOF

# create backup
echo "creating backup of master"
/usr/bin/innobackupex --defaults-file=master.cnf \
    --host=127.0.0.1 \
    --port=3306 \
    --user=repl_user \
    --password=repl_password \
    --no-timestamp \
    $DATA_DIR > /dev/null 2>&1

# prepare datadir
echo "preparing data directory"
/usr/bin/innobackupex --defaults-file=slave.cnf \
    --apply-log \
    $DATA_DIR > /dev/null 2>&1

# extract GTID from xtrabackup binlog info
GTID=$(cat $DATA_DIR/xtrabackup_binlog_info | awk '{print $3}')
echo "extracted gtid from xtrabackup_binlog_info, gtid = $GTID"

# start server
echo "starting server"
/usr/sbin/mysqld --defaults-extra-file=slave.cnf > slave.log 2>&1 &

# wait for server to start
echo "waiting for server to start"
sleep 5

# stop slave
echo "stopping slave"
exec_repl "STOP SLAVE"

# install semi-sync slave plugin
echo "installing semi-sync slave plugin"
exec_repl "INSTALL PLUGIN rpl_semi_sync_slave SONAME 'semisync_slave.so'"

# enable semi-sync
echo "enable semi-sync slave"
exec_repl "SET GLOBAL rpl_semi_sync_slave_enabled = 1"

echo "set semi-sync trace level"
exec_repl "SET GLOBAL rpl_semi_sync_slave_trace_level = 17"

# update slave gtid pos
echo "update gtid_slave_pos"
exec_repl "SET GLOBAL gtid_slave_pos = '$GTID'"

# change master
echo "execute change master"
exec_repl "CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=3306, MASTER_USER='repl_user', MASTER_PASSWORD='repl_password', MASTER_CONNECT_RETRY=10, MASTER_USE_GTID=current_pos"

# stop slave io thread
echo "stopping slave io thread"
exec_repl "STOP SLAVE IO_THREAD"

# start slave io thread
echo "starting slave io thread"
exec_repl "START SLAVE IO_THREAD"

# start slave
echo "starting slave"
exec_repl "START SLAVE"

trigger.sh

#!/bin/bash

set -e

PORT="3306"
DUMP_SRC="https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-category.sql.gz"
DUMP="$PWD/dump.sql.gz"

if [[ ! -f $DUMP ]]; then
    echo "test dump missing, downloading"
    curl -s $DUMP_SRC > $DUMP
fi

echo "creating test database"
MYSQL_PWD="repl_password" /usr/bin/mysql --host 127.0.0.1 --port $PORT --user repl_user -e "CREATE DATABASE test"

echo "restoring dump"
zcat $DUMP | MYSQL_PWD="repl_password" /usr/bin/mysql --host 127.0.0.1 --port $PORT --user repl_user -D "test"