Test with Redis Sentinel Only

gistfile1.txt

root@kienlt-redis-sentinel-1:~# redis-cli -p 6800 -a 123123 info replication|head -n3
# Replication
role:master
connected_slaves:1
root@kienlt-redis-sentinel-2:~# redis-cli -p 6800 -a 123123 info replication|head -n 3
# Replication
role:slave
master_host:10.0.0.10
==> Shutdown server 10.0.0.10
[16:42:35] Success: 10, Failure: 0
Connection error while setting key:10: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:11: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:12: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:13: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:14: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:15: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:16: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:17: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:18: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:19: Error 111 connecting to 10.0.0.10:6800. Connection refused.
[16:42:40] Success: 0, Failure: 10
==> Failover took 2min20s??
[16:45:00] Success: 10, Failure: 0
[16:45:05] Success: 10, Failure: 0
[16:45:10] Success: 10, Failure: 0
[16:45:15] Success: 10, Failure: 0
[16:45:21] Success: 10, Failure: 0
[16:45:26] Success: 10, Failure: 0
[16:45:31] Success: 10, Failure: 0
[16:45:36] Success: 10, Failure: 0
[16:45:41] Success: 10, Failure: 0
[16:45:46] Success: 10, Failure: 0
root@kienlt-redis-sentinel-2:~# systemctl stop redis-server-6800 redis-sentinel-8800.service 
[16:51:21] Success: 10, Failure: 0
[16:51:26] Success: 10, Failure: 0
[16:51:32] Success: 10, Failure: 0
[16:51:37] Success: 10, Failure: 0
[16:51:42] Success: 10, Failure: 0
[16:51:47] Success: 10, Failure: 0
[16:51:52] Success: 10, Failure: 0
Connection error while setting key:840: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:841: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:842: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:843: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:844: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:845: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:846: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:847: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:848: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:849: Error 111 connecting to 10.1.0.10:6800. Connection refused.
[16:51:57] Success: 0, Failure: 10
[16:52:02] Success: 10, Failure: 0
[16:52:08] Success: 10, Failure: 0
[16:52:13] Success: 10, Failure: 0
[16:52:18] Success: 10, Failure: 0
[16:52:23] Success: 10, Failure: 0
[16:52:28] Success: 10, Failure: 0
[16:52:33] Success: 10, Failure: 0
[16:52:38] Success: 10, Failure: 0
==> Failover took 5s
root@kienlt-redis-sentinel-1:~# redis-cli -p 6800 -a 123123 info replication|head -n3
# Replication
role:master
connected_slaves:1
root@kienlt-redis-sentinel-1:~# systemctl stop redis-server-6800 redis-sentinel-8800.service
[16:54:51] Success: 10, Failure: 0
[16:54:56] Success: 10, Failure: 0
[16:55:01] Success: 10, Failure: 0
[16:55:06] Success: 10, Failure: 0
[16:55:11] Success: 10, Failure: 0
[16:55:16] Success: 10, Failure: 0
[16:55:21] Success: 10, Failure: 0
Connection error while setting key:1250: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:1251: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:1252: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:1253: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:1254: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:1255: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:1256: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:1257: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:1258: Error 111 connecting to 10.0.0.10:6800. Connection refused.
Connection error while setting key:1259: Error 111 connecting to 10.0.0.10:6800. Connection refused.
[16:55:26] Success: 0, Failure: 10
[16:55:32] Success: 10, Failure: 0
[16:55:37] Success: 10, Failure: 0
[16:55:42] Success: 10, Failure: 0
[16:55:47] Success: 10, Failure: 0
[16:55:52] Success: 10, Failure: 0
[16:55:57] Success: 10, Failure: 0
[16:56:02] Success: 10, Failure: 0
==> Failover took 5s
root@kienlt-redis-sentinel-2:~# redis-cli -p 6800 -a 123123 info replication|head -n 3
# Replication
role:master
connected_slaves:1
==> Shutdown 2nd times of master to check redis failover.
### New Client is fine
 ./redis_benchmark.py 
[16:58:39] Success: 10, Failure: 0
[16:58:44] Success: 10, Failure: 0
[16:58:49] Success: 10, Failure: 0
[16:58:55] Success: 10, Failure: 0
[16:59:00] Success: 10, Failure: 0
[16:59:05] Success: 10, Failure: 0
[16:59:10] Success: 10, Failure: 0
[16:59:15] Success: 10, Failure: 0
[16:59:20] Success: 10, Failure: 0
### But current connected client
[16:57:03] Success: 10, Failure: 0
[16:57:08] Success: 10, Failure: 0
[16:57:13] Success: 10, Failure: 0
[16:57:18] Success: 10, Failure: 0
[16:57:24] Success: 10, Failure: 0
[16:57:29] Success: 10, Failure: 0
[16:57:34] Success: 10, Failure: 0
[16:57:39] Success: 10, Failure: 0
[16:57:44] Success: 10, Failure: 0
[16:57:49] Success: 10, Failure: 0
Connection error while setting key:1540: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:1541: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:1542: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:1543: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:1544: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:1545: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:1546: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:1547: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:1548: Error 111 connecting to 10.1.0.10:6800. Connection refused.
Connection error while setting key:1549: Error 111 connecting to 10.1.0.10:6800. Connection refused.
[16:57:54] Success: 0, Failure: 10
[17:00:14] Success: 10, Failure: 0
[17:00:19] Success: 10, Failure: 0
[17:00:24] Success: 10, Failure: 0
[17:00:29] Success: 10, Failure: 0
[17:00:34] Success: 10, Failure: 0
[17:00:39] Success: 10, Failure: 0
[17:00:44] Success: 10, Failure: 0
==> Took 2min20s?
### Change to Golang with same logic for test. Change timeout to 10 seconds and 1 second sleep 
# go run redis_benchmark.go
redis: 2024/08/07 17:42:25 sentinel.go:661: sentinel: new master="8800-sentinel-data-crm" addr="10.0.0.10:6800"
[17:42:25] Success: 10, Failure: 0
[17:42:26] Success: 10, Failure: 0
[17:42:27] Success: 10, Failure: 0
[17:42:28] Success: 10, Failure: 0
[17:42:29] Success: 10, Failure: 0
[17:42:30] Success: 10, Failure: 0
[17:42:31] Success: 10, Failure: 0
[17:42:32] Success: 10, Failure: 0
[17:42:33] Success: 10, Failure: 0
[17:42:34] Success: 10, Failure: 0
[17:42:36] Success: 10, Failure: 0
[17:42:37] Success: 10, Failure: 0
[17:42:38] Success: 10, Failure: 0
[17:42:39] Success: 10, Failure: 0
[17:42:40] Success: 10, Failure: 0
[17:42:41] Success: 10, Failure: 0
[17:42:42] Success: 10, Failure: 0
[17:42:43] Success: 10, Failure: 0
[17:42:44] Success: 10, Failure: 0
[17:42:45] Success: 10, Failure: 0
[17:42:46] Success: 10, Failure: 0
[17:42:47] Success: 10, Failure: 0
[17:42:49] Success: 10, Failure: 0
[17:42:50] Success: 10, Failure: 0
[17:42:51] Success: 10, Failure: 0
[17:42:52] Success: 10, Failure: 0
[17:42:53] Success: 10, Failure: 0
[17:42:54] Success: 10, Failure: 0
[17:42:55] Success: 10, Failure: 0
[17:42:56] Success: 10, Failure: 0
[17:42:57] Success: 10, Failure: 0
[17:42:58] Success: 10, Failure: 0
[17:42:59] Success: 10, Failure: 0
[17:43:00] Success: 10, Failure: 0
[17:43:01] Success: 10, Failure: 0
redis: 2024/08/07 17:43:02 pubsub.go:159: redis: discarding bad PubSub connection: EOF
redis: 2024/08/07 17:43:03 sentinel.go:587: sentinel: GetMasterAddrByName name="8800-sentinel-data-crm" failed: dial tcp 10.0.0.10:8800: connect: connection refused
redis: 2024/08/07 17:43:03 sentinel.go:587: sentinel: GetMasterAddrByName name="8800-sentinel-data-crm" failed: dial tcp 10.0.0.10:8800: connect: connection refused
redis: 2024/08/07 17:43:03 sentinel.go:514: sentinel: GetMasterAddrByName master="8800-sentinel-data-crm" failed: dial tcp 10.0.0.10:8800: connect: connection refused
Error while setting key:350: dial tcp 10.0.0.10:6800: connect: connection refused
Retrying connection in 1 seconds...
redis: 2024/08/07 17:43:06 sentinel.go:661: sentinel: new master="8800-sentinel-data-crm" addr="10.0.0.10:6800"
Connection error: dial tcp 10.0.0.10:6800: connect: connection refused
Retrying connection in 2 seconds...
redis: 2024/08/07 17:43:08 sentinel.go:661: sentinel: new master="8800-sentinel-data-crm" addr="10.0.0.10:6800"
redis: 2024/08/07 17:43:09 sentinel.go:661: sentinel: new master="8800-sentinel-data-crm" addr="10.1.0.10:6800"
redis: 2024/08/07 17:43:09 sentinel.go:661: sentinel: new master="8800-sentinel-data-crm" addr="10.1.0.10:6800"
redis: 2024/08/07 17:43:09 sentinel.go:661: sentinel: new master="8800-sentinel-data-crm" addr="10.1.0.10:6800"
redis: 2024/08/07 17:43:09 sentinel.go:661: sentinel: new master="8800-sentinel-data-crm" addr="10.1.0.10:6800"
[17:43:18] Success: 9, Failure: 1
[17:43:20] Success: 10, Failure: 0
[17:43:21] Success: 10, Failure: 0
[17:43:22] Success: 10, Failure: 0
[17:43:23] Success: 10, Failure: 0
[17:43:24] Success: 10, Failure: 0
[17:43:25] Success: 10, Failure: 0
[17:43:26] Success: 10, Failure: 0
[17:43:27] Success: 10, Failure: 0
[17:43:28] Success: 10, Failure: 0
[17:43:29] Success: 10, Failure: 0
[17:43:30] Success: 10, Failure: 0
[17:43:31] Success: 10, Failure: 0
[17:43:33] Success: 10, Failure: 0
[17:43:34] Success: 10, Failure: 0
[17:43:35] Success: 10, Failure: 0
[17:43:36] Success: 10, Failure: 0
[17:43:37] Success: 10, Failure: 0
[17:43:38] Success: 10, Failure: 0
[17:43:39] Success: 10, Failure: 0
==> Much better took only 20 seconds after shutdown