Playbook to show some of orchestrator's capabilities, used in DB AMA presentation.

orchestrator-demo-playbook.sh

#!/bin/bash
# This playbook assumes you have cloned https://github.com/openark/orchestrator
# and ran: ./script/dock system
# which landed you in orchestrator's playground environment.
# Further information available on the welcome screen once you've ran the docker image.

# FYI, orchestrator's config file is at /etc/orchestrator.conf.json

orchestrator-client -c topology-tabulated -alias ci
orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t'


# Refactoring:

orchestrator-client -c relocate -i 127.0.0.1:10113 -d 127.0.0.1:10112
orchestrator-client -c relocate -i 127.0.0.1:10113 -d 127.0.0.1:10111
orchestrator-client -c relocate-replicas -i 127.0.0.1:10111 -d 127.0.0.1:10112

orchestrator-client -c which-cluster-instances -alias ci
orchestrator-client -c which-cluster-master -i 127.0.0.1:10112
orchestrator-client -c which-master -i 127.0.0.1:10112
orchestrator-client -c which-master -i 127.0.0.1:10113
orchestrator-client -c which-replicas -i 127.0.0.1:10112

# Scripting samples:

orchestrator-client -c which-cluster-instances -alias ci | while read i ; do
  orchestrator-client -c relocate -i "$i" -d 127.0.0.1:10111
done

orchestrator-client -c enable-semi-sync-master -i 127.0.0.1:10111
orchestrator-client -c which-replicas -i 127.0.0.1:10111 | while read i ; do
  orchestrator-client -c enable-semi-sync-replica -i "$i"
done

orchestrator-client -c which-cluster-instances -alias ci | while read i ; do
  orchestrator-client -c disable-semi-sync-master -i "$i"
  orchestrator-client -c disable-semi-sync-replica -i "$i"
done

# Examples: GTID, Errant GTIS

~/sandboxes/ci/s2 -e "flush slow logs"

orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t'
orchestrator-client -c which-gtid-errant -i 127.0.0.1:10113

orchestrator-client -c replication-analysis

# Digress: use of API
orchestrator-client -c api -path all-instances
orchestrator-client -c api -path all-instances | jq -C . | less -r
orchestrator-client -c api -path all-instances | jq -C '.[] | {Port: .Key.Port, ServerUUID: .ServerUUID, ExecutedGtidSet: .ExecutedGtidSet, GtidErrant:.GtidErrant}'

orchestrator-client -c api -path cluster/ci | jq '.[] | select(.GtidErrant!="")'
orchestrator-client -c api -path cluster/ci | jq '.[] | select(.GtidErrant!="") | {Port: .Key.Port}'

# Digress: further API show off
orchestrator-client -c api -path cluster/ci | jq '.[] | select(.SlaveHosts | length == 0) | .Key.Port'
orchestrator-client -c api -path cluster/ci | jq '.[] | select(.MasterKey.Port != 0) | .Key.Port'
orchestrator-client -c api -path cluster/ci | jq -r '.[] | .Version' | grep -c "5.7.26"

# Back to GTID :)

# Assuming we've validated the errant transaction didn't change data:
orchestrator-client -c gtid-errant-inject-empty -i 127.0.0.1:10113

# or...
~/sandboxes/ci/s2 -e "flush slow logs"
sleep 3
orchestrator-client -c gtid-errant-reset-master -i 127.0.0.1:10113

# or...
~/sandboxes/ci/s2 -e "flush slow logs"
sleep 3
orchestrator-client -c api -path gtid-errant-reset-master/127.0.0.1/10113 | jq -C .

# Demonstrate how HAProxy routes 13306 traffic to current master:
mysql -uci -pci -h 127.0.0.1 --port 13306 -e "select @@hostname, @@port"
consul kv get -recurse mysql/master/ci

# Graceful takeovers:
orchestrator-client -c graceful-master-takeover-auto -alias ci -d 127.0.0.1:10112
orchestrator-client -c graceful-master-takeover-auto -alias ci
orchestrator-client -c register-candidate -i 127.0.0.1:10113 --promotion-rule prefer
# Let orchestrator pick the promoted replica, expect it to be 10113
orchestrator-client -c graceful-master-takeover-auto -alias ci
orchestrator-client -c graceful-master-takeover-auto -alias ci -d 127.0.0.1:10111
# Expect the following to fail due to '"PreventCrossRegionMasterFailover": true' config
orchestrator-client -c graceful-master-takeover-auto -alias ci -d 127.0.0.1:10114


orchestrator-client -c which-cluster-instances -alias ci | while read i ; do
  orchestrator-client -c relocate -i "$i" -d 127.0.0.1:10111
done


orchestrator-client -c ack-all-recoveries --reason demo

mysqladmin -uci -pci -h 127.0.0.1 --port 13306 processlist
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 shutdown
orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t'

# for this environment, orchestrator's config allows failovers 5 seconds apart.
# In production the config would be higher.

sleep 5
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 processlist
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 shutdown
while sleep 0.5 ; do orchestrator-client -c replication-analysis ; done

redeploy-ci-env

# Failover:

# A complex scenario. We suggest to orchestrator that 127.0.0.1:10113 is preferred to be promoted in case of failover.
# But we set up the topology such that 127.0.0.1:10113 is the most lagging replica,
# and such that 127.0.0.1:10114 (which may not be promoted due to PreventCrossRegionMasterFailover) is the most up-to-date replica.
orchestrator-client -c register-candidate -i 127.0.0.1:10113 --promotion-rule prefer
orchestrator-client -c stop-replica -i 127.0.0.1:10113
sleep 3
orchestrator-client -c stop-replica -i 127.0.0.1:10112
sleep 3
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 processlist
mysqladmin -uci -pci -h 127.0.0.1 --port 13306 shutdown
while sleep 0.5 ; do orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t' ; echo ""; done