synapse-compress-state.sh

#!/bin/bash

# >>> https://github.com/matrix-org/rust-synapse-compress-state <<<
# use this tool to compress room states, it can free up hundreds of gigabytes!
# the synapse-compress-state.sh script helps you to loop through many rooms automatically
# be aware that depending on your installation and hardware, this will take hours/days!
#
# synapse-compress-state.sh is far from perfect and inspired by https://github.com/matrix-org/synapse/blob/master/contrib/purge_api/purge_history.sh
# DO BACKUPS! BE CAREFUL! DON'T BLINDLY COPY AND RUN THIS SCRIPT!
# you in theory don't need to turn synapse off but to actually free up the disk space you need to vacuum your db which locks the tables
#
# don't do it if you are close to running out of memory! it can take 6G of RAM for big rooms like matrixhq,
# in some special cases like the haskell room at matrix.org, it can go over 20G of RAM, make at least sure to have enough swap,
# swapfiles may come in handy as they are usually more flexible than partitions: https://wiki.archlinux.org/index.php/Swap#Swap_file
# note that you CAN use multible swapfiles to increase swap!
# ...or consider nuking such a room from your server and lock it out with the limit_remote_rooms.complexity setting!

# emergency brake
set -e

# build a list of rooms with the postgresql command 'SELECT room_id FROM rooms'
# example all rooms: \copy (select room_id from rooms) to '/home/synapse/rooms.csv';
# example >50 users: \copy (select q.room_id from (select count(*) as numberofusers, room_id from current_state_events where type ='m.room.member' group by room_id) as q left join room_aliases a on q.room_id=a.room_id where q.numberofusers > 50 order by numberofusers desc) to '/home/synapse/rooms.csv';
#
# hint: adding ' to end of lines with vim: :%s/$/\'/g
ROOMS_ARRAY=(
#'!cURbafjkfsMDVwdRDQ:matrix.org'
#'!JAnMCKgLGDxzxjtjfE:matrix.org'
# ..continue like that, don't use commas(!)
)

for ROOM in "${ROOMS_ARRAY[@]}"; do
        echo "################################## $(date) ################# "
        ROOM=${ROOM%#*}
        # adjust according to your setup!
        # building synapse-compress-state:
        # https://doc.rust-lang.org/cargo/getting-started/installation.html
        # use the 'cargo build --release' command once ur ready
        # 
        # putting the credentials right into this script like this is not ideal in terms of security,
        # think twice if it is a good idea on your setup this way!
        #
        # the output file can be large, /tmp may not be enough!
        /path/to/rust-synapse-compress-state/target/debug/synapse-compress-state -p "postgresql://dbuser:dbpassword@localhost/synapse" -r $ROOM -o "/home/synapse/out.sql" -t
        # this can be simplified, i at first thought something was broken, as synapse-compress-state sometimes returns
        # empty .sql files so this part did not do anything in the beginning
        /usr/bin/psql -U synapse -d synapse -f "/data/home/synapse/out.sql"
        echo "The psql command ran..."
        # just in case
        rm /data/home/synapse/out.sql
        echo "Deleted old.sql..."
done