Created
November 12, 2020 15:43
-
-
Save phivid/4a1159beda9d2bc805f77c3ddf8b1f61 to your computer and use it in GitHub Desktop.
influx_restore.bash
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| #set -e | |
| # Script to restore InfluxDB data from first full backup found before $RESTORE_DATE until $RESTORE_DATE. | |
| # Pre-requisites: | |
| # RESTORE_DATE : Date is set in format YYYYMMDDTHHMMSSZ (UTC time). Can be "latest". | |
| # AZ_ENV : Environment to work in. | |
| # DATABASES : Databases to import. | |
| # TO_DATABASES : Databases to import to new databases. | |
| # TENANT_ID : Tenant ID to work with. | |
| # AZCOPY_SPA_CLIENT_SECRET : Sercret for SP | |
| # AZ_USERNAME : Service Principal for Azure login | |
| # AZ_RG : Azure Resource group from where to retrieve Backups from. | |
| # AZ_STORAGEACCOUNT : Azure Storage account where to retrieve Backups from. | |
| # INFLUXDB_HOST : Host InfluxDB | |
| # INFLUXDB_PORT : Port InfluxDB | |
| # INFLUX_BR_PORT : Port for Backup Restore InfluxDB | |
| SLACK_URI="" | |
| LOCAL_DIR="/tmp" | |
| # Put DATABASES and TO_NEW_DATABASES into array | |
| ARRAY_DB=($DATABASES) | |
| echo " List of Databases to restore: ${ARRAY_DB[*]}" | |
| ARRAY_TO_DB=($TO_DATABASES) | |
| echo " List of Databases to restore IN: ${ARRAY_TO_DB[*]}" | |
| ## Be sure to use kubeconfig that match the targeted environment to restore in. | |
| #kubectl port-forward service/influxdb1 8088:8088 -n data > /dev/null & | |
| #kubectl port-forward service/influxdb1 8086:8086 -n data > /dev/null & | |
| if [[ $RESTORE_DATE == "latest" ]] | |
| then | |
| RESTORE_DATE=$(date -u +%Y%m%dT%H%M%SZ) | |
| fi | |
| az login --tenant $TENANT_ID --service-principal -u $AZ_USERNAME -p $AZCOPY_SPA_CLIENT_SECRET > /dev/null | |
| if [[ $? -ne 0 ]] | |
| then | |
| curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: issue during login to Azure storage account for retrieving INFLUXDB databases ['"$DATABASES"'] : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI | |
| exit 1 | |
| fi | |
| KEY=$(az storage account keys list --account-name $AZ_STORAGEACCOUNT --resource-group $AZ_RG --query '[0].value' --output tsv) | |
| if [[ $KEY == "" ]] | |
| then | |
| curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: issue on Azure storage account Key retrieval during recovery process of INFLUXDB databases ['"$DATABASES"'] : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI | |
| exit 1 | |
| fi | |
| echo " Start of InfluxDB restore for a recovery at ${RESTORE_DATE} from ${ARRAY_DB[*]} respectively to ${ARRAY_TO_DB[*]}... - `date +%Y/%m/%d-%H:%M:%S`" | |
| for directory in `az storage fs file list --account-name $AZ_STORAGEACCOUNT --account-key $KEY -f influxdb --recursive=false 2>/dev/null | jq -r '.[].name' | sort -r` | |
| do | |
| if [[ "$(echo $directory | cut -d'-' -f1)" < "$RESTORE_DATE" ]] | |
| then | |
| if [[ $directory != *"-full" ]] | |
| then | |
| LIST="$LIST $directory" | |
| elif [[ $directory == *"-full" ]] | |
| then | |
| LIST="$LIST $directory" | |
| break | |
| fi | |
| fi | |
| done | |
| LIST=$(echo "$LIST" | awk '{ for (i=NF; i>1; i--) printf("%s ",$i); print $1; }') | |
| echo " List of Directories to retrieve for restoring data at ${RESTORE_DATE}: $LIST" | |
| export AZCOPY_SPA_CLIENT_SECRET | |
| azcopy login --service-principal --application-id $AZ_USERNAME --tenant-id $TENANT_ID | |
| for selected_directory in $LIST | |
| do | |
| echo " Copy $selected_directory locally ..." | |
| azcopy cp "https://${AZ_STORAGEACCOUNT}.blob.core.windows.net/influxdb/$selected_directory" "${LOCAL_DIR}/" --recursive=true | |
| if [[ $? -ne 0 ]] | |
| then | |
| curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: issue during copy of INFLUXDB Backup https://'"${AZ_STORAGEACCOUNT}"'.blob.core.windows.net/influxdb/'"$selected_directory"' locally : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI | |
| echo " Local copy of $selected_directory failed." | |
| exit 1 | |
| else | |
| echo " Local copy of $selected_directory completed." | |
| fi | |
| TS=$(echo $selected_directory | cut -d'-' -f1) | |
| #ORI: for database in $DATABASES | |
| for index in ${!ARRAY_DB[*]} | |
| do | |
| #influx -host $INFLUXDB_HOST -port $INFLUXDB_PORT -execute "SHOW DATABASES" | grep ^${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}$ > /dev/null | |
| influx -execute "SHOW DATABASES" | grep ^${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}$ > /dev/null | |
| DB_EXISTS_CODE=$? | |
| if [[ $DB_EXISTS_CODE -eq 0 ]] | |
| then | |
| #Direct load for temporary side load DB | |
| ERROR_SIDE_LOAD=1 | |
| ERR_COUNT=0 | |
| while [[ $ERROR_SIDE_LOAD -ne 0 && $ERR_COUNT -lt 3 ]] | |
| do | |
| echo "This is try number $((ERR_COUNT+1)) for direct load for temporary side load DB." | |
| sleep 3 | |
| #influxd restore -portable -host $INFLUXDB_HOST:$INFLUX_BR_PORT -db ${ARRAY_DB[$index]} -newdb ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} ${LOCAL_DIR}/$selected_directory | |
| influxd restore -portable -db ${ARRAY_DB[$index]} -newdb ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} ${LOCAL_DIR}/$selected_directory | |
| ERROR_SIDE_LOAD=$? | |
| ERR_COUNT=$((ERR_COUNT+1)) | |
| done | |
| if [[ $ERROR_SIDE_LOAD -ne 0 ]] | |
| then | |
| curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: influxd restore command failed for '"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}"'-restored-'"${TS}"' : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI | |
| echo " Import of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} failed." | |
| exit 1 | |
| else | |
| echo " Import of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} completed." | |
| fi | |
| #Side load for existing DB | |
| ERROR_SIDE_LOAD=1 | |
| ERR_COUNT=0 | |
| while [[ $ERROR_SIDE_LOAD -ne 0 && $ERR_COUNT -lt 3 ]] | |
| do | |
| echo "This is try number $((ERR_COUNT+1)) for side load." | |
| sleep 3 | |
| #influx -host $INFLUXDB_HOST -port $INFLUXDB_PORT -database ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} -execute "SELECT * INTO ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}..:MEASUREMENT FROM /.*/ GROUP BY *" | |
| influx -database ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} -execute "SELECT * INTO ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}..:MEASUREMENT FROM /.*/ GROUP BY *" | |
| ERROR_SIDE_LOAD=$? | |
| ERR_COUNT=$((ERR_COUNT+1)) | |
| done | |
| if [[ $ERROR_SIDE_LOAD -ne 0 ]] | |
| then | |
| curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: injection of '"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}"'-restored-'"${TS}"' into '"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}"' failed : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI | |
| echo " Injection of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} into ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]} failed." | |
| exit 1 | |
| else | |
| echo " Injection of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} into ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]} completed." | |
| fi | |
| #influx -host $INFLUXDB_HOST -port $INFLUXDB_PORT -execute "DROP DATABASE \"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS}\"" > /dev/null | |
| influx -execute "DROP DATABASE \"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS}\"" > /dev/null | |
| else | |
| #Direct load for NON existing DB | |
| ERROR_SIDE_LOAD=1 | |
| ERR_COUNT=0 | |
| while [[ $ERROR_SIDE_LOAD -ne 0 && $ERR_COUNT -lt 3 ]] | |
| do | |
| echo "This is try number $((ERR_COUNT+1)) for direct load in NON existing target Database." | |
| sleep 3 | |
| if [[ -z ${ARRAY_TO_DB[$index]} ]]; | |
| then | |
| #influxd restore -portable -host $INFLUXDB_HOST:$INFLUX_BR_PORT -db ${ARRAY_DB[$index]} ${LOCAL_DIR}/$selected_directory | |
| influxd restore -portable -db ${ARRAY_DB[$index]} ${LOCAL_DIR}/$selected_directory | |
| else | |
| #influxd restore -portable -host $INFLUXDB_HOST:$INFLUX_BR_PORT -db ${ARRAY_DB[$index]} -newdb ${ARRAY_TO_DB[$index]} ${LOCAL_DIR}/$selected_directory | |
| influxd restore -portable -db ${ARRAY_DB[$index]} -newdb ${ARRAY_TO_DB[$index]} ${LOCAL_DIR}/$selected_directory | |
| fi | |
| #ORI: influxd restore -portable -host $INFLUXDB_HOST:$INFLUX_BR_PORT -db $database ${LOCAL_DIR}/$selected_directory | |
| ERROR_SIDE_LOAD=$? | |
| ERR_COUNT=$((ERR_COUNT+1)) | |
| done | |
| if [[ $ERROR_SIDE_LOAD -ne 0 ]] | |
| then | |
| curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopAlert", "text":"['"$AZ_ENV"'] Caution: influxd restore command failed for '"${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}"' : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":x:"}' $SLACK_URI | |
| echo " Import of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} failed." | |
| exit 1 | |
| else | |
| echo " Import of ${ARRAY_TO_DB[$index]:-$ARRAY_DB[$index]}-restored-${TS} completed." | |
| fi | |
| fi | |
| done | |
| rm -rf ${LOCAL_DIR}/$selected_directory | |
| done | |
| #kill $(jobs -p) | |
| curl -X POST -H 'Content-type: application/json' --data '{"username": "RestoreCopInfo", "text":"['"$AZ_ENV"'] OK: Restore for recovery at '"$RESTORE_DATE"' from InfluxDB databases ['"${ARRAY_DB[*]}"'] into InfluxDB databases ['"${ARRAY_TO_DB[*]}"'] completed : '"`date +%Y/%m/%d-%H:%M:%S`"'.", "icon_emoji": ":white_check_mark:"}' $SLACK_URI | |
| echo "OK: Restore Completed." | |
| exit 0 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment