Last active
May 30, 2022 18:18
-
-
Save Maxr1998/516cc49ab29e3b928ecbe474b20b1fe9 to your computer and use it in GitHub Desktop.
Script that allows to stream Google Takeout tar archives into borg, to download the archives on a high-bandwidth VPS and only copy the changes to your NAS.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# Target repository on your NAS | |
export BORG_REPO="ssh://user@host/..." | |
# Disable prompts that would break the script | |
export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes | |
export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes | |
# Direct download links to the Google Takeout archives, extract with your browser's network debugger (F12). | |
# You want the URL redirected to (with the same format like below), since it's already authenticated. | |
LINKS=( | |
"https://...-apidata.googleusercontent.com/download/storage/v1/b/takeout-xx/o/..." | |
"https://...-apidata.googleusercontent.com/download/storage/v1/b/takeout-xx/o/..." | |
"..." | |
) | |
# Build input archives for bsdtar. They will be streamed through curl and named pipes. | |
TAR_ARGS=() | |
for link in "${LINKS[@]}"; do | |
pipe=$(echo $link | md5sum | cut -c -32) | |
mkfifo $pipe | |
# Cleanup pipes when finished | |
trap "rm -f $pipe" EXIT | |
curl $link -o $pipe 2> /dev/null & | |
TAR_ARGS+=("@$pipe") | |
done | |
# borg arguments. Create regular checkpoints in case something fails, and display progress. | |
BORG_ARGS=( | |
import-tar | |
--verbose | |
--filter AME | |
--list | |
--progress | |
--stats | |
--show-rc | |
--checkpoint-interval 60 | |
--compression lz4 | |
::'archive-{now}' | |
) | |
# Merge archives with bsdtar and import into borg | |
eval "bsdtar -cf - ${TAR_ARGS[*]}" | borg ${BORG_ARGS[@]} - |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment