Created
June 3, 2016 02:32
-
-
Save hyamamoto/91d8fb827d8ae2ae1d439fc9837ac007 to your computer and use it in GitHub Desktop.
Send file Amazon S3 bucket (aws sync) to Akamai NetStorage directory (secure rsync). Comment out 2 lines for --delete behavior.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
# | |
# Sync Amason S3 Bucket with Akamai NetStorage | |
# | |
# Dependency: | |
# "aws" command : AWS CLI | |
# "rsync" command :yeah, the one. | |
# .aws/credentials : AWS Credential file (for aws command) | |
# .ssj/netstorage.ppk : Private Key for your NetStorage account | |
# | |
# Scripted by Hiroshi Yamamoto ([email protected]) | |
# Licensed under WTFPL version 2.0 (http://www.wtfpl.net/) | |
# Remove comments if you prefer "--delete" behavior | |
# AWS_SYNC_OPT=--delete | |
# RSYNC_OPT=--delete | |
# Setting: Credential | |
SSH_KEY=".ssh/netstorage.ppk" | |
AWS_CREDENTIAL=".aws/credentials" | |
# Setting: Amazon S3 Disk | |
# Ref: https://docs.aws.amazon.com/en_us/general/latest/gr/rande.html#s3_region | |
SRC_S3=s3://YOURBUCKET/src_dir | |
SRC_S3_REGION=us-east-1 | |
# Setting: Akamai NetStorage Disk | |
# Ref: https://control.akamai.com/dl/customers/NS/NetStorage_UserGuide.pdf (P38) | |
[email protected]:/mirror_dir | |
# Setting: Local Temporary directory | |
SYNC_TEMP=./temp_sync | |
# make sandwitchs | |
AWS=aws | |
RSYNC=rsync | |
echo "Sync: Starting." | |
hash $AWS 2>/dev/null || { echo >&2 "aws command not installed. aborting."; exit 1; } | |
hash $RSYNC 2>/dev/null || { echo >&2 "rsync command not installed. aborting."; exit 1; } | |
# let's picnic | |
echo "Sync: Preparation: Checking or creating a temporary directory [$SYNC_TEMP]." | |
mkdir -p $SYNC_TEMP | |
echo "Sync: Downloading: Amazon S3 [$SRC_S3] -> Local [$SYNC_TEMP]" | |
AWS_SHARED_CREDENTIALS_FILE=$AWS_CREDENTIAL $AWS s3 sync $AWS_SYNC_OPT --region $SRC_S3_REGION $SRC_S3 $SYNC_TEMP | |
if [[ $? -ne 0 ]] ; then | |
echo "Sync: Downloading: Aborted. Check [$SYNC_TEMP] directory if necessary." | |
exit 1 | |
fi | |
echo "Sync: Downloading: Done" | |
echo "Sync: Uploading: Local [$SYNC_TEMP] -> Akamai NetStorage [$DEST_NS] " | |
$RSYNC -av -e "ssh -2 -i $SSH_KEY" $RSYNC_OPT $SYNC_TEMP/ $DEST_NS | |
if [[ $? -ne 0 ]] ; then | |
echo "Sync: Uploading: Aborted. Check [$SYNC_TEMP] directory if necessary." | |
exit 2 | |
fi | |
echo "Sync: Uploading: Done" | |
# wanna clean up the mess? | |
echo "Sync: Done. Clean up the [$SYNC_TEMP] directory if necessary." |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$ ./.sh | |
Sync: Starting. | |
Sync: Preparation: Checking or creating a temporary directory [./temp_sync]. | |
Sync: Downloading: Amazon S3 [s3://YOURBUCKET/src_dir] -> Local [./temp_sync] | |
download: s3://YOURBUCKET/src_dir/dir1/file1.html to temp_sync/ | |
download: s3://YOURBUCKET/src_dir/dir2/file2.html to temp_sync/ | |
... | |
... | |
download: s3://YOURBUCKET/src_dir/dir99/file1.html to temp_sync/ | |
download: s3://YOURBUCKET/src_dir/dir99/file2.html to temp_sync/ | |
Sync: Downloading: Done | |
Sync: Uploading: Local [./temp_sync] -> Akamai NetStorage [[email protected]:/mirror_dir] | |
Warning: Permanently added 'YOURNS.upload.akamai.com,XXX.XXX.XXX.XXX' (DSA) to the list of known hosts. | |
logged in | |
building file list ... done | |
.d..tp... temp_sync/ | |
.d..tp... temp_sync/dir1/ | |
.f...p... temp_sync/dir1/file1.html | |
.f...p... temp_sync/dir1/file2.html | |
... | |
... | |
... | |
.f...p... temp_sync/dir99/file1.html | |
.f...p... temp_sync/dir99/file2.html | |
sent 51531 bytes received 12062 bytes 514.92 bytes/sec | |
total size is 19825324 speedup is 311.75 | |
Sync: Uploading: Done | |
Sync: Done. Clean up the [./temp_sync] directory if necessary. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
how can i transfer akamai netstorage to s3 bucket