-
-
Save thimslugga/428af290e4a5994b78b5b2642e8fd390 to your computer and use it in GitHub Desktop.
Copied from https://s3.amazonaws.com/jacquecp/pv-hvm-latest.sh, discovered at https://forums.aws.amazon.com/thread.jspa?messageID=648055
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
VER="0.94.2"; | |
SCRIPTTITLE="PV - HVM - version $VER"; | |
RED=$(tput setaf 1) | |
GREEN=$(tput setaf 2) | |
NORMAL=$(tput sgr0) | |
# Configure logging | |
tmp="/tmp" | |
logfile="$tmp/$(basename $0).$$.log" | |
# Gather required information from meta-data | |
export metadata_document="$(curl http://169.254.169.254/latest/dynamic/instance-identity/document -s -m 5)" | |
export workinginstanceid="$(curl http://169.254.169.254/latest/meta-data/instance-id -m 5 -s)" | |
export localami="$(curl http://169.254.169.254/latest/meta-data/ami-id -s -m 5)" | |
export iamrole="$(curl http://169.254.169.254/latest/meta-data/iam/info/ -s -m 5)" | |
# Exit if all data could not be gathered from meta-data | |
if [[ -z $metadata_document ]] || [[ -z $workinginstanceid ]] || [[ -z $localami ]] || [[ -z $iamrole ]]; then | |
echo -e "Cannot access metadata - http://169.254.169.254. Exiting..." | |
exit 255 | |
fi | |
# Define required variables (without jq, as jq might not be installed yet) | |
accountid="$(echo $metadata_document | tr , '\n' | \grep -i "accountId" | awk -F "\"" '{ print $4 }')" | |
region="$(echo $metadata_document | tr , '\n' | \grep -i "region" | awk -F "\"" '{ print $4 }')" | |
az="$(echo $metadata_document | tr , '\n' | \grep -i "availabilityZone" | awk -F "\"" '{ print $4 }')" | |
AWS_DEFAULT_REGION="$region" | |
sysroot="/mnt" | |
# Progress bar functions | |
# -------------vvv------------- | |
MINIPROGRESS(){ | |
for f in $(seq 2); do | |
echo -ne "."; | |
sleep .5; | |
done | |
} | |
PROGRESS(){ | |
for f in $(seq 10); do | |
echo -ne "."; | |
sleep .5; | |
done | |
} | |
MAXPROGRESS(){ | |
for f in $(seq 15); do | |
echo -ne "."; | |
sleep 3; | |
done | |
} | |
# -------------^^^------------- | |
# Ensure the script execute as root - required for certain script actions - installing package, mounting partitions, etc. | |
CHECK_ROOT(){ | |
if [[ $EUID -ne 0 ]]; then | |
echo -e "\n${RED}ERROR${NORMAL}: This script must be run as root."; | |
echo -e " 1. Re-run this script as root."; | |
echo -e " 2. Ensure aws-cli is configured for the root user.\n"; | |
exit 1 | |
fi | |
} | |
# Ensure the script is executed on on a RHEL or Ubuntu based instance | |
CHECK_OS(){ | |
LOGGING "\nChecking working instance OS"; | |
LOGGING "----------------------------\n"; | |
if [[ $(file /etc/*_version) = *ERROR* ]] ||\ | |
[[ $(grep -i "Ubuntu" /etc/*_version 2>/dev/null) != *Ubuntu* ]] &&\ | |
[[ $(file /etc/*-release) != *ERROR* ]]; then | |
LOGGING "Working instance - RHEL Based"; | |
os="rhel"; | |
elif [[ $(file /etc/*-release) = *ERROR* ]] ||\ | |
[[ $(grep -i "Ubuntu" /etc/*-release 2>/dev/null) = *Ubuntu* ]] &&\ | |
[[ $(file /etc/*_version) != *ERROR* ]]; then | |
LOGGING "Working instance - Ubuntu"; | |
os="ubuntu"; | |
else | |
LOGGING "Unsupported distributions"; | |
os="unsupported"; | |
echo -e "Only Debian & RHEL based Distributions currently supported"; | |
exit 1 | |
fi | |
} | |
# Install the awscli tools | |
CHECK_AWSCLI(){ | |
if [[ -z $os ]]; then | |
CHECK_OS | |
fi | |
LOGGING "\nChecking awscli tools installation"; | |
LOGGING "----------------------------------\n"; | |
if hash aws 2>/dev/null; then | |
LOGGING "AWSCLI installed" | |
if [[ $os = "rhel" ]]; then | |
sudo yum update aws-cli -y > /dev/null 2>&1 | |
fi | |
aws=true | |
else | |
LOGGING "AWSCLI not installed - installing"; | |
aws=false | |
echo -e "Installing aws-cli"; | |
if [[ $os = "ubuntu" ]]; then | |
sudo dpkg --configure -a && sudo apt-get update > /dev/null 2>&1 && sudo apt-get install -y python-pip > /dev/null 2>&1 && sudo pip install --upgrade awscli > /dev/null 2>&1 | |
elif [[ $os = "rhel" ]]; then | |
sudo yum install python-pip -y > /dev/null 2>&1 && sudo pip install --upgrade awscli > /dev/null 2>&1 | |
fi | |
if hash aws 2>/dev/null; then | |
aws=true | |
else | |
LOGGING "Failed to install AWSCLI"; | |
echo -e "Cannot install awscli"; | |
exit | |
fi | |
fi | |
} | |
# Install jq - used to extract resource information from awscli calls | |
CHECK_JQ(){ | |
if [[ -z $os ]]; then | |
CHECK_OS | |
fi | |
LOGGING "\nChecking jq installation"; | |
LOGGING "------------------------\n"; | |
if hash jq 2>/dev/null; then | |
LOGGING "JQ installed" | |
jq=true | |
else | |
LOGGING "JQ not installed - installing"; | |
echo -e "Installing jq\n"; | |
curl -s http://stedolan.github.io/jq/download/linux64/jq -o /usr/local/bin/jq && chmod +x /usr/local/bin/jq; | |
if hash jq 2>/dev/null; then | |
jq=true | |
else | |
LOGGING "Failed to install JQ"; | |
echo -e "Cannot install jq"; | |
exit | |
fi | |
fi | |
} | |
# Install pv for dd progress | |
CHECK_PV(){ | |
if [[ -z $os ]]; then | |
CHECK_OS | |
fi | |
LOGGING "\nChecking pv installation"; | |
LOGGING "----------------------------------\n"; | |
if hash pv 2>/dev/null; then | |
LOGGING "PV installed" | |
pv=true | |
else | |
LOGGING "PV not installed - installing"; | |
pv=false | |
if [[ $os = "ubuntu" ]]; then | |
sudo dpkg --configure -a && sudo apt-get update > /dev/null 2>&1 && sudo apt-get install -y pv > /dev/null 2>&1 | |
elif [[ $os = "rhel" ]]; then | |
sudo yum install pv -y > /dev/null 2>&1 | |
fi | |
if hash pv 2>/dev/null; then | |
pv=true | |
else | |
pv=false | |
LOGGING "PV installation failed"; | |
fi | |
fi | |
} | |
# Exponential backoff on API call rate limit (Currently just random backoff - to be changed to exponential) | |
#API_CALL(){ | |
# api_call_check="RateLimit" | |
# while [[ $api_call_check =~ "RateLimit" ]]; do | |
# export api_call_check="$(aws --output json --region $region $* 2>&1)" | |
# echo $api_call_check >> $logfile | |
# if [[ $api_call_check =~ "RateLimit" ]]; then | |
# local waittime="$(( ( RANDOM % 10 ) + 5 ))" | |
# echo -e "Current API call (${RED}$2${NORMAL}) has been rate limited. Waiting $waittime seconds to try again."; | |
# sleep $waittime | |
# fi | |
# done | |
#} | |
API_CALL(){ | |
api_call_check="RateLimit" | |
while [[ $api_call_check =~ "RateLimit" ]]; do | |
echo -e "API CALL:\n---------------------\naws --output json --region $region $*" >> $logfile; | |
export api_call_check="$(aws --output json --region $region $* 2>&1)" | |
echo -e "$api_call_check\n---------------------\n" >> $logfile | |
if [[ $api_call_check =~ "RateLimit" ]]; then | |
local waittime="$(( ( RANDOM % 10 ) + 5 ))" | |
echo -e "Current API call ($2) has been rate limited. Waiting $waittime seconds to try again."; | |
sleep $waittime | |
fi | |
done | |
} | |
# Logging functions | |
LOGGING(){ | |
echo -e "$1" >> $logfile | |
} | |
LOGGINGVAR(){ | |
sed -i "s/$1"/"$2"/g $logfile | |
} | |
# Checking for IAM Role / User, testing IAM permissions for all API calls and exit with meaningful message if any problems are found | |
# -------------vvv------------- | |
# Check for IAM role or .aws/credentials and configure if missing | |
AWSCLI_CREDS(){ | |
LOGGING "\nChecking required AWS credentials"; | |
LOGGING "---------------------------------\n"; | |
#export iamrole="$(curl http://169.254.169.254/latest/meta-data/iam/info/ -s -m 5)" | |
if [[ -f $HOME/.aws/config ]]; then | |
export AWS_CONFIG_FILE=$HOME/.aws/config | |
LOGGING "Found $HOME/.aws/config"; | |
echo -e "AWSCLI configuration file found - ${GREEN}$AWS_CONFIG_FILE${NORMAL}\n" | |
echo -ne "Checking basic API call permissions"; | |
API_CALL ec2 describe-instances --instance-ids $workinginstanceid | |
if [[ $api_call_check =~ "UnauthorizedOperation" ]] || [[ $api_call_check =~ "AuthFailure" ]]; then | |
MINIPROGRESS | |
LOGGING "Failed to do basic aws ec2 describe-instances call using credentials configured in awscli configuration file." | |
echo -e "${RED}[FAIL]${NORMAL}\n" | |
PERMFAIL | |
else | |
MINIPROGRESS; | |
LOGGING "Basic aws ec2 describe-instances call using IAM role succeeded."; | |
echo -e "${GREEN}[OK]${NORMAL}\n" | |
fi | |
elif [[ $iamrole =~ "Success" ]]; then | |
LOGGING "IAM Role found - $(echo $iamrole | jq -r '.InstanceProfileArn')"; | |
echo -e "IAM Role found - ${GREEN}$(echo $iamrole | jq -r '.InstanceProfileArn')${NORMAL}\n"; | |
sleep 1 | |
echo -ne "Checking basic API call permissions"; | |
API_CALL ec2 describe-instances --instance-ids $workinginstanceid | |
if [[ $api_call_check =~ "UnauthorizedOperation" ]] || [[ $api_call_check =~ "AuthFailure" ]]; then | |
MINIPROGRESS; | |
LOGGING "Failed to do basic aws ec2 describe-instances call using IAM role. Configuring awscli config file"; | |
echo -e "${RED}[FAIL]${NORMAL}\n" | |
echo -e "Configuring aws-cli"; | |
aws configure | |
if [[ -f $HOME/.aws/config ]]; then | |
LOGGING "IAM Credentials configured. AWS_CONFIG_FILE has been set."; | |
export AWS_CONFIG_FILE=$HOME/.aws/config | |
echo -ne "Rechecking basic API call permissions"; | |
API_CALL ec2 describe-instances --instance-ids $workinginstanceid | |
if [[ $api_call_check =~ "UnauthorizedOperation" ]] || [[ $api_call_check =~ "AuthFailure" ]]; then | |
MINIPROGRESS | |
LOGGING "Failed to do basic aws ec2 describe-instances call using credentials configured in awscli configuration file." | |
echo -e "${RED}[FAIL]${NORMAL}\n" | |
PERMFAIL | |
else | |
MINIPROGRESS; | |
LOGGING "Basic aws ec2 describe-instances call using IAM permissions succeeded."; | |
echo -e "${GREEN}[OK]${NORMAL}\n" | |
fi | |
else | |
echo -e "IAM Role or IAM permissions required to continue. Neither found.\nExiting..." | |
exit | |
fi | |
else | |
MINIPROGRESS; | |
LOGGING "Basic aws ec2 describe-instances call using IAM role succeeded."; | |
echo -e "${GREEN}[OK]${NORMAL}\n" | |
fi | |
else | |
LOGGING "$HOME/.aws/config not found. IAM role not found. Configuring awscli config file"; | |
echo -e "No IAM role found, configuring aws-cli"; | |
aws configure | |
if [[ -f $HOME/.aws/config ]]; then | |
export AWS_CONFIG_FILE=$HOME/.aws/config | |
LOGGING "IAM Credentials configured. AWS_CONFIG_FILE has been set."; | |
echo -ne "\nChecking basic API call permissions"; | |
API_CALL ec2 describe-instances --instance-ids $workinginstanceid | |
if [[ $api_call_check =~ "UnauthorizedOperation" ]] || [[ $api_call_check =~ "AuthFailure" ]]; then | |
MINIPROGRESS | |
LOGGING "Failed to do basic aws ec2 describe-instances call using credentials configured in awscli configuration file." | |
echo -e "${RED}[FAIL]${NORMAL}\n" | |
PERMFAIL | |
else | |
MINIPROGRESS; | |
LOGGING "Basic aws ec2 describe-instances call using IAM permissions succeeded."; | |
echo -e "${GREEN}[OK]${NORMAL}\n" | |
fi | |
echo -e "IAM Role or IAM permissions required to continue. Neither found.\nExiting..." | |
else | |
exit | |
fi | |
fi | |
export workinginstancejson=$api_call_check | |
if [[ $workinginstancejson == "" ]]; then | |
echo "Failed to get working instance data. Cannot continue. Exiting script."; | |
LOGGING "Failed to get working instance data - initial describe-instances call failed - cannot continue to check permissions" | |
exit $? | |
fi | |
} | |
# Permission check for all required calls | |
PERMISSIONS(){ | |
LOGGING "\nChecking required IAM permissions"; | |
LOGGING "---------------------------------\n"; | |
local localvol="$(echo $workinginstancejson | jq -r '.Reservations[].Instances[0].BlockDeviceMappings[].Ebs.VolumeId' | head -n 1)" | |
local localami="$(echo $workinginstancejson | jq -r '.Reservations[].Instances[0].ImageId')" | |
# IAM permission checks - format: CHKPERMISSIONS "display message" "aws command with --dry-run" | |
CHKPERMISSIONS "describe-instances" "API_CALL ec2 describe-instances --instance-id $workinginstanceid --dry-run"; | |
CHKPERMISSIONS "start-instances" "API_CALL ec2 start-instances --instance-id $workinginstanceid --dry-run"; | |
CHKPERMISSIONS "stop-instances" "API_CALL ec2 stop-instances --instance-id $workinginstanceid --dry-run"; | |
CHKPERMISSIONS "run-instances" "API_CALL ec2 run-instances --image-id $localami --instance-type m3.medium --dry-run"; | |
CHKPERMISSIONS "describe-volumes" "API_CALL ec2 describe-volumes --volume-ids $localvol --dry-run"; | |
CHKPERMISSIONS "create-volumes" "API_CALL ec2 create-volume --size 1 --availability-zone $az --dry-run"; | |
CHKPERMISSIONS "attach-volumes" "API_CALL ec2 attach-volume --volume-id $localvol --instance-id $workinginstanceid --device /dev/sdf --dry-run" "echo $api_call_check"; | |
CHKPERMISSIONS "detach-volumes" "API_CALL ec2 detach-volume --volume-id $localvol --dry-run"; | |
CHKPERMISSIONS "delete-volumes" "API_CALL ec2 delete-volume --volume-id $localvol --dry-run"; | |
CHKPERMISSIONS "describe-snapshots" "API_CALL ec2 describe-snapshots --snapshot-id snap-testing --dry-run"; | |
CHKPERMISSIONS "create-snapshot" "API_CALL ec2 create-snapshot --volume-id $localvol --description createsnapshot.$$ --dry-run"; | |
CHKPERMISSIONS "copy-snapshot" "API_CALL ec2 copy-snapshot --source-region $region --destination-region $region --source-snapshot-id snap-abc123ab --description "CHKPERMISSIONS-copy-snapshot" --dry-run"; | |
CHKPERMISSIONS "register-image" "API_CALL ec2 register-image --name registername.$$ --description registerimage.$$ --architecture x86_64 --root-device-name /dev/sda1 --virtualization-type hvm --dry-run"; | |
CHKPERMISSIONS "create-image" "API_CALL ec2 create-image --instance $workinginstanceid --name createimage.$$ --dry-run"; | |
CHKPERMISSIONS "describe-images" "API_CALL ec2 describe-images --image-id $localami --dry-run"; | |
if [[ $failed_check = "1" ]]; then | |
PERMFAIL | |
fi | |
LOGGING "\nPermission check completed\n--------------------------\n"; | |
} | |
CHKPERMISSIONS(){ | |
LOGGING "Checking $1"; | |
#permcheck="$($2 2>&1)" | |
$2 2>&1 | |
permcheck="$api_call_check" | |
if [[ $permcheck =~ "UnauthorizedOperation" ]] || [[ $permcheck =~ "Unable to locate credentials" ]]; then | |
printf "%-25s%-25s\n" "$1" "${RED}[FAIL]${NORMAL}" | |
LOGGING "^^^^^ FAILED ^^^^^\n$permcheck\n"; | |
export failed_check="1" | |
else | |
printf "%-25s%-25s\n" "$1" "${GREEN}[OK]${NORMAL}" | |
fi | |
} | |
PERMFAIL(){ | |
echo -e ""; | |
echo -e ""; | |
if [[ -f $HOME/.aws/credentials ]]; then | |
PERMFAIL_LOG "access key" credentials | |
elif [[ -f $HOME/.aws/config ]]; then | |
PERMFAIL_LOG "access key" config | |
elif [[ $(curl http://169.254.169.254/latest/meta-data/iam/info/ -s -m 5) =~ "Success" ]]; then | |
PERMFAIL_LOG role | |
else | |
echo -e "Ensure that the IAM user or role has the correct permissions."; | |
LOGGING "Ensure that the IAM user or role has the correct permissions."; | |
fi | |
FAILMESSAGE | |
exit $? | |
} | |
PERMFAIL_LOG(){ | |
if [[ $1 == "access key" ]]; then | |
local key="$(grep -A 1 "default" $HOME/.aws/$2 | grep -i "aws_access_key_id" | awk '{ print$3 }')" | |
elif [[ $1 == "role" ]]; then | |
local key="$(echo $iamrole | jq -r '.InstanceProfileArn')" | |
fi | |
echo -e "Ensure $1: ${RED}$key${NORMAL} has the correct permissions."; | |
LOGGING "\nEnsure $1: $key has the correct permissions."; | |
} | |
FAILMESSAGE(){ | |
echo -e ""; | |
echo -e "\e[1mSample IAM Policy:\e[0m"; | |
echo -e "{"; | |
echo -e " \"Statement\": ["; | |
echo -e " {"; | |
echo -e " \"Sid\": \"Stmt1394176741257\","; | |
echo -e " \"Action\": ["; | |
echo -e " \"ec2:DescribeInstances\","; | |
echo -e " \"ec2:RunInstances\","; | |
echo -e " \"ec2:StartInstances\","; | |
echo -e " \"ec2:StopInstances\","; | |
echo -e " \"ec2:DescribeVolumes\","; | |
echo -e " \"ec2:AttachVolume\","; | |
echo -e " \"ec2:CreateVolume\","; | |
echo -e " \"ec2:DetachVolume\","; | |
echo -e " \"ec2:DeleteVolume\","; | |
echo -e " \"ec2:DescribeImages\","; | |
echo -e " \"ec2:RegisterImage\","; | |
echo -e " \"ec2:DescribeSnapshots\","; | |
echo -e " \"ec2:CreateSnapshot\","; | |
echo -e " \"ec2:CopySnapshot\","; | |
echo -e " \"ec2:CreateImage\""; | |
echo -e " ],"; | |
echo -e " \"Effect\": \"Allow\","; | |
echo -e " \"Resource\": \"*\""; | |
echo -e " }"; | |
echo -e " ]"; | |
echo -e "}"; | |
echo -e ""; | |
echo -e "See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-policies-for-amazon-ec2.html"; | |
echo -e ""; | |
LOGGING ""; | |
LOGGING '{'; | |
LOGGING ' "Statement": ['; | |
LOGGING ' {'; | |
LOGGING ' "Sid": "Stmt1394176741257",'; | |
LOGGING ' "Action": ['; | |
LOGGING ' "ec2:DescribeInstances",'; | |
LOGGING ' "ec2:RunInstances",'; | |
LOGGING ' "ec2:StartInstances",'; | |
LOGGING ' "ec2:StopInstances",'; | |
LOGGING ' "ec2:DescribeVolumes",'; | |
LOGGING ' "ec2:AttachVolume",'; | |
LOGGING ' "ec2:CreateVolume",'; | |
LOGGING ' "ec2:DetachVolume",'; | |
LOGGING ' "ec2:DeleteVolume",'; | |
LOGGING ' "ec2:DescribeImages",'; | |
LOGGING ' "ec2:RegisterImage",'; | |
LOGGING ' "ec2:DescribeSnapshots",'; | |
LOGGING ' "ec2:CreateSnapshot",'; | |
LOGGING ' "ec2:CopySnapshot",'; | |
LOGGING ' "ec2:CreateImage"'; | |
LOGGING ' ],'; | |
LOGGING ' "Effect": "Allow",'; | |
LOGGING ' "Resource": "*"'; | |
LOGGING ' }'; | |
LOGGING ' ]'; | |
LOGGING '}'; | |
LOGGING "See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-policies-for-amazon-ec2.html"; | |
} | |
# -------------^^^------------- | |
# Start actual script | |
# -------------vvv------------- | |
# Display help section | |
if [[ $@ =~ "--help" ]]; then | |
echo -e "\nUsage: $0 <instance id> <options>\nExample: $0 i-xxxxxxxx --skip-checks \n\nSwitches:" | |
echo -e " --help Displays this help section"; | |
echo -e " --skip-checks Skip all prerequisite checks, including tools &\n permisions."; | |
echo -e " --force-reboot Reboot the source instance, without confirmation, when\n creating the temporary AMI"; | |
echo -e " --cleanup Cleanup temporary resource without prompting\n"; | |
exit 0 | |
elif [[ $@ =~ "--version" ]]; then | |
echo -e "\n$SCRIPTTITLE\n" | |
exit 0 | |
fi | |
# Print script banner | |
BANNER(){ | |
clear | |
echo -e "${RED}\n#####################################################################${NORMAL}"; | |
#echo -e ""; | |
echo -ne "\n${GREEN}"; | |
printf "%*s\n" $(((${#SCRIPTTITLE}+70)/2)) "$SCRIPTTITLE" | |
echo -e "${RED}"; | |
#echo -e ""; | |
echo -e "#####################################################################${NORMAL}\n"; | |
} | |
# Start logging | |
rm -rf $logfile; | |
touch $logfile; | |
LOGGING "\n$(/bin/basename $0) script log:\n-------------------------------\n\nResources used:\nvara\nvarb\nvarc\nvard\nvare\nvarf\nvarg\n\nWork log:\n--------\n" | |
# Checking prerequisites | |
if [[ ! $@ =~ "--skip-checks" ]]; then | |
LOGGING "----------------------"; | |
LOGGING "Checking prerequisites"; | |
LOGGING "---------vvv----------"; | |
BANNER; | |
echo -e "Checking script prerequisite\n"; | |
# Checking if script is executed as root | |
CHECK_ROOT | |
# Checking if this is a supported OS | |
CHECK_OS | |
# Checking is required packages are installed | |
CHECK_AWSCLI | |
CHECK_JQ | |
CHECK_PV | |
# Checking awscli credentials (config file or IAM role) | |
AWSCLI_CREDS | |
# Checking required IAM permissions | |
PERMISSIONS | |
LOGGING "------------^^^------------"; | |
LOGGING "Required prerequisites met"; | |
LOGGING "---------------------------\n"; | |
else | |
BANNER; | |
# Checking awscli credentials (config file or IAM role) | |
AWSCLI_CREDS | |
LOGGING "\nScript executed with --skip-checks.\nNo prerequisites checked.\n\n---------------------------------\n"; | |
fi | |
LOGGINGVAR "vara" "Temporary instance: $workinginstanceid"; | |
LOGGINGVAR "varb" "Availability Zone: $az"; | |
LOGGINGVAR "varg" "Region: $region"; | |
### Start ### | |
BANNER; | |
# Define / check source instance for valid instance id, if instance exist in same region & account | |
# -------------vvv------------- | |
srcinstancejson="$tmp/srcdescribe-instances" | |
sourcevolumes="$tmp/srcvol-list" | |
sourcesnapshots="$tmp/srcsnap-list" | |
srcvolumejson="$tmp/srcdescribe-volumes" | |
srcimagejson="$tmp/srcimage" | |
check_response="false" | |
ERROR(){ | |
echo -e "${RED}ERROR${NORMAL}: Script executed on source PV instance."; | |
echo -e " Script cannot be executed from source PV instance."; | |
echo -e " Launch a temoprary Amazon Linux instance to run the script."; | |
echo -e " Exiting script\n"; | |
LOGGING "ERROR: Script executed on source PV instance.\n"; | |
exit 1; | |
} | |
if [[ ! $1 =~ ^(--.*) ]]; then | |
sourceinstance=$1 | |
fi | |
if [[ -n $sourceinstance ]]; then | |
if [[ $sourceinstance =~ ^(i-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9])$ ]] || [[ $sourceinstance =~ ^(i-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9])$ ]]; then | |
# if [[ $sourceinstance =~ ^(i-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9])$ ]]; then | |
if [[ $sourceinstance == $workinginstanceid ]]; then | |
ERROR | |
fi | |
else | |
echo -e "${RED}$1${NORMAL} is not a valid instance id. Please provide a valid instance id:\e[0m"; | |
while [[ ${check_response} == "false" ]]; do | |
read -p "> "; | |
if [[ $sourceinstance =~ ^(i-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9])$ ]] || [[ $sourceinstance =~ ^(i-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9])$ ]]; then | |
# if [[ $sourceinstance =~ ^(i-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9])$ ]]; then | |
if [[ $sourceinstance == $workinginstanceid ]]; then | |
ERROR | |
fi | |
check_response="true" | |
else | |
echo -e "${RED}***Incorrect format${NORMAL} (Format: i-xxxxxxxx)"; | |
echo; | |
echo -e "${RED}Source PV instance:${NORMAL}"; | |
fi | |
done; | |
fi | |
else | |
echo -e "\e[1mSource PV instance:\e[0m"; | |
while [[ ${check_response} == "false" ]]; do | |
read -p "> "; | |
if [[ $REPLY =~ ^(i-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9])$ ]] || [[ $REPLY =~ ^(i-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9])$ ]]; then | |
# if [[ $REPLY =~ ^(i-[a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9][a-z0-9])$ ]]; then | |
if [[ $REPLY == $workinginstanceid ]]; then | |
ERROR | |
fi | |
check_response="true" | |
sourceinstance=$REPLY | |
else | |
echo -e "\${RED}***Incorrect format***${NORMAL} (Format: i-xxxxxxxx)"; | |
echo; | |
echo -e "${RED}Source PV instance:${NORMAL}"; | |
fi | |
done; | |
fi | |
unset -f ERROR | |
# Check if instance exists in region / account | |
API_CALL ec2 describe-instances --instance-ids $sourceinstance | |
if [[ $api_call_check =~ "InvalidInstanceID" ]] ; then | |
echo -e "${RED}$sourceinstance${NORMAL} does not exist in ${GREEN}$region${NORMAL}, or in account ${GREEN}$accountid${NORMAL}\n"; | |
echo -e "Confirm instance id and try again.\n"; | |
LOGGING "$sourceinstance does not exist in $region, or in account $accountid\n"; | |
exit 1; | |
fi | |
# Save source instance data - to be reused later | |
API_CALL ec2 describe-instances --instance-ids $sourceinstance | |
echo $api_call_check > $srcinstancejson | |
LOGGING "Source instance - $sourceinstance - json saved to $srcinstancejson:"; | |
LOGGING "-----------------------------------------------------------------------\n$(cat $srcinstancejson)${NORMAL}\n-----------------------------------------------------------------------\n\n" | |
LOGGINGVAR "varc" "Source Instance: $sourceinstance" | |
### Various sanity checks on source instance | |
# Check device names for trailing numbers | |
if [[ $(cat $srcinstancejson | jq -r '.Reservations[].Instances[].BlockDeviceMappings[] | select(.DeviceName != "/dev/sda1") | .DeviceName') =~ [1-9] ]]; then | |
echo -e "${RED}ERROR:${NORMAL} Instance ${RED}$sourceinstance${NORMAL} contains volumes attached as devices with\n trailing numbers:\n${GREEN}$(cat $srcinstancejson | jq -r '.Reservations[].Instances[].BlockDeviceMappings[] | select(.DeviceName != "/dev/sda1") | .DeviceName')${NORMAL}\n\nHVM instances / AMIs do not support the use of trailing numbers on\ndevice names:\n${GREEN}http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html${NORMAL}\n\nPlease re-attach these volumes device names without trailing numbers\nand try again. Exiting...\n"; | |
LOGGING "$sourceinstance contains volumes attached as devices with trailing numbers.\n$(cat $srcinstancejson | jq -r '.Reservations[].Instances[].BlockDeviceMappings')"; | |
exit 255 | |
fi | |
# Check if source instance is HVM | |
if [[ $(cat $srcinstancejson | jq -r '.Reservations[].Instances[].VirtualizationType') == "hvm" ]]; then | |
echo -e "${RED}ERROR:${NORMAL} Instance ${RED}$sourceinstance${NORMAL} is already HVM. No need to convert it.\n\nExiting...\n"; | |
LOGGING "$sourceinstance is an HVM instances:\n$(cat $srcinstancejson | jq '.Reservations[].Instances[]' | grep Virtualization)"; | |
exit 255 | |
fi | |
# Check if source instance is a Marketplace instance | |
if [[ $(cat $srcinstancejson | jq -r '.Reservations[].Instances[].ProductCodes[].ProductCodeType') == "marketplace" ]]; then | |
echo -e "${RED}ERROR:${NORMAL} Instance ${RED}$sourceinstance${NORMAL} has been launched from a Marketplacei AMI.\n Instances launched from a Marketplace AMI cannot be converted.\n\nYou should backup your data and migrate your workload to a new\nHVM instance manually. Exiting...\n"; | |
LOGGING "$sourceinstance is a Marketplace instance and is not supported.\n$(cat $srcinstancejson | jq -r '.Reservations[].Instances[].ProductCodes')"; | |
exit 255 | |
fi | |
# -------------^^^------------- | |
API_CALL ec2 describe-volumes --volume-ids $(cat $srcinstancejson | jq -r '.Reservations[].Instances[].BlockDeviceMappings[] | select(.Ebs) | .Ebs.VolumeId') | |
echo $api_call_check | jq -r '.Volumes[] | .Attachments[].Device, "-", .VolumeId, "-", .VolumeType, "-", .Iops' |\ | |
tr '\n' ' ' |\ | |
sed -e 's*\/d*\n\/d*g' |\ | |
grep vol > $sourcevolumes | |
LOGGING "Source volumes / devices:" | |
LOGGING "------------------------\n$(cat $sourcevolumes)\n------------------------\n\n" | |
# Create temporary AMI from source instance. | |
# -------------vvv------------- | |
AMI_CHECK(){ | |
if [[ $tempami =~ "Server.InternalError" ]]; then | |
sleep 30 | |
check_ami_response="false" | |
while [[ $check_ami_response != "true" ]]; do | |
API_CALL ec2 create-image --instance-id $sourceinstance --name Temp-Image-PV-HVM-Conversion.$$ --description Temporary_AMI_created_from_source_instance_$sourceinstance --reboot | |
tempami="(echo $api_call_check | jq -r '.ImageId')" | |
API ec2 describe-images --image-ids $tempami | |
if [[ ! $api_call_check =~ "Server.InternalError" ]]; then | |
check_ami_response="true" | |
else | |
sleep 10 | |
fi | |
done | |
fi | |
} | |
LOGGING "Temporary AMI:\n-------------\n"; | |
if [[ ! $@ =~ "--force-reboot" ]]; then | |
API_CALL ec2 describe-instances --instance-ids $sourceinstance | |
if [[ $(echo $api_call_check | jq -r '.Reservations[].Instances[].State.Name') != "stopped" ]]; then | |
echo -e "Source instance is running." | |
check_response="false" | |
echo -e "\nReboot instance to create temporary AMI? (y/n)"; | |
while [[ $check_response == "false" ]]; do | |
read -p "> "; | |
if [[ $REPLY =~ ^([yY][eE][sS]|[yY])$ ]]; then | |
echo -e "Creating temporary AMI from instance: ${RED}$sourceinstance${NORMAL}"; | |
API_CALL ec2 create-image --instance-id $sourceinstance --name Temp-Image-PV-HVM-Conversion.$$ --description Temporary_AMI_created_from_source_instance_$sourceinstance --reboot | |
tempami=$(echo $api_call_check | jq -r '.ImageId') | |
LOGGING "Temporary AMI creation started - with reboot option selected:\n$api_call_check" | |
# AMI_CHECK | |
check_response="true" | |
elif [[ $REPLY =~ ^([nN][oO]|[nN])$ ]]; then | |
echo -e "\nFilesystem integrity cannot be guaranteed. Creating temporary AMI\nfrom ${RED}$sourceinstance${NORMAL}, without reboot..."; | |
API_CALL ec2 create-image --instance-id $sourceinstance --name Temp-Image-PV-HVM-Conversion.$$ --description Temporary_AMI_created_from_source_instance_$sourceinstance --no-reboot | |
tempami=$(echo $api_call_check | jq -r '.ImageId') | |
create_ami_response="true" | |
LOGGING "Temporary AMI creation started - with no-reboot option selected:\n$api_call_check" | |
check_response="true" | |
else | |
echo -e "Yes or No"; | |
fi | |
done | |
else | |
echo -e "\nCreating temporary AMI from instance: ${RED}$sourceinstance${NORMAL}"; | |
API_CALL ec2 create-image --instance-id $sourceinstance --name Temp-Image-PV-HVM-Conversion.$$ --description Temporary_AMI_created_from_source_instance_$sourceinstance --no-reboot | |
tempami=$(echo $api_call_check | jq -r '.ImageId') | |
create_ami_response="true" | |
LOGGING "Temporary AMI creation started - source instance in stopped state:\n$api_call_check" | |
fi | |
else | |
echo -e "\nCreating temporary AMI from instance: ${RED}$sourceinstance${NORMAL}\n--force-reboot option was used:\nRebooting ${RED}$sourceinstance${NORMAL} (if running)"; | |
API_CALL ec2 create-image --instance-id $sourceinstance --name Temp-Image-PV-HVM-Conversion.$$ --description Temporary_AMI_created_from_source_instance_$sourceinstance --reboot | |
tempami=$(echo $api_call_check | jq -r '.ImageId') | |
LOGGING "Temporary AMI creation started - source instance was rebooted (--force-reboot option):\n$api_call_check" | |
fi | |
echo -e "\nThis process may take a very long time if the instance has large\nvolumes attached.\nPlease be patient and allow this process to finish.\n"; | |
echo -ne "Creating AMI"; | |
MINIPROGRESS | |
checkprogress="incomplete" | |
while [[ $checkprogress != "completed" ]]; do | |
API_CALL ec2 describe-images --image-ids $tempami | |
if [[ $(echo $api_call_check | jq -r '.Images[].State') != "available" ]]; then | |
if [[ $(echo $api_call_check | jq -r '.Images[].State') == "pending" ]]; then | |
MAXPROGRESS | |
elif [[ $(echo $api_call_check | jq -r '.Images[].State') == "failed" ]]; then | |
echo -e "${RED}[FAIL]${NORMAL}\n" | |
echo -e "\nAMI Creation failed - Reason:\n${RED}$(echo $api_call_check | jq -r '.Images[].StateReason.Message')${NORMAL}" | |
LOGGING "AMI Creation failed. Reason:\n$(echo $api_call_check | jq -r '.Images[].StateReason.Message')\n" | |
exit 255 | |
fi | |
else | |
checkprogress="completed" | |
fi | |
done | |
API_CALL ec2 describe-images --image-ids $tempami | |
echo $api_call_check > $srcimagejson | |
# Get block device mapping from source image created previously, removing any "VirtualName": "ephemeral0" from the results. | |
cat $srcimagejson | jq -r '.Images[].BlockDeviceMappings[] | select(.Ebs) | .DeviceName, "*", .Ebs.SnapshotId, "*", .Ebs.VolumeSize, "*", .Ebs.VolumeType, "*", .Ebs.Iops, ","' | tr '\n' ' ' | sed -e 's*\ ,*\n*g' -e 's* \/*\/*g' -e 's*\ **g' > $sourcesnapshots | |
for completed in $(cat $srcimagejson | jq -r '.Images[].BlockDeviceMappings[] | select(.Ebs) | .Ebs.SnapshotId'); do | |
MINIPROGRESS | |
checkprogess="incomplete" | |
while [[ $checkprogress != "completed" ]]; do | |
API_CALL ec2 describe-snapshots --snapshot-ids $completed | |
if [[ $(echo $api_call_check | jq -r '.Snapshots[].State') != "completed" ]]; then | |
PROGRESS | |
else | |
checkprogress="completed" | |
LOGGING "Snapshot - $completed completed successfully"; | |
fi | |
done; | |
done; | |
echo -e "${GREEN}[OK]${NORMAL}"; | |
LOGGING "\nAMI:\n$(cat $srcimagejson)\n" | |
LOGGING "Source snapshots / devices:" | |
LOGGING "$(cat $sourcesnapshots)\n-------------\n" | |
LOGGINGVAR "vard" "Temporary AMI: $tempami" | |
# -------------^^^------------- | |
# Creating source volume, attaching, fsck and resizing | |
# -------------vvv------------- | |
# Creating source root volume | |
echo -ne "\nCreating temporary source volume"; | |
srcrootsnap="$(cat $sourcesnapshots | grep -i 'sda\|xvde' | awk -F "*" '{ print $2 }')" | |
srcvolsize="$(cat $sourcesnapshots | grep -i 'sda\|xvde' | awk -F "*" '{ print $3 }')" | |
srcvoltype="$(cat $sourcesnapshots | grep -i 'sda\|xvde' | awk -F "*" '{ print $4 }')" | |
if [[ $srcvoltype == "io1" ]]; then | |
srcvoliops="$(cat $sourcesnapshots | grep -i 'sda\|xvde' | awk -F "*" '{ print $5 }')" | |
API_CALL ec2 create-volume --snapshot-id $srcrootsnap --availability-zone $az --size $srcvolsize --volume-type $srcvoltype --iops $srcvoliops | |
LOGGING "Source volume create:\n--------------------\nVolume: $(echo $api_call_check | jq -r '.VolumeId')\nSnapshot: $srcrootsnap\nSize: $srcvolsize\nType: $srcvoltype\nIOPS: $srcvoliops" | |
LOGGING "$api_call_check\n" | |
else | |
API_CALL ec2 create-volume --snapshot-id $srcrootsnap --availability-zone $az --size $srcvolsize --volume-type $srcvoltype | |
LOGGING "Source volume create:\n--------------------\nVolume: $(echo $api_call_check | jq -r '.VolumeId')\nSnapshot: $srcrootsnap\nSize: $srcvolsize\nType: $srcvoltype" | |
LOGGING "$api_call_check" | |
fi | |
srcvolume="$(echo $api_call_check | jq -r '.VolumeId')" | |
PROGRESS | |
check_progress="incomplete" | |
while [[ $check_progress != "completed" ]]; do | |
API_CALL ec2 describe-volumes --volume-ids $srcvolume | |
if [[ $(echo $api_call_check | jq -r '.Volumes[].State') != "available" ]]; then | |
PROGRESS | |
else | |
check_progress="completed" | |
fi | |
done | |
echo -e "${GREEN}$srcvolume${NORMAL}"; | |
LOGGINGVAR "vare" "Source volume: $srcvolume" | |
# Attaching source root volume to temporary instance | |
sourcedev="/dev/sdj" | |
sourcedevicemapping=$(echo $workinginstancejson | jq -r '.Reservations[].Instances[].BlockDeviceMappings[].DeviceName') | |
if [[ -z $(echo $sourcedevicemapping | \grep "$sourcedev") ]]; then | |
echo -ne "\nAttaching ${GREEN}$srcvolume${NORMAL} as ${RED}$sourcedev${NORMAL}"; | |
MINIPROGRESS | |
API_CALL ec2 attach-volume --volume-id $srcvolume --instance-id $workinginstanceid --device $sourcedev | |
check_state="false" | |
while [[ $check_state != "true" ]]; do | |
MAXPROGRESS | |
API_CALL ec2 describe-volumes --volume-ids $srcvolume | |
if [[ $(echo $api_call_check | jq -r '.Volumes[].State') != "in-use" ]]; then | |
MINIPROGRESS | |
else | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
check_state="true" | |
LOGGING "Attached $srcvolume:"; | |
LOGGING "$api_call_check" | |
fi | |
done | |
else | |
localdevice=('' {b..z}) | |
echo -e "\nDefault device name, ${RED}$sourcedev${NORMAL} in use. Select another device:\n"; | |
for (( I = 1; I <= 25; ++I )); do | |
echo -e "/dev/sd${localdevice[I]}" | grep -v "$(lsblk -l | awk '{ print $1 }' | grep -i "^x" | sed -e 's/^xv/s/g')" | |
done | |
check_response="false" | |
while [[ $check_response != "true" ]]; do | |
read -p "> "; | |
if [[ $REPLY =~ ^([\/]dev[\/]sd[b-z])$ ]]; then | |
if [[ -z $(echo $sourcedevicemapping | \grep "$REPLY") ]]; then | |
check_response="true" | |
sourcedev="$REPLY" | |
else | |
check_response="false" | |
BANNER; | |
echo -e "Device name, ${RED}$REPLY${NORMAL} in use. Select another device:\n"; | |
for (( I = 1; I <= 25; ++I )); do | |
echo -e "/dev/sd${localdevice[I]}" | grep -v "$(lsblk -l | awk '{ print $1 }' | grep -i "^x" | sed -e 's/^xv/s/g')" | |
done | |
fi | |
else | |
check_response="false" | |
BANNER | |
echo -e "${RED}$REPLY${NORMAL} is not a valid selection. Select from the list of devices below:\n"; | |
for (( I = 1; I <= 25; ++I )); do | |
echo -e "/dev/sd${localdevice[I]}" | grep -v "$(lsblk -l | awk '{ print $1 }' | grep -i "^x" | sed -e 's/^xv/s/g')" | |
done | |
fi | |
done | |
BANNER | |
echo -ne "Attaching ${GREEN}$srcvolume${NORMAL} as ${RED}$sourcedev${NORMAL}" | |
API_CALL ec2 attach-volume --volume-id $srcvolume --instance-id $workinginstanceid --device $sourcedev | |
check_state="false" | |
while [[ $check_state != "true" ]]; do | |
MAXPROGRESS | |
API_CALL ec2 describe-volumes --volume-ids $srcvolume | |
if [[ $(echo $api_call_check | jq -r '.Volumes[].State') != "in-use" ]]; then | |
MINIPROGRESS | |
else | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
check_state="true" | |
LOGGING "Attached $srcvolume:"; | |
LOGGING "$api_call_check\n" | |
fi | |
done | |
fi | |
# Get xen device names | |
echo -e "\nPreparing source device:\n"; | |
sourcedev1="$(echo "$sourcedev" | awk -F 'd' '{print $NF}')" | |
while [[ -z $source ]]; do | |
for f in $(lsblk -l | grep -i "xvd$sourcedev1" | awk '{ print $1 }'); do | |
mount /dev/$f $sysroot > /dev/null 2>&1; | |
if [[ -d $sysroot/etc ]]; then | |
export source="/dev/$f"; | |
fi; | |
umount $sysroot > /dev/null 2>&1; | |
done; | |
LOGGING "Attached as: $sourcedev\nInternal device: $source\n--------------------\n" | |
done | |
echo -e "This process will take a very long time (${RED}Up to a couple of hours${NORMAL}) if\nchecking a large source volume. Please be patient and allow this\nprocess to finish.\n"; | |
LOGGING "Checking source volume:\n----------------------\n"; | |
echo -e "Checking ${RED}$source${NORMAL}:"; | |
# Check if grub had been installed | |
mount $source $sysroot | |
which $sysroot/sbin/grub > /dev/null 2>&1 | |
status_0=$? | |
which $sysroot/usr/sbin/grub > /dev/null 2>&1 | |
status_1=$? | |
if [[ "$status_0" -ne 0 ]] && [[ "$status_1" -ne 0 ]] ; then | |
echo -e "Grub has not been installed on $sourceinstance. Please install grub and run the script again. Exiting...\n"; | |
LOGGING "Grub is not installed on $sourceinstance"; | |
umount $sysroot | |
sleep 5 | |
echo -ne "Removing temporary resources:\n----------------------------\n"; | |
LOGGING "Removing temporary resources:"; | |
MINIPROGRESS | |
API_CALL ec2 deregister-image --image-id $tempami | |
LOGGING "Deregistering $tempami:\n$api_call_check\n"; | |
MINIPROGRESS | |
for cleanup in $(cat $sourcesnapshots | awk -F "*" '{ print $2 }'); do | |
API_CALL ec2 delete-snapshot --snapshot-id $cleanup | |
LOGGING "Deleted $cleanup"; | |
done; | |
API_CALL ec2 detach-volume --volume-id $srcvolume | |
check_response="false" | |
while [[ $check_response != "true" ]]; do | |
API_CALL ec2 describe-volumes --volume-ids $srcvolume | |
if [[ $(echo $api_call_check | jq -r '.Volumes[].State') != "available" ]]; then | |
PROGRESS | |
else | |
check_response="true" | |
LOGGING "\n$srcvolume detached:\n$api_call_check\n"; | |
fi | |
done | |
API_CALL ec2 delete-volume --volume-id $srcvolume | |
LOGGING "$srcvolume deleted"; | |
for delete in $srcinstancejson $sourcevolumes $sourcesnapshots $srcvolumejson $srcimagejson $finalsnap $mapping; do | |
(rm -vf $delete) | tee -a $logfile > /dev/null 2>&1 | |
done; | |
LOGGING "Temporary files deleted\n\n----------------------------\n" | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
exit 1 | |
else | |
echo -e "\nConfirmed grub installation on ${GREEN}$srcvolume${NORMAL}\n"; | |
LOGGING "Confirmed grub installation on $srcvolume\n"; | |
umount $sysroot | |
sleep 2 | |
fi | |
# Execute fsck before resizing source partition | |
echo -e "Checking $source for any errors:"; | |
LOGGING "e2fsck $source:"; | |
(e2fsck -vfp "$source") 2>&1 | tee -a $logfile | |
# Check to see if fsck was successful and resize source partition | |
status=$? | |
if [[ "$status" = 0 ]] ; then | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
LOGGING "fsck check completed successfully"; | |
echo -e "Reducing ${RED}$source${NORMAL} size:"; | |
LOGGING "\nReducing $source size:"; | |
# Create a variable with the output of resize2fs, to be used to decided the size of the destination volume later and then log the output. | |
resize_check="$(resize2fs -d 16 -M "$source" 2>&1)" && echo $resize_check >> $logfile | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
else | |
echo -e "${RED}[FAIL]${NORMAL}\n $source will not be resized."; | |
export fsck_failed="true" | |
LOGGING "fsck check on $source failed - skip resizing."; | |
fi | |
echo -e "Dumping ${RED}$source${NORMAL} info:"; | |
dump=$(dumpe2fs -h "$source") | |
block_size=$(awk -F':[ \t]+' '/^Block size:/ {print $2}' <<<"$dump") | |
block_count=$(awk -F':[ \t]+' '/^Block count:/ {print $2}' <<<"$dump") | |
LOGGING "$source dump2fs output:\n$dump\n\n$source block size: $block_size\n$source block count: ${RED}$block_count${NORMAL}\n----------------------\n"; | |
echo -e "${GREEN}[OK]${NORMAL}"; | |
# -------------^^^------------- | |
# Creating destination volume and partitioning | |
# -------------vvv------------- | |
# Create destination root volume | |
# Set destination volume size, based on fsck result of source partition | |
BANNER; | |
echo -ne "\nCreating temporary destination volume"; | |
# If the volume could not be resized, create the temporary destination volume size as source volume + 1GB, otherwise dd will fail with the destination volume running out of space (as a partition will be created on the destination volume. | |
if [[ $fsck_failed == "true" ]] || [[ $resize_check =~ "Nothing to do" ]]; then | |
dstvolsize=$(expr $srcvolsize + 1) | |
LOGGING "Temporary destination volume size increased to ${dstvolsize}GB, as resize2fs could not resize the volume. Reason:" | |
if [[ $fsck_failed == "true" ]]; then | |
LOGGING "fsck check failed, resulting in resizing the volume not being executed" | |
elif [[ $resize_check =~ "Nothing to do" ]]; then | |
LOGGING "resize2fs could not shrik the filesystem on the source volume - already at minimum size" | |
fi | |
else | |
dstvolsize="$srcvolsize" | |
LOGGING "Temporary destination volume size = $dstvolume" | |
fi | |
dstvoltype="$srcvoltype" | |
if [[ $dstvoltype == "io1" ]]; then | |
dstvoliops="$(cat $sourcesnapshots | grep -i 'sda\|xvde' | awk -F "*" '{ print $5 }')" | |
API_CALL ec2 create-volume --availability-zone $az --size $dstvolsize --volume-type $dstvoltype --iops $dstvoliops | |
LOGGING "Destination volume create:\n--------------------\nVolume: $(echo $api_call_check | jq -r '.VolumeId')\nSize: $dstvolsize\nType: $dstvoltype\nIOPS: $dstvoliops"; | |
LOGGING "$api_call_check\n"; | |
else | |
API_CALL ec2 create-volume --availability-zone $az --size $dstvolsize --volume-type $dstvoltype | |
LOGGING "Destination volume create:\n--------------------\nVolume: $(echo $api_call_check | jq -r '.VolumeId')\nSize: $dstvolsize\nType: $dstvoltype"; | |
LOGGING "$api_call_check\n"; | |
fi | |
dstvolume="$(echo $api_call_check | jq -r '.VolumeId')" | |
check_progress="incomplete" | |
MINIPROGRESS | |
while [[ $check_progress != "completed" ]]; do | |
API_CALL ec2 describe-volumes --volume-ids $dstvolume | |
if [[ $(echo $api_call_check | jq -r '.Volumes[].State') != "available" ]]; then | |
PROGRESS | |
else | |
check_progress="completed" | |
fi | |
done | |
echo -e "${GREEN}$dstvolume${NORMAL}"; | |
LOGGINGVAR "varf" "Desitnation volume - $dstvolume" | |
# Refresh workinginstancejson, to include attached source devices | |
API_CALL ec2 describe-instances --instance-ids $workinginstanceid | |
workinginstancejson="$api_call_check" | |
# Attaching destination volume to temporary instance | |
destdev="/dev/sdk" | |
destdevicemapping=$(echo $workinginstancejson | jq -r '.Reservations[].Instances[].BlockDeviceMappings[].DeviceName') | |
if [[ -z $(echo $destdevicemapping | \grep "$destdev") ]]; then | |
echo -ne "\nAttaching ${GREEN}$dstvolume${NORMAL} as ${RED}$destdev${NORMAL}"; | |
MINIPROGRESS | |
API_CALL ec2 attach-volume --volume-id $dstvolume --instance-id $workinginstanceid --device $destdev | |
check_state="false" | |
while [[ $check_state != "true" ]]; do | |
MAXPROGRESS | |
API_CALL ec2 describe-volumes --volume-ids $dstvolume | |
if [[ $(echo $api_call_check | jq -r '.Volumes[].State') != "in-use" ]]; then | |
MINIPROGRESS | |
else | |
echo "${GREEN}[OK]${NORMAL}"; | |
check_state="true" | |
LOGGING "Attached $dstvolume:"; | |
LOGGING "$api_call_check" | |
fi | |
done | |
else | |
localdevice=('' {b..z}) | |
echo -e "\nDefault device name, ${RED}$destdev${NORMAL} in use. Select another device:\n"; | |
for (( I = 1; I <= 25; ++I )); do | |
echo -e "/dev/sd${localdevice[I]}" | grep -v "$(lsblk -l | awk '{ print $1 }' | grep -i "^x" | sed -e 's/^xv/s/g')" | |
done | |
check_response="false" | |
while [[ $check_response != "true" ]]; do | |
read -p "> "; | |
if [[ $REPLY =~ ^([\/]dev[\/]sd[b-z])$ ]]; then | |
if [[ -z $(echo $destdevicemapping | \grep "$REPLY") ]]; then | |
check_response="true" | |
destdev="$REPLY" | |
else | |
check_response="false" | |
BANNER; | |
echo -e "Device name, ${RED}$REPLY${NORMAL} in use. Select another device:\n"; | |
for (( I = 1; I <= 25; ++I )); do | |
echo -e "/dev/sd${localdevice[I]}" | grep -v "$(lsblk -l | awk '{ print $1 }' | grep -i "^x" | sed -e 's/^xv/s/g')" | |
done | |
fi | |
else | |
check_response="false" | |
BANNER | |
echo -e "${RED}$REPLY${NORMAL} is not a valid selection. Select from the list of devices below:\n"; | |
for (( I = 1; I <= 25; ++I )); do | |
echo -e "/dev/sd${localdevice[I]}" | grep -v "$(lsblk -l | awk '{ print $1 }' | grep -i "^x" | sed -e 's/^xv/s/g')" | |
done | |
fi | |
done | |
BANNER | |
echo -ne "Attaching ${GREEN}$dstvolume${NORMAL} as ${RED}$destdev${NORMAL}" | |
API_CALL ec2 attach-volume --volume-id $dstvolume --instance-id $workinginstanceid --device $destdev | |
check_state="false" | |
while [[ $check_state != "true" ]]; do | |
MAXPROGRESS | |
API_CALL ec2 describe-volumes --volume-ids $dstvolume | |
if [[ $(echo $api_call_check | jq -r '.Volumes[].State') != "in-use" ]]; then | |
MINIPROGRESS | |
else | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
check_state="true" | |
LOGGING "Attached $dstvolume:"; | |
LOGGING "$api_call_check\n" | |
fi | |
done | |
fi | |
# Get xen device names | |
destdev1="$(echo "$destdev" | awk -F 'd' '{print $NF}')" | |
while [[ -z $destination ]]; do | |
if [[ $(file "/dev/xvd$destdev1" | \grep -o block) == "block" ]]; then | |
destination="/dev/xvd$destdev1" | |
sleep 5; | |
fi | |
done | |
LOGGING "Attached as: $destdev\nInternal device: $destination\n--------------------\n" | |
# Partition destination volume | |
echo -e "Partitioning ${RED}$destination${NORMAL}:"; | |
LOGGING "Partitioning $destination:"; | |
(parted $destination --script 'mklabel msdos mkpart primary 1M -1s print quit') 2>&1 | tee -a $logfile | |
(partprobe $destination) | tee -a $logfile | |
(udevadm settle) | tee -a $logfile | |
echo -e "\n${GREEN}[OK]${NORMAL}\n"; | |
# -------------^^^------------- | |
# Duplicating source partition to destination partition, resizing and mounting and preparing destination partition | |
# -------------vvv------------- | |
# Duplicate source to destination | |
echo -e "Duplicating ${RED}$source${NORMAL} to ${RED}$destination${NORMAL}:"; | |
LOGGING "Duplicated $source to $destination device\n"; | |
if [[ $pv = true ]]; then | |
total_size="$(( $block_count * $block_size))" | |
(dd if=$source bs=$block_size count=$block_count | pv -tpaeb --no-splice -s $total_size | dd of=${destination}1 bs=$block_size count=$block_count) | tee -a $logfile | |
else | |
(dd if=$source bs=$block_size count=$block_count of=${destination}1) | tee -a $logfile | |
fi | |
echo -e "\n${GREEN}[OK]${NORMAL}\n"; | |
# Resizing destination | |
echo -e "Resizing ${RED}$destination${NORMAL} to fill volume:\n"; | |
LOGGING "Resizing ${destinatio}1:"; | |
(resize2fs -d 16 ${destination}1) 2>&1 | tee -a $logfile | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
# Mount destination filesystem to mount point | |
echo -e "Mount ${RED}$destination${NORMAL} to ${GREEN}$sysroot${NORMAL}"; | |
LOGGING "Mount $destination to $sysroot:"; | |
(mount -v ${destination}1 $sysroot) 2>&1 | tee -a $logfile | |
echo -e "\n${GREEN}[OK]${NORMAL}\n"; | |
# Prepare devices on destination for grub installation | |
echo -e "Preparing ${RED}$destination${NORMAL} for chroot environment:"; | |
LOGGING "Copy $destination and ${destination}1 to $sysroot/dev/"; | |
(cp -va $destination ${destination}1 $sysroot/dev/) | tee -a $logfile > /dev/null 2>&1 | |
LOGGING "---------------------\n"; | |
echo -e "\n${GREEN}[OK]${NORMAL}\n"; | |
# -------------^^^------------- | |
# Making changes to destination volume | |
# -------------vvv------------- | |
BANNER | |
echo -e "\nConverting ${RED}$dstvolume${NORMAL} to HVM:\n"; | |
LOGGING "Convert $dstvolume to HVM:\n------------------------"; | |
# Installing grub | |
LOGGING "Installing grub:"; | |
(rm -vf $sysroot/boot/grub/*stage*) | tee -a $logfile > /dev/null 2>&1 | |
#(cp -v $sysroot/usr/*/grub/*/*stage* $sysroot/boot/grub/) | tee -a $logfile > /dev/null 2>&1 | |
(find $sysroot/usr -type f -path "*grub*stage*" -exec cp -fv {} $sysroot/boot/grub/ \;) | tee -a $logfile > /dev/null 2>&1 | |
(rm -vf $sysroot/boot/grub/device.map) | tee -a $logfile > /dev/null 2>&1 | |
echo -e "${NORMAL}Installing grub\n"; | |
(cat <<EOF | chroot $sysroot grub --batch | |
device (hd0) $destination | |
root (hd0,0) | |
setup (hd0) | |
EOF | |
) 2>&1 | tee -a $logfile | |
LOGGING "Removing temp device files:"; | |
(rm -vf ${sysroot}${destination} ${sysroot}${destination}1) | tee -a $logfile > /dev/null 2>&1 | |
echo -e "\n${GREEN}[OK]${NORMAL}\n"; | |
# Fixing grub.conf / menu.lst | |
echo -e "Fixing grub config"; | |
LOGGING "\nModified grub config:"; | |
(e2label ${destination}1 root) | tee -a $logfile > /dev/null 2>&1 & | |
sleep 10; # Give everything time to setting before hacking grub.conf | |
for grub in grub.conf menu.lst; do | |
if [[ -L $sysroot/boot/grub/$grub ]]; then | |
LOGGING "$sysroot/boot/grub/$grub is a symlink" | |
else | |
sed -i 's/(hd0)/(hd0,0)/' $sysroot/boot/grub/$grub > /dev/null 2>&1 | |
sed -i 's/root=\([^ ]*\)/root=LABEL=root/' $sysroot/boot/grub/$grub > /dev/null 2>&1 | |
sed -i 's/console*\([^ ]*\)//' $sysroot/boot/grub/$grub > /dev/null 2>&1 | |
if [[ -f $sysroot/boot/grub/$grub ]]; then | |
if [[ ! $(cat $sysroot/boot/grub/$grub | grep -i "^kernel") =~ "console" ]]; then | |
sed -i '/^kernel/{s/$/ console\=ttyS0/}' $sysroot/boot/grub/$grub; | |
fi | |
fi | |
fi | |
done | |
if [[ -f $sysroot/boot/grub/menu.lst ]]; then | |
LOGGING "$(cat $sysroot/boot/grub/menu.lst)\n"; | |
else | |
LOGGING "$(cat $sysroot/boot/grub/grub.conf)\n"; | |
fi | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
# Fix fstab | |
echo -e "Fixing fstab config"; | |
LOGGING "Modified fstab config:"; | |
sed -i 's/LABEL=[^\s]/LABEL=root/i' $sysroot/etc/fstab | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
LOGGING "$(cat $sysroot/etc/fstab)\n"; | |
# Unmount $sysroot | |
LOGGING "Unmount $dstvolume:"; | |
(umount $sysroot) | tee -a $logfile > /dev/null 2>&1 | |
sleep 5; | |
LOGGING "\n$dstvolume successfully converted to HVM"; | |
LOGGING "--------------------------------------\n"; | |
# -------------^^^------------- | |
# Finalize process and create AMI | |
# -------------vvv------------- | |
# Creating snapshot and registering AMI | |
LOGGING "Registering snapshot(s) and creating image (AMI):\n------------------------------------------------\n"; | |
BANNER; | |
# Figure out root snap / volume size with final AMI creation... | |
finalsnap="$tmp/final-snapshots" | |
rm -f $finalsnap | |
echo -ne "Creating a snapshot of ${RED}$dstvolume${NORMAL}"; | |
check_response="false" | |
API_CALL ec2 create-snapshot --volume-id $dstvolume --description converted_HVMAMI_root | |
rootsnap="$(echo $api_call_check | jq -r '.SnapshotId')" | |
rootsnap_size="$(echo $api_call_check | jq -r '.VolumeSize')" | |
LOGGING "Creating $rootsnap:"; | |
while [[ $check_response != "true" ]]; do | |
MAXPROGRESS | |
API_CALL ec2 describe-snapshots --snapshot-ids $rootsnap | |
if [[ $(echo $api_call_check | jq -r '.Snapshots[].State') != "completed" ]]; then | |
PROGRESS | |
else | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
check_response="true" | |
echo -ne "$rootsnap*/dev/sda1*" >> $finalsnap; | |
(echo -ne $rootsnap_size; cat $sourcesnapshots | grep -i '/dev/sda\|/dev/xvde' | awk -F "*" '{print "*"$4"*"$5 }') >> $finalsnap | |
LOGGING "$api_call_check\n"; | |
fi | |
done | |
# Copy additional snapshots | |
echo -ne "Finalizing snapshot(s) for final AMI"; | |
for snap in $(cat $sourcesnapshots | grep -v '/dev/sda\|/dev/xvde'); do | |
region=$region | |
srcdev="$(echo $snap | awk -F "*" '{ print $1 }')"; | |
srcsnap="$(echo $snap | awk -F "*" '{ print $2 }')"; | |
srcsize="$(echo $snap | awk -F "*" '{ print $3 }')"; | |
srctype="$(echo $snap | awk -F "*" '{ print $4 }')"; | |
srciops="$(echo $snap | awk -F "*" '{ print $5 }')"; | |
check_response="false" | |
API_CALL ec2 copy-snapshot --source-region $region --destination-region $region --source-snapshot-id $srcsnap --description converted_HVMAMI_additionalvolume | |
tempsnap="$(echo $api_call_check | jq -r '.SnapshotId')" | |
LOGGING "Creating $tempsnap from $srcsnap:"; | |
while [[ $check_response != "true" ]]; do | |
API_CALL ec2 describe-snapshots --snapshot-ids $tempsnap | |
if [[ $(echo $api_call_check | jq -r '.Snapshots[].State') != "completed" ]]; then | |
MAXPROGRESS | |
else | |
check_response="true" | |
echo -ne "$tempsnap*$srcdev*" >> $finalsnap; | |
echo -e "$(cat $sourcesnapshots | grep -i $srcsnap | awk -F "*" '{ print $3"*"$4"*"$5 }')" >> $finalsnap; | |
LOGGING "$api_call_check\n"; | |
fi | |
done | |
done | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
# Create block device mapping file | |
mapping="$tmp/mapping" | |
echo -e "[\n" > $mapping; | |
for snap in $(cat $finalsnap); do | |
snapshot="$(echo $snap | awk -F "*" '{ print $1 }')"; | |
device="$(echo $snap | awk -F "*" '{ print $2 }')"; | |
size="$(echo $snap | awk -F "*" '{ print $3 }')"; | |
voltype="$(echo $snap | awk -F "*" '{ print $4 }')"; | |
iops="$(echo $snap | awk -F "*" '{ print $5 }')"; | |
if [[ $voltype == "io1" ]]; then | |
echo -e " {\n \"VirtualName\": \"ebs\",\n \"DeviceName\": \"$device\",\n \"Ebs\": {\n \"SnapshotId\": \"$snapshot\",\n \"VolumeSize\": $size,\n \"VolumeType\": \"$voltype\",\n \"Iops\": $iops\n }\n }," >> $mapping; | |
else | |
echo -e " {\n \"VirtualName\": \"ebs\",\n \"DeviceName\": \"$device\",\n \"Ebs\": {\n \"SnapshotId\": \"$snapshot\",\n \"VolumeSize\": $size,\n \"VolumeType\": \"$voltype\"\n }\n }," >> $mapping; | |
fi | |
done | |
# Add ephemeral volumes from initila src image (if any) and add to mapping file | |
cat $srcimagejson | jq -r '.Images[].BlockDeviceMappings[] | select(.VirtualName)' | sed -e 's/\}/\},/g' >> $mapping | |
# Remove trailing "," from jq output and end off the mapping file with "]" | |
head -n -1 $mapping > $tmp/tmpmap; | |
\mv $tmp/tmpmap $mapping; | |
echo -e " }\n]" >> $mapping; | |
LOGGING "BlockDeviceMapping for final AMI:\n$(cat $mapping)"; | |
# Register AMI | |
echo -ne "Registering final HVM AMI"; | |
API_CALL ec2 register-image --name Converted_HVM_${sourceinstance}.$$ --description HVM_AMI_created_from_source_instance_${sourceinstance} --architecture x86_64 --root-device-name /dev/sda1 --virtualization-type hvm --block-device-mappings file://$mapping | |
ami="$(echo $api_call_check | jq -r '.ImageId')" | |
PROGRESS | |
check_response="false" | |
while [[ $check_response != "true" ]]; do | |
API_CALL ec2 describe-images --image-id $ami | |
if [[ $(echo $api_call_check | jq -r '.Images[].State') != "available" ]]; then | |
PROGRESS | |
else | |
check_response="true" | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
LOGGING "Final AMI - $ami:\n$api_call_check\n"; | |
fi | |
done | |
echo -e "\nHVM AMI creation complete. AMI = ${GREEN}$ami${NORMAL}\n"; | |
echo -e "$ami" > $tmp/ami-d | |
sleep 2 | |
### Cleanup | |
BANNER | |
CLEANUP(){ | |
echo -ne "\nRemoving temporary resources"; | |
MINIPROGRESS | |
API_CALL ec2 deregister-image --image-id $tempami | |
LOGGING "Deregistered temporaory ami - $tempami:\n$api_call_check"; | |
MINIPROGRESS | |
for cleanup in $(cat $sourcesnapshots | awk -F "*" '{ print $2 }'); do | |
API_CALL ec2 delete-snapshot --snapshot-id $cleanup | |
LOGGING "Deleted $cleanup"; | |
MINIPROGRESS | |
done | |
API_CALL ec2 detach-volume --volume-id $srcvolume | |
check_subresponse="false" | |
while [[ $check_subresponse != "true" ]]; do | |
API_CALL ec2 describe-volumes --volume-ids $srcvolume | |
if [[ $(echo $api_call_check | jq -r '.Volumes[].State') != "available" ]]; then | |
PROGRESS | |
else | |
check_subresponse="true" | |
LOGGING "Detached $srcvolume:\n$api_call_check"; | |
fi | |
done | |
API_CALL ec2 detach-volume --volume-id $dstvolume | |
check_subresponse="false" | |
while [[ $check_subresponse != "true" ]]; do | |
API_CALL ec2 describe-volumes --volume-ids $dstvolume | |
if [[ $(echo $api_call_check | jq -r '.Volumes[].State') != "available" ]]; then | |
PROGRESS | |
else | |
check_subresponse="true" | |
LOGGING "Detached $dstvolume:\n$api_call_check"; | |
fi | |
done | |
API_CALL ec2 delete-volume --volume-id $srcvolume | |
API_CALL ec2 delete-volume --volume-id $dstvolume | |
LOGGING "Detached $srcvolume & $dstvolume\n"; | |
LOGGING "Deleting temporary files:"; | |
for delete in $srcinstancejson $sourcevolumes $sourcesnapshots $srcvolumejson $srcimagejson $finalsnap $mapping; do | |
(rm -vf $delete) | tee -a $logfile > /dev/null 2>&1 | |
done; | |
echo -e "${GREEN}[OK]${NORMAL}\n"; | |
} | |
echo -e "Cleaning up all temporary resources:\n"; | |
if [[ $@ =~ "--cleanup" ]]; then | |
CLEANUP | |
else | |
echo -e "Continue with clean up of temprary resources? It's suggested to test ${GREEN}$ami${NORMAL} first,\nbefore removing temporary resource. (y/n)"; | |
check_response="false" | |
while [[ $check_response == "false" ]]; do | |
read -p "> "; | |
if [[ $REPLY =~ ^([yY][eE][sS]|[yY])$ ]]; then | |
check_response="true"; | |
CLEANUP | |
elif [[ $REPLY =~ ^([nN][oO]|[nN])$ ]]; then | |
check_response="true"; | |
echo -e "\nManually remove the following resource once you have successfully tested ${GREEN}$ami${NORMAL}:"; | |
echo -e " 1. Deregister temporary AMI of source instance: ${RED}$tempami${NORMAL}"; | |
echo -e " 2. Delete the following snapshots used by ${RED}$tempami${NORMAL}:"; | |
for cleanup in $(cat $sourcesnapshots | awk -F "*" '{ print $2 }'); do | |
echo -e " ${RED}$cleanup${NORMAL}"; | |
done | |
echo -e " 3. Unmount ${RED}$dstvolume${NORMAL} from mount point /mnt"; | |
echo -e " 4. Detach and delete temporary volumes used during the conversion process:"; | |
echo -e " ${RED}$srcvolume${NORMAL}"; | |
echo -e " ${RED}$dstvolume${NORMAL}"; | |
echo -e " 5. Delete the following temporary files:"; | |
for delete in $srcinstancejson $sourcevolumes $sourcesnapshots $srcvolumejson $srcimagejson $finalsnap $mapping; do | |
echo -e " ${RED}$delete${NORMAL}"; | |
done; | |
LOGGING "Manual cleanup selected. Ensure the removal of the following resource:"; | |
LOGGING "Temporary AMI: $tempami"; | |
LOGGING "Snapshots: $(cat $sourcesnapshots | awk -F "*" '{ print $2 }')"; | |
LOGGING "Volumes: $srcvolume, $dstvolume"; | |
LOGGING "Delete the following files manually:\n$srcinstancejson\n$sourcevolumes\n$sourcesnapshots\n$srcvolumejson\n$srcimagejson\n$finalsnap\n$mapping\nDone"; | |
else | |
echo -e "yes or no?" | |
fi | |
done; | |
fi | |
BANNER | |
echo -e "PV to HVM conversion script completed successfully.\nNew HVM AMI created from instance $sourceinstance:\n ${RED}$ami${NORMAL}\n\nLog file located at:${RED}$logfile${NORMAL}.\n"; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment