Bash script for automated testing of OpenStack freezer-agent's backup/restore interaction with Nova, Cinder and Glance on a DevStack.

freezer-agent-test.sh

# Test script for freezer-agent implementing automated testing of backup and
# restore actions using the OpenStack APIs only (Glance, Cinder, Nova) and
# their resources. Includes basic validation of backup/restore results.
# NOTE: Tests are limited to "local" storage mode, not covering S3/Swift
# (except for cinder-backup) or other backup targets.

# WARNING: Use this script with caution, it might enact destructive actions
#          on resources within the project currently authenticated in!
#          Meant for use on a DevStack.

# USAGE: <this-script>.sh [cleanup] [nova] [cinder] [cindernative] [glance]

# cleanup      = do a pre-cleanup of resources before starting tests
#                if specified as the only argument, do cleanup only
#
# nova         = test single-server and project-wide nova backup/restore
#
# cinder       = test single-volume backup/restore (via image download)
#
# cindernative = test single-volume backup/restore via cinder-backup
#
# glance       = test single-image and project-wide glance backups

# global test settings
PREFIX="T3ST-freezer-"
BACKUP_DIR="/tmp/TEST_freezer_backups"
LOGFILE="/opt/stack/logs/${PREFIX}agent.log"
DO_CLEANUP_AFTER_FAIL=1
KEEP_BACKUP_DIR_AFTER_TESTS=1

# existing devstack resources
IMAGE_NAME="cirros-0.6.3-x86_64-disk"
IMAGE_TYPE="qcow2"
VOLUME_SIZE=1
VM_FLAVOR="cirros256"
VM_NETWORK="private"

# Authenticate using the demo user in demo project on DevStack.
source openrc demo demo

# runtime variables
TESTS_CONCLUDED=""

function cleanup() {
    # note: images created by Freezer are marked as shared
    openstack image list --shared -f value -c id \
        | xargs -I% openstack image delete %
    openstack image list -f value -c id -c name | grep restore_ \
        | cut -d' ' -f1 | xargs -I% openstack image delete %
    openstack image list -f value -c id -c name | grep snapshot_of \
        | cut -d' ' -f1 | xargs -I% openstack image delete %
    openstack volume list -f value -c id -c name | grep None \
        | cut -d' ' -f1 | xargs -I% openstack volume delete %
    openstack volume list -f value -c id -c name | grep "$PREFIX" \
        | cut -d' ' -f1 | xargs -I% openstack volume delete %
    openstack volume snapshot list -f value -c id -c name \
        | grep backup_snapshot_for_volume | cut -d' ' -f1 \
        | xargs -I% openstack volume snapshot delete %
    openstack volume snapshot list -f value -c id -c name | grep None \
        | cut -d' ' -f1 | xargs -I% openstack volume snapshot delete %
    openstack server list -f value -c id -c name | grep "$PREFIX" \
        | cut -d' ' -f1 | xargs -I% openstack server delete %
}

function panic() {
    set +x
    if [ "$DO_CLEANUP_AFTER_FAIL" = "1" ]; then
        cleanup
    fi
    echo "Freezer FAILED!"
    exit 1
}

function wait_for_resource_with_state () {
    set +x
    RESOURCE_TYPE=$1
    IDENTIFIER=$2
    EXPECTED_STATE=$3
    WAIT_SECONDS=2
    MAX_RETRIES=15
    echo ""
    echo -n "Waiting for $RESOURCE_TYPE '$IDENTIFIER' "
    echo "to reach state '$EXPECTED_STATE' ..."

    CURRENT_STATE=$(
        openstack $RESOURCE_TYPE list | grep "$IDENTIFIER" | awk '{print $6}'
    )
    TRY_COUNT=1
    while [ "$CURRENT_STATE" != "$EXPECTED_STATE" ]
    do
        if [ $TRY_COUNT -gt $MAX_RETRIES ]; then
            echo -n "ERROR: $RESOURCE_TYPE '$IDENTIFIER' failed to reach "
            echo "state '$EXPECTED_STATE' after $MAX_RETRIES retries."
            echo ""
            panic
        fi
        echo -n "The $RESOURCE_TYPE '$IDENTIFIER' is not in state "
        echo -n "'$EXPECTED_STATE' (was '$CURRENT_STATE'), waiting "
        echo "$WAIT_SECONDS seconds..."
        sleep $WAIT_SECONDS
        CURRENT_STATE=$(
            openstack $RESOURCE_TYPE list \
            | grep "$IDENTIFIER" | awk '{print $6}'
        )
        TRY_COUNT=$((TRY_COUNT + 1))
    done
    echo -n "The $RESOURCE_TYPE '$IDENTIFIER' has reached "
    echo "state '$EXPECTED_STATE'."
    set -x
}

function wait_for_resources_absent() {
    set +x
    RESOURCE_TYPE=$1
    WAIT_SECONDS=2
    MAX_RETRIES=20
    echo ""
    echo "Waiting for $RESOURCE_TYPE resources to vanish ... "

    CURRENT_COUNT=$(
        openstack $RESOURCE_TYPE list -f value -c id | wc -l
    )
    TRY_COUNT=1
    while [ "$CURRENT_COUNT" != "0" ]
    do
        if [ $TRY_COUNT -gt $MAX_RETRIES ]; then
            echo -n "ERROR: some $RESOURCE_TYPE resources still exist "
            echo "after $MAX_RETRIES retries."
            echo ""
            panic
        fi
        echo -n "There are still $RESOURCE_TYPE resources, waiting "
        echo "$WAIT_SECONDS seconds..."
        sleep $WAIT_SECONDS
        CURRENT_COUNT=$(
            openstack $RESOURCE_TYPE list -f value -c id | wc -l
        )
        TRY_COUNT=$((TRY_COUNT + 1))
    done
    echo "All $RESOURCE_TYPE resources have now disappeared."
    set -x

}

function test_glance_single() {
    # SINGLE IMAGE :: BACKUP
    mkdir -p $BACKUP_DIR/glance_single
    IMG_ID=$(openstack image show $IMAGE_NAME -f value -c id)
    freezer-agent --log-file $LOGFILE --debug \
        --mode glance --engine glance --glance-image-id $IMG_ID \
        --storage local --container $BACKUP_DIR/glance_single \
        --backup-name single_image_backup --noincremental --no-incremental YES
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    EXPECTED_PATH="$BACKUP_DIR/glance_single/data/glance/$(hostname)_single_image_backup"
    test -d "$EXPECTED_PATH"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # SINGLE IMAGE :: RESTORE
    freezer-agent --log-file $LOGFILE --debug \
        --action restore --mode glance --engine glance \
        --backup-name single_image_backup --glance-image-id $IMG_ID \
        --storage local --container $BACKUP_DIR/glance_single
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    openstack image show "restore_$IMAGE_NAME" -f value -c id
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # SINGLE IMAGE :: TEARDOWN
    openstack image delete "restore_$IMAGE_NAME"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    TESTS_CONCLUDED="$TESTS_CONCLUDED glance_single"
}

function test_glance_project() {
    # MULTI IMAGE :: SETUP
    IMAGE2_NAME="${PREFIX}img-1"
    IMAGE3_NAME="${PREFIX}img-2"
    openstack image save --file /tmp/freezer_test_image.img "$IMAGE_NAME"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    openstack image create --file /tmp/freezer_test_image.img \
        --disk-format $IMAGE_TYPE $IMAGE2_NAME
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    openstack image create --file /tmp/freezer_test_image.img \
        --disk-format $IMAGE_TYPE $IMAGE3_NAME
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    mkdir -p $BACKUP_DIR/glance_multi

    # MULTI IMAGE :: BACKUP
    PRJ_ID=$(openstack project show $OS_PROJECT_NAME -f value -c id)
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    freezer-agent --log-file $LOGFILE --debug \
        --mode glance --engine glance --project-id $PRJ_ID \
        --storage local --container $BACKUP_DIR/glance_multi \
        --backup-name multi_image_backup \
        --noincremental --no-incremental YES

    # verification
    IMAGE_ID_1=$(openstack image show -f value -c id $IMAGE_NAME)
    IMAGE_ID_2=$(openstack image show -f value -c id $IMAGE2_NAME)
    IMAGE_ID_3=$(openstack image show -f value -c id $IMAGE3_NAME)
    test -f "$BACKUP_DIR/glance_multi/project_glance_$PRJ_ID"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    EXPECTED_PATH="$BACKUP_DIR/glance_multi/data/glance"
    EXPECTED_PATH="$EXPECTED_PATH/$(hostname)_multi_image_backup"
    test -d "$EXPECTED_PATH"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    test -d "$EXPECTED_PATH/$IMAGE_ID_1"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    test -d "$EXPECTED_PATH/$IMAGE_ID_2"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    test -d "$EXPECTED_PATH/$IMAGE_ID_3"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # MULTI IMAGE :: RESTORE
    freezer-agent --log-file $LOGFILE --debug --action restore \
        --mode glance --engine glance --project-id $PRJ_ID \
        --storage local --container $BACKUP_DIR/glance_multi \
        --backup-name multi_image_backup \
        --noincremental --no-incremental YES

    # verification
    openstack image show "restore_$IMAGE_NAME" -f value -c id
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    openstack image show "restore_$IMAGE2_NAME" -f value -c id
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    openstack image show "restore_$IMAGE3_NAME" -f value -c id
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # MULTI IMAGE :: TEARDOWN
    openstack image delete "$IMAGE2_NAME"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    openstack image delete "$IMAGE3_NAME"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    TESTS_CONCLUDED="$TESTS_CONCLUDED glance_project"
}

function test_cinder_single() {
    VOL_NAME="${PREFIX}vol"
    # SINGLE VOLUME :: SETUP
    openstack volume create --size $VOLUME_SIZE \
        --image $IMAGE_NAME $VOL_NAME
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    wait_for_resource_with_state volume $VOL_NAME available

    # SINGLE VOLUME :: BACKUP
    VOL_ID=$(openstack volume show -f value -c id $VOL_NAME)
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    freezer-agent --log-file $LOGFILE --debug \
    --mode cinder --cinder-vol-id $VOL_ID \
    --storage local --container $BACKUP_DIR/cinder_single \
    --backup-name single_volume_backup \
    --noincremental --no-incremental YES
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    EXPECTED_PATH="$BACKUP_DIR/cinder_single/$VOL_ID"
    test -d "$EXPECTED_PATH"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # SINGLE VOLUME :: RESTORE
    # note: the volume name will be the same, so first delete the old one
    openstack volume delete $VOL_ID
    freezer-agent --log-file $LOGFILE --debug --action restore \
    --mode cinder --cinder-vol-id $VOL_ID \
    --storage local --container $BACKUP_DIR/cinder_single \
    --backup-name single_volume_backup
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    openstack volume show -f value -c id $VOL_NAME
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    NEW_ID=$(openstack volume show -f value -c id $VOL_NAME)
    if [ "$VOL_ID" = "$NEW_ID" ]; then
        echo "ERROR: expected restored volume with different ID"
        panic
    fi

    # SINGLE VOLUME :: TEARDOWN
    openstack volume delete $VOL_NAME

    TESTS_CONCLUDED="$TESTS_CONCLUDED cinder_single"
}

function test_cindernative() {
    VOL_NAME="${PREFIX}vol"
    # CINDERNATIVE :: SETUP
    openstack volume create --size $VOLUME_SIZE \
        --image $IMAGE_NAME $VOL_NAME
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    wait_for_resource_with_state volume $VOL_NAME available

    # CINDERNATIVE :: BACKUP
    VOL_ID=$(openstack volume show -f value -c id $VOL_NAME)
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    freezer-agent --log-file $LOGFILE --debug \
    --mode cindernative --cindernative-vol-id $VOL_ID \
    --container ${PREFIX}cindernative \
    --backup-name native_cinder_backup \
    --noincremental --no-incremental YES
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    openstack volume backup list --volume $VOL_NAME -f value -c id
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    COUNT=$(
        openstack volume backup list --volume $VOL_NAME -f value -c id \
        | wc -l
    )
    if [ "$COUNT" != "1" ]; then
        echo "ERROR: expected 1 volume backup to exist for volume $VOL_NAME"
        panic
    fi
    # volume will be in "backing-up" state temporarily
    wait_for_resource_with_state volume $VOL_NAME available

    # CINDERNATIVE :: RESTORE
    # note: the volume name will be the same, so first delete the old one
    freezer-agent --log-file $LOGFILE --debug --action restore \
    --mode cinder --cindernative-vol-id $VOL_ID \
    --backup-name native_cinder_backup \
    --container ${PREFIX}cindernative
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    STATUS=$(openstack volume show $VOL_NAME -f value -c status)
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    if [ "$STATUS" != "restoring-backup" ]; then
        echo -n "ERROR: expected volume $VOL_NAME status to be "
        echo "'restoring-backup' (was '$STATUS')"
        panic
    fi
    wait_for_resource_with_state volume $VOL_NAME available

    # CINDERNATIVE :: TEARDOWN
    openstack volume backup list --volume $VOL_ID -f value -c id \
        | xargs -I% openstack volume backup delete %
    openstack volume delete $VOL_NAME

    TESTS_CONCLUDED="$TESTS_CONCLUDED cindernative"
}

function test_nova_single() {
    VM_NAME="${PREFIX}vm"
    # SINGLE SERVER :: SETUP
    openstack server create --image $IMAGE_NAME --network $VM_NETWORK \
        --flavor $VM_FLAVOR $VM_NAME
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    wait_for_resource_with_state server $VM_NAME ACTIVE

    VM_ID=$(openstack server show -f value -c id $VM_NAME)
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # SINGLE SERVER :: BACKUP
    freezer-agent --log-file $LOGFILE --debug \
    --mode nova --engine nova --nova-inst-id $VM_ID \
    --storage local --container $BACKUP_DIR/nova_single \
    --backup-name single_server_backup \
    --noincremental --no-incremental YES
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    EXPECTED_PATH="$BACKUP_DIR/nova_single/data/nova"
    EXPECTED_PATH="$EXPECTED_PATH/$(hostname)_single_server_backup/$VM_ID"
    test -d "$EXPECTED_PATH"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # SINGLE SERVER :: RESTORE
    openstack server delete $VM_ID
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    freezer-agent --log-file $LOGFILE --debug --action restore \
    --mode nova --engine nova --nova-inst-id $VM_ID \
    --storage local --container $BACKUP_DIR/nova_single \
    --backup-name single_server_backup \
    --noincremental --no-incremental YES
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    openstack server show -f value -c id $VM_NAME
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    NEW_ID=$(openstack server show -f value -c id $VM_NAME)
    if [ "$VM_ID" = "$NEW_ID" ]; then
        echo "ERROR: expected restored server with different ID"
        panic
    fi
    wait_for_resource_with_state server $VM_NAME ACTIVE

    # SINGLE SERVER :: TEARDOWN
    openstack server delete $NEW_ID

    TESTS_CONCLUDED="$TESTS_CONCLUDED nova_single"
}

function test_nova_project() {
    VM_NAME1="${PREFIX}vm1"
    VM_NAME2="${PREFIX}vm2"
    VM_NAME3="${PREFIX}vm3"
    # MULTI SERVER :: SETUP
    openstack server create --image $IMAGE_NAME --network $VM_NETWORK \
        --flavor $VM_FLAVOR $VM_NAME1
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    openstack server create --image $IMAGE_NAME --network $VM_NETWORK \
        --flavor $VM_FLAVOR $VM_NAME2
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    openstack server create --image $IMAGE_NAME --network $VM_NETWORK \
        --flavor $VM_FLAVOR $VM_NAME3
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    wait_for_resource_with_state server $VM_NAME1 ACTIVE
    wait_for_resource_with_state server $VM_NAME2 ACTIVE
    wait_for_resource_with_state server $VM_NAME3 ACTIVE

    VM_ID1=$(openstack server show -f value -c id $VM_NAME1)
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    VM_ID2=$(openstack server show -f value -c id $VM_NAME2)
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    VM_ID3=$(openstack server show -f value -c id $VM_NAME3)
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # MULTI SERVER :: BACKUP
    PRJ_ID=$(openstack project show $OS_PROJECT_NAME -f value -c id)
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    freezer-agent --log-file $LOGFILE --debug \
    --mode nova --engine nova --project-id $PRJ_ID \
    --storage local --container $BACKUP_DIR/nova_multi \
    --backup-name multi_server_backup \
    --noincremental --no-incremental YES
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    EXPECTED_PATH="$BACKUP_DIR/nova_multi/data/nova"
    EXPECTED_PATH="$EXPECTED_PATH/$(hostname)_multi_server_backup"
    test -d "$EXPECTED_PATH"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    test -d "$EXPECTED_PATH/$VM_ID1"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    test -d "$EXPECTED_PATH/$VM_ID2"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    test -d "$EXPECTED_PATH/$VM_ID3"
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # MULTI SERVER :: RESTORE
    openstack server delete $VM_ID1
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    openstack server delete $VM_ID2
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi
    openstack server delete $VM_ID3
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # delete all existing servers
    wait_for_resources_absent server

    freezer-agent --log-file $LOGFILE --debug --action restore \
    --mode nova --engine nova --project-id $PRJ_ID \
    --storage local --container $BACKUP_DIR/nova_multi \
    --backup-name multi_server_backup \
    --noincremental --no-incremental YES
    ecode=$?; if [ $ecode -ne 0 ]; then panic; fi

    # verification
    wait_for_resource_with_state server $VM_NAME1 ACTIVE
    wait_for_resource_with_state server $VM_NAME2 ACTIVE
    wait_for_resource_with_state server $VM_NAME3 ACTIVE

    # MULTI SERVER :: TEARDOWN
    openstack server delete $VM_NAME1
    openstack server delete $VM_NAME2
    openstack server delete $VM_NAME3

    TESTS_CONCLUDED="$TESTS_CONCLUDED nova_project"
}

DO_PRECLEANUP=false
TEST_GLANCE=false
TEST_CINDER=false
TEST_CINDERNATIVE=false
TEST_NOVA=false
for name in "$@"; do
    case $name in
        glance)
            TEST_GLANCE=true
            echo "INFO: tests enabled for: glance"
            ;;
        cinder)
            TEST_CINDER=true
            echo "INFO: tests enabled for: cinder"
            ;;
        cindernative)
            TEST_CINDERNATIVE=true
            echo "INFO: tests enabled for: cindernative"
            ;;
        nova)
            TEST_NOVA=true
            VM_COUNT=$(
                openstack server list -f value -c id \
                | wc -l
            )
            if [ "$VM_COUNT" != "0" ]; then
                set +x
                echo "ERROR: there are existing servers, cannot execute nova "
                echo "tests; please cleanup all servers and restart the tests"
                exit 1
            fi
            echo "INFO: tests enabled for: nova"
            ;;
        cleanup)
            DO_PRECLEANUP=true
            echo "INFO: pre-cleanup enabled!"
            ;;
        *)
            echo "ERROR: no test mode '$name'"
            exit 1
            ;;
    esac
done
echo "INFO: purging backup directory at $BACKUP_DIR"
rm -rf "$BACKUP_DIR"
sleep 1.5

set -x
SCHEDULE_POSTCLEANUP=false
if [ "$DO_PRECLEANUP" = true ] ; then
    echo "INFO: executing pre-cleanup ..."
    cleanup
fi
if [ "$TEST_GLANCE" = true ] ; then
    test_glance_single
    test_glance_project
    SCHEDULE_POSTCLEANUP=true
fi
if [ "$TEST_CINDER" = true ] ; then
    test_cinder_single
    # note: cinder mode has no project-id i.e. multi-volume functionality!
    SCHEDULE_POSTCLEANUP=true
fi
if [ "$TEST_CINDERNATIVE" = true ] ; then
    test_cindernative
    SCHEDULE_POSTCLEANUP=true
fi
if [ "$TEST_NOVA" = true ] ; then
    test_nova_single
    test_nova_project
    SCHEDULE_POSTCLEANUP=true
fi

set +x
if [ "$SCHEDULE_POSTCLEANUP" = true ] ; then
    echo "INFO: executing post-cleanup ..."
    if [ "$KEEP_BACKUP_DIR_AFTER_TESTS" != "1" ]; then
        echo "INFO: purging backup directory at $BACKUP_DIR"
        rm -rf "$BACKUP_DIR"
    else
        echo "INFO: preserving backup directory at $BACKUP_DIR"
    fi
    cleanup
fi
echo "PASSED the following tests:$TESTS_CONCLUDED"
exit 0