Verifies content addressed blocks in S3

verify-blocks.sh

s3Location="$1"

function verifyMd5 {
    objectLocation="$1"
    objectName="$(basename ${objectLocation})"
    mc cat "${objectLocation}" \
        | md5sum -c <(echo "${objectName}  -") --status \
            && echo "${objectLocation} = OK" \
            || echo "${objectLocation} = BAD"
    exit 0
}

export -f verifyMd5
mc ls --json "${s3Location}" \
    | jq -r 'select(.type=="file") | .key' \
    | parallel verifyMd5 "${s3Location}/{}"

To get more information about the failure, see: https://gist.github.com/colin-nolan/66fbcaf787ef46dd4b89c101c6647636.