Last active
November 9, 2017 05:14
-
-
Save anujith-singh/3d85c721d86c7bfe1e79773734f38ba8 to your computer and use it in GitHub Desktop.
Deletes or to correctly describe aborts partially uploaded files from Amazon S3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import sys | |
import json | |
bucket_name = sys.argv[1] | |
limit = sys.argv[2] | |
# Usage Ex: python delete_partial_files_S3.py <my_bucket_name> <limit> | |
# output = os.popen("aws s3api list-multipart-uploads --bucket " + bucket_name).read() | |
# if not output: | |
# print "No partilly uploaded files" | |
# sys.exit() | |
# output = json.loads(output) | |
# print len(output['Uploads']) # prints the number of partilly uploaded files | |
# sys.exit() | |
output = os.popen("aws s3api list-multipart-uploads --bucket " + bucket_name + " --max-items " + limit).read() | |
if not output: | |
print "No partilly uploaded files" | |
sys.exit() | |
output = json.loads(output) | |
uploads = output['Uploads'] | |
next_token = None | |
if 'NextToken' in output: | |
next_token = output['NextToken'] | |
def delete_partially_uploaded_files(uploads, next_token, bucket_name, limit): | |
for upload in uploads: | |
key = upload['Key'].replace(" ", "\ ") | |
upload_id = upload['UploadId'] | |
print "Deleting \nkey: " + key + "\nupload_id: " + upload_id | |
# os.system("aws s3api abort-multipart-upload --bucket " + bucket_name + " --key " + key + " --upload-id " + upload_id) | |
print "DELETED" | |
print "" | |
if next_token: | |
# Uncomment the below line to paginate while listing, NOT TO BE USED WHILE DELETING/ABORTING | |
# output = os.popen("aws s3api list-multipart-uploads --bucket " + bucket_name + " --max-items " + limit + " --starting-token " + next_token).read() | |
# --starting-token has not been used as it doesn't work as expected while DELETING/ABORTING multipart upload | |
output = os.popen("aws s3api list-multipart-uploads --bucket " + bucket_name + " --max-items " + limit).read() | |
output = json.loads(output) | |
uploads2 = output['Uploads'] | |
if 'NextToken' in output: | |
next_token2 = output['NextToken'] | |
else: | |
next_token2 = None | |
delete_partially_uploaded_files(uploads2, next_token2, bucket_name, limit) | |
delete_partially_uploaded_files(uploads, next_token, bucket_name, limit) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment