Created
October 13, 2020 19:24
-
-
Save AvinashDalvi89/31fc1fc450edb2c4af19d47f36ab1259 to your computer and use it in GitHub Desktop.
Create tar from S3 bucket files using boto3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3 | |
import tarfile | |
import os.path | |
s3Client = boto3.client('s3') | |
s3object = boto3.resource('s3') | |
def lambda_handler(event, context): | |
agtBucket = "angularbuildbucket" | |
key="" | |
tar = tarfile.open('/tmp/example.tar', 'w') | |
source_dir="/tmp/" | |
for fname in get_matching_s3_keys(bucket=agtBucket, prefix=key, suffix='.js'): | |
print(fname) | |
file_obj = s3object.Object(agtBucket, fname) | |
#file_content = file_obj.get()['Body'].read() | |
#tar.add(file_content) | |
s3object.Bucket(agtBucket).download_file(fname, '/tmp/'+fname) | |
tar.add(source_dir, arcname=os.path.basename(source_dir)) | |
tar.close() | |
s3object.meta.client.upload_file(source_dir+"example.tar", agtBucket, 'example.tar') | |
def get_matching_s3_keys(bucket, prefix='', suffix=''): | |
""" | |
Generate the keys in an S3 bucket. | |
:param bucket: Name of the S3 bucket. | |
:param prefix: Only fetch keys that start with this prefix (optional). | |
:param suffix: Only fetch keys that end with this suffix (optional). | |
""" | |
kwargs = {'Bucket': bucket, 'Prefix': prefix} | |
while True: | |
resp = s3Client.list_objects_v2(**kwargs) | |
for obj in resp['Contents']: | |
key = obj['Key'] | |
if key.endswith(suffix): | |
yield key | |
try: | |
kwargs['ContinuationToken'] = resp['NextContinuationToken'] | |
except KeyError: | |
break |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment