Created
February 28, 2020 20:43
-
-
Save slaughtr/ea5d25ac9366e2b89b5744ccd99532b6 to your computer and use it in GitHub Desktop.
List all objects in S3 bucket (without prefixes)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const getAllObjects = async s3Params => { | |
const AWS = require('aws-sdk'); | |
const S3 = new AWS.S3(); | |
let results = []; | |
let gotAllObjs = false; | |
// Loop until we get all keys - This does NOT do any retries or backoff. | |
while (!gotAllObjs) { | |
if (!s3Params.ContinuationToken || s3Params.ContinuationToken !== 'DONE') { | |
const s3listres = await S3.listObjectsV2(s3Params).promise(); | |
// assign ContinuationToken to data.NextContinuationToken | |
if (s3listres.NextContinuationToken) s3Params.ContinuationToken = s3listres.NextContinuationToken; | |
else s3Params.ContinuationToken = 'DONE'; | |
results = [...results, ...s3listres.Contents]; | |
console.log('Found this many files: ', results.length); | |
} else if (s3Params.ContinuationToken === 'DONE') { | |
gotAllObjs = true; | |
console.log('Got all objects names from S3'); | |
} | |
} | |
return [ ...new Set(results) ]; | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment