Make buckets has no object even if the bucket is versioning-enabled

go.mod

module niku/cleanup-version-enabled-bucket

go 1.17

require (
	github.com/aws/aws-sdk-go-v2 v1.11.2
	github.com/aws/aws-sdk-go-v2/config v1.11.0
	github.com/aws/aws-sdk-go-v2/service/s3 v1.21.0
)

require (
	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect
	github.com/aws/aws-sdk-go-v2/credentials v1.6.4 // indirect
	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect
	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect
	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect
	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect
	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0 // indirect
	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect
	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.2 // indirect
	github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 // indirect
	github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 // indirect
	github.com/aws/smithy-go v1.9.0 // indirect
)

main.go

package main

import (
	"context"
	"flag"
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
)

func isTargetBucket(bucket types.Bucket) bool {
  // You can define target bucket
	return true
}

func collectTargetBuckets(client *s3.Client) ([]types.Bucket, error) {
	var result []types.Bucket
	listBucketsInput := s3.ListBucketsInput{}
	listBucketsOutput, err := client.ListBuckets(context.TODO(), &listBucketsInput)
	if err != nil {
		return result, err
	}
	for _, bucket := range listBucketsOutput.Buckets {
		if isTargetBucket(bucket) {
			result = append(result, bucket)
		}
	}
	return result, nil
}

func collectTargetObjectVersionsByEachBucket(client *s3.Client, input []types.Bucket) (map[string][]types.ObjectIdentifier, error) {
	result := make(map[string][]types.ObjectIdentifier)
	for _, bucket := range input {
		bucketName := *bucket.Name
		var keyMarker *string
		var versionIdMarker *string
		for {
			// It looks Maxkeys effects length of (Versions + DeleteMarkers).
			// i.e., If object versions in a bucket are over 1000, ListObjectVersions returns N Versions and 1000 - N DeleteMakers.
			listObjectVersionsInput := s3.ListObjectVersionsInput{
				Bucket:          aws.String(bucketName),
				KeyMarker:       keyMarker,
				VersionIdMarker: versionIdMarker,
			}
			listObjectVersionsOutput, err := client.ListObjectVersions(context.TODO(), &listObjectVersionsInput)
			if err != nil {
				return result, err
			}
			// To make a versioning bucket has no object, it needs deleting all Versions and DeleteMarkers.
			for _, version := range listObjectVersionsOutput.Versions {
				objectIdentifier := types.ObjectIdentifier{
					Key:       version.Key,
					VersionId: version.VersionId,
				}
				result[bucketName] = append(result[bucketName], objectIdentifier)
			}
			for _, deleteMarker := range listObjectVersionsOutput.DeleteMarkers {
				objectIdentifier := types.ObjectIdentifier{
					Key:       deleteMarker.Key,
					VersionId: deleteMarker.VersionId,
				}
				result[bucketName] = append(result[bucketName], objectIdentifier)
			}
			keyMarker = listObjectVersionsOutput.NextKeyMarker
			versionIdMarker = listObjectVersionsOutput.NextVersionIdMarker
			count := len(result[bucketName])
			fmt.Printf("%s:%d\n", bucketName, count)
			if keyMarker == nil && versionIdMarker == nil {
				break
			}
		}
	}
	return result, nil
}

func makeDeleteObjectsInputIncludingVersion(client *s3.Client, input map[string][]types.ObjectIdentifier) []s3.DeleteObjectsInput {
	var result []s3.DeleteObjectsInput
	for bucketName, objectVersions := range input {
		// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
		// The request contains a list of up to 1000 keys that you want to delete.
		batchSize := 1000
		for i := 0; i < len(objectVersions); i += batchSize {
			j := i + batchSize
			if len(objectVersions) < j {
				j = len(objectVersions)
			}
			var objectIdentifiers []types.ObjectIdentifier
			for _, objectVersion := range objectVersions[i:j] {
				// To delete versioned object, it needs both value Key and VersionId.
				objectIdentifier := types.ObjectIdentifier{
					Key:       aws.String(*objectVersion.Key),
					VersionId: objectVersion.VersionId,
				}
				objectIdentifiers = append(objectIdentifiers, objectIdentifier)
			}
			delete := types.Delete{
				Objects: objectIdentifiers,
			}
			deleteObjectsInput := s3.DeleteObjectsInput{
				Bucket: aws.String(bucketName),
				Delete: &delete,
			}
			result = append(result, deleteObjectsInput)
		}
	}
	return result
}

func deleteObjects(client *s3.Client, deleteObjectsInput s3.DeleteObjectsInput) error {
	_, err := client.DeleteObjects(context.TODO(), &deleteObjectsInput)
	if err != nil {
		return err
	}
	return nil
}

func main() {
	var deleteConfirmed bool
	flag.BoolVar(&deleteConfirmed, "delete-confirmed", false, "if you confirm deleting objects including versions, make this flug true. otherwise it runs dry-run.")
	flag.Parse()

	cfg, err := config.LoadDefaultConfig(context.TODO())
	if err != nil {
		panic("configuration error, " + err.Error())
	}
	client := s3.NewFromConfig(cfg)

	targetBuckets, err := collectTargetBuckets(client)
	if err != nil {
		fmt.Println("collectTargetBuckets err: " + err.Error())
		panic(err)
	}

	targetObjectVersionsByEachBucket, err := collectTargetObjectVersionsByEachBucket(client, targetBuckets)
	if err != nil {
		fmt.Println("collectTargetObjectVersionsByEachBucket err: " + err.Error())
		panic(err)
	}

	for k, v := range targetObjectVersionsByEachBucket {
		fmt.Printf("%s:%d\n", k, len(v))
	}

	deleteObjectsInputs := makeDeleteObjectsInputIncludingVersion(client, targetObjectVersionsByEachBucket)

	for times, deleteObjectsInput := range deleteObjectsInputs {
		fmt.Printf("%d, %s, %d\n", times, *deleteObjectsInput.Bucket, len(deleteObjectsInput.Delete.Objects))
		if deleteConfirmed {
			if err := deleteObjects(client, deleteObjectsInput); err != nil {
				fmt.Println("Got an error deleteing objects:" + err.Error())
			}
		}
	}

	fmt.Println("Finished")
}