Copy S3 bucket from one account to another with versions

aws-s3-copy-with-versions.rb

require 'aws-sdk-s3'
require 'concurrent'


source_bucket = 'xxx'
dest_bucket = 'xxx'
acl = 'public-read'

client = Aws::S3::Client.new(
  region: 'ap-southeast-2',
  credentials:  Aws::Credentials.new('XXX', 'XXX'),
)
pool = Concurrent::FixedThreadPool.new(100)
sized_queue = SizedQueue.new(3000)

objects = {}
resp = OpenStruct.new(is_truncated: true)
while resp.is_truncated
  resp = client.list_object_versions({
    bucket: source_bucket,
    max_keys: 1000,
    key_marker: resp.next_key_marker,
    version_id_marker: resp.next_version_id_marker
  })
  resp.versions.each do |v|
    objects[v.key] ||= []
    objects[v.key] << v
    objects[v.key].sort_by!(&:last_modified)
  end
  puts "Next key marker: #{resp.next_key_marker}, O: #{objects.keys.size} Q: #{sized_queue.size}"
  # puts resp.versions[0]

  objects.keys[0..500].each do |k|
    # puts k
    versions = objects[k]
    objects.delete(k)
    sized_queue.push(versions)
    pool.post do
      v = sized_queue.pop
      # versions sorted by last_modified
      v.each do |vv|
        puts "running #{source_bucket}/#{vv.key}?versionId=#{vv.version_id}"

        begin
          client.copy_object({
            bucket: dest_bucket,
            copy_source: "#{source_bucket}/#{vv.key}?versionId=#{vv.version_id}",
            key: vv.key,
            acl: acl,
          })
        rescue => e
          puts e.message
        end
      end
    end
  end
end

pool.shutdown
pool.wait_for_termination