s3-copy-inspiration.py

import os
import json
import uuid

import boto3
from boto3.s3.transfer import TransferConfig



s3 = boto3.resource('s3')

KB = 1024
MB = KB * 1024
GB = MB * 1024

# https://stackoverflow.com/questions/46556972/what-is-optimal-setting-for-multipart-threshold-and-mutilpart-chunksize-while-do
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html
_setting_multipart_threshold = 8 * MB
_setting_max_concurrency = 800 # Lambda has at most 1024 threads, leave some overhead
_setting_multipart_chunksize = 1 * GB
_setting_num_download_attempts = 5
_setting_max_io_queue = 10000
_setting_io_chunksize = 256 * KB
_setting_use_threads = True

config = TransferConfig(
  multipart_threshold=_setting_multipart_threshold,
  max_concurrency=_setting_max_concurrency,
  multipart_chunksize=_setting_multipart_chunksize,
  num_download_attempts=_setting_num_download_attempts,
  max_io_queue=_setting_max_io_queue,
  io_chunksize=_setting_io_chunksize,
  use_threads=_setting_use_threads,
)

copy_source = {
  'Bucket': 'bucket-name',
  'Key': 'file-name'
}

dest_bucket = 'bucket-name'
dest_file = str(uuid.uuid4())

s3.meta.client.copy(
  CopySource=copy_source,
  Bucket=dest_bucket,
  Key=dest_file,
  Config=config,
)