Sync full directories to/from S3

s3dir.py

import os
from metaflow import S3

def put_dir(local_root, s3root):
    root = os.path.abspath(local_root)
    objs = []
    for p, _, files in os.walk(root):
        for f in files:
            path = os.path.join(p, f)
            key = os.path.relpath(path, start=root)
            objs.append((os.path.join(s3root, key), path))
    with S3() as s3:
        s3.put_files(objs)


def get_dir(s3root, local_root):
    with S3(s3root=s3root) as s3:
        objs = s3.get_all()
        for obj in objs:
            path = os.path.join(local_root, obj.key)
            os.makedirs(os.path.dirname(path), exist_ok=True)
            os.rename(obj.path, path)


if __name__ == "__main__":
    import sys

    print(f"Uploading directory {sys.argv[1]} to {sys.argv[2]}")
    put_dir(sys.argv[1], sys.argv[2])
    print(f"Downloading to {sys.argv[3]}")
    get_dir(sys.argv[2], sys.argv[3])