Loop over S3 videos looking for ones that need new thumbs.

s3videopager.py

#!/usr/bin/env python3
# https://boto3.amazonaws.com/v1/documentation/api/latest/guide/paginators.html
"""Loop over S3 videos looking for ones that need new thumbs.

Finds all the video/assetid/ prefixes and loops over them, then for each
with enough keys, looks to see if they have thumbs (so skip this one) or a
~orig. file we can use to create thumbs.
"""

import os

import boto3

S3_NAME = os.environ['S3_NAME']

client = boto3.client('s3')
paginator = client.get_paginator('list_objects')
page_iter = paginator.paginate(Bucket=S3_NAME, Prefix='video/', Delimiter='/')
for page in page_iter:  # returns 1000 page['Contents'] or page['CommonPrefixes'])
    for prefixobj in page['CommonPrefixes']:  # video/hst2005_640x480/
        video_prefix = prefixobj['Prefix']
        objs = client.list_objects_v2(Bucket=S3_NAME, Prefix=video_prefix)
        keys = [obj['Key'] for obj in objs['Contents']]
        if len(keys) < 3:  # need at least metadata.json, collection.json, ...~orig.*
            continue
        got_thumb = False
        orig_video = False
        for key in keys:
            if key.endswith('~small.jpg'):
                got_thumb = key
                break           # so we don't need to thumb this one
            if '~orig.' in key:
                orig_video = key
        if got_thumb:
            print('gotthumb %s' % got_thumb)
            # we've got thumbs, no need to do anything, ignore
            continue            # we've got a thumb, so skip to next video
        if orig_video:
            print('CREATE   %s' % orig_video)
            # Call code to create at video_prefix with orig_video
        else:
            print('wtf?     %s' % keys)
            # this is probably bogus from testing or broken code, ignore