Management command to upload only files that have been changed to S3 The S3 bucket is assumed to be like: Bucket |-static |-media

statics3upload.py

from django.core.management.base import BaseCommand
from django.conf import settings
from storages.backends.s3boto import S3BotoStorage
from boto.s3.connection import S3Connection
from boto.s3.key import Key
import os
from hashlib import md5
from subprocess import call

# Assuming the project structure is like this:
# BASE_DIR
# |- .git
# |- PROJECT_DIR
# |  |- APPS_DIR
# |  |- settings.py
# |  |- STATIC_DIR (static)
# |-STATIC_ROOT(static)
# |-VIRTUALENV_DIR

# In the settings file:
# PROJECT_PATH = os.path.dirname(os.path.abspath(__file__))
# BASE_DIR = os.path.dirname(os.path.dirname(__file__))

class Command(BaseCommand):
    args = ''
    help = 'Closes the specified poll for voting'
    can_import_settings = True
    def __init__(self,*args, **kwargs):
        super(Command, self).__init__(*args, **kwargs)
        conn = S3Connection(settings.AWS_ACCESS_KEY_ID,settings.AWS_SECRET_ACCESS_KEY)
        self.bucket = conn.get_bucket(settings.AWS_STORAGE_BUCKET_NAME)
    def handle(self, *args, **options):
        bucket_dict = {}
        for k in self.bucket.get_all_keys(prefix="static"):
            # k.etag returns u'"xxxxxxxxxxx30bb87cbdac24f55xxxx"' hence strip the quotes from left and right
            bucket_dict[k.name] = k.etag.rstrip('"').lstrip('"')
            # pass
        static_dir = os.path.join(settings.PROJECT_PATH,'static')
        os.chdir(settings.BASE_DIR) # Git root
        for root, dirs, files in os.walk(static_dir):
            for f in files:
                fullpath = os.path.join(root,f)
                key = fullpath.replace(settings.PROJECT_PATH+"/","")
                if bucket_dict.get(key,None) == md5(open(fullpath).read()).hexdigest():
                    # if the key exists on the server side already and its hash is equal to the hash of local file
                    # go to the next file
                    continue
                else:
                    # Check if the key exists
                    if self.bucket.get_key(key) is None:
                        # Check if the key is ignored in the local repository
                        if not fullpath.endswith(".gitignore") or bool(call(["git","check-ignore","%s"%(fullpath)])):
                            # If not (that is check-ignore returns True
                            # Create the key
                            key_to_upload_to = self.bucket.new_key(key)
                        else:
                            # Else go to the next file
                            continue
                    else:
                        # Get the existing key if it already exists
                        key_to_upload_to = self.bucket.get_key(key)
                    uploaded_bytes = key_to_upload_to.set_contents_from_filename(fullpath)
                    print ("Uploaded %s bytes of %s to S3"%(uploaded_bytes,fullpath))