Eases pain of sharing halconfig with teammates by syncing local version with AWS S3 and Secrets Manager

halconfig.py

#!/usr/bin/env python3

import argparse
import base64
import boto3
import datetime
import distutils.dir_util
import json
import os
import re
import subprocess
import tarfile
import tempfile

from os import listdir, chdir, path
from pathlib import Path
from shutil import copyfile

HAL_CONFIG_SECRET_NAME = "devops/spinnaker/hal/config"
HAL_BUCKET_NAME = "upside-opsdata-private"
HAL_BACKUP_BUCKET_PREFIX = "halyard/backup.tar"
TEMP_DIR = tempfile.TemporaryDirectory()
AWS_REGION_NAME = "us-west-2"


def get_secrets():
    session = boto3.session.Session(profile_name="upside-cent")
    client = session.client(
        service_name='secretsmanager',
        region_name=AWS_REGION_NAME
    )
    get_secret_value_response = client.get_secret_value(
        SecretId=HAL_CONFIG_SECRET_NAME
    )
    if 'SecretString' in get_secret_value_response:
        secrets = get_secret_value_response['SecretString']
    else:
        secrets = base64.b64decode(get_secret_value_response['SecretBinary'])

    return json.loads(secrets)

def get_backup_from_S3():
    session = boto3.session.Session(profile_name="upside-cent")
    client = session.client(
        service_name='s3',
        region_name=AWS_REGION_NAME
    )
    client.download_file(HAL_BUCKET_NAME, HAL_BACKUP_BUCKET_PREFIX, f'{TEMP_DIR.name}/backup-old.tar')

def upload_backup_to_S3():
    session = boto3.session.Session(profile_name="upside-cent")
    client = session.client(
        service_name='s3',
        region_name=AWS_REGION_NAME
    )
    client.upload_file(f'{TEMP_DIR.name}/backup.tar', HAL_BUCKET_NAME, HAL_BACKUP_BUCKET_PREFIX)

def interpolate_secrets(file_path):
    with open(file_path, "r") as config_file:
        data = ""
        for line in config_file:
            results =re.search(r'{{\s(\S+)\s}}', line)
            if results:
                data += line.replace(results.group(0), secrets[results.group(1)])
            else:
                data += line

    with open(file_path, "w") as config_file:
        config_file.write(data)

def update_backup_tar():
    old_tar = tarfile.open(f'{TEMP_DIR.name}/backup-old.tar')
    old_tar.extractall(f'{TEMP_DIR.name}/backup')
    old_tar.close()

    # copy local repo config to backup (to be interpolated)
    src = str(Path(__file__).parent.absolute()) + "/.hal/"
    dest = f'{TEMP_DIR.name}/local'
    distutils.dir_util.copy_tree(src, dest)

    # interpolate local repo config
    for subdir, dirs, files in os.walk(dest):
        for file in files:
            interpolate_secrets(os.path.join(subdir, file))

    # overwrite backed up config with interpolated local repo config
    src = f'{TEMP_DIR.name}/local'
    dest = f'{TEMP_DIR.name}/backup'
    distutils.dir_util.copy_tree(src, dest)

    new_tar = tarfile.open(f'{TEMP_DIR.name}/backup.tar', "x")
    chdir(f'{TEMP_DIR.name}/backup')
    for name in listdir("./"):
        new_tar.add(name)
    new_tar.close()

def create_hal_backup():
    subprocess.run(["hal", "backup", "create", "--quiet"], check=True)
    home = path.expanduser("~")
    filename = f'halyard-{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M")}'
    for f in listdir(home):
        if re.search(filename, f):
            copyfile(f'{home}/{f}', f'{TEMP_DIR.name}/backup-old.tar')
            break


parser = argparse.ArgumentParser()
parser.add_argument('-b', '--backup', action='store_true', help="backup using hal backup create and upload to S3", default=False)
parser.add_argument('-r', '--restore', action='store_true', help="restore halconfig from hal template", default=False)

args = parser.parse_args()

if not args.restore and not args.backup:
    print("Please add either --backup or --restore flag.")
    exit(1)
elif args.restore and args.backup:
    print("Please do not use both --backup and --restore flags.")
    exit(1)

secrets = get_secrets()

if args.restore:
    get_backup_from_S3()
    update_backup_tar()

    subprocess.run(["hal", "backup", "restore", "--backup-path", f"{TEMP_DIR.name}/backup.tar" , "--quiet"], check=True)

    print("Successfully generated halconfig from S3, git, and Secrets Manager...")

if args.backup:
    create_hal_backup()
    update_backup_tar()
    upload_backup_to_S3()

    print("Successfully backed up halconfig to S3...")

TEMP_DIR.cleanup()