azPyDeploy.py

# This script needs
# $ pip install fabric

from os import environ

from fabric import Connection

# Create a file `secret.py` in the same directory as this one
# and add in it the credentials to connect to the server and GitHub.
# Here is a template.
#############
# File secret.py

#from os import environ, path
#
### Connection
#environ['REMOTE_HOST'] = '172.104.239.248'
#environ['REMOTE_USER'] = '****'
#environ['REMOTE_PASSWORD'] = '********+'
#
## Python venv
#environ['VENV_NAME'] = 'prod-api'
#
### Git
#environ['GIT_DIR'] = '~/app'
#environ['GIT_DEFAULT_BRANCH'] = 'main'
#environ['REPO_URL'] = 'https://github.com/digitalocean/sample-flask.git'
#############
import secret


def create_conn():
    # Switch the two lines if you connect via PEM Key
    # instead of password.
    params = {
        #'key_filename': environ['SSH_KEY_PATH']}
        'password': environ['REMOTE_PASSWORD']
    }
    conn = Connection(
        host=environ['REMOTE_HOST'],
        user=environ['REMOTE_USER'],
        connect_kwargs=params,
    )
    return conn


######################
# Internal Functions #
######################


def _create_vm(conn):
    _install_packages(conn)
    _install_python(conn)
    _install_venv(conn)


def _install_packages(conn):
    conn.sudo('apt-get -y update')
    conn.sudo('apt-get -y upgrade')
    conn.sudo('apt-get install -y build-essential')
    #conn.sudo('apt-get install -y checkinstall')
    conn.sudo('apt-get install -y libreadline-gplv2-dev')
    conn.sudo('apt-get install -y libncurses-dev')
    conn.sudo('apt-get install -y libncursesw5-dev')
    conn.sudo('apt-get install -y libssl-dev')
    conn.sudo('apt-get install -y libsqlite3-dev')
    conn.sudo('apt-get install -y tk-dev')
    conn.sudo('apt-get install -y libgdbm-dev')
    conn.sudo('apt-get install -y libpq-dev')
    conn.sudo('apt-get install -y libc6-dev')
    conn.sudo('apt-get install -y libbz2-dev')
    conn.sudo('apt-get install -y zlib1g-dev')
    conn.sudo('apt-get install -y openssl')
    conn.sudo('apt-get install -y libffi-dev')
    conn.sudo('apt-get install -y python3-dev')
    conn.sudo('apt-get install -y python3-setuptools')
    conn.sudo('apt-get install -y uuid-dev')
    conn.sudo('apt-get install -y lzma-dev')
    conn.sudo('apt-get install -y wget')
    conn.sudo('apt-get install -y git')
    conn.sudo('apt-get install -y postgresql')


def _install_python(conn):
    """Install python 3.7 in the remote machine."""

    res = conn.run('python3 --version')
    if '3.7' in res.stdout.strip():
        # Python >= 3.7 already exists
        return

    conn.run('rm -rf /tmp/Python3.7 && mkdir /tmp/Python3.7')

    with conn.cd('/tmp/Python3.7'):
        conn.run('wget https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tar.xz')
        conn.run('tar xvf Python-3.7.0.tar.xz')

    with conn.cd ('/tmp/Python3.7/Python-3.7.0'):
        conn.run('./configure --enable-optimizations')
        conn.run('make')

    # see https://github.com/pyinvoke/invoke/issues/459
    conn.sudo('bash -c "cd /tmp/Python3.7/Python-3.7.0 && make altinstall"')


def _install_venv(conn):
    """Install virtualenv, virtualenvwrapper."""

    res = conn.run('which python3.7')
    res = res.stdout.strip()
    py_path = res

    conn.sudo('apt install -y virtualenvwrapper')

    # for a standard Debian distro
    venv_sh = '/usr/share/virtualenvwrapper/virtualenvwrapper.sh'

    conn.run('echo >> ~/.bashrc')  # new line
    conn.run(f'echo source {venv_sh} >> ~/.bashrc')
    conn.run('echo >> ~/.bashrc')  # new line
    conn.run('echo export LC_ALL=en_US.UTF-8 >> ~/.bashrc')
    conn.run('source ~/.bashrc')
    env = environ['VENV_NAME']
    with conn.prefix(f'source {venv_sh}'):
        conn.run(f'mkvirtualenv -p {py_path} {env}')


def _pull_repo(conn, branch=None, commit=None):
    if branch and commit:
        raise ValueError('Cannot provide both branch name and commit hash')
    source = environ['GIT_DIR']
    if not branch:
        branch = environ['GIT_DEFAULT_BRANCH']
    repo = environ['REPO_URL']
    if commit:
        print('Hash provided. Resetting to that commit.')
        conn.run(
            f"cd {source} && "
            'git stash && '
            f'git reset --hard {commit} && '
            'git checkout -B tmp_branch'
        )
    else:
        if conn.run(f'test -e {source}/.git', warn=True).ok:
            print('Repo already exists.')
            # run("cd %s && git pull upstream %s" % (source_dir, branch))
            #conn.run(f'cd {source} && git fetch origin {branch}')
            #conn.run(f'cd {source} && git reset --hard origin/{branch}')
        else:
            print('Repo did not exist. Creating it...')
            conn.run(f'git clone {repo} {source}')
            conn.run(f'cd {source} && git remote set-url origin {repo}')
        print('Checking out the requested branch...')
        conn.run(f'cd {source} && git fetch origin && git checkout {branch} && git pull origin {branch}')
    current_hash = conn.run(f'cd {source} && git log -n 1 --format=%H', hide='both')
    current_hash = current_hash.stdout.strip()
    print(f'Checked out {current_hash}')
    return current_hash


def _install_project(conn):
    repo_path = environ['GIT_DIR']
    venv_name = environ['VENV_NAME']
    venv_sh = 'source /usr/share/virtualenvwrapper/virtualenvwrapper.sh'
    with conn.cd(repo_path):
        with conn.prefix(
            f'{venv_sh} && workon {venv_name}'
        ):
            conn.run('pip install --upgrade pip')
            conn.run('pip install -r requirements.txt')
            # If your project as a `setup.py` then
            # install project.
            #conn.run('pip install -e .')


def _restart_web(conn):
    try:
        conn.sudo('pkill gunicorn')
    except:
        pass # may not be running at all.
    repo_path = environ['GIT_DIR']
    venv_name = environ['VENV_NAME']
    venv_sh = 'source /usr/share/virtualenvwrapper/virtualenvwrapper.sh'
    with conn.cd(repo_path):
        with conn.prefix(
            f'{venv_sh} && workon {venv_name}'
        ):
            conn.run("gunicorn app:app -b 0.0.0.0:8080 -w 3 --daemon")


#####################################
# Functions used from the __main__ ##
#####################################


def create_vm(**kwargs):
    _create_vm(create_conn())


def pull_repo(**kwargs):
    conn = create_conn()
    _pull_repo(conn, **kwargs)


def install_project(**kwargs):
    _install_project(create_conn())


def restart_web(**kwargs):
    _restart_web(create_conn())


def main(tasks):
    if len(tasks) <= 1:
        print('No task name found')
        return
    i = 1
    while i < len(tasks):
        try:
            fn = getattr(sys.modules[__name__], tasks[i])
        except AttributeError:
            print(f'Cannot find task {tasks[i]}. Quit.')
            return
        params = {}
        j = i + 1
        while j < len(tasks) and '=' in tasks[j]:
            k, v = tasks[j].split('=')
            params[k] = v
            j += 1
        i = j
        print(f'Function is {fn}')
        print(f'args are {params}')
        fn(**params)


if __name__ == '__main__':
    '''
    Run it with
    >>python main <task1> <key1-task1>=<value1-task1> <key2-task1>=<value2-task2> <task2> <key1-task2>=<value1-task2>

    E.g.
    $ python main create_vm
    '''
    import sys
    tasks = sys.argv
    main(tasks)