Manages python dependencies in requirements.txt file maintaining the ordering of the dependencies and any comments. Adds, removes, and updates, dependencies.

requires

#!/usr/bin/env python
# requires
# Creates a requirements.txt file using pip freeze.
#
# Author:   Benjamin Bengfort <[email protected]>
# Created:  Fri Jan 22 08:50:31 2016 -0500
#
# Copyright (C) 2016 Bengfort.com
# For license information, see LICENSE.txt
#
# ID: requires.py [] [email protected] $

"""
Creates a requirements.txt file with pip freeze, but does a better job at
dealing with commented packages in the freeze file.
"""

##########################################################################
## Imports
##########################################################################

import os
import pip
import sys
import shutil
import argparse

from platform import python_version as pyvers

try:
    from pip.operations import freeze
except ImportError:
    from pip._internal.operations import freeze


##########################################################################
## Command Description
##########################################################################

DESCRIPTION = "Creates better requirements.txt files using pip freeze."
EPILOG      = "An alternative is to use pip freeze -r requirements.txt"
VERSION     = "requires v1.1 | pip v{} | python v{}".format(pip.__version__, pyvers())

ARGUMENTS   = {
    ('-o', '--output'): {
        'metavar': 'PATH',
        'default': None,
        'type': str,
        'help': 'specify a path to write freeze file to',
    },
    ("-e", "--edit"): {
        'action': 'store_true',
        'default': False,
        'help': 'edit the requirements.txt file directly',
    },
    'requirements': {
        'nargs': '?',
        'default': 'requirements.txt',
        'help': 'text file containing requirements',
    },
}


##########################################################################
## Primary Functionality
##########################################################################

# Arguments that may be in the requirements.txt
reqargs = (
    '-r', '--requirement',
    '-Z', '--always-unzip',
    '-f', '--find-links',
    '-i', '--index-url',
    '--extra-index-url',
)

# Parsable operators for semantic versioning
operators = ("==", ">=", ">", "!=", "~=", "<", "<=")


def parse(dep):
    """
    Parses a dependency string and returns the name and the part.
    """

    if dep.startswith("-e") or dep.startswith("--editable"):
        if dep.startswith('-e'):
            name = dep[2:].strip()
        else:
            name = dep[len('--editable'):].strip().lstrip('=')

        return name, dep

    for operator in operators:
        if operator in dep:
            return dep.split(operator, 1)[0].strip(), dep

    return dep, dep


def packages(**kwargs):
    """
    Uses pip freeze to yield a list of packages installed.
    TODO: Directly implement pip freeze: https://github.com/pypa/pip
    """
    for dep in freeze.freeze(**kwargs):
        yield parse(dep)


def requires(args):
    """
    Compares the output of pip freeze with the contents of a requirements
    file and appropriately merges them (including skipping comments).
    """

    # Get the currently installed packages
    installed = dict(packages())
    removed   = []

    # If the requirements file exists, begin comparison
    if os.path.exists(args.requirements):
        with open(args.requirements, 'r') as rf:
            for line in rf:
                line = line.strip()

                # Newlines or arguments in the requirements kept as is
                if not line or line.startswith(reqargs):
                    yield line
                    continue

                # Handling commented lines in the requirements file
                if line.startswith("#"):
                    comment = line.lstrip("# ")
                    name, dep = parse(comment)

                    if name in installed:
                        yield "# {}".format(installed.pop(name))
                        continue

                    yield line
                    continue

                # Handle other dependencies in the requirements
                name, dep = parse(line)
                if name in installed:
                    yield installed.pop(name)
                else:
                    removed.append(name)

        # All remaining dependencies are added
        if installed:
            yield "\n## The following requirements were added by pip freeze:"


    # Yield the remaining sorted list of dependencies
    for dep in sorted(installed.values(), key=lambda x: x.lower()):
        yield dep

    # Yield the uninstalled dependencies
    if removed:
        yield "\n## The following requirements are no longer installed:"
        for name in removed: yield "# {}".format(name)


def write(args, output):
    path = args.requirements if args.edit else args.output
    if path:
        with open(path, 'w') as of:
            of.write("\n".join(output)+"\n")
    else:
        print("\n".join(output)+"\n")


##########################################################################
## Main Method
##########################################################################

def main(*args):

    # Construct the argument parser
    parser = argparse.ArgumentParser(
        description=DESCRIPTION, epilog=EPILOG
    )

    # Add version argument
    parser.add_argument('--version', action='version', version=VERSION)

    # Add the arguments from the definition above
    for keys, kwargs in ARGUMENTS.items():
        if not isinstance(keys, tuple):
            keys = (keys,)
        parser.add_argument(*keys, **kwargs)

    # Handle the input from the command line
    args   = parser.parse_args()

    # Determine the outpath
    if args.output and args.edit:
        parser.error("cannot specify both --edit and --output")
        return

    try:
        output = list(requires(args))
        write(args, output)
    except Exception as e:
        parser.error(str(e))
        return

    # Exit successfully
    parser.exit(0)

if __name__ == '__main__':
    main(*sys.argv[1:])