fedora-review-justification.py

"""
Find out how many packages are added to Fedora every year and how many of them
are reviews using the `fedora-review` tool.

I suppose that this script is harsh on Bugzilla's load, please don't overuse it.

See http://frostyx.cz/posts/please-keep-fedora-review-alive
"""

import bugzilla


BZ_URL = "https://bugzilla.redhat.com/xmlrpc.cgi"
BZ_PAGE_SIZE = 1000


def _iterate_query(querydata):
    """
    Iterate Bugzilla query until all results are fetched.

    Taken from:
    https://github.com/python-bugzilla/python-bugzilla/issues/149#issuecomment-921971629
    """
    bz = bugzilla.Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi')
    results = bz.query(querydata)
    if len(results) == BZ_PAGE_SIZE:
        last_result_id = results[-1].id
        querydata['f1'] = 'bug_id'
        querydata['o1'] = 'greaterthan'
        querydata['v1'] = last_result_id
        print("Last ID: {0}".format(last_result_id))
        results += _iterate_query(querydata)
    return results


def get_bugs():
    bz = bugzilla.Bugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi')
    query = bz.build_query(
        product="Fedora",
        component="Package Review",
        status="CLOSED",
    )
    # query["blocks"] = ftibug.id
    query["limit"] = BZ_PAGE_SIZE
    query["offset"] = 0
    query["order_by"] = "bug_id"
    return _iterate_query(query)


def fedora_review_used(bug):
    for comment in bug.getcomments():
        if "===== MUST items =====" in comment["text"]:
            return True
        if "Package Review\n==============" in comment["text"]:
            return True
        if "==== Generic ====" in comment["text"]:
            return True
        if "review.txt" in comment["text"]:
            return True
    return False


def generator_used(bug):
    right = bug.summary.split("Review Request:")[-1]
    name = right.split(" - ")[0].strip()
    year = bug.creation_time.timetuple().tm_year

    # https://github.com/fedora-python/pyp2rpm
    if name.startswith("python-") and year >= 2012:
        return True

    # https://pagure.io/fedora-rust/rust2rpm
    if name.startswith("rust-") and year >= 2017:
        return True

    # https://pagure.io/GoSIG/go2rpm.git
    if name.startswith("golang-") and year >= 2019:
        return True

    # https://github.com/fedora-ruby/gem2rpm
    if name.startswith("rubygem-") and year >= 2007:
        return True

    # https://pagure.io/r2spec.git
    if name.startswith("R-") and year >= 2008:
        return True

    # https://github.com/juhp/cabal-rpm
    if name.startswith(("haskell-", "ghc-")) and year >= 2012:
        return True

    # https://github.com/pear/PEAR_Command_Packaging/tree/master/tests/make-rpm-spec
    if name.startswith(("php-", "ghc-")) and year >= 2012:
        return True

    return False


def dump_dat_file(results):
    path = "fedora-review-stats.dat"
    with open(path, "w") as fp:
        fp.write("# Stats of fedora-review usage\n")
        fp.write("@ fedora-review used, package generator probably used, fedora-review not used\n")
        for year, stats in sorted(results.items()):
            line = "{0},{1},{2},{3}\n".format(
                year,
                stats["used"],
                stats["generated"],
                stats["not-used"],
            )
            fp.write(line)
    return path


def dump_bz_url(bug, suffix):
    path = "fedora-review-{0}.txt".format(suffix)
    with open(path, "a+") as fp:
        fp.write("{0}\n".format(bug.weburl))


results = {}
bugs = get_bugs()
for i, bug in enumerate(bugs):
    print("Processing bug [{0}/{1}]".format(i, len(bugs)))

    ignored_resolutions = ["DUPLICATE", "NOTABUG", "DEFERRED", "WONTFIX",
                           "CANTFIX"]
    if bug.resolution in ignored_resolutions:
        print("Skipping resolution: #{0} {1}".format(bug.id, bug.resolution))
        continue

    if not bug.summary.startswith("Review Request:"):
        print("Skipping wrong summary: #{0}".format(bug.id))
        continue

    year = bug.last_change_time.timetuple().tm_year
    results.setdefault(year, {})
    results[year].setdefault("used", 0)
    results[year].setdefault("not-used", 0)
    results[year].setdefault("generated", 0)

    if fedora_review_used(bug):
        key = "used"
    elif generator_used(bug):
        key = "generated"
    else:
        key = "not-used"

    results[year][key] += 1
    dump_bz_url(bug, key)


path = dump_dat_file(results)
print("Written: {0}".format(path))