Collect documentation requests from commits

doc-requests.py

#!/usr/bin/env python

from __future__ import print_function
import sys
import re
import subprocess
import argparse
import csv


# {{{ Helpers

def popen(cmdline):
    """ Wrapper around Popen.subprocess() that redirects the output to a pipe,
        correctly handles encoding and raises a RuntimeError if the executable
        was not found. Works on both Python 2 and 3.
    """
    popen_kwargs = {
        'stdout': subprocess.PIPE,
    }
    if sys.version_info[0] == 3:
        popen_kwargs['encoding'] = 'utf-8'

    if sys.version_info[0] == 2:
        global FileNotFoundError
        FileNotFoundError = OSError

    try:
        return subprocess.Popen(cmdline, **popen_kwargs)
    except FileNotFoundError as e:
        raise RuntimeError("Unable to find '{}' executable: {}".format(
            cmdline[0], str(e)))


# }}} Helpers


class Commit:
    """ Represents a result of parsing a git commit from git log.
    """

    COMMIT_LINE_RE = re.compile(r'^commit ([0-9a-f]{40})')
    MERGE_RE = re.compile(r'Merge: ')
    AUTHOR_RE = re.compile(r'^Author: (.*)$')
    DATE_RE = re.compile(r'^Date: (.*)$')
    MESSAGE_LINE_RE = re.compile(r'^[ ]{4}(.*)$')
    DOCBOT_START_RE = re.compile(r'^[ ]{4}@TarantoolBot document$')
    DOCBOT_TITLE_RE = re.compile(r'^[ ]{4}Title: (.*)$')

    def __init__(self):
        self._commit_id = None
        self._author = None
        self._date = None
        self._subject = None
        self._body = None
        self._docbot_started = False
        self._docbot_title = None
        self._docbot_body = None

    def commit_id(self):
        return self._commit_id

    def subject(self):
        return self._subject

    def has_docbot(self):
        return self._docbot_started

    def docbot_title(self):
        return self._docbot_title

    def docbot_body(self):
        return self._docbot_body

    def __str__(self):
        return 'commit {} ("{}")'.format(self._commit_id[:12], self._subject)

    def __repr__(self):
        return str(self)

    def add_line(self, line):
        m = self.COMMIT_LINE_RE.match(line)
        if m:
            assert self._commit_id is None
            self._commit_id = m.group(1)
            return

        m = self.MERGE_RE.match(line)
        if m:
            # Ignore.
            return

        m = self.AUTHOR_RE.match(line)
        if m:
            assert self._author is None
            self._author = m.group(1)
            return

        m = self.DATE_RE.match(line)
        if m:
            assert self._date is None
            self._date = m.group(1)
            return

        # The empty line after the headers and before the message.
        if line == '':
            return

        m = self.MESSAGE_LINE_RE.match(line)
        if m:
            xline = m.group(1)
            if self._subject is None:
                self._subject = xline
                return
            if self._body is None and xline == '':
                return
            if self._body is None:
                self._body = ''
            self._body += xline + '\n'

            m = self.DOCBOT_START_RE.match(line)
            if m:
                assert not self._docbot_started
                self._docbot_started = True
                self._docbot_body = ''
                return

            if self._docbot_started:
                self._docbot_body += xline + '\n'
                m = self.DOCBOT_TITLE_RE.match(line)
                if m:
                    assert self._docbot_title is None
                    self._docbot_title = m.group(1)

            return

        raise RuntimeError('Unexpected line: {}'.format(line))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Collect documentation requests from commits')
    parser.add_argument('--body', action='store_true',
                        help='Whether to store docbot body')
    args = parser.parse_args()
    store_body = args.body

    process = popen(['git', 'log', '--reverse'])

    commits = []
    cur = Commit()
    for line in process.stdout:
        line = line.rstrip()
        if line.startswith('commit'):
            if cur.has_docbot():
                commits.append(cur)
            cur = Commit()
        cur.add_line(line)
    process.wait()

    if store_body:
        output_file = 'doc-requests-with-bodies.csv'
    else:
        output_file = 'doc-requests.csv'
    with open(output_file, 'w') as f:
        header = [
            'commit id',
            'subject',
            'docbot title',
        ]
        if store_body:
            header.append('docbot body')
        print(','.join(header), file=f)

        w = csv.writer(f)

        for commit in commits:
            data = [
                commit.commit_id(),
                commit.subject(),
                commit.docbot_title(),
            ]
            if store_body:
                data.append(commit.docbot_body())
            w.writerow(data)
    print('Written {}'.format(output_file), file=sys.stderr)