strip_article.py

T = TypeVar('T')
K = TypeVar('K')
def group_by(iterable: Iterable[T], key: Callable[[T], K]) -> Dict[K, List[T]]:
    groups: Dict[K, List[T]] = {}
    for item in iterable:
        k = key(item)
        if k in groups: groups[k].append(item)
        else: groups[k] = [item]
    return groups

usable_form_pattern = re.compile(r'^\s*[^\s*]')  # form does not start with *

def strip_article(article: Iterable[List[str]]) -> Iterable[List[str]]:
    """Removes unused (marked with *) and identical forms from an article.
    Unions gammatical features of identical forms.
    """
    usable_forms = (row for row in article if usable_form_pattern.match(row[0]))
    
    forms_grouped_by_raw_trans = group_by(usable_forms, lambda row: row[2])
    for _, grouped_rows in forms_grouped_by_raw_trans.items():
        result_row = grouped_rows[0]
        gram_features = (feature for row in grouped_rows for feature in row[1].split())
        unique_gram_features = mit.unique_everseen(gram_features)
        result_row[1] = f' {" ".join(unique_gram_features)} '
        yield result_row