List Common Voice duplicate sentences for Portuguese

list_pt_same_normalized.py

#!/usr/bin/env python3
"""Find Portuguese sentences with different sentence_ids that match after normalization.

Normalization: lowercase, drop commas and periods, collapse whitespace. A group is reported when two or more distinct
sentence_ids share that key.

Usage:
  python3 list_pt_same_normalized.py validated.tsv pt-same-text.tsv
"""

from __future__ import annotations

import argparse
import csv
import re
from collections import defaultdict
from pathlib import Path

SPACE_RE = re.compile(r"\s+")
PUNCTUATION_RE = re.compile(r"[.,]")


def normalize(sentence: str) -> str:
    text = sentence.lower()
    text = PUNCTUATION_RE.sub("", text)
    return SPACE_RE.sub(" ", text).strip()


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("tsv", type=Path, help="Common Voice validated.tsv")
    parser.add_argument("output", type=Path, help="TSV of groups that collapse to the same text")
    args = parser.parse_args()

    # sentence_id -> set of original sentences and the normalized key
    by_key: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
    with args.tsv.open(newline="") as in_fobj:
        reader = csv.DictReader(in_fobj, delimiter="\t", quoting=csv.QUOTE_NONE)
        for row in reader:
            sentence_id = row.get("sentence_id") or ""
            sentence = (row.get("sentence") or "").strip()
            if not sentence_id or not sentence:
                continue
            by_key[normalize(sentence)][sentence_id].add(sentence)

    duplicate_groups = [(key, ids) for key, ids in by_key.items() if len(ids) > 1]
    duplicate_groups.sort(key=lambda item: (-len(item[1]), item[0]))
    with args.output.open("w") as out_fobj:
        writer = csv.DictWriter(out_fobj, delimiter="\t", fieldnames=("key", "sentence", "sentence_id"))
        writer.writeheader()
        for key, ids in duplicate_groups:
            for sentence_id, sentences in sorted(ids.items()):
                for sentence in sorted(sentences):
                    writer.writerow({"key": key, "sentence": sentence, "sentence_id": sentence_id})
    print(f"{len(duplicate_groups)} duplicate groups with more than one sentence_id saved to {args.output}")


if __name__ == "__main__":
    main()

pt-same-text.tsv