Find unreferenced strings in Monitor

find_unreferenced_strings_monitor.py

#!/usr/bin/env python3
import argparse
import os
import signal
import subprocess
import sys
from compare_locales import parser

# Capture CTRL+C
unreferenced_entities = []
processed_files = []


def signal_handler(sig, frame):
    print_output()
    sys.exit(0)


signal.signal(signal.SIGINT, signal_handler)


def print_output():
    if processed_files:
        print("\nProcessed files:")
        for f in processed_files:
            print(f"  - {f}")
    if unreferenced_entities:
        print("Unreference entities:")
        for e in unreferenced_entities:
            print(f"  - {e}")


def grep_message(entity, repository_path, file_path):
    """
    For standard messages, we need to ignore the /locales folder.

    For terms, we only search in reference .ftl files, but need to exclude the
    file where they are defined.
    """
    if entity.startswith("-"):
        grep_cmd = [
            "grep",
            "-R",
            "-F",
            "-q",
            "--include=*.ftl",
            f"--exclude={file_path}",
            "--", # otherwise, grep gets confused by the hyphen in "-someterm"
            entity,
            os.path.join(repository_path, "locales", "en"),
            os.path.join(repository_path, "locales-pending"),
        ]
    else:
        grep_cmd = [
            "grep",
            "-R",
            "-F",
            "-q",
            "--include=*.ts",
            "--include=*.tsx",
            f"--exclude-dir={os.path.join(repository_path, 'locales')}",
            f"--exclude-dir={os.path.join(repository_path, 'locales-pending')}",
            entity,
            repository_path,
        ]
    result = subprocess.run(grep_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    return result.returncode == 0


def extract_string_list(project_path):
    ref_paths = [
        os.path.join(project_path, "locales", "en"),
        os.path.join(project_path, "locales-pending"),
    ]
    string_list = {}

    for ref_path in ref_paths:
        file_list = []
        for root, dirs, files in os.walk(ref_path, followlinks=False):
            for file in files:
                if file.endswith(".ftl"):
                    file_list.append(os.path.join(root, file))
        file_list.sort()

        for file in file_list:
            rel_path = os.path.relpath(file, ref_path)
            file_parser = parser.getParser(".ftl")
            file_parser.readFile(file)
            try:
                entities = file_parser.parse()
                for entity in entities:
                    # Ignore Junk
                    if isinstance(entity, parser.Junk):
                        continue
                    if rel_path not in string_list:
                        string_list[rel_path] = []
                    string_list[rel_path].append(str(entity))
            except Exception as e:
                print(f"Error parsing file: {file}")
                print(e)

    return string_list


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--path", dest="repo_path", help="Path to repository clone", required=True
    )
    args = parser.parse_args()

    # Path to project clone
    string_list = extract_string_list(args.repo_path)

    # Excluded files and IDs
    excluded_files = [
        "data-classes.ftl",
    ]
    excluded_ids = []

    for file_path, entities in string_list.items():
        if file_path in excluded_files:
            continue
        for entity in entities:
            # Skip if the entity is in the exclusion list
            if entity in excluded_ids:
                continue

            sys.stdout.write(".")
            sys.stdout.flush()

            if not grep_message(entity, args.repo_path, file_path):
                unreferenced_entities.append(f"{file_path}:{entity}")
        processed_files.append(file_path)

    # Print output if execution wasn't interrupted
    print_output()


if __name__ == "__main__":
    main()