Process C-like preprocessors permutations (by creating individual files for each permutation)

process_preprocessors_permutations.py

#!/usr/bin/env python3
import argparse
from enum import Enum
from io import TextIOWrapper, StringIO
from os.path import splitext
from pathlib import Path
from typing import Union
import re
import subprocess
import tempfile

VERSION = "0.1.0"

TOKENS_REGEX_PATTERN = r"^[\S\t]*?#[\S\t]*?(include|elif|else|ifdef|ifndef|endif|if).*$"
TOKENS_REGEX = re.compile(TOKENS_REGEX_PATTERN, re.IGNORECASE | re.UNICODE)
TOKENS_TO_IGNORE_REGEX_PATTERN = r"^[\S\t]*?#[\S\t]*?(define|undef|line|error|embed|pragma).*$"
TOKENS_TO_IGNORE_REGEX = re.compile(TOKENS_TO_IGNORE_REGEX_PATTERN, re.IGNORECASE | re.UNICODE)

EMSCRIPTEN_INTERPOLATION_REGEX_PATTERN = r"\{\{\{.+?\}\}\}"
EMSCRIPTEN_INTERPOLATION_REGEX = re.compile(EMSCRIPTEN_INTERPOLATION_REGEX_PATTERN, re.IGNORECASE | re.UNICODE | re.DOTALL)

verbose = False

class Fragment(list[Union["Fragment", "Either", str]]):
    def __str__(self) -> str:
        return f"<Fragment {[x for x in self]}>"

    def __repr__(self) -> str:
        return f"<Fragment {[x for x in self]}>"

    def copy(self) -> "Fragment":
        return self.copy()


class Either:
    def __init__(self):
        self.if_statements: list[Fragment] = []
        self.else_statement: Fragment | None = None

    def __str__(self) -> str:
        return f"<Either if_statements: {[x for x in self.if_statements]}, else_statement: {self.else_statement} >"

    def __repr__(self) -> str:
        return f"<Either if_statements: {[x for x in self.if_statements]}, else_statement: {self.else_statement} >"


class RenderedFile(list[str]):
    def copy(self) -> "RenderedFile":
        return RenderedFile(super().copy())


def parse_lines(lines: list[str]) -> Fragment:
    current_fragment = Fragment()
    current_either = Either()
    current_buffer: list[str] = []
    current_level = 0

    in_else = False

    for line in lines:
        ignore_match = TOKENS_TO_IGNORE_REGEX.match(line)
        if ignore_match is not None:
            continue

        preprocessor_match = TOKENS_REGEX.match(line)
        if preprocessor_match is None:
            current_buffer.append(line)
            continue

        # We are currently parsing a token.
        token = preprocessor_match.groups()[-1]

        if token == "include":
            # TODO: actually parse include. (ONLY IF ASKED)
            # current_fragment += parse_lines(lines_of_the_include)
            continue

        if token == "if" or token == "ifdef" or token == "ifndef":
            current_level += 1
            if current_level == 1:
                # We only care about level 1, as we delegate other levels to
                # a recursive call.
                current_fragment += current_buffer
                current_either = Either()
                current_buffer = []
                continue
        elif token == "endif":
            current_level -= 1
            if current_level < 0:
                raise ValueError("current_level < 0")
            if current_level == 0:
                if len(current_buffer) > 0:
                    if in_else:
                        new_fragment = parse_lines(current_buffer)
                        current_either.else_statement = new_fragment
                    else:
                        new_fragment = parse_lines(current_buffer)
                        current_either.if_statements.append(new_fragment)
                    in_else = False
                    current_buffer = []
                current_fragment.append(current_either)
                continue
        else:
            if current_level == 1:
                if len(current_buffer) > 0:
                    new_fragment = parse_lines(current_buffer)
                    current_either.if_statements.append(new_fragment)
                    current_buffer = []

                in_else = token == "else"
                continue

        current_buffer.append(line)

    if len(current_buffer) > 0:
        current_fragment += current_buffer

    return current_fragment


def render_either(either: Either) -> list[RenderedFile]:
    files_rendered: list[RenderedFile] = []

    for if_statement in either.if_statements:
        files_rendered += render_fragment(if_statement)

    if either.else_statement is not None:
        files_rendered += render_fragment(either.else_statement)

    return files_rendered


def render_fragment(fragment: Fragment, root = False, eslint=False) -> list[RenderedFile]:
    files_rendered: list[RenderedFile] = []
    content_so_far: list[str] = []
    previous_element_type = type(None)

    program_len = len(fragment)
    for index, element in enumerate(fragment):
        if previous_element_type == Either and len(files_rendered) > 1 and root and eslint:
            worst_case_scenario = files_rendered[0]
            for i in range(1, len(files_rendered)):
                worst_case_contender = files_rendered[i]
                if len(worst_case_contender) > len(worst_case_scenario):
                    worst_case_scenario = worst_case_contender
            worst_case_scenario_string = "".join(worst_case_scenario)
            try:
                subprocess.run(["eslint", "--no-config-lookup", "--stdin", "--ext=js"], input=worst_case_scenario_string, text=True, check=True, capture_output=True)
                files_rendered = [worst_case_scenario]
            except subprocess.CalledProcessError as e:
                if verbose:
                    print(e.output)
                pass

        if root and verbose:
            print(f"[{len(files_rendered)}] Rendering element {index} of {program_len}. Type: {type(element)}")

        if type(element) == str:
            content_so_far.append(element)
        elif type(element) == Either:
            if len(files_rendered) == 0:
                new_file = RenderedFile()
                new_file += content_so_far
                files_rendered.append(new_file)
            else:
                for file_rendered in files_rendered:
                    file_rendered += content_so_far
            content_so_far = []

            new_files_rendered: list[RenderedFile] = []
            for file_rendered in files_rendered:
                for if_statement in element.if_statements:
                    if_rendered_files = render_fragment(if_statement)
                    for if_rendered_file in if_rendered_files:
                        new_file = file_rendered.copy()
                        new_file += if_rendered_file
                        new_files_rendered.append(new_file)
                if element.else_statement is None:
                    # We need to render the "without if"
                    new_files_rendered.append(file_rendered.copy())
                else:
                    else_rendered_files = render_fragment(element.else_statement)
                    for else_rendered_file in else_rendered_files:
                        content = file_rendered.copy()
                        content += else_rendered_file
                        new_files_rendered.append(content)
            files_rendered = new_files_rendered

        elif type(element) == Fragment:
            files_rendered += render_fragment(element)

        previous_element_type = type(element)

    if len(content_so_far) > 0:
        if len(files_rendered) == 0:
            content = RenderedFile()
            content += content_so_far
            files_rendered.append(content)
        else:
            new_files_rendered: list[RenderedFile] = []
            for file_rendered in files_rendered:
                content = RenderedFile()
                content += file_rendered + content_so_far
                new_files_rendered.append(content)
            files_rendered = new_files_rendered

    return files_rendered


def main(args: argparse.Namespace) -> None:
    global verbose
    verbose = args.verbose

    target_directory: Path = args.target_directory
    target_directory.mkdir(exist_ok=True)

    filename: TextIOWrapper = args.filename
    real_file_name_splitext = splitext(Path(filename.name).name)
    real_file_name_without_ext = real_file_name_splitext[0]
    real_file_name_only_ext = real_file_name_splitext[1][1:]

    with filename as f:
        lines = f.readlines()

        if args.replace_emscripten_interpolation:
            joined_lines = "".join(lines)
            joined_lines = re.sub(EMSCRIPTEN_INTERPOLATION_REGEX, "true", joined_lines)
            lines = joined_lines.split("\n")
            del joined_lines

        top_fragment = parse_lines(lines)

        rendered_files = render_fragment(top_fragment, root=True, eslint=args.eslint)

        for index, rendered_file in enumerate(rendered_files):
            rendered_filename = f"{real_file_name_without_ext}_{index}.{real_file_name_only_ext}"
            with open(target_directory.joinpath(rendered_filename), "w") as tf:
                if verbose:
                   print(f"Writing '{rendered_filename}'")
                tf.writelines(rendered_file)
        print(f"Done. Created {len(rendered_files)} files.")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Creates a file for every preprocessor permutation"
    )
    parser.add_argument(
        "filename",
        type=argparse.FileType('r', encoding="utf-8"),
        help="File to process"
    )
    parser.add_argument(
        "target_directory",
        type=Path,
        help="Directory to output permutations"
    )
    parser.add_argument(
        "-v", "--version",
        action="version",
        version=f"%(prog)s {VERSION}",
        help="Get the version"
    )
    parser.add_argument(
        "-V", "--verbose",
        action="store_true",
        dest="verbose",
        help="Toggle verbosity"
    )
    parser.add_argument(
        "--eslint",
        action="store_true",
        dest="eslint",
        help="Use eslint on JavaScript files in order to test their validity."
    )
    parser.add_argument(
        "--replace-emscripten-interpolation",
        action="store_true",
        dest="replace_emscripten_interpolation",
        help="Replace `{{{ interpolation }}}` tags with `true`"
    )
    args = parser.parse_args()
    main(args)