IncludesSmasher

README.md

Scan a folder hierarchy for .cpp, .h and .hpp files.

Build a simplisitc #include dependency graph, and compute a transitive score for each .cpp files.

Display a score at the bottom. The lower the better.

This can be used to speedup compilation by making sure the score goes as low as possible.

$ ./includes_smasher.py ~/src/foss/IXWebSocket/ixwebsocket 
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXHttpClient.cpp 71
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXGetFreePort.cpp 39
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocketPerMessageDeflate.cpp 20
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSelectInterruptFactory.cpp 19
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSetThreadName.cpp 1
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXCancellationRequest.cpp 2
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSocketTLSOptions.cpp 1
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXUserAgent.cpp 1
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXDNSLookup.cpp 30
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSocketMbedTLS.cpp 73
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXBench.cpp 3
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocketHttpHeaders.cpp 26
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocketCloseConstants.cpp 2
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSelectInterruptPipe.cpp 8
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocketProxyServer.cpp 234
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXStrCaseCompare.cpp 1
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSocketOpenSSL.cpp 35
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSocketAppleSSL.cpp 31
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocketServer.cpp 547
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocketPerMessageDeflateOptions.cpp 2
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXHttp.cpp 37
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXUuid.cpp 1
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSocketConnect.cpp 57
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSocket.cpp 47
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocketPerMessageDeflateCodec.cpp 12
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSocketServer.cpp 96
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXNetSystem.cpp 21
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXHttpServer.cpp 455
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXUdpSocket.cpp 47
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocket.cpp 252
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSocketFactory.cpp 110
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocketHandshake.cpp 102
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXConnectionState.cpp 5
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXExponentialBackoff.cpp 1
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSelectInterruptEvent.cpp 9
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXWebSocketTransport.cpp 209
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXUrlParser.cpp 1
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXSelectInterrupt.cpp 3
/Users/benjamin.sergeant/src/foss/IXWebSocket/ixwebsocket/IXGzipCodec.cpp 4
total deps 2615

includes_smasher.py

#!/usr/bin/env python3
'''
To sort by the file whose include complexity is the highest

./includes_smasher.py path/to/folder | sort -k 2 -n
'''

import sys
import os
import collections
import argparse
from typing import Dict, List


def walk_source_files(dn: str):
    for root, _, files in os.walk(dn):
        for path in files:
            _, ext = os.path.splitext(path)
            if ext not in ('.cpp', '.c', '.h', '.hpp'):
                continue

            abspath = os.path.join(root, path)
            yield abspath


def find_includes(path: str) -> list[str]:
    includes = []

    try:
        with open(path) as f:
            content = f.read()
    except:
        return []

    lines = content.splitlines()
    for line in lines:
        if '#include' in line:
            tokens = line.split()
            if len(tokens) > 1:
                filename = tokens[1]
                filename = filename.replace('"', '').replace("'", '')
                filename = filename.replace('<', '').replace('>', '')

                if filename != '#include':
                    includes.append(filename)

    return includes


# Thank you ChatGPT
def find_descendants(filename: str, includes: Dict[str, Iterable[str]]) -> List[str]:
    """
    Iterative DFS using an explicit stack.
    - Appends every encountered include (duplicates allowed).
    - Avoids infinite recursion by guarding cycles *per current path* (not globally).
    """
    result: List[str] = []

    # Stack holds (current_node, iterator over its children, path_set)
    stack: List[Tuple[str, Iterable[str], Set[str]]] = []

    # Start at the root filename
    root_children = list(includes.get(filename, []))
    stack.append((filename, iter(root_children), {filename}))

    while stack:
        node, it, path = stack[-1]
        try:
            child = next(it)
            result.append(child)  # record every include edge (even if repeated via other paths)

            # If child is not on the *current* path, descend
            if child not in path:
                child_children = list(includes.get(child, []))
                # push new frame with updated path
                stack.append((child, iter(child_children), path | {child}))
            # else: cycle detected on this path—do not descend, but we already counted the include
        except StopIteration:
            # done with this node
            stack.pop()

    return result


def run(root):
    includes_counter = collections.defaultdict(int)
    includes = collections.defaultdict(list)

    paths = list(walk_source_files(root))

    # First path collect all header -> [header1, header2, ...] relationships
    for path in paths:
        path_filename = os.path.basename(path)
        path_includes = find_includes(path)
        _, ext = os.path.splitext(path)

        for include in path_includes:
            includes_counter[include] += 1
            filename = os.path.basename(include)

            # trim includes
            if ext in ('.h', '.hpp'):
                includes[path_filename].append(filename)

    total_deps = 0

    for path in paths:
        _, ext = os.path.splitext(path)
        if ext not in ('.cpp', '.h'):
            continue

        deps_count = 0
        path_includes = find_includes(path)

        all_descendants = []
        for include in path_includes:
            filename = os.path.basename(include)

            descendants = find_descendants(filename, includes)
            deps_count += len(descendants)

            for descendant in descendants:
                all_descendants.append(descendant)

        print(path, deps_count)
        total_deps += deps_count

        if args.save and os.path.basename(path) == args.filename:
            with open('/tmp/deps', 'w') as f:
                f.write('\n'.join(all_descendants))

        if args.verbose:
            for header in all_descendants:
                print('\t' + header)

    print('total deps', total_deps)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("root", help="Root folder to explore")
    parser.add_argument("--filename", help=".cpp file to filter", default='foo.cpp')
    parser.add_argument("--save", help="Save deps for a given file to disk", action='store_true')
    parser.add_argument("--verbose", "-v", help="Print deps for each files", action='store_true')
    args = parser.parse_args()

    run(args.root)