Generic Binder Compilation Using CMake

compile.py

# GistID: 6ac4980910a08a2c3410cd5fc800cd17
"""

This module allows to compile cpp code into python modules using binder anc
cmake.

The following options are available:

* BINDER_REPO: a `Path` object to the cloned repo of binder
* SOURCE: a string-like to the C++ source code
* INCLUDE: a string-like to the C++ includes
* CMAKE_DIR: a `Path` object to the directory used for cmake and binding
* BINDER_CFG: a string-like to the binder configuration file
* CMAKE_VERSION: a string-like containing the cmake minimum version
* REBUILD: if True, first deletes the binding generated by binder

Cleaning
--------

#. To clean the compilation files, but not the bindings, enter the `CMAKE_DIR`
   and use: `ninja clean`.
#. To clean the bindings but not the compilation files, delete
   `{CMAKE_DIR}/binder`
#. To clean both bindings and compilation, remove `CMAKE_DIR`

Example
-------

In your `setup.py`, use:

```
import compile
compile.compile_chain('cpp_namespace', 'output_module', 'destination')
```

And in `destination/script.py`:

```
from output_module import MyObject, myfunc
```

"""
import glob
import os
from pathlib import Path
import shutil
import subprocess
from distutils.sysconfig import get_python_inc

# Overall script settings
BINDER_REPO = Path("/opt/binder")
SOURCE = f'{os.getcwd()}/include'
INCLUDE = SOURCE
CMAKE_DIR = Path('cmake_bindings')
BINDER_CFG = 'binder.cfg'
CMAKE_VERSION = '3.21'
REBUILD = False

bindings_dir = Path('binder')
full_bindings_dir = CMAKE_DIR / bindings_dir
binder_executable = next(
    (BINDER_REPO / 'build').glob('llvm-*/build_*/bin/binder'))
binder_source = BINDER_REPO / "source"
pybind_source = BINDER_REPO / "build" / "pybind11" / "include"
use_binder_cfg = True
cpp_std = 'c++20'


def make_all_includes():
    all_includes = []
    all_include_filename = 'all_cmake_includes.hpp'
    for filename in (glob.glob(f'{SOURCE}/**/*.hpp', recursive=True) +
                     glob.glob(f'{SOURCE}/**/*.cpp', recursive=True) +
                     glob.glob(f'{SOURCE}/**/*.h', recursive=True) +
                     glob.glob(f'{SOURCE}/**/*.cc', recursive=True) +
                     glob.glob(f'{SOURCE}/**/*.c', recursive=True)):
        with open(filename, 'r') as fh:
            for line in fh:
                if line.startswith('#include'):
                    all_includes.append(line.strip())
    all_includes = list(set(all_includes))
    # This is to ensure that the list is always the same and doesn't
    # depend on the filesystem state.  Not technically necessary, but
    # will cause inconsistent errors without it.
    all_includes.sort()
    with open(all_include_filename, 'w') as fh:
        for include in all_includes:
            fh.write(f'{include}\n')
    return all_include_filename


def make_bindings_code(all_includes_fn, cpp_namespace, python_module):
    if REBUILD:
        shutil.rmtree(full_bindings_dir, ignore_errors=True)
    if not os.path.exists(full_bindings_dir):
        os.makedirs(full_bindings_dir)
    command = (f'{binder_executable} --root-module {python_module} '
               f'--prefix {os.getcwd()}/{full_bindings_dir}/ '
               f'--bind {cpp_namespace} ' +
               ('--config ' + BINDER_CFG if use_binder_cfg else '') +
               f' {all_includes_fn} -- -std={cpp_std} '
               f'-I{INCLUDE} -DNDEBUG -v').split()
    print('BINDER COMMAND:', ' '.join(command))
    subprocess.check_call(command)
    sources_to_compile = []
    with open(f'{full_bindings_dir}/{python_module}.sources', 'r') as fh:
        for line in fh:
            sources_to_compile.append(bindings_dir / line.strip())
    return sources_to_compile


def compile_sources(sources_to_compile, python_module):
    back_dir = os.getcwd()
    os.chdir(CMAKE_DIR)
    lines_to_write = []
    lines_to_write.append(f'cmake_minimum_required(VERSION {CMAKE_VERSION})')
    lines_to_write.append(f'project({python_module})')

    for include_dir in [
            binder_source, SOURCE, INCLUDE, pybind_source,
            get_python_inc()
    ]:
        lines_to_write.append(f'include_directories({include_dir})')
    lines_to_write.append(
        'set_property(GLOBAL PROPERTY POSITION_INDEPENDENT_CODE ON)')  # -fPIC
    lines_to_write.append('add_definitions(-DNDEBUG)')

    lines_to_write.append(f'add_library({python_module} SHARED')
    for source in sources_to_compile:
        lines_to_write.append(f'\t{source}')
    lines_to_write.append(')')
    lines_to_write.append(
        f'set_target_properties({python_module} PROPERTIES PREFIX "")')
    lines_to_write.append(
        f'set_target_properties({python_module} PROPERTIES SUFFIX ".so")')

    with open('CMakeLists.txt', 'w') as f:
        for line in lines_to_write:
            f.write(f'{line}\n')
    # Done making CMakeLists.txt
    subprocess.call('cmake -G Ninja'.split())
    subprocess.call('ninja')
    os.chdir(back_dir)


def compile_chain(cpp_namespace, python_module, dst_dir=None):
    """
    This function uses binder to bind `cpp_namespace` into a `python_module`
    and moves the compiled '.so' library to `dst_dir`.
    """

    all_includes_fn = make_all_includes()
    sources_to_compile = make_bindings_code(all_includes_fn, cpp_namespace,
                                            python_module)
    compile_sources(sources_to_compile, python_module)

    if dst_dir is not None:
        dst_dir = Path(dst_dir)
        compiled_name = python_module + '.so'
        if os.path.exists(dst_dir / compiled_name):
            os.remove(dst_dir / compiled_name)
        shutil.move(CMAKE_DIR / compiled_name, dst_dir)