Per-subinterpreter GIL

.gitignore

subinterpreters

Makefile

GIL_MODE = PyInterpreterConfig_OWN_GIL
N = 37
PYTHON_BUILD ?= 3.12

CFLAGS = $(shell pkg-config --cflags python-$(PYTHON_BUILD)) -Wall -Wextra -Werror -pthread -ggdb3 -gdwarf-5 -DGIL_MODE=$(GIL_MODE) -DN=$(N)
LDFLAGS = $(shell pkg-config --libs python-$(PYTHON_BUILD)) -lpython$(PYTHON_BUILD) -pthread

all: subinterpreters

clean:
	rm -f subinterpreters
.PHONY: clean

subinterpreters.c

#define PY_SSIZE_T_CLEAN
#include <Python.h>

#include <assert.h>
#include <stdio.h>
#include <sched.h>
#include <unistd.h>
#include <pthread.h>

#define xstr(s) str(s)
#define str(s) #s

#ifndef N
# define N 37
#endif

#ifndef GIL_MODE
# define GIL_MODE PyInterpreterConfig_OWN_GIL
#endif

static const char *prog =
"def fib(n):\n"
"    return 1 if n <= 2 else fib(n - 1) + fib(n - 2)\n"
"\n"
"print(fib(" xstr(N) "))\n";

static void do_work() {
	PyRun_SimpleString(prog);
}

static pthread_mutex_t go = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t go_cond = PTHREAD_COND_INITIALIZER;

static void * start_thread(void * Py_UNUSED(arg)) {
	PyThreadState *ts = NULL, *main_ts = NULL;
	PyInterpreterConfig config = {
		.use_main_obmalloc = 0,
		.allow_fork = 0,
		.allow_exec = 0,
		.allow_threads = 0,
		.allow_daemon_threads = 0,
		.check_multi_interp_extensions = 1,
		.gil = GIL_MODE,
	};
	PyStatus status = { 0 };

	/* We, supposedly, need to hold "the" GIL before
	 * we can construct another interpreter.
	 */
	main_ts = PyThreadState_New(PyInterpreterState_Main());
	assert(main_ts != NULL);
	PyEval_RestoreThread(main_ts);

	/* If this succeeds, the current thread state will be
	 * from the new interpreter (so, `ts`) already, we'll
	 * hold its GIL, and the one from the current thread
	 * state is released.
	 */
	status = Py_NewInterpreterFromConfig(&ts, &config);
	if(PyStatus_Exception(status)) {
		fprintf(stderr, "Failed to create new interpreter: %s (in %s)\n", status.err_msg, status.func == NULL ? "NULL" : status.func);
	} else {
		/* Go back to main interpreter to clean up threadstate */
		ts = PyThreadState_Swap(main_ts);
	}

	/* Clean up main interpreter threadstate */
	PyThreadState_Clear(main_ts);
	PyThreadState_DeleteCurrent();

	/* Signal the main thread we're good to go */
	pthread_mutex_lock(&go);
	pthread_cond_signal(&go_cond);
	pthread_mutex_unlock(&go);

	/* If interpreter creation failed earlier, bail out */
	if(PyStatus_Exception(status)) {
		goto out;
	}

	/* Switch threadstate to the subinterpreter one and acquire its GIL */
	PyEval_RestoreThread(ts);

	/* Burn some CPU */
	do_work();	

	/* Clean up */
	Py_EndInterpreter(ts);

out:
	return NULL;
}

int main(int Py_UNUSED(argc), char ** Py_UNUSED(argv)) {
	pthread_t thread = { 0 };
	int rc = 1;
	void *out = NULL;

	Py_InitializeEx(1);

	rc = pthread_create(&thread, NULL, start_thread, NULL);
	if(rc != 0) {
		perror("pthread_create");
		rc = 1;
		goto out;
	}

	/* Allow the thread to acquire the main interpreter lock and start its interpreter */
	Py_BEGIN_ALLOW_THREADS
	pthread_mutex_lock(&go);
	pthread_cond_wait(&go_cond, &go);
	pthread_mutex_unlock(&go);
	Py_END_ALLOW_THREADS

	/* Burn some CPU, in the main interpreter */
	do_work();

	/* Wait for the thread to complete its job */
	rc = pthread_join(thread, &out);
	if(rc != 0) {
		perror("pthread_join");
		rc = 1;
		goto out;
	}

	rc = 0;

out:
	rc = Py_FinalizeEx();
	if(rc != 0) {
		rc = 1;
		goto out;
	}

	return rc;
}