Call C function from Reaper JSFX PoC

jsfx_addfunc.c

#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <math.h>

#include "ns-eel.h"

#ifndef I_KNOW_WHAT_I_AM_DOING
#error Hardcoded offsets are for 32-bit 5.95 and possibly other versions
#endif

#define BASE 0x10000000UL

typedef void (*pNSEEL_addfunc_ret_type_t)(const char *, int, int, NSEEL_PPPROC, void *, eel_function_table *);
static pNSEEL_addfunc_ret_type_t pNSEEL_addfunc_ret_type = (pNSEEL_addfunc_ret_type_t)(0x10002220UL - BASE);
static NSEEL_PPPROC NSEEL_PProc_THIS = (NSEEL_PPPROC)(0x10008b00UL - BASE);
static void NSEEL_addfunc_ret_type(const char *name, int np, int ret_type, NSEEL_PPPROC pproc, void *fptr, eel_function_table *destination) {
	pNSEEL_addfunc_ret_type(name, np, ret_type, pproc, fptr, destination);
}

static EEL_F NSEEL_CGEN_CALL add(void *opaque, EEL_F *a, EEL_F *b) {
	return *a + *b;
}
static int NSEEL_CGEN_CALL setReaEQCircleSize(void *opaque, EEL_F *size) {
	HMODULE reaeq = GetModuleHandle("reaeq.dll");
	if (reaeq == NULL) {
		return 0;
	}
	DWORD textSize = 0x31000;
	DWORD prot;
	if (!VirtualProtect(reaeq+0x1000, textSize, PAGE_EXECUTE_READWRITE, &prot)) {
		return 0;
	}
	*(float *)(0x1001416CUL - BASE + (DWORD)reaeq) = *size;
	char sz = lround(*size) + 1;
	*(char *)(0x10016D04UL - BASE + (DWORD)reaeq) = -sz;
	*(char *)(0x10016D43UL - BASE + (DWORD)reaeq) = sz;
	*(char *)(0x10016DF2UL - BASE + (DWORD)reaeq) = sz;
	*(char *)(0x10016E4FUL - BASE + (DWORD)reaeq) = -sz;
	DWORD t;
	VirtualProtect(reaeq+0x1000, textSize, prot, &t);
	return 1;
}

static DWORD (*JesusonicAPI_init)(DWORD, DWORD, DWORD);

static DWORD init(DWORD arg1, DWORD arg2, DWORD arg3) {
	DWORD ret = JesusonicAPI_init(arg1, arg2, arg3);
	NSEEL_addfunc_retval("myadd", 2, NSEEL_PProc_THIS, add);
	NSEEL_addfunc_retbool("setReaEQCircleSize", 1, NSEEL_PProc_THIS, setReaEQCircleSize);
	return ret;
}

__declspec(dllexport)
int ReaperPluginEntry(HINSTANCE hInstance, void *rec) {
	if (rec == NULL) { return 0; }
	HMODULE jsfx = GetModuleHandle("jsfx.dll");
	if (jsfx == NULL) { return 0; }
	pNSEEL_addfunc_ret_type = (pNSEEL_addfunc_ret_type_t)((DWORD)pNSEEL_addfunc_ret_type + (DWORD)jsfx);
	NSEEL_PProc_THIS = (NSEEL_PPPROC)((DWORD)NSEEL_PProc_THIS + (DWORD)jsfx);
	void **JesusonicAPI = (void **)GetProcAddress(jsfx, "JesusonicAPI");
	if (JesusonicAPI == NULL) { return 0; }
	JesusonicAPI_init = (DWORD (*)(DWORD,DWORD,DWORD))JesusonicAPI[1];
	JesusonicAPI[1] = init;
	return 1;
}

I forgot what I originally did, but in the WDL function for registering string functions

void EEL_string_register()
{
  NSEEL_addfunc_retval("strlen",1,NSEEL_PProc_THIS,&_eel_strlen);
  NSEEL_addfunc_retval("strcat",2,NSEEL_PProc_THIS,&_eel_strcat);
  NSEEL_addfunc_retval("strcpy",2,NSEEL_PProc_THIS,&_eel_strcpy);
  ...

we can use that to find the corresponding function in the binary.

If you use Ghidra on reaper.exe (x64 version 6.40 in this example), search for strlen

                             s_strlen_140a0e3ac        XREF[1]:     FUN_1406c4b80:1406c4bab(*)  
   140a0e3ac     ds       "strlen"
   140a0e3b3     ??       00h

go to the XREF'd function (FUN_1406c4b80), and that would be EEL_string_register. The decompiled code is

void FUN_1406c4b80(void)

{
  undefined8 *puVar1;
  
  puVar1 = DAT_140beadf0;
  if (DAT_140beadf0 == (undefined8 *)0x0) {
    puVar1 = &DAT_140beadf8;
  }
  FUN_1400d79e4("strlen",1,1,FUN_1400cee44,FUN_1406ba590,puVar1);
  ...

FUN_1400d79e4 is NSEEL_addfunc_retval but is actually NSEEL_addfunc_ret_type
FUN_1400cee44 is NSEEL_PProc_THIS
#define BASE should be 0x140000000 (see Memory Map...Image Base in Ghidra)

awesome, thanks ess7. one correction, your original code requires the functions from jsfx.dll, not reaper.exe. that may also help them change less frequently (depending on how often they update jsfx.dll).

I managed to get them from Ghidra following your v. helpful instructions. For Reaper 6.4 (x64), they are:

#define BASE 0x180000000
static pNSEEL_addfunc_ret_type_t pNSEEL_addfunc_ret_type = (pNSEEL_addfunc_ret_type_t)(0x18000ceb8 - BASE);
static NSEEL_PPPROC NSEEL_PProc_THIS = (NSEEL_PPPROC)(0x180004338 - BASE);

And as it's 64bit, change JesusonicAPI_init() and pointer math to use LONG_PTR instead of DWORD (or just int):

static LONG_PTR (*JesusonicAPI_init)(LONG_PTR, LONG_PTR, LONG_PTR);

I just remembered I dumped the undocumented functions exported by reaper_plugin_info_t GetFunc() and the NSEEL functions are there:
https://stash.reaper.fm/v/34480/Undocumented%20API%20functions.txt

REAPER_PLUGIN_DLL_EXPORT int REAPER_PLUGIN_ENTRYPOINT(REAPER_PLUGIN_HINSTANCE hInstance, reaper_plugin_info_t *rec) {
  ...
  pNSEEL_addfunc_ret_type = (pNSEEL_addfunc_ret_type_t)rec->GetFunc("NSEEL_addfunc_ret_type");
  NSEEL_PProc_THIS = (NSEEL_PPPROC)rec->GetFunc("NSEEL_PProc_THIS");
  ...

I'm getting the correct addresses but the function is not registered. Even EEL_string_register is not called by Reaper but the string functions work. Still investigating.

By the way I wanted to reply to the Reaper forum threads about this topic. Tried registering, waited for approval, but they banned my IP.

Added a new gist that works on 6.40 x64:
https://gist.github.com/ess7/6c75c111194a9159d05472aadc956c01

I'll add some commentary later. If it doesn't work try a "Full recompile/reset"