Implement darwin cross-compile tooling for node-capnp

This commit introduces tooling to allow a linux host to compile
node-capnp for a darwin target. To do this, tooling is also provided to
allow for cross-compiling capnproto itself, as this didn't really seem
to function before. It's all still a little bit hacky and relies upon
you having the osxcross toolchain available, but I have at least a
moderate belief that it might work.

Approved-by: Henry Harrod

Author: p--b

.gitignore

@@ -1,2 +1,5 @@
 node_modules
 build
+capnp-darwin.env
+build-capnp/
+package-lock.json

README-DARWIN.md

@@ -0,0 +1,19 @@
+So, you want to cross-compile for darwin? Have fun.
+
+Jokes, I've done all the work for you. Simply:
+
+0. Make the osxcross darwin clang toolchain available on your PATH (rewind-server has a handy script
+   to do all the building for you)
+1. `./build-capnp-darwin.sh` (this builds capnproto so we have darwin libs and headers available)
+2. `. capnp-darwin.env` (this exports `CAPNP_LIBDIR` and `CAPNP_INCDIR` so capnp can be found by the node
+   module build script)
+3. `npm run build:darwin` (this builds the capnp.node native module for darwin)
+4. ...
+5. Profit!!!
+
+NOTE that the generated capnp dylibs will not yet have been patched up to have the correct link
+paths, and so are still named `*.so` - you'll want to use `install_name_tool` to fix them up before
+bundling them into your application. The node module binary itself *IS* patched by this tooling (see
+`build.js` for more fascinating details).
+
+(ymmv, e&oe...)

build-capnp-darwin.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Fourier Rewind
+# Copyright Fourier Audio Ltd. 2021. All Rights Reserved.
+
+set -euf -o pipefail
+
+o64-clang --version
+o64-clang++ --version
+
+# Get directory of this script in a relatively robust fashion (pun intended); see
+# http://www.binaryphile.com/bash/2020/01/12/determining-the-location-of-your-script-in-bash.html
+HERE=$(cd "$(dirname "$BASH_SOURCE")"; cd -P "$(dirname "$(readlink "$BASH_SOURCE" || echo -e "$BASH_SOURCE")")"; pwd)
+
+BUILD="${HERE}/build-capnp"
+
+rm -rf "${BUILD}"
+mkdir -p "${BUILD}"
+pushd $_
+
+curl -O https://capnproto.org/capnproto-c++-0.8.0.tar.gz
+tar zxf capnproto-c++-0.8.0.tar.gz
+pushd capnproto-c++-0.8.0
+
+CC=o64-clang CXX=o64-clang++ ./configure --build=x86_64-apple-darwin --host=x86_64-linux-gnu
+
+# libtool seems to decide that we must build shared libraries with -nostdlib as the linker will
+# necessarily link against the wrong stdlib. I don't believe this to be true for our toolchain, but
+# I'm also not enough of a GNU greybeard to know how to dissuade libtool from emitting this linker
+# flag. So, we'll do it the brute-force-and-ignorance way.
+sed -i 's/ -nostdlib//g' ./libtool
+
+# Similarly, the configure script seems to misdetect the value of the max_cmd_length variable,
+# setting it to an empty string. This results in the libtool linker command going bang approximately
+# half way through. Force the value to be set.
+ARG_MAX=`getconf ARG_MAX`
+sed -i "s/max_cmd_len=$/max_cmd_len=${ARG_MAX}/" ./libtool
+
+# The capnp makefile tries to run the capnpc tool as a smoke test approximately half way through the
+# process. Unsurprisingly, when we are cross-compiling, this approach is met with limited success.
+# Expunge this from the makefile, replacing the test_capnp_outputs rule with an empty recipe.
+sed -i 's/(test_capnpc_outputs): test_capnpc_middleman$/(test_capnpc_outputs):\n\t@:/' Makefile
+
+CC=o64-clang CXX=o64-clang++ make -j
+
+make install-data DESTDIR=capnp-root
+
+export CAPNP_LIBDIR=`pwd`/.libs
+export CAPNP_INCDIR=`pwd`/capnp-root/usr/local/include
+
+popd
+popd
+
+cat << EOF > capnp-darwin.env
+export CAPNP_LIBDIR="${CAPNP_LIBDIR}"
+export CAPNP_INCDIR="${CAPNP_INCDIR}"
+EOF
+
+echo "Written env file $(pwd)/capnp-darwin.env !";

build.js

@@ -31,25 +31,56 @@ var force = false, debug = false;
 var
 	arch = process.arch,
 	platform = process.platform,
-	v8 = /[0-9]+\.[0-9]+/.exec(process.versions.v8)[0];
+	v8 = /[0-9]+\.[0-9]+/.exec(process.versions.v8)[0],
+  environment = { ...process.env };
+
+var patchLibPath = false, patchTool = null;
+
 var args = process.argv.slice(2).filter(function(arg) {
 	if (arg === '-f') {
 		force = true;
 		return false;
 	} else if (arg.substring(0, 13) === '--target_arch') {
 		arch = arg.substring(14);
+    force = true; // No point trying to run a binary that isn't the host's arch
+	} else if (arg.substring(0, 13) === '--target_plat') {
+		platform = arg.substring(14);
+    force = true; // No point trying to run a binary that isn't the host's platform
+    environment = patchEnvironment(environment);
+    return false;
 	} else if (arg === '--debug') {
 		debug = true;
-	}
+	} else if (arg.substring(0, 12) === '--patch_path') {
+    patchLibPath = arg.substring(13);
+
+    if (!environment.hasOwnProperty("PATCH_TOOL")) {
+      console.error("--patch_path was requested but PATCH_TOOL env was not set; aborting");
+      console.error("be sure to set --target_plat first if cross-compiling as it will set PATCH_TOOL for you");
+      process.exit(1);
+    }
+
+    return false;
+  }
+
 	return true;
 });
+
+// If we are cross-compiling for darwin, we must instruct gyp to generate a makefile which is
+// compatible with the MacOS tooling. YOU MUST use the -f variant (NOT --format=) because it
+// overrides -f set by node-gyp, and node-gyp isn't clever enough to know that they're aliases.
+var configure_args = (platform != process.platform && platform == "darwin")
+                   ? ["-f", "make-mac"]
+                   : [];
+
 if (!{ia32: true, x64: true, arm: true}.hasOwnProperty(arch)) {
 	console.error('Unsupported (?) architecture: `'+ arch+ '`');
 	process.exit(1);
 }
 
 // Test for pre-built library
 var modPath = platform+ '-'+ arch+ '-v8-'+ v8;
+var command = process.platform === 'win32' ? 'node-gyp.cmd' : 'node-gyp';
+
 if (!force) {
 	try {
 		fs.statSync(path.join(__dirname, 'bin', modPath, 'capnp.node'));
@@ -70,36 +101,46 @@ if (!force) {
 	build();
 }
 
-// Build it
-function build() {
+function nodeGyp(task, arguments = [], gyp_args = []) {
+  var final_args = [task].concat(arguments, "--", gyp_args);
+  console.log(`Executing ${command} with the following args: `);
+  console.log(final_args);
+
 	var sp = cp.spawn(
-		process.platform === 'win32' ? 'node-gyp.cmd' : 'node-gyp',
-		['rebuild'].concat(args),
-		{customFds: [0, 1, 2]});
+    command,
+    final_args,
+		{
+      stdio: 'inherit',
+      env: environment,
+    });
 
-	sp
-	.on('close', function(){ afterBuild(); })
-	.on('exit', function(err) {
+	sp.on('exit', function(err) {
 		if (err) {
 			if (err === 127) {
 				console.error(
 					'node-gyp not found! Please upgrade your install of npm! You need at least 1.1.5 (I think) '+
 					'and preferably 1.1.30.'
 				);
 			} else {
-				console.error('Build failed');
+				console.error(`Build step ${task} failed with exit code ${err}`);
 			}
 			return process.exit(err);
 		}
 	});
 
-  if (sp.stdout) {
-    sp.stdout.pipe(process.stdout);
-  }
+  return sp;
+}
 
-  if (sp.stderr) {
-    sp.stderr.pipe(process.stderr);
-  }
+// Build it
+function build() {
+  console.log("Building with the following environment:");
+  console.log(environment);
+
+  nodeGyp("clean").on('close', () => {
+    nodeGyp("configure", args, configure_args).on('close', () => {
+      nodeGyp("build", args).on('close', () => afterBuild());
+    });
+  });
 }
 
 // Move it to expected location
@@ -120,6 +161,10 @@ function afterBuild() {
 		process.exit(1);
 	}
 
+  if (patchLibPath !== false) {
+    patchLibs(environment.PATCH_TOOL, patchLibPath, targetPath);
+  }
+
   try {
     fs.renameSync(targetPath, installPath);
     console.log('Installed in `'+ installPath+ '`');
@@ -128,3 +173,73 @@ function afterBuild() {
   }
 }
 
+/**
+ * Rewrite the dynamic library references in the produced bundle to have appropriate paths for
+ * relative layout within the final application bundle.
+ *
+ * @param patchTool - command name to use for editing the binary
+ * @param patchPath - the path to be stripped out of the binary
+ * @param target - The target binary to be modified
+ */
+function patchLibs(patchTool, patchPath, target) {
+  var libVersion = "0.8.0";
+  var libExt = "so";
+  var libs = [
+        "libkj",
+        "libkj-async",
+        "libcapnp",
+        "libcapnpc",
+        "libcapnp-rpc",
+  ];
+
+  var patchArgs = libs.flatMap((lib) => {
+    return ["-change", `${patchPath}/${lib}-${libVersion}.${libExt}`, `@executable_path/${lib}-${libVersion}.dylib`];
+  }).concat([target]);
+
+  console.log(`Invoking binary patch tool ${patchTool} with the following arguments:`);
+  console.log(patchArgs);
+
+  /*
+   * In future if this errors, we may need to set headerpad_max_install_names in LDFLAGS too...
+   * (this provides more space for the dylib names in the object so that they can be rewritten
+   * post-link) - looks like we don't need to do this for now though?
+   */
+  var patchProcess = cp.spawnSync(patchTool, patchArgs, {
+    stdio: 'inherit',
+  });
+
+  if (patchProcess.error) {
+    console.error(`Failed to patch binary: ${patchProcess.error}`);
+    process.exit(1);
+  } else if (patchProcess.status !== 0) {
+    console.error(`Failed to patch binary: ${patchTool} exited with code ${patchProcess.status}`);
+    process.exit(1);
+  } else {
+    console.log("Binary patched successfully.");
+  }
+}
+
+/**
+ * Modify the provided environment object to be appropriate for cross-compilation for a darwin
+ * target from a linux host
+ */
+function patchEnvironment(env) {
+  if (!env.hasOwnProperty("CAPNP_LIBDIR")) {
+    console.error("CAPNP_LIBDIR must be set to a place I can find compiled libcapnp, libkj and friends for the target plat/arch");
+    process.exit(1);
+  }
+
+  if (!env.hasOwnProperty("CAPNP_INCDIR")) {
+    console.error("CAPNP_INCDIR must be set to a place I can find headers for libcapnp and libkj!");
+    process.exit(1);
+  }
+
+  return {
+    CC: "o64-clang",
+    CXX: "o64-clang++",
+    CXXFLAGS: "-mmacosx-version-min=10.7 -std=c++14 -stdlib=libc++ -I" + env.CAPNP_INCDIR,
+    LDFLAGS: "-L" + env.CAPNP_LIBDIR,
+    PATCH_TOOL: "x86_64-apple-darwin20.2-install_name_tool",
+    ...env
+  };
+}

package.json

@@ -22,7 +22,8 @@
   "types": "src/node-capnp/capnp.d.ts",
   "scripts": {
     "install": "node ./build.js --dist-url=https://electronjs.org/headers --target=12.0.2",
-    "test": "node src/node-capnp/capnp-test.js"
+    "test": "node src/node-capnp/capnp-test.js",
+    "build:darwin": "npm run install -- --target_plat=darwin --patch_path=.libs"
   },
   "dependencies": {
     "nan": "^2.7.0"

src/node-capnp/capnp.js

@@ -46,7 +46,9 @@ if (false) {
   }
 }
 
-var v8capnp = require("../../bin/linux-x64-v8-8.4/capnp.node");
+var v8capnp = process.platform == "darwin"
+            ? require("../../bin/darwin-x64-v8.8.4/capnp.node")
+            : require("../../bin/linux-x64-v8-8.4/capnp.node");
 
 var importPath = [];
 for (var i in module.paths) {