declarative natives build script, MSVC for Windows targets + FreeBSD x86 & Windows ARM natives (#231)

the MSVC targets are a much better option than gnullvm for windows ARM
(tier 3 toolchain vs tier 1).

not sure if there's a nice approach to sticking everything we need in
one container but with caching the duplication isnt too bad.
also not sure if the dev commands should be changed to call local cargo
directly, since anyone working on the natives will probably be doing
that anyway.

* *actually* ensures non-`x86_64-apple-darwin` release builds get
`codegen-units = 1`, after moving to a workspace, since workspaces
ignore profile overrides in member crates' Cargo.toml.
* native build script now declaratively generates tasks from a single
list of targets, which also handles copying/etc as opposed to the
previous ad-hoc handwritten tasks
* the natives are now LZMA compressed and decompressed at load time
* should fix #40, though loader improvements may be needed to detect
freebsd and avoid loading the linux natives.
* fixes #31 
* might fix #206 (by changing how the natives file is named)

---------

Co-authored-by: Random-Scientist <Random-Scientist@users.noreply.github.com>
Co-authored-by: mcrcortex <18544518+MCRcortex@users.noreply.github.com>
Co-authored-by: Ocelot <FinntheRaider@gmail.com>

Author: 3 people

.github/workflows/build-rust.yml

@@ -37,7 +37,7 @@ jobs:
           key: ${{ runner.os }}-node-${{ hashFiles('common/src/main/rust/**/Cargo.lock') }}
 
       - name: Run Gradle Task
-        run: ./gradlew common:buildImage common:buildRustNatives
+        run: ./gradlew common:buildImages common:buildRustNatives
 
       - name: Take ownership of the workspace
         run: sudo chown -R $USER:$USER .

README.md

@@ -27,7 +27,7 @@ View the [Sable Developer Wiki](https://github.com/ryanhcode/sable/wiki) for doc
 # Building Rust Natives
 
 1. Install Docker from https://www.docker.com/get-started/ or from your relevant package manager
-2. Run `gradlew common:buildImage` (only has to be done once)
+2. Run `gradlew common:buildImages` (only has to be done once)
 3. Run `gradlew common:buildRustNatives`
 
 ### Thanks

common/build.gradle

@@ -1,16 +1,53 @@
 import org.apache.tools.ant.taskdefs.condition.Os
 
+import org.tukaani.xz.LZMA2Options
+import org.tukaani.xz.XZOutputStream
+
+import java.nio.file.Files
+buildscript {
+    dependencies {
+        classpath 'org.tukaani:xz:1.10'
+    }
+}
+
 plugins {
     id 'multiloader-common'
     id 'net.neoforged.moddev'
 }
 
+def mac(arch) {
+    [triple: "${arch}-apple-darwin", arch: arch, os: "macos", ext: "dylib", lib: true]
+}
+def linux(arch) {
+    [triple: "${arch}-unknown-linux-gnu", arch: arch, os: "linux", ext: "so", lib: true]
+}
+def freebsd(arch) {
+    [triple: "${arch}-unknown-freebsd", arch: arch, os: "freebsd", ext: ".so", lib: true]
+}
+def windows(arch) {
+    [triple: "${arch}-pc-windows-msvc", arch: arch, os: "windows", ext: "dll", lib: false]
+}
+
+def supportedTargets = [
+    mac("x86_64"),
+    mac("aarch64"),
+    linux("x86_64"),
+    linux("aarch64"),
+    windows("x86_64"),
+    windows("aarch64"),
+    freebsd("x86_64"),
+    // freebsd("aarch64"),
+]
+def pinnedRustVersion = 'nightly-2026-01-29'
 def rustRootDir = file("src/main/rust")
 def rustProjectDir = file("$rustRootDir/rapier")
 def nativesDir = file("src/main/resources/natives/sable_rapier")
 def cargoCacheDir = project.layout.buildDirectory.file("cargo").get().asFile
 def docker = Os.isFamily(Os.FAMILY_MAC) ? "/usr/local/bin/docker" : "docker"
-def dockerCommand = [
+def xWinImage = 'sable-build-xwin'
+def zigbuildImage = 'sable-build-zigbuild'
+
+def baseDockerCommand = [
         docker,
         'run',
         '--rm',
@@ -22,8 +59,9 @@ def dockerCommand = [
         "$cargoCacheDir/registry:/usr/local/cargo/registry",
         '-w',
         '/io/rapier',
-        'sable-build'
 ]
+project.ext.zigbuildCargo = [baseDockerCommand, zigbuildImage, 'cargo'].flatten()
+project.ext.xWinCargo = [baseDockerCommand, xWinImage, 'cargo'].flatten()
 
 tasks.register('createContainersDirectory') {
     doLast {
@@ -33,116 +71,112 @@ tasks.register('createContainersDirectory') {
     }
 }
 
-tasks.register('buildImage', Exec) {
+
+tasks.register('buildZigbuildImage', Exec) {
     group = 'rust'
     workingDir rustRootDir
-    commandLine docker, 'build', '-t', 'sable-build', '.'
+    commandLine docker, 'build', '-t', zigbuildImage, 'container/zigbuild', '--build-arg', "RUST_VERSION=${pinnedRustVersion}"
     dependsOn createContainersDirectory
 }
-
-tasks.register('cleanRust', Exec) {
+tasks.register('buildXWinImage', Exec) {
     group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'clean'
+    workingDir rustRootDir
+    commandLine docker, 'build', '-t', xWinImage, 'container/xwin', '--build-arg', "RUST_VERSION=${pinnedRustVersion}"
     dependsOn createContainersDirectory
 }
 
-tasks.register('compileRust', Exec) {
+tasks.register('buildImages') {
     group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild'
-    dependsOn createContainersDirectory
-    finalizedBy copyRustNativesDev
+    dependsOn buildZigbuildImage, buildXWinImage
 }
 
-tasks.register('compileRustMacAArch64', Exec) {
+tasks.register('cleanRust', Exec) {
     group = 'rust'
     workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'aarch64-apple-darwin'
+    commandLine project.ext.zigbuildCargo // only need to use zigbuild cargo since both containers share the same target folder
+    args 'clean'
     dependsOn createContainersDirectory
 }
 
-tasks.register('compileRustMacX86', Exec) {
-    group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'x86_64-apple-darwin'
-    dependsOn createContainersDirectory
+def compileRustTaskName = { target ->
+    "compileRust-${target.os}-${target.arch}"
 }
 
-tasks.register('compileRustLinuxAArch64', Exec) {
-    group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'aarch64-unknown-linux-gnu.2.17'
-    dependsOn createContainersDirectory
-}
+def commandLineForTarget(target) {
+    if(target.triple.contains('msvc')) {
+        [project.ext.xWinCargo, 'xwin', 'build', '--release', '--target', target.triple].flatten()
+        // support meme platforms, this does nothing by default because the target is disabled
+    } else if(target.triple == 'aarch64-unknown-freebsd') {
+        [project.ext.zigbuildCargo, 'zigbuild', '-Z', 'build-std', '--release', '--target', target.triple].flatten()
+    } else {
+        [project.ext.zigbuildCargo, 'zigbuild', '--release', '--target', target.triple].flatten()
+    }
 
-tasks.register('compileRustLinux', Exec) {
-    group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'x86_64-unknown-linux-gnu.2.17'
-    dependsOn createContainersDirectory
 }
-
-tasks.register('compileRustWindows', Exec) {
-    group = 'rust'
-    workingDir rustProjectDir
-    commandLine dockerCommand
-    args 'cargo', 'zigbuild', '--release', '--target', 'x86_64-pc-windows-gnu'
-    dependsOn createContainersDirectory
+def nativesNameForTarget(target) {
+    "sable_rapier_${target.arch}_${target.os}.${target.ext}"
+}
+supportedTargets.forEach { target ->
+    tasks.register(compileRustTaskName(target), Exec) {
+        group = 'rust'
+        workingDir rustProjectDir
+        description = "Cross-compiles natives for the ${target.triple} target"
+        commandLine = commandLineForTarget(target)
+    }
 }
 
 tasks.register('buildRustNatives') {
     group = 'build'
     description = 'Compiles all Rust natives and moves them to resources.'
 
-    dependsOn compileRustMacAArch64, compileRustMacX86, compileRustLinux, compileRustLinuxAArch64, compileRustWindows
+    dependsOn = supportedTargets.stream().map(compileRustTaskName).collect()
     finalizedBy copyRustNatives
 }
 
 tasks.register('copyRustNatives', Copy) {
     group = 'rust'
-
     into nativesDir
     mustRunAfter(buildRustNatives)
 
-    def moves = [
-            [src: "target/aarch64-apple-darwin/release/libsable_rapier.dylib", dest: "sable_rapier_aarch64_macos.dylib"],
-            [src: "target/x86_64-apple-darwin/release/libsable_rapier.dylib", dest: "sable_rapier_x86_64_macos.dylib"],
-            [src: "target/x86_64-unknown-linux-gnu/release/libsable_rapier.so", dest: "sable_rapier_x86_64_linux.so"],
-            [src: "target/aarch64-unknown-linux-gnu/release/libsable_rapier.so", dest: "sable_rapier_aarch64_linux.so"],
-            [src: "target/x86_64-pc-windows-gnu/release/sable_rapier.dll", dest: "sable_rapier_x86_64_windows.dll"]
-    ]
-
-    moves.forEach { map ->
-        from(file("$rustRootDir/${map.src}")) {
-            rename { map.dest }
+    supportedTargets.forEach { target ->
+        from(file("$rustRootDir/target/${target.triple}/release/${ if(target.lib) {"lib"} else {""}}sable_rapier.${target.ext}")) {
+            rename { nativesNameForTarget(target) }
         }
     }
+    finalizedBy packRustNatives
 }
 
 tasks.register('copyRustNativesDev', Copy) {
     group = 'rust'
+    into nativesDir
 
+    supportedTargets.forEach { target ->
+        from(file("$rustRootDir/target/debug/${ if(target.lib) {"lib"} else {""}}sable_rapier.${target.ext}")) {
+            rename { nativesNameForTarget(target) }
+        }
+    }
+    finalizedBy packRustNatives
+}
+tasks.register('packRustNatives', Tar) {
+    group = 'rust'
+    archiveFile.set file("${nativesDir}/sable_rapier_binaries.tar.xz")
     into nativesDir
-    mustRunAfter(compileRust)
-
-    def moves = [
-            [src: "target/debug/libsable_rapier.dylib", dest: "sable_rapier_aarch64_macos.dylib"],
-            [src: "target/debug/libsable_rapier.dylib", dest: "sable_rapier_x86_64_macos.dylib"],
-            [src: "target/debug/libsable_rapier.so", dest: "sable_rapier_aarch64_linux.so"],
-            [src: "target/debug/libsable_rapier.so", dest: "sable_rapier_x86_64_linux.so"],
-            [src: "target/debug/sable_rapier.dll", dest: "sable_rapier_aarch64_windows.dll"],
-            [src: "target/debug/sable_rapier.dll", dest: "sable_rapier_x86_64_windows.dll"]
-    ]
-    moves.forEach { map ->
-        from(file("$rustRootDir/${map.src}")) {
-            rename { map.dest }
+
+    mustRunAfter copyRustNatives, copyRustNativesDev
+    supportedTargets.forEach { target ->
+        var f = nativesNameForTarget(target);
+        // No, you can't use rename here in case you were wondering.
+        from(file("${nativesDir}/${f}")) { eachFile { setPath f } }
+    }
+    doLast {
+        byte[] bytes = Files.readAllBytes(getArchiveFile().get().asFile.toPath());
+        try (var f = new FileOutputStream(getArchiveFile().get().asFile))  {
+            try (var x = new XZOutputStream(f, new LZMA2Options(LZMA2Options.PRESET_MAX))) {
+                x.write(bytes);
+            }
+        }
+        supportedTargets.forEach { t ->
+            delete(file("${nativesDir}/${nativesNameForTarget(t)}"))
         }
     }
 }

common/src/main/java/dev/ryanhcode/sable/physics/impl/rapier/Rapier3D.java

@@ -9,9 +9,13 @@
 import net.minecraft.Util;
 import net.minecraft.Util.OS;
 import net.minecraft.server.level.ServerLevel;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.jetbrains.annotations.ApiStatus;
 import org.joml.Matrix3dc;
 import org.joml.Vector3dc;
+import org.tukaani.xz.XZInputStream;
 
 import java.io.FileNotFoundException;
 import java.io.InputStream;
@@ -57,19 +61,33 @@ private static String getNativeName() {
 
     private static void loadLibrary() {
         final String nativeName = getNativeName();
-        try (final InputStream is = Rapier3D.class.getResourceAsStream("/natives/" + LIB_NAME + "/" + nativeName)) {
+        try (final InputStream is = Rapier3D.class.getResourceAsStream("/natives/" + LIB_NAME + "/sable_rapier_binaries.tar.xz")) {
             if (is == null) {
-                throw new FileNotFoundException(LIB_NAME);
+                throw new FileNotFoundException("sable_rapier_binaries.tar.xz");
+            }
+            try (final XZInputStream is2 = new XZInputStream(is);
+                 final TarArchiveInputStream ti = new TarArchiveInputStream(is2)) {
+
+                TarArchiveEntry entry;
+                while ((entry = ti.getNextEntry()) != null) {
+                    if (entry.getName().equals(nativeName)) {
+                        final String[] split = nativeName.split("\\.");
+                        final Path tempFile = Files.createTempFile(split[0], "." + split[1]);
+                        Files.copy(ti, tempFile, StandardCopyOption.REPLACE_EXISTING);
+                        System.load(tempFile.toAbsolutePath().toString());
+                        ENABLED = true;
+                        return;
+                    }
+                }
+
+                throw new FileNotFoundException(nativeName);
             }
-
-            final Path tempFile = Files.createTempFile(LIB_TMP_DIR_PREFIX, null);
-            Files.copy(is, tempFile, StandardCopyOption.REPLACE_EXISTING);
-            System.load(tempFile.toAbsolutePath().toString());
-            ENABLED = true;
         } catch (final Throwable t) {
             ENABLED = false;
 
-            Sable.LOGGER.error("Sable has failed to load the natives needed for its Rapier pipeline. Native library name {}. Please report with system details and logs to {}", nativeName, Sable.ISSUE_TRACKER_URL, t);
+            Sable.LOGGER.error(
+                    "Sable has failed to load the natives needed for its Rapier pipeline. Native library name {}. Please report with system details and logs to {}",
+                    nativeName, Sable.ISSUE_TRACKER_URL, t);
         }
     }
 

common/src/main/resources/natives/sable_rapier/sable_rapier_aarch64_linux.so

@@ -14,3 +14,11 @@ jni = "0.21.1"
 colored = "2.1.0"
 fern = { version = "0.6.2", features = ["colored"] }
 humantime = "2.1.0"
+
+
+[profile.release]
+lto = "thin"
+codegen-units = 1
+
+[profile.bench]
+debug = true