erts: Allow erlang:load_nif/2 to load a NIF from an in-memory binary

Add support for passing `#{memory => NifSO::binary(),
label => string():binary()}` as the first argument to erlang:load_nif/2.
When a tuple is given, the NIF shared object is loaded directly from
the binary image in memory instead of from the file system.

This addition is needed so that escripts can package SO dependencies
inside and be distributed as a single executable.

Implementation
--------------
* erts_dlopen_mem() — new static helper in erl_nif.c (Linux/POSIX only).
  On Linux >= 3.17 it uses memfd_create(2) to create an anonymous
  in-memory file, writes the SO image into it, and calls dlopen(3) via
  /proc/self/fd/<fd>.  On older Linux kernels and other POSIX systems it
  falls back to shm_open(3) + /dev/shm/<name>.  The implementation is
  derived from the https://github.com/saleyn/memfd_create proof of
  concept.

* erts_load_nif() — extended argument parsing:
  - Plain filename() continues to work unchanged.
  - #{memory => Binary, label => Filename} map: Binary is extracted via
    erts_get_aligned_binary_bytes(), passed to erts_dlopen_mem(), and
    the resulting dlopen handle is used directly.  The Filename string
    serves only as a label inside the memory-backed file descriptor.
  - The open-from-memory step is performed before the existing else-if
    validation chain (version checks, module-name check, create_lib)
    so both code paths share the same validation and bookkeeping.
  - erts_sys_ddll_open() is skipped (via !load_from_mem guard) when a
    handle was already obtained from memory.

* The feature is guarded by #if defined(HAVE_DLOPEN) && defined(__unix__).
  On unsupported platforms load_nif/2 returns {error,{load_failed,…}}.

Testing
-------
* nif_SUITE: new test case load_nif_from_mem
  - Happy path: reads nif_mod.1.so into a binary, calls
    erlang:load_nif({Path, Binary}, []) via nif_mod:load_nif_lib_from_mem/3,
    and asserts lib_version() == 1.
  - Error path: non-binary second element returns {error,{bad_lib,_}}.

* nif_mod.erl: new exported helper load_nif_lib_from_mem/3 so that the
  load_nif call site lives inside the NIF module (required by the BIF).

Author: saleyn

.gitignore

@@ -213,6 +213,7 @@ JAVADOC-GENERATED
 /erts/emulator/test/*_native_SUITE.erl
 /erts/emulator/test/*_SUITE_data/Makefile
 /erts/emulator/test/*_stripped_types_SUITE.erl
+/erts/emulator/test/nif_SUITE_data/*.so
 /erts/test/install_SUITE_data/install_bin
 /erts/test/autoimport_SUITE_data/erlang.xml
 /erts/emulator/make_test_dir/

erts/emulator/beam/erl_nif.c

@@ -44,7 +44,6 @@
 #endif
 
 #include "erl_nif.h"
-
 #include "sys.h"
 #include "global.h"
 #include "erl_binary.h"
@@ -74,6 +73,154 @@
 #include <limits.h>
 #include <stddef.h> /* offsetof */
 
+#undef SHMOPEN_USE_LINUX_MEMFD
+#undef SHMOPEN_USE_SHM_ANON
+#undef SHMOPEN_USE_SHM_MKSTEMP
+#undef SHMOPEN_USE_POSIX
+#undef USE_ERTS_DLOPEN_MEM
+
+#ifdef HAVE_DLFCN_H
+#include <dlfcn.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#if defined(__linux__)
+#  if defined(__linux__)
+#    include <sys/syscall.h>
+#  endif
+#  include <sys/utsname.h>
+#  ifdef SYS_memfd_create
+#    define SHMOPEN_USE_LINUX_MEMFD
+#  else
+#    define SHMOPEN_USE_SHM_POSIX
+#  endif
+#  define USE_ERTS_DLOPEN_MEM
+#elif defined(__FreeBSD__)
+#  define SHMOPEN_USE_SHM_ANON
+#  define USE_ERTS_DLOPEN_MEM
+#elif defined(__OpenBSD__)
+#  define SHMOPEN_USE_SHM_MKSTEMP
+#  define USE_ERTS_DLOPEN_MEM
+#else
+#  if defined(__APPLE__)  || defined(__MACH__)      || defined(__DARWIN__) || \
+      defined(__NetBSD__) || defined(__DragonFly__) || defined(__HAIKU__)  || \
+      defined(__sun)
+#    define SHMOPEN_USE_POSIX
+#    define USE_ERTS_DLOPEN_MEM
+#  endif
+#endif
+
+#define ERTS_NIF_MEM_NAME_MAX 64
+#define ERTS_MIN_KERNEL_VSN   317
+
+#ifdef USE_ERTS_DLOPEN_MEM
+/* Always declare erts_dlopen_mem for all platforms */
+static void *erts_dlopen_mem(const char *filename, const void *mem, size_t size);
+
+static char* erts_shm_name(const char *pfx, const char *sfx, char* buf, size_t buf_len)
+{
+    /* Generate a unique name for the shared memory object. */
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    enif_snprintf(buf, buf_len, "%s-%-06ld%s", pfx, (long)tv.tv_usec, sfx);
+    return buf;
+}
+
+/*
+ * Load a shared object from memory using memfd_create (Linux >= 3.17) or
+ * shm_open (other POSIX systems).  Returns a dlopen(3) handle on success,
+ * or NULL on failure.
+ *
+ * filename - a label associated with the in-memory file (may be NULL)
+ * mem      - pointer to the SO image in memory
+ * size     - byte length of the SO image
+ */
+static void *erts_dlopen_mem(const char *filename, const void *mem, size_t size)
+{
+    char path[PATH_MAX];
+    char shm_name[NAME_MAX];
+    int  shm_fd = -1;
+    void *handle = NULL;
+#if defined(SHMOPEN_USE_SHM_MKSTEMP) || defined(SHMOPEN_USE_POSIX)
+    int  need_unlink = 0;
+#endif
+
+    path[0]     = '\0';
+    shm_name[0] = '\0';
+
+#if defined(SHMOPEN_USE_LINUX_MEMFD)
+    if (!filename)
+        filename = erts_shm_name("erl-nif-mem-", ".so", shm_name, sizeof(shm_name));
+    shm_fd = memfd_create(filename, MFD_CLOEXEC);
+    if (shm_fd < 0)
+        return NULL;
+    enif_snprintf(path, sizeof(path), "/proc/self/fd/%d", shm_fd);
+
+#elif defined(SHMOPEN_USE_SHM_ANON)
+    if (filename)
+        return NULL;
+    shm_fd = shm_open(SHM_ANON, O_RDWR, 0);
+    if (shm_fd < 0)
+        return NULL;
+# if defined(__APPLE__)
+    if (fcntl(shm_fd, F_GETPATH, path) < 0)
+        goto err;
+# else
+    enif_snprintf(path, sizeof(path), "/dev/fd/%d", shm_fd);
+# endif
+
+#elif defined(SHMOPEN_USE_SHM_MKSTEMP)
+    enif_snprintf(path, sizeof(path), "/tmp/%s-XXXXXX", filename ? filename : "enif-shm");
+    shm_fd = shm_mkstemp(path);
+    if (shm_fd < 0)
+        return NULL;
+    need_unlink = 1;
+
+#else /* SHMOPEN_USE_POSIX */
+    if (filename) {
+        enif_snprintf(shm_name, sizeof(shm_name), "%s", filename);
+    } else {
+        erts_shm_name("enif-shm-", ".so", shm_name, sizeof(shm_name));
+    }
+    enif_snprintf(path, sizeof(path), "/dev/shm/%s", shm_name);
+    shm_fd = shm_open(path, O_RDWR | O_CREAT | O_EXCL, 0600);
+    if (shm_fd < 0)
+        return NULL;
+    need_unlink = 1;
+#endif
+
+    if (ftruncate(shm_fd, size) || write(shm_fd, mem, size) != (ssize_t)size)
+        goto err;
+
+    handle = dlopen(path, RTLD_NOW | RTLD_LOCAL);
+
+err:
+# if defined(SHMOPEN_USE_SHM_MKSTEMP)
+    if (need_unlink)
+        unlink(path);
+# elif defined(SHMOPEN_USE_POSIX)
+    if (need_unlink)
+        shm_unlink(path);
+# endif
+    if (shm_fd >= 0)
+        close(shm_fd);
+    return handle;
+}
+#endif /* USE_ERTS_DLOPEN_MEM */
+
 #define ERTS_NIF_HALT_INFO_FLAG_BLOCK               (1 << 0)
 #define ERTS_NIF_HALT_INFO_FLAG_HALTING             (1 << 1)
 #define ERTS_NIF_HALT_INFO_FLAG_WAITING             (1 << 2)
@@ -4721,6 +4868,8 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args)
     Eterm ret = am_ok;
     int veto;
     int is_static = 0;
+    int load_from_mem = 0;       /* non-zero when loading from in-memory binary */
+    Eterm nif_binary = THE_NON_VALUE;
     struct erl_module_nif* lib = NULL;
     struct erl_module_instance* this_mi;
     struct erl_module_instance* prev_mi;
@@ -4731,12 +4880,41 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args)
         /* since lib_name is used in error messages */
         encoding = ERL_FILENAME_UTF8;
     }
+
+    /*
+     * Accept either:
+     *   Filename::string()|binary()              -- load from file (existing behaviour)
+     *   #{memory := NifSO}                       -- load from memory, auto-generated label
+     *   #{memory := NifSO, label := Filename}    -- load from memory with explicit label
+     */
+    if (is_map(filename)) {
+        const Eterm *mem_val = erts_maps_get(am_memory, filename);
+        const Eterm *lbl_val = erts_maps_get(am_label, filename);
+        if (mem_val == NULL) {
+            return load_nif_error(c_p, "bad_lib",
+                "load_nif/2: map argument must contain a 'memory' key");
+        }
+        if (!is_bitstring(*mem_val) || TAIL_BITS(bitstring_size(*mem_val)) != 0) {
+            return load_nif_error(c_p, "bad_lib",
+                "load_nif/2: 'memory' value must be a binary");
+        }
+        nif_binary    = *mem_val;
+        load_from_mem = 1;
+        if (lbl_val != NULL) {
+            filename = *lbl_val;   /* fall through to filename decoding for label */
+        } else {
+            lib_name = NULL;   /* anonymous: erts_dlopen_mem will generate a name */
+            goto after_lib_name;
+        }
+    }
+
     lib_name = erts_convert_filename_to_encoding(filename, NULL, 0,
                                                  ERTS_ALC_T_TMP, 1, 0, encoding,
-						 NULL, 0);
+                                                 NULL, 0);
     if (!lib_name) {
         return THE_NON_VALUE;
     }
+after_lib_name:;
 
     /* Find calling module */
     caller = erts_find_function_from_pc(I);
@@ -4769,11 +4947,52 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args)
 	this_mi = module_p->on_load;
     }
 
+    /* If the caller passed #{memory => NifSO::binary(), label => Filename},
+     * open the shared object from memory now so that `handle` is ready for
+     * the common else-if chain below (which runs
+     * erts_sys_ddll_load_nif_init / call_nif_init, version checks, and
+     * create_lib for both file and memory loads). */
+    if (!is_static && load_from_mem) {
+#if defined(USE_ERTS_DLOPEN_MEM)
+        const byte *bin_bytes = NULL;
+        Uint  bin_size  = 0;
+        const byte *tmp_alloc = NULL;
+        static const int max_path_len = PATH_MAX - 9; /* space for "/dev/shm/" */
+        if (lib_name != NULL) {
+            int lib_name_len = sys_strlen(lib_name);
+            if (lib_name_len > max_path_len - 1) { /* -1 for null terminator added */
+                ret = load_nif_error(c_p, "load_failed",
+                    "NIF library path too long: %d (max %d bytes)",
+                    lib_name_len, max_path_len);
+                goto error;
+            }
+        }
+        bin_bytes = erts_get_aligned_binary_bytes(nif_binary, &bin_size, &tmp_alloc);
+        if (!bin_bytes) {
+            ret = load_nif_error(c_p, "load_failed",
+                "Failed to access NIF binary data");
+            goto error;
+        }
+        handle = erts_dlopen_mem(lib_name, bin_bytes, (size_t)bin_size);
+        erts_free_aligned_binary_bytes(tmp_alloc);
+        if (!handle) {
+            ret = load_nif_error(c_p, "load_failed",
+                "Failed to load NIF library from memory: '%s'",
+                dlerror());
+            goto error;
+        }
+#else
+        ret = load_nif_error(c_p, "load_failed",
+            "Loading NIF from memory is not supported on this platform");
+        goto error;
+#endif
+    }
+
     if (this_mi->nif != NULL) {
         ret = load_nif_error(c_p,"reload","NIF library already loaded"
                              " (reload disallowed since OTP 20).");
     }
-    else if (!is_static &&
+    else if (!is_static && !load_from_mem &&
              (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) {
 	const char slogan[] = "Failed to load NIF library";
 	if (strstr(errdesc.str, lib_name) != NULL) {
@@ -5031,7 +5250,8 @@ Eterm erts_load_nif(Process *c_p, ErtsCodePtr I, Eterm filename, Eterm args)
 	erts_sys_ddll_free_error(&errdesc);
     }
 
-    erts_free(ERTS_ALC_T_TMP, lib_name);
+    if (lib_name)
+        erts_free(ERTS_ALC_T_TMP, lib_name);
 
     BIF_RET(ret);
 }

erts/emulator/test/nif_SUITE.erl

@@ -34,6 +34,7 @@
          init_per_group/2, end_per_group/2,
 	 init_per_testcase/2, end_per_testcase/2,
          basic/1, reload_error/1, upgrade/1, heap_frag/1,
+         load_nif_from_mem/1,
          t_on_load/1,
          t_nifs_attrib/1,
          t_load_race/1,
@@ -220,7 +221,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() ->
     [basic,
-     non_exported_nif]
+     non_exported_nif,
+     load_nif_from_mem]
         ++
     [{group, G} || G <- api_groups()]
         ++
@@ -357,6 +359,64 @@ non_exported_nif(Config) when is_list(Config) ->
     false = lists:member({lib_version,0}, ?MODULE:module_info(exports)),
     ok.
 
+
+%% Test loading a NIF library from an in-memory binary image via
+%% erlang:load_nif(#{memory := Binary, ...}, LoadInfo).
+load_nif_from_mem(Config) when is_list(Config) ->
+    case os:type() of
+        {unix, _} -> run_load_nif_from_mem(Config);
+        _ ->
+            %% Should fail with load_failed on unsupported platforms
+            {error, {load_failed, _}} =
+                erlang:load_nif(#{memory => <<1,2,3,4>>}, []),
+            ok
+    end.
+
+run_load_nif_from_mem(Config) ->
+    Data = proplists:get_value(data_dir, Config),
+    ModFile = filename:join(Data, "nif_mod"),
+    {ok, nif_mod, ModBin} = compile:file(ModFile, [binary, return_errors]),
+    {module, nif_mod} = erlang:load_module(nif_mod, ModBin),
+
+    %% Derive the .so path the same way nif_mod:load_nif_lib/2 does,
+    %% but with the .so extension appended explicitly for file:read_file/1.
+    SoPath = filename:join(Data, "nif_mod.1.so"),
+    {ok, SoBin} = file:read_file(SoPath),
+
+    %% --- happy path: map with label ---
+    ok = nif_mod:load_nif_lib_from_mem(Config, 1, SoBin),
+    1 = nif_mod:lib_version(),
+
+    %% cleanup: delete + purge so the module can be reloaded for error tests
+    true = erlang:delete_module(nif_mod),
+    true = erlang:purge_module(nif_mod),
+    receive unloaded -> ok after 1000 -> ok end,
+
+    %% --- error: 'memory' value is not a binary ---
+    {module, nif_mod} = erlang:load_module(nif_mod, ModBin),
+    {error, {bad_lib, _}} = nif_mod:load_nif_lib_from_mem(Config, 1, not_a_binary),
+    true = erlang:delete_module(nif_mod),
+    true = erlang:purge_module(nif_mod),
+    receive unloaded -> ok after 1000 -> ok end,
+
+    %% --- error: label longer than PATH_MAX-9 ---
+    {module, nif_mod} = erlang:load_module(nif_mod, ModBin),
+    LongPath = lists:duplicate(4096-9, $x),
+    {error, {load_failed, _}} = nif_mod:load_nif_mem_path(LongPath, SoBin),
+    true = erlang:delete_module(nif_mod),
+    true = erlang:purge_module(nif_mod),
+    receive unloaded -> ok after 1000 -> ok end,
+
+    %% --- anonymous load: map with only 'memory' key ---
+    {module, nif_mod} = erlang:load_module(nif_mod, ModBin),
+    ok = nif_mod:load_nif_mem_anon(SoBin),
+    1 = nif_mod:lib_version(),
+    true = erlang:delete_module(nif_mod),
+    true = erlang:purge_module(nif_mod),
+    receive unloaded -> ok after 1000 -> ok end,
+
+    ok.
+
 %% Test old reload feature now always fails
 reload_error(Config) when is_list(Config) ->
     TmpMem = tmpmem(),