[erts] Make it possible to use the systems libzstd

Author: rickard-green

.github/scripts/otp-compliance.es

@@ -1297,6 +1297,8 @@ create_vendor_relations(NewVendorPackages, #{~"packages" := Packages, ~"relation
                                   case ID of
                                       ~"SPDXRef-otp-erts-zlib" ->
                                           create_spdx_relation('OPTIONAL_COMPONENT_OF', ID, RootId);
+                                      ~"SPDXRef-otp-erts-zstd" ->
+                                          create_spdx_relation('OPTIONAL_COMPONENT_OF', ID, RootId);
                                       ~"SPDXRef-otp-erts-asmjit" ->
                                           create_spdx_relation('OPTIONAL_COMPONENT_OF', ID, RootId);
                                       ~"SPDXRef-otp-erts-tcl" ->

erts/config.h.in

@@ -84,6 +84,9 @@
 /* Define if builtin zlib should be used */
 #undef ERTS_USE_BUILTIN_ZLIB
 
+/* Define if builtin zstd should be used */
+#undef ERTS_USE_BUILTIN_ZSTD
+
 /* Define to 1 if alternatives to embedded 3pps should be used, 0 if not, and
    -1 to use the default */
 #undef ERTS_USE_EMBEDDED_3PP_ALTS

erts/configure

@@ -670,6 +670,7 @@ LIBSCTP
 SYSTEMD_DAEMON_LIBS
 SOCKET_LIBS
 Z_LIB
+LIBZSTD
 ERTS_USE_BUILTIN_ERRNO_ID
 TERMCAP_LIB
 THR_DEFS
@@ -855,6 +856,7 @@ enable_ppc_lwsync_instruction
 with_threadnames
 enable_use_embedded_3pp_alternatives
 enable_builtin_errno_id
+enable_builtin_zstd
 enable_builtin_zlib
 enable_esock
 enable_esock_use_rcvsndtimeo
@@ -1610,6 +1612,8 @@ Optional Features:
   --disable-builtin-errno-id
                           try to find an alternative on the system to use for
                           errno-id
+  --enable-builtin-zstd   force use of our own built-in zstd
+  --disable-builtin-zstd  try to find a static libzstd on the system to use
   --enable-builtin-zlib   force use of our own built-in zlib
   --disable-builtin-zlib  try to find a zlib on the system to use
   --enable-esock          enable builtin socket (as a nif) support (default)
@@ -16667,12 +16671,14 @@ then :
   enableval=$enable_use_embedded_3pp_alternatives;  case "$enableval" in
     no)
       enable_builtin_errno_id=yes
+      enable_builtin_zstd=yes
       enable_builtin_zlib=yes
       use_embedded_3pp_alts=no
       use_embedded_3pp_alts_val=0
       ;;
     *)
       enable_builtin_errno_id=no
+      enable_builtin_zstd=no
       enable_builtin_zlib=no
       use_embedded_3pp_alts=yes
       use_embedded_3pp_alts_val=1
@@ -16682,6 +16688,7 @@ then :
 else case e in #(
   e)
   enable_builtin_errno_id=yes
+  enable_builtin_zstd=yes
   enable_builtin_zlib=no
   use_embedded_3pp_alts=default
   use_embedded_3pp_alts_val=-1
@@ -16731,6 +16738,82 @@ ERTS_USE_BUILTIN_ERRNO_ID=$enable_builtin_errno_id
 
 
 
+# Check whether --enable-builtin-zstd was given.
+if test ${enable_builtin_zstd+y}
+then :
+  enableval=$enable_builtin_zstd;  case "$enableval" in
+      no) enable_builtin_zstd=no ;;
+      *)  enable_builtin_zstd=yes ;;
+    esac
+fi
+
+
+LIBZSTD=
+
+if test "$enable_builtin_zstd" = "no"
+then :
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for static libzstd of version 1.5.6 or higher" >&5
+printf %s "checking for static libzstd of version 1.5.6 or higher... " >&6; }
+  zstd_save_LIBS=$LIBS
+  LIBS="-l:libzstd.a $LIBS"
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h>
+#include <zdict.h>
+
+int
+main (void)
+{
+
+#if ZSTD_VERSION_NUMBER >= 10506
+    (void) ZSTD_createCCtx_advanced(ZSTD_defaultCMem);
+#else
+#error "No libzstd of version 1.5.6 or higher found"
+error
+#endif
+
+  ;
+  return 0;
+}
+
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+
+ LIBZSTD="-l:libzstd.a"
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+
+else case e in #(
+  e)
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no, using builtin zstd instead" >&5
+printf "%s\n" "no, using builtin zstd instead" >&6; }
+ ;;
+esac
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+
+LIBS="$zstd_save_LIBS"
+
+
+fi
+
+
+
+if test "$LIBZSTD" = ""
+then :
+
+
+printf "%s\n" "#define ERTS_USE_BUILTIN_ZSTD 1" >>confdefs.h
+
+
+fi
+
+
 # Check whether --enable-builtin-zlib was given.
 if test ${enable_builtin_zlib+y}
 then :

erts/configure.ac

@@ -1259,12 +1259,14 @@ AS_HELP_STRING([--disable-use-embedded-3pp-alternatives],
 [ case "$enableval" in
     no)
       enable_builtin_errno_id=yes
+      enable_builtin_zstd=yes
       enable_builtin_zlib=yes
       use_embedded_3pp_alts=no
       use_embedded_3pp_alts_val=0
       ;;
     *)
       enable_builtin_errno_id=no
+      enable_builtin_zstd=no
       enable_builtin_zlib=no
       use_embedded_3pp_alts=yes
       use_embedded_3pp_alts_val=1
@@ -1273,6 +1275,7 @@ AS_HELP_STRING([--disable-use-embedded-3pp-alternatives],
 ],
 [
   enable_builtin_errno_id=yes
+  enable_builtin_zstd=yes
   enable_builtin_zlib=no
   use_embedded_3pp_alts=default
   use_embedded_3pp_alts_val=-1
@@ -1308,6 +1311,59 @@ AS_IF([test $enable_builtin_errno_id = yes],
 ERTS_USE_BUILTIN_ERRNO_ID=$enable_builtin_errno_id
 AC_SUBST(ERTS_USE_BUILTIN_ERRNO_ID)
 
+dnl -------------
+dnl zstd
+dnl -------------
+
+AC_ARG_ENABLE(builtin-zstd,
+AS_HELP_STRING([--enable-builtin-zstd],
+               [force use of our own built-in zstd])
+AS_HELP_STRING([--disable-builtin-zstd],
+               [try to find a static libzstd on the system to use]),
+  [ case "$enableval" in
+      no) enable_builtin_zstd=no ;;
+      *)  enable_builtin_zstd=yes ;;
+    esac ])
+
+LIBZSTD=
+
+AS_IF([test "$enable_builtin_zstd" = "no"],
+ [
+  AC_MSG_CHECKING(for static libzstd of version 1.5.6 or higher)
+  zstd_save_LIBS=$LIBS
+  LIBS="-l:libzstd.a $LIBS"
+  AC_LINK_IFELSE(
+     [AC_LANG_PROGRAM([[
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h>
+#include <zdict.h>
+]],[[
+#if ZSTD_VERSION_NUMBER >= 10506
+    (void) ZSTD_createCCtx_advanced(ZSTD_defaultCMem);
+#else
+#error "No libzstd of version 1.5.6 or higher found"
+error
+#endif
+]])
+],
+[
+ LIBZSTD="-l:libzstd.a"
+ AC_MSG_RESULT(yes)
+],[
+ AC_MSG_RESULT([no, using builtin zstd instead])
+])
+
+LIBS="$zstd_save_LIBS"
+
+ ])
+
+AC_SUBST(LIBZSTD)
+
+AS_IF([test "$LIBZSTD" = ""],
+      [
+        AC_DEFINE(ERTS_USE_BUILTIN_ZSTD, [1], [Define if builtin zstd should be used])
+      ])
+
 dnl -------------
 dnl zlib
 dnl -------------

erts/emulator/Makefile.in

@@ -32,6 +32,7 @@ DTRACE_ENABLED=@DTRACE_ENABLED@
 DTRACE_ENABLED_2STEP=@DTRACE_ENABLED_2STEP@
 USE_VM_PROBES=@USE_VM_PROBES@
 LIBS = @LIBS@
+LIBZSTD=@LIBZSTD@
 Z_LIB=@Z_LIB@
 CROSS_COMPILING = @CROSS_COMPILING@
 NO_INLINE_FUNCTIONS=false
@@ -398,6 +399,10 @@ else
 LIBS += $(OPENSSL_LIB)
 endif
 
+ifdef LIBZSTD
+# Use system libzstd
+LIBS += $(LIBZSTD)
+else
 ZSTD_DIR = $(ERL_TOP)/erts/emulator/zstd/obj/$(TARGET)/$(TYPE)
 ZSTD_LIB = $(ZSTD_DIR)/$(LIB_PREFIX)zstd$(LIB_SUFFIX)
 DEPLIBS += $(ZSTD_LIB)
@@ -408,6 +413,7 @@ else
 # Build on darwin fails if -l$(ZSTD_LIB) is used
 LIBS    += $(ZSTD_LIB)
 endif
+endif
 
 LIBSCTP = @LIBSCTP@
 
@@ -842,7 +848,10 @@ COMMON_INCLUDES = -Ibeam -Isys/$(ERLANG_OSTYPE) -Isys/common -I$(TARGET)
 ifndef Z_LIB
 COMMON_INCLUDES += -Izlib 
 endif
-COMMON_INCLUDES += -Ipcre -Iryu -Iopenssl/include -Izstd
+ifndef LIBZSTD
+COMMON_INCLUDES += -Izstd
+endif
+COMMON_INCLUDES += -Ipcre -Iryu -Iopenssl/include
 COMMON_INCLUDES += -I../include -I../include/$(TARGET)
 COMMON_INCLUDES += -I../include/internal -I../include/internal/$(TARGET)
 

erts/emulator/beam/erl_bif_info.c

@@ -3589,7 +3589,11 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
         xtra -= 2;
 
         hp = erts_produce_heap(&hfact, 2, xtra);
+#ifdef ERTS_USE_BUILTIN_ZSTD
         included = CONS(hp, AM_zstd, included);
+#else
+        excluded = CONS(hp, AM_zstd, excluded);
+#endif
         xtra -= 2;
 
         hp = erts_produce_heap(&hfact, 2, xtra);

erts/emulator/nifs/common/zstd_nif.c

@@ -28,9 +28,16 @@
 #define STATIC_ERLANG_NIF 1
 
 #include "erl_nif.h"
+#include "config.h"
 #define ZSTD_STATIC_LINKING_ONLY
-#include "erl_zstd.h"
-#include "erl_zdict.h"
+
+#ifdef ERTS_USE_BUILTIN_ZSTD
+#  include "erl_zstd.h"
+#  include "erl_zdict.h"
+#else
+#  include <zstd.h>
+#  include <zdict.h>
+#endif
 
 static ErlNifResourceType *compress_type;
 static ErlNifResourceType *decompress_type;