Support transport-level compression

.github/workflows/docker/Dockerfile

@@ -11,6 +11,8 @@ RUN apt-get update && \
     ninja-build \
     pkg-config \
     libssl-dev \
+    liblz4-dev \
+    libzstd-dev \
     libz-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*

.github/workflows/docker/build_test_ntf.sh

@@ -41,6 +41,9 @@ build_ntf() {
                 --output /workspace/ntf-core/build \
                 --jobs ${jobs} \
                 --standalone \
+                --with-zlib \
+                --with-zstd \
+                --with-lz4 \
                 --without-warnings \
                 --without-warnings-as-errors \
                 --from-continuous-integration
@@ -57,6 +60,13 @@ build_ntf() {
         exit ${rc}
     fi
 
+    make ntca_checksum.t
+    rc=${?}
+    if [ ${rc} -ne 0 ]; then
+        echo "Failed to build: rc = ${rc}"
+        exit ${rc}
+    fi
+
     make build_test
     rc=${?}
     if [ ${rc} -ne 0 ]; then

cmake/templates/ntccfg_config.h

@@ -109,6 +109,15 @@ namespace ntccfg {
 // driver for a proactor. This driver is available on Linux.
 #define NTC_BUILD_WITH_IORING @NTF_BUILD_WITH_IORING@
 
+// Build with support for lz4 in the default compression driver.
+#define NTC_BUILD_WITH_LZ4 @NTF_BUILD_WITH_LZ4@
+
+// Build with support for zstd in the default compression driver.
+#define NTC_BUILD_WITH_ZSTD @NTF_BUILD_WITH_ZSTD@
+
+// Build with support for zlib in the default compression driver.
+#define NTC_BUILD_WITH_ZLIB @NTF_BUILD_WITH_ZLIB@
+
 // Build with support for OpenSSL as the driver for TLS.
 #define NTC_BUILD_WITH_OPENSSL @NTF_BUILD_WITH_OPENSSL@
 

configure

@@ -303,6 +303,18 @@ else
     NTF_CONFIGURE_WITH_IORING=0
 fi
 
+if [[ -z "${NTF_CONFIGURE_WITH_LZ4}" ]]; then
+    NTF_CONFIGURE_WITH_LZ4=0
+fi
+
+if [[ -z "${NTF_CONFIGURE_WITH_ZSTD}" ]]; then
+    NTF_CONFIGURE_WITH_ZSTD=0
+fi
+
+if [[ -z "${NTF_CONFIGURE_WITH_ZLIB}" ]]; then
+    NTF_CONFIGURE_WITH_ZLIB=0
+fi
+
 if [[ -z "${NTF_CONFIGURE_WITH_OPENSSL}" ]]; then
     NTF_CONFIGURE_WITH_OPENSSL=1
 fi
@@ -490,6 +502,10 @@ usage()
     echo "    --with-iocp                        Enable the proactor driver implemented with I/O completion ports (Windows only) [${NTF_CONFIGURE_WITH_IOCP}]"
     echo "    --with-ioring                      Enable the proactor driver implemented with 'io_uring' (Linux only) [${NTF_CONFIGURE_WITH_IORING}]"
 
+    echo "    --with-lz4                         Enable the compression driver implemented with lz4 [${NTF_CONFIGURE_WITH_LZ4}]"
+    echo "    --with-zstd                        Enable the compression driver implemented with zstd [${NTF_CONFIGURE_WITH_ZSTD}]"
+    echo "    --with-zlib                        Enable the compression driver implemented with zlib [${NTF_CONFIGURE_WITH_ZLIB}]"
+
     echo "    --with-openssl                     Enable the encryption driver implemented with OpenSSL [${NTF_CONFIGURE_WITH_OPENSSL}]"
 
     echo "    --with-dynamic-load-balancing      Enable processing I/O on any thread, rather than a single thread [${NTF_CONFIGURE_WITH_DYNAMIC_LOAD_BALANCING}]"
@@ -666,6 +682,13 @@ while true ; do
         --with-ioring)
             NTF_CONFIGURE_WITH_IORING=1 ; shift ;;
 
+        --with-lz4)
+            NTF_CONFIGURE_WITH_LZ4=1 ; shift ;;
+        --with-zstd)
+            NTF_CONFIGURE_WITH_ZSTD=1 ; shift ;;
+        --with-zlib)
+            NTF_CONFIGURE_WITH_ZLIB=1 ; shift ;;
+
         --with-openssl)
             NTF_CONFIGURE_WITH_OPENSSL=1 ; shift ;;
 
@@ -778,6 +801,13 @@ while true ; do
         --without-ioring)
             NTF_CONFIGURE_WITH_IORING=0 ; shift ;;
 
+        --without-lz4)
+            NTF_CONFIGURE_WITH_LZ4=0 ; shift ;;
+        --without-zstd)
+            NTF_CONFIGURE_WITH_ZSTD=0 ; shift ;;
+        --without-zlib)
+            NTF_CONFIGURE_WITH_ZLIB=0 ; shift ;;
+
         --without-openssl)
             NTF_CONFIGURE_WITH_OPENSSL=0 ; shift ;;
 
@@ -906,6 +936,10 @@ export NTF_CONFIGURE_WITH_KQUEUE
 export NTF_CONFIGURE_WITH_IOCP
 export NTF_CONFIGURE_WITH_IORING
 
+export NTF_CONFIGURE_WITH_LZ4
+export NTF_CONFIGURE_WITH_ZSTD
+export NTF_CONFIGURE_WITH_ZLIB
+
 export NTF_CONFIGURE_WITH_OPENSSL
 
 export NTF_CONFIGURE_WITH_DYNAMIC_LOAD_BALANCING

configure.cmd

@@ -109,6 +109,18 @@ IF NOT DEFINED NTF_CONFIGURE_WITH_IOCP (
     set NTF_CONFIGURE_WITH_IOCP=1
 )
 
+IF NOT DEFINED NTF_CONFIGURE_WITH_LZ4 (
+    set NTF_CONFIGURE_WITH_LZ4=0
+)
+
+IF NOT DEFINED NTF_CONFIGURE_WITH_ZSTD (
+    set NTF_CONFIGURE_WITH_ZSTD=0
+)
+
+IF NOT DEFINED NTF_CONFIGURE_WITH_ZLIB (
+    set NTF_CONFIGURE_WITH_ZLIB=0
+)
+
 IF NOT DEFINED NTF_CONFIGURE_WITH_OPENSSL (
     set NTF_CONFIGURE_WITH_OPENSSL=1
 )
@@ -333,6 +345,16 @@ if not "%1"=="" (
         set NTF_CONFIGURE_WITH_IOCP=1
     )
 
+    if "%1"=="--with-lz4" (
+        set NTF_CONFIGURE_WITH_LZ4=1
+    )
+    if "%1"=="--with-zstd" (
+        set NTF_CONFIGURE_WITH_ZSTD=1
+    )
+    if "%1"=="--with-zlib" (
+        set NTF_CONFIGURE_WITH_ZLIB=1
+    )
+
     if "%1"=="--with-openssl" (
         set NTF_CONFIGURE_WITH_OPENSSL=1
     )
@@ -465,6 +487,16 @@ if not "%1"=="" (
         set NTF_CONFIGURE_WITH_IOCP=0
     )
 
+    if "%1"=="--without-lz4" (
+        set NTF_CONFIGURE_WITH_LZ4=0
+    )
+    if "%1"=="--without-zstd" (
+        set NTF_CONFIGURE_WITH_ZSTD=0
+    )
+    if "%1"=="--without-zlib" (
+        set NTF_CONFIGURE_WITH_ZLIB=0
+    )
+
     if "%1"=="--without-openssl" (
         set NTF_CONFIGURE_WITH_OPENSSL=0
     )
@@ -860,6 +892,10 @@ echo     --with-select                    Enable the reactor driver implemented
 echo     --with-poll                      Enable the reactor driver implemented with 'poll'
 echo     --with-iocp                      Enable the proactor driver that depends on I/O completion ports
 
+echo     --with-lz4                       Enable the compression driver implemented with lz4
+echo     --with-zstd                      Enable the compression driver implemented with zstd
+echo     --with-zlib                      Enable the compression driver implemented with zlib
+
 echo     --with-openssl                   Enable the encryption driver implemented with OpenSSL
 
 echo     --with-dynamic-load-balancing    Enable processing I/O on any thread, rather than a single thread

groups/ntc/CMakeLists.txt

@@ -28,7 +28,7 @@ add_library(ntc STATIC)
 bbs_setup_target_uor(
     ntc
     NO_EMIT_PKG_CONFIG_FILE
-    PRIVATE_PACKAGES ntcm ntcs ntcq ntcd ntcdns ntctls ntcu ntcr ntcp ntco)
+    PRIVATE_PACKAGES ntcm ntcs ntcq ntcd ntcdns ntctlc ntctls ntcu ntcr ntcp ntco)
 
 target_include_directories(
     ntc

groups/ntc/group/ntc.mem

@@ -6,6 +6,7 @@ ntcs
 ntcq
 ntcd
 ntcdns
+ntctlc
 ntctls
 ntcu
 ntcr

groups/ntc/ntc.cpp

@@ -41,6 +41,11 @@
 #include <ntca_bindeventtype.cpp>
 #include <ntca_bindoptions.cpp>
 #include <ntca_bindtoken.cpp>
+#include <ntca_checksum.cpp>
+#include <ntca_checksumtype.cpp>
+#include <ntca_compressionconfig.cpp>
+#include <ntca_compressiongoal.cpp>
+#include <ntca_compressiontype.cpp>
 #include <ntca_connectcontext.cpp>
 #include <ntca_connectevent.cpp>
 #include <ntca_connecteventtype.cpp>
@@ -49,6 +54,8 @@
 #include <ntca_datagramsocketevent.cpp>
 #include <ntca_datagramsocketeventtype.cpp>
 #include <ntca_datagramsocketoptions.cpp>
+#include <ntca_deflatecontext.cpp>
+#include <ntca_deflateoptions.cpp>
 #include <ntca_downgradecontext.cpp>
 #include <ntca_downgradeevent.cpp>
 #include <ntca_downgradeeventtype.cpp>
@@ -97,6 +104,8 @@
 #include <ntca_getservicenameeventtype.cpp>
 #include <ntca_getservicenameoptions.cpp>
 #include <ntca_interfaceconfig.cpp>
+#include <ntca_inflatecontext.cpp>
+#include <ntca_inflateoptions.cpp>
 #include <ntca_listenersocketevent.cpp>
 #include <ntca_listenersocketeventtype.cpp>
 #include <ntca_listenersocketoptions.cpp>
@@ -127,6 +136,8 @@
 #include <ntca_sendeventtype.cpp>
 #include <ntca_sendoptions.cpp>
 #include <ntca_sendtoken.cpp>
+#include <ntca_serializationconfig.cpp>
+#include <ntca_serializationtype.cpp>
 #include <ntca_shutdowncontext.cpp>
 #include <ntca_shutdownevent.cpp>
 #include <ntca_shutdowneventtype.cpp>
@@ -170,6 +181,8 @@
 #include <ntci_closecallbackfactory.cpp>
 #include <ntci_closefuture.cpp>
 #include <ntci_closeresult.cpp>
+#include <ntci_compression.cpp>
+#include <ntci_compressiondriver.cpp>
 #include <ntci_connectcallback.cpp>
 #include <ntci_connectcallbackfactory.cpp>
 #include <ntci_connectfuture.cpp>
@@ -244,6 +257,7 @@
 #include <ntci_sender.cpp>
 #include <ntci_sendfuture.cpp>
 #include <ntci_sendresult.cpp>
+#include <ntci_serialization.cpp>
 #include <ntci_strand.cpp>
 #include <ntci_strandfactory.cpp>
 #include <ntci_streamsocket.cpp>
@@ -263,6 +277,8 @@
 #include <ntci_upgradable.cpp>
 #include <ntci_upgradecallback.cpp>
 #include <ntci_upgradecallbackfactory.cpp>
+#include <ntci_upgradefuture.cpp>
+#include <ntci_upgraderesult.cpp>
 #include <ntci_user.cpp>
 #include <ntci_waiter.cpp>
 
@@ -326,6 +342,7 @@
 // ntcd
 
 #include <ntcd_blobbufferfactory.cpp>
+#include <ntcd_compression.cpp>
 #include <ntcd_datagramsocket.cpp>
 #include <ntcd_datapool.cpp>
 #include <ntcd_datautil.cpp>
@@ -353,6 +370,10 @@
 #include <ntcdns_utility.cpp>
 #include <ntcdns_vocabulary.cpp>
 
+// ntctlc
+
+#include <ntctlc_plugin.cpp>
+
 // ntctls
 
 #include <ntctls_plugin.cpp>

groups/ntc/ntc.t.cpp

@@ -41,6 +41,7 @@
 // #include <ntca_bindeventtype.t.cpp>
 // #include <ntca_bindoptions.t.cpp>
 // #include <ntca_bindtoken.t.cpp>
+#include <ntca_checksum.t.cpp>
 // #include <ntca_connectcontext.t.cpp>
 // #include <ntca_connectevent.t.cpp>
 // #include <ntca_connecteventtype.t.cpp>
@@ -326,6 +327,7 @@
 // ntcd
 
 #include <ntcd_blobbufferfactory.t.cpp>
+#include <ntcd_compression.t.cpp>
 #include <ntcd_datagramsocket.t.cpp>
 #include <ntcd_datapool.t.cpp>
 #include <ntcd_datautil.t.cpp>
@@ -353,6 +355,10 @@
 #include <ntcdns_utility.t.cpp>
 #include <ntcdns_vocabulary.t.cpp>
 
+// ntctlc
+
+#include <ntctlc_plugin.t.cpp>
+
 // ntctls
 
 #include <ntctls_plugin.t.cpp>