qatzstd refactor(Do not merge just for review code)

api/BUILD

@@ -73,6 +73,7 @@ proto_library(
     visibility = ["//visibility:public"],
     deps = [
         "//contrib/envoy/extensions/compression/qatzip/compressor/v3alpha:pkg",
+        "//contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/http/dynamo/v3:pkg",
         "//contrib/envoy/extensions/filters/http/golang/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/http/language/v3alpha:pkg",

api/contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha/BUILD

@@ -0,0 +1,12 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = [
+        "//envoy/config/core/v3:pkg",
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+    ],
+)

api/contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha/qatzstd.proto

@@ -0,0 +1,72 @@
+syntax = "proto3";
+
+package envoy.extensions.compression.qatzstd.compressor.v3alpha;
+
+import "envoy/config/core/v3/base.proto";
+
+import "google/protobuf/wrappers.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.compression.qatzstd.compressor.v3alpha";
+option java_outer_classname = "QatzstdProto";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/compression/qatzstd/compressor/v3alpha";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Qatzstd Compressor]
+// [#extension: envoy.compression.qatzstd.compressor]
+
+// [#next-free-field: 8]
+message Qatzstd {
+  // Reference to http://facebook.github.io/zstd/zstd_manual.html
+  enum Strategy {
+    DEFAULT = 0;
+    FAST = 1;
+    DFAST = 2;
+    GREEDY = 3;
+    LAZY = 4;
+    LAZY2 = 5;
+    BTLAZY2 = 6;
+    BTOPT = 7;
+    BTULTRA = 8;
+    BTULTRA2 = 9;
+  }
+
+  // Set compression parameters according to pre-defined compression level table.
+  // Note that exact compression parameters are dynamically determined,
+  // depending on both compression level and source content size (when known).
+  // Value 0 means default, and default level is 3.
+  // Setting a level does not automatically set all other compression parameters
+  // to default. Setting this will however eventually dynamically impact the compression
+  // parameters which have not been manually set. The manually set
+  // ones will 'stick'.
+  google.protobuf.UInt32Value compression_level = 1;
+
+  // A 32-bits checksum of content is written at end of frame. If not set, defaults to false.
+  bool enable_checksum = 2;
+
+  // The higher the value of selected strategy, the more complex it is,
+  // resulting in stronger and slower compression.
+  // Special: value 0 means "use default strategy".
+  Strategy strategy = 3 [(validate.rules).enum = {defined_only: true}];
+
+  // A dictionary for compression. Zstd offers dictionary compression, which greatly improves
+  // efficiency on small files and messages. Each dictionary will be generated with a dictionary ID
+  // that can be used to search the same dictionary during decompression.
+  // Please refer to `zstd manual <https://github.com/facebook/zstd/blob/dev/programs/zstd.1.md#dictionary-builder>`_
+  // to train a specific dictionary for compression.
+  config.core.v3.DataSource dictionary = 4;
+
+  // Value for compressor's next output buffer. If not set, defaults to 4096.
+  google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}];
+
+
+  // Enable QAT to accelerate zstd compresstion or not. If not set, defaults to false.
+  bool enable_qat_zstd = 6;
+
+  // Fall back to software for QAT-zstd when input size is less than this value.
+  // Valid only enable_qat_zstd is true. 0 means no fallback at all. If not set, defaults to 4000.
+  google.protobuf.UInt32Value qat_zstd_fallback_threshold = 7 [(validate.rules).uint32 = {lte: 65536 gte: 0}];
+}

api/versioning/BUILD

@@ -10,6 +10,7 @@ proto_library(
     visibility = ["//visibility:public"],
     deps = [
         "//contrib/envoy/extensions/compression/qatzip/compressor/v3alpha:pkg",
+        "//contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha:pkg",
         "//contrib/envoy/extensions/config/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/http/dynamo/v3:pkg",
         "//contrib/envoy/extensions/filters/http/golang/v3alpha:pkg",

bazel/foreign_cc/qatzstd.patch

@@ -0,0 +1,85 @@
+diff --git a/src/Makefile b/src/Makefile
+index 1abf10d..a0c7e9a 100644
+--- a/src/Makefile
++++ b/src/Makefile
+@@ -41,6 +41,7 @@ LIBDIR     ?= $(INSTALLDIR)/lib
+ INCLUDEDIR ?= $(INSTALLDIR)/include
+
+ CP ?= cp
++MKDIR ?= mkdir
+
+ ENABLE_USDM_DRV ?= 0
+ ifneq ($(ICP_ROOT), )
+@@ -55,10 +56,8 @@ ifneq ($(ICP_ROOT), )
+ else
+ 	QATFLAGS = -DINTREE
+ 	LDFLAGS  = -lqat
+-	ifneq ($(ENABLE_USDM_DRV), 0)
+-		QATFLAGS += -DENABLE_USDM_DRV
+-		LDFLAGS	 += -lusdm
+-	endif
++	QATFLAGS += -DENABLE_USDM_DRV
++	LDFLAGS	 += -lusdm
+ endif
+
+ ifdef ZSTDLIB
+@@ -69,8 +68,8 @@ CFLAGS += -Wall -Werror -Wextra -Wcast-align -Wshadow -Wstrict-aliasing=1 \
+ 		  -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
+ 		  -Wundef -Wpointer-arith -Wvla -Wformat=2 -Winit-self \
+ 		  -Wfloat-equal -Wwrite-strings -Wredundant-decls -Wc++-compat \
+-		  -pedantic -fstack-protector-strong -fPIE -fPIC \
+-		  -fno-delete-null-pointer-checks -fwrapv -fno-strict-overflow
++		  -pedantic -fstack-protector-strong \
++		  -fno-delete-null-pointer-checks -fwrapv 
+
+ DEBUGLEVEL ?=0
+
+@@ -81,27 +80,30 @@ else
+ 	QATFLAGS += -O3
+ endif
+
++$(info INSTALLDIR="$(INSTALLDIR)")
++$(info CPPFLAGS="$(CPPFLAGS)")
++
+ qatseqprod.o: qatseqprod.c
+-	$(CC) -c $(CFLAGS) $(QATFLAGS) $(DEBUGFLAGS) $^ -o $@
++	$(CC) -c $(CPPFLAGS) $(CFLAGS) $(QATFLAGS) $(DEBUGFLAGS) $^ -o $@
+
+ lib: qatseqprod.o
+ 	$(AR) rc libqatseqprod.a $^
+-	$(CC) -shared $^ $(LDFLAGS) -o libqatseqprod.so
++	@echo qatseqprod library successfully build
+
+ .PHONY: install
+ install: lib
++	$(MKDIR) -p $(LIBDIR)
++	$(MKDIR) -p $(INCLUDEDIR)
+ 	$(CP) libqatseqprod.a $(LIBDIR)
+-	$(CP) libqatseqprod.so $(LIBDIR)
+ 	$(CP) qatseqprod.h $(INCLUDEDIR)
+ 	@echo qatseqprod library successfully installed
+
+ .PHONY: uninstall
+ uninstall:
+ 	$(RM) $(LIBDIR)/libqatseqprod.a
+-	$(RM) $(LIBDIR)/libqatseqprod.so
+ 	$(RM) $(INCLUDEDIR)/qatseqprod.h
+ 	@echo qatseqprod library successfully uninstalled
+
+ clean:
+ 	$(RM) *.o
+-	$(RM) libqatseqprod.a libqatseqprod.so
++	$(RM) libqatseqprod.a 
+diff --git a/test/Makefile b/test/Makefile
+index dff0c8e..4ba01b2 100644
+--- a/test/Makefile
++++ b/test/Makefile
+@@ -34,7 +34,7 @@
+ # #######################################################################
+ LIB     = ../src
+
+-LDFLAGS = $(LIB)/libqatseqprod.a -I$(LIB)
++LDFLAGS = $(LIB)/libqatseqprod.a -I$(LIB) -L$(LIB) -l:libqatseqprod.a -l:libqat.a -l:libusdm.a -l:libzstd.a -lpthread -lcrypto
+
+ ifneq ($(ICP_ROOT), )
+ 	LDFLAGS += -lqat_s -lusdm_drv_s -Wl,-rpath,$(ICP_ROOT)/build -L$(ICP_ROOT)/build

bazel/repositories.bzl

@@ -280,6 +280,7 @@ def envoy_dependencies(skip_targets = []):
     _com_github_intel_ipp_crypto_crypto_mb()
     _com_github_intel_qatlib()
     _com_github_intel_qatzip()
+    _com_github_qat_zstd()
     _com_github_lz4_lz4()
     _com_github_jbeder_yaml_cpp()
     _com_github_libevent_libevent()
@@ -526,6 +527,14 @@ def _com_github_intel_qatzip():
         build_file_content = BUILD_ALL_CONTENT,
     )
 
+def _com_github_qat_zstd():
+    external_http_archive(
+        name = "com_github_qat_zstd",
+        build_file_content = BUILD_ALL_CONTENT,
+        patch_args = ["-p1"],
+        patches = ["@envoy//bazel/foreign_cc:qatzstd.patch"],
+    )
+
 def _com_github_lz4_lz4():
     external_http_archive(
         name = "com_github_lz4_lz4",

bazel/repository_locations.bzl

@@ -431,7 +431,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         urls = ["https://github.com/intel/qatlib/archive/refs/tags/{version}.tar.gz"],
         use_category = ["dataplane_ext"],
         release_date = "2023-02-24",
-        extensions = ["envoy.tls.key_providers.qat","envoy.compression.qatzip.compresso"],
+        extensions = ["envoy.tls.key_providers.qat", "envoy.compression.qatzip.compressor","envoy.compression.qatzstd.compressor"],
         cpe = "N/A",
         license = "BSD-3-Clause",
         license_url = "https://github.com/intel/qatlib/blob/{version}/LICENSE",
@@ -451,6 +451,21 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         license = "BSD-3-Clause",
         license_url = "https://github.com/intel/QATzip/blob/{version}/LICENSE",
     ),
+    com_github_qat_zstd = dict(
+        project_name = "QAT-ZSTD-Plugin",
+        project_desc = "Intel® QuickAssist Technology ZSTD Plugin (QAT ZSTD Plugin)",
+        project_url = "https://github.com/intel/QAT-ZSTD-Plugin/",
+        version = "0.1.0",
+        sha256 = "74c5bfbb3b0c6f1334e128ee0b43958d1d34751a4762e54e8f970c443e445f33",
+        strip_prefix = "QAT-ZSTD-Plugin-{version}",
+        urls = ["https://github.com/intel/QAT-ZSTD-Plugin/archive/refs/tags/v{version}.tar.gz"],
+        use_category = ["dataplane_ext"],
+        extensions = [
+            "envoy.compression.qatzstd.compressor",
+        ],
+        release_date = "2023-09-08",
+        cpe = "N/A",
+    ),
     com_github_luajit_luajit = dict(
         project_name = "LuaJIT",
         project_desc = "Just-In-Time compiler for Lua",

contrib/contrib_build_config.bzl

@@ -5,84 +5,85 @@ CONTRIB_EXTENSIONS = {
     #
 
     "envoy.compression.qatzip.compressor":                      "//contrib/qat/compression/qatzip/compressor/source:config",
+    "envoy.compression.qatzstd.compressor":                      "//contrib/qat/compression/qatzstd/compressor/source:config",
+
+    # #
+    # # HTTP filters
+    # #
 
-    #
-    # HTTP filters
-    #
+    # "envoy.filters.http.dynamo":                                "//contrib/dynamo/filters/http/source:config",
+    # "envoy.filters.http.golang":                                "//contrib/golang/filters/http/source:config",
+    # "envoy.filters.http.language":                              "//contrib/language/filters/http/source:config_lib",
+    # "envoy.filters.http.squash":                                "//contrib/squash/filters/http/source:config",
+    # "envoy.filters.http.sxg":                                   "//contrib/sxg/filters/http/source:config",
 
-    "envoy.filters.http.dynamo":                                "//contrib/dynamo/filters/http/source:config",
-    "envoy.filters.http.golang":                                "//contrib/golang/filters/http/source:config",
-    "envoy.filters.http.language":                              "//contrib/language/filters/http/source:config_lib",
-    "envoy.filters.http.squash":                                "//contrib/squash/filters/http/source:config",
-    "envoy.filters.http.sxg":                                   "//contrib/sxg/filters/http/source:config",
+    # #
+    # # Network filters
+    # #
 
-    #
-    # Network filters
-    #
+    # "envoy.filters.network.client_ssl_auth":                    "//contrib/client_ssl_auth/filters/network/source:config",
+    # "envoy.filters.network.kafka_broker":                       "//contrib/kafka/filters/network/source:kafka_broker_config_lib",
+    # "envoy.filters.network.kafka_mesh":                         "//contrib/kafka/filters/network/source/mesh:config_lib",
+    # "envoy.filters.network.mysql_proxy":                        "//contrib/mysql_proxy/filters/network/source:config",
+    # "envoy.filters.network.postgres_proxy":                     "//contrib/postgres_proxy/filters/network/source:config",
+    # "envoy.filters.network.rocketmq_proxy":                     "//contrib/rocketmq_proxy/filters/network/source:config",
+    # "envoy.filters.network.generic_proxy":                      "//contrib/generic_proxy/filters/network/source:config",
+    # "envoy.filters.network.golang":                             "//contrib/golang/filters/network/source:config",
 
-    "envoy.filters.network.client_ssl_auth":                    "//contrib/client_ssl_auth/filters/network/source:config",
-    "envoy.filters.network.kafka_broker":                       "//contrib/kafka/filters/network/source:kafka_broker_config_lib",
-    "envoy.filters.network.kafka_mesh":                         "//contrib/kafka/filters/network/source/mesh:config_lib",
-    "envoy.filters.network.mysql_proxy":                        "//contrib/mysql_proxy/filters/network/source:config",
-    "envoy.filters.network.postgres_proxy":                     "//contrib/postgres_proxy/filters/network/source:config",
-    "envoy.filters.network.rocketmq_proxy":                     "//contrib/rocketmq_proxy/filters/network/source:config",
-    "envoy.filters.network.generic_proxy":                      "//contrib/generic_proxy/filters/network/source:config",
-    "envoy.filters.network.golang":                             "//contrib/golang/filters/network/source:config",
+    # #
+    # # Sip proxy
+    # #
 
-    #
-    # Sip proxy
-    #
+    # "envoy.filters.network.sip_proxy":                          "//contrib/sip_proxy/filters/network/source:config",
+    # "envoy.filters.sip.router":                                 "//contrib/sip_proxy/filters/network/source/router:config",
 
-    "envoy.filters.network.sip_proxy":                          "//contrib/sip_proxy/filters/network/source:config",
-    "envoy.filters.sip.router":                                 "//contrib/sip_proxy/filters/network/source/router:config",
+    # #
+    # # Private key providers
+    # #
 
-    #
-    # Private key providers
-    #
+    # "envoy.tls.key_providers.cryptomb":                         "//contrib/cryptomb/private_key_providers/source:config",
+    # "envoy.tls.key_providers.qat":                              "//contrib/qat/private_key_providers/source:config",
+    # "envoy.tls.key_providers.sgx":                              "//contrib/sgx/private_key_providers/source:config",
 
-    "envoy.tls.key_providers.cryptomb":                         "//contrib/cryptomb/private_key_providers/source:config",
-    "envoy.tls.key_providers.qat":                              "//contrib/qat/private_key_providers/source:config",
-    "envoy.tls.key_providers.sgx":                              "//contrib/sgx/private_key_providers/source:config",
+    # #
+    # # Socket interface extensions
+    # #
 
-    #
-    # Socket interface extensions
-    #
+    # "envoy.bootstrap.vcl":                                      "//contrib/vcl/source:config",
 
-    "envoy.bootstrap.vcl":                                      "//contrib/vcl/source:config",
+    # #
+    # # Input matchers
+    # #
 
-    #
-    # Input matchers
-    #
+    # "envoy.matching.input_matchers.hyperscan":                  "//contrib/hyperscan/matching/input_matchers/source:config",
 
-    "envoy.matching.input_matchers.hyperscan":                  "//contrib/hyperscan/matching/input_matchers/source:config",
+    # #
+    # # Connection Balance extensions
+    # #
 
-    #
-    # Connection Balance extensions
-    #
+    # "envoy.network.connection_balance.dlb":                     "//contrib/network/connection_balance/dlb/source:connection_balancer",
 
-    "envoy.network.connection_balance.dlb":                     "//contrib/network/connection_balance/dlb/source:connection_balancer",
+    # #
+    # # Regex engines
+    # #
 
-    #
-    # Regex engines
-    #
+    # "envoy.regex_engines.hyperscan":                            "//contrib/hyperscan/regex_engines/source:config",
 
-    "envoy.regex_engines.hyperscan":                            "//contrib/hyperscan/regex_engines/source:config",
+    # #
+    # # Extensions for generic proxy
+    # #
+    # "envoy.filters.generic.router":                             "//contrib/generic_proxy/filters/network/source/router:config",
+    # "envoy.generic_proxy.codecs.dubbo":                         "//contrib/generic_proxy/filters/network/source/codecs/dubbo:config",
 
-    #
-    # Extensions for generic proxy
-    #
-    "envoy.filters.generic.router":                             "//contrib/generic_proxy/filters/network/source/router:config",
-    "envoy.generic_proxy.codecs.dubbo":                         "//contrib/generic_proxy/filters/network/source/codecs/dubbo:config",
+    # #
+    # # xDS delegates
+    # #
 
-    #
-    # xDS delegates
-    #
-
-    "envoy.xds_delegates.kv_store":                            "//contrib/config/source:kv_store_xds_delegate",
+    # "envoy.xds_delegates.kv_store":                            "//contrib/config/source:kv_store_xds_delegate",
 
-    #
-    # cluster specifier plugin
-    #
+    # #
+    # # cluster specifier plugin
+    # #
 
-    "envoy.router.cluster_specifier_plugin.golang":             "//contrib/golang/router/cluster_specifier/source:config",
+    # "envoy.router.cluster_specifier_plugin.golang":             "//contrib/golang/router/cluster_specifier/source:config",
 }