Skip to content
This repository was archived by the owner on Nov 20, 2024. It is now read-only.

qatzstd refactor(Do not merge just for review code) #251

Open
wants to merge 5 commits into
base: release-1.19-intel
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ proto_library(
visibility = ["//visibility:public"],
deps = [
"//contrib/envoy/extensions/compression/qatzip/compressor/v3alpha:pkg",
"//contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha:pkg",
"//contrib/envoy/extensions/filters/http/dynamo/v3:pkg",
"//contrib/envoy/extensions/filters/http/golang/v3alpha:pkg",
"//contrib/envoy/extensions/filters/http/language/v3alpha:pkg",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.

load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")

licenses(["notice"]) # Apache 2

api_proto_package(
deps = [
"//envoy/config/core/v3:pkg",
"@com_github_cncf_udpa//udpa/annotations:pkg",
],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
syntax = "proto3";

package envoy.extensions.compression.qatzstd.compressor.v3alpha;

import "envoy/config/core/v3/base.proto";

import "google/protobuf/wrappers.proto";

import "udpa/annotations/status.proto";
import "validate/validate.proto";

option java_package = "io.envoyproxy.envoy.extensions.compression.qatzstd.compressor.v3alpha";
option java_outer_classname = "QatzstdProto";
option java_multiple_files = true;
option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/compression/qatzstd/compressor/v3alpha";
option (udpa.annotations.file_status).package_version_status = ACTIVE;

// [#protodoc-title: Qatzstd Compressor]
// [#extension: envoy.compression.qatzstd.compressor]

// [#next-free-field: 8]
message Qatzstd {
// Reference to http://facebook.github.io/zstd/zstd_manual.html
enum Strategy {
DEFAULT = 0;
FAST = 1;
DFAST = 2;
GREEDY = 3;
LAZY = 4;
LAZY2 = 5;
BTLAZY2 = 6;
BTOPT = 7;
BTULTRA = 8;
BTULTRA2 = 9;
}

// Set compression parameters according to pre-defined compression level table.
// Note that exact compression parameters are dynamically determined,
// depending on both compression level and source content size (when known).
// Value 0 means default, and default level is 3.
// Setting a level does not automatically set all other compression parameters
// to default. Setting this will however eventually dynamically impact the compression
// parameters which have not been manually set. The manually set
// ones will 'stick'.
google.protobuf.UInt32Value compression_level = 1;

// A 32-bits checksum of content is written at end of frame. If not set, defaults to false.
bool enable_checksum = 2;

// The higher the value of selected strategy, the more complex it is,
// resulting in stronger and slower compression.
// Special: value 0 means "use default strategy".
Strategy strategy = 3 [(validate.rules).enum = {defined_only: true}];

// A dictionary for compression. Zstd offers dictionary compression, which greatly improves
// efficiency on small files and messages. Each dictionary will be generated with a dictionary ID
// that can be used to search the same dictionary during decompression.
// Please refer to `zstd manual <https://github.com/facebook/zstd/blob/dev/programs/zstd.1.md#dictionary-builder>`_
// to train a specific dictionary for compression.
config.core.v3.DataSource dictionary = 4;

// Value for compressor's next output buffer. If not set, defaults to 4096.
google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}];


// Enable QAT to accelerate zstd compresstion or not. If not set, defaults to false.
bool enable_qat_zstd = 6;

// Fall back to software for QAT-zstd when input size is less than this value.
// Valid only enable_qat_zstd is true. 0 means no fallback at all. If not set, defaults to 4000.
google.protobuf.UInt32Value qat_zstd_fallback_threshold = 7 [(validate.rules).uint32 = {lte: 65536 gte: 0}];
}
1 change: 1 addition & 0 deletions api/versioning/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ proto_library(
visibility = ["//visibility:public"],
deps = [
"//contrib/envoy/extensions/compression/qatzip/compressor/v3alpha:pkg",
"//contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha:pkg",
"//contrib/envoy/extensions/config/v3alpha:pkg",
"//contrib/envoy/extensions/filters/http/dynamo/v3:pkg",
"//contrib/envoy/extensions/filters/http/golang/v3alpha:pkg",
Expand Down
85 changes: 85 additions & 0 deletions bazel/foreign_cc/qatzstd.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
diff --git a/src/Makefile b/src/Makefile
index 1abf10d..a0c7e9a 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -41,6 +41,7 @@ LIBDIR ?= $(INSTALLDIR)/lib
INCLUDEDIR ?= $(INSTALLDIR)/include

CP ?= cp
+MKDIR ?= mkdir

ENABLE_USDM_DRV ?= 0
ifneq ($(ICP_ROOT), )
@@ -55,10 +56,8 @@ ifneq ($(ICP_ROOT), )
else
QATFLAGS = -DINTREE
LDFLAGS = -lqat
- ifneq ($(ENABLE_USDM_DRV), 0)
- QATFLAGS += -DENABLE_USDM_DRV
- LDFLAGS += -lusdm
- endif
+ QATFLAGS += -DENABLE_USDM_DRV
+ LDFLAGS += -lusdm
endif

ifdef ZSTDLIB
@@ -69,8 +68,8 @@ CFLAGS += -Wall -Werror -Wextra -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wundef -Wpointer-arith -Wvla -Wformat=2 -Winit-self \
-Wfloat-equal -Wwrite-strings -Wredundant-decls -Wc++-compat \
- -pedantic -fstack-protector-strong -fPIE -fPIC \
- -fno-delete-null-pointer-checks -fwrapv -fno-strict-overflow
+ -pedantic -fstack-protector-strong \
+ -fno-delete-null-pointer-checks -fwrapv

DEBUGLEVEL ?=0

@@ -81,27 +80,30 @@ else
QATFLAGS += -O3
endif

+$(info INSTALLDIR="$(INSTALLDIR)")
+$(info CPPFLAGS="$(CPPFLAGS)")
+
qatseqprod.o: qatseqprod.c
- $(CC) -c $(CFLAGS) $(QATFLAGS) $(DEBUGFLAGS) $^ -o $@
+ $(CC) -c $(CPPFLAGS) $(CFLAGS) $(QATFLAGS) $(DEBUGFLAGS) $^ -o $@

lib: qatseqprod.o
$(AR) rc libqatseqprod.a $^
- $(CC) -shared $^ $(LDFLAGS) -o libqatseqprod.so
+ @echo qatseqprod library successfully build

.PHONY: install
install: lib
+ $(MKDIR) -p $(LIBDIR)
+ $(MKDIR) -p $(INCLUDEDIR)
$(CP) libqatseqprod.a $(LIBDIR)
- $(CP) libqatseqprod.so $(LIBDIR)
$(CP) qatseqprod.h $(INCLUDEDIR)
@echo qatseqprod library successfully installed

.PHONY: uninstall
uninstall:
$(RM) $(LIBDIR)/libqatseqprod.a
- $(RM) $(LIBDIR)/libqatseqprod.so
$(RM) $(INCLUDEDIR)/qatseqprod.h
@echo qatseqprod library successfully uninstalled

clean:
$(RM) *.o
- $(RM) libqatseqprod.a libqatseqprod.so
+ $(RM) libqatseqprod.a
diff --git a/test/Makefile b/test/Makefile
index dff0c8e..4ba01b2 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -34,7 +34,7 @@
# #######################################################################
LIB = ../src

-LDFLAGS = $(LIB)/libqatseqprod.a -I$(LIB)
+LDFLAGS = $(LIB)/libqatseqprod.a -I$(LIB) -L$(LIB) -l:libqatseqprod.a -l:libqat.a -l:libusdm.a -l:libzstd.a -lpthread -lcrypto

ifneq ($(ICP_ROOT), )
LDFLAGS += -lqat_s -lusdm_drv_s -Wl,-rpath,$(ICP_ROOT)/build -L$(ICP_ROOT)/build
9 changes: 9 additions & 0 deletions bazel/repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ def envoy_dependencies(skip_targets = []):
_com_github_intel_ipp_crypto_crypto_mb()
_com_github_intel_qatlib()
_com_github_intel_qatzip()
_com_github_qat_zstd()
_com_github_lz4_lz4()
_com_github_jbeder_yaml_cpp()
_com_github_libevent_libevent()
Expand Down Expand Up @@ -526,6 +527,14 @@ def _com_github_intel_qatzip():
build_file_content = BUILD_ALL_CONTENT,
)

def _com_github_qat_zstd():
external_http_archive(
name = "com_github_qat_zstd",
build_file_content = BUILD_ALL_CONTENT,
patch_args = ["-p1"],
patches = ["@envoy//bazel/foreign_cc:qatzstd.patch"],
)

def _com_github_lz4_lz4():
external_http_archive(
name = "com_github_lz4_lz4",
Expand Down
17 changes: 16 additions & 1 deletion bazel/repository_locations.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -431,7 +431,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
urls = ["https://github.com/intel/qatlib/archive/refs/tags/{version}.tar.gz"],
use_category = ["dataplane_ext"],
release_date = "2023-02-24",
extensions = ["envoy.tls.key_providers.qat","envoy.compression.qatzip.compresso"],
extensions = ["envoy.tls.key_providers.qat", "envoy.compression.qatzip.compressor","envoy.compression.qatzstd.compressor"],
cpe = "N/A",
license = "BSD-3-Clause",
license_url = "https://github.com/intel/qatlib/blob/{version}/LICENSE",
Expand All @@ -451,6 +451,21 @@ REPOSITORY_LOCATIONS_SPEC = dict(
license = "BSD-3-Clause",
license_url = "https://github.com/intel/QATzip/blob/{version}/LICENSE",
),
com_github_qat_zstd = dict(
project_name = "QAT-ZSTD-Plugin",
project_desc = "Intel® QuickAssist Technology ZSTD Plugin (QAT ZSTD Plugin)",
project_url = "https://github.com/intel/QAT-ZSTD-Plugin/",
version = "0.1.0",
sha256 = "74c5bfbb3b0c6f1334e128ee0b43958d1d34751a4762e54e8f970c443e445f33",
strip_prefix = "QAT-ZSTD-Plugin-{version}",
urls = ["https://github.com/intel/QAT-ZSTD-Plugin/archive/refs/tags/v{version}.tar.gz"],
use_category = ["dataplane_ext"],
extensions = [
"envoy.compression.qatzstd.compressor",
],
release_date = "2023-09-08",
cpe = "N/A",
),
com_github_luajit_luajit = dict(
project_name = "LuaJIT",
project_desc = "Just-In-Time compiler for Lua",
Expand Down
121 changes: 61 additions & 60 deletions contrib/contrib_build_config.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -5,84 +5,85 @@ CONTRIB_EXTENSIONS = {
#

"envoy.compression.qatzip.compressor": "//contrib/qat/compression/qatzip/compressor/source:config",
"envoy.compression.qatzstd.compressor": "//contrib/qat/compression/qatzstd/compressor/source:config",

# #
# # HTTP filters
# #

#
# HTTP filters
#
# "envoy.filters.http.dynamo": "//contrib/dynamo/filters/http/source:config",
# "envoy.filters.http.golang": "//contrib/golang/filters/http/source:config",
# "envoy.filters.http.language": "//contrib/language/filters/http/source:config_lib",
# "envoy.filters.http.squash": "//contrib/squash/filters/http/source:config",
# "envoy.filters.http.sxg": "//contrib/sxg/filters/http/source:config",

"envoy.filters.http.dynamo": "//contrib/dynamo/filters/http/source:config",
"envoy.filters.http.golang": "//contrib/golang/filters/http/source:config",
"envoy.filters.http.language": "//contrib/language/filters/http/source:config_lib",
"envoy.filters.http.squash": "//contrib/squash/filters/http/source:config",
"envoy.filters.http.sxg": "//contrib/sxg/filters/http/source:config",
# #
# # Network filters
# #

#
# Network filters
#
# "envoy.filters.network.client_ssl_auth": "//contrib/client_ssl_auth/filters/network/source:config",
# "envoy.filters.network.kafka_broker": "//contrib/kafka/filters/network/source:kafka_broker_config_lib",
# "envoy.filters.network.kafka_mesh": "//contrib/kafka/filters/network/source/mesh:config_lib",
# "envoy.filters.network.mysql_proxy": "//contrib/mysql_proxy/filters/network/source:config",
# "envoy.filters.network.postgres_proxy": "//contrib/postgres_proxy/filters/network/source:config",
# "envoy.filters.network.rocketmq_proxy": "//contrib/rocketmq_proxy/filters/network/source:config",
# "envoy.filters.network.generic_proxy": "//contrib/generic_proxy/filters/network/source:config",
# "envoy.filters.network.golang": "//contrib/golang/filters/network/source:config",

"envoy.filters.network.client_ssl_auth": "//contrib/client_ssl_auth/filters/network/source:config",
"envoy.filters.network.kafka_broker": "//contrib/kafka/filters/network/source:kafka_broker_config_lib",
"envoy.filters.network.kafka_mesh": "//contrib/kafka/filters/network/source/mesh:config_lib",
"envoy.filters.network.mysql_proxy": "//contrib/mysql_proxy/filters/network/source:config",
"envoy.filters.network.postgres_proxy": "//contrib/postgres_proxy/filters/network/source:config",
"envoy.filters.network.rocketmq_proxy": "//contrib/rocketmq_proxy/filters/network/source:config",
"envoy.filters.network.generic_proxy": "//contrib/generic_proxy/filters/network/source:config",
"envoy.filters.network.golang": "//contrib/golang/filters/network/source:config",
# #
# # Sip proxy
# #

#
# Sip proxy
#
# "envoy.filters.network.sip_proxy": "//contrib/sip_proxy/filters/network/source:config",
# "envoy.filters.sip.router": "//contrib/sip_proxy/filters/network/source/router:config",

"envoy.filters.network.sip_proxy": "//contrib/sip_proxy/filters/network/source:config",
"envoy.filters.sip.router": "//contrib/sip_proxy/filters/network/source/router:config",
# #
# # Private key providers
# #

#
# Private key providers
#
# "envoy.tls.key_providers.cryptomb": "//contrib/cryptomb/private_key_providers/source:config",
# "envoy.tls.key_providers.qat": "//contrib/qat/private_key_providers/source:config",
# "envoy.tls.key_providers.sgx": "//contrib/sgx/private_key_providers/source:config",

"envoy.tls.key_providers.cryptomb": "//contrib/cryptomb/private_key_providers/source:config",
"envoy.tls.key_providers.qat": "//contrib/qat/private_key_providers/source:config",
"envoy.tls.key_providers.sgx": "//contrib/sgx/private_key_providers/source:config",
# #
# # Socket interface extensions
# #

#
# Socket interface extensions
#
# "envoy.bootstrap.vcl": "//contrib/vcl/source:config",

"envoy.bootstrap.vcl": "//contrib/vcl/source:config",
# #
# # Input matchers
# #

#
# Input matchers
#
# "envoy.matching.input_matchers.hyperscan": "//contrib/hyperscan/matching/input_matchers/source:config",

"envoy.matching.input_matchers.hyperscan": "//contrib/hyperscan/matching/input_matchers/source:config",
# #
# # Connection Balance extensions
# #

#
# Connection Balance extensions
#
# "envoy.network.connection_balance.dlb": "//contrib/network/connection_balance/dlb/source:connection_balancer",

"envoy.network.connection_balance.dlb": "//contrib/network/connection_balance/dlb/source:connection_balancer",
# #
# # Regex engines
# #

#
# Regex engines
#
# "envoy.regex_engines.hyperscan": "//contrib/hyperscan/regex_engines/source:config",

"envoy.regex_engines.hyperscan": "//contrib/hyperscan/regex_engines/source:config",
# #
# # Extensions for generic proxy
# #
# "envoy.filters.generic.router": "//contrib/generic_proxy/filters/network/source/router:config",
# "envoy.generic_proxy.codecs.dubbo": "//contrib/generic_proxy/filters/network/source/codecs/dubbo:config",

#
# Extensions for generic proxy
#
"envoy.filters.generic.router": "//contrib/generic_proxy/filters/network/source/router:config",
"envoy.generic_proxy.codecs.dubbo": "//contrib/generic_proxy/filters/network/source/codecs/dubbo:config",
# #
# # xDS delegates
# #

#
# xDS delegates
#

"envoy.xds_delegates.kv_store": "//contrib/config/source:kv_store_xds_delegate",
# "envoy.xds_delegates.kv_store": "//contrib/config/source:kv_store_xds_delegate",

#
# cluster specifier plugin
#
# #
# # cluster specifier plugin
# #

"envoy.router.cluster_specifier_plugin.golang": "//contrib/golang/router/cluster_specifier/source:config",
# "envoy.router.cluster_specifier_plugin.golang": "//contrib/golang/router/cluster_specifier/source:config",
}
Loading