Merge pull request #5637 from sysown/v3.0_new_zstd

feat: add mysql-zstd_compression_level variable

Author: renecannao

.gitignore

@@ -211,5 +211,9 @@ scripts/mcp/DiscoveryAgent/ClaudeCode_Headless/tmp/
 test/tap/tests/test_cluster_sync_config/test_cluster_sync_nomonitor/cluster_sync_node_stderr.txt
 test/tap/tests/test_cluster_sync_config/test_cluster_sync_withmonitor/cluster_sync_node_stderr.txt
 
+# unit test binaries (built but not tracked)
+test/tap/tests/unit/*-t
+
 # test-scripts runtime dependencies (created by run-tests-isolated.bash)
 test-scripts/deps/
+.worktrees/

CLAUDE.md

@@ -55,22 +55,39 @@ The same codebase produces three product tiers via feature flags:
 
 Tests use TAP (Test Anything Protocol) with Docker-based backend infrastructure.
 
+### Running TAP tests — DO NOT manually set up Docker containers
+
+**ALWAYS use `run-tests-isolated.bash`**. It handles infrastructure setup, ProxySQL start, test execution, and cleanup. Never manually create Docker networks, start containers, or run init scripts — the runner does all of that.
+
 ```bash
-# Build and run all TAP tests
-make build_tap_tests
-cd test/tap && make
+# Set up infrastructure (backends + ProxySQL container)
+WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=mysql84-g1 test/infra/control/ensure-infras.bash
 
-# Run specific test groups
-cd test/tap/tests && make
-cd test/tap/tests_with_deps && make
+# Run all tests for a TAP group
+WORKSPACE=$(pwd) INFRA_ID=dev-$USER TAP_GROUP=mysql84-g1 test/infra/control/run-tests-isolated.bash
 
-# Test infrastructure (Docker environments)
-# Located in test/infra/ with docker-compose configs for:
-# mysql57, mysql84, mariadb10, pgsql16, pgsql17, clickhouse23, etc.
+# Build test binaries first (requires proxysql binary)
+make build_tap_tests          # release
+make build_tap_test_debug     # debug
 ```
 
+Available TAP groups are defined in `test/tap/groups/groups.json`. Group names follow the pattern `<infra>-g<N>` (e.g., `mysql84-g1`, `legacy-g2`, `pgsql16-g1`).
+
+### DO NOT
+
+- **DO NOT** manually create Docker networks (`docker network create`)
+- **DO NOT** manually start containers (`docker start`, `docker run`)
+- **DO NOT** run `docker-compose-init.bash` directly — use `ensure-infras.bash`
+- **DO NOT** symlink build artifacts between worktrees — build in each worktree separately
+- **DO NOT** copy source files between worktrees or repos
+- **DO NOT** run `cd test/tap/tests && make` and expect tests to pass without infrastructure
+
+### Test file conventions
+
 Test files follow the naming pattern `test_*.cpp` or `*-t.cpp` in `test/tap/tests/`.
 
+Test binaries are built via a pattern rule in `test/tap/tests/Makefile`: `make <testname>-t` compiles `<testname>-t.cpp` into `<testname>-t`. No special Makefile target is needed for new tests — just add the `.cpp` file and register it in `groups.json`.
+
 ## Architecture
 
 ### Build Pipeline

include/MySQL_Thread.h

@@ -488,6 +488,7 @@ class MySQL_Threads_Handler
 		bool default_reconnect;
 		bool have_compress;
 		int protocol_compression_level;
+		int zstd_compression_level;
 		bool have_ssl;
 		bool multiplexing;
 //		bool stmt_multiplexing;

include/proxysql_structs.h

@@ -1296,6 +1296,7 @@ __thread int mysql_thread___poll_timeout_on_failure;
 __thread bool mysql_thread___connection_warming;
 __thread bool mysql_thread___have_compress;
 __thread int mysql_thread___protocol_compression_level;
+__thread int mysql_thread___zstd_compression_level;
 __thread bool mysql_thread___have_ssl;
 __thread bool mysql_thread___multiplexing;
 __thread bool mysql_thread___log_unhealthy_connections;
@@ -1629,6 +1630,7 @@ extern __thread int mysql_thread___poll_timeout_on_failure;
 extern __thread bool mysql_thread___connection_warming;
 extern __thread bool mysql_thread___have_compress;
 extern __thread int mysql_thread___protocol_compression_level;
+extern __thread int mysql_thread___zstd_compression_level;
 extern __thread bool mysql_thread___have_ssl;
 extern __thread bool mysql_thread___multiplexing;
 extern __thread bool mysql_thread___log_unhealthy_connections;

lib/MySQL_Protocol.cpp

@@ -2300,7 +2300,7 @@ void MySQL_Protocol::PPHR_SetConnAttrs(MyProt_tmp_auth_vars& vars1, account_deta
 	const uint8_t zstd_compression_level =
 		(vars1.zstd_compression_level > 0 && vars1.zstd_compression_level <= ZSTD_maxCLevel())
 			? vars1.zstd_compression_level
-			: static_cast<uint8_t>(std::min<int>(ZSTD_maxCLevel(), std::max<int>(1, mysql_thread___protocol_compression_level)));
+			: static_cast<uint8_t>(mysql_thread___zstd_compression_level);
 
 	myconn->options.compression_zstd = false;
 	myconn->options.zstd_compression_level = 0;

lib/MySQL_Thread.cpp

@@ -30,6 +30,7 @@ using json = nlohmann::json;
 #include "MySQL_Resolution.h"
 
 #include <fcntl.h>
+#include <zstd.h>
 
 using std::vector;
 using std::function;
@@ -528,6 +529,7 @@ static char * mysql_thread_variables_names[]= {
 	(char *)"evaluate_replication_lag_on_servers_load",
 	(char *)"proxy_protocol_networks",
 	(char *)"protocol_compression_level",
+	(char *)"zstd_compression_level",
 	(char *)"ignore_min_gtid_annotations",
 	(char *)"fast_forward_grace_close_ms",
 #ifdef PROXYSQLFFTO
@@ -1453,6 +1455,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() {
 	variables.log_mysql_warnings_enabled=false;
 	variables.data_packets_history_size=0;
 	variables.protocol_compression_level=3;
+	variables.zstd_compression_level=3;
 	variables.ignore_min_gtid_annotations=false;
 	// status variables
 	status_variables.mirror_sessions_current=0;
@@ -2669,6 +2672,7 @@ char ** MySQL_Threads_Handler::get_variables_list() {
 		VariablesPointers_int["handle_warnings"]			   = make_tuple(&variables.handle_warnings,				  0,			  1, false);
 		VariablesPointers_int["evaluate_replication_lag_on_servers_load"] = make_tuple(&variables.evaluate_replication_lag_on_servers_load, 0, 1, false);
 		VariablesPointers_int["protocol_compression_level"]    = make_tuple(&variables.protocol_compression_level,   -1,              9, false);
+		VariablesPointers_int["zstd_compression_level"]       = make_tuple(&variables.zstd_compression_level,       1,              ZSTD_maxCLevel(), false);
 
 		// logs
 		VariablesPointers_int["auditlog_filesize"]     = make_tuple(&variables.auditlog_filesize,    1024*1024, 1*1024*1024*1024, false);
@@ -4733,6 +4737,7 @@ void MySQL_Thread::refresh_variables() {
 	REFRESH_VARIABLE_INT(poll_timeout_on_failure);
 	REFRESH_VARIABLE_BOOL(have_compress);
 	REFRESH_VARIABLE_INT(protocol_compression_level);
+	REFRESH_VARIABLE_INT(zstd_compression_level);
 	REFRESH_VARIABLE_BOOL(have_ssl);
 	REFRESH_VARIABLE_BOOL(multiplexing);
 	REFRESH_VARIABLE_BOOL(log_unhealthy_connections);
@@ -4817,6 +4822,7 @@ MySQL_Thread::MySQL_Thread() {
 	mysql_thread___ssl_p2s_crlpath=NULL;
 
 	mysql_thread___protocol_compression_level=3;
+	mysql_thread___zstd_compression_level=3;
 
 	last_maintenance_time=0;
 	last_move_to_idle_thread_time=0;

lib/mysql_data_stream.cpp

@@ -29,8 +29,8 @@ static int get_zstd_compression_level(const MySQL_Connection* myconn) {
 	if (zstd_level > 0 && zstd_level <= ZSTD_maxCLevel()) {
 		return zstd_level;
 	}
-	if (mysql_thread___protocol_compression_level > 0 && mysql_thread___protocol_compression_level <= ZSTD_maxCLevel()) {
-		return mysql_thread___protocol_compression_level;
+	if (mysql_thread___zstd_compression_level > 0 && mysql_thread___zstd_compression_level <= ZSTD_maxCLevel()) {
+		return mysql_thread___zstd_compression_level;
 	}
 	return ZSTD_CLEVEL_DEFAULT;
 }

test/tap/groups/groups.json

@@ -99,6 +99,8 @@
   "mysql-test_malformed_packet-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ],
   "mysql-test_ssl_CA-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ],
   "mysql-watchdog_test-t" : [ "legacy-g4","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g4","mysql90-g4","mysql95-g4" ],
+  "mysql-zstd_compression_level-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ],
+  "mysql-zstd_compression_level_libmysql-t" : [ "mysql84-g1","mysql90-g1","mysql95-g1" ],
   "mysql_encode_unit-t" : [ "unit-tests-g1" ],
   "mysql_error_classifier_unit-t" : [ "unit-tests-g1" ],
   "mysql_hostgroup_attributes-servers_defaults-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ],

test/tap/tests/Makefile

@@ -263,6 +263,9 @@ mysql_reconnect_libmariadb-t: mysql_reconnect.cpp $(TAP_LDIR)/libtap.so
 mysql_reconnect_libmysql-t: mysql_reconnect.cpp $(TAP_LDIR)/libtap_mysql8.a
 	$(CXX) -DLIBMYSQL_HELPER8 -DDISABLE_WARNING_COUNT_LOGGING $< -I$(TEST_MYSQL8_IDIR) -I$(TEST_MYSQL8_EDIR) -L$(TEST_MYSQL8_LDIR) -lmysqlclient -ltap_mysql8 -lresolv $(CUSTOMARGS) -o $@
 
+mysql-zstd_compression_level_libmysql-t: mysql-zstd_compression_level-t.cpp $(TAP_LDIR)/libtap_mysql8.a
+	$(CXX) -DLIBMYSQL_HELPER8 -DDISABLE_WARNING_COUNT_LOGGING $< -I$(TEST_MYSQL8_IDIR) -I$(TEST_MYSQL8_EDIR) -L$(TEST_MYSQL8_LDIR) -lmysqlclient -ltap_mysql8 -lresolv $(CUSTOMARGS) -o $@
+
 fast_forward_grace_close_libmysql-t: fast_forward_grace_close.cpp $(TAP_LDIR)/libtap_mysql8.a
 	$(CXX) -DLIBMYSQL_HELPER8 -DDISABLE_WARNING_COUNT_LOGGING $< -I$(TEST_MYSQL8_IDIR) -I$(TEST_MYSQL8_EDIR) -L$(TEST_MYSQL8_LDIR) -lmysqlclient -ltap_mysql8 -lresolv $(CUSTOMARGS) -o $@