Merge pull request #5708 from sysown/issue-5686-galera-vars

feat: add Galera wsrep variables and MariaDB SET STATEMENT ... FOR support

Author: renecannao

include/proxysql_structs.h

@@ -250,6 +250,8 @@ enum mysql_variable_name {
 	SQL_UNIQUE_CHECKS,
 	SQL_WSREP_OSU_METHOD,
 	SQL_WSREP_SYNC_WAIT,
+	SQL_WSREP_TRX_FRAGMENT_SIZE,
+	SQL_WSREP_TRX_FRAGMENT_UNIT,
 	SQL_NAME_LAST_HIGH_WM,
 };
 
@@ -1854,6 +1856,8 @@ mysql_variable_st mysql_tracked_variables[] {
 	{ SQL_UNIQUE_CHECKS,              SETTING_VARIABLE, true,  false, false, true,  (char *)"unique_checks",              NULL, (char *)"" , false} ,
 	{ SQL_WSREP_OSU_METHOD,           SETTING_VARIABLE, true,  false, false, false, (char *)"wsrep_osu_method",           NULL, (char *)"" , false} ,
 	{ SQL_WSREP_SYNC_WAIT,			  SETTING_VARIABLE, false, false, true,  false, (char *)"wsrep_sync_wait",			  (char *)"wsrep_sync_wait", (char *)"0" , false} ,
+	{ SQL_WSREP_TRX_FRAGMENT_SIZE,   SETTING_VARIABLE, false, false, true,  false, (char *)"wsrep_trx_fragment_size",    NULL, (char *)"0" , false} ,
+	{ SQL_WSREP_TRX_FRAGMENT_UNIT,   SETTING_VARIABLE, true,  false, false, false, (char *)"wsrep_trx_fragment_unit",    NULL, (char *)"" , false} ,
 	/*
 	variables that will need input validation:
 	binlog_row_image

lib/MySQL_Session.cpp

@@ -132,7 +132,8 @@ static const std::set<std::string> mysql_variables_numeric = {
 	"sql_select_limit",
 	"timestamp",
 	"tmp_table_size",
-	"wsrep_sync_wait"
+	"wsrep_sync_wait",
+	"wsrep_trx_fragment_size"
 };
 static const std::set<std::string> mysql_variables_strings = {
 	"default_storage_engine",
@@ -143,6 +144,7 @@ static const std::set<std::string> mysql_variables_strings = {
 	"log_slow_filter",
 	"optimizer_switch",
 	"wsrep_osu_method",
+	"wsrep_trx_fragment_unit",
 };
 
 #include "proxysql_find_charset.h"
@@ -2686,6 +2688,10 @@ bool MySQL_Session::handler_again___status_SETTING_GENERIC_VARIABLE(int *_rc, co
 					||
 					(myerr == 1193) // variable is not found
 					||
+					(myerr == 1210) // Incorrect arguments to SET
+					||
+					(myerr == 1231) // Variable can't be set to the value
+					||
 					(myerr == 1651) // Query cache is disabled
 					||
 					(is_perm_track_err(myerr, var_name)) // Special permitted tracking errors (~= '1193')
@@ -6515,6 +6521,11 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
 			if (pos != nq.npos) {
 				nq.erase(pos + 1); // remove trailing spaces and semicolumns
 			}
+			// detect MariaDB SET STATEMENT ... FOR syntax
+			// pass through to backend without hostgroup locking
+			if (strncasecmp(nq.c_str(), (char *)"SET STATEMENT ", 14) == 0 && strcasestr(nq.c_str(), (char *)" FOR ")) {
+				return false;
+			}
 			if (
 				(
 					match_regexes && (match_regexes[1]->match(dig))

test/README.md

@@ -54,16 +54,24 @@ Common test groups (defined in `tap/groups/groups.json`):
 
 ## Prerequisites
 
-1. Build the CI base image (one-time setup):
+1. Obtain the CI base image (one-time setup):
+
+   Pull from GHCR if available:
+   ```bash
+   docker pull ghcr.io/sysown/proxysql-ci-base:latest
+   docker tag ghcr.io/sysown/proxysql-ci-base:latest proxysql-ci-base:latest
+   ```
+
+   Or build locally:
    ```bash
    cd test/infra/docker-base
    docker build --network host -t proxysql-ci-base:latest .
    cd ../../../
    ```
 
-2. Build ProxySQL and TAP tests:
+2. Build ProxySQL and TAP tests in debug mode:
    ```bash
-   make -j$(nproc) && make -j$(nproc) build_tap_test
+   make debug -j$(nproc) && make -j$(nproc) build_tap_test_debug
    ```
 
 ## Where logs actually live after a run

test/tap/groups/groups.json

@@ -400,6 +400,7 @@
   "test_session_status_flags-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ],
   "test_set_character_results-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4","set_parser_algorithm_3-g1" ],
   "test_set_collation-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4","set_parser_algorithm_3-g1" ],
+  "test_set_statement_for-t" : [ "legacy-g2","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g2","mysql90-g2","mysql95-g2" ],
   "test_simple_embedded_HTTP_server-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ],
   "test_sqlite3_from_unixtime-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ],
   "test_sqlite3_pass_exts-t" : [ "legacy-g9","mysql-auto_increment_delay_multiplex=0-g4","mysql-multiplexing=false-g4","mysql-query_digests=0-g4","mysql-query_digests_keep_comment=1-g4","mysql84-g9","mysql90-g4","mysql95-g4" ],

test/tap/tests/generate_set_session_csv.cpp

@@ -303,6 +303,11 @@ int main() {
 	vars["wsrep_osu_method"] = std::make_unique<variable>("wsrep_osu_method", true, false, false);
 	add_values_and_quotes("wsrep_osu_method", {"TOI","RSU"});
 
+	vars["wsrep_trx_fragment_size"] = std::make_unique<variable>("wsrep_trx_fragment_size", false, true, false);
+	vars["wsrep_trx_fragment_size"]->add(int_values_small, 100);
+	vars["wsrep_trx_fragment_unit"] = std::make_unique<variable>("wsrep_trx_fragment_unit", false, false, false);
+	add_values_and_quotes("wsrep_trx_fragment_unit", {"BYTES", "ROWS", "STATEMENTS"});
+
 	vars["sql_quote_show_create"] = std::make_unique<variable>("sql_quote_show_create", true, false, true);
 	vars["sql_quote_show_create"]->add(bool_values);
 

test/tap/tests/set_testing-240-t.cpp

@@ -49,6 +49,7 @@ int uniquequeries=0;
 int histograms=-1;
 
 bool is_mariadb = false;
+bool is_cluster = false;
 unsigned int g_connect_OK=0;
 unsigned int g_connect_ERR=0;
 unsigned int g_select_OK=0;
@@ -256,6 +257,21 @@ void * my_conn_thread(void *arg) {
 					vars[el.key()] = el.value();
 				}
 			}
+			else if (el.key() == "wsrep_sync_wait") {
+				if (is_cluster) {
+					vars[el.key()] = el.value();
+				}
+			}
+			else if (el.key() == "wsrep_trx_fragment_size") {
+				if (is_cluster) {
+					vars[el.key()] = el.value();
+				}
+			}
+			else if (el.key() == "wsrep_trx_fragment_unit") {
+				if (is_cluster) {
+					vars[el.key()] = el.value();
+				}
+			}
 			else {
 				vars[el.key()] = el.value();
 			}
@@ -558,7 +574,7 @@ int main(int argc, char *argv[]) {
 		MYSQL_QUERY(proxysql_admin, update.c_str());
 		MYSQL_QUERY(proxysql_admin, "LOAD MYSQL QUERY RULES TO RUNTIME");
 
-		if (detect_version(cl, is_mariadb) != 0) {
+		if (detect_version(cl, is_mariadb, is_cluster) != 0) {
 			diag("Cannot detect MySQL version");
 			return exit_status();
 		}

test/tap/tests/set_testing-240.h

@@ -21,6 +21,8 @@ const std::vector<std::string> possible_unknown_variables = {
 	"group_replication_consistency",
 	"query_cache_type",
 	"wsrep_osu_method",
+	"wsrep_trx_fragment_size",
+	"wsrep_trx_fragment_unit",
 	"wsrep_sync_wait",
 	"max_execution_time",
 	"optimizer_use_condition_selectivity",
@@ -464,7 +466,7 @@ bool check_session_track_gtids(const std::string& expVal, const std::string& sVa
 	return res;
 }
 
-int detect_version(CommandLine& cl, bool& is_mariadb) {
+int detect_version(CommandLine& cl, bool& is_mariadb, bool& is_cluster) {
 	MYSQL* mysql = mysql_init(NULL);
 	if (!mysql)
 		return 1;
@@ -487,6 +489,15 @@ int detect_version(CommandLine& cl, bool& is_mariadb) {
 		}
 	}
 	mysql_free_result(result);
+	MYSQL_QUERY(mysql, "SHOW VARIABLES LIKE 'wsrep_sync_wait'");
+	result = mysql_store_result(mysql);
+	unsigned long long nr = mysql_num_rows(result);
+	if (nr == 0) {
+		is_cluster = false;
+	} else {
+		is_cluster = true;
+	}
+	mysql_free_result(result);
 	mysql_close(mysql);
 	return 0;
 }

test/tap/tests/set_testing-multi-t.cpp

@@ -165,6 +165,16 @@ void * my_conn_thread(void *arg) {
 					vars[el.key()] = el.value();
 				}
 			}
+			else if (el.key() == "wsrep_trx_fragment_size") {
+				if (is_cluster) {
+					vars[el.key()] = el.value();
+				}
+			}
+			else if (el.key() == "wsrep_trx_fragment_unit") {
+				if (is_cluster) {
+					vars[el.key()] = el.value();
+				}
+			}
 			else if (el.key() == "transaction_read_only") {
 				if (is_mariadb) {
 					vars["tx_read_only"] = el.value();
@@ -287,13 +297,16 @@ void * my_conn_thread(void *arg) {
 					(el.key() == "session_track_gtids" && !check_session_track_gtids(el.value(), s.value(), k.value()))
 				))
 			) {
-				__sync_fetch_and_add(&g_failed, 1);
-				testPassed = false;
-				fprintf(stderr, "Test failed for this case %s->%s.\n\nmysql data %s\n\n proxysql data %s\n\n csv data %s\n\n\n",
+				if (k != mysql_vars.end() && s != proxysql_vars.end() && s.value() == el.value() && k.value() == UNKNOWNVAR) {
+					variables_tested++;
+				} else {
+					__sync_fetch_and_add(&g_failed, 1);
+					testPassed = false;
+					fprintf(stderr, "Test failed for this case %s->%s.\n\nmysql data %s\n\n proxysql data %s\n\n csv data %s\n\n\n",
 						el.value().dump().c_str(), el.key().c_str(), mysql_vars.dump().c_str(), proxysql_vars.dump().c_str(), vars.dump().c_str());
-				ok(testPassed, "mysql connection [%p], thread_id [%lu], command [%s]", mysql, mysql->thread_id, testCases[r2].command.c_str());
-				// In case of failing test, exit completely.
-				exit(EXIT_FAILURE);
+					ok(testPassed, "mysql connection [%p], thread_id [%lu], command [%s]", mysql, mysql->thread_id, testCases[r2].command.c_str());
+					exit(EXIT_FAILURE);
+				}
 			} else {
 				variables_tested++;
 			}

test/tap/tests/set_testing-t.cpp

@@ -172,6 +172,16 @@ void * my_conn_thread(void *arg) {
 					vars[el.key()] = el.value();
 				}
 			}
+			else if (el.key() == "wsrep_trx_fragment_size") {
+				if (is_cluster) {
+					vars[el.key()] = el.value();
+				}
+			}
+			else if (el.key() == "wsrep_trx_fragment_unit") {
+				if (is_cluster) {
+					vars[el.key()] = el.value();
+				}
+			}
 			else if (el.key() == "transaction_read_only") {
 				if (is_mariadb) {
 					vars["tx_read_only"] = el.value();
@@ -325,7 +335,9 @@ void * my_conn_thread(void *arg) {
 					(el.key() == "session_track_gtids" && !check_session_track_gtids(el.value(), s.value(), k.value()))
 				))
 			) {
-				if (el.key() == "wsrep_sync_wait" && k == mysql_vars.end() && (s.value() == el.value())) {
+				if (k != mysql_vars.end() && s != proxysql_vars["conn"].end() && s.value() == el.value() && k.value() == UNKNOWNVAR) {
+					variables_tested++;
+				} else if (el.key() == "wsrep_sync_wait" && k == mysql_vars.end() && (s.value() == el.value())) {
 					variables_tested++;
 				} else {
 					__sync_fetch_and_add(&g_failed, 1);
@@ -336,9 +348,6 @@ void * my_conn_thread(void *arg) {
 							proxysql_vars["conn"].size(), proxysql_vars["conn"].dump(2).c_str(),
 							vars.dump(2).c_str());
 					diag("FAILED FOR: connections mysql[%p] proxysql[%s], thread_id [%lu], command [%s]", mysql, paddress.c_str(), mysql->thread_id, testCases[r2].command.c_str());
-					//ok(testPassed, "connections mysql[%p] proxysql[%s], thread_id [%lu], command [%s]", mysql, paddress.c_str(), mysql->thread_id, testCases[r2].command.c_str());
-					// In case of failing test, exit completely.
-					//exit(EXIT_FAILURE);
 				}
 			} else {
 				variables_tested++;

test/tap/tests/set_testing-t.csv

@@ -68,6 +68,10 @@
 "set max_join_size=18446744073709551615", "{'max_join_size':'18446744073709551615'}"
 "set wsrep_sync_wait=1", "{'wsrep_sync_wait':'1'}"
 "set wsrep_sync_wait=0", "{'wsrep_sync_wait':'0'}"
+"set wsrep_trx_fragment_size=1000", "{'wsrep_trx_fragment_size':'1000'}"
+"set wsrep_trx_fragment_size=0", "{'wsrep_trx_fragment_size':'0'}"
+"set wsrep_trx_fragment_unit='ROWS'", "{'wsrep_trx_fragment_unit':'ROWS'}"
+"set wsrep_trx_fragment_unit='BYTES'", "{'wsrep_trx_fragment_unit':'BYTES'}"
 "set group_concat_max_len=2048", "{'group_concat_max_len':'2048'}"
 "set group_concat_max_len=4096", "{'group_concat_max_len':'4096'}"
 "set tx_isolation='READ-COMMITTED', group_concat_max_len=4096", "{'transaction_isolation':'READ-COMMITTED', 'group_concat_max_len':'4096'}"