Skip to content

Commit 51a6439

Browse files
wazir-ahmedwazir-ahmed
committed
test: add shun recovery timing regression coverage
Signed-off-by: Wazir Ahmed <wazir@proxysql.com>
1 parent 4742050 commit 51a6439

2 files changed

Lines changed: 145 additions & 0 deletions

File tree

‎test/tap/groups/groups.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@
231231
"reg_test_5233_set_warning-t" : [ "legacy-g1","mysql-auto_increment_delay_multiplex=0-g1","mysql-multiplexing=false-g1","mysql-query_digests=0-g1","mysql-query_digests_keep_comment=1-g1","mysql84-g1","mysql90-g1","mysql95-g1" ],
232232
"reg_test_5306-show_warnings_with_comment-t" : [ "legacy-g2","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql84-g2","mysql90-g2","mysql95-g2" ],
233233
"reg_test_5389-flush_logs_no_drop-t" : [ "legacy-g4","mysql84-g4","mysql90-g4","mysql95-g4" ],
234+
"reg_test_5546-shun_recovery-t" : [ "legacy-g4","mysql84-g4" ],
234235
"reg_test__ssl_client_busy_wait-t" : [ "legacy-g2","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g2","mysql90-g2","mysql95-g2" ],
235236
"reg_test_com_change_user_malformed_packet-t" : [ "mysql84-g1","mysql95-g1" ],
236237
"reg_test_compression_split_packets-t" : [ "legacy-g2","mysql-auto_increment_delay_multiplex=0-g2","mysql-multiplexing=false-g2","mysql-query_digests=0-g2","mysql-query_digests_keep_comment=1-g2","mysql84-g2","mysql90-g2","mysql95-g2" ],

‎test/tap/tests/reg_test_5546-shun_recovery-t.cpp‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
/**
2+
* @file reg_test_5546-shun_recovery-t.cpp
3+
* @brief Regression test for #5546: monitor ping shun must not recover before the next ping window.
4+
*
5+
* The test creates a dummy unreachable backend, configures monitor ping failures to shun it after one
6+
* failed check, and uses a recovery time much shorter than the monitor ping interval. After the server is
7+
* shunned, a query routed to the dummy hostgroup exercises the server-selection unshun path. The server must
8+
* remain SHUNNED instead of becoming ONLINE before the monitor has had enough time to re-check it.
9+
*/
10+
11+
#include <string>
12+
#include <unistd.h>
13+
14+
#include "mysql.h"
15+
16+
#include "command_line.h"
17+
#include "tap.h"
18+
#include "utils.h"
19+
20+
using std::string;
21+
22+
constexpr int kHostgroup = 5546;
23+
constexpr int kBackendPort = 3305;
24+
constexpr int kMonitorPingIntervalMs = 5000;
25+
constexpr int kShunRecoverySec = 1;
26+
constexpr int kPollIntervalMs = 100;
27+
constexpr int kShunWaitTimeoutMs = 10000;
28+
constexpr const char* kBackendHost = "127.0.0.1";
29+
30+
string get_runtime_server_status(MYSQL* admin) {
31+
const string query =
32+
"SELECT COALESCE((SELECT status FROM runtime_mysql_servers WHERE hostgroup_id=" +
33+
std::to_string(kHostgroup) + " AND hostname='" + kBackendHost + "' AND port=" +
34+
std::to_string(kBackendPort) + "), 'MISSING')";
35+
ext_val_t<string> status { mysql_query_ext_val(admin, query, string { "ERROR" }) };
36+
if (status.err != EXIT_SUCCESS) {
37+
diag("Failed to fetch runtime status: %s", get_ext_val_err(admin, status).c_str());
38+
return "ERROR";
39+
}
40+
return status.val;
41+
}
42+
43+
bool wait_for_server_status(MYSQL* admin, const string& expected_status, int timeout_ms) {
44+
for (int waited_ms = 0; waited_ms <= timeout_ms; waited_ms += kPollIntervalMs) {
45+
const string status = get_runtime_server_status(admin);
46+
if (status == expected_status) {
47+
diag("Server reached expected status hostgroup=%d host=%s port=%d status=%s waited_ms=%d",
48+
kHostgroup, kBackendHost, kBackendPort, status.c_str(), waited_ms);
49+
return true;
50+
}
51+
usleep(kPollIntervalMs * 1000);
52+
}
53+
54+
diag("Timed out waiting for status %s; last status=%s", expected_status.c_str(), get_runtime_server_status(admin).c_str());
55+
return false;
56+
}
57+
58+
void cleanup(MYSQL* admin) {
59+
const string delete_query =
60+
"DELETE FROM mysql_servers WHERE hostgroup_id=" + std::to_string(kHostgroup) +
61+
" AND hostname='" + kBackendHost + "' AND port=" + std::to_string(kBackendPort);
62+
run_q(admin, delete_query.c_str());
63+
run_q(admin, "LOAD MYSQL SERVERS TO RUNTIME");
64+
run_q(admin, "SET mysql-monitor_enabled='true'");
65+
run_q(admin, "SET mysql-monitor_ping_interval='8000'");
66+
run_q(admin, "SET mysql-monitor_ping_max_failures='3'");
67+
run_q(admin, "SET mysql-shun_recovery_time_sec='10'");
68+
run_q(admin, "LOAD MYSQL VARIABLES TO RUNTIME");
69+
}
70+
71+
void setup(MYSQL* admin) {
72+
cleanup(admin);
73+
run_q(admin, "SET mysql-monitor_enabled='true'");
74+
run_q(admin, ("SET mysql-monitor_ping_interval='" + std::to_string(kMonitorPingIntervalMs) + "'").c_str());
75+
run_q(admin, "SET mysql-monitor_ping_max_failures='1'");
76+
run_q(admin, ("SET mysql-shun_recovery_time_sec='" + std::to_string(kShunRecoverySec) + "'").c_str());
77+
run_q(admin, "LOAD MYSQL VARIABLES TO RUNTIME");
78+
79+
const string insert_query =
80+
"INSERT INTO mysql_servers (hostgroup_id, hostname, port, status, max_connections, comment) VALUES (" +
81+
std::to_string(kHostgroup) + ", '" + kBackendHost + "', " + std::to_string(kBackendPort) +
82+
", 'ONLINE', 100, 'reg_test_5546')";
83+
run_q(admin, insert_query.c_str());
84+
run_q(admin, "LOAD MYSQL SERVERS TO RUNTIME");
85+
}
86+
87+
int main(int argc, char** argv) {
88+
plan(2);
89+
90+
CommandLine cl;
91+
if (cl.getEnv()) {
92+
diag("Failed to get the required environmental variables.");
93+
return EXIT_FAILURE;
94+
}
95+
96+
MYSQL* admin = mysql_init(NULL);
97+
MYSQL* proxy = mysql_init(NULL);
98+
99+
if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, nullptr, cl.admin_port, nullptr, 0)) {
100+
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(admin));
101+
return EXIT_FAILURE;
102+
}
103+
104+
if (!mysql_real_connect(proxy, cl.host, cl.username, cl.password, nullptr, cl.port, nullptr, 0)) {
105+
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy));
106+
mysql_close(admin);
107+
return EXIT_FAILURE;
108+
}
109+
110+
setup(admin);
111+
112+
const bool shunned = wait_for_server_status(admin, "SHUNNED", kShunWaitTimeoutMs);
113+
ok(shunned, "Unreachable backend is SHUNNED after one monitor ping failure");
114+
115+
bool stayed_shunned = shunned;
116+
if (shunned) {
117+
const string routed_query = "DO /* ;hostgroup=" + std::to_string(kHostgroup) + " */ 1";
118+
const int sleep_ms = kShunRecoverySec * 1000;
119+
for (int waited_ms = 0; waited_ms < kMonitorPingIntervalMs; waited_ms += sleep_ms) {
120+
sleep(kShunRecoverySec);
121+
122+
const int query_rc = mysql_query(proxy, routed_query.c_str());
123+
diag("Routed query after recovery window returned rc=%d errno=%d error=\"%s\" waited_ms=%d",
124+
query_rc, mysql_errno(proxy), mysql_error(proxy), waited_ms + sleep_ms);
125+
126+
const string status = get_runtime_server_status(admin);
127+
diag("Observed status during one ping interval waited_ms=%d status=%s",
128+
waited_ms + sleep_ms, status.c_str());
129+
stayed_shunned = status == "SHUNNED";
130+
if (!stayed_shunned) {
131+
diag("Server changed status after recovery-window selection status=%s", status.c_str());
132+
break;
133+
}
134+
}
135+
}
136+
ok(stayed_shunned, "SHUNNED backend does not become ONLINE before the next monitor ping window (#5546)");
137+
138+
cleanup(admin);
139+
140+
mysql_close(proxy);
141+
mysql_close(admin);
142+
143+
return exit_status();
144+
}

0 commit comments

Comments
 (0)