Skip to content

Commit 9116d39

Browse files
renecannaorenecannao
authored
Merge pull request #5944 from sysown/test/passthrough-auth-pool-reuse
test(passthrough-auth): adversarial pool-reuse (force-new probe) test
2 parents 20bbe92 + e0d4b14 commit 9116d39

2 files changed

Lines changed: 364 additions & 0 deletions

File tree

‎test/tap/groups/groups.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,7 @@
408408
"test_passthrough_auth_invalidation-t" : [ "mysql84-g4","mysql90-g4","mysql95-g4" ],
409409
"test_passthrough_auth_metrics-t" : [ "mysql84-g4","mysql90-g4","mysql95-g4" ],
410410
"test_passthrough_auth_misconfig_warning-t" : [ "no-infra-g1" ],
411+
"test_passthrough_auth_pool_reuse-t" : [ "mysql84-g4","mysql90-g4","mysql95-g4" ],
411412
"test_passthrough_auth_ratelimit-t" : [ "mysql84-g4","mysql90-g4","mysql95-g4" ],
412413
"test_passthrough_auth_security-t" : [ "mysql84-g4","mysql90-g4","mysql95-g4" ],
413414
"test_passthrough_auth_unknown_user-t" : [ "mysql84-g4","mysql90-g4","mysql95-g4" ],

‎test/tap/tests/test_passthrough_auth_pool_reuse-t.cpp‎

Lines changed: 363 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,363 @@
1+
/**
2+
* @file test_passthrough_auth_pool_reuse-t.cpp
3+
* @brief Adversarial test for the pass-through "force-new" probe invariant.
4+
*
5+
* This test guards the single load-bearing correctness claim of the
6+
* pass-through design: a pass-through backend probe MUST acquire a
7+
* FRESH backend connection (get_MyConn_from_pool(..., ff=true)) and
8+
* actually authenticate the borrowed credential against the backend --
9+
* it must NEVER be satisfied by a connection already sitting idle in the
10+
* pool.
11+
*
12+
* Why this matters
13+
* ----------------
14+
* ProxySQL's connection pool reuses backend connections keyed by
15+
* USERNAME only: requires_CHANGE_USER() compares the username and
16+
* match_tracked_options() compares client capability flags -- neither
17+
* checks the password. So a warm pooled connection authenticated for
18+
* `alice` with the CORRECT password X is, structurally, a candidate to
19+
* satisfy a request for `alice`. If the probe path ever reused such a
20+
* connection, a client presenting `alice` + a WRONG password Y would be
21+
* handed the already-authenticated connection and let in WITHOUT the
22+
* wrong password ever being validated -- a silent authentication
23+
* bypass. The `ff=true` (force-new) acquisition in
24+
* handler_again___status_AUTHENTICATING_BACKEND_FOR_CLIENT is what
25+
* prevents this; see doc/internal/passthrough_authentication.md §6.3.
26+
*
27+
* The existing test_passthrough_auth_e2e-t proves wrong passwords are
28+
* rejected on a COLD cache. It does NOT prove they are rejected when a
29+
* warm, correctly-authenticated pooled connection for the same user is
30+
* available to be (mis)reused. That warm-pool window is exactly where a
31+
* force-new regression would hide, and it is what this test constructs
32+
* deliberately.
33+
*
34+
* Attack shape reproduced
35+
* -----------------------
36+
* 1. Legitimate client connects as U with the REAL password over TLS.
37+
* The probe succeeds, U's credential is cached, and a backend
38+
* connection authenticated as U/REAL is returned to the pool.
39+
* 2. Flush ONLY the pass-through credential cache for U (the pooled
40+
* backend connection is untouched). Now the next connect for U
41+
* cannot fast-path on the cache -- it MUST re-probe -- while a warm
42+
* U/REAL connection is sitting free in the pool.
43+
* 3. Attacker connects as U with a WRONG password over TLS.
44+
* Correct behavior: force-new probe -> backend rejects (1045) ->
45+
* generic access denied. A force-new regression would instead reuse
46+
* the warm U/REAL connection and return errno 0 (BYPASS).
47+
* 4. Positive control: U with the REAL password still succeeds, proving
48+
* the deny in step 3 was password-specific, not a blanket breakage.
49+
*
50+
* Requires a MySQL 8+ backend (caching_sha2_password). Registered in the
51+
* mysql84+/mysql9x groups alongside the other pass-through e2e tests.
52+
*
53+
* Spec reference: doc/internal/passthrough_authentication.md §6.3, §11.1
54+
*/
55+
#include <climits>
56+
#include <cstring>
57+
#include <string>
58+
#include <vector>
59+
60+
#include "mysql.h"
61+
#include "mysqld_error.h"
62+
63+
#include "tap.h"
64+
#include "command_line.h"
65+
#include "utils.h"
66+
67+
using std::string;
68+
using std::vector;
69+
70+
static constexpr const char* TEST_USER = "tap_passthrough_reuse_user";
71+
static constexpr const char* TEST_BACKEND_PW = "p4ssth0ugh-r3us3-r34l!";
72+
static constexpr const char* WRONG_PW = "definitely-not-the-real-password";
73+
74+
/** @brief MySQL 8+ backend hostgroup in the TAP infra (mirrors e2e test). */
75+
static uint32_t MYSQL8_HG = get_env_int("TAP_MYSQL8_BACKEND_HG", 30);
76+
77+
/** @brief Run a query; diag on failure. */
78+
static int do_query(MYSQL* mysql, const string& q) {
79+
if (mysql_query(mysql, q.c_str())) {
80+
diag("Query failed: %s -- %s", q.c_str(), mysql_error(mysql));
81+
return EXIT_FAILURE;
82+
}
83+
return EXIT_SUCCESS;
84+
}
85+
86+
/** @brief Row count of stats_mysql_passthrough_auth_cache (-1 on error). */
87+
static int cache_entry_count(MYSQL* admin) {
88+
if (mysql_query(admin, "SELECT COUNT(*) FROM stats_mysql_passthrough_auth_cache")) {
89+
diag("SELECT stats_mysql_passthrough_auth_cache failed: %s", mysql_error(admin));
90+
return -1;
91+
}
92+
MYSQL_RES* res = mysql_store_result(admin);
93+
if (!res) return -1;
94+
MYSQL_ROW row = mysql_fetch_row(res);
95+
int n = (row && row[0]) ? atoi(row[0]) : -1;
96+
mysql_free_result(res);
97+
return n;
98+
}
99+
100+
/** @brief Read a single counter from stats_mysql_passthrough_auth_metrics (-1 on error). */
101+
static long long metric(MYSQL* admin, const char* name) {
102+
const string q =
103+
string("SELECT metric_value FROM stats_mysql_passthrough_auth_metrics "
104+
"WHERE metric_name='") + name + "'";
105+
if (mysql_query(admin, q.c_str())) {
106+
diag("SELECT metric '%s' failed: %s", name, mysql_error(admin));
107+
return -1;
108+
}
109+
MYSQL_RES* res = mysql_store_result(admin);
110+
if (!res) return -1;
111+
MYSQL_ROW row = mysql_fetch_row(res);
112+
long long v = (row && row[0]) ? atoll(row[0]) : -1;
113+
mysql_free_result(res);
114+
return v;
115+
}
116+
117+
/** @brief Sum of ConnFree across servers in a hostgroup (0 if none/err). */
118+
static int free_conns(MYSQL* admin, uint32_t hg) {
119+
const string q =
120+
string("SELECT IFNULL(SUM(ConnFree),0) FROM stats_mysql_connection_pool "
121+
"WHERE hostgroup=") + std::to_string(hg);
122+
if (mysql_query(admin, q.c_str())) {
123+
diag("SELECT ConnFree failed: %s", mysql_error(admin));
124+
return 0;
125+
}
126+
MYSQL_RES* res = mysql_store_result(admin);
127+
if (!res) return 0;
128+
MYSQL_ROW row = mysql_fetch_row(res);
129+
int n = (row && row[0]) ? atoi(row[0]) : 0;
130+
mysql_free_result(res);
131+
return n;
132+
}
133+
134+
/**
135+
* @brief Connect through ProxySQL over TLS with the given credentials.
136+
* Optionally run a trivial query first (to force a backend
137+
* connection to be established and returned to the pool).
138+
* @return mysql_errno (0 on success).
139+
*
140+
* TLS is required: the caching_sha2_password full-auth exchange only
141+
* yields the cleartext to ProxySQL over a secured channel, and
142+
* mysql-passthrough_auth_require_tls defaults to 'true'. Same CLIENT_SSL
143+
* + mysql_ssl_set(all-NULL) pattern as test_passthrough_auth_e2e-t.
144+
*/
145+
static unsigned int try_connect(const CommandLine& cl, const char* user,
146+
const char* pass, bool run_query) {
147+
MYSQL* m = mysql_init(NULL);
148+
if (!m) return UINT_MAX;
149+
mysql_options(m, MYSQL_DEFAULT_AUTH, "caching_sha2_password");
150+
mysql_ssl_set(m, NULL, NULL, NULL, NULL, NULL);
151+
const MYSQL* res = mysql_real_connect(
152+
m, cl.host, user, pass, NULL, cl.port, NULL, CLIENT_SSL);
153+
unsigned int err = res ? 0 : mysql_errno(m);
154+
if (!res) {
155+
diag("Frontend connect user='%s' pass='%s' failed: errno=%u msg='%s'",
156+
user, pass, err, mysql_error(m));
157+
} else if (run_query) {
158+
/* Establish an actual backend connection so a warm one is pooled. */
159+
if (mysql_query(m, "SELECT 1")) {
160+
diag("warm-up SELECT 1 failed: %s", mysql_error(m));
161+
} else {
162+
MYSQL_RES* r = mysql_store_result(m);
163+
if (r) mysql_free_result(r);
164+
}
165+
}
166+
mysql_close(m);
167+
return err;
168+
}
169+
170+
int main() {
171+
CommandLine cl;
172+
173+
/*
174+
* Plan:
175+
* Setup (6): backend conn, admin conn, backend user, empty-pw row,
176+
* passthrough enabled, cache starts empty.
177+
* Body (11):
178+
* [warm] (2) legit connect+query ok; cache populated (n=1)
179+
* [flush] (2) per-user cache flush ok; cache empty (n=0)
180+
* [pre] (1) a warm backend connection is free in the pool
181+
* [ATTACK](1) wrong-pw connect DENIED (1045) -- force-new invariant
182+
* [corr] (2) probes_failed_credentials +1; probes_ok unchanged
183+
* [corr] (1) cache still empty after wrong-pw (n=0)
184+
* [ctrl] (2) real-pw connect still succeeds; cache repopulated
185+
* Cleanup (2): DROP USER; mysql_users + variables restored.
186+
* Total: 6 + 11 + 2 = 19.
187+
*/
188+
plan(19);
189+
190+
if (cl.getEnv()) {
191+
diag("CommandLine getEnv() failed");
192+
return exit_status();
193+
}
194+
195+
/* -------- backend & admin connections -------- */
196+
MYSQL* backend = mysql_init(NULL);
197+
const MYSQL* bcr = mysql_real_connect(
198+
backend, cl.mysql_host, cl.mysql_username, cl.mysql_password,
199+
NULL, cl.mysql_port, NULL, 0);
200+
ok(bcr != NULL, "Connected to backend MySQL at %s:%d", cl.mysql_host, cl.mysql_port);
201+
if (!bcr) { mysql_close(backend); return exit_status(); }
202+
203+
MYSQL* admin = mysql_init(NULL);
204+
const MYSQL* acr = mysql_real_connect(
205+
admin, cl.admin_host, cl.admin_username, cl.admin_password,
206+
NULL, cl.admin_port, NULL, 0);
207+
ok(acr != NULL, "Connected to ProxySQL admin at %s:%d", cl.admin_host, cl.admin_port);
208+
if (!acr) { mysql_close(backend); mysql_close(admin); return exit_status(); }
209+
210+
/* -------- backend user with caching_sha2_password -------- */
211+
do_query(backend, string("DROP USER IF EXISTS '") + TEST_USER + "'@'%'");
212+
const string create_user =
213+
string("CREATE USER '") + TEST_USER + "'@'%' "
214+
"IDENTIFIED WITH 'caching_sha2_password' BY '" + TEST_BACKEND_PW + "'";
215+
bool user_ok =
216+
(do_query(backend, create_user) == EXIT_SUCCESS) &&
217+
(do_query(backend, string("GRANT SELECT ON *.* TO '") + TEST_USER + "'@'%'") == EXIT_SUCCESS);
218+
ok(user_ok, "Backend user '%s' provisioned with caching_sha2_password", TEST_USER);
219+
220+
/* -------- mysql_users empty-password row -------- */
221+
do_query(admin, string("DELETE FROM mysql_users WHERE username='") + TEST_USER + "'");
222+
const string insert =
223+
string("INSERT INTO mysql_users (username, password, default_hostgroup, active) VALUES ('")
224+
+ TEST_USER + "', '', " + std::to_string(MYSQL8_HG) + ", 1)";
225+
bool row_ok =
226+
(do_query(admin, insert) == EXIT_SUCCESS) &&
227+
(do_query(admin, "LOAD MYSQL USERS TO RUNTIME") == EXIT_SUCCESS);
228+
ok(row_ok, "Empty-password row inserted for '%s' (default_hostgroup=%u)", TEST_USER, MYSQL8_HG);
229+
230+
/* -------- enable pass-through (empty-password mode, TLS gate on) -------- */
231+
const vector<string> enable_queries {
232+
"SET mysql-passthrough_auth_enabled='true'",
233+
"SET mysql-passthrough_auth_require_tls='true'",
234+
"SET mysql-passthrough_auth_empty_password='true'",
235+
"SET mysql-passthrough_auth_unknown_users='false'",
236+
"SET mysql-default_authentication_plugin='caching_sha2_password'",
237+
/*
238+
* Keep the per-user failure allowance comfortably above the single
239+
* wrong-pw attempt this test makes, so the ATTACK step is denied by
240+
* the credential check (1045), never masked by a rate-limit lockout.
241+
*/
242+
"SET mysql-passthrough_auth_max_failures_per_user='10'",
243+
"SET mysql-passthrough_auth_max_failures_per_ip='100'",
244+
"LOAD MYSQL VARIABLES TO RUNTIME",
245+
};
246+
bool cfg_ok = true;
247+
for (const string& q : enable_queries) {
248+
if (do_query(admin, q) != EXIT_SUCCESS) { cfg_ok = false; break; }
249+
}
250+
ok(cfg_ok, "Pass-through enabled (empty-pw mode, TLS gate on, default_auth=caching_sha2)");
251+
252+
/*
253+
* The probe->backend leg needs TLS too (MySQL 8.4+ backends reject the
254+
* plaintext caching_sha2_password full-auth). Mark the hostgroup's
255+
* servers use_ssl=1, mirroring test_passthrough_auth_e2e-t.
256+
*/
257+
do_query(admin, string("UPDATE mysql_servers SET use_ssl=1 WHERE hostgroup_id=")
258+
+ std::to_string(MYSQL8_HG));
259+
do_query(admin, "LOAD MYSQL SERVERS TO RUNTIME");
260+
261+
/* Start from a clean cache regardless of prior test state. */
262+
do_query(admin, "PROXYSQL FLUSH PASSTHROUGH_AUTH_CACHE");
263+
ok(cache_entry_count(admin) == 0, "Cache starts with zero entries");
264+
265+
/* ============================================================
266+
* [warm] Legitimate connect: probe succeeds, credential cached, and
267+
* a backend connection authed as U/REAL lands in the pool.
268+
* ============================================================ */
269+
{
270+
const unsigned int err = try_connect(cl, TEST_USER, TEST_BACKEND_PW, /*run_query=*/true);
271+
ok(err == 0, "[warm] Legit connect with real password succeeds (errno=%u)", err);
272+
ok(cache_entry_count(admin) == 1, "[warm] Cache populated after legit connect");
273+
}
274+
275+
/* ============================================================
276+
* [flush] Clear ONLY the credential cache for U. The warm backend
277+
* connection stays in the pool -- this is the whole point.
278+
* ============================================================ */
279+
{
280+
const string flush =
281+
string("PROXYSQL FLUSH PASSTHROUGH_AUTH_CACHE FOR USER '") + TEST_USER + "'";
282+
ok(do_query(admin, flush) == EXIT_SUCCESS, "[flush] Per-user cache flush succeeds");
283+
ok(cache_entry_count(admin) == 0, "[flush] Credential cache empty after flush");
284+
}
285+
286+
/* ============================================================
287+
* [pre] Confirm the warm-pool window actually exists. If no backend
288+
* connection is free, the reuse hazard isn't being exercised;
289+
* we surface that explicitly rather than passing vacuously.
290+
* ============================================================ */
291+
const int free_before = free_conns(admin, MYSQL8_HG);
292+
ok(free_before >= 1,
293+
"[pre] A warm backend connection is free in the pool (ConnFree=%d) -- reuse window is real",
294+
free_before);
295+
296+
/* Baselines for the corroborating metric assertions. */
297+
const long long ok_before = metric(admin, "probes_ok");
298+
const long long fail_before = metric(admin, "probes_failed_credentials");
299+
300+
/* ============================================================
301+
* [ATTACK] U + WRONG password, cache cold, warm U/REAL conn pooled.
302+
* MUST be denied (1045). errno 0 here == force-new regression
303+
* == silent auth bypass via pooled-connection reuse.
304+
* ============================================================ */
305+
{
306+
const unsigned int err = try_connect(cl, TEST_USER, WRONG_PW, /*run_query=*/false);
307+
ok(err == ER_ACCESS_DENIED_ERROR,
308+
"[ATTACK] Wrong password DENIED despite warm pooled U/REAL conn "
309+
"(errno=%u, expected %u; errno 0 would mean pooled-conn reuse bypass)",
310+
err, (unsigned)ER_ACCESS_DENIED_ERROR);
311+
}
312+
313+
/* ============================================================
314+
* [corr] The denial came from a real backend probe rejection, not
315+
* from a short-circuit: the credential-failure counter must
316+
* advance and the success counter must NOT.
317+
* ============================================================ */
318+
{
319+
const long long fail_after = metric(admin, "probes_failed_credentials");
320+
const long long ok_after = metric(admin, "probes_ok");
321+
ok(fail_before >= 0 && fail_after == fail_before + 1,
322+
"[corr] probes_failed_credentials advanced by 1 (%lld -> %lld) -- backend actually rejected",
323+
fail_before, fail_after);
324+
ok(ok_before >= 0 && ok_after == ok_before,
325+
"[corr] probes_ok unchanged (%lld -> %lld) -- no false success recorded",
326+
ok_before, ok_after);
327+
}
328+
ok(cache_entry_count(admin) == 0, "[corr] Cache still empty after wrong-pw attempt");
329+
330+
/* ============================================================
331+
* [ctrl] Positive control: the real password still authenticates,
332+
* proving the ATTACK deny was password-specific, not a blanket
333+
* breakage of the pass-through path.
334+
* ============================================================ */
335+
{
336+
const unsigned int err = try_connect(cl, TEST_USER, TEST_BACKEND_PW, /*run_query=*/false);
337+
ok(err == 0, "[ctrl] Real password still succeeds after the attack (errno=%u)", err);
338+
ok(cache_entry_count(admin) == 1, "[ctrl] Cache repopulated by the control connect");
339+
}
340+
341+
/* -------- cleanup (run every step even on failure) -------- */
342+
do_query(admin, "PROXYSQL FLUSH PASSTHROUGH_AUTH_CACHE");
343+
{
344+
const int rc = do_query(backend, string("DROP USER IF EXISTS '") + TEST_USER + "'@'%'");
345+
ok(rc == EXIT_SUCCESS, "Cleanup: DROP USER on backend");
346+
}
347+
{
348+
int rc = EXIT_SUCCESS;
349+
rc |= do_query(admin, string("DELETE FROM mysql_users WHERE username='") + TEST_USER + "'");
350+
rc |= do_query(admin, "LOAD MYSQL USERS TO RUNTIME");
351+
rc |= do_query(admin, "SET mysql-passthrough_auth_enabled='false'");
352+
rc |= do_query(admin, "SET mysql-passthrough_auth_require_tls='true'");
353+
rc |= do_query(admin, "SET mysql-passthrough_auth_max_failures_per_user='3'");
354+
rc |= do_query(admin, "SET mysql-passthrough_auth_max_failures_per_ip='10'");
355+
rc |= do_query(admin, "SET mysql-default_authentication_plugin='mysql_native_password'");
356+
rc |= do_query(admin, "LOAD MYSQL VARIABLES TO RUNTIME");
357+
ok(rc == EXIT_SUCCESS, "Cleanup: mysql_users + variables restored");
358+
}
359+
360+
mysql_close(backend);
361+
mysql_close(admin);
362+
return exit_status();
363+
}

0 commit comments

Comments
 (0)