Skip to content

Commit 9aae4fe

Browse files
committed
fix(sharding): defer alter cluster add until primary view
- Prevent premature ADD queries during rejoin or failover - Avoid "nodes not ready" errors and potential SST donor crashes - Add check to verify target cluster status before execution
1 parent d1a50f9 commit 9aae4fe

1 file changed

Lines changed: 34 additions & 0 deletions

File tree

src/Plugin/Sharding/Queue.php

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,9 +192,43 @@ protected function shouldSkipQuery(array $query): bool {
192192
}
193193
}
194194

195+
// Defer ALTER CLUSTER ... ADD until its target internal cluster has re-formed a primary
196+
// view on this node. During a rejoin/failover rebalance the queue keeps processing even
197+
// while internal clusters are non-primary (to avoid deadlock), but issuing the ADD before
198+
// the returning member has re-synced is premature: the daemon reports "nodes not ready"
199+
// and, in the worst case, crashes in the SST donor setup (SetDonor4Joiner). Waiting here —
200+
// without burning a try — lets the bootstrap + Galera reconnect bring the cluster primary
201+
// first. Primary is reached independently of this ADD, so this cannot deadlock.
202+
if ($this->isAlterClusterAddTableQuery($query['query'])
203+
&& !$this->isQueryClusterPrimary($query['query'])) {
204+
Buddy::debugvv("Sharding queue: defer {$query['id']} — target cluster not primary yet");
205+
return true;
206+
}
207+
195208
return !$this->attemptToUpdateStatus($query, 'processing', 0);
196209
}
197210

211+
/**
212+
* Whether the internal cluster targeted by an ALTER CLUSTER statement is primary on this node.
213+
* Unknown/unparseable cluster -> treat as ready (do not block).
214+
* @param string $query
215+
* @return bool
216+
*/
217+
protected function isQueryClusterPrimary(string $query): bool {
218+
$name = $this->extractClusterNameFromQuery($query);
219+
if ($name === null) {
220+
return true;
221+
}
222+
try {
223+
$res = $this->client->sendRequest("SHOW STATUS LIKE 'cluster_{$name}_status'")->getResult();
224+
/** @var array{0?:array{data?:array<array{Value?:string}>}} $res */
225+
$status = $res[0]['data'][0]['Value'] ?? 'primary';
226+
return $status === 'primary';
227+
} catch (\Throwable $e) {
228+
return true;
229+
}
230+
}
231+
198232
/**
199233
* Helper to process the query from the queue
200234
* @param Node $node

0 commit comments

Comments
 (0)