Skip to content

Commit 371b9f0

Browse files
asdf2014asdf2014
committed
ZOOKEEPER-2789: Reassign ZXID to a 24-bit epoch / 40-bit counter with rolling upgrade
Merge current master into the reassign_zxid branch and rework the change into an opt-in, rolling-upgradeable wide-counter zxid layout. The 32-bit counter overflowing forces a leader re-election roughly every 49.7 days at 1k writes/s (ZOOKEEPER-1277). This adds a 24-bit epoch / 40-bit counter layout that pushes that to ~34.9 years while still allowing ~1915 years of hourly elections before the epoch overflows. - ZxidLayout / ZxidLayoutState: the legacy 32/32 and wide 24/40 layouts plus a per-QuorumPeer record of the epoch from which the ensemble is wide. The static ZxidUtils helpers remain the legacy-fixed layout for the on-the-wire epoch carrier zxids of the handshake. - Opt-in via zookeeper.wideCounterZxidEnabled (default off). A newly elected leader switches at its new epoch, persists it to a new zxidLayoutSwitchEpoch file, and announces it in the LEADERINFO handshake (learner protocol version bumped to 0x11000). Learners below 0x11000 are refused once switched. - The switch coincides with an epoch bump, so the numeric order of zxids is preserved; layoutFor(zxid) selects the decode layout per zxid so existing legacy snapshots/txnlogs are read correctly across the boundary without being rewritten. One-way switch with an anti-downgrade guard. - Replace hardcoded zxid bit operations with layout-aware calls across Leader, LearnerHandler, Learner, Follower, ObserverMaster, ZKDatabase, FastLeaderElection, Commands and LogChopper.
1 parent 0cb298e commit 371b9f0

28 files changed

Lines changed: 1719 additions & 74 deletions

‎zookeeper-server/src/main/java/org/apache/zookeeper/server/ZKDatabase.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import org.apache.zookeeper.server.quorum.Leader.PureRequestProposal;
5959
import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
6060
import org.apache.zookeeper.server.util.SerializeUtils;
61+
import org.apache.zookeeper.server.util.ZxidLayoutState;
6162
import org.apache.zookeeper.server.util.ZxidUtils;
6263
import org.apache.zookeeper.txn.TxnDigest;
6364
import org.apache.zookeeper.txn.TxnHeader;
@@ -85,6 +86,13 @@ public class ZKDatabase {
8586

8687
private final boolean allowDiscontinuousProposals = Boolean.getBoolean("zookeeper.test.allowDiscontinuousProposals");
8788

89+
/** How the zxids in this database are split into epoch and counter; set by the owning QuorumPeer. */
90+
private volatile ZxidLayoutState zxidLayoutState = ZxidLayoutState.legacyOnly();
91+
92+
public void setZxidLayoutState(ZxidLayoutState zxidLayoutState) {
93+
this.zxidLayoutState = zxidLayoutState;
94+
}
95+
8896
/**
8997
* Default value is to use snapshot if txnlog size exceeds 1/3 the size of snapshot
9098
*/
@@ -332,7 +340,8 @@ public void addCommittedProposal(Request request) {
332340
return;
333341
} else if (!allowDiscontinuousProposals
334342
&& request.zxid != maxCommittedLog + 1
335-
&& ZxidUtils.getEpochFromZxid(request.zxid) <= ZxidUtils.getEpochFromZxid(maxCommittedLog)) {
343+
&& zxidLayoutState.layoutFor(request.zxid).getEpochFromZxid(request.zxid)
344+
<= zxidLayoutState.layoutFor(maxCommittedLog).getEpochFromZxid(maxCommittedLog)) {
336345
String msg = String.format(
337346
"Committed proposal cached out of order: 0x%s is not the next proposal of 0x%s",
338347
ZxidUtils.zxidToString(request.zxid),

‎zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import org.apache.zookeeper.server.quorum.ReadOnlyZooKeeperServer;
7272
import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
7373
import org.apache.zookeeper.server.util.RateLimiter;
74+
import org.apache.zookeeper.server.util.ZxidLayout;
7475
import org.apache.zookeeper.server.util.ZxidUtils;
7576
import org.eclipse.jetty.http.HttpStatus;
7677
import org.slf4j.Logger;
@@ -1211,8 +1212,9 @@ public CommandResponse runGet(ZooKeeperServer zkServer, Map<String, String> kwar
12111212
response.put("voting", voting);
12121213
long lastProcessedZxid = zkServer.getZKDatabase().getDataTreeLastProcessedZxid();
12131214
response.put("last_zxid", "0x" + ZxidUtils.zxidToString(lastProcessedZxid));
1214-
response.put("zab_epoch", ZxidUtils.getEpochFromZxid(lastProcessedZxid));
1215-
response.put("zab_counter", ZxidUtils.getCounterFromZxid(lastProcessedZxid));
1215+
ZxidLayout layout = peer.getZxidLayoutState().layoutFor(lastProcessedZxid);
1216+
response.put("zab_epoch", layout.getEpochFromZxid(lastProcessedZxid));
1217+
response.put("zab_counter", layout.getCounterFromZxid(lastProcessedZxid));
12161218
response.put("zabstate", zabState.name().toLowerCase());
12171219
} else {
12181220
response.put("voting", false);

‎zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/TxnLogToolkit.java‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.apache.zookeeper.server.TxnLogEntry;
5656
import org.apache.zookeeper.server.util.LogChopper;
5757
import org.apache.zookeeper.server.util.SerializeUtils;
58+
import org.apache.zookeeper.server.util.ZxidLayoutState;
5859
import org.apache.zookeeper.txn.CheckVersionTxn;
5960
import org.apache.zookeeper.txn.CreateContainerTxn;
6061
import org.apache.zookeeper.txn.CreateTTLTxn;
@@ -117,6 +118,12 @@ Options getOptions() {
117118

118119
// chop mode
119120
private long zxid = -1L;
121+
// chop mode: the epoch from which the ensemble uses the wide-counter zxid
122+
// layout (the content of the zxidLayoutSwitchEpoch file), or -1 when the
123+
// log is entirely in the legacy layout. Only affects the gap diagnostics,
124+
// which decompose zxids into epoch/counter; the chop boundary itself is a
125+
// purely numeric comparison and is layout-independent.
126+
private long zxidLayoutSwitchEpoch = -1L;
120127

121128
/**
122129
* @param args Command line arguments
@@ -164,8 +171,15 @@ public TxnLogToolkit(
164171
}
165172

166173
public TxnLogToolkit(String txnLogFileName, String zxidName) throws TxnLogToolkitException {
174+
this(txnLogFileName, zxidName, null);
175+
}
176+
177+
public TxnLogToolkit(String txnLogFileName, String zxidName, String switchEpochName) throws TxnLogToolkitException {
167178
txnLogFile = loadTxnFile(txnLogFileName);
168179
zxid = Long.decode(zxidName);
180+
if (switchEpochName != null) {
181+
zxidLayoutSwitchEpoch = Long.decode(switchEpochName);
182+
}
169183
}
170184

171185
private File loadTxnFile(String txnLogFileName) throws TxnLogToolkitException {
@@ -253,9 +267,14 @@ public void dump(Scanner scanner) throws Exception {
253267

254268
public void chop() {
255269
File targetFile = new File(txnLogFile.getParentFile(), txnLogFile.getName() + ".chopped" + zxid);
270+
ZxidLayoutState layoutState = ZxidLayoutState.legacyOnly();
271+
if (zxidLayoutSwitchEpoch >= 0) {
272+
layoutState = new ZxidLayoutState();
273+
layoutState.switchAt(zxidLayoutSwitchEpoch);
274+
}
256275
try (InputStream is = new BufferedInputStream(new FileInputStream(txnLogFile));
257276
OutputStream os = new BufferedOutputStream(new FileOutputStream(targetFile))) {
258-
if (!LogChopper.chop(is, os, zxid)) {
277+
if (!LogChopper.chop(is, os, zxid, layoutState)) {
259278
throw new TxnLogToolkitException(
260279
ExitCode.INVALID_INVOCATION.getValue(),
261280
"Failed to chop %s",
@@ -457,8 +476,13 @@ private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolki
457476
// Chop mode options
458477
Option chopOpt = new Option("c", "chop", false, "Chop mode. Chop txn file to a zxid.");
459478
Option zxidOpt = new Option("z", "zxid", true, "Used with chop. Zxid to which to chop.");
479+
Option switchEpochOpt = new Option("s", "switch-epoch", true,
480+
"Used with chop. The wide-counter zxid layout switch epoch (content of the zxidLayoutSwitchEpoch "
481+
+ "file) when the log spans the layout switch, so gap diagnostics decode zxids correctly. "
482+
+ "Optional; the chop result is the same with or without it.");
460483
options.addOption(chopOpt);
461484
options.addOption(zxidOpt);
485+
options.addOption(switchEpochOpt);
462486

463487
try {
464488
CommandLine cli = parser.parse(options, args);
@@ -469,7 +493,7 @@ private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolki
469493
printHelpAndExit(1, options);
470494
}
471495
if (cli.hasOption("chop") && cli.hasOption("zxid")) {
472-
return new TxnLogToolkit(cli.getArgs()[0], cli.getOptionValue("zxid"));
496+
return new TxnLogToolkit(cli.getArgs()[0], cli.getOptionValue("zxid"), cli.getOptionValue("switch-epoch"));
473497
}
474498
return new TxnLogToolkit(cli.hasOption("recover"), cli.hasOption("verbose"), cli.getArgs()[0], cli.hasOption("yes"));
475499
} catch (ParseException e) {
@@ -479,7 +503,7 @@ private static TxnLogToolkit parseCommandLine(String[] args) throws TxnLogToolki
479503

480504
private static void printHelpAndExit(int exitCode, Options options) {
481505
HelpFormatter help = new HelpFormatter();
482-
help.printHelp(120, "TxnLogToolkit [-dhrvc] <txn_log_file_name> (-z <zxid>)", "", options, "");
506+
help.printHelp(120, "TxnLogToolkit [-dhrvc] <txn_log_file_name> (-z <zxid>) [-s <switch_epoch>]", "", options, "");
483507
ServiceUtils.requestSystemExit(exitCode);
484508
}
485509

‎zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FastLeaderElection.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState;
3737
import org.apache.zookeeper.server.quorum.flexible.QuorumOracleMaj;
3838
import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
39-
import org.apache.zookeeper.server.util.ZxidUtils;
4039
import org.slf4j.Logger;
4140
import org.slf4j.LoggerFactory;
4241

@@ -284,7 +283,7 @@ public void run() {
284283
}
285284
} else {
286285
LOG.info("Backward compatibility mode (28 bits), server id: {}", response.sid);
287-
rpeerepoch = ZxidUtils.getEpochFromZxid(rzxid);
286+
rpeerepoch = self.getZxidLayoutState().layoutFor(rzxid).getEpochFromZxid(rzxid);
288287
}
289288

290289
// check if we have a version that includes config. If so extract config info from message.

‎zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Follower.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ void followLeader() throws InterruptedException {
9393
}
9494
//check to see if the leader zxid is lower than ours
9595
//this should never happen but is just a safety check
96-
long newEpoch = ZxidUtils.getEpochFromZxid(newEpochZxid);
96+
long newEpoch = self.getZxidLayoutState().current().getEpochFromZxid(newEpochZxid);
9797
if (newEpoch < self.getAcceptedEpoch()) {
9898
LOG.error("Proposed leader epoch "
9999
+ ZxidUtils.zxidToString(newEpochZxid)

‎zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ protected void setupRequestProcessors() {
7777
LinkedBlockingQueue<Request> pendingTxns = new LinkedBlockingQueue<>();
7878

7979
public void logRequest(Request request) {
80-
if ((request.zxid & 0xffffffffL) != 0) {
80+
if (self.getZxidLayoutState().current().getCounterFromZxid(request.zxid) != 0) {
8181
pendingTxns.add(request);
8282
}
8383
syncProcessor.processRequest(request);

‎zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Leader.java‎

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@
6868
import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType;
6969
import org.apache.zookeeper.server.quorum.auth.QuorumAuthServer;
7070
import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
71-
import org.apache.zookeeper.server.util.ZxidUtils;
71+
import org.apache.zookeeper.server.util.ZxidLayout;
72+
import org.apache.zookeeper.server.util.ZxidLayoutState;
7273
import org.apache.zookeeper.util.ServiceUtils;
7374
import org.slf4j.Logger;
7475
import org.slf4j.LoggerFactory;
@@ -654,15 +655,15 @@ void lead() throws IOException, InterruptedException {
654655

655656
long epoch = getEpochToPropose(self.getMyId(), self.getAcceptedEpoch());
656657

657-
zk.setZxid(ZxidUtils.makeZxid(epoch, 0));
658+
zk.setZxid(self.getZxidLayoutState().current().makeZxid(epoch, 0));
658659

659660
synchronized (this) {
660661
lastProposed = zk.getZxid();
661662
}
662663

663664
newLeaderProposal.packet = new QuorumPacket(NEWLEADER, zk.getZxid(), null, null);
664665

665-
if ((newLeaderProposal.packet.getZxid() & 0xffffffffL) != 0) {
666+
if (self.getZxidLayoutState().current().getCounterFromZxid(newLeaderProposal.packet.getZxid()) != 0) {
666667
LOG.info("NEWLEADER proposal has Zxid of {}", Long.toHexString(newLeaderProposal.packet.getZxid()));
667668
}
668669

@@ -755,7 +756,7 @@ void lead() throws IOException, InterruptedException {
755756
String initialZxid = System.getProperty("zookeeper.testingonly.initialZxid");
756757
if (initialZxid != null) {
757758
long zxid = Long.parseLong(initialZxid);
758-
zk.setZxid((zk.getZxid() & 0xffffffff00000000L) | zxid);
759+
zk.setZxid(self.getZxidLayoutState().current().clearCounter(zk.getZxid()) | zxid);
759760
}
760761

761762
if (!System.getProperty("zookeeper.leaderServes", "yes").equals("no")) {
@@ -1065,7 +1066,7 @@ public synchronized void processAck(long sid, long zxid, SocketAddress followerA
10651066
LOG.trace("outstanding proposals all");
10661067
}
10671068

1068-
if ((zxid & 0xffffffffL) == 0) {
1069+
if (self.getZxidLayoutState().current().getCounterFromZxid(zxid) == 0) {
10691070
/*
10701071
* We no longer process NEWLEADER ack with this method. However,
10711072
* the learner sends an ack back to the leader after it gets
@@ -1274,7 +1275,15 @@ public synchronized long getLastProposed() {
12741275
* Returns the current epoch of the leader.
12751276
*/
12761277
public long getEpoch() {
1277-
return ZxidUtils.getEpochFromZxid(lastProposed);
1278+
// Read lastProposed through the synchronized getter: everywhere else
1279+
// it is accessed under the Leader monitor, so reading it unsynchronized
1280+
// here would be an inconsistent-synchronization data race.
1281+
return self.getZxidLayoutState().current().getEpochFromZxid(getLastProposed());
1282+
}
1283+
1284+
@Override
1285+
public ZxidLayoutState getZxidLayoutState() {
1286+
return self.getZxidLayoutState();
12781287
}
12791288

12801289
@SuppressWarnings("serial")
@@ -1298,11 +1307,13 @@ public Proposal propose(Request request) throws XidRolloverException {
12981307
ServiceUtils.requestSystemExit(ExitCode.UNEXPECTED_ERROR.getValue());
12991308
}
13001309
/**
1301-
* Address the rollover issue. All lower 32bits set indicate a new leader
1302-
* election. Force a re-election instead. See ZOOKEEPER-1277
1310+
* Address the rollover issue. An exhausted counter part indicates a
1311+
* new leader election. Force a re-election instead. See ZOOKEEPER-1277
13031312
*/
1304-
if ((request.zxid & 0xffffffffL) == 0xffffffffL) {
1305-
String msg = "zxid lower 32 bits have rolled over, forcing re-election, and therefore new epoch start";
1313+
ZxidLayout layout = self.getZxidLayoutState().current();
1314+
if (layout.getCounterFromZxid(request.zxid) == layout.getMaxCounter()) {
1315+
String msg = "zxid counter (lower " + layout.getCounterBits()
1316+
+ " bits) has rolled over, forcing re-election, and therefore new epoch start";
13061317
shutdown(msg);
13071318
throw new XidRolloverException(msg);
13081319
}
@@ -1480,6 +1491,7 @@ public long getEpochToPropose(long sid, long lastAcceptedEpoch) throws Interrupt
14801491
QuorumVerifier verifier = self.getQuorumVerifier();
14811492
if (connectingFollowers.contains(self.getMyId()) && verifier.containsQuorum(connectingFollowers)) {
14821493
waitingForNewEpoch = false;
1494+
maybeSwitchZxidLayout(epoch);
14831495
self.setAcceptedEpoch(epoch);
14841496
connectingFollowers.notifyAll();
14851497
} else {
@@ -1501,6 +1513,27 @@ public long getEpochToPropose(long sid, long lastAcceptedEpoch) throws Interrupt
15011513
}
15021514
}
15031515

1516+
/**
1517+
* Switches the ensemble to the wide-counter zxid layout at the moment
1518+
* the new epoch is established, if enabled and not yet switched. Doing
1519+
* it inside the epoch decision point guarantees that every zxid of the
1520+
* new epoch — the leader's own as well as those handed to the
1521+
* LearnerHandlers — is composed with one consistent layout.
1522+
*/
1523+
private void maybeSwitchZxidLayout(long newEpoch) throws IOException {
1524+
if (!self.isWideCounterZxidEnabled() || self.getZxidLayoutState().isSwitched()) {
1525+
return;
1526+
}
1527+
if (newEpoch > ZxidLayout.WIDE_COUNTER.getMaxEpoch()) {
1528+
LOG.warn("Cannot switch to the {} zxid layout: epoch 0x{} exceeds its largest representable epoch 0x{}",
1529+
ZxidLayout.WIDE_COUNTER,
1530+
Long.toHexString(newEpoch),
1531+
Long.toHexString(ZxidLayout.WIDE_COUNTER.getMaxEpoch()));
1532+
return;
1533+
}
1534+
self.recordZxidLayoutSwitch(newEpoch);
1535+
}
1536+
15041537
@Override
15051538
public ZKDatabase getZKDatabase() {
15061539
return zk.getZKDatabase();

0 commit comments

Comments
 (0)