diff --git a/.coderabbit.yaml b/.coderabbit.yaml index e987f1b31..198c0277a 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -5,3 +5,7 @@ reviews: enabled: true drafts: true ignore_title_keywords: ["WIP"] + # A feature branch collects its pull requests on an integration branch before it is merged into + # main, so review those pull requests too, not only the ones that target the default branch. + base_branches: + - ".*" diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java index dc2876c2c..3cb9bd4b3 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java @@ -171,6 +171,7 @@ import com.linecorp.centraldogma.server.internal.mirror.DefaultMirroringServicePlugin; import com.linecorp.centraldogma.server.internal.mirror.MirrorAccessControl; import com.linecorp.centraldogma.server.internal.mirror.MirrorRunner; +import com.linecorp.centraldogma.server.internal.replication.RecoveryPayloadBuilder; import com.linecorp.centraldogma.server.internal.replication.ZooKeeperCommandExecutor; import com.linecorp.centraldogma.server.internal.storage.project.DefaultProjectManager; import com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager; @@ -963,7 +964,7 @@ private CommandExecutor newZooKeeperCommandExecutor( sessionManager, encryptionStorageManager, /* onTakeLeadership */ null, /* onReleaseLeadership */ null, /* onTakeZoneLeadership */ null, /* onReleaseZoneLeadership */ null), - meterRegistry, zone, + meterRegistry, zone, new RecoveryPayloadBuilder(pm), onTakeLeadership, onReleaseLeadership, onTakeZoneLeadership, onReleaseZoneLeadership); } @@ -1028,7 +1029,8 @@ private void configureHttpApi(ServerBuilder sb, .annotatedService(new ServerStatusService(executor, statusManager, repoStatusManager)) .annotatedService(new ProjectServiceV1(projectApiManager, executor, repoStatusManager)) .annotatedService(new RepositoryServiceV1(executor, mds, encryptionStorageManager, - repoStatusManager)) + repoStatusManager, + new RecoveryPayloadBuilder(pm))) .annotatedService(new CredentialServiceV1(projectApiManager, executor)) .annotatedService(new VariableServiceV1(pm, executor)); if (LOGBACK_ENABLED) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java b/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java index 18952b466..c29cf92ab 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/Command.java @@ -68,6 +68,8 @@ @Type(value = UpdateProjectStatusCommand.class, name = "UPDATE_PROJECT_STATUS"), @Type(value = UpdateRepositoryStatusCommand.class, name = "UPDATE_REPOSITORY_STATUS"), @Type(value = ForcePushCommand.class, name = "FORCE_PUSH_COMMAND"), + @Type(value = RecoverRepositoryCommand.class, name = "RECOVER_REPOSITORY"), + @Type(value = RecoverRepositoryRequestCommand.class, name = "RECOVER_REPOSITORY_REQUEST"), }) public interface Command { @@ -526,6 +528,38 @@ static Command forcePush(Command delegate) { return new ForcePushCommand<>(delegate); } + /** + * Returns a new {@link Command} which recovers a diverged repository from a source replica by resetting + * to {@code resetToRevision} and replaying {@code commits} up to {@code headRevision}. See + * {@link RecoverRepositoryCommand}. + */ + static Command recoverRepository(Author author, String projectName, String repositoryName, + int sourceServerId, Revision resetToRevision, + Revision headRevision, Iterable commits) { + requireNonNull(author, "author"); + requireNonNull(projectName, "projectName"); + requireNonNull(repositoryName, "repositoryName"); + requireNonNull(resetToRevision, "resetToRevision"); + requireNonNull(headRevision, "headRevision"); + requireNonNull(commits, "commits"); + return new RecoverRepositoryCommand(null, author, projectName, repositoryName, sourceServerId, + resetToRevision, headRevision, commits); + } + + /** + * Returns a new {@link Command} which asks the source replica to originate a recovery. See + * {@link RecoverRepositoryRequestCommand}. + */ + static Command recoverRepositoryRequest(Author author, String projectName, String repositoryName, + int sourceServerId, Revision fromRevision) { + requireNonNull(author, "author"); + requireNonNull(projectName, "projectName"); + requireNonNull(repositoryName, "repositoryName"); + requireNonNull(fromRevision, "fromRevision"); + return new RecoverRepositoryRequestCommand(null, author, projectName, repositoryName, + sourceServerId, fromRevision); + } + /** * Returns the {@link CommandType} of the command. */ diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java index 2730e69ca..736e764ea 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandType.java @@ -49,6 +49,8 @@ public enum CommandType { UPDATE_SERVER_STATUS(Void.class), UPDATE_PROJECT_STATUS(Void.class), UPDATE_REPOSITORY_STATUS(Void.class), + RECOVER_REPOSITORY(Revision.class), + RECOVER_REPOSITORY_REQUEST(Void.class), // The result type of FORCE_PUSH is Object because it can be any type. FORCE_PUSH(Object.class); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/RecoverRepositoryCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/RecoverRepositoryCommand.java new file mode 100644 index 000000000..d98c17983 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/RecoverRepositoryCommand.java @@ -0,0 +1,137 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.command; + +import static java.util.Objects.requireNonNull; + +import java.util.List; +import java.util.Objects; + +import org.jspecify.annotations.Nullable; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.common.collect.ImmutableList; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Revision; + +/** + * A {@link Command} which recovers a diverged repository from a source replica. It is originated by the + * source replica (the single source of truth) and applied identically on every replica, itself included: a + * replica already converged with {@link #commits()} is left untouched, and every other one resets its git + * repository and commit-id database to {@link #resetToRevision()} and replays {@link #commits()} up to + * {@link #headRevision()}. Because the replayed commits carry the original author, timestamp and + * self-contained changes, a replay reproduces the source's commit ids; each one is verified against + * {@link ReplayCommit#expectedCommitId()}, and a mismatch aborts the recovery and rolls the replica back. + * + *

The convergence check is by content, not by replica: the source is normally the replica that is + * already converged, but a commit that lands on it between the payload build and the apply (a force push, + * which read-only does not block) makes the source replay over itself too, discarding that commit. + * + *

This is a {@link RepositoryCommand} so that it is scoped to a single repository (lock scope and + * read-only failure blast radius) and is not rejected while the repository/project is read-only. + */ +public final class RecoverRepositoryCommand extends RepositoryCommand { + + private final int sourceServerId; + private final Revision resetToRevision; + private final Revision headRevision; + private final List commits; + + @JsonCreator + RecoverRepositoryCommand(@JsonProperty("timestamp") @Nullable Long timestamp, + @JsonProperty("author") @Nullable Author author, + @JsonProperty("projectName") String projectName, + @JsonProperty("repositoryName") String repositoryName, + @JsonProperty("sourceServerId") int sourceServerId, + @JsonProperty("resetToRevision") Revision resetToRevision, + @JsonProperty("headRevision") Revision headRevision, + @JsonProperty("commits") Iterable commits) { + super(CommandType.RECOVER_REPOSITORY, timestamp, author, projectName, repositoryName); + this.sourceServerId = sourceServerId; + this.resetToRevision = requireNonNull(resetToRevision, "resetToRevision"); + this.headRevision = requireNonNull(headRevision, "headRevision"); + this.commits = ImmutableList.copyOf(requireNonNull(commits, "commits")); + } + + /** + * Returns the ZooKeeper server ID of the source replica whose repository the {@link #commits()} were + * taken from. It records where a recovery came from; it is not consulted when the command is applied, + * which decides by content (see the class javadoc). + */ + @JsonProperty + public int sourceServerId() { + return sourceServerId; + } + + /** + * Returns the {@link Revision} to which a replica resets its repository before replaying + * {@link #commits()}. + */ + @JsonProperty + public Revision resetToRevision() { + return resetToRevision; + } + + /** + * Returns the head {@link Revision} of the source repository, which is also the result of this command. + */ + @JsonProperty + public Revision headRevision() { + return headRevision; + } + + /** + * Returns the ordered {@link ReplayCommit}s to replay after resetting to {@link #resetToRevision()}. + */ + @JsonProperty + public List commits() { + return commits; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RecoverRepositoryCommand)) { + return false; + } + final RecoverRepositoryCommand that = (RecoverRepositoryCommand) obj; + return super.equals(that) && + sourceServerId == that.sourceServerId && + resetToRevision.equals(that.resetToRevision) && + headRevision.equals(that.headRevision) && + commits.equals(that.commits); + } + + @Override + public int hashCode() { + return Objects.hash(sourceServerId, resetToRevision, headRevision, commits) * 31 + super.hashCode(); + } + + @Override + ToStringHelper toStringHelper() { + return super.toStringHelper() + .add("sourceServerId", sourceServerId) + .add("resetToRevision", resetToRevision) + .add("headRevision", headRevision) + .add("commits", commits.size()); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/RecoverRepositoryRequestCommand.java b/server/src/main/java/com/linecorp/centraldogma/server/command/RecoverRepositoryRequestCommand.java new file mode 100644 index 000000000..0976f8096 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/RecoverRepositoryRequestCommand.java @@ -0,0 +1,96 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.command; + +import static java.util.Objects.requireNonNull; + +import java.util.Objects; + +import org.jspecify.annotations.Nullable; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects.ToStringHelper; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Revision; + +/** + * A {@link Command} which asks the source replica to originate a {@link RecoverRepositoryCommand}. It is + * originated by a non-source replica that received the recovery request (e.g. behind a load balancer) and is + * applied as a no-op on every replica; the source replica reacts to it (off the replication-log replay + * thread) by building and originating the actual {@link RecoverRepositoryCommand}. + */ +public final class RecoverRepositoryRequestCommand extends RepositoryCommand { + + private final int sourceServerId; + private final Revision fromRevision; + + @JsonCreator + RecoverRepositoryRequestCommand(@JsonProperty("timestamp") @Nullable Long timestamp, + @JsonProperty("author") @Nullable Author author, + @JsonProperty("projectName") String projectName, + @JsonProperty("repositoryName") String repositoryName, + @JsonProperty("sourceServerId") int sourceServerId, + @JsonProperty("fromRevision") Revision fromRevision) { + super(CommandType.RECOVER_REPOSITORY_REQUEST, timestamp, author, projectName, repositoryName); + this.sourceServerId = sourceServerId; + this.fromRevision = requireNonNull(fromRevision, "fromRevision"); + } + + /** + * Returns the ZooKeeper server ID of the source replica that should originate the recovery. + */ + @JsonProperty + public int sourceServerId() { + return sourceServerId; + } + + /** + * Returns the first {@link Revision} to replay. Recovery replays {@code fromRevision..sourceHead}. + */ + @JsonProperty + public Revision fromRevision() { + return fromRevision; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RecoverRepositoryRequestCommand)) { + return false; + } + final RecoverRepositoryRequestCommand that = (RecoverRepositoryRequestCommand) obj; + return super.equals(that) && + sourceServerId == that.sourceServerId && + fromRevision.equals(that.fromRevision); + } + + @Override + public int hashCode() { + return Objects.hash(sourceServerId, fromRevision) * 31 + super.hashCode(); + } + + @Override + ToStringHelper toStringHelper() { + return super.toStringHelper() + .add("sourceServerId", sourceServerId) + .add("fromRevision", fromRevision); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/ReplayCommit.java b/server/src/main/java/com/linecorp/centraldogma/server/command/ReplayCommit.java new file mode 100644 index 000000000..5ff61a9d2 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/ReplayCommit.java @@ -0,0 +1,177 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.command; + +import static java.util.Objects.requireNonNull; + +import java.util.List; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Revision; + +/** + * A single commit replayed onto a diverged replica during a {@link RecoverRepositoryCommand}. It carries the + * original commit metadata and a self-contained set of {@link Change}s so that every replica reconstructs an + * identical commit (and thus an identical commit id) when it is applied on top of the previous revision. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class ReplayCommit { + + private final Revision revision; + private final long timestampMillis; + private final Author author; + private final String summary; + private final String detail; + private final Markup markup; + private final List> changes; + private final String expectedCommitId; + + /** + * Creates a new instance. + */ + @JsonCreator + public ReplayCommit(@JsonProperty("revision") Revision revision, + @JsonProperty("timestampMillis") long timestampMillis, + @JsonProperty("author") Author author, + @JsonProperty("summary") String summary, + @JsonProperty("detail") String detail, + @JsonProperty("markup") Markup markup, + @JsonProperty("changes") Iterable> changes, + @JsonProperty("expectedCommitId") String expectedCommitId) { + this.revision = requireNonNull(revision, "revision"); + this.timestampMillis = timestampMillis; + this.author = requireNonNull(author, "author"); + this.summary = requireNonNull(summary, "summary"); + this.detail = requireNonNull(detail, "detail"); + this.markup = requireNonNull(markup, "markup"); + this.changes = ImmutableList.copyOf(requireNonNull(changes, "changes")); + this.expectedCommitId = requireNonNull(expectedCommitId, "expectedCommitId"); + } + + /** + * Returns the {@link Revision} produced by this commit. + */ + @JsonProperty + public Revision revision() { + return revision; + } + + /** + * Returns the commit time in milliseconds. + */ + @JsonProperty + public long timestampMillis() { + return timestampMillis; + } + + /** + * Returns the {@link Author} of the commit. + */ + @JsonProperty + public Author author() { + return author; + } + + /** + * Returns the human-readable summary of the commit. + */ + @JsonProperty + public String summary() { + return summary; + } + + /** + * Returns the human-readable detail of the commit. + */ + @JsonProperty + public String detail() { + return detail; + } + + /** + * Returns the {@link Markup} of the {@link #detail()}. + */ + @JsonProperty + public Markup markup() { + return markup; + } + + /** + * Returns the self-contained {@link Change}s of the commit. + */ + @JsonProperty + public List> changes() { + return changes; + } + + /** + * Returns the commit id the replayed commit must produce. A replica that produces a different one + * aborts the recovery instead of writing a history that diverges from the source, so this is what + * makes a recovery verifiable rather than merely hopeful. + */ + @JsonProperty + public String expectedCommitId() { + return expectedCommitId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ReplayCommit)) { + return false; + } + final ReplayCommit that = (ReplayCommit) o; + return timestampMillis == that.timestampMillis && + revision.equals(that.revision) && + author.equals(that.author) && + summary.equals(that.summary) && + detail.equals(that.detail) && + markup == that.markup && + changes.equals(that.changes) && + Objects.equals(expectedCommitId, that.expectedCommitId); + } + + @Override + public int hashCode() { + return Objects.hash(revision, timestampMillis, author, summary, detail, markup, changes, + expectedCommitId); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("revision", revision) + .add("timestampMillis", timestampMillis) + .add("author", author) + .add("summary", summary) + .add("markup", markup) + .add("changes", changes.size()) + .add("expectedCommitId", expectedCommitId) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java index 759253be5..2faa24a05 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java @@ -28,6 +28,7 @@ import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.centraldogma.common.ReadOnlyException; +import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.server.auth.Session; import com.linecorp.centraldogma.server.auth.SessionManager; import com.linecorp.centraldogma.server.internal.management.RepoStatusManager; @@ -244,6 +245,15 @@ private CompletableFuture doExecute0(ExecutionContext ctx, Command com return doExecute0(ctx, ((ForcePushCommand) command).delegate()); } + if (command instanceof RecoverRepositoryRequestCommand) { + // Applied as a no-op on every replica; the source replica reacts to it in the ZooKeeper layer. + return (CompletableFuture) CompletableFuture.completedFuture(null); + } + + if (command instanceof RecoverRepositoryCommand) { + return (CompletableFuture) recoverRepository((RecoverRepositoryCommand) command); + } + throw new UnsupportedOperationException(command.toString()); } @@ -280,21 +290,34 @@ private CompletableFuture removeProject(RemoveProjectCommand c) { return CompletableFuture.supplyAsync(() -> { projectManager.remove(c.projectName()); return null; - }, repositoryWorker); + }, repositoryWorker).thenRun(repoStatusManager::refreshReadOnlyMetrics); } private CompletableFuture unremoveProject(UnremoveProjectCommand c) { return CompletableFuture.supplyAsync(() -> { projectManager.unremove(c.projectName()); return null; - }, repositoryWorker); + }, repositoryWorker).thenRun(repoStatusManager::refreshReadOnlyMetrics); } private CompletableFuture purgeProject(PurgeProjectCommand c) { return CompletableFuture.supplyAsync(() -> { projectManager.markForPurge(c.projectName()); return null; - }, repositoryWorker); + }, repositoryWorker).thenCompose(unused -> { + if (projectManager.exists(c.projectName())) { + // markForPurge() was a no-op because the project was not in the removed state; + // keep its status so read-only enforcement is not silently defeated. + return CompletableFuture.completedFuture(null); + } + // Best-effort cleanup: a failure here must not fail the already-applied purge command. + return repoStatusManager.removeProjectStatus(c.projectName(), c.author()) + .exceptionally(cause -> { + logger.warn("Failed to remove the status of the purged project: {}", + c.projectName(), cause); + return null; + }); + }); } // Repository operations @@ -331,21 +354,35 @@ private CompletableFuture removeRepository(RemoveRepositoryCommand c) { return CompletableFuture.supplyAsync(() -> { projectManager.get(c.projectName()).repos().remove(c.repositoryName()); return null; - }, repositoryWorker); + }, repositoryWorker).thenRun(repoStatusManager::refreshReadOnlyMetrics); } private CompletableFuture unremoveRepository(UnremoveRepositoryCommand c) { return CompletableFuture.supplyAsync(() -> { projectManager.get(c.projectName()).repos().unremove(c.repositoryName()); return null; - }, repositoryWorker); + }, repositoryWorker).thenRun(repoStatusManager::refreshReadOnlyMetrics); } private CompletableFuture purgeRepository(PurgeRepositoryCommand c) { return CompletableFuture.supplyAsync(() -> { projectManager.get(c.projectName()).repos().markForPurge(c.repositoryName()); return null; - }, repositoryWorker); + }, repositoryWorker).thenCompose(unused -> { + if (projectManager.exists(c.projectName()) && + projectManager.get(c.projectName()).repos().exists(c.repositoryName())) { + // markForPurge() was a no-op because the repository was not in the removed state; + // keep its status so read-only enforcement is not silently defeated. + return CompletableFuture.completedFuture(null); + } + // Best-effort cleanup: a failure here must not fail the already-applied purge command. + return repoStatusManager.removeRepoStatus(c.projectName(), c.repositoryName(), c.author()) + .exceptionally(cause -> { + logger.warn("Failed to remove the status of the purged repository:" + + " {}/{}", c.projectName(), c.repositoryName(), cause); + return null; + }); + }); } private CompletableFuture migrateToEncryptedRepository(MigrateToEncryptedRepositoryCommand c) { @@ -405,6 +442,24 @@ private Repository repo(RepositoryCommand c) { return projectManager.get(c.projectName()).repos().get(c.repositoryName()); } + private CompletableFuture recoverRepository(RecoverRepositoryCommand c) { + // The recovery is a pure function of its payload: the manager no-ops when the repository is already + // converged (the source, or a healthy replica), otherwise it resets to c.resetToRevision() and + // replays c.commits(). Every replica therefore reaches the same state and returns the same head + // revision, which is what the replication log compares. + // + // The read-only precondition is deliberately not re-checked here. It is stored under a different + // execution path (the whole server) than this command (the repository), so an operator making the + // repository writable again races the replay: replicas would disagree on the check, and the replica + // that failed it would skip this log entry for good and stay diverged. The precondition is enforced + // once, where it is a decision rather than a race: on the replica that originates the recovery. + return CompletableFuture.supplyAsync(() -> { + projectManager.get(c.projectName()).repos() + .recoverRepository(c.repositoryName(), c.resetToRevision(), c.commits()); + return c.headRevision(); + }, repositoryWorker); + } + private CompletableFuture rewrapAllKeys() { if (!encryptionStorageManager.enabled()) { throw new IllegalStateException("Encryption is not enabled."); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RecoverRepositoryRequest.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RecoverRepositoryRequest.java new file mode 100644 index 000000000..ec2080e99 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RecoverRepositoryRequest.java @@ -0,0 +1,73 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.api; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; + +public final class RecoverRepositoryRequest { + + private final int fromRevision; + private final int sourceServerId; + + @JsonCreator + public RecoverRepositoryRequest(@JsonProperty("fromRevision") int fromRevision, + @JsonProperty("sourceServerId") int sourceServerId) { + checkArgument(fromRevision >= 2, "fromRevision: %s (expected: >= 2)", fromRevision); + checkArgument(sourceServerId > 0, "sourceServerId: %s (expected: > 0)", sourceServerId); + this.fromRevision = fromRevision; + this.sourceServerId = sourceServerId; + } + + @JsonProperty("fromRevision") + public int fromRevision() { + return fromRevision; + } + + @JsonProperty("sourceServerId") + public int sourceServerId() { + return sourceServerId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RecoverRepositoryRequest)) { + return false; + } + final RecoverRepositoryRequest that = (RecoverRepositoryRequest) o; + return fromRevision == that.fromRevision && sourceServerId == that.sourceServerId; + } + + @Override + public int hashCode() { + return fromRevision * 31 + sourceServerId; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("fromRevision", fromRevision) + .add("sourceServerId", sourceServerId) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RecoverRepositoryResponse.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RecoverRepositoryResponse.java new file mode 100644 index 000000000..3fa2bb991 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RecoverRepositoryResponse.java @@ -0,0 +1,80 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.api; + +import static java.util.Objects.requireNonNull; + +import java.util.Objects; + +import org.jspecify.annotations.Nullable; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class RecoverRepositoryResponse { + + private final RecoveryStatus status; + @Nullable + private final Integer headRevision; + + public RecoverRepositoryResponse(RecoveryStatus status, @Nullable Integer headRevision) { + this.status = requireNonNull(status, "status"); + this.headRevision = headRevision; + } + + @JsonProperty("status") + public RecoveryStatus status() { + return status; + } + + /** + * Returns the source replica's head revision, which every other replica converges to once it replays + * the recovery, or {@code null} if the recovery was only requested. + */ + @Nullable + @JsonProperty("headRevision") + public Integer headRevision() { + return headRevision; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RecoverRepositoryResponse)) { + return false; + } + final RecoverRepositoryResponse that = (RecoverRepositoryResponse) o; + return status == that.status && Objects.equals(headRevision, that.headRevision); + } + + @Override + public int hashCode() { + return status.hashCode() * 31 + Objects.hashCode(headRevision); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("status", status) + .add("headRevision", headRevision) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RecoveryStatus.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RecoveryStatus.java new file mode 100644 index 000000000..1774175bd --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RecoveryStatus.java @@ -0,0 +1,33 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.api; + +/** + * The outcome of a repository recovery request. + */ +public enum RecoveryStatus { + /** + * The request landed on the source replica, which originated the recovery. The other replicas apply + * it when they replay it from the replication log, so this does not mean the cluster has converged. + */ + COMPLETED, + /** + * The source replica has been asked over the replication log to originate the recovery + * asynchronously, best-effort: a failure is only reported in the source replica's log. + */ + REQUESTED +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1.java index b69d7ec72..c77e63669 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1.java @@ -41,6 +41,7 @@ import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.armeria.server.annotation.Blocking; import com.linecorp.armeria.server.annotation.Consumes; import com.linecorp.armeria.server.annotation.Delete; import com.linecorp.armeria.server.annotation.Get; @@ -66,6 +67,8 @@ import com.linecorp.centraldogma.server.internal.api.converter.CreateApiResponseConverter; import com.linecorp.centraldogma.server.internal.management.RepoStatusManager; import com.linecorp.centraldogma.server.internal.management.RepositoryState; +import com.linecorp.centraldogma.server.internal.replication.RecoveryPayloadBuilder; +import com.linecorp.centraldogma.server.internal.replication.ZooKeeperCommandExecutor; import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.User; import com.linecorp.centraldogma.server.storage.encryption.EncryptionStorageManager; @@ -73,6 +76,7 @@ import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.Repository; +import com.linecorp.centraldogma.server.storage.repository.RepositoryHead; import io.micrometer.core.instrument.Tag; @@ -87,14 +91,17 @@ public class RepositoryServiceV1 extends AbstractService { private final MetadataService mds; private final EncryptionStorageManager encryptionStorageManager; private final RepoStatusManager repoStatusManager; + private final RecoveryPayloadBuilder recoveryPayloadBuilder; public RepositoryServiceV1(CommandExecutor executor, MetadataService mds, EncryptionStorageManager encryptionStorageManager, - RepoStatusManager repoStatusManager) { + RepoStatusManager repoStatusManager, + RecoveryPayloadBuilder recoveryPayloadBuilder) { super(executor); this.repoStatusManager = repoStatusManager; this.mds = requireNonNull(mds, "mds"); this.encryptionStorageManager = requireNonNull(encryptionStorageManager, "encryptionStorageManager"); + this.recoveryPayloadBuilder = requireNonNull(recoveryPayloadBuilder, "recoveryPayloadBuilder"); } /** @@ -294,6 +301,117 @@ public CompletableFuture updateStatus(Project project, .thenApply(state -> newRepositoryDto(repository, newStatus)); } + /** + * GET /projects/{projectName}/repos/{repoName}/head + * + *

Returns the head of the repository on the replica that served the request, identified by + * both its revision and its commit ID. Diverged replicas may report the same revision, so a matching + * revision proves nothing; only the commit ID proves that they hold the same history. It is how an + * administrator confirms that a recovery converged before making the repository writable again. + */ + @Get("/projects/{projectName}/repos/{repoName}/head") + @RequiresSystemAdministrator + @Blocking + public RepositoryHead head(Repository repository) { + return repository.head(); + } + + /** + * POST /projects/{projectName}/repos/{repoName}/recover + * + *

Recovers the repository from a diverged state, using the repository of the replica whose server ID + * is {@code sourceServerId} as the single source of truth: every other replica resets its repository to + * just before {@code fromRevision} and replays the source's commits up to the source's head, so all + * replicas converge to identical commit IDs. + * + *

Replicated (ZooKeeper) mode only. The repository must be read-only before recovery so that no new + * commit can be originated while the recovery is in flight (the precondition is re-verified when the + * command is applied), and it stays read-only afterwards. Note that a force-push races recovery + * deliberately — it bypasses read-only, so a force-pushed commit that lands between the payload build + * and the apply is discarded by the replay, on the source replica too. + * + *

Neither result means the cluster has converged: the replicas other than the source apply the + * recovery when they replay it from the replication log. {@code COMPLETED} means the source replica + * originated the recovery; {@code REQUESTED} means the source replica was asked to originate it over + * the replication log, best-effort — a failure is only reported in the source replica's log. The + * administrator confirms convergence with {@code GET .../head} on every replica before making the + * repository writable again. Recovery should not run during a rolling upgrade: a replica that does not + * know the recovery commands yet skips them and turns read-only. + */ + @Post("/projects/{projectName}/repos/{repoName}/recover") + @Consumes("application/json") + @RequiresSystemAdministrator + public CompletableFuture recover(ServiceRequestContext ctx, + Project project, + Repository repository, + Author author, + RecoverRepositoryRequest request) { + final ZooKeeperCommandExecutor zkExecutor = validateRecoveryPrerequisites(ctx, project, repository, + request); + final String projectName = project.name(); + final String repoName = repository.name(); + final int sourceServerId = request.sourceServerId(); + final Revision fromRevision = new Revision(request.fromRevision()); + ctx.setRequestTimeoutMillis(Long.MAX_VALUE); // Disable the request timeout for recovery. + + if (zkExecutor.replicaId() == sourceServerId) { + // This replica is the source of truth; build the payload from the local storage and originate + // the recovery command directly. + logger.info("Originating a recovery of {}/{} from revision {} as the source replica.", + projectName, repoName, fromRevision); + return CompletableFuture + .supplyAsync(() -> recoveryPayloadBuilder.build(author, projectName, repoName, + sourceServerId, fromRevision), + ctx.blockingTaskExecutor()) + .thenCompose(this::execute) + .thenApply(headRevision -> new RecoverRepositoryResponse( + RecoveryStatus.COMPLETED, headRevision.major())); + } + + // Ask the source replica to originate the recovery via the replication log. + logger.info("Requesting a recovery of {}/{} from revision {} to the source replica {}.", + projectName, repoName, fromRevision, sourceServerId); + return execute(Command.recoverRepositoryRequest(author, projectName, repoName, sourceServerId, + fromRevision)) + .thenApply(unused -> new RecoverRepositoryResponse(RecoveryStatus.REQUESTED, null)); + } + + private ZooKeeperCommandExecutor validateRecoveryPrerequisites(ServiceRequestContext ctx, Project project, + Repository repository, + RecoverRepositoryRequest request) { + if (InternalProjectInitializer.INTERNAL_PROJECT_DOGMA.equals(project.name()) || + Project.isInternalRepo(repository.name())) { + // Internal repository content is written by content transformers without text normalization, + // so a replay cannot reproduce it byte-identically. + return HttpApiUtil.throwResponse( + ctx, HttpStatus.FORBIDDEN, + "Cannot recover an internal repository: %s/%s", project.name(), repository.name()); + } + if (!(executor() instanceof ZooKeeperCommandExecutor)) { + throw new IllegalArgumentException( + "Repository recovery is only supported in replicated (ZooKeeper) mode."); + } + if (repository.isEncrypted()) { + throw new IllegalArgumentException( + "Recovery is not supported for an encrypted repository: " + + project.name() + '/' + repository.name()); + } + final ZooKeeperCommandExecutor zkExecutor = (ZooKeeperCommandExecutor) executor(); + if (!zkExecutor.replicationConfig().servers().containsKey(request.sourceServerId())) { + throw new IllegalArgumentException( + "sourceServerId: " + request.sourceServerId() + " (expected: one of " + + zkExecutor.replicationConfig().servers().keySet() + ')'); + } + if (getReplicationStatus(repository) != ReplicationStatus.READ_ONLY) { + return HttpApiUtil.throwResponse( + ctx, HttpStatus.CONFLICT, + "The repository must be read-only before recovery so that no new commit can be " + + "originated while the recovery is in flight: %s/%s. Change the status to READ_ONLY " + + "first.", project.name(), repository.name()); + } + return zkExecutor; + } + /** * POST /projects/{projectName}/repos/{repoName}/migrate/file * diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/ReplicaInfo.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/ReplicaInfo.java new file mode 100644 index 000000000..da1f6d68b --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/ReplicaInfo.java @@ -0,0 +1,88 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.api.sysadmin; + +import static java.util.Objects.requireNonNull; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; + +/** + * A replica of the cluster, from the static replication configuration. + */ +public final class ReplicaInfo { + + private final int serverId; + private final String host; + private final boolean current; + + public ReplicaInfo(int serverId, String host, boolean current) { + this.serverId = serverId; + this.host = requireNonNull(host, "host"); + this.current = current; + } + + /** + * Returns the ZooKeeper server ID of the replica. + */ + @JsonProperty("serverId") + public int serverId() { + return serverId; + } + + /** + * Returns the host name of the replica. + */ + @JsonProperty("host") + public String host() { + return host; + } + + /** + * Returns whether this replica is the one that served the request. + */ + @JsonProperty("current") + public boolean current() { + return current; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ReplicaInfo)) { + return false; + } + final ReplicaInfo that = (ReplicaInfo) o; + return serverId == that.serverId && current == that.current && host.equals(that.host); + } + + @Override + public int hashCode() { + return (serverId * 31 + host.hashCode()) * 31 + Boolean.hashCode(current); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("serverId", serverId) + .add("host", host) + .add("current", current) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/ServerStatusService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/ServerStatusService.java index 34e99383c..20f1b863f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/ServerStatusService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/ServerStatusService.java @@ -16,9 +16,13 @@ package com.linecorp.centraldogma.server.internal.api.sysadmin; +import static com.google.common.collect.ImmutableList.toImmutableList; + import java.util.List; import java.util.concurrent.CompletableFuture; +import com.google.common.collect.ImmutableList; + import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.server.HttpStatusException; import com.linecorp.armeria.server.annotation.Consumes; @@ -71,6 +75,25 @@ public List readOnlyRepositories() { return repoStatusManager.readOnlyStatuses(); } + /** + * GET /replicas + * + *

Returns the replicas of the cluster from the static replication configuration, marking the one + * that served this request as {@code current}. Returns an empty list when the server is not running + * in replicated (ZooKeeper) mode. + */ + @Get("/replicas") + public List replicas() { + if (!(executor() instanceof ZooKeeperCommandExecutor)) { + return ImmutableList.of(); + } + final ZooKeeperCommandExecutor zkExecutor = (ZooKeeperCommandExecutor) executor(); + return zkExecutor.replicationConfig().servers().entrySet().stream() + .map(entry -> new ReplicaInfo(entry.getKey(), entry.getValue().host(), + entry.getKey() == zkExecutor.replicaId())) + .collect(toImmutableList()); + } + /** * PUT /status * diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManager.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManager.java index 9ad3346e4..55088ca8f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManager.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; @@ -30,13 +31,17 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; -import com.google.common.collect.ImmutableList; +import com.spotify.futures.CompletableFutures; import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.CentralDogmaException; +import com.linecorp.centraldogma.common.ChangeConflictException; import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.ProjectNotFoundException; import com.linecorp.centraldogma.common.RedundantChangeException; import com.linecorp.centraldogma.common.ReplicationStatus; +import com.linecorp.centraldogma.common.RepositoryNotFoundException; import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.internal.storage.repository.crud.CrudContext; @@ -45,6 +50,7 @@ import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; +import com.linecorp.centraldogma.server.storage.repository.HasRevision; import com.linecorp.centraldogma.server.storage.repository.Repository; import com.linecorp.centraldogma.server.storage.repository.RepositoryListener; @@ -83,7 +89,8 @@ public RepoStatusManager(ServerStatusManager statusManager, ProjectManager pm, crudRepository = new StandaloneCrudOperation<>(RepositoryState.class, pm); // read-only scope metrics - Gauge.builder("repository.read.only.count", statusMap, Map::size).register(meterRegistry); + Gauge.builder("repository.read.only.count", this, RepoStatusManager::activeReadOnlyCount) + .register(meterRegistry); readOnlyScopeGauge = MultiGauge.builder("repository.read.only").register(meterRegistry); } @@ -91,6 +98,9 @@ public void initialize() { final Repository dogmaRepo = pm.get(InternalProjectInitializer.INTERNAL_PROJECT_DOGMA).repos() .get(Project.REPO_DOGMA); dogmaRepo.addListener(RepositoryListener.of("/status/**/*.json", entries -> { + // This snapshot only ever contains files that still exist, so a stale/late-delivered + // snapshot must never evict a key it does not observe. Deletions on purge are therefore + // applied directly by removeRepoStatus/removeProjectStatus, not reconciled here. for (Entry entry : entries.values()) { final RepositoryState repoState; try { @@ -107,7 +117,7 @@ public void initialize() { statusMap.put(getKey(repoState.projectName(), repoState.repoName()), repoState); } } - updateReadOnlyScopeMetrics(); + refreshReadOnlyMetrics(); })); } @@ -155,7 +165,10 @@ public RepositoryState getRepoStatus(String projectName, String repoName) { * as its repository name. */ public List readOnlyStatuses() { - return ImmutableList.copyOf(statusMap.values()); + // Hide entries whose project/repository was removed; the preserved file restores it on unremove. + return statusMap.values().stream() + .filter(state -> isActive(state.projectName(), state.repoName())) + .collect(toImmutableList()); } @Nullable @@ -184,6 +197,11 @@ private static String getKey(String projectName, String repoName) { return projectName + '/' + repoName; } + private static boolean isStorageAbsent(Throwable cause) { + final Throwable peeled = Exceptions.peel(cause); + return peeled instanceof ProjectNotFoundException || peeled instanceof RepositoryNotFoundException; + } + public boolean isWritable(String projectName, String repoName) { if (!statusManager.serverStatus().writable()) { return false; @@ -213,20 +231,133 @@ public CompletableFuture updateProjectStatus(String projectName, Author au return updateRepoStatus(projectName, Project.REPO_DOGMA, author, newStatus); } + /** + * Deletes the replication status file of the specified repository. Called when a repository is purged + * so that neither the status nor its metrics leak the removed entry. + * + *

The deletion is never gated on the in-memory cache: the cache is loaded by an asynchronously + * registered listener, so it can still be empty while the replication log is replayed after a + * restart, and skipping the deletion then would leave this replica's status storage — and hence its + * {@code dogma/dogma} repository — behind the rest of the cluster. Deleting a status file that does + * not exist is a no-op on every replica. + */ + public CompletableFuture removeRepoStatus(String projectName, String repoName, Author author) { + final String description = "Delete the replication status of '" + projectName + '/' + repoName + '\''; + return crudOperation().delete(crudContext(projectName), repoName, author, description) + .handle((revision, cause) -> { + if (cause != null) { + final Throwable peeled = Exceptions.peel(cause); + if (!(peeled instanceof ChangeConflictException || + peeled instanceof RedundantChangeException)) { + return Exceptions.throwUnsafely(peeled); + } + // The status file does not exist; still evict the cache below. + } + // The listener does not observe deletions, so evict directly. + statusMap.remove(getKey(projectName, repoName)); + refreshReadOnlyMetrics(); + return null; + }); + } + + /** + * Deletes all replication status files of the specified project, if any. Called when a project is + * purged so that neither the statuses nor their metrics leak the removed entries. The entries are + * enumerated from the replicated status files rather than the in-memory cache, which can still be + * empty while the replication log is replayed after a restart. + * + *

The status files are read asynchronously. Never block on the returned future from the repository + * worker, which is the executor the read itself is scheduled on. + */ + public CompletableFuture removeProjectStatus(String projectName, Author author) { + final CompletableFuture>> statesFuture; + try { + statesFuture = crudOperation().findAll(crudContext(projectName)); + } catch (RuntimeException e) { + return completedIfStatusStorageAbsent(e); + } + return statesFuture.handle((states, cause) -> { + if (cause != null) { + return RepoStatusManager.completedIfStatusStorageAbsent(cause); + } + // Clean up each repository independently so a single failure does not skip the rest. + CompletableFuture future = CompletableFuture.completedFuture(null); + for (HasRevision state : states) { + final String repoName = state.object().repoName(); + future = future.thenCompose(unused -> removeRepoStatus(projectName, repoName, author) + .exceptionally(repoCause -> { + logger.warn("Failed to remove the replication status of '{}/{}'.", + projectName, repoName, repoCause); + return null; + })); + } + return future; + }).thenCompose(Function.identity()); + } + + /** + * Returns a future completed with {@code null} if the cause only means that the internal status + * storage does not exist yet, so no status was ever replicated; a failed future otherwise. The + * lookup may fail either synchronously or through the returned future, so both are funnelled here. + */ + private static CompletableFuture completedIfStatusStorageAbsent(Throwable cause) { + if (isStorageAbsent(cause)) { + return CompletableFuture.completedFuture(null); + } + return CompletableFutures.exceptionallyCompletedFuture(cause); + } + private static CrudContext crudContext(String projectName) { final String targetPath = PATH_PREFIX + projectName + '/'; return new CrudContext(InternalProjectInitializer.INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, targetPath); } - private void updateReadOnlyScopeMetrics() { - readOnlyScopeGauge.register( - statusMap.values().stream() - .>map(state -> MultiGauge.Row.of( - Tags.of("project", state.projectName(), - "repo", state.repoName()), - 1)) - .collect(toImmutableList()), - true); + /** + * Re-registers the {@code repository.read.only} gauge. Invoked by the repository listener on + * status changes and by the command executor when a repository/project is removed, restored or + * purged (which change {@link #isActive} without touching the status files). + */ + public synchronized void refreshReadOnlyMetrics() { + try { + readOnlyScopeGauge.register( + statusMap.values().stream() + .filter(state -> isActive(state.projectName(), state.repoName())) + .>map(state -> MultiGauge.Row.of( + Tags.of("project", state.projectName(), + "repo", state.repoName()), + 1)) + .collect(toImmutableList()), + true); + } catch (Exception e) { + // Never let a metrics refresh failure propagate into the command that triggered it. + logger.warn("Failed to refresh the read-only scope metrics.", e); + } + } + + private double activeReadOnlyCount() { + return statusMap.values().stream() + .filter(state -> isActive(state.projectName(), state.repoName())) + .count(); + } + + /** + * Returns {@code true} if the project and repository of a read-only entry still exist, i.e. they + * have not been soft-removed or purged. + */ + private boolean isActive(String projectName, String repoName) { + try { + if (!pm.exists(projectName)) { + return false; + } + if (Project.REPO_DOGMA.equals(repoName)) { + // Project-scoped entry; the project itself exists. + return true; + } + return pm.get(projectName).repos().exists(repoName); + } catch (CentralDogmaException e) { + // The project/repository was removed concurrently; treat it as inactive. + return false; + } } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/RecoveryPayloadBuilder.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/RecoveryPayloadBuilder.java new file mode 100644 index 000000000..5cf6da063 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/RecoveryPayloadBuilder.java @@ -0,0 +1,75 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.replication; + +import static java.util.Objects.requireNonNull; + +import java.util.List; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.command.Command; +import com.linecorp.centraldogma.server.command.RecoverRepositoryCommand; +import com.linecorp.centraldogma.server.command.RecoverRepositoryRequestCommand; +import com.linecorp.centraldogma.server.command.ReplayCommit; +import com.linecorp.centraldogma.server.storage.project.ProjectManager; + +/** + * Builds a self-contained {@link RecoverRepositoryCommand} from the local storage. Invoked only on the + * source replica of a recovery, whose repository is the single source of truth. + */ +public final class RecoveryPayloadBuilder { + + private final ProjectManager projectManager; + + /** + * Creates a new instance. + */ + public RecoveryPayloadBuilder(ProjectManager projectManager) { + this.projectManager = requireNonNull(projectManager, "projectManager"); + } + + /** + * Builds a {@link RecoverRepositoryCommand} that carries the commits of + * {@code request.fromRevision()..HEAD} of the local repository, so that every other replica can + * converge to the local history by replaying them. + */ + public Command build(RecoverRepositoryRequestCommand request) { + requireNonNull(request, "request"); + return build(request.author(), request.projectName(), request.repositoryName(), + request.sourceServerId(), request.fromRevision()); + } + + /** + * Builds a {@link RecoverRepositoryCommand} that carries the commits of {@code fromRevision..HEAD} of + * the local repository. The reset revision and the head revision are derived here, so that a recovery + * originated directly by the source replica and one originated in reaction to a request are built by + * the same rules. + */ + public Command build(Author author, String projectName, String repositoryName, + int sourceServerId, Revision fromRevision) { + requireNonNull(author, "author"); + requireNonNull(projectName, "projectName"); + requireNonNull(repositoryName, "repositoryName"); + requireNonNull(fromRevision, "fromRevision"); + final List commits = + projectManager.get(projectName).repos().buildRecoveryPayload(repositoryName, fromRevision); + final Revision headRevision = commits.get(commits.size() - 1).revision(); + return Command.recoverRepository(author, projectName, repositoryName, sourceServerId, + fromRevision.backward(1), headRevision, commits); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ReplicationLogContext.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ReplicationLogContext.java index 9dde77325..0c03e1057 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ReplicationLogContext.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ReplicationLogContext.java @@ -17,7 +17,6 @@ package com.linecorp.centraldogma.server.internal.replication; import java.util.Arrays; -import java.util.Base64; import java.util.Objects; import org.jspecify.annotations.Nullable; @@ -89,7 +88,8 @@ public String toString() { .add("replayRevision", replayRevision) .add("meta", meta) .add("log", log) - .add("bytes", bytes == null ? null : Base64.getEncoder().encodeToString(bytes)) + // A recovery payload can be tens of megabytes; never dump it into a log line. + .add("bytes", bytes == null ? null : bytes.length + " bytes") .toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java index 320c4a32d..ee3f96262 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java @@ -93,6 +93,8 @@ import com.linecorp.centraldogma.server.command.ForcePushCommand; import com.linecorp.centraldogma.server.command.NormalizableCommit; import com.linecorp.centraldogma.server.command.ProjectCommand; +import com.linecorp.centraldogma.server.command.RecoverRepositoryCommand; +import com.linecorp.centraldogma.server.command.RecoverRepositoryRequestCommand; import com.linecorp.centraldogma.server.command.RepositoryCommand; import com.linecorp.centraldogma.server.command.UpdateServerStatusCommand; import com.linecorp.centraldogma.server.internal.command.DefaultExecutionContext; @@ -148,6 +150,9 @@ public final class ZooKeeperCommandExecutor @Nullable private final String zone; + @Nullable + private final RecoveryPayloadBuilder recoveryPayloadBuilder; + // Failing to acquire a lock is a critical problem, so we wait as much as we can. private long lockTimeoutNanos = TimeUnit.MINUTES.toNanos(1); @@ -156,6 +161,8 @@ public final class ZooKeeperCommandExecutor private volatile RetryPolicy retryPolicy = RETRY_POLICY_NEVER; private volatile ExecutorService executor; private volatile ExecutorService logWatcherExecutor; + @Nullable + private volatile ExecutorService recoveryOriginatorExecutor; private volatile PathChildrenCache logWatcher; private volatile OldLogRemover oldLogRemover; private volatile ExecutorService leaderSelectorExecutor; @@ -377,12 +384,14 @@ public ZooKeeperCommandExecutor(ZooKeeperReplicationConfig cfg, File dataDir, CommandExecutor delegate, MeterRegistry meterRegistry, @Nullable String zone, + @Nullable RecoveryPayloadBuilder recoveryPayloadBuilder, @Nullable Consumer onTakeLeadership, @Nullable Consumer onReleaseLeadership, @Nullable Consumer onTakeZoneLeadership, @Nullable Consumer onReleaseZoneLeadership) { super(onTakeLeadership, onReleaseLeadership, onTakeZoneLeadership, onReleaseZoneLeadership); + this.recoveryPayloadBuilder = recoveryPayloadBuilder; this.cfg = requireNonNull(cfg, "cfg"); requireNonNull(dataDir, "dataDir"); revisionFile = new File(dataDir.getAbsolutePath() + File.separatorChar + "last_revision"); @@ -429,6 +438,13 @@ public int replicaId() { return cfg.serverId(); } + /** + * Returns the replication configuration of this cluster. + */ + public ZooKeeperReplicationConfig replicationConfig() { + return cfg; + } + public CommandExecutor unwrap() { return delegate; } @@ -473,6 +489,15 @@ protected void doStart(@Nullable Runnable onTakeLeadership, logWatcher.getListenable().addListener(this, MoreExecutors.directExecutor()); logWatcher.start(); + // Building a recovery payload blocks for as long as it takes to read the whole replayed range, + // so it gets a thread of its own rather than one of the few ForkJoinPool.commonPool() threads + // that the rest of the JVM shares. + recoveryOriginatorExecutor = ExecutorServiceMetrics.monitor( + meterRegistry, + Executors.newSingleThreadExecutor( + new DefaultThreadFactory("recovery-originator", true)), + "recoveryOriginator"); + // Start the leader selection. oldLogRemover = new OldLogRemover(); leaderSelectorExecutor = ExecutorServiceMetrics.monitor( @@ -692,6 +717,11 @@ protected void doStop(@Nullable Runnable onReleaseLeadership, listenerInfo = null; logger.info("Stopping the worker threads"); boolean interrupted = shutdown(executor); + final ExecutorService recoveryOriginatorExecutor = this.recoveryOriginatorExecutor; + if (recoveryOriginatorExecutor != null) { + interrupted |= shutdown(recoveryOriginatorExecutor); + this.recoveryOriginatorExecutor = null; + } logger.info("Stopped the worker threads"); try { @@ -842,6 +872,9 @@ private synchronized void replayLogs(long targetRevision, boolean force) { if (command instanceof UpdateServerStatusCommand) { updateZkCommandStatusLater((UpdateServerStatusCommand) command); } + if (command instanceof RecoverRepositoryRequestCommand) { + reactToRecoveryRequestLater((RecoverRepositoryRequestCommand) command); + } } catch (Throwable t) { try { // Skip the failed log so the remaining logs can still be replayed. @@ -892,6 +925,57 @@ private void updateZkCommandStatusLater(UpdateServerStatusCommand command) { } } + /** + * Reacts to a replayed {@link RecoverRepositoryRequestCommand}. Only the replica whose server ID matches + * the source server ID reacts, by building a self-contained {@link RecoverRepositoryCommand} from its + * local storage and originating it via the replication log. The reaction runs off the replay thread + * because originating a command replays pending logs by itself. + */ + private void reactToRecoveryRequestLater(RecoverRepositoryRequestCommand command) { + if (command.sourceServerId() != replicaId()) { + return; + } + final RecoveryPayloadBuilder recoveryPayloadBuilder = this.recoveryPayloadBuilder; + if (recoveryPayloadBuilder == null) { + logger.error("Cannot originate a recovery for {}/{}; no {} is configured.", + command.projectName(), command.repositoryName(), + RecoveryPayloadBuilder.class.getSimpleName()); + return; + } + final ExecutorService recoveryOriginatorExecutor = this.recoveryOriginatorExecutor; + if (recoveryOriginatorExecutor == null) { + logger.warn("Cannot originate a recovery for {}/{}; the replica is stopping.", + command.projectName(), command.repositoryName()); + return; + } + final String repoName = command.projectName() + '/' + command.repositoryName(); + recoveryOriginatorExecutor.execute(() -> { + final Command recoverCommand; + try { + recoverCommand = recoveryPayloadBuilder.build(command); + } catch (Throwable t) { + logger.error("Failed to build the recovery payload of {}; recovery is not originated.", + repoName, t); + return; + } + logger.info("Originating a recovery of {} as the source replica: {}", repoName, recoverCommand); + try { + execute(recoverCommand).handle((revision, cause) -> { + if (cause != null) { + logger.error("Failed to originate a recovery of {}.", repoName, cause); + } else { + logger.info("Successfully originated a recovery of {}. head: {}", repoName, + revision); + } + return null; + }); + } catch (Throwable t) { + // execute() throws synchronously when the executor is stopping or the server is read-only. + logger.error("Failed to originate a recovery of {}.", repoName, t); + } + }); + } + @Override public void childEvent(CuratorFramework unused, PathChildrenCacheEvent event) throws Exception { if (event.getType() != PathChildrenCacheEvent.Type.CHILD_ADDED) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryManagerWrapper.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryManagerWrapper.java index 592e629b0..3ca6b2c2e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryManagerWrapper.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryManagerWrapper.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -35,7 +36,9 @@ import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.CentralDogmaException; import com.linecorp.centraldogma.common.RepositoryNotFoundException; +import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.internal.Util; +import com.linecorp.centraldogma.server.command.ReplayCommit; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.repository.Repository; import com.linecorp.centraldogma.server.storage.repository.RepositoryManager; @@ -87,6 +90,27 @@ public void fallbackToFileRepository(String repositoryName) { } } + @Override + public boolean recoverRepository(String repositoryName, Revision resetToRevision, + List commits) { + if (!delegate.recoverRepository(repositoryName, resetToRevision, commits)) { + // Already converged, so the delegate kept its Repository instance and this wrapper must keep + // its own: rebuilding it would throw away every cache entry keyed on the current instance. + return false; + } + repos.replace(repositoryName, repoWrapper.apply(delegate.get(repositoryName))); + final BiConsumer callback = postMigrationCallback; + if (callback != null) { + callback.accept(repositoryName, repos.get(repositoryName)); + } + return true; + } + + @Override + public List buildRecoveryPayload(String repositoryName, Revision fromRevision) { + return delegate.buildRecoveryPayload(repositoryName, fromRevision); + } + @Override public void close(Supplier failureCauseSupplier) { delegate.close(failureCauseSupplier); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java index 435be3e16..26d4dbd51 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java @@ -45,6 +45,7 @@ import com.linecorp.centraldogma.server.storage.repository.EntryTransformer; import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.Repository; +import com.linecorp.centraldogma.server.storage.repository.RepositoryHead; import com.linecorp.centraldogma.server.storage.repository.RepositoryListener; public class RepositoryWrapper implements Repository { @@ -60,6 +61,11 @@ public final T unwrap() { return (T) repo; } + @Override + public RepositoryHead head() { + return unwrap().head(); + } + @Override public org.eclipse.jgit.lib.Repository jGitRepository() { return unwrap().jGitRepository(); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java index 91bc7d1ab..f0a8b1388 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java @@ -50,6 +50,7 @@ import com.linecorp.centraldogma.server.storage.repository.DiffResultType; import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.Repository; +import com.linecorp.centraldogma.server.storage.repository.RepositoryHead; import com.linecorp.centraldogma.server.storage.repository.RepositoryListener; final class CachingRepository implements Repository { @@ -65,6 +66,11 @@ final class CachingRepository implements Repository { this.cache = requireNonNull(cache, "cache"); } + @Override + public RepositoryHead head() { + return repo.head(); + } + @Override public org.eclipse.jgit.lib.Repository jGitRepository() { return repo.jGitRepository(); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java index 13a421cc1..85cdb134e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java @@ -34,6 +34,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -106,6 +107,7 @@ import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.FindOptions; import com.linecorp.centraldogma.server.storage.repository.Repository; +import com.linecorp.centraldogma.server.storage.repository.RepositoryHead; import com.linecorp.centraldogma.server.storage.repository.RepositoryListener; /** @@ -169,6 +171,7 @@ class GitRepository implements Repository { @VisibleForTesting final CommitWatchers commitWatchers = new CommitWatchers(); private final AtomicReference> closePending = new AtomicReference<>(); + private final AtomicBoolean closeScheduled = new AtomicBoolean(); private final CompletableFuture closeFuture = new CompletableFuture<>(); private final List listeners = new CopyOnWriteArrayList<>(); @@ -232,9 +235,9 @@ List listeners() { */ void close(Supplier failureCauseSupplier) { requireNonNull(failureCauseSupplier, "failureCauseSupplier"); - if (closePending.compareAndSet(null, failureCauseSupplier)) { + closePending.compareAndSet(null, failureCauseSupplier); + if (closeScheduled.compareAndSet(false, true)) { repositoryWorker.execute(() -> { - // MUST acquire gcLock first to prevent a dead lock rwLock.writeLock().lock(); try { closeRepository(commitIdDatabase, jGitRepository); @@ -242,7 +245,7 @@ void close(Supplier failureCauseSupplier) { try { rwLock.writeLock().unlock(); } finally { - commitWatchers.close(failureCauseSupplier); + commitWatchers.close(closePending.get()); closeFuture.complete(null); } } @@ -252,6 +255,42 @@ void close(Supplier failureCauseSupplier) { closeFuture.join(); } + /** + * Closes this repository on the calling thread instead of dispatching to the repository worker. A + * recovery runs on a repository-worker thread, so {@link #close(Supplier)} would make it wait for a task + * queued back to the same pool, which deadlocks once every worker is busy. + */ + void closeInline(Supplier failureCauseSupplier) { + requireNonNull(failureCauseSupplier, "failureCauseSupplier"); + closePending.compareAndSet(null, failureCauseSupplier); + if (closeScheduled.compareAndSet(false, true)) { + rwLock.writeLock().lock(); + try { + closeRepository(commitIdDatabase, jGitRepository); + } finally { + try { + rwLock.writeLock().unlock(); + } finally { + commitWatchers.close(closePending.get()); + closeFuture.complete(null); + } + } + } else { + closeFuture.join(); + } + } + + /** + * Marks this repository as closed so that a new or lock-blocked read fails fast with the given cause, + * without releasing the underlying resources yet. {@link #close(Supplier)} must still be called + * afterwards. Unlike {@link #close(Supplier)}, this never blocks, so it is safe to call while holding + * the write lock. + */ + void markClosePending(Supplier failureCauseSupplier) { + requireNonNull(failureCauseSupplier, "failureCauseSupplier"); + closePending.compareAndSet(null, failureCauseSupplier); + } + static void closeRepository(@Nullable CommitIdDatabase commitIdDatabase, org.eclipse.jgit.lib.@Nullable Repository jGitRepository) { if (commitIdDatabase != null) { @@ -284,6 +323,19 @@ public org.eclipse.jgit.lib.Repository jGitRepository() { return jGitRepository; } + @Override + public RepositoryHead head() { + readLock(); + try { + // A recovery force-moves master and rebuilds the commit-id database of the directory this + // instance is open on, so the pair is only coherent under the read lock. + final Revision headRevision = this.headRevision; + return new RepositoryHead(headRevision, commitIdDatabase.get(headRevision).name()); + } finally { + readUnlock(); + } + } + @Override public Project parent() { return parent; @@ -912,6 +964,18 @@ private CompletableFuture commit( }, repositoryWorker); } + /** + * Commits on the calling thread instead of dispatching to the repository worker. A recovery replays + * its commits from a repository-worker thread while it holds the write lock of the repository it is + * replacing, so blocking on a task queued back to that same pool would deadlock it. + */ + CommitResult blockingCommit(Revision baseRevision, long commitTimeMillis, Author author, String summary, + String detail, Markup markup, Iterable> changes) { + final CommitExecutor commitExecutor = + new CommitExecutor(this, commitTimeMillis, author, summary, detail, markup, false); + return commitExecutor.execute(baseRevision, normBaseRevision -> changes); + } + /** * Removes {@code \r} and appends {@code \n} on the last line if it does not end with {@code \n}. */ @@ -945,6 +1009,33 @@ static void doRefUpdate(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk } } + /** + * Force-updates {@code ref} to {@code commitId}, allowing a non-fast-forward (backward) move. Used by + * repository recovery to reset {@code refs/heads/master} to an earlier revision. + */ + static void doForceRefUpdate(org.eclipse.jgit.lib.Repository jGitRepository, RevWalk revWalk, + String ref, ObjectId commitId) throws IOException { + if (ref.startsWith(Constants.R_TAGS)) { + throw new StorageException("Using a tag is not allowed. ref: " + ref); + } + + final RefUpdate refUpdate = jGitRepository.updateRef(ref); + refUpdate.setNewObjectId(commitId); + refUpdate.setForceUpdate(true); + + final Result res = refUpdate.update(revWalk); + switch (res) { + case NEW: + case FAST_FORWARD: + case FORCED: + case NO_CHANGE: + // Expected + break; + default: + throw new StorageException("unexpected forced refUpdate state: " + res); + } + } + @Override public CompletableFuture findLatestRevision(Revision lastKnownRevision, String pathPattern, boolean errorOnEntryNotFound) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java index 6e8cda603..fa67856f5 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepositoryManager.java @@ -50,6 +50,8 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Utf8; +import com.google.common.collect.ImmutableList; import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.CentralDogmaException; @@ -58,6 +60,8 @@ import com.linecorp.centraldogma.common.RepositoryExistsException; import com.linecorp.centraldogma.common.RepositoryNotFoundException; import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.command.CommitResult; +import com.linecorp.centraldogma.server.command.ReplayCommit; import com.linecorp.centraldogma.server.internal.JGitUtil; import com.linecorp.centraldogma.server.internal.storage.DirectoryBasedStorageManager; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; @@ -269,6 +273,243 @@ public void fallbackToFileRepository(String repositoryName) { } } + @Override + public boolean recoverRepository(String repositoryName, Revision resetToRevision, + List commits) { + requireNonNull(repositoryName, "repositoryName"); + requireNonNull(resetToRevision, "resetToRevision"); + requireNonNull(commits, "commits"); + logger.info("Starting to recover the repository '{}' (reset to {}, replay {} commits).", + projectRepositoryName(repositoryName), resetToRevision, commits.size()); + final long startTime = System.nanoTime(); + final GitRepository old = (GitRepository) get(repositoryName); + if (old.isEncrypted()) { + throw new StorageException("recovery is not supported for an encrypted repository: " + + projectRepositoryName(repositoryName)); + } + if (commits.isEmpty()) { + return false; + } + + // Skip if the repository is already converged with the source (the source replica itself, or a + // healthy replica): its HEAD revision and commit id already match the last replayed commit. This + // keeps the source untouched and makes recovery idempotent. + final ReplayCommit lastCommit = commits.get(commits.size() - 1); + final String expectedHeadCommitId = lastCommit.expectedCommitId(); + final Revision currentHead = old.normalizeNow(Revision.HEAD); + if (currentHead.equals(lastCommit.revision())) { + final ObjectId currentHeadCommitId = old.commitIdDatabase().get(currentHead); + if (currentHeadCommitId != null && expectedHeadCommitId.equals(currentHeadCommitId.name())) { + logger.info("Repository '{}' is already converged at {} ({}); nothing to recover.", + projectRepositoryName(repositoryName), currentHead, expectedHeadCommitId); + return false; + } + } + + // Remember the current HEAD so the old repository can be restored if recovery fails. + final Revision originalHeadRevision = old.normalizeNow(Revision.HEAD); + if (resetToRevision.major() > originalHeadRevision.major()) { + throw new StorageException( + "cannot recover " + projectRepositoryName(repositoryName) + ": the local head " + + originalHeadRevision + " is below the reset revision " + resetToRevision + + "; this replica is missing the shared base history."); + } + final ObjectId originalHeadCommitId = old.commitIdDatabase().get(originalHeadRevision); + final ObjectId resetToCommitId = old.commitIdDatabase().get(resetToRevision); + final File repoDir = old.repoDir(); + + // Quiesce the old repository so no read observes a partially rebuilt commit-id database while the new + // repository is opened on the same directory. + old.writeLock(); + GitRepository neo = null; + boolean swapped = false; + try { + // Force-move master backward to the reset revision, then reopen so that openFileRepository() + // rebuilds the commit-id database to match the new HEAD. + forceMoveMaster(old.jGitRepository(), resetToCommitId); + neo = openFileRepository(parent, repoDir, repositoryWorker, cache); + + // Replay the source commits so every replica converges to the same commit ids. + // + // The commits are applied on this thread rather than through the asynchronous commit(), which + // dispatches to the repository worker: this method already runs on a repository-worker thread + // and holds the write lock of the repository it is replacing, so a read of that repository + // parks another worker thread on the lock. Waiting here for a task queued back to the same + // pool would deadlock it once every worker is parked. + for (ReplayCommit commit : commits) { + final Revision revision = commit.revision(); + final CommitResult result = neo.blockingCommit( + revision.backward(1), commit.timestampMillis(), commit.author(), commit.summary(), + commit.detail(), commit.markup(), commit.changes()); + if (!revision.equals(result.revision())) { + throw new StorageException("unexpected replayed revision: " + result.revision() + + " (expected: " + revision + ')'); + } + final String expectedCommitId = commit.expectedCommitId(); + final String actualCommitId = neo.commitIdDatabase().get(revision).name(); + if (!expectedCommitId.equals(actualCommitId)) { + throw new StorageException( + "commit id mismatch while recovering '" + + projectRepositoryName(repositoryName) + "' at " + revision + " (expected: " + + expectedCommitId + ", actual: " + actualCommitId + "). Revisions up to " + + resetToRevision + " may have diverged, or the content is not reproducible " + + "byte-identically (e.g. written by a content transformer); the repository " + + "was rolled back."); + } + } + + if (!replaceChild(repositoryName, old, neo)) { + throw new StorageException("failed to replace the repository after recovery: " + + projectRepositoryName(repositoryName)); + } + swapped = true; + // The old instance shares the on-disk state that was just rewritten, so a reader that was + // blocked on its lock must fail fast instead of reading through the stale instance. + old.markClosePending(() -> new CentralDogmaException( + projectRepositoryName(repositoryName) + " is recovered. Try again.")); + } catch (Throwable t) { + throw new StorageException("failed to recover the repository '" + + projectRepositoryName(repositoryName) + "' (reset to " + + resetToRevision + ')', t); + } finally { + if (!swapped) { + // Roll back so the (still diverged) old repository stays internally consistent. Both closes + // run on this thread: the write lock of `old` is still held, so waiting for the repository + // worker here would deadlock it. + if (neo != null) { + try { + neo.closeInline(() -> new CentralDogmaException("should never reach here")); + } catch (Throwable t2) { + logger.warn("Failed to close the partially recovered repository '{}'.", + projectRepositoryName(repositoryName), t2); + } + } + try { + forceMoveMaster(old.jGitRepository(), originalHeadCommitId); + old.commitIdDatabase().rebuild(old.jGitRepository()); + } catch (Throwable t2) { + // The reset already rewrote the commit-id database that `old` shares, so a failed + // rollback leaves it inconsistent with the repository. Fail every read through this + // instance rather than serving a half-rewritten history; a restart rebuilds it. + old.markClosePending(() -> new CentralDogmaException( + projectRepositoryName(repositoryName) + " is corrupted by a failed recovery " + + "rollback. Restart this replica to rebuild it.")); + logger.error("Failed to roll back the repository '{}' after a failed recovery. " + + "It is left unreadable until this replica is restarted.", + projectRepositoryName(repositoryName), t2); + } + } + old.writeUnLock(); + } + + // The swap already succeeded, so the recovery must be reported as successful and the old + // repository must be released even if a listener transfer or the callback fails. + try { + for (RepositoryListener listener : old.listeners()) { + neo.addListener(listener); + } + final BiConsumer callback = postMigrationCallback; + if (callback != null) { + callback.accept(repositoryName, neo); + } + } catch (Throwable t) { + logger.warn("Failed to hand over the listeners of the recovered repository '{}'.", + projectRepositoryName(repositoryName), t); + } finally { + old.closeInline(() -> new CentralDogmaException( + projectRepositoryName(repositoryName) + " is recovered. Try again.")); + } + logger.info("Recovered the repository '{}' to {} in {} seconds.", + projectRepositoryName(repositoryName), neo.normalizeNow(Revision.HEAD), + TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime)); + return true; + } + + private static void forceMoveMaster(org.eclipse.jgit.lib.Repository jGitRepository, ObjectId commitId) { + try (RevWalk revWalk = newRevWalk(jGitRepository.newObjectReader())) { + GitRepository.doForceRefUpdate(jGitRepository, revWalk, R_HEADS_MASTER, commitId); + } catch (IOException e) { + throw new StorageException("failed to force-move " + R_HEADS_MASTER + " to " + commitId.name(), e); + } + } + + @Override + public List buildRecoveryPayload(String repositoryName, Revision fromRevision) { + requireNonNull(repositoryName, "repositoryName"); + requireNonNull(fromRevision, "fromRevision"); + final GitRepository repo = (GitRepository) get(repositoryName); + if (repo.isEncrypted()) { + throw new StorageException("recovery is not supported for an encrypted repository: " + + projectRepositoryName(repositoryName)); + } + final Revision headRevision = repo.normalizeNow(Revision.HEAD); + if (headRevision.major() < 2) { + throw new IllegalArgumentException( + "the repository has no replayable revision: " + projectRepositoryName(repositoryName) + + " (head: " + headRevision + ')'); + } + final int from = fromRevision.major(); + if (fromRevision.isRelative() || from < 2 || from > headRevision.major()) { + throw new IllegalArgumentException( + "fromRevision: " + fromRevision + " (expected: an absolute revision in [2, " + + headRevision.major() + "])"); + } + + final int commitCount = headRevision.major() - from + 1; + validateRecoveryPayloadSize(projectRepositoryName(repositoryName), commitCount, 0); + long payloadBytes = 0; + final ImmutableList.Builder commits = + ImmutableList.builderWithExpectedSize(commitCount); + for (int i = from; i <= headRevision.major(); i++) { + final Revision revision = new Revision(i); + final Commit commit = repo.history(revision, revision, Repository.ALL_PATH).join().get(0); + final Map> changes = + repo.diff(revision.backward(1), revision, Repository.ALL_PATH, + DiffResultType.PATCH_TO_TEXT_UPSERT).join(); + for (Change change : changes.values()) { + payloadBytes += estimateSize(change); + } + validateRecoveryPayloadSize(projectRepositoryName(repositoryName), commitCount, payloadBytes); + commits.add(new ReplayCommit(revision, commit.when(), commit.author(), commit.summary(), + commit.detail(), commit.markup(), changes.values(), + repo.commitIdDatabase().get(revision).name())); + } + return commits.build(); + } + + // The payload crosses the replication log as a single entry and is materialized in memory by every + // replica, so an unbounded payload could exhaust the heap cluster-wide and stall replication. + @VisibleForTesting + static int maxRecoveryCommits = 10_000; + @VisibleForTesting + static long maxRecoveryPayloadBytes = 64 * 1024 * 1024; + + private void validateRecoveryPayloadSize(String name, int commitCount, long payloadBytes) { + if (commitCount > maxRecoveryCommits) { + throw new IllegalArgumentException( + "the recovery of " + name + " spans too many revisions: " + commitCount + + " (maximum: " + maxRecoveryCommits + "). Recover from a later revision."); + } + if (payloadBytes > maxRecoveryPayloadBytes) { + throw new IllegalArgumentException( + "the recovery payload of " + name + " is larger than " + maxRecoveryPayloadBytes + + " bytes. Recover from a later revision."); + } + } + + // Counts UTF-8 bytes rather than chars, so that non-ASCII content is not under-counted several-fold. + // JSON escaping is not counted, so this stays a lower bound on the serialized size. + private static long estimateSize(Change change) { + final Object content = change.content(); + long size = Utf8.encodedLength(change.path()); + if (content instanceof String) { + size += Utf8.encodedLength((String) content); + } else if (content != null) { + size += Utf8.encodedLength(content.toString()); + } + return size; + } + @Override protected Repository openChild(File childDir) throws Exception { requireNonNull(childDir, "childDir"); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java index f850ddec0..3ac8ecc1c 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java @@ -83,6 +83,15 @@ public interface Repository { */ org.eclipse.jgit.lib.Repository jGitRepository(); + /** + * Returns the head of this repository: its head {@link Revision} together with the ID of the commit + * that revision points at, read as one so the two always describe the same commit. Blocks while the + * repository is being rewritten, and fails once it has been replaced. + */ + default RepositoryHead head() { + throw new UnsupportedOperationException(); + } + /** * Returns the parent {@link Project} of this {@link Repository}. */ diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryHead.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryHead.java new file mode 100644 index 000000000..3f7566622 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryHead.java @@ -0,0 +1,68 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.storage.repository; + +import static java.util.Objects.requireNonNull; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; + +import com.linecorp.centraldogma.common.Revision; + +/** + * The head of a repository on a single replica, identified by both its revision and its commit ID. The + * two are read together so that they always describe the same commit. Replicas of the same repository may + * report the same revision even when their histories have diverged, so only the commit ID proves that they + * hold the same history. + */ +public final class RepositoryHead { + + private final Revision revision; + private final String commitId; + + /** + * Creates a new instance. + */ + public RepositoryHead(Revision revision, String commitId) { + this.revision = requireNonNull(revision, "revision"); + this.commitId = requireNonNull(commitId, "commitId"); + } + + /** + * Returns the head revision. + */ + @JsonProperty("revision") + public Revision revision() { + return revision; + } + + /** + * Returns the ID of the commit the head revision points at. + */ + @JsonProperty("commitId") + public String commitId() { + return commitId; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("revision", revision) + .add("commitId", commitId) + .toString(); + } +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryManager.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryManager.java index b57eea3c9..e6a52b8ab 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryManager.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/RepositoryManager.java @@ -16,8 +16,11 @@ package com.linecorp.centraldogma.server.storage.repository; +import java.util.List; import java.util.function.BiConsumer; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.command.ReplayCommit; import com.linecorp.centraldogma.server.storage.StorageManager; import com.linecorp.centraldogma.server.storage.project.Project; @@ -40,6 +43,30 @@ public interface RepositoryManager extends StorageManager { */ void fallbackToFileRepository(String repositoryName); + /** + * Recovers the specified repository by resetting it to {@code resetToRevision} and replaying the given + * {@code commits} on top of it. Used to reconcile a diverged replica with a source replica. See + * {@link com.linecorp.centraldogma.server.command.RecoverRepositoryCommand}. + * + * @return {@code true} if the repository was rewritten and its {@link Repository} instance replaced; + * {@code false} if it was already converged with {@code commits} and thus left untouched, which + * is the outcome on the source replica and on every replica that did not diverge. + */ + default boolean recoverRepository(String repositoryName, Revision resetToRevision, + List commits) { + throw new UnsupportedOperationException(); + } + + /** + * Builds the {@link ReplayCommit}s of {@code fromRevision..HEAD} of the specified repository, to be + * carried by a {@link com.linecorp.centraldogma.server.command.RecoverRepositoryCommand}. Invoked only + * on the source replica of a recovery. {@code fromRevision} must be an absolute revision greater than 1 + * and not greater than the HEAD revision. + */ + default List buildRecoveryPayload(String repositoryName, Revision fromRevision) { + throw new UnsupportedOperationException(); + } + /** * Sets a callback that is invoked after a repository is migrated or fallen back. * The callback receives the repository name and the new {@link Repository} instance. diff --git a/server/src/test/java/com/linecorp/centraldogma/server/command/RecoverRepositoryCommandTest.java b/server/src/test/java/com/linecorp/centraldogma/server/command/RecoverRepositoryCommandTest.java new file mode 100644 index 000000000..769904ea0 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/command/RecoverRepositoryCommandTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.command; + +import static com.linecorp.centraldogma.testing.internal.TestUtil.assertJsonConversion; + +import org.junit.jupiter.api.Test; + +import com.google.common.collect.ImmutableList; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Revision; + +class RecoverRepositoryCommandTest { + + // The command crosses the replication log as JSON, so the wire format must stay stable. + @Test + void testJsonConversion() { + assertJsonConversion( + new RecoverRepositoryCommand( + 1234L, Author.SYSTEM, "foo", "bar", 2, new Revision(2), new Revision(4), + ImmutableList.of( + new ReplayCommit(new Revision(3), 5678L, + new Author("Marge Simpson", "marge@simpsonsworld.com"), + "summary3", "detail3", Markup.PLAINTEXT, + ImmutableList.of( + Change.ofTextUpsert("/memo.txt", "Bon voyage!"), + Change.ofRemoval("/old.txt")), + "1111111111111111111111111111111111111111"), + new ReplayCommit(new Revision(4), 6789L, Author.SYSTEM, + "summary4", "", Markup.PLAINTEXT, + ImmutableList.of(Change.ofTextUpsert("/memo.txt", "v4")), + "0123456789012345678901234567890123456789"))), + Command.class, + '{' + + " \"type\": \"RECOVER_REPOSITORY\"," + + " \"timestamp\": 1234," + + " \"author\": {" + + " \"name\": \"system\"," + + " \"email\": \"system@localhost.localdomain\"" + + " }," + + " \"projectName\": \"foo\"," + + " \"repositoryName\": \"bar\"," + + " \"sourceServerId\": 2," + + " \"resetToRevision\": 2," + + " \"headRevision\": 4," + + " \"commits\": [{" + + " \"revision\": 3," + + " \"timestampMillis\": 5678," + + " \"author\": {" + + " \"name\": \"Marge Simpson\"," + + " \"email\": \"marge@simpsonsworld.com\"" + + " }," + + " \"summary\": \"summary3\"," + + " \"detail\": \"detail3\"," + + " \"markup\": \"PLAINTEXT\"," + + " \"changes\": [{" + + " \"type\": \"UPSERT_TEXT\"," + + " \"path\": \"/memo.txt\"," + + " \"content\": \"Bon voyage!\"" + + " }, {" + + " \"type\": \"REMOVE\"," + + " \"path\": \"/old.txt\"" + + " }]," + + " \"expectedCommitId\": \"1111111111111111111111111111111111111111\"" + + " }, {" + + " \"revision\": 4," + + " \"timestampMillis\": 6789," + + " \"author\": {" + + " \"name\": \"system\"," + + " \"email\": \"system@localhost.localdomain\"" + + " }," + + " \"summary\": \"summary4\"," + + " \"detail\": \"\"," + + " \"markup\": \"PLAINTEXT\"," + + " \"changes\": [{" + + " \"type\": \"UPSERT_TEXT\"," + + " \"path\": \"/memo.txt\"," + + " \"content\": \"v4\"" + + " }]," + + " \"expectedCommitId\": \"0123456789012345678901234567890123456789\"" + + " }]" + + '}'); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/command/RecoverRepositoryRequestCommandTest.java b/server/src/test/java/com/linecorp/centraldogma/server/command/RecoverRepositoryRequestCommandTest.java new file mode 100644 index 000000000..821784159 --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/command/RecoverRepositoryRequestCommandTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.command; + +import static com.linecorp.centraldogma.testing.internal.TestUtil.assertJsonConversion; + +import org.junit.jupiter.api.Test; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Revision; + +class RecoverRepositoryRequestCommandTest { + + // The command crosses the replication log as JSON, so the wire format must stay stable. + @Test + void testJsonConversion() { + assertJsonConversion( + new RecoverRepositoryRequestCommand(1234L, Author.SYSTEM, "foo", "bar", 2, new Revision(3)), + Command.class, + '{' + + " \"type\": \"RECOVER_REPOSITORY_REQUEST\"," + + " \"timestamp\": 1234," + + " \"author\": {" + + " \"name\": \"system\"," + + " \"email\": \"system@localhost.localdomain\"" + + " }," + + " \"projectName\": \"foo\"," + + " \"repositoryName\": \"bar\"," + + " \"sourceServerId\": 2," + + " \"fromRevision\": 3" + + '}'); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1Test.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1Test.java index 01d032e50..64b3c8f57 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1Test.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryServiceV1Test.java @@ -313,6 +313,29 @@ void updateInternalRepositoryStatus() { assertThat(credential.status()).isSameAs(HttpStatus.CREATED); } + @Test + void recoverRepository_gating() { + final String repoName = "recoverRepo"; + assertThat(createRepository(systemAdminClient, repoName).status()).isEqualTo(HttpStatus.CREATED); + + // A non-admin user cannot start a recovery. + final AggregatedHttpResponse userRes = + userClient.blocking().prepare() + .post(REPOS_PREFIX + '/' + repoName + "/recover") + .contentJson(new RecoverRepositoryRequest(2, 1)) + .execute(); + assertThat(userRes.status()).isEqualTo(HttpStatus.FORBIDDEN); + + // Recovery is rejected in standalone (non-replicated) mode. + final AggregatedHttpResponse adminRes = + systemAdminClient.blocking().prepare() + .post(REPOS_PREFIX + '/' + repoName + "/recover") + .contentJson(new RecoverRepositoryRequest(2, 1)) + .execute(); + assertThat(adminRes.status()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(adminRes.contentUtf8()).contains("replicated"); + } + private static ResponseEntity updateStatus(ReplicationStatus status, String repoName) { final BlockingWebClient client = systemAdminClient.blocking(); return client.prepare() diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryStatusMetricsTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryStatusMetricsTest.java new file mode 100644 index 000000000..08911ee5e --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/RepositoryStatusMetricsTest.java @@ -0,0 +1,116 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.api; + +import static com.linecorp.centraldogma.internal.api.v1.HttpApiV1Constants.API_V1_PATH_PREFIX; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.linecorp.armeria.client.BlockingWebClient; +import com.linecorp.armeria.client.WebClientBuilder; +import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.centraldogma.common.ReplicationStatus; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +/** + * Verifies that the {@code repository.read.only} metric is refreshed through the + * {@link com.linecorp.centraldogma.server.command.StandaloneCommandExecutor} hooks as a read-only + * repository is soft-removed, restored and purged. Unlike {@code RepoStatusManagerTest}, this drives + * the real command-executor path so the hooks themselves are exercised (not simulated). + */ +class RepositoryStatusMetricsTest { + + static final MeterRegistry meterRegistry = new SimpleMeterRegistry(); + + @RegisterExtension + static final CentralDogmaExtension dogma = new CentralDogmaExtension() { + @Override + protected void configure(CentralDogmaBuilder builder) { + builder.meterRegistry(meterRegistry); + } + + @Override + protected void configureHttpClient(WebClientBuilder builder) { + builder.addHeader(HttpHeaderNames.AUTHORIZATION, "Bearer anonymous"); + } + }; + + @Test + void readOnlyScopeMetricFollowsRepositoryLifecycle() { + final BlockingWebClient client = dogma.httpClient().blocking(); + dogma.client().createProject("mp").join(); + dogma.client().createRepository("mp", "mr").join(); + setReadOnly(client, "mp", "mr"); + + // Marked read-only -> a metric row exists. + await().untilAsserted(() -> assertThat(readOnlyGauge("mp", "mr")).isNotNull()); + + // Soft-remove -> the removeRepository hook must drop the metric row. + dogma.client().removeRepository("mp", "mr").join(); + await().untilAsserted(() -> assertThat(readOnlyGauge("mp", "mr")).isNull()); + + // Restore -> the unremoveRepository hook must bring it back. + dogma.client().unremoveRepository("mp", "mr").join(); + await().untilAsserted(() -> assertThat(readOnlyGauge("mp", "mr")).isNotNull()); + + // Remove + purge -> the purgeRepository hook must delete it permanently. + dogma.client().removeRepository("mp", "mr").join(); + dogma.client().purgeRepository("mp", "mr").join(); + await().untilAsserted(() -> assertThat(readOnlyGauge("mp", "mr")).isNull()); + } + + @Test + void purgingANonRemovedRepositoryKeepsItsReadOnlyStatus() { + final BlockingWebClient client = dogma.httpClient().blocking(); + dogma.client().createProject("kp").join(); + dogma.client().createRepository("kp", "kr").join(); + setReadOnly(client, "kp", "kr"); + await().untilAsserted(() -> assertThat(readOnlyGauge("kp", "kr")).isNotNull()); + + // Purge without removing first: markForPurge() is a no-op, so the read-only status must be + // kept. Deleting it here would silently defeat read-only enforcement on a still-active repo. + try { + dogma.client().purgeRepository("kp", "kr").join(); + } catch (Exception e) { + // Purging a non-removed repository may be rejected at the metadata layer; the only thing + // under test here is that the read-only status was not deleted by the command. + } + // The read-only status (and hence its metric row) is still there. + assertThat(readOnlyGauge("kp", "kr")).isNotNull(); + } + + private static void setReadOnly(BlockingWebClient client, String project, String repo) { + client.prepare() + .put(API_V1_PATH_PREFIX + "projects/" + project + "/repos/" + repo + "/status") + .contentJson(new UpdateRepositoryStatusRequest(ReplicationStatus.READ_ONLY)) + .execute(); + } + + private static Gauge readOnlyGauge(String project, String repo) { + return meterRegistry.find("repository.read.only") + .tags("project", project, "repo", repo) + .gauge(); + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ServerStatusServiceTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ServerStatusServiceTest.java index 70a80d9cc..dee12221e 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ServerStatusServiceTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ServerStatusServiceTest.java @@ -95,6 +95,49 @@ void readOnlyRepositories() { }); } + @Test + void readOnlyRepositoryHiddenWhenRemovedAndPurged() { + final BlockingWebClient client = dogma.httpClient().blocking(); + + dogma.client().createProject("del").join(); + dogma.client().createRepository("del", "ro").join(); + final AggregatedHttpResponse updateRes = + client.prepare() + .put(API_V1_PATH_PREFIX + "projects/del/repos/ro/status") + .contentJson(new UpdateRepositoryStatusRequest(ReplicationStatus.READ_ONLY)) + .execute(); + assertThat(updateRes.status()).isEqualTo(HttpStatus.OK); + + // The repository is listed while read-only. + await().untilAsserted(() -> { + final AggregatedHttpResponse res = client.get(API_V1_PATH_PREFIX + "status/repos/read-only"); + assertThat(res.status()).isEqualTo(HttpStatus.OK); + assertThatJson(res.contentUtf8()).isArray().ofLength(1); + assertThatJson(res.contentUtf8()).node("[0].projectName").isEqualTo("del"); + assertThatJson(res.contentUtf8()).node("[0].repoName").isEqualTo("ro"); + }); + + // Soft-removing the repository hides it from the read-only list. + dogma.client().removeRepository("del", "ro").join(); + await().untilAsserted(() -> assertThat( + client.get(API_V1_PATH_PREFIX + "status/repos/read-only").status()) + .isEqualTo(HttpStatus.NO_CONTENT)); + + // Purging the removed repository keeps it gone. + dogma.client().purgeRepository("del", "ro").join(); + await().untilAsserted(() -> assertThat( + client.get(API_V1_PATH_PREFIX + "status/repos/read-only").status()) + .isEqualTo(HttpStatus.NO_CONTENT)); + } + + @Test + void replicas_emptyInStandaloneMode() { + final BlockingWebClient client = dogma.httpClient().blocking(); + // An empty list is returned as 204 No Content in standalone (non-replicated) mode. + assertThat(client.get(API_V1_PATH_PREFIX + "replicas").status()) + .isEqualTo(HttpStatus.NO_CONTENT); + } + @Test void updateStatus_setUnwritable() { final AggregatedHttpResponse res = updateStatus(ServerStatus.REPLICATION_ONLY); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManagerTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManagerTest.java index 1a14e42a8..60f9a7434 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManagerTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManagerTest.java @@ -20,12 +20,21 @@ import static org.assertj.core.api.Assertions.tuple; import static org.awaitility.Awaitility.await; +import java.util.Map; + +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.extension.RegisterExtension; import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.ReplicationStatus; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; +import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.testing.internal.ProjectManagerExtension; import io.micrometer.core.instrument.Gauge; @@ -33,6 +42,7 @@ import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +@Timeout(120) class RepoStatusManagerTest { @RegisterExtension @@ -86,30 +96,205 @@ void projectStatus_cruTest() { @Test void readOnlyStatuses_and_metrics() { - assertThat(statusManager.readOnlyStatuses()).isEmpty(); - assertThat(readOnlyCount()).isZero(); + final ProjectManager pm = pmExtension.projectManager(); + pm.create("test_prj", Author.SYSTEM); + pm.get("test_prj").repos().create("test_repo", Author.SYSTEM); + pm.create("test_prj2", Author.SYSTEM); + try { + assertThat(statusManager.readOnlyStatuses()).isEmpty(); + assertThat(readOnlyCount()).isZero(); - statusManager.updateRepoStatus("test_prj", "test_repo", Author.DEFAULT, ReplicationStatus.READ_ONLY) - .join(); - statusManager.updateProjectStatus("test_prj2", Author.DEFAULT, ReplicationStatus.READ_ONLY).join(); + statusManager.updateRepoStatus("test_prj", "test_repo", Author.DEFAULT, + ReplicationStatus.READ_ONLY).join(); + statusManager.updateProjectStatus("test_prj2", Author.DEFAULT, ReplicationStatus.READ_ONLY) + .join(); - await().untilAsserted(() -> assertThat(statusManager.readOnlyStatuses()) - .extracting(RepositoryState::projectName, RepositoryState::repoName, RepositoryState::status) - .containsExactlyInAnyOrder( - tuple("test_prj", "test_repo", ReplicationStatus.READ_ONLY), - tuple("test_prj2", "dogma", ReplicationStatus.READ_ONLY))); + await().untilAsserted(() -> assertThat(statusManager.readOnlyStatuses()) + .extracting(RepositoryState::projectName, RepositoryState::repoName, + RepositoryState::status) + .containsExactlyInAnyOrder( + tuple("test_prj", "test_repo", ReplicationStatus.READ_ONLY), + tuple("test_prj2", "dogma", ReplicationStatus.READ_ONLY))); - assertThat(readOnlyCount()).isEqualTo(2); - // A project-scoped entry is the one whose repository is "dogma". - assertThat(readOnlyGauge("test_prj", "test_repo")).isOne(); - assertThat(readOnlyGauge("test_prj2", "dogma")).isOne(); + assertThat(readOnlyCount()).isEqualTo(2); + // A project-scoped entry is the one whose repository is "dogma". + assertThat(readOnlyGauge("test_prj", "test_repo")).isOne(); + assertThat(readOnlyGauge("test_prj2", "dogma")).isOne(); - // Reverting to WRITABLE removes the entry from the list and the metrics. - statusManager.updateRepoStatus("test_prj", "test_repo", Author.DEFAULT, ReplicationStatus.WRITABLE) + // Reverting to WRITABLE removes the entry from the list and the metrics. + statusManager.updateRepoStatus("test_prj", "test_repo", Author.DEFAULT, + ReplicationStatus.WRITABLE).join(); + statusManager.updateProjectStatus("test_prj2", Author.DEFAULT, ReplicationStatus.WRITABLE).join(); + await().untilAsserted(() -> assertThat(statusManager.readOnlyStatuses()).isEmpty()); + assertThat(readOnlyCount()).isZero(); + } finally { + statusManager.removeRepoStatus("test_prj", "test_repo", Author.DEFAULT).join(); + statusManager.removeProjectStatus("test_prj2", Author.DEFAULT).join(); + } + } + + @Test + void softDeletedRepositoryIsHiddenAndRestored() { + final ProjectManager pm = pmExtension.projectManager(); + pm.create("readonly_soft", Author.SYSTEM); + pm.get("readonly_soft").repos().create("repo", Author.SYSTEM); + try { + statusManager.updateRepoStatus("readonly_soft", "repo", Author.DEFAULT, + ReplicationStatus.READ_ONLY).join(); + await().untilAsserted(() -> assertThat(statusManager.readOnlyStatuses()) + .extracting(RepositoryState::projectName, RepositoryState::repoName) + .containsExactly(tuple("readonly_soft", "repo"))); + assertThat(readOnlyCount()).isOne(); + assertThat(readOnlyGauge("readonly_soft", "repo")).isOne(); + + // Soft-removing the repository hides it from the list and metrics but keeps the status file. + pm.get("readonly_soft").repos().remove("repo"); + statusManager.refreshReadOnlyMetrics(); + assertThat(statusManager.readOnlyStatuses()).isEmpty(); + assertThat(readOnlyCount()).isZero(); + assertThat(readOnlyGaugeOrNull("readonly_soft", "repo")).isNull(); + // The status is preserved, so getRepoStatus() still reports READ_ONLY. + assertThat(statusManager.getRepoStatus("readonly_soft", "repo").status()) + .isEqualTo(ReplicationStatus.READ_ONLY); + + // Restoring the repository brings the read-only status back. + pm.get("readonly_soft").repos().unremove("repo"); + statusManager.refreshReadOnlyMetrics(); + assertThat(statusManager.readOnlyStatuses()) + .extracting(RepositoryState::projectName, RepositoryState::repoName) + .containsExactly(tuple("readonly_soft", "repo")); + assertThat(readOnlyCount()).isOne(); + assertThat(readOnlyGauge("readonly_soft", "repo")).isOne(); + } finally { + // Clean up the shared state so other tests start from an empty read-only list. + statusManager.removeRepoStatus("readonly_soft", "repo", Author.DEFAULT).join(); + } + } + + @Test + void softDeletedProjectIsHiddenAndRestored() { + final ProjectManager pm = pmExtension.projectManager(); + pm.create("readonly_prj_soft", Author.SYSTEM); + pm.get("readonly_prj_soft").repos().create("repo", Author.SYSTEM); + try { + statusManager.updateRepoStatus("readonly_prj_soft", "repo", Author.DEFAULT, + ReplicationStatus.READ_ONLY).join(); + statusManager.updateProjectStatus("readonly_prj_soft", Author.DEFAULT, + ReplicationStatus.READ_ONLY).join(); + await().untilAsserted(() -> assertThat(statusManager.readOnlyStatuses()) + .extracting(RepositoryState::projectName, RepositoryState::repoName) + .containsExactlyInAnyOrder(tuple("readonly_prj_soft", "repo"), + tuple("readonly_prj_soft", "dogma"))); + assertThat(readOnlyCount()).isEqualTo(2); + + // Soft-removing the whole project hides both the repo-scope and project-scope entries. + pm.remove("readonly_prj_soft"); + statusManager.refreshReadOnlyMetrics(); + assertThat(statusManager.readOnlyStatuses()).isEmpty(); + assertThat(readOnlyCount()).isZero(); + assertThat(readOnlyGaugeOrNull("readonly_prj_soft", "repo")).isNull(); + assertThat(readOnlyGaugeOrNull("readonly_prj_soft", "dogma")).isNull(); + + // Restoring the project brings both entries back. + pm.unremove("readonly_prj_soft"); + statusManager.refreshReadOnlyMetrics(); + assertThat(statusManager.readOnlyStatuses()) + .extracting(RepositoryState::projectName, RepositoryState::repoName) + .containsExactlyInAnyOrder(tuple("readonly_prj_soft", "repo"), + tuple("readonly_prj_soft", "dogma")); + assertThat(readOnlyCount()).isEqualTo(2); + } finally { + statusManager.removeProjectStatus("readonly_prj_soft", Author.DEFAULT).join(); + } + } + + @Test + void purgingRepositoryRemovesStatusAndMetrics() { + final ProjectManager pm = pmExtension.projectManager(); + pm.create("readonly_purge", Author.SYSTEM); + pm.get("readonly_purge").repos().create("repo", Author.SYSTEM); + try { + statusManager.updateRepoStatus("readonly_purge", "repo", Author.DEFAULT, + ReplicationStatus.READ_ONLY).join(); + await().untilAsserted(() -> assertThat(statusManager.readOnlyStatuses()).hasSize(1)); + assertThat(readOnlyCount()).isOne(); + + // Purging deletes the status file and evicts the cache entry and metrics directly. + statusManager.removeRepoStatus("readonly_purge", "repo", Author.DEFAULT).join(); + assertThat(statusManager.readOnlyStatuses()).isEmpty(); + assertThat(readOnlyCount()).isZero(); + assertThat(readOnlyGaugeOrNull("readonly_purge", "repo")).isNull(); + // The status file is gone, so the repository is writable again even though it still exists. + assertThat(statusManager.getRepoStatus("readonly_purge", "repo").status()) + .isEqualTo(ReplicationStatus.WRITABLE); + } finally { + statusManager.removeRepoStatus("readonly_purge", "repo", Author.DEFAULT).join(); + } + } + + @Test + void purgingProjectRemovesAllStatuses() { + final ProjectManager pm = pmExtension.projectManager(); + pm.create("readonly_proj", Author.SYSTEM); + pm.get("readonly_proj").repos().create("repo", Author.SYSTEM); + try { + statusManager.updateRepoStatus("readonly_proj", "repo", Author.DEFAULT, + ReplicationStatus.READ_ONLY).join(); + statusManager.updateProjectStatus("readonly_proj", Author.DEFAULT, + ReplicationStatus.READ_ONLY).join(); + await().untilAsserted(() -> assertThat(statusManager.readOnlyStatuses()).hasSize(2)); + + statusManager.removeProjectStatus("readonly_proj", Author.DEFAULT).join(); + assertThat(statusManager.readOnlyStatuses()).isEmpty(); + assertThat(readOnlyCount()).isZero(); + } finally { + statusManager.removeProjectStatus("readonly_proj", Author.DEFAULT).join(); + } + } + + /** + * The purge-time cleanup must not trust the in-memory cache: a replica that replays a purge before the + * status listener's initial snapshot lands would otherwise keep the status file forever, while its + * peers delete it - leaving its internal dogma repository behind the rest of the cluster. + */ + @Test + void purgeCleanupDoesNotTrustAColdCache() { + statusManager.updateRepoStatus("cold_purge", "repo", Author.DEFAULT, ReplicationStatus.READ_ONLY) .join(); - statusManager.updateProjectStatus("test_prj2", Author.DEFAULT, ReplicationStatus.WRITABLE).join(); - await().untilAsserted(() -> assertThat(statusManager.readOnlyStatuses()).isEmpty()); - assertThat(readOnlyCount()).isZero(); + statusManager.updateProjectStatus("cold_purge2", Author.DEFAULT, ReplicationStatus.READ_ONLY).join(); + awaitStatus("cold_purge", "repo", ReplicationStatus.READ_ONLY); + awaitStatus("cold_purge2", "dogma", ReplicationStatus.READ_ONLY); + assertThat(statusFiles("cold_purge")).isNotEmpty(); + assertThat(statusFiles("cold_purge2")).isNotEmpty(); + + // A manager whose listener was never registered has an empty cache; it must still delete the + // replicated status files. + final RepoStatusManager coldManager = newColdManager(); + coldManager.removeRepoStatus("cold_purge", "repo", Author.DEFAULT).join(); + coldManager.removeProjectStatus("cold_purge2", Author.DEFAULT).join(); + + assertThat(statusFiles("cold_purge")).isEmpty(); + assertThat(statusFiles("cold_purge2")).isEmpty(); + } + + /** + * Reads the replicated status files of the project straight out of the internal dogma repository, + * bypassing every in-memory cache. + */ + private Map> statusFiles(String projectName) { + return pmExtension.projectManager() + .get(InternalProjectInitializer.INTERNAL_PROJECT_DOGMA) + .repos().get(Project.REPO_DOGMA) + .find(Revision.HEAD, "/status/" + projectName + "/*.json").join(); + } + + /** + * Returns a manager whose {@link RepoStatusManager#initialize()} was never called, so its status cache + * stays empty and every answer must come from the replicated status files. + */ + private RepoStatusManager newColdManager() { + return new RepoStatusManager(pmExtension.serverStatusManager(), pmExtension.projectManager(), + new SimpleMeterRegistry()); } /** @@ -137,4 +322,11 @@ private double readOnlyGauge(String projectName, String repoName) { .containsExactlyInAnyOrder(Tag.of("project", projectName), Tag.of("repo", repoName)); return gauge.value(); } + + @Nullable + private Gauge readOnlyGaugeOrNull(String projectName, String repoName) { + return meterRegistry.find("repository.read.only") + .tags("project", projectName, "repo", repoName) + .gauge(); + } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/Replica.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/Replica.java index 5255d0260..98fe7f7bc 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/Replica.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/Replica.java @@ -81,7 +81,7 @@ protected void doStop(@Nullable Runnable onReleaseLeadership, protected CompletableFuture doExecute(ExecutionContext ctx, Command command) { return (CompletableFuture) delegate.apply(command); } - }, meterRegistry, null, null, null, null, null); + }, meterRegistry, null, null, null, null, null, null); commandExecutor.setLockTimeoutMillis(10000); startFuture = start ? commandExecutor.start() : null; diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperRepositoryRecoveryIntegrationTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperRepositoryRecoveryIntegrationTest.java new file mode 100644 index 000000000..60f1ac71a --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperRepositoryRecoveryIntegrationTest.java @@ -0,0 +1,460 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.replication; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.client.BlockingWebClient; +import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.ResponseEntity; +import com.linecorp.armeria.common.util.UnmodifiableFuture; +import com.linecorp.centraldogma.client.CentralDogma; +import com.linecorp.centraldogma.client.CentralDogmaRepository; +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Entry; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Query; +import com.linecorp.centraldogma.common.ReplicationStatus; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.jsonpatch.JsonPatchOperation; +import com.linecorp.centraldogma.internal.api.v1.RepositoryDto; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.server.CentralDogmaConfig; +import com.linecorp.centraldogma.server.command.Command; +import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.command.RepositoryCommand; +import com.linecorp.centraldogma.server.command.StandaloneCommandExecutor; +import com.linecorp.centraldogma.server.internal.api.RecoverRepositoryRequest; +import com.linecorp.centraldogma.server.internal.api.UpdateRepositoryStatusRequest; +import com.linecorp.centraldogma.server.internal.api.sysadmin.UpdateServerStatusRequest; +import com.linecorp.centraldogma.server.internal.api.sysadmin.UpdateServerStatusRequest.Scope; +import com.linecorp.centraldogma.server.management.ServerStatus; +import com.linecorp.centraldogma.server.plugin.Plugin; +import com.linecorp.centraldogma.server.plugin.PluginContext; +import com.linecorp.centraldogma.server.plugin.PluginTarget; +import com.linecorp.centraldogma.testing.internal.CentralDogmaReplicationExtension; + +/** + * Drives a repository into a truly diverged state (one replica applied a command the others did not) and + * verifies that {@code POST /api/v1/projects/{p}/repos/{r}/recover} reconverges every replica onto the + * designated source replica's history — through both the direct (endpoint on the source) and the + * request-and-react (endpoint on a non-source replica) paths. + */ +@Timeout(120) +class ZooKeeperRepositoryRecoveryIntegrationTest { + + private static final String TEST_REPO = "test-repo"; + private static final AtomicInteger testCounter = new AtomicInteger(); + private static final FaultInjector faultInjector = new FaultInjector(); + + // serverId is 1-based while servers() is 0-based: serverById(2) == servers().get(1). + private static final int DIVERGED_SERVER_ID = 2; + private static final int SOURCE_SERVER_ID = 1; + + @RegisterExtension + static final CentralDogmaReplicationExtension replica = new CentralDogmaReplicationExtension(3) { + @Override + protected void configureEach(int serverId, CentralDogmaBuilder builder) { + if (serverId == DIVERGED_SERVER_ID) { + builder.plugins(faultInjector); + } + } + }; + + private static CentralDogma client0; + + private String testProject; + + @BeforeEach + void beforeEach() { + client0 = replica.servers().get(0).client(); + + // Reset a server-wide REPLICATION_ONLY escalation from a previous test, if any. + replica.servers().get(0).blockingHttpClient() + .prepare() + .put("/api/v1/status") + .contentJson(new UpdateServerStatusRequest(ServerStatus.WRITABLE, Scope.ALL)) + .execute(); + for (int i = 0; i < 3; i++) { + final BlockingWebClient adminClient = replica.servers().get(i).blockingHttpClient(); + await().untilAsserted(() -> assertThat(getServerStatus(adminClient)) + .isEqualTo(ServerStatus.WRITABLE)); + } + + testProject = "recovery-project-" + testCounter.incrementAndGet(); + client0.createProject(testProject).join(); + client0.createRepository(testProject, TEST_REPO).join(); + } + + @Test + void recoverDivergedReplicaViaSourceReplica() { + driveRepoIntoDivergedReadOnly(); + + // POST the recovery to the source replica itself; it builds the payload and originates directly. + final AggregatedHttpResponse response = + recover(adminClientOf(SOURCE_SERVER_ID), new RecoverRepositoryRequest(3, SOURCE_SERVER_ID)); + assertThat(response.status()).isEqualTo(HttpStatus.OK); + assertThat(response.contentUtf8()).contains("\"COMPLETED\""); + assertThat(response.contentUtf8()).contains("\"headRevision\":3"); + + // Recovery is idempotent: running it again converges to the same head and changes nothing. + final AggregatedHttpResponse second = + recover(adminClientOf(SOURCE_SERVER_ID), new RecoverRepositoryRequest(3, SOURCE_SERVER_ID)); + assertThat(second.status()).isEqualTo(HttpStatus.OK); + assertThat(second.contentUtf8()).contains("\"COMPLETED\""); + assertThat(second.contentUtf8()).contains("\"headRevision\":3"); + + assertClusterConvergedAndUsable(); + } + + @Test + void recoverRejectedForOutOfRangeFromRevision() { + driveRepoIntoDivergedReadOnly(); + + // The source head is r3, so replaying from r99 is impossible; the direct path surfaces it as 400. + final AggregatedHttpResponse response = + recover(adminClientOf(SOURCE_SERVER_ID), new RecoverRepositoryRequest(99, SOURCE_SERVER_ID)); + assertThat(response.status()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(response.contentUtf8()).contains("fromRevision"); + } + + @Test + void replicasEndpointListsClusterRoster() { + for (int serverId = 1; serverId <= 3; serverId++) { + final ResponseEntity response = adminClientOf(serverId) + .prepare() + .get("/api/v1/replicas") + .asJson(JsonNode.class) + .execute(); + assertThat(response.status()).isEqualTo(HttpStatus.OK); + final JsonNode replicas = response.content(); + assertThat(replicas.size()).isEqualTo(3); + int currentCount = 0; + for (JsonNode replica : replicas) { + assertThat(replica.get("host").asText()).isNotEmpty(); + if (replica.get("current").asBoolean()) { + currentCount++; + // The replica marked as current is the one that served the request. + assertThat(replica.get("serverId").asInt()).isEqualTo(serverId); + } + } + assertThat(currentCount).isEqualTo(1); + } + } + + @Test + void recoverDivergedReplicaViaNonSourceReplica() { + driveRepoIntoDivergedReadOnly(); + + // POST the recovery to the diverged, non-source replica; it asks the source over the replication + // log and the source reacts by originating the actual recovery command. + final AggregatedHttpResponse response = + recover(adminClientOf(DIVERGED_SERVER_ID), new RecoverRepositoryRequest(2, SOURCE_SERVER_ID)); + assertThat(response.status()).isEqualTo(HttpStatus.OK); + assertThat(response.contentUtf8()).contains("\"REQUESTED\""); + + assertClusterConvergedAndUsable(); + } + + /** + * The read-only precondition is checked here, when the recovery is originated, and deliberately not + * again when the command is applied. The status lives under a different execution path than this + * command, so an operator making the repository writable again races the replay: replicas would + * disagree on an apply-time check, and a replica that failed it would skip the log entry for good and + * stay diverged in silence. The apply is instead a pure function of the payload, so every replica + * reaches the same state - at the documented cost of discarding a commit that raced the recovery. + */ + @Test + void recoverRejectedWhileWritable() { + // The repository is writable; recovery must be rejected so that no concurrent push can race the + // payload build. + final AggregatedHttpResponse response = + recover(adminClientOf(SOURCE_SERVER_ID), new RecoverRepositoryRequest(2, SOURCE_SERVER_ID)); + assertThat(response.status()).isEqualTo(HttpStatus.CONFLICT); + assertThat(response.contentUtf8()).contains("read-only"); + } + + @Test + void recoverRejectedForUnknownSourceServer() { + driveRepoIntoDivergedReadOnly(); + + final AggregatedHttpResponse response = + recover(adminClientOf(SOURCE_SERVER_ID), new RecoverRepositoryRequest(2, 42)); + assertThat(response.status()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(response.contentUtf8()).contains("sourceServerId"); + } + + @Test + void recoverRejectedForInternalRepository() { + // Internal repository content is written by content transformers, so it cannot be reproduced + // byte-identically by a replay. + final AggregatedHttpResponse response = + adminClientOf(SOURCE_SERVER_ID) + .prepare() + .post("/api/v1/projects/{project}/repos/{repo}/recover") + .pathParam("project", testProject) + .pathParam("repo", "dogma") + .contentJson(new RecoverRepositoryRequest(2, SOURCE_SERVER_ID)) + .execute(); + assertThat(response.status()).isEqualTo(HttpStatus.FORBIDDEN); + assertThat(response.contentUtf8()).contains("internal repository"); + } + + /** + * Produces the recovery scenario: the fault-injected replica applies an extra commit directly (not via + * the replication log), so replaying the next replicated commit fails there and the repository goes + * read-only cluster-wide, with the fault-injected replica truly diverged from the others. + * + *

Source history: r1 (creation), r2 {@code {"a": 1}}, r3 {@code {"a": 2}}. Diverged replica: + * r1, r2 and a local r3 {@code {"a": 3}}; the legitimate r3 was skipped. + */ + private void driveRepoIntoDivergedReadOnly() { + final CentralDogmaRepository repo = client0.forRepo(testProject, TEST_REPO); + repo.commit("seed", Change.ofJsonUpsert("/a.json", "{ \"a\": 1 }")).push().join(); + + faultInjector.injectFault(Command.push( + Author.DEFAULT, testProject, TEST_REPO, Revision.HEAD, + "inject fault", "", Markup.PLAINTEXT, + ImmutableList.of(Change.ofJsonUpsert("/a.json", "{ \"a\": 3 }")))); + + repo.commit("divergence", + Change.ofJsonPatch("/a.json", + JsonPatchOperation.safeReplace("/a", new IntNode(1), new IntNode(2)))) + .push() + .join(); + + // The failed replay escalates the repository into read-only cluster-wide. Assert it on a writable + // replica; a locally read-only replica composes its server status into every per-repo status. + await().untilAsserted(() -> assertThat( + getRepoStatus(adminClientOf(SOURCE_SERVER_ID), testProject, TEST_REPO).status()) + .isEqualTo(ReplicationStatus.READ_ONLY)); + + // The diverged replica really diverged: it holds its locally applied content instead of the + // replicated one. + await().ignoreExceptions().untilAsserted(() -> assertThat( + jsonValueOn(DIVERGED_SERVER_ID, "a")).isEqualTo(3)); + assertThat(jsonValueOn(SOURCE_SERVER_ID, "a")).isEqualTo(2); + } + + /** + * Asserts the post-recovery invariants: every replica converged to the source history, the repository + * stayed read-only until made writable, and afterwards a new push replicates cleanly to all replicas + * with no new read-only escalation. + */ + private void assertClusterConvergedAndUsable() { + // Every replica converges to the source content at the source head. Two replicas of a diverged + // repository share the head revision, so the commit ID is what actually proves convergence. + for (int serverId = 1; serverId <= 3; serverId++) { + final int id = serverId; + await().ignoreExceptions().untilAsserted(() -> { + assertThat(headRevisionOn(id).major()).isEqualTo(3); + assertThat(jsonValueOn(id, "a")).isEqualTo(2); + assertThat(headCommitIdOn(id)).isEqualTo(headCommitIdOn(SOURCE_SERVER_ID)); + }); + } + + // The repository stays read-only after recovery until the operator makes it writable. + assertThat(getRepoStatus(adminClientOf(SOURCE_SERVER_ID), testProject, TEST_REPO).status()) + .isEqualTo(ReplicationStatus.READ_ONLY); + final ResponseEntity writableRes = + adminClientOf(SOURCE_SERVER_ID) + .prepare() + .put("/api/v1/projects/{project}/repos/{repo}/status") + .pathParam("project", testProject) + .pathParam("repo", TEST_REPO) + .contentJson(new UpdateRepositoryStatusRequest(ReplicationStatus.WRITABLE)) + .asJson(RepositoryDto.class) + .execute(); + assertThat(writableRes.status()).isEqualTo(HttpStatus.OK); + + // A new push replays cleanly on every replica — including the recovered one — and triggers no new + // read-only escalation. + client0.forRepo(testProject, TEST_REPO) + .commit("after-recovery", Change.ofJsonUpsert("/a.json", "{ \"a\": 4 }")) + .push() + .join(); + for (int serverId = 1; serverId <= 3; serverId++) { + final int id = serverId; + await().ignoreExceptions().untilAsserted(() -> { + assertThat(headRevisionOn(id).major()).isEqualTo(4); + assertThat(jsonValueOn(id, "a")).isEqualTo(4); + }); + } + assertThat(getRepoStatus(adminClientOf(SOURCE_SERVER_ID), testProject, TEST_REPO).status()) + .isEqualTo(ReplicationStatus.WRITABLE); + } + + private AggregatedHttpResponse recover(BlockingWebClient client, RecoverRepositoryRequest request) { + return client.prepare() + .post("/api/v1/projects/{project}/repos/{repo}/recover") + .pathParam("project", testProject) + .pathParam("repo", TEST_REPO) + .contentJson(request) + .execute(); + } + + /** + * Returns the head commit ID of the repository as reported by the given replica. A diverged replica + * holds a different commit at the same revision, so this is the only value that distinguishes them. + */ + private String headCommitIdOn(int serverId) { + final ResponseEntity response = + adminClientOf(serverId) + .prepare() + .get("/api/v1/projects/{project}/repos/{repo}/head") + .pathParam("project", testProject) + .pathParam("repo", TEST_REPO) + .asJson(JsonNode.class) + .execute(); + assertThat(response.status()).isEqualTo(HttpStatus.OK); + return response.content().get("commitId").asText(); + } + + @Test + void headEndpointDistinguishesADivergedReplica() { + driveRepoIntoDivergedReadOnly(); + + // Same head revision on both replicas... + assertThat(headRevisionOn(DIVERGED_SERVER_ID)).isEqualTo(headRevisionOn(SOURCE_SERVER_ID)); + // ...but a different commit, which is exactly what makes the revision alone useless for + // verifying a recovery. + assertThat(headCommitIdOn(DIVERGED_SERVER_ID)).isNotEqualTo(headCommitIdOn(SOURCE_SERVER_ID)); + + recover(adminClientOf(SOURCE_SERVER_ID), new RecoverRepositoryRequest(3, SOURCE_SERVER_ID)); + await().ignoreExceptions().untilAsserted(() -> assertThat(headCommitIdOn(DIVERGED_SERVER_ID)) + .isEqualTo(headCommitIdOn(SOURCE_SERVER_ID))); + } + + private Revision headRevisionOn(int serverId) { + return replica.serverById(serverId).client() + .forRepo(testProject, TEST_REPO) + .normalize(Revision.HEAD) + .join(); + } + + private int jsonValueOn(int serverId, String field) { + final Entry entry = replica.serverById(serverId).client() + .forRepo(testProject, TEST_REPO) + .file(Query.ofJson("/a.json")) + .get() + .join(); + return entry.content().get(field).asInt(); + } + + private static BlockingWebClient adminClientOf(int serverId) { + return replica.serverById(serverId).blockingHttpClient(); + } + + private static RepositoryDto getRepoStatus(BlockingWebClient client, String projectName, + String repoName) { + final ResponseEntity response = + client.prepare() + .get("/api/v1/projects/{project}/repos/{repo}") + .pathParam("project", projectName) + .pathParam("repo", repoName) + .asJson(RepositoryDto.class) + .execute(); + assertThat(response.status()).isEqualTo(HttpStatus.OK); + return response.content(); + } + + private static ServerStatus getServerStatus(BlockingWebClient client) { + final ResponseEntity response = + client.prepare() + .get("/api/v1/status") + .asJson(ServerStatus.class) + .execute(); + assertThat(response.status()).isEqualTo(HttpStatus.OK); + return response.content(); + } + + private static final class FaultInjector implements Plugin { + + private StandaloneCommandExecutor commandExecutor; + private CommandExecutor zkCommandExecutor; + + /** + * Originates the command through the fault-injected replica's replicated executor. + */ + T zkExecute(Command command) { + return zkCommandExecutor.execute(command).join(); + } + + /** + * Applies the command directly on the fault-injected replica's local storage, bypassing the + * replication log, so that replica diverges from the rest of the cluster. + */ + void injectFault(Command command) { + final RepositoryCommand repositoryCommand = (RepositoryCommand) command; + final String projectName = repositoryCommand.projectName(); + final String repoName = repositoryCommand.repositoryName(); + // Wait for the seed commit to be replayed; otherwise the fault fires one log entry too early. + final Revision originHead = + client0.forRepo(projectName, repoName).normalize(Revision.HEAD).join(); + final CentralDogma injectorClient = replica.serverById(DIVERGED_SERVER_ID).client(); + await().untilAsserted(() -> { + final Revision localHead = + injectorClient.forRepo(projectName, repoName).normalize(Revision.HEAD).join(); + assertThat(localHead.major()).isGreaterThanOrEqualTo(originHead.major()); + }); + commandExecutor.execute(command).join(); + } + + @Override + public boolean isEnabled(CentralDogmaConfig config) { + return true; + } + + @Override + public PluginTarget target(CentralDogmaConfig config) { + return PluginTarget.ALL_REPLICAS; + } + + @Override + public CompletionStage start(PluginContext context) { + zkCommandExecutor = context.commandExecutor(); + commandExecutor = + (StandaloneCommandExecutor) ((ZooKeeperCommandExecutor) zkCommandExecutor).unwrap(); + return UnmodifiableFuture.completedFuture(null); + } + + @Override + public CompletionStage stop(PluginContext context) { + return UnmodifiableFuture.completedFuture(null); + } + + @Override + public Class configType() { + return getClass(); + } + } +} diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RecoverRepositoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RecoverRepositoryTest.java new file mode 100644 index 000000000..ab846835a --- /dev/null +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/git/RecoverRepositoryTest.java @@ -0,0 +1,288 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.server.internal.storage.repository.git; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; +import com.google.common.util.concurrent.MoreExecutors; + +import com.linecorp.centraldogma.common.Author; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.common.Markup; +import com.linecorp.centraldogma.common.Revision; +import com.linecorp.centraldogma.common.RevisionNotFoundException; +import com.linecorp.centraldogma.server.command.ReplayCommit; +import com.linecorp.centraldogma.server.storage.StorageException; +import com.linecorp.centraldogma.server.storage.encryption.NoopEncryptionStorageManager; +import com.linecorp.centraldogma.server.storage.project.Project; + +// A recovery that deadlocks would otherwise hang the build until the CI job is killed, which says nothing +// about which test broke. +@Timeout(60) +class RecoverRepositoryTest { + + private static final String REPO = "test_repo"; + + private Path tempDir; + + @BeforeEach + void setUp(@TempDir Path tempDir) { + this.tempDir = tempDir; + } + + @Test + void skipsWhenAlreadyConverged() { + final GitRepositoryManager mgr = newRepositoryManager(); + final GitRepository repo = (GitRepository) mgr.create(REPO, Author.SYSTEM); + pushMixedRevisions(repo); + + final Revision head = repo.normalizeNow(Revision.HEAD); + final String headId = commitId(repo, head); + final List payload = mgr.buildRecoveryPayload(REPO, new Revision(3)); + + // The source (and any healthy replica) is already at the target -> recovery is a no-op and the + // GitRepository instance is left untouched (not swapped). + mgr.recoverRepository(REPO, new Revision(2), payload); + assertThat(mgr.get(REPO)).isSameAs(repo); + assertThat(repo.normalizeNow(Revision.HEAD)).isEqualTo(head); + assertThat(commitId(repo, head)).isEqualTo(headId); + } + + @Test + void resetsAndReplaysToConverge() { + final GitRepositoryManager mgr = newRepositoryManager(); + final GitRepository source = (GitRepository) mgr.create(REPO, Author.SYSTEM); + pushMixedRevisions(source); + + // Capture the source-of-truth head and per-revision commit ids, then build the replay payload + // through the production path. + final Revision sourceHead = source.normalizeNow(Revision.HEAD); // r5 + final String sourceHeadId = commitId(source, sourceHead); + final String sourceId3 = commitId(source, new Revision(3)); + final String sourceId4 = commitId(source, new Revision(4)); + final List payload = mgr.buildRecoveryPayload(REPO, new Revision(3)); + + // Diverge: push a 6th revision with different content so the repository is ahead of the payload. + source.commit(new Revision(5), 6000L, Author.SYSTEM, "diverged", "", Markup.PLAINTEXT, + ImmutableList.of(Change.ofTextUpsert("/f.txt", "diverged")), false).join(); + assertThat(source.normalizeNow(Revision.HEAD)).isEqualTo(new Revision(6)); + + // Recover: reset to r2 and replay r3..r5 (a multi-file commit, a JSON commit and a removal) + // -> converge back to the exact source commit ids, dropping r6. + mgr.recoverRepository(REPO, new Revision(2), payload); + + final GitRepository recovered = (GitRepository) mgr.get(REPO); + assertThat(recovered).isNotSameAs(source); // the instance was swapped + assertThat(recovered.normalizeNow(Revision.HEAD)).isEqualTo(sourceHead); // r5, not r6 + assertThat(commitId(recovered, new Revision(3))).isEqualTo(sourceId3); + assertThat(commitId(recovered, new Revision(4))).isEqualTo(sourceId4); + assertThat(commitId(recovered, sourceHead)).isEqualTo(sourceHeadId); + // The replayed content matches the source history: /f.txt was removed at r5 and /g.txt remains. + assertThat(recovered.getOrNull(sourceHead, "/f.txt").join()).isNull(); + assertThat(recovered.getOrNull(sourceHead, "/g.txt").join().contentAsText()).isEqualTo("g\n"); + // The diverged r6 no longer exists. + assertThatThrownBy(() -> recovered.commitIdDatabase().get(new Revision(6))) + .isInstanceOf(RevisionNotFoundException.class); + } + + /** + * A recovery is applied on a repository-worker thread, so it must never wait for another + * repository-worker task: the pool is fixed-size, and a read of the repository being recovered parks a + * worker on the write lock the recovery holds. Driving it through a one-thread pool deadlocks outright + * if any step hops back onto the pool. The other tests here cannot catch this - they pass a + * ForkJoinPool, whose join() spawns a compensation thread and papers over the self-dependency. + */ + @Test + void recoveryNeverWaitsOnTheRepositoryWorkerPool() throws Exception { + final ExecutorService repositoryWorker = Executors.newFixedThreadPool(1); + try { + final Project project = mock(Project.class); + lenient().when(project.name()).thenReturn("test_project"); + final GitRepositoryManager mgr = new GitRepositoryManager( + project, tempDir.toFile(), repositoryWorker, MoreExecutors.directExecutor(), null, + NoopEncryptionStorageManager.INSTANCE); + final GitRepository repo = (GitRepository) mgr.create(REPO, Author.SYSTEM); + pushMixedRevisions(repo); + final List payload = mgr.buildRecoveryPayload(REPO, new Revision(3)); + + // Diverge, so the recovery resets and replays instead of short-circuiting as converged. + repo.commit(new Revision(5), 6000L, Author.SYSTEM, "diverged", "", Markup.PLAINTEXT, + ImmutableList.of(Change.ofTextUpsert("/g.txt", "diverged")), false).join(); + + // Apply it the way StandaloneCommandExecutor does: from the repository worker. + final Future recovered = repositoryWorker.submit( + () -> mgr.recoverRepository(REPO, new Revision(2), payload)); + assertThat(recovered.get(30, TimeUnit.SECONDS)).isTrue(); + assertThat(mgr.get(REPO).normalizeNow(Revision.HEAD)).isEqualTo(new Revision(5)); + } finally { + repositoryWorker.shutdownNow(); + } + } + + @Test + void rollsBackWhenACommitIdDoesNotMatch() { + final GitRepositoryManager mgr = newRepositoryManager(); + final GitRepository repo = (GitRepository) mgr.create(REPO, Author.SYSTEM); + pushMixedRevisions(repo); + final List payload = new ArrayList<>(mgr.buildRecoveryPayload(REPO, new Revision(3))); + + // Diverge so recovery does not short-circuit as already-converged. + repo.commit(new Revision(5), 6000L, Author.SYSTEM, "diverged", "", Markup.PLAINTEXT, + ImmutableList.of(Change.ofTextUpsert("/g.txt", "diverged")), false).join(); + final Revision headBefore = repo.normalizeNow(Revision.HEAD); // r6 + final String headIdBefore = commitId(repo, headBefore); + + // Corrupt the expected commit id of the last replayed commit so the apply detects divergence. + final ReplayCommit last = payload.get(payload.size() - 1); + payload.set(payload.size() - 1, new ReplayCommit( + last.revision(), last.timestampMillis(), last.author(), last.summary(), last.detail(), + last.markup(), last.changes(), "0000000000000000000000000000000000000000")); + + assertThatThrownBy(() -> mgr.recoverRepository(REPO, new Revision(2), payload)) + .isInstanceOf(StorageException.class); + + // The repository must be rolled back to its pre-recovery HEAD and stay usable. + final GitRepository afterFailure = (GitRepository) mgr.get(REPO); + assertThat(afterFailure.normalizeNow(Revision.HEAD)).isEqualTo(headBefore); + assertThat(commitId(afterFailure, headBefore)).isEqualTo(headIdBefore); + // A subsequent read still works (the commit-id database is consistent). + assertThat(afterFailure.getOrNull(headBefore, "/g.txt").join().contentAsText()) + .isEqualTo("diverged\n"); + } + + @Test + void rejectsAnOutOfRangeFromRevision() { + final GitRepositoryManager mgr = newRepositoryManager(); + final GitRepository repo = (GitRepository) mgr.create(REPO, Author.SYSTEM); + pushMixedRevisions(repo); // head == r5 + + assertThatThrownBy(() -> mgr.buildRecoveryPayload(REPO, new Revision(1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("[2, 5]"); + assertThatThrownBy(() -> mgr.buildRecoveryPayload(REPO, new Revision(6))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("[2, 5]"); + assertThatThrownBy(() -> mgr.buildRecoveryPayload(REPO, new Revision(-1))) + .isInstanceOf(IllegalArgumentException.class); + + // A repository with only its creation commit has nothing to replay. + mgr.create("empty_repo", Author.SYSTEM); + assertThatThrownBy(() -> mgr.buildRecoveryPayload("empty_repo", new Revision(2))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("no replayable revision"); + } + + /** + * The payload crosses the replication log as one entry and is materialized in memory by every + * replica, so buildRecoveryPayload() itself must refuse to build an unbounded one. + */ + @Test + void rejectsAnOversizedRecoveryPayload() { + final GitRepositoryManager mgr = newRepositoryManager(); + final GitRepository repo = (GitRepository) mgr.create(REPO, Author.SYSTEM); + pushMixedRevisions(repo); // head == r5, so r3..r5 is 3 commits + + final int commitLimit = GitRepositoryManager.maxRecoveryCommits; + final long byteLimit = GitRepositoryManager.maxRecoveryPayloadBytes; + try { + GitRepositoryManager.maxRecoveryCommits = 2; + assertThatThrownBy(() -> mgr.buildRecoveryPayload(REPO, new Revision(3))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("too many revisions"); + // Two commits fit. + assertThat(mgr.buildRecoveryPayload(REPO, new Revision(4))).hasSize(2); + + GitRepositoryManager.maxRecoveryCommits = commitLimit; + GitRepositoryManager.maxRecoveryPayloadBytes = 1; + assertThatThrownBy(() -> mgr.buildRecoveryPayload(REPO, new Revision(3))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("larger than"); + } finally { + GitRepositoryManager.maxRecoveryCommits = commitLimit; + GitRepositoryManager.maxRecoveryPayloadBytes = byteLimit; + } + } + + @Test + void rejectsAReplicaMissingTheResetBase() { + final GitRepositoryManager mgr = newRepositoryManager(); + final GitRepository source = (GitRepository) mgr.create(REPO, Author.SYSTEM); + pushMixedRevisions(source); // head == r5 + final List payload = mgr.buildRecoveryPayload(REPO, new Revision(5)); + + // A replica whose head is below the reset revision (r4) lacks the shared base history. + final GitRepositoryManager lagging = newRepositoryManager(Files.createTempDir()); + final GitRepository laggingRepo = (GitRepository) lagging.create(REPO, Author.SYSTEM); + laggingRepo.commit(new Revision(1), 2000L, Author.SYSTEM, "add f", "d", Markup.PLAINTEXT, + ImmutableList.of(Change.ofTextUpsert("/f.txt", "v2")), false).join(); + + assertThatThrownBy(() -> lagging.recoverRepository(REPO, new Revision(4), payload)) + .isInstanceOf(StorageException.class) + .hasMessageContaining("missing the shared base history"); + } + + /** + * Pushes r2..r5 covering the change shapes recovery must replay byte-identically: a text upsert (r2), + * a multi-file commit (r3), a JSON upsert (r4) and a removal (r5). + */ + private static void pushMixedRevisions(GitRepository repo) { + repo.commit(new Revision(1), 2000L, Author.SYSTEM, "add f", "detail2", Markup.PLAINTEXT, + ImmutableList.of(Change.ofTextUpsert("/f.txt", "v2")), false).join(); + repo.commit(new Revision(2), 3000L, Author.SYSTEM, "add g and h", "detail3", Markup.PLAINTEXT, + ImmutableList.of(Change.ofTextUpsert("/g.txt", "g"), + Change.ofTextUpsert("/h.txt", "h")), false).join(); + repo.commit(new Revision(3), 4000L, Author.SYSTEM, "add json", "detail4", Markup.PLAINTEXT, + ImmutableList.of(Change.ofJsonUpsert("/a.json", "{ \"a\": 1 }")), false).join(); + repo.commit(new Revision(4), 5000L, Author.SYSTEM, "remove f", "detail5", Markup.PLAINTEXT, + ImmutableList.of(Change.ofRemoval("/f.txt")), false).join(); + } + + private static String commitId(GitRepository repo, Revision revision) { + return repo.commitIdDatabase().get(revision).name(); + } + + private GitRepositoryManager newRepositoryManager() { + return newRepositoryManager(tempDir.toFile()); + } + + private static GitRepositoryManager newRepositoryManager(java.io.File rootDir) { + final Project mock = mock(Project.class); + lenient().when(mock.name()).thenReturn("test_project"); + return new GitRepositoryManager(mock, rootDir, ForkJoinPool.commonPool(), + MoreExecutors.directExecutor(), null, + NoopEncryptionStorageManager.INSTANCE); + } +} diff --git a/webapp/build.gradle b/webapp/build.gradle index c0519dae2..c82addddf 100644 --- a/webapp/build.gradle +++ b/webapp/build.gradle @@ -71,6 +71,13 @@ tasks.register('runTestShiroServer', JavaExec) { mainClass = "com.linecorp.centraldogma.webapp.ShiroCentralDogmaTestServer" } +tasks.register('runTestReplicatedShiroServer', JavaExec) { + group = "Execution" + description = "Run the Central Dogma server with Apache Shiro in replicated (ZooKeeper) mode" + classpath = sourceSets.test.runtimeClasspath + mainClass = "com.linecorp.centraldogma.webapp.ReplicatedShiroCentralDogmaTestServer" +} + if (!rootProject.hasProperty('noLint')) { tasks.register('eslint', NpmTask) { dependsOn(tasks.named('npmInstall')) diff --git a/webapp/e2e/repo-status.spec.ts b/webapp/e2e/repo-status.spec.ts index e7cb5e1bf..d5ad427c3 100644 --- a/webapp/e2e/repo-status.spec.ts +++ b/webapp/e2e/repo-status.spec.ts @@ -232,5 +232,17 @@ test.describe.serial('Repository Status', () => { // The repository leaves the read-only list. await expect(table.locator('tr', { hasText: FORM_REPO })).toHaveCount(0, { timeout: 10000 }); }); + + test('hides the Repository Recovery tab in standalone mode', async ({ page }) => { + // The e2e backend is a single standalone server, so /api/v1/replicas is empty and the + // ZooKeeper-only recovery tab must not be offered. + await page.goto('/app/settings/repo-status'); + await expect(page.getByRole('tab', { name: 'Repository Status' })).toBeVisible(); + await expect(page.getByRole('tab', { name: 'Repository Recovery' })).toHaveCount(0); + + // Navigating to the page directly explains why the feature is unavailable. + await page.goto('/app/settings/recovery'); + await expect(page.getByText(/only available when the server runs in replicated/)).toBeVisible(); + }); }); }); diff --git a/webapp/javaTest/java/com/linecorp/centraldogma/webapp/ReplicatedShiroCentralDogmaTestServer.java b/webapp/javaTest/java/com/linecorp/centraldogma/webapp/ReplicatedShiroCentralDogmaTestServer.java new file mode 100644 index 000000000..2517d1a9b --- /dev/null +++ b/webapp/javaTest/java/com/linecorp/centraldogma/webapp/ReplicatedShiroCentralDogmaTestServer.java @@ -0,0 +1,116 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.centraldogma.webapp; + +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.PASSWORD; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.PASSWORD2; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.USERNAME; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.USERNAME2; +import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.getAccessToken; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import org.apache.shiro.config.Ini; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; + +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.centraldogma.client.armeria.ArmeriaCentralDogmaBuilder; +import com.linecorp.centraldogma.common.Change; +import com.linecorp.centraldogma.server.CentralDogma; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; +import com.linecorp.centraldogma.server.ZooKeeperReplicationConfig; +import com.linecorp.centraldogma.server.ZooKeeperServerConfig; +import com.linecorp.centraldogma.server.auth.shiro.ShiroAuthProviderFactory; + +/** + * A {@link ShiroCentralDogmaTestServer} variant that runs a two-replica ZooKeeper cluster in one JVM + * (server 1 on port 36462, server 2 on port 36463), so that replication-only features (e.g. repository + * recovery, including its request-to-the-source path) can be exercised from the web UI during + * development. + */ +final class ReplicatedShiroCentralDogmaTestServer { + + private static final int PORT1 = 36462; + private static final int PORT2 = 36463; + + private static final Map SERVERS = ImmutableMap.of( + 1, new ZooKeeperServerConfig("127.0.0.1", 36466, 36467, 36468, null, null), + 2, new ZooKeeperServerConfig("127.0.0.1", 36469, 36470, 36471, null, null)); + + @SuppressWarnings("UncommentedMain") + public static void main(String[] args) throws IOException { + final CentralDogma server1 = newServer(1, PORT1); + final CentralDogma server2 = newServer(2, PORT2); + // A two-node quorum needs both peers; start them concurrently. + final var start1 = server1.start(); + final var start2 = server2.start(); + start1.join(); + start2.join(); + scaffold(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + server1.close(); + server2.close(); + })); + } + + private static CentralDogma newServer(int serverId, int port) throws IOException { + final Path rootDir = Files.createTempDirectory("dogma-replicated-test-" + serverId); + return new CentralDogmaBuilder(rootDir.toFile()) + .port(port, SessionProtocol.HTTP) + .systemAdministrators(USERNAME) + .cors("http://127.0.0.1:36462", "http://127.0.0.1:36463", "http://127.0.0.1:3000", + "http://localhost:36462", "http://localhost:36463", "http://localhost:3000") + .authProviderFactory(new ShiroAuthProviderFactory(unused -> { + final Ini iniConfig = new Ini(); + final Ini.Section users = iniConfig.addSection("users"); + users.put(USERNAME, PASSWORD); + users.put(USERNAME2, PASSWORD2); + return iniConfig; + })) + .replication(new ZooKeeperReplicationConfig( + serverId, SERVERS, "test-secret-for-replication-0123456789abcdef")) + .build(); + } + + private static void scaffold() throws UnknownHostException, JsonProcessingException { + final String token = getAccessToken(WebClient.of("http://127.0.0.1:" + PORT1), USERNAME, PASSWORD, + "appId", true); + final com.linecorp.centraldogma.client.CentralDogma client = new ArmeriaCentralDogmaBuilder() + .host("127.0.0.1", PORT1) + .accessToken(token) + .build(); + client.createProject("foo").join(); + client.createRepository("foo", "bar").join(); + client.forRepo("foo", "bar") + .commit("add a.json", Change.ofJsonUpsert("/a.json", "{ \"a\": 1 }")) + .push() + .join(); + client.forRepo("foo", "bar") + .commit("update a.json", Change.ofJsonUpsert("/a.json", "{ \"a\": 2 }")) + .push() + .join(); + } + + private ReplicatedShiroCentralDogmaTestServer() {} +} diff --git a/webapp/src/dogma/features/api/apiSlice.ts b/webapp/src/dogma/features/api/apiSlice.ts index bfa1b3256..6e95af5ef 100644 --- a/webapp/src/dogma/features/api/apiSlice.ts +++ b/webapp/src/dogma/features/api/apiSlice.ts @@ -46,6 +46,7 @@ import { UpdateServerStatusRequest, } from 'dogma/features/settings/server-status/ServerStatusDto'; import { ReplicationStatus, RepositoryStatus } from 'dogma/features/settings/repo-status/RepoStatusDto'; +import { RecoverRepositoryResponse, ReplicaInfo } from 'dogma/features/settings/recovery/RecoveryDto'; import Router from 'next/router'; import { VariableDto } from 'dogma/features/project/settings/variables/VariableDto'; import { XdsApp, XdsClientStatus, XdsSnapshot } from 'dogma/features/xds/ControlPlaneStatusDto'; @@ -483,6 +484,23 @@ export const apiSlice = createApi({ // The project and repository lists carry the replication status, so they go stale as well. invalidatesTags: ['RepoStatus', 'Project', 'Repo'], }), + getReplicas: builder.query({ + query: () => '/api/v1/replicas', + // An empty list is returned as 204 No Content in standalone (non-replicated) mode. + transformResponse: (response: ReplicaInfo[] | null) => response ?? [], + }), + recoverRepository: builder.mutation< + RecoverRepositoryResponse, + { projectName: string; repoName: string; fromRevision: number; sourceServerId: number } + >({ + query: ({ projectName, repoName, fromRevision, sourceServerId }) => ({ + url: `/api/v1/projects/${projectName}/repos/${repoName}/recover`, + method: 'POST', + body: { fromRevision, sourceServerId }, + }), + // Recovery rewrites the repository history on the other replicas. + invalidatesTags: ['Repo'], + }), getProjectCredentials: builder.query({ query: (projectName) => `/api/v1/projects/${projectName}/credentials`, transformResponse: (response: CredentialDto[]) => addIdFromCredentialNames(response), @@ -704,6 +722,9 @@ export const { // Repository Status useGetReadOnlyReposQuery, useUpdateRepositoryStatusMutation, + // Repository Recovery + useGetReplicasQuery, + useRecoverRepositoryMutation, // Credential useGetProjectCredentialsQuery, useGetCredentialQuery, diff --git a/webapp/src/dogma/features/settings/SettingView.tsx b/webapp/src/dogma/features/settings/SettingView.tsx index 63329ed3b..e74747601 100644 --- a/webapp/src/dogma/features/settings/SettingView.tsx +++ b/webapp/src/dogma/features/settings/SettingView.tsx @@ -20,6 +20,7 @@ import Link from 'next/link'; import { Breadcrumbs } from 'dogma/common/components/Breadcrumbs'; import { useRouter } from 'next/router'; import { useAppSelector } from 'dogma/hooks'; +import { useGetReplicasQuery } from 'dogma/features/api/apiSlice'; import { GrSystem } from 'react-icons/gr'; interface SettingsViewProps { @@ -27,7 +28,12 @@ interface SettingsViewProps { children: ReactNode; } -type TabName = 'Mirror Access Control' | 'Application Identities' | 'Server Status' | 'Repository Status'; +type TabName = + | 'Mirror Access Control' + | 'Application Identities' + | 'Server Status' + | 'Repository Status' + | 'Repository Recovery'; export interface TapInfo { name: TabName; @@ -35,15 +41,20 @@ export interface TapInfo { admin: boolean; } +// 'Repository Recovery' must stay last: it is hidden in standalone mode, and the Tabs index is +// computed over this whole array, so hiding a non-trailing entry would misalign the tab highlight. const TABS: TapInfo[] = [ { name: 'Application Identities', path: 'app-identities', admin: false }, { name: 'Mirror Access Control', path: 'mirror-access', admin: true }, { name: 'Server Status', path: 'server-status', admin: true }, { name: 'Repository Status', path: 'repo-status', admin: true }, + { name: 'Repository Recovery', path: 'recovery', admin: true }, ]; const SettingView = ({ currentTab, children }: SettingsViewProps) => { const { user } = useAppSelector((state) => state.auth); + // Repository recovery exists only in replicated (ZooKeeper) mode, where the replica list is non-empty. + const { data: replicas } = useGetReplicasQuery(undefined, { skip: !user?.systemAdmin }); const tabIndex = TABS.findIndex((tab) => tab.name === currentTab); const router = useRouter(); @@ -66,6 +77,9 @@ const SettingView = ({ currentTab, children }: SettingsViewProps) => { if (tab.admin && !user?.systemAdmin) { return null; } + if (tab.path === 'recovery' && (!replicas || replicas.length === 0)) { + return null; + } let link = `/app/settings`; if (tab.path !== '') { link += `/${tab.path}`; diff --git a/webapp/src/dogma/features/settings/recovery/RecoverRepositoryForm.tsx b/webapp/src/dogma/features/settings/recovery/RecoverRepositoryForm.tsx new file mode 100644 index 000000000..502bf29d4 --- /dev/null +++ b/webapp/src/dogma/features/settings/recovery/RecoverRepositoryForm.tsx @@ -0,0 +1,398 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +import { + Alert, + AlertIcon, + Box, + Button, + Code, + Flex, + FormControl, + FormHelperText, + FormLabel, + Heading, + NumberDecrementStepper, + NumberIncrementStepper, + NumberInput, + NumberInputField, + NumberInputStepper, + Text, + useClipboard, + useColorMode, + useDisclosure, +} from '@chakra-ui/react'; +import { OptionBase, Select } from 'chakra-react-select'; +import Prism from 'prismjs'; +import 'prismjs/components/prism-bash'; +import 'prismjs/themes/prism.css'; +import { useEffect, useState } from 'react'; +import { + useGetProjectsQuery, + useGetReplicasQuery, + useGetReposQuery, + useRecoverRepositoryMutation, +} from 'dogma/features/api/apiSlice'; +import { ProjectDto } from 'dogma/features/project/ProjectDto'; +import { RepoDto } from 'dogma/features/repo/RepoDto'; +import { RecoverRepositoryResponse, ReplicaInfo } from 'dogma/features/settings/recovery/RecoveryDto'; +import { RecoveryConfirmModal } from 'dogma/features/settings/recovery/RecoveryConfirmModal'; +import ErrorMessageParser from 'dogma/features/services/ErrorMessageParser'; +import { newNotification } from 'dogma/features/notification/notificationSlice'; +import { useAppDispatch } from 'dogma/hooks'; + +interface Option extends OptionBase { + value: string; + label: string; +} + +interface SourceOption extends OptionBase { + value: number; + label: string; + host: string; +} + +interface RecoveryResult { + projectName: string; + repoName: string; + sourceServerId: number; + response: RecoverRepositoryResponse; +} + +/** + * Returns the reason of a rejected recovery without the Java stack trace the server sends to a system + * administrator, which would otherwise bury the message that matters. + */ +export function conciseErrorMessage(error: unknown): string { + const parsed = ErrorMessageParser.parse(error); + const lines: string[] = []; + for (const line of parsed.split('\n')) { + // A stack frame, or the exception class repeating the message that was already shown. + if (/^\s+at\s/.test(line) || /^\s*(?:[a-z][\w$]*\.)+[A-Z][\w$]*(?:Exception|Error)\b/.test(line)) { + break; + } + lines.push(line); + } + const concise = lines.join('\n').trim(); + return concise || parsed; +} + +/** + * Builds a copy-pastable shell script that compares the head of the recovered repository on every + * replica. The replicas other than the source apply the recovery when they replay it, and a failure is + * only reported in the source replica's log, so this is how an administrator confirms convergence before + * making the repository writable again. + * + *

It compares the head *commit ID*, not the revision: diverged replicas of the same repository always + * report the same revision, so a revision alone can never tell them apart. + * + *

Over HTTPS each curl adds {@code -k}: it reaches a replica by its own host name, which a + * certificate issued for the load balancer's name does not cover. + */ +export function buildVerificationScript(result: RecoveryResult, replicas: ReplicaInfo[]): string { + const { projectName, repoName, sourceServerId } = result; + const origin = process.env.NEXT_PUBLIC_HOST || window.location.origin; + const url = new URL(origin); + const https = url.protocol === 'https:'; + // The roster carries no port, so every address is seeded from the URL this page was served on and the + // operator corrects the rest. When that collapses two replicas onto one address, the script would poll the + // same server twice and report a convergence it never checked, so say so rather than let it read as a pass. + const port = url.port || (https ? '443' : '36462'); + const entries = replicas.map((replica) => `${replica.serverId}=${replica.host}:${port}`); + const collided = new Set(replicas.map((replica) => replica.host)).size < replicas.length; + return [ + "CD_TOKEN=''", + `# Address of each replica, as serverId=host:port. Server ${sourceServerId} is the source.`, + ...(collided + ? [ + '# WARNING: replicas share a host, so they were all given the same port and some entries below', + '# now point at the SAME server. Give each its own port first, or this check will poll one', + '# replica twice and wrongly look converged.', + ] + : ['# The roster carries no port, so the ports are a guess. Correct any that differs.']), + `REPLICAS='${entries.join(' ')}'`, + '', + `# Every replica of ${projectName}/${repoName} must report the same head commit ID as the source.`, + '# The revision proves nothing: diverged replicas report the same revision.', + 'for replica in $REPLICAS; do', + ' id="${replica%%=*}"; addr="${replica#*=}"', + ` commit=$(curl -sf${https ? 'k' : ''} -m 10 -H "Authorization: Bearer $CD_TOKEN" \\`, + ` "${url.protocol}//$addr/api/v1/projects/${projectName}/repos/${repoName}/head" \\`, + " | jq -r '.commitId // empty')", + ' echo "server $id $addr ${commit:-REQUEST FAILED}"', + 'done', + "dupes=$(printf '%s\\n' $REPLICAS | cut -d= -f2 | sort | uniq -d)", + '[ -z "$dupes" ] || echo "WARNING: polled twice, so this proves nothing: $dupes"', + '', + '# Converged only if every line shows the same commit ID. REQUEST FAILED is not a pass: that replica', + `# was not reached. A failed recovery is reported only in the log of the source (server ${sourceServerId}).`, + ].join('\n'); +} + +const RecoverRepositoryForm = () => { + const { colorMode } = useColorMode(); + const { isOpen, onOpen, onClose } = useDisclosure(); + const dispatch = useAppDispatch(); + + const [project, setProject] = useState

panel
+ , + // eslint-disable-next-line @typescript-eslint/no-explicit-any + { preloadedState: adminPreloadedState as any }, + ); +}; + +describe('SettingView', () => { + beforeEach(() => jest.clearAllMocks()); + + it('hides the Repository Recovery tab in standalone mode without misaligning the highlight', () => { + renderSettingView([]); + + expect(screen.queryByRole('tab', { name: 'Repository Recovery' })).toBeNull(); + // The Recovery entry must stay last in TABS: hiding it must not shift which tab is highlighted. + expect(screen.getByRole('tab', { name: 'Repository Status' })).toHaveAttribute('aria-selected', 'true'); + }); + + it('shows the Repository Recovery tab in replicated mode', () => { + renderSettingView([{ serverId: 1, host: '127.0.0.1', current: true }]); + + expect(screen.getByRole('tab', { name: 'Repository Recovery' })).toBeInTheDocument(); + expect(screen.getByRole('tab', { name: 'Repository Status' })).toHaveAttribute('aria-selected', 'true'); + }); +});