Add a Repository Status page to view and manage read-only repositories

Motivation:

Central Dogma can put an individual repository or a whole project into
read-only mode (scoped replication-failure read-only, line#1305), but there was
no way to see which repositories or projects are currently read-only, nor to
set or clear that status, from the web UI.

Modifications:

- Add `GET /api/v1/status/repos/read-only` (system administrator only) that
  returns the projects and repositories that are currently read-only. A
  project-scoped entry uses `dogma` as its repository name.
- Serialize `RepositoryState.updatedAt` as an ISO-8601 string via `@JsonFormat`
  so clients render the real timestamp instead of an epoch number.
- Expose read-only scope metrics: a `repository.read.only.count` gauge and a
  `repository.read.only` multi-gauge tagged by project, repo and scope.
- Add a "Repository Status" settings tab (system administrator only) that
  - lists the read-only projects and repositories with their scope, status and
    last-updated time;
  - offers a form to make a repository read-only, with project and repository
    auto-complete; and
  - offers a per-row action to revert a repository or project to writable.
  Both changes require re-typing the full `project/repository` name to confirm,
  to guard against accidental clicks.
- Add the `updateRepositoryStatus` mutation and a `RepoStatus` cache tag so the
  list refreshes after a change.
- Add unit tests, a Playwright e2e test and a server test for the new endpoint.

Result:

- A system administrator can now view all read-only repositories and projects
  and toggle read-only status from the web UI, with a type-to-confirm safeguard
  against mistakes.

Author: ikhoon

server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java

@@ -623,7 +623,7 @@ private CommandExecutor startCommandExecutor(
         }
 
         statusManager = new ServerStatusManager(cfg.dataDir());
-        repoStatusManager = new RepoStatusManager(statusManager, pm);
+        repoStatusManager = new RepoStatusManager(statusManager, pm, meterRegistry);
         logger.info("Startup mode: {}", statusManager.serverStatus());
         final CommandExecutor executor;
         final ReplicationMethod replicationMethod = cfg.replicationConfig().method();
@@ -1020,7 +1020,7 @@ private void configureHttpApi(ServerBuilder sb,
         assert statusManager != null && repoStatusManager != null;
         final ContextPathServicesBuilder apiV1ServiceBuilder = sb.contextPath(API_V1_PATH_PREFIX);
         apiV1ServiceBuilder
-                .annotatedService(new ServerStatusService(executor, statusManager))
+                .annotatedService(new ServerStatusService(executor, statusManager, repoStatusManager))
                 .annotatedService(new ProjectServiceV1(projectApiManager, executor))
                 .annotatedService(new RepositoryServiceV1(executor, mds, encryptionStorageManager,
                                                           repoStatusManager))

server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/ServerStatusService.java

@@ -16,6 +16,7 @@
 
 package com.linecorp.centraldogma.server.internal.api.sysadmin;
 
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 
 import com.linecorp.armeria.common.HttpStatus;
@@ -29,6 +30,8 @@
 import com.linecorp.centraldogma.server.internal.api.AbstractService;
 import com.linecorp.centraldogma.server.internal.api.auth.RequiresSystemAdministrator;
 import com.linecorp.centraldogma.server.internal.api.sysadmin.UpdateServerStatusRequest.Scope;
+import com.linecorp.centraldogma.server.internal.management.RepoStatusManager;
+import com.linecorp.centraldogma.server.internal.management.RepositoryState;
 import com.linecorp.centraldogma.server.internal.management.ServerStatusManager;
 import com.linecorp.centraldogma.server.internal.replication.ZooKeeperCommandExecutor;
 import com.linecorp.centraldogma.server.management.ServerStatus;
@@ -38,10 +41,13 @@
 public final class ServerStatusService extends AbstractService {
 
     private final ServerStatusManager serverStatusManager;
+    private final RepoStatusManager repoStatusManager;
 
-    public ServerStatusService(CommandExecutor executor, ServerStatusManager serverStatusManager) {
+    public ServerStatusService(CommandExecutor executor, ServerStatusManager serverStatusManager,
+                               RepoStatusManager repoStatusManager) {
         super(executor);
         this.serverStatusManager = serverStatusManager;
+        this.repoStatusManager = repoStatusManager;
     }
 
     /**
@@ -54,6 +60,17 @@ public ServerStatus status() {
         return ServerStatus.of(executor().isWritable(), executor().isStarted());
     }
 
+    /**
+     * GET /status/repos/read-only
+     *
+     * <p>Returns the projects and repositories that are currently read-only. A project-scoped entry uses
+     * {@code dogma} as its repository name.
+     */
+    @Get("/status/repos/read-only")
+    public List<RepositoryState> readOnlyRepositories() {
+        return repoStatusManager.readOnlyStatuses();
+    }
+
     /**
      * PUT /status
      *

server/src/main/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManager.java

@@ -16,7 +16,10 @@
 
 package com.linecorp.centraldogma.server.internal.management;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
+
 import java.time.Instant;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -27,6 +30,7 @@
 
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.databind.JsonMappingException;
+import com.google.common.collect.ImmutableList;
 
 import com.linecorp.armeria.common.util.Exceptions;
 import com.linecorp.centraldogma.common.Author;
@@ -44,6 +48,11 @@
 import com.linecorp.centraldogma.server.storage.repository.Repository;
 import com.linecorp.centraldogma.server.storage.repository.RepositoryListener;
 
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.MultiGauge;
+import io.micrometer.core.instrument.Tags;
+
 /**
  * Manages the replication status of repositories and projects. The replication status is stored in a special
  * repository named "dogma" in dogma project
@@ -65,11 +74,17 @@ public final class RepoStatusManager {
     @Nullable
     private final StandaloneCrudOperation<RepositoryState> crudRepository;
     private final ServerStatusManager statusManager;
+    private final MultiGauge readOnlyScopeGauge;
 
-    public RepoStatusManager(ServerStatusManager statusManager, ProjectManager pm) {
+    public RepoStatusManager(ServerStatusManager statusManager, ProjectManager pm,
+                             MeterRegistry meterRegistry) {
         this.pm = pm;
         this.statusManager = statusManager;
         crudRepository = new StandaloneCrudOperation<>(RepositoryState.class, pm);
+
+        // read-only scope metrics
+        Gauge.builder("repository.read.only.count", statusMap, Map::size).register(meterRegistry);
+        readOnlyScopeGauge = MultiGauge.builder("repository.read.only").register(meterRegistry);
     }
 
     public void initialize() {
@@ -92,6 +107,7 @@ public void initialize() {
                     statusMap.put(getKey(repoState.projectName(), repoState.repoName()), repoState);
                 }
             }
+            updateReadOnlyScopeMetrics();
         }));
     }
 
@@ -133,6 +149,15 @@ public RepositoryState getRepoStatus(String projectName, String repoName) {
         return new RepositoryState(projectName, repoName, ReplicationStatus.WRITABLE, null);
     }
 
+    /**
+     * Returns the {@link RepositoryState}s of all projects and repositories that are currently not
+     * {@link ReplicationStatus#WRITABLE}. A project-scoped read-only entry uses {@link Project#REPO_DOGMA}
+     * as its repository name.
+     */
+    public List<RepositoryState> readOnlyStatuses() {
+        return ImmutableList.copyOf(statusMap.values());
+    }
+
     @Nullable
     private RepositoryState getRepoStatus0(String projectName, String repoName) {
         if (!statusManager.serverStatus().writable()) {
@@ -193,4 +218,17 @@ private static CrudContext crudContext(String projectName) {
         return new CrudContext(InternalProjectInitializer.INTERNAL_PROJECT_DOGMA,
                                Project.REPO_DOGMA, targetPath);
     }
+
+    private void updateReadOnlyScopeMetrics() {
+        readOnlyScopeGauge.register(
+                statusMap.values().stream()
+                         .<MultiGauge.Row<?>>map(state -> MultiGauge.Row.of(
+                                 Tags.of("project", state.projectName(),
+                                         "repo", state.repoName(),
+                                         "scope", state.repoName().equals(Project.REPO_DOGMA)
+                                                  ? "project" : "repository"),
+                                 1))
+                         .collect(toImmutableList()),
+                true);
+    }
 }

server/src/main/java/com/linecorp/centraldogma/server/internal/management/RepositoryState.java

@@ -24,6 +24,7 @@
 import org.jspecify.annotations.Nullable;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -39,6 +40,8 @@ public final class RepositoryState {
     private final String projectName;
     private final String repoName;
     private final ReplicationStatus status;
+    // Serialize as an ISO-8601 string to match the rest of the API (e.g. RepositoryDto.createdAt).
+    @JsonFormat(shape = JsonFormat.Shape.STRING)
     @Nullable
     private final Instant updatedAt;
 

server/src/test/java/com/linecorp/centraldogma/server/internal/api/ServerStatusServiceTest.java

@@ -20,6 +20,7 @@
 import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.awaitility.Awaitility.await;
 
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -32,6 +33,7 @@
 import com.linecorp.armeria.common.AggregatedHttpResponse;
 import com.linecorp.armeria.common.HttpHeaderNames;
 import com.linecorp.armeria.common.HttpStatus;
+import com.linecorp.centraldogma.common.ReplicationStatus;
 import com.linecorp.centraldogma.server.internal.api.sysadmin.UpdateServerStatusRequest;
 import com.linecorp.centraldogma.server.internal.api.sysadmin.UpdateServerStatusRequest.Scope;
 import com.linecorp.centraldogma.server.management.ServerStatus;
@@ -62,6 +64,37 @@ void status() {
         assertThat(res.contentUtf8()).isEqualTo("\"WRITABLE\"");
     }
 
+    @Test
+    void readOnlyRepositories() {
+        final BlockingWebClient client = dogma.httpClient().blocking();
+
+        // No read-only repository initially (an empty result is returned as 204 No Content).
+        assertThat(client.get(API_V1_PATH_PREFIX + "status/repos/read-only").status())
+                .isEqualTo(HttpStatus.NO_CONTENT);
+
+        // Set a repository read-only.
+        dogma.client().createProject("foo").join();
+        dogma.client().createRepository("foo", "bar").join();
+        final AggregatedHttpResponse updateRes =
+                client.prepare()
+                      .put(API_V1_PATH_PREFIX + "projects/foo/repos/bar/status")
+                      .contentJson(new UpdateRepositoryStatusRequest(ReplicationStatus.READ_ONLY))
+                      .execute();
+        assertThat(updateRes.status()).isEqualTo(HttpStatus.OK);
+
+        // The read-only repository should be listed.
+        await().untilAsserted(() -> {
+            final AggregatedHttpResponse res = client.get(API_V1_PATH_PREFIX + "status/repos/read-only");
+            assertThat(res.status()).isEqualTo(HttpStatus.OK);
+            assertThatJson(res.contentUtf8()).isArray().ofLength(1);
+            assertThatJson(res.contentUtf8()).node("[0].projectName").isEqualTo("foo");
+            assertThatJson(res.contentUtf8()).node("[0].repoName").isEqualTo("bar");
+            assertThatJson(res.contentUtf8()).node("[0].status").isEqualTo("READ_ONLY");
+            // updatedAt must serialize as an ISO-8601 string, not an epoch number (regression guard).
+            assertThatJson(res.contentUtf8()).node("[0].updatedAt").isString();
+        });
+    }
+
     @Test
     void updateStatus_setUnwritable() {
         final AggregatedHttpResponse res = updateStatus(ServerStatus.REPLICATION_ONLY);

server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresRoleTest.java

@@ -92,7 +92,8 @@ protected void configure(ServerBuilder sb) throws Exception {
                     NoopEncryptionStorageManager.INSTANCE,
                     com.google.common.collect.ImmutableMap.of());
             final ServerStatusManager statusManager = new ServerStatusManager(dataDir);
-            final RepoStatusManager repoStatusManager = new RepoStatusManager(statusManager, pm);
+            final RepoStatusManager repoStatusManager =
+                    new RepoStatusManager(statusManager, pm, NoopMeterRegistry.get());
             final CommandExecutor executor = new StandaloneCommandExecutor(
                     pm, ForkJoinPool.commonPool(), statusManager, repoStatusManager, null,
                     NoopEncryptionStorageManager.INSTANCE,

server/src/test/java/com/linecorp/centraldogma/server/internal/management/RepoStatusManagerTest.java

@@ -17,6 +17,7 @@
 package com.linecorp.centraldogma.server.internal.management;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.tuple;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -26,16 +27,22 @@
 import com.linecorp.centraldogma.common.ReplicationStatus;
 import com.linecorp.centraldogma.testing.internal.ProjectManagerExtension;
 
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+
 class RepoStatusManagerTest {
 
     @RegisterExtension
     static ProjectManagerExtension pmExtension = new ProjectManagerExtension();
 
     private RepoStatusManager statusManager;
+    private MeterRegistry meterRegistry;
 
     @BeforeEach
     void setUp() {
-        statusManager = new RepoStatusManager(pmExtension.serverStatusManager(), pmExtension.projectManager());
+        meterRegistry = new SimpleMeterRegistry();
+        statusManager = new RepoStatusManager(pmExtension.serverStatusManager(), pmExtension.projectManager(),
+                                              meterRegistry);
         statusManager.initialize();
     }
 
@@ -75,4 +82,41 @@ void projectStatus_cruTest() {
         assertThat(entity).isEqualTo(
                 new RepositoryState("test_prj", "dogma", ReplicationStatus.WRITABLE, null));
     }
+
+    @Test
+    void readOnlyStatuses_and_metrics() {
+        assertThat(statusManager.readOnlyStatuses()).isEmpty();
+        assertThat(readOnlyCount()).isZero();
+
+        statusManager.updateRepoStatus("test_prj", "test_repo", Author.DEFAULT, ReplicationStatus.READ_ONLY)
+                     .join();
+        statusManager.updateProjectStatus("test_prj2", Author.DEFAULT, ReplicationStatus.READ_ONLY).join();
+
+        assertThat(statusManager.readOnlyStatuses())
+                .extracting(RepositoryState::projectName, RepositoryState::repoName, RepositoryState::status)
+                .containsExactlyInAnyOrder(
+                        tuple("test_prj", "test_repo", ReplicationStatus.READ_ONLY),
+                        tuple("test_prj2", "dogma", ReplicationStatus.READ_ONLY));
+
+        assertThat(readOnlyCount()).isEqualTo(2);
+        assertThat(scopeGauge("test_prj", "test_repo", "repository")).isOne();
+        assertThat(scopeGauge("test_prj2", "dogma", "project")).isOne();
+
+        // Reverting to WRITABLE removes the scope from the list and the metrics.
+        statusManager.updateRepoStatus("test_prj", "test_repo", Author.DEFAULT, ReplicationStatus.WRITABLE)
+                     .join();
+        statusManager.updateProjectStatus("test_prj2", Author.DEFAULT, ReplicationStatus.WRITABLE).join();
+        assertThat(statusManager.readOnlyStatuses()).isEmpty();
+        assertThat(readOnlyCount()).isZero();
+    }
+
+    private double readOnlyCount() {
+        return meterRegistry.get("repository.read.only.count").gauge().value();
+    }
+
+    private double scopeGauge(String projectName, String repoName, String scope) {
+        return meterRegistry.get("repository.read.only")
+                            .tags("project", projectName, "repo", repoName, "scope", scope)
+                            .gauge().value();
+    }
 }

testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/ProjectManagerExtension.java

@@ -172,8 +172,10 @@ public void recreateProjectManager() {
      * Override this method to customize a {@link CommandExecutor}.
      */
     protected CommandExecutor newCommandExecutor(ProjectManager projectManager, Executor worker, File dataDir) {
+        final RepoStatusManager repoStatusManager =
+                new RepoStatusManager(serverStatusManager, projectManager, NoopMeterRegistry.get());
         return new StandaloneCommandExecutor(projectManager, worker, serverStatusManager,
-                                             new RepoStatusManager(serverStatusManager, projectManager), null,
+                                             repoStatusManager, null,
                                              NoopEncryptionStorageManager.INSTANCE, null, null, null,
                                              null);
     }