HBASE-23996 Add metric for the split WAL procedure

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystemSource.java

@@ -21,6 +21,13 @@
 import org.apache.hadoop.hbase.metrics.BaseSource;
 import org.apache.yetus.audience.InterfaceAudience;
 
+/**
+ * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based
+ *             distributed WAL splitter; see SplitWALManager. These metrics ({@code hlogSplitTime},
+ *             {@code hlogSplitSize}, {@code metaHlogSplitTime}, {@code metaHlogSplitSize}) are only
+ *             emitted by the deprecated ZK-coordinated WAL split path.
+ */
+@Deprecated
 @InterfaceAudience.Private
 public interface MetricsMasterFileSystemSource extends BaseSource {
 

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java

@@ -21,6 +21,13 @@
 import org.apache.hadoop.metrics2.MetricHistogram;
 import org.apache.yetus.audience.InterfaceAudience;
 
+/**
+ * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based
+ *             distributed WAL splitter; see SplitWALManager. Implementation of the deprecated
+ *             {@link MetricsMasterFileSystemSource}, whose metrics are only emitted by the
+ *             ZK-coordinated WAL split path.
+ */
+@Deprecated
 @InterfaceAudience.Private
 public class MetricsMasterFilesystemSourceImpl extends BaseSourceImpl
   implements MetricsMasterFileSystemSource {

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java

@@ -92,6 +92,7 @@ public interface MetricsMasterSource extends BaseSource {
   String OFFLINE_REGION_COUNT_DESC = "Number of Offline Regions";
 
   String SERVER_CRASH_METRIC_PREFIX = "serverCrash";
+  String SPLIT_WAL_METRIC_PREFIX = "splitWAL";
   String OLD_WAL_DIR_SIZE_DESC = "size of old WALs directory in bytes";
 
   /**
@@ -114,4 +115,7 @@ public interface MetricsMasterSource extends BaseSource {
 
   /** Returns {@link OperationMetrics} containing common metrics for server crash operation */
   OperationMetrics getServerCrashMetrics();
+
+  /** Returns {@link OperationMetrics} containing common metrics for split WAL operation */
+  OperationMetrics getSplitWALMetrics();
 }

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java

@@ -39,6 +39,7 @@ public class MetricsMasterSourceImpl extends BaseSourceImpl implements MetricsMa
   private MutableFastCounter clusterWriteRequestsCounter;
 
   private OperationMetrics serverCrashMetrics;
+  private OperationMetrics splitWALMetrics;
 
   public MetricsMasterSourceImpl(MetricsMasterWrapper masterWrapper) {
     this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, masterWrapper);
@@ -64,6 +65,7 @@ public void init() {
      * BaseSourceImpl#registry} to register the new metrics.
      */
     serverCrashMetrics = new OperationMetrics(registry, SERVER_CRASH_METRIC_PREFIX);
+    splitWALMetrics = new OperationMetrics(registry, SPLIT_WAL_METRIC_PREFIX);
   }
 
   @Override
@@ -144,4 +146,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
   public OperationMetrics getServerCrashMetrics() {
     return serverCrashMetrics;
   }
+
+  @Override
+  public OperationMetrics getSplitWALMetrics() {
+    return splitWALMetrics;
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java

@@ -58,8 +58,11 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * ZooKeeper based implementation of {@link SplitLogManagerCoordination}
+ * ZooKeeper based implementation of {@link SplitLogManagerCoordination}.
+ * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based
+ *             distributed WAL splitter; see SplitWALManager.
  */
+@Deprecated
 @InterfaceAudience.Private
 public class ZKSplitLogManagerCoordination extends ZKListener
   implements SplitLogManagerCoordination {

hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java

@@ -57,9 +57,11 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * ZooKeeper based implementation of {@link SplitLogWorkerCoordination} It listen for changes in
- * ZooKeeper and
+ * ZooKeeper based implementation of {@link SplitLogWorkerCoordination}.
+ * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based
+ *             distributed WAL splitter; see SplitWALManager.
  */
+@Deprecated
 @InterfaceAudience.Private
 public class ZkSplitLogWorkerCoordination extends ZKListener implements SplitLogWorkerCoordination {
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java

@@ -43,6 +43,7 @@ public class MetricsMaster {
   private MetricsMasterQuotaSource masterQuotaSource;
 
   private ProcedureMetrics serverCrashProcMetrics;
+  private ProcedureMetrics splitWALProcMetrics;
 
   public MetricsMaster(MetricsMasterWrapper masterWrapper) {
     masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class)
@@ -53,6 +54,7 @@ public MetricsMaster(MetricsMasterWrapper masterWrapper) {
       .getInstance(MetricsMasterQuotaSourceFactory.class).create(masterWrapper);
 
     serverCrashProcMetrics = convertToProcedureMetrics(masterSource.getServerCrashMetrics());
+    splitWALProcMetrics = convertToProcedureMetrics(masterSource.getSplitWALMetrics());
   }
 
   // for unit-test usage
@@ -130,11 +132,16 @@ public void incrementQuotaObserverTime(final long executionTime) {
     masterQuotaSource.incrementSpaceQuotaObserverChoreTime(executionTime);
   }
 
-  /** Returns Set of metrics for assign procedure */
+  /** Returns Set of metrics for server crash procedure */
   public ProcedureMetrics getServerCrashProcMetrics() {
     return serverCrashProcMetrics;
   }
 
+  /** Returns Set of metrics for split WAL procedure */
+  public ProcedureMetrics getSplitWALProcMetrics() {
+    return splitWALProcMetrics;
+  }
+
   /**
    * This is utility function that converts {@link OperationMetrics} to {@link ProcedureMetrics}.
    * NOTE: Procedure framework in hbase-procedure module accesses metrics common to most procedures

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFileSystem.java

@@ -20,6 +20,13 @@
 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
 import org.apache.yetus.audience.InterfaceAudience;
 
+/**
+ * @deprecated since 2.4.0 and in 3.0.0, to be removed in 4.0.0, replaced by procedure-based
+ *             distributed WAL splitter; see SplitWALManager. These metrics ({@code hlogSplitTime},
+ *             {@code hlogSplitSize}, {@code metaHlogSplitTime}, {@code metaHlogSplitSize}) are only
+ *             emitted by the deprecated ZK-coordinated WAL split path.
+ */
+@Deprecated
 @InterfaceAudience.Private
 public class MetricsMasterFileSystem {
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SplitWALProcedure.java

@@ -21,6 +21,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.master.SplitWALManager;
+import org.apache.hadoop.hbase.procedure2.ProcedureMetrics;
 import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
 import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
 import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
@@ -193,6 +194,11 @@ protected void afterReplay(MasterProcedureEnv env) {
     }
   }
 
+  @Override
+  protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) {
+    return env.getMasterServices().getMasterMetrics().getSplitWALProcMetrics();
+  }
+
   @Override
   protected void toStringClassDetails(StringBuilder builder) {
     builder.append(getProcName());

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java

@@ -198,6 +198,8 @@ public void testDefaultMasterMetrics() throws Exception {
 
     metricsHelper.assertCounter(MetricsMasterSource.SERVER_CRASH_METRIC_PREFIX + "SubmittedCount",
       0, masterSource);
+    metricsHelper.assertCounter(MetricsMasterSource.SPLIT_WAL_METRIC_PREFIX + "SubmittedCount", 0,
+      masterSource);
     metricsHelper.assertGauge("oldWALsDirSize", master.getMasterWalManager().getOldWALsDirSize(),
       masterSource);
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitWALManager.java

@@ -31,6 +31,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.CompatibilityFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtil;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.ServerName;
@@ -45,6 +46,7 @@
 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
 import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
 import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.test.MetricsAssertHelper;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -69,6 +71,8 @@
 public class TestSplitWALManager {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestSplitWALManager.class);
+  private static final MetricsAssertHelper METRICS_HELPER =
+    CompatibilityFactory.getInstance(MetricsAssertHelper.class);
   private static HBaseTestingUtil TEST_UTIL;
   private HMaster master;
   private SplitWALManager splitWALManager;
@@ -231,6 +235,9 @@ private void splitLogsTestHelper(HBaseTestingUtil testUtil) throws Exception {
     // load table
     testUtil.loadTable(testUtil.getConnection().getTable(TABLE_NAME), FAMILY);
     ProcedureExecutor<MasterProcedureEnv> masterPE = hmaster.getMasterProcedureExecutor();
+    MetricsMasterSource masterSource = hmaster.getMasterMetrics().getMetricsSource();
+    long splitWALSubmittedBase = METRICS_HELPER
+      .getCounter(MetricsMasterSource.SPLIT_WAL_METRIC_PREFIX + "SubmittedCount", masterSource);
     ServerName metaServer = testUtil.getHBaseCluster().getServerHoldingMeta();
     ServerName testServer = testUtil.getHBaseCluster().getRegionServerThreads().stream()
       .map(rs -> rs.getRegionServer().getServerName()).filter(rs -> rs != metaServer).findAny()
@@ -239,6 +246,9 @@ private void splitLogsTestHelper(HBaseTestingUtil testUtil) throws Exception {
     assertEquals(1, procedures.size());
     ProcedureTestingUtility.submitAndWait(masterPE, procedures.get(0));
     assertEquals(0, splitWALManager.getWALsToSplit(testServer, false).size());
+    // The SplitWALProcedure above should have been reported to the split WAL metric.
+    METRICS_HELPER.assertCounter(MetricsMasterSource.SPLIT_WAL_METRIC_PREFIX + "SubmittedCount",
+      splitWALSubmittedBase + 1, masterSource);
 
     // Validate the old WAL file archive dir
     Path walRootDir = hmaster.getMasterFileSystem().getWALRootDir();
@@ -251,6 +261,9 @@ private void splitLogsTestHelper(HBaseTestingUtil testUtil) throws Exception {
     ProcedureTestingUtility.submitAndWait(masterPE, procedures.get(0));
     assertEquals(0, splitWALManager.getWALsToSplit(metaServer, true).size());
     assertEquals(1, splitWALManager.getWALsToSplit(metaServer, false).size());
+    // The meta SplitWALProcedure should also have been counted by the split WAL metric.
+    METRICS_HELPER.assertCounter(MetricsMasterSource.SPLIT_WAL_METRIC_PREFIX + "SubmittedCount",
+      splitWALSubmittedBase + 2, masterSource);
     // There should be archiveFileCount + 1 WALs after SplitWALProcedure finish
     assertEquals(archiveFileCount + 1, walFS.listStatus(walArchivePath).length,
       "Splitted WAL files should be archived");