OAK-11609 : add support for metrics export to Prometheus via pushgateway for VersionGarbageCollector (#2216)

shodaaan · web-flow · commit 4b7c14a6dd4d · 2025-04-25T17:42:54.000+02:00
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/NodeStoreFixtureProvider.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/NodeStoreFixtureProvider.java
@@ -100,15 +100,19 @@ private static boolean oldSegmentStore(Options options) {
         return !manifest.exists();
     }
 
+    public static StatisticsProvider createStatsProvider(Whiteboard wb, Closer closer) {
+        ScheduledExecutorService executorService =
+                MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1));
+        MetricStatisticsProvider statsProvider = new MetricStatisticsProvider(getPlatformMBeanServer(), executorService);
+        closer.register(statsProvider);
+        closer.register(() -> reportMetrics(statsProvider));
+        wb.register(MetricRegistry.class, statsProvider.getRegistry(), emptyMap());
+        return statsProvider;
+    }
+
     private static StatisticsProvider createStatsProvider(Options options, Whiteboard wb, Closer closer) {
         if (options.getCommonOpts().isMetricsEnabled()) {
-            ScheduledExecutorService executorService =
-                    MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1));
-            MetricStatisticsProvider statsProvider = new MetricStatisticsProvider(getPlatformMBeanServer(), executorService);
-            closer.register(statsProvider);
-            closer.register(() -> reportMetrics(statsProvider));
-            wb.register(MetricRegistry.class, statsProvider.getRegistry(), emptyMap());
-            return statsProvider;
+            return createStatsProvider(wb, closer);
         }
         return StatisticsProvider.NOOP;
     }
@@ -163,7 +167,7 @@ public void close() throws IOException {
         }
     }
 
-    private static class ClosingWhiteboard implements Whiteboard {
+    public static class ClosingWhiteboard implements Whiteboard {
         private final Whiteboard delegate;
         private final Closer closer;
 
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixture.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixture.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import org.apache.jackrabbit.oak.plugins.document.FullGCMetricsExporter;
+
+/**
+ * Fixture encapsulating FullGC metrics exporter instance of T
+ * @param <T>
+ */
+public interface FullGCMetricsExporterFixture<T> extends FullGCMetricsExporter<T>, MetricsExporterFixture<T> {
+}
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixtureProvider.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixtureProvider.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import com.codahale.metrics.MetricRegistry;
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.dropwizard.DropwizardExports;
+import io.prometheus.client.exporter.PushGateway;
+import org.apache.jackrabbit.oak.run.MetricsExporterFixtureProvider.ExportMetricsArgs;
+import org.apache.jackrabbit.oak.run.MetricsExporterFixtureProvider.ExporterType;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.getService;
+
+/**
+ * Initializes metrics exported fixture for fullGC. For use in oak-run RevisionsCommand.
+ */
+class FullGCMetricsExporterFixtureProvider {
+
+    private static final Logger log = LoggerFactory.getLogger(NodeStoreFixtureProvider.class);
+
+    @Nullable
+    static FullGCMetricsExporterFixture<PushGateway> create(RevisionsCommand.RevisionsOptions options, Whiteboard wb) {
+        if (options.exportMetrics()) {
+            CollectorRegistry collectorRegistry = new CollectorRegistry();
+            wb.register(CollectorRegistry.class, collectorRegistry, emptyMap());
+
+            MetricRegistry metricRegistry = getService(wb, MetricRegistry.class);
+
+            ExportMetricsArgs metricsArgs = new ExportMetricsArgs(options.exportMetricsArgs());
+            if (metricsArgs.getExporterType() == ExporterType.pushgateway) {
+                PushGateway pg = new PushGateway(metricsArgs.getPushUri());
+                new DropwizardExports(metricRegistry).register(collectorRegistry);
+
+                wb.register(PushGateway.class, pg, emptyMap());
+                return new FullGCMetricsExporterFixture<>() {
+                    public ExporterType getExporterType() {
+                        return ExporterType.pushgateway;
+                    }
+
+                    public PushGateway getMetricsExporter() {
+                        return pg;
+                    }
+
+                    @Override
+                    public void close() {
+                        pushMetrics(collectorRegistry, pg, metricsArgs);
+                    }
+
+                    /**
+                     * Push the metricsMap that is passed from VersionGarbageCollector to pushgateway.
+                     */
+                    @Override
+                    public void onIterationComplete() {
+                        pushMetrics(collectorRegistry, pg, metricsArgs);
+                    }
+                };
+            }
+        }
+        return null;
+    }
+
+    private static void pushMetrics(CollectorRegistry collectorRegistry, PushGateway pg, ExportMetricsArgs metricsArgs) {
+        try {
+            log.info("Pushing metrics to pushgateway: ", metricsArgs.getPushMap());
+            pg.pushAdd(collectorRegistry, PushGateway.class.getName(), metricsArgs.getPushMap());
+        } catch (IOException e) {
+            log.error("Error pushing metrics to pushgateway", e);
+        }
+    }
+}
diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
@@ -18,6 +18,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
@@ -42,6 +43,7 @@
 import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder;
 import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
 import org.apache.jackrabbit.oak.plugins.document.FormatVersion;
+import org.apache.jackrabbit.oak.plugins.document.FullGCMetricsExporter;
 import org.apache.jackrabbit.oak.plugins.document.MissingLastRevSeeker;
 import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
 import org.apache.jackrabbit.oak.plugins.document.RevisionContextWrapper;
@@ -50,12 +52,16 @@
 import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCInfo;
 import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
 import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
 import org.apache.jackrabbit.oak.run.commons.Command;
 import org.apache.jackrabbit.oak.plugins.document.VersionGCOptions;
 import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector;
 import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
 import org.apache.jackrabbit.oak.spi.gc.LoggingGCMonitor;
+import org.apache.jackrabbit.oak.spi.whiteboard.DefaultWhiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
 import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
 import org.bson.Document;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -75,6 +81,7 @@
 import static org.apache.jackrabbit.oak.run.Utils.asCloseable;
 import static org.apache.jackrabbit.oak.run.Utils.createDocumentMKBuilder;
 import static org.apache.jackrabbit.oak.run.Utils.getMongoConnection;
+import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.getService;
 
 /**
  * Gives information about current node revisions state.
@@ -118,7 +125,7 @@ public RevisionsCommand(boolean exitWhenDone) {
         this.exitWhenDone = exitWhenDone;
     }
 
-    private static class RevisionsOptions extends Utils.NodeStoreOptions {
+    static class RevisionsOptions extends Utils.NodeStoreOptions {
 
         static final String CMD_INFO = "info";
         static final String CMD_COLLECT = "collect";
@@ -148,6 +155,7 @@ private static class RevisionsOptions extends Utils.NodeStoreOptions {
         final OptionSpec<Boolean> embeddedVerification;
         final OptionSpec<Integer> fullGcMode;
         final OptionSpec<Boolean> fullGCAuditLoggingEnabled;
+        final OptionSpec<String> exportMetrics;
 
         RevisionsOptions(String usage) {
             super(usage);
@@ -211,6 +219,8 @@ private static class RevisionsOptions extends Utils.NodeStoreOptions {
                     .withOptionalArg().ofType(Long.class).defaultsTo(TimeUnit.DAYS.toSeconds(1));
             fullGCAuditLoggingEnabled = parser.accepts("fullGCAuditLoggingEnabled", "Enable audit logging for Full GC")
                     .withOptionalArg().ofType(Boolean.class).defaultsTo(FALSE);
+            exportMetrics = parser.accepts("exportMetrics",
+                    "type, URI to export the metrics and optional metadata all delimeted by semi-colon(;)").withRequiredArg();
         }
 
         public RevisionsOptions parse(String[] args) {
@@ -313,6 +323,14 @@ boolean doCompaction() {
         Boolean isFullGCAuditLoggingEnabled() {
             return options.has(fullGCAuditLoggingEnabled);
         }
+
+        boolean exportMetrics() {
+            return options.has(exportMetrics);
+        }
+
+        String exportMetricsArgs() {
+            return exportMetrics.value(options);
+        }
     }
 
     @Override
@@ -382,7 +400,6 @@ private VersionGarbageCollector bootstrapVGC(RevisionsOptions options, Closer cl
         builder.setFullGCBatchSize(options.getFullGcBatchSize());
         builder.setFullGCProgressSize(options.getFullGcProgressSize());
         builder.setFullGcMaxAgeMillis(SECONDS.toMillis(options.getFullGcMaxAge()));
-        builder.setFullGCAuditLoggingEnabled(options.isFullGCAuditLoggingEnabled());
 
         // create a VersionGCSupport while builder is read-write
         VersionGCSupport gcSupport = builder.createVersionGCSupport();
@@ -416,7 +433,6 @@ private VersionGarbageCollector bootstrapVGC(RevisionsOptions options, Closer cl
         System.out.println("FullGcProgressSize is : " + options.getFullGcProgressSize());
         System.out.println("FullGcMaxAgeInSecs is : " + options.getFullGcMaxAge());
         System.out.println("FullGcMaxAgeMillis is : " + builder.getFullGcMaxAgeMillis());
-        System.out.println("FullGCAuditLoggingEnabled is : " + options.isFullGCAuditLoggingEnabled());
         VersionGarbageCollector gc = createVersionGC(builder.build(), gcSupport, options.isDryRun(), builder);
 
         VersionGCOptions gcOptions = gc.getOptions();
@@ -460,11 +476,22 @@ private void printInfo(VersionGarbageCollector gc, RevisionsOptions options) thr
 
     private void collect(final RevisionsOptions options, Closer closer, boolean fullGCEnabled) throws IOException {
         VersionGarbageCollector gc = bootstrapVGC(options, closer, fullGCEnabled);
-        // Set a default statistics provider
-        gc.setStatisticsProvider(new DefaultStatisticsProvider(Executors.newSingleThreadScheduledExecutor()));
+
+        // setup metrics exporter
+        Whiteboard whiteboard = new NodeStoreFixtureProvider.ClosingWhiteboard(new DefaultWhiteboard(), closer);
+        StatisticsProvider statsProvider = NodeStoreFixtureProvider.createStatsProvider(whiteboard, closer);
+        whiteboard.register(StatisticsProvider.class, statsProvider, Collections.emptyMap());
+        gc.setStatisticsProvider(statsProvider, true);
+
+        FullGCMetricsExporter metricsExporter = FullGCMetricsExporterFixtureProvider.create(options, whiteboard);
+        gc.setFullGCMetricsExporter(metricsExporter);
+
         ExecutorService executor = Executors.newSingleThreadExecutor();
         final Semaphore finished = new Semaphore(0);
         try {
+            // register metrics exporter to closer
+            closer.register(metricsExporter);
+
             // collect until shutdown hook is called
             final AtomicBoolean running = new AtomicBoolean(true);
             Runtime.getRuntime().addShutdownHook(new Thread(() -> {
@@ -490,6 +517,7 @@ private void collect(final RevisionsOptions options, Closer closer, boolean full
             }
             System.out.println("retrieving gc info");
             printInfo(gc, options);
+        } catch (Exception e) {
         } finally {
             finished.release();
             if (options.isDryRun()) {
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCMetricsExporter.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCMetricsExporter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.io.Closeable;
+import java.util.Map;
+
+/**
+ * Exporter interface for setting dependency for VersionGarbageCollector that allows
+ * for export of fullGC metrics to Prometheus via pushgateway.
+ * @param <T>
+ */
+public interface FullGCMetricsExporter<T> extends Closeable {
+
+    /**
+     * Called from VersionGarbageCollector when a fullGC iteration completes.
+     */
+    void onIterationComplete();
+}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImpl.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImpl.java
@@ -37,6 +37,7 @@
  */
 class FullGCStatsCollectorImpl implements FullGCStatsCollector {
 
+    static final String OAK_RUN_METRICS_PREFIX = "oak_FullGC";
     static final String FULL_GC = "FullGC";
     static final String READ_DOC = "READ_DOC";
     static final String DELETED_ORPHAN_NODE = "DELETED_ORPHAN_NODE";
@@ -83,9 +84,15 @@ class FullGCStatsCollectorImpl implements FullGCStatsCollector {
 
     private final CounterStats counter;
     private final CounterStats failureCounter;
+    private static String METRICS_QUALIFIED_NAME_PREFIX;
 
     FullGCStatsCollectorImpl(StatisticsProvider provider) {
+        this(provider, false);
+    }
+
+    FullGCStatsCollectorImpl(StatisticsProvider provider, boolean isOakRunJob) {
         this.provider = provider;
+        this.METRICS_QUALIFIED_NAME_PREFIX = isOakRunJob ? OAK_RUN_METRICS_PREFIX : FULL_GC;
 
         readDoc = meter(provider, READ_DOC);
         deletedOrphanNode = meter(provider, DELETED_ORPHAN_NODE);
@@ -232,11 +239,10 @@ private static CounterStats counter(StatisticsProvider provider, String name) {
     }
 
     private static String qualifiedName(String metricName) {
-        return FULL_GC + "." + metricName;
+        return METRICS_QUALIFIED_NAME_PREFIX + "." + metricName;
     }
 
     private MeterStats getMeter(Map<GCPhase, MeterStats> map, GCPhase phase, String name) {
         return map.computeIfAbsent(phase, p -> meter(provider, name + "." + p.name()));
     }
-
 }
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
