ZOOKEEPER-4952:Reduce the GC overhead of Prometheus reject exception handling

Author: Li Wang <liwang@apple.com>

Author: liwang

zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/main/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProvider.java

@@ -33,7 +33,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -677,7 +677,7 @@ private Optional<ExecutorService> createExecutor() {
                 numWorkerThreads,
                 0L,
                 TimeUnit.MILLISECONDS,
-                queue, new PrometheusWorkerThreadFactory());
+                queue, new PrometheusWorkerThreadFactory(), new PrometheusRejectedExecutionHandler());
         LOG.info("Executor service was created with numWorkerThreads {} and maxQueueSize {}",
                 numWorkerThreads,
                 maxQueueSize);
@@ -714,14 +714,21 @@ public Thread newThread(final Runnable runnable) {
         }
     }
 
+    private static class PrometheusRejectedExecutionHandler implements RejectedExecutionHandler {
+        private static boolean maxQueueSizeExceeded = false;
+
+        @Override
+        public void rejectedExecution(final Runnable r, final ThreadPoolExecutor e) {
+            if (!maxQueueSizeExceeded) {
+                maxQueueSizeExceeded = true;
+                LOG.warn("Prometheus metrics queue size exceeded the max");
+            }
+        }
+    }
+
     private void reportMetrics(final Runnable task) {
         if (executorOptional.isPresent()) {
-            try {
-                executorOptional.get().submit(task);
-            } catch (final RejectedExecutionException e) {
-                rateLogger.rateLimitLog("Prometheus metrics reporting task queue size exceeded the max",
-                        String.valueOf(maxQueueSize));
-            }
+            executorOptional.get().submit(task);
         } else {
             task.run();
         }