add slow request sampler

core/src/main/java/com/linecorp/armeria/common/util/AndSampler.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.linecorp.armeria.common.util;
+
+/**
+ * Sample if both of the samplers sample.
+ */
+final class AndSampler<T> implements Sampler<T> {
+
+    private final Sampler<T> left;
+    private final Sampler<T> right;
+
+    AndSampler(Sampler<T> left, Sampler<T> right) {
+        this.left = left;
+        this.right = right;
+    }
+
+    @Override
+    public boolean isSampled(T t) {
+        // Assign the variables otherwise the short-circuiting will cause sampler to not be used.
+        final boolean leftSampled = left.isSampled(t);
+        final boolean rightSampled = right.isSampled(t);
+        return leftSampled && rightSampled;
+    }
+
+    @Override
+    public String toString() {
+        return left.toString() + " and " + right.toString();
+    }
+}

core/src/main/java/com/linecorp/armeria/common/util/OrSampler.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.linecorp.armeria.common.util;
+
+/**
+ * Sample if one of the samplers samples.
+ */
+final class OrSampler<T> implements Sampler<T> {
+
+    private final Sampler<T> left;
+    private final Sampler<T> right;
+
+    OrSampler(Sampler<T> left, Sampler<T> right) {
+        this.left = left;
+        this.right = right;
+    }
+
+    @Override
+    public boolean isSampled(T t) {
+        // Assign the variables otherwise the short-circuiting will cause sampler to not be used.
+        final boolean leftSampled = left.isSampled(t);
+        final boolean rightSampled = right.isSampled(t);
+        return leftSampled || rightSampled;
+    }
+
+    @Override
+    public String toString() {
+        return left.toString() + " or " + right.toString();
+    }
+}

core/src/main/java/com/linecorp/armeria/common/util/Sampler.java

@@ -42,6 +42,64 @@
  */
 @FunctionalInterface
 public interface Sampler<T> {
+
+    /**
+     * Returns a sampler that applies logical or operator to both samplers decisions.
+     */
+    default Sampler<T> or(Sampler<T> other) {
+        return new OrSampler<>(this, other);
+    }
+
+    /**
+     * Returns a sampler that applies logical and operator to both samplers decisions.
+     */
+    default Sampler<T> and(Sampler<T> other) {
+        return new AndSampler<>(this, other);
+    }
+
+    /**
+     * Returns a sampler that returns {@code true} if the value is greater than the given value.
+     */
+    static <T extends Comparable<T>> Sampler<T> greaterThan(T val) {
+        return object -> object.compareTo(val) > 0;
+    }
+
+    /**
+     * Returns a sampler that returns {@code true} if the value is less than or equal to the given value.
+     */
+    static <T extends Comparable<T>> Sampler<T> greaterThanOrEqual(T val) {
+        return object -> object.compareTo(val) >= 0;
+    }
+
+    /**
+     * Returns a sampler that returns {@code true} if the value is less than the given value.
+     */
+    static <T extends Comparable<T>> Sampler<T> lessThan(T val) {
+        return object -> object.compareTo(val) < 0;
+    }
+
+    /**
+     * Returns a sampler that returns {@code true} if the value is less than or equal to the given value.
+     */
+    static <T extends Comparable<T>> Sampler<T> lessThanOrEqual(T val) {
+        return object -> object.compareTo(val) <= 0;
+    }
+
+    /**
+     * Returns a sampler that returns {@code true} if the value is equal to the given value.
+     */
+    static <T extends Comparable<T>> Sampler<T> equal(T val) {
+        return object -> object.compareTo(val) == 0;
+    }
+
+    /**
+     * Returns a sampler that returns {@code true} if the value is inside the percentile distribution
+     * in the given time window.
+     */
+    static Sampler<Long> percentile(float percentile, long windowMilliseconds) {
+        return new TimeWindowPercentileSampler(percentile, windowMilliseconds);
+    }
+
     /**
      * Returns a probabilistic sampler which samples at the specified {@code probability}
      * between {@code 0.0} and {@code 1.0}.

core/src/main/java/com/linecorp/armeria/common/util/TimeWindowPercentileSampler.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2023 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.linecorp.armeria.common.util;
+
+import java.time.Duration;
+
+import io.micrometer.core.instrument.Clock;
+import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
+import io.micrometer.core.instrument.distribution.HistogramSnapshot;
+import io.micrometer.core.instrument.distribution.TimeWindowPercentileHistogram;
+
+/**
+ * Sample if the value is less than the percentile of the values in the last window.
+ */
+final class TimeWindowPercentileSampler implements Sampler<Long> {
+
+    private final float percentile;
+    private final long windowLengthMillis;
+
+    private final TimeWindowPercentileHistogram histogram;
+    private static final long SNAPSHOT_UPDATE_MILLIS = 1000L;
+    private long lastSnapshotMillis = 0L;
+    private HistogramSnapshot histogramSnapshot;
+
+    TimeWindowPercentileSampler(float percentile, long windowLengthMillis) {
+        this.percentile = percentile;
+        this.windowLengthMillis = windowLengthMillis;
+
+        final DistributionStatisticConfig distributionStatisticConfig =
+                DistributionStatisticConfig.builder()
+                                           .percentiles(percentile)
+                                           .percentilePrecision(2)
+                                           .minimumExpectedValue(1.0)
+                                           .maximumExpectedValue(Double.POSITIVE_INFINITY)
+                                           .expiry(Duration.ofMillis(windowLengthMillis))
+                                           .bufferLength(3)
+                                           .build();
+        this.histogram = new TimeWindowPercentileHistogram(Clock.SYSTEM, distributionStatisticConfig, true);
+        this.histogramSnapshot = histogram.takeSnapshot(0, 0, 0);
+        this.lastSnapshotMillis = System.currentTimeMillis();
+    }
+
+    static TimeWindowPercentileSampler create(float percentile, long windowLengthMillis) {
+        return new TimeWindowPercentileSampler(percentile, windowLengthMillis);
+    }
+
+    @Override
+    public boolean isSampled(Long t) {
+        histogram.recordLong(t);
+
+        if (lastSnapshotMillis + SNAPSHOT_UPDATE_MILLIS < System.currentTimeMillis()) {
+            histogramSnapshot = histogram.takeSnapshot(0, 0, 0);
+            lastSnapshotMillis = System.currentTimeMillis();
+        }
+
+        final Double percentileValue = histogramSnapshot.percentileValues()[0].value();
+        return percentileValue.longValue() <= t;
+    }
+
+    @Override
+    public String toString() {
+        return "SlidingWindowPercentileSampler with " + windowLengthMillis + " ms window and " + percentile +
+               " percentile";
+    }
+}

core/src/main/java/com/linecorp/armeria/server/annotation/decorator/LoggingDecorator.java

@@ -83,6 +83,28 @@
      */
     float failureSamplingRate() default -1.0f;
 
+    /**
+     * Sample the requests if they are slower than the {@code slowRequestSamplingPercentile} percent
+     * of the requests.
+     */
+    float slowRequestSamplingPercentile() default -1.0f;
+
+    /**
+     * Slow request percentiles are calculated over the last {@code slowRequestSamplingWindowMilliseconds}.
+     */
+    long slowRequestSamplingWindowMilliseconds() default 10 * 60 * 1000;
+
+    /**
+     * Don't sample the requests if they are faster than the {@code slowRequestSamplingLowerBoundMilliseconds}.
+     * Should be used with {@link #slowRequestSamplingUpperBoundMilliseconds()}.
+     */
+    long slowRequestSamplingLowerBoundMilliseconds() default 0L;
+
+    /**
+     * Always sample the requests if they are slower than the {@code slowRequestSamplingUpperBoundMilliseconds}.
+     */
+    long slowRequestSamplingUpperBoundMilliseconds() default Long.MAX_VALUE;
+
     /**
      * The order of decoration, where a {@link Decorator} of lower value will be applied first.
      */

core/src/main/java/com/linecorp/armeria/server/annotation/decorator/LoggingDecoratorFactoryFunction.java

@@ -21,6 +21,7 @@
 import com.linecorp.armeria.server.HttpService;
 import com.linecorp.armeria.server.annotation.DecoratorFactoryFunction;
 import com.linecorp.armeria.server.logging.LoggingService;
+import com.linecorp.armeria.server.logging.LoggingServiceBuilder;
 
 /**
  * A factory which creates a {@link LoggingService} decorator.
@@ -38,15 +39,23 @@ public final class LoggingDecoratorFactoryFunction implements DecoratorFactoryFu
         final float failureSamplingRate =
                 parameter.failureSamplingRate() >= 0.0f ? parameter.failureSamplingRate()
                                                         : parameter.samplingRate();
-        return LoggingService.builder()
-                             .logWriter(LogWriter.builder()
-                                                 .requestLogLevel(parameter.requestLogLevel())
-                                                 .successfulResponseLogLevel(
-                                                         parameter.successfulResponseLogLevel())
-                                                 .failureResponseLogLevel(parameter.failureResponseLogLevel())
-                                                 .build())
-                             .successSamplingRate(successSamplingRate)
-                             .failureSamplingRate(failureSamplingRate)
-                             .newDecorator();
+        final LogWriter logWriter = LogWriter.builder()
+                                             .requestLogLevel(parameter.requestLogLevel())
+                                             .successfulResponseLogLevel(parameter.successfulResponseLogLevel())
+                                             .failureResponseLogLevel(parameter.failureResponseLogLevel())
+                                             .build();
+        final LoggingServiceBuilder builder
+                = LoggingService.builder()
+                                .logWriter(logWriter)
+                                .successSamplingRate(successSamplingRate)
+                                .failureSamplingRate(failureSamplingRate);
+
+        if (parameter.slowRequestSamplingPercentile() >= 0.0f) {
+            builder.slowRequestSamplingPercentile(parameter.slowRequestSamplingPercentile(),
+                                                  parameter.slowRequestSamplingWindowMilliseconds(),
+                                                  parameter.slowRequestSamplingLowerBoundMilliseconds(),
+                                                  parameter.slowRequestSamplingUpperBoundMilliseconds());
+        }
+        return builder.newDecorator();
     }
 }

core/src/main/java/com/linecorp/armeria/server/logging/LoggingService.java

@@ -56,17 +56,23 @@ public static LoggingServiceBuilder builder() {
 
     LoggingService(HttpService delegate, LogWriter logWriter,
                    Sampler<? super ServiceRequestContext> successSampler,
-                   Sampler<? super ServiceRequestContext> failureSampler) {
+                   Sampler<? super ServiceRequestContext> failureSampler,
+                   Sampler<Long> slowRequestSampler) {
         super(requireNonNull(delegate, "delegate"));
         this.logWriter = requireNonNull(logWriter, "logWriter");
         requireNonNull(successSampler, "successSampler");
         requireNonNull(failureSampler, "failureSampler");
         sampler = requestLog -> {
             final ServiceRequestContext ctx = (ServiceRequestContext) requestLog.context();
+            final boolean isSlow = slowRequestSampler.isSampled(requestLog.totalDurationNanos());
+            final boolean successOrFailure;
             if (ctx.config().successFunction().isSuccess(ctx, requestLog)) {
-                return successSampler.isSampled(ctx);
+                successOrFailure = successSampler.isSampled(ctx);
+            } else {
+                successOrFailure = failureSampler.isSampled(ctx);
             }
-            return failureSampler.isSampled(ctx);
+
+            return successOrFailure || isSlow;
         };
     }
 

core/src/main/java/com/linecorp/armeria/server/logging/LoggingServiceBuilder.java

@@ -50,6 +50,8 @@ public final class LoggingServiceBuilder extends LoggingDecoratorBuilder {
 
     private Sampler<? super ServiceRequestContext> failureSampler = Sampler.always();
 
+    private Sampler<Long> slowRequestSampler = Sampler.never();
+
     LoggingServiceBuilder() {}
 
     /**
@@ -107,6 +109,14 @@ public LoggingServiceBuilder failureSampler(
         return this;
     }
 
+    /**
+     * Sets the {@link Sampler} that determines whether a request is slow enough to be logged.
+     */
+    public LoggingServiceBuilder slowRequestSampler(Sampler<Long> slowRequestSampler) {
+        this.slowRequestSampler = requireNonNull(slowRequestSampler, "slowRequestSampler");
+        return this;
+    }
+
     /**
      * Sets the rate at which to sample requests to log. Any number between {@code 0.0} and {@code 1.0} will
      * cause a random sample of the failure requests to be logged.
@@ -118,12 +128,57 @@ public LoggingServiceBuilder failureSamplingRate(float failureSamplingRate) {
         return failureSampler(Sampler.random(failureSamplingRate));
     }
 
+    /**
+     * Sets conditions to sample slow requests.
+     *
+     * <p>If {slowRequestPercentile} is 0.99, {windowMilliseconds} is 60000, we will sample requests that are
+     * slower than 99% of the requests within 1-minute time window. This will make sure our sampling adapts the
+     * traffic pattern. For example, traffic can be very low during the night thus our average response time
+     * will be pretty low. Therefore, our p99s should be lower than the midday traffic.
+     * Otherwise, we won't log any p99s during night.</p>
+     *
+     * <p>If we set {slowRequestSamplingLowerBoundMilliseconds} to 100, we won't sample any request that took
+     * less than 1000ms. This is useful to filter out requests that are too fast to be considered
+     * as slow requests. If your endpoint performs pretty healthy, you shouldn't see any slow request logs.</p>
+     *
+     * <p>If we set {slowRequestSamplingUpperBoundMilliseconds} to 1000, we will sample any request that
+     * took more than 1000 milliseconds. This is useful to make sure we log slow requests even if
+     * they are not in the p99 percentile. If your service is unhealthy, this will make sure any slow request
+     * is logged into your system.</p>
+     *
+     * @param slowRequestPercentile percentile of slow requests.
+     * @param windowMilliseconds window size to calculate percentile.
+     * @param slowRequestSamplingLowerBoundMilliseconds lower bound of slow requests.
+     *          Any request that took less than this amount of time won't be sampled.
+     * @param slowRequestSamplingUpperBoundMilliseconds upper bound of slow requests.
+     *          Any request that took more than this amount of time will be sampled regardless
+     *          of their percentile.
+     */
+    public LoggingServiceBuilder slowRequestSamplingPercentile(float slowRequestPercentile,
+                                                               long windowMilliseconds,
+                                                               long slowRequestSamplingLowerBoundMilliseconds,
+                                                               long slowRequestSamplingUpperBoundMilliseconds) {
+        final Sampler<Long> percentileMatches;
+        if (0.0 <= slowRequestPercentile && slowRequestPercentile <= 1.0) {
+            percentileMatches = Sampler.never();
+        } else {
+            percentileMatches = Sampler.percentile(slowRequestPercentile, windowMilliseconds);
+        }
+
+        final Sampler<Long> isSlow = Sampler.greaterThanOrEqual(
+                slowRequestSamplingLowerBoundMilliseconds * 1000);
+        final Sampler<Long> isVerySlow = Sampler.greaterThan(slowRequestSamplingUpperBoundMilliseconds * 1000);
+        return slowRequestSampler(
+                isVerySlow.or(percentileMatches.and(isSlow))
+        );
+    }
+
     /**
      * Returns a newly-created {@link LoggingService} decorating {@link HttpService} based on the properties
      * of this builder.
      */
     public LoggingService build(HttpService delegate) {
-        return new LoggingService(delegate, logWriter(), successSampler, failureSampler);
+        return new LoggingService(delegate, logWriter(), successSampler, failureSampler, slowRequestSampler);
     }
 
     /**