Skip to content

Commit 6697608

Browse files
committed
Fix decay for quantile digests which are more frequently read than modified
1 parent 402174e commit 6697608

File tree

2 files changed

+63
-3
lines changed

2 files changed

+63
-3
lines changed

stats/src/main/java/com/facebook/airlift/stats/QuantileDigest.java

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import java.util.concurrent.TimeUnit;
3333
import java.util.concurrent.atomic.AtomicInteger;
3434
import java.util.concurrent.atomic.AtomicLong;
35+
import java.util.function.LongBinaryOperator;
3536

3637
import static com.facebook.airlift.stats.QuantileDigest.MiddleFunction.DEFAULT;
3738
import static com.google.common.base.Preconditions.checkArgument;
@@ -328,6 +329,14 @@ public void scale(double scaleFactor)
328329
*/
329330
public List<Long> getQuantilesLowerBound(List<Double> quantiles)
330331
{
332+
if (alpha > 0.0) {
333+
long nowInSeconds = TimeUnit.NANOSECONDS.toSeconds(ticker.read());
334+
if (nowInSeconds - landmarkInSeconds >= RESCALE_THRESHOLD_SECONDS) {
335+
rescale(nowInSeconds);
336+
compress(); // rescale affects weights globally, so force compression
337+
}
338+
}
339+
331340
checkArgument(Ordering.natural().isOrdered(quantiles), "quantiles must be sorted in increasing order");
332341
for (double quantile : quantiles) {
333342
checkArgument(quantile >= 0 && quantile <= 1, "quantile must be between [0,1]");
@@ -379,6 +388,14 @@ public boolean process(int node)
379388
*/
380389
public List<Long> getQuantilesUpperBound(List<Double> quantiles)
381390
{
391+
if (alpha > 0.0) {
392+
long nowInSeconds = TimeUnit.NANOSECONDS.toSeconds(ticker.read());
393+
if (nowInSeconds - landmarkInSeconds >= RESCALE_THRESHOLD_SECONDS) {
394+
rescale(nowInSeconds);
395+
compress(); // rescale affects weights globally, so force compression
396+
}
397+
}
398+
382399
checkArgument(Ordering.natural().isOrdered(quantiles), "quantiles must be sorted in increasing order");
383400
for (double quantile : quantiles) {
384401
checkArgument(quantile >= 0 && quantile <= 1, "quantile must be between [0,1]");
@@ -411,9 +428,12 @@ public boolean process(int node)
411428

412429
// we finished the traversal without consuming all quantiles. This means the remaining quantiles
413430
// correspond to the max known value
414-
while (iterator.hasNext()) {
415-
builder.add(max);
416-
iterator.next();
431+
if (iterator.hasNext()) {
432+
long max = getMax();
433+
while (iterator.hasNext()) {
434+
builder.add(max);
435+
iterator.next();
436+
}
417437
}
418438

419439
return builder.build();
@@ -624,7 +644,13 @@ void compress()
624644
{
625645
double bound = Math.floor(weightedCount / calculateCompressionFactor());
626646

647+
AtomicLong max = new AtomicLong(Integer.MIN_VALUE);
648+
AtomicLong min = new AtomicLong(Integer.MAX_VALUE);
627649
postOrderTraversal(root, node -> {
650+
if (counts[node] >= ZERO_WEIGHT_THRESHOLD) {
651+
max.accumulateAndGet(upperBound(node), Math::max);
652+
min.accumulateAndGet(lowerBound(node), Math::min);
653+
}
628654
// if children's weights are 0 remove them and shift the weight to their parent
629655
int left = lefts[node];
630656
int right = rights[node];
@@ -655,6 +681,15 @@ void compress()
655681
// root's count may have decayed to ~0
656682
if (root != -1 && counts[root] < ZERO_WEIGHT_THRESHOLD) {
657683
root = tryRemove(root);
684+
if (root < 0) {
685+
this.min = Long.MAX_VALUE;
686+
this.max = Long.MIN_VALUE;
687+
}
688+
// If decay is being used, the min and max values may have decayed as well
689+
else if (alpha > 0) {
690+
this.min = min.get();
691+
this.max = max.get();
692+
}
658693
}
659694
}
660695

stats/src/test/java/com/facebook/airlift/stats/TestQuantileDigest.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -433,6 +433,31 @@ public void testDecayedCountsWithClockIncrementSmallerThanRescaleThreshold()
433433
assertEquals(digest.getCount(), 15.0);
434434
}
435435

436+
@Test
437+
public void testDecayedQuantilesWithNoMergeOrAdd()
438+
throws Exception
439+
{
440+
TestingTicker ticker = new TestingTicker();
441+
QuantileDigest digest = new QuantileDigest(0.01, ExponentialDecay.computeAlpha(0.5, 60), ticker);
442+
443+
addAll(digest, asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
444+
445+
assertEquals(digest.getConfidenceFactor(), 0.0);
446+
assertEquals(digest.getQuantile(0.5), 5);
447+
448+
// Decay the digest
449+
ticker.increment(60, TimeUnit.SECONDS);
450+
assertEquals(digest.getQuantile(0.5), 5);
451+
452+
// Allow further decay
453+
ticker.increment(6, TimeUnit.MINUTES);
454+
assertEquals(digest.getQuantile(0.5), 5);
455+
456+
// Values have decayed to 0
457+
ticker.increment(60, TimeUnit.MINUTES);
458+
assertEquals(digest.getQuantile(0.5), Long.MIN_VALUE); // Default value for empty digests
459+
}
460+
436461
@Test
437462
public void testMinMax()
438463
throws Exception

0 commit comments

Comments
 (0)