Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,8 @@ public double getRank(final double value) {
if (value < firstMean) {
if ((firstMean - minValue_) > 0) {
if (value == minValue_) { return 0.5 / centroidsWeight_; }
return (1.0 + (((value - minValue_) / (firstMean - minValue_)) * ((centroidWeights_[0] / 2.0) - 1.0)));
return (1.0 + (((value - minValue_) / (firstMean - minValue_))
* ((centroidWeights_[0] / 2.0) - 1.0))) / centroidsWeight_;
}
return 0; // should never happen
}
Expand Down Expand Up @@ -255,7 +256,7 @@ public double getQuantile(final double rank) {
}
final double lastWeight = centroidWeights_[numCentroids_ - 1];
if ((lastWeight > 1) && ((centroidsWeight_ - weight) <= (lastWeight / 2.0))) {
return maxValue_ + (((centroidsWeight_ - weight - 1.0) / ((lastWeight / 2.0) - 1.0))
return maxValue_ - (((centroidsWeight_ - weight - 1.0) / ((lastWeight / 2.0) - 1.0))
* (maxValue_ - centroidMeans_[numCentroids_ - 1]));
}

Expand All @@ -277,7 +278,7 @@ public double getQuantile(final double rank) {
}
final double w1 = weight - weightSoFar - leftWeight;
final double w2 = (weightSoFar + dw) - weight - rightWeight;
return weightedAverage(centroidMeans_[i], w1, centroidMeans_[i + 1], w2);
return weightedAverage(centroidMeans_[i], w2, centroidMeans_[i + 1], w1);
}
weightSoFar += dw;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,59 @@ public void deserializeNaNSingleValue() {
assertThrows(SketchesArgumentException.class, () -> TDigestDouble.heapify(MemorySegment.ofArray(bytes)));
}

@Test
public void rankBelowFirstCentroidMean() {
// the format allows a first centroid of weight greater than 1, so the left tail of
// getRank() must stay normalized just like the right tail
final byte[] bytes = serializeNonEmpty();
MemorySegment.ofArray(bytes).set(ValueLayout.JAVA_DOUBLE_UNALIGNED, 16, -1); // min
MemorySegment.ofArray(bytes).set(ValueLayout.JAVA_LONG_UNALIGNED, 40, 100L); // first weight
final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes));
final double totalWeight = td.getTotalWeight();
assertEquals(td.getRank(-1), 0.5 / totalWeight);
assertEquals(td.getRank(-0.5), (1.0 + (((100 / 2.0) - 1.0) * 0.5)) / totalWeight);
double previous = 0;
for (int i = 0; i <= 100; i++) {
final double rank = td.getRank(-1 + (i / 100.0));
assertTrue((rank >= 0) && (rank <= 1), "rank out of [0, 1]: " + rank);
assertTrue(rank >= previous, "rank not monotonic: " + rank + " after " + previous);
previous = rank;
}
}

@Test
public void quantilesAreMonotonic() {
final TDigestDouble td = new TDigestDouble((short) 100);
for (int i = 0; i < 10000; i++) {
td.update(i);
}
double previous = td.getMinValue();
for (int i = 0; i <= 1000; i++) {
final double quantile = td.getQuantile(i / 1000.0);
assertTrue(quantile >= previous, "quantile not monotonic: " + quantile + " after " + previous);
assertTrue((quantile >= td.getMinValue()) && (quantile <= td.getMaxValue()),
"quantile out of [min, max]: " + quantile);
previous = quantile;
}
}

@Test
public void quantileAboveLastCentroidMean() {
final byte[] bytes = serializeNonEmpty();
final MemorySegment seg = MemorySegment.ofArray(bytes);
final int numCentroids = seg.get(ValueLayout.JAVA_INT_UNALIGNED, 8);
final long lastWeightOffset = 40 + ((numCentroids - 1) * 16L);
seg.set(ValueLayout.JAVA_LONG_UNALIGNED, lastWeightOffset, 100L);
final TDigestDouble td = TDigestDouble.heapify(seg);
double previous = td.getMinValue();
for (int i = 0; i <= 1000; i++) {
final double quantile = td.getQuantile(i / 1000.0);
assertTrue(quantile >= previous, "quantile not monotonic: " + quantile + " after " + previous);
assertTrue(quantile <= td.getMaxValue(), "quantile above max: " + quantile);
previous = quantile;
}
}

@Test
public void deserializeFromReferenceImplementationDouble() {
final byte[] bytes = TestUtil.getFileBytes(resPath, "tdigest_ref_k100_n10000_double.sk");
Expand Down