Skip to content

Commit 46b3fb9

Browse files
Relying on upstream version of datasketch and stop using HLL 4. (#2936)
We were relying on a fork for: a bugfix in LIST serialization a better API exposing a new Coupon type, required for caching coupons. We also stop using HLL8 in hope to fix https://datadoghq.atlassian.net/browse/CLOUDPREM-625 Co-authored-by: Paul Masurel <paul.masurel@datadoghq.com>
1 parent fbe620b commit 46b3fb9

2 files changed

Lines changed: 9 additions & 5 deletions

File tree

Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ tantivy-bitpacker = { version = "0.10", path = "./bitpacker" }
6565
common = { version = "0.11", path = "./common/", package = "tantivy-common" }
6666
tokenizer-api = { version = "0.7", path = "./tokenizer-api", package = "tantivy-tokenizer-api" }
6767
sketches-ddsketch = { version = "0.4", features = ["use_serde"] }
68-
datasketches = { git = "https://github.com/fulmicoton-dd/datasketches-rust", rev = "7635fb8" }
68+
datasketches = { version = "0.3.0", features = ["hll"] }
6969
futures-util = { version = "0.3.28", optional = true }
7070
futures-channel = { version = "0.3.28", optional = true }
7171
fnv = "1.0.7"

src/aggregation/metric/cardinality.rs

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -166,8 +166,12 @@ impl CouponCache {
166166
let should_use_dense =
167167
highest_term_ord < 1_000_000u64 || highest_term_ord < num_terms as u64 * 3u64;
168168
if should_use_dense {
169-
let mut coupon_map: Vec<Coupon> = vec![Coupon::EMPTY; highest_term_ord as usize + 1];
170-
for (term_ord, coupon) in term_ords.into_iter().zip(coupons.into_iter()) {
169+
// We don't really care about the value here. We will populate all the values we will
170+
// read anyway.
171+
let uninitialized_coupon = Coupon::from_hash(0);
172+
let mut coupon_map: Vec<Coupon> =
173+
vec![uninitialized_coupon; highest_term_ord as usize + 1];
174+
for (term_ord, coupon) in term_ords.into_iter().zip(coupons) {
171175
coupon_map[term_ord as usize] = coupon;
172176
}
173177
CouponCache::Dense {
@@ -821,7 +825,7 @@ impl<'de> Deserialize<'de> for CardinalityCollector {
821825
impl CardinalityCollector {
822826
fn new(salt: u8) -> Self {
823827
Self {
824-
sketch: HllSketch::new(LG_K, HllType::Hll4),
828+
sketch: HllSketch::new(LG_K, HllType::Hll8),
825829
salt,
826830
}
827831
}
@@ -852,7 +856,7 @@ impl CardinalityCollector {
852856
let mut union = HllUnion::new(LG_K);
853857
union.update(&self.sketch);
854858
union.update(&right.sketch);
855-
self.sketch = union.to_sketch(HllType::Hll4);
859+
self.sketch = union.to_sketch(HllType::Hll8);
856860
Ok(())
857861
}
858862
}

0 commit comments

Comments
 (0)