Skip to content

Commit 321f0a2

Browse files
hanefihanefi
authored
Round hll cardinality results to 10 decimal places (#112)
1 parent d051e1e commit 321f0a2

6 files changed

+34
-42
lines changed

‎expected/cumulative_add_cardinality_correction.out

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -56,13 +56,13 @@ SELECT curr.recno,
5656
FROM test_msgfjqhm prev, test_msgfjqhm curr
5757
WHERE curr.recno > 1
5858
AND curr.recno = prev.recno + 1
59-
AND curr.cardinality !=
60-
hll_cardinality(hll_add(prev.union_compressed_multiset,
61-
hll_hashval(curr.raw_value)))
59+
AND round(curr.cardinality::numeric, 10) !=
60+
round(hll_cardinality(hll_add(prev.union_compressed_multiset,
61+
hll_hashval(curr.raw_value)))::numeric,
62+
10)
6263
ORDER BY curr.recno;
63-
recno | cardinality | hll_cardinality
64-
-------+------------------+------------------
65-
1792 | 4250.71186178904 | 4250.71186178904
66-
(1 row)
64+
recno | cardinality | hll_cardinality
65+
-------+-------------+-----------------
66+
(0 rows)
6767

6868
DROP TABLE test_msgfjqhm;

‎expected/cumulative_add_comprehensive_promotion.out

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -56,27 +56,13 @@ SELECT curr.recno,
5656
FROM test_ptwysrqk prev, test_ptwysrqk curr
5757
WHERE curr.recno > 1
5858
AND curr.recno = prev.recno + 1
59-
AND curr.cardinality !=
60-
hll_cardinality(hll_add(prev.union_compressed_multiset,
61-
curr.raw_value::hll_hashval))
59+
AND round(curr.cardinality::numeric, 10) !=
60+
round(hll_cardinality(hll_add(prev.union_compressed_multiset,
61+
curr.raw_value::hll_hashval))::numeric,
62+
10)
6263
ORDER BY curr.recno;
63-
recno | cardinality | hll_cardinality
64-
-------+------------------+------------------
65-
4343 | 4250.71186178904 | 4250.71186178904
66-
4344 | 4250.71186178904 | 4250.71186178904
67-
4345 | 4250.71186178904 | 4250.71186178904
68-
4346 | 4250.71186178904 | 4250.71186178904
69-
4347 | 4250.71186178904 | 4250.71186178904
70-
4348 | 4250.71186178904 | 4250.71186178904
71-
4349 | 4250.71186178904 | 4250.71186178904
72-
4350 | 4250.71186178904 | 4250.71186178904
73-
4351 | 4250.71186178904 | 4250.71186178904
74-
4352 | 4250.71186178904 | 4250.71186178904
75-
4353 | 4250.71186178904 | 4250.71186178904
76-
4354 | 4250.71186178904 | 4250.71186178904
77-
4355 | 4250.71186178904 | 4250.71186178904
78-
4356 | 4250.71186178904 | 4250.71186178904
79-
4357 | 4250.71186178904 | 4250.71186178904
80-
(15 rows)
64+
recno | cardinality | hll_cardinality
65+
-------+-------------+-----------------
66+
(0 rows)
8167

8268
DROP TABLE test_ptwysrqk;

‎expected/cumulative_union_sparse_promotion.out

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ SELECT recno,
5151
union_cardinality,
5252
hll_cardinality(union_compressed_multiset)
5353
FROM test_bsnvqefe
54-
WHERE union_cardinality != hll_cardinality(union_compressed_multiset)
54+
WHERE round(union_cardinality::numeric, 10) !=
55+
round(hll_cardinality(union_compressed_multiset)::numeric, 10)
5556
ORDER BY recno;
5657
recno | union_cardinality | hll_cardinality
5758
-------+-------------------+-----------------
@@ -81,9 +82,10 @@ SELECT curr.recno,
8182
FROM test_bsnvqefe prev, test_bsnvqefe curr
8283
WHERE curr.recno > 1
8384
AND curr.recno = prev.recno + 1
84-
AND curr.union_cardinality !=
85-
hll_cardinality(hll_union(curr.compressed_multiset,
86-
prev.union_compressed_multiset))
85+
AND round(curr.union_cardinality::numeric, 10) !=
86+
round(hll_cardinality(hll_union(curr.compressed_multiset,
87+
prev.union_compressed_multiset))::numeric,
88+
10)
8789
ORDER BY curr.recno;
8890
recno | union_cardinality | hll_cardinality
8991
-------+-------------------+-----------------

‎sql/cumulative_add_cardinality_correction.sql

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,10 @@ SELECT curr.recno,
4444
FROM test_msgfjqhm prev, test_msgfjqhm curr
4545
WHERE curr.recno > 1
4646
AND curr.recno = prev.recno + 1
47-
AND curr.cardinality !=
48-
hll_cardinality(hll_add(prev.union_compressed_multiset,
49-
hll_hashval(curr.raw_value)))
47+
AND round(curr.cardinality::numeric, 10) !=
48+
round(hll_cardinality(hll_add(prev.union_compressed_multiset,
49+
hll_hashval(curr.raw_value)))::numeric,
50+
10)
5051
ORDER BY curr.recno;
5152

5253
DROP TABLE test_msgfjqhm;

‎sql/cumulative_add_comprehensive_promotion.sql

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,10 @@ SELECT curr.recno,
4444
FROM test_ptwysrqk prev, test_ptwysrqk curr
4545
WHERE curr.recno > 1
4646
AND curr.recno = prev.recno + 1
47-
AND curr.cardinality !=
48-
hll_cardinality(hll_add(prev.union_compressed_multiset,
49-
curr.raw_value::hll_hashval))
47+
AND round(curr.cardinality::numeric, 10) !=
48+
round(hll_cardinality(hll_add(prev.union_compressed_multiset,
49+
curr.raw_value::hll_hashval))::numeric,
50+
10)
5051
ORDER BY curr.recno;
5152

5253
DROP TABLE test_ptwysrqk;

‎sql/cumulative_union_sparse_promotion.sql

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ SELECT recno,
3939
union_cardinality,
4040
hll_cardinality(union_compressed_multiset)
4141
FROM test_bsnvqefe
42-
WHERE union_cardinality != hll_cardinality(union_compressed_multiset)
42+
WHERE round(union_cardinality::numeric, 10) !=
43+
round(hll_cardinality(union_compressed_multiset)::numeric, 10)
4344
ORDER BY recno;
4445

4546
-- Test union of incremental multiset.
@@ -63,9 +64,10 @@ SELECT curr.recno,
6364
FROM test_bsnvqefe prev, test_bsnvqefe curr
6465
WHERE curr.recno > 1
6566
AND curr.recno = prev.recno + 1
66-
AND curr.union_cardinality !=
67-
hll_cardinality(hll_union(curr.compressed_multiset,
68-
prev.union_compressed_multiset))
67+
AND round(curr.union_cardinality::numeric, 10) !=
68+
round(hll_cardinality(hll_union(curr.compressed_multiset,
69+
prev.union_compressed_multiset))::numeric,
70+
10)
6971
ORDER BY curr.recno;
7072

7173
-- Test aggregate accumulation

0 commit comments

Comments
 (0)