Skip to content

Commit ce91e6f

Browse files
leerholeerho
authored
Merge pull request #198 from DataSketches/MoveHeapifyAndWrap
Move Heapify and wrap to base DoublesUnion class,
2 parents df2237b + 7a5bce4 commit ce91e6f

10 files changed

Lines changed: 127 additions & 30 deletions

‎src/main/java/com/yahoo/sketches/quantiles/DoublesUnion.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,51 @@ public static DoublesUnionBuilder builder() {
2323
return new DoublesUnionBuilder();
2424
}
2525

26+
/**
27+
* Returns a Heap Union object that has been initialized with the data from the given sketch.
28+
* @param sketch A DoublesSketch to be used as a source of data only and will not be modified.
29+
* @return a DoublesUnion object
30+
*/
31+
public static DoublesUnion heapify(final DoublesSketch sketch) {
32+
return DoublesUnionImpl.heapifyInstance(sketch);
33+
}
34+
35+
/**
36+
* Returns a Heap Union object that has been initialized with the data from the given memory
37+
* image of a sketch.
38+
*
39+
* @param srcMem A memory image of a DoublesSketch to be used as a source of data,
40+
* but will not be modified.
41+
* @return a Union object
42+
*/
43+
public static DoublesUnion heapify(final Memory srcMem) {
44+
return DoublesUnionImpl.heapifyInstance(srcMem);
45+
}
46+
47+
/**
48+
* Returns a read-only Union object that wraps off-heap data of the given memory image of
49+
* a sketch. The data structures of the Union remain off-heap.
50+
*
51+
* @param mem A memory region to be used as the data structure for the sketch
52+
* and will be modified.
53+
* @return a Union object
54+
*/
55+
public static DoublesUnion wrap(final Memory mem) {
56+
return DoublesUnionImplR.wrapInstance(mem);
57+
}
58+
59+
/**
60+
* Returns an updatable Union object that wraps off-heap data of the given memory image of
61+
* a sketch. The data structures of the Union remain off-heap.
62+
*
63+
* @param mem A memory region to be used as the data structure for the sketch
64+
* and will be modified.
65+
* @return a Union object
66+
*/
67+
public static DoublesUnion wrap(final WritableMemory mem) {
68+
return DoublesUnionImpl.wrapInstance(mem);
69+
}
70+
2671
/**
2772
* Returns true if this union is empty
2873
* @return true if this union is empty

‎src/main/java/com/yahoo/sketches/quantiles/DoublesUnionBuilder.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,9 @@ public DoublesUnion build(final WritableMemory dstMem) {
7070
* Returns a Heap Union object that has been initialized with the data from the given sketch.
7171
* @param sketch A DoublesSketch to be used as a source of data only and will not be modified.
7272
* @return a DoublesUnion object
73+
* @deprecated moved to DoublesUnion
7374
*/
75+
@Deprecated
7476
public static DoublesUnion heapify(final DoublesSketch sketch) {
7577
return DoublesUnionImpl.heapifyInstance(sketch);
7678
}
@@ -82,7 +84,9 @@ public static DoublesUnion heapify(final DoublesSketch sketch) {
8284
* @param srcMem A memory image of a DoublesSketch to be used as a source of data,
8385
* but will not be modified.
8486
* @return a Union object
87+
* @deprecated moved to DoublesUnion
8588
*/
89+
@Deprecated
8690
public static DoublesUnion heapify(final Memory srcMem) {
8791
return DoublesUnionImpl.heapifyInstance(srcMem);
8892
}
@@ -94,7 +98,9 @@ public static DoublesUnion heapify(final Memory srcMem) {
9498
* @param mem A memory region to be used as the data structure for the sketch
9599
* and will be modified.
96100
* @return a Union object
101+
* @deprecated moved to DoublesUnion
97102
*/
103+
@Deprecated
98104
public static DoublesUnion wrap(final Memory mem) {
99105
return DoublesUnionImplR.wrapInstance(mem);
100106
}
@@ -106,7 +112,9 @@ public static DoublesUnion wrap(final Memory mem) {
106112
* @param mem A memory region to be used as the data structure for the sketch
107113
* and will be modified.
108114
* @return a Union object
115+
* @deprecated moved to DoublesUnion
109116
*/
117+
@Deprecated
110118
public static DoublesUnion wrap(final WritableMemory mem) {
111119
return DoublesUnionImpl.wrapInstance(mem);
112120
}

‎src/test/java/com/yahoo/sketches/kll/DeprecatedAndMiscTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
*/
1515
public class DeprecatedAndMiscTest {
1616

17+
@SuppressWarnings("deprecation")
1718
@Test
1819
public void checkDeprecatedRankError() {
1920
KllFloatsSketch sketch = new KllFloatsSketch();

‎src/test/java/com/yahoo/sketches/quantiles/DeprecatedAndMiscTest.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright 2018, Yahoo! Inc. Licensed under the terms of the
3+
* Apache License 2.0. See LICENSE file at the project root for terms.
4+
*/
5+
6+
package com.yahoo.sketches.quantiles;
7+
8+
import static com.yahoo.sketches.quantiles.HeapUpdateDoublesSketchTest.buildAndLoadQS;
9+
10+
import java.util.Comparator;
11+
12+
import org.testng.annotations.Test;
13+
14+
import com.yahoo.memory.Memory;
15+
import com.yahoo.memory.WritableMemory;
16+
17+
/**
18+
* @author Lee Rhodes
19+
*/
20+
public class DeprecatedAndMiscTest {
21+
22+
@SuppressWarnings({ "deprecation", "unused" })
23+
@Test
24+
public void checkDeprecatedRankError() {
25+
DoublesSketch ds = buildAndLoadQS(64, 64);
26+
double err = ds.getNormalizedRankError();
27+
err = DoublesSketch.getNormalizedRankError(64);
28+
DoublesUnion du1 = DoublesUnionBuilder.heapify(ds);
29+
30+
Memory mem = Memory.wrap(ds.toByteArray());
31+
DoublesUnion du2 = DoublesUnionBuilder.heapify(mem);
32+
33+
DoublesUnion du3 = DoublesUnionBuilder.wrap(mem);
34+
35+
WritableMemory wmem = WritableMemory.wrap(ds.toByteArray());
36+
DoublesUnion du4 = DoublesUnionBuilder.wrap(wmem);
37+
38+
ItemsSketch<String> is = ItemsSketch.getInstance(64, Comparator.naturalOrder());
39+
err = is.getNormalizedRankError();
40+
err = ItemsSketch.getNormalizedRankError(64);
41+
}
42+
43+
}

‎src/test/java/com/yahoo/sketches/quantiles/DirectUpdateDoublesSketchTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,11 @@ public void checkBigMinMax () {
5757
assertTrue(resultsA[0] == 1.0);
5858
assertTrue(resultsA[1] == 999.0);
5959

60-
DoublesUnion union1 = DoublesUnionBuilder.heapify(qs1);
60+
DoublesUnion union1 = DoublesUnion.heapify(qs1);
6161
union1.update(qs2);
6262
DoublesSketch result1 = union1.getResult();
6363

64-
DoublesUnion union2 = DoublesUnionBuilder.heapify(qs2);
64+
DoublesUnion union2 = DoublesUnion.heapify(qs2);
6565
union2.update(qs3);
6666
DoublesSketch result2 = union2.getResult();
6767

@@ -106,11 +106,11 @@ public void checkSmallMinMax () {
106106
assert (resultsA[1] == 5.0);
107107
assert (resultsA[2] == 8.0);
108108

109-
DoublesUnion union1 = DoublesUnionBuilder.heapify(qs1);
109+
DoublesUnion union1 = DoublesUnion.heapify(qs1);
110110
union1.update(qs2);
111111
DoublesSketch result1 = union1.getResult();
112112

113-
DoublesUnion union2 = DoublesUnionBuilder.heapify(qs2);
113+
DoublesUnion union2 = DoublesUnion.heapify(qs2);
114114
union2.update(qs3);
115115
DoublesSketch result2 = union2.getResult();
116116

‎src/test/java/com/yahoo/sketches/quantiles/DoublesUnionBuilderTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@ public void checkBuilds() {
2929
bldr.setMaxK(128);
3030
DoublesUnion union = bldr.build(); //virgin union
3131

32-
union = DoublesUnionBuilder.heapify(srcMem);
32+
union = DoublesUnion.heapify(srcMem);
3333
DoublesSketch qs2 = union.getResult();
3434
assertEquals(qs1.getCompactStorageBytes(), qs2.getCompactStorageBytes());
3535

36-
union = DoublesUnionBuilder.heapify(qs2);
36+
union = DoublesUnion.heapify(qs2);
3737
DoublesSketch qs3 = union.getResult();
3838
assertEquals(qs2.getCompactStorageBytes(), qs3.getCompactStorageBytes());
3939
assertFalse(qs2 == qs3);
@@ -56,12 +56,12 @@ public void checkDeprecated1() {
5656
bldr.setMaxK(128);
5757
DoublesUnion union = bldr.build(); //virgin union
5858

59-
union = DoublesUnionBuilder.heapify(srcMem); //heapify
59+
union = DoublesUnion.heapify(srcMem); //heapify
6060
DoublesSketch qs2 = union.getResult();
6161
assertEquals(qs1.getCompactStorageBytes(), qs2.getCompactStorageBytes());
6262
assertEquals(qs1.getUpdatableStorageBytes(), qs2.getUpdatableStorageBytes());
6363

64-
union = DoublesUnionBuilder.heapify(qs2); //heapify again
64+
union = DoublesUnion.heapify(qs2); //heapify again
6565
DoublesSketch qs3 = union.getResult();
6666
assertEquals(qs2.getCompactStorageBytes(), qs3.getCompactStorageBytes());
6767
assertEquals(qs2.getUpdatableStorageBytes(), qs3.getUpdatableStorageBytes());

‎src/test/java/com/yahoo/sketches/quantiles/DoublesUnionImplTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -483,15 +483,15 @@ public void checkUnionUpdateLogic2Direct() {
483483
@Test
484484
public void checkResultAndReset() {
485485
final DoublesSketch qs1 = buildAndLoadQS(256, 0);
486-
final DoublesUnion union = DoublesUnionBuilder.heapify(qs1);
486+
final DoublesUnion union = DoublesUnion.heapify(qs1);
487487
final DoublesSketch qs2 = union.getResultAndReset();
488488
assertEquals(qs2.getK(), 256);
489489
}
490490

491491
@Test
492492
public void checkResultAndResetDirect() {
493493
final DoublesSketch qs1 = buildAndLoadDQS(256, 0);
494-
final DoublesUnion union = DoublesUnionBuilder.heapify(qs1);
494+
final DoublesUnion union = DoublesUnion.heapify(qs1);
495495
final DoublesSketch qs2 = union.getResultAndReset();
496496
assertEquals(qs2.getK(), 256);
497497
}
@@ -665,7 +665,7 @@ public void checkWrapInstance() {
665665

666666
final byte[] byteArr = sketch.toByteArray(false);
667667
final WritableMemory mem = WritableMemory.wrap(byteArr);
668-
final DoublesUnion union = DoublesUnionBuilder.wrap(mem);
668+
final DoublesUnion union = DoublesUnion.wrap(mem);
669669
Assert.assertFalse(union.isEmpty());
670670
assertTrue(union.isDirect());
671671
final DoublesSketch sketch2 = union.getResult();
@@ -699,7 +699,7 @@ public void emptyUnionSerDeIssue195() {
699699
DoublesUnion union = DoublesUnion.builder().build();
700700
byte[] byteArr = union.toByteArray();
701701
Memory mem = Memory.wrap(byteArr);
702-
DoublesUnion union2 = DoublesUnionBuilder.heapify(mem);
702+
DoublesUnion union2 = DoublesUnion.heapify(mem);
703703
Assert.assertEquals(mem.getCapacity(), 8L);
704704
Assert.assertTrue(union2.isEmpty());
705705
}

‎src/test/java/com/yahoo/sketches/quantiles/HeapUpdateDoublesSketchTest.java‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ public void checkEndToEnd() {
5353
}
5454
}
5555
assertEquals(qs.getN() + qs2.getN(), n);
56-
DoublesUnion union = DoublesUnionBuilder.heapify(qs);
56+
DoublesUnion union = DoublesUnion.heapify(qs);
5757
union.update(qs2);
5858
DoublesSketch result = union.getResult();
5959

@@ -164,11 +164,11 @@ public void checkBigMinMax () {
164164
assertTrue(resultsA[0] == 1.0);
165165
assertTrue(resultsA[1] == 999.0);
166166

167-
DoublesUnion union1 = DoublesUnionBuilder.heapify(qs1);
167+
DoublesUnion union1 = DoublesUnion.heapify(qs1);
168168
union1.update(qs2);
169169
DoublesSketch result1 = union1.getResult();
170170

171-
DoublesUnion union2 = DoublesUnionBuilder.heapify(qs2);
171+
DoublesUnion union2 = DoublesUnion.heapify(qs2);
172172
union2.update(qs3);
173173
DoublesSketch result2 = union2.getResult();
174174

@@ -213,11 +213,11 @@ public void checkSmallMinMax () {
213213
assert (resultsA[1] == 5.0);
214214
assert (resultsA[2] == 8.0);
215215

216-
DoublesUnion union1 = DoublesUnionBuilder.heapify(qs1);
216+
DoublesUnion union1 = DoublesUnion.heapify(qs1);
217217
union1.update(qs2);
218218
DoublesSketch result1 = union1.getResult();
219219

220-
DoublesUnion union2 = DoublesUnionBuilder.heapify(qs2);
220+
DoublesUnion union2 = DoublesUnion.heapify(qs2);
221221
union2.update(qs3);
222222
DoublesSketch result2 = union2.getResult();
223223

@@ -406,7 +406,7 @@ public void checkMerge() {
406406
int n = 1000000;
407407
DoublesSketch qs1 = buildAndLoadQS(k,n,0);
408408
DoublesSketch qs2 = buildAndLoadQS(k,0,0); //empty
409-
DoublesUnion union = DoublesUnionBuilder.heapify(qs2);
409+
DoublesUnion union = DoublesUnion.heapify(qs2);
410410
union.update(qs1);
411411
DoublesSketch result = union.getResult();
412412
double med1 = qs1.getQuantile(0.5);
@@ -420,7 +420,7 @@ public void checkReverseMerge() {
420420
int k = PreambleUtil.DEFAULT_K;
421421
DoublesSketch qs1 = buildAndLoadQS(k, 1000, 0);
422422
DoublesSketch qs2 = buildAndLoadQS(2*k,1000, 1000);
423-
DoublesUnion union = DoublesUnionBuilder.heapify(qs2);
423+
DoublesUnion union = DoublesUnion.heapify(qs2);
424424
union.update(qs1); //attempt merge into larger k
425425
DoublesSketch result = union.getResult();
426426
assertEquals(result.getK(), k);
@@ -431,7 +431,7 @@ public void checkInternalBuildHistogram() {
431431
int k = PreambleUtil.DEFAULT_K;
432432
int n = 1000000;
433433
DoublesSketch qs = buildAndLoadQS(k,n,0);
434-
double eps = qs.getNormalizedRankError();
434+
double eps = qs.getNormalizedRankError(true);
435435
//println("EPS:"+eps);
436436
double[] spts = {100000, 500000, 900000};
437437
double[] fracArr = qs.getPMF(spts);
@@ -744,7 +744,7 @@ public void quantilesCheckViaMemory() {
744744
}
745745

746746
static String getRanksTable(DoublesSketch qs, double[] ranks) {
747-
double rankError = qs.getNormalizedRankError();
747+
double rankError = qs.getNormalizedRankError(false);
748748
double[] values = qs.getQuantiles(ranks);
749749
double maxV = qs.getMaxValue();
750750
double minV = qs.getMinValue();
@@ -788,7 +788,7 @@ static String getRanksTable(DoublesSketch qs, double[] ranks) {
788788
public void checkKisTwo() {
789789
int k = 2;
790790
UpdateDoublesSketch qs1 = DoublesSketch.builder().setK(k).build();
791-
double err = qs1.getNormalizedRankError();
791+
double err = qs1.getNormalizedRankError(false);
792792
assertTrue(err < 1.0);
793793
byte[] arr = qs1.toByteArray(true); //8
794794
assertEquals(arr.length, DoublesSketch.getCompactStorageBytes(k, 0));
@@ -801,7 +801,7 @@ public void checkKisTwo() {
801801
public void checkKisTwoDeprecated() {
802802
int k = 2;
803803
UpdateDoublesSketch qs1 = DoublesSketch.builder().setK(k).build();
804-
double err = qs1.getNormalizedRankError();
804+
double err = qs1.getNormalizedRankError(false);
805805
assertTrue(err < 1.0);
806806
byte[] arr = qs1.toByteArray(true); //8
807807
assertEquals(arr.length, DoublesSketch.getCompactStorageBytes(k, 0));
@@ -854,9 +854,9 @@ public void testIt() {
854854

855855
int k = 1024;
856856
DoublesSketch qsk = new DoublesSketchBuilder().setK(k).build();
857-
DoublesUnion u1 = DoublesUnionBuilder.heapify(qsk);
857+
DoublesUnion u1 = DoublesUnion.heapify(qsk);
858858
u1.getResult().putMemory(mem);
859-
DoublesUnion u2 = DoublesUnionBuilder.heapify(mem);
859+
DoublesUnion u2 = DoublesUnion.heapify(mem);
860860
DoublesSketch qsk2 = u2.getResult();
861861
assertTrue(qsk2.isEmpty());
862862
}

‎src/test/java/com/yahoo/sketches/quantiles/ItemsSketchTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ public void estimation() {
130130
Assert.assertEquals(quantiles[1], Integer.valueOf(500), 17); // median
131131
Assert.assertEquals(quantiles[2], Integer.valueOf(1000)); // max value
132132

133-
double normErr = sketch.getNormalizedRankError();
133+
double normErr = sketch.getNormalizedRankError(true);
134134
Assert.assertEquals(normErr, .0172, .001);
135135
println(""+normErr);
136136

‎src/test/java/com/yahoo/sketches/quantiles/ReadOnlyMemoryTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ public void heapifyUnionFromSparse() {
180180
s1.update(1);
181181
s1.update(2);
182182
Memory mem = Memory.wrap(s1.toByteArray(false));
183-
DoublesUnion u = DoublesUnionBuilder.heapify(mem);
183+
DoublesUnion u = DoublesUnion.heapify(mem);
184184
u.update(3);
185185
DoublesSketch s2 = u.getResult();
186186
Assert.assertEquals(s2.getMinValue(), 1.0);
@@ -193,7 +193,7 @@ public void heapifyUnionFromCompact() {
193193
s1.update(1);
194194
s1.update(2);
195195
Memory mem = Memory.wrap(s1.toByteArray(true));
196-
DoublesUnion u = DoublesUnionBuilder.heapify(mem);
196+
DoublesUnion u = DoublesUnion.heapify(mem);
197197
u.update(3);
198198
DoublesSketch s2 = u.getResult();
199199
Assert.assertEquals(s2.getMinValue(), 1.0);
@@ -206,7 +206,7 @@ public void wrapUnionFromSparse() {
206206
s1.update(1);
207207
s1.update(2);
208208
Memory mem = Memory.wrap(s1.toByteArray(false));
209-
DoublesUnion u = DoublesUnionBuilder.wrap(mem);
209+
DoublesUnion u = DoublesUnion.wrap(mem);
210210
DoublesSketch s2 = u.getResult();
211211
Assert.assertEquals(s2.getMinValue(), 1.0);
212212
Assert.assertEquals(s2.getMaxValue(), 2.0);
@@ -254,7 +254,7 @@ public void wrapUnionFromCompact() {
254254
s1.update(1);
255255
s1.update(2);
256256
Memory mem = Memory.wrap(s1.toByteArray(true));
257-
DoublesUnionBuilder.wrap(mem);
257+
DoublesUnion.wrap(mem);
258258
fail();
259259
}
260260

0 commit comments

Comments
 (0)