Skip to content

Commit 3b446ee

Browse files
leerholeerho
authored
Merge pull request #450 from apache/prep_for_items_sketch
Preparatory work for generic items.
2 parents ee84291 + e4271bf commit 3b446ee

14 files changed

Lines changed: 189 additions & 215 deletions

‎src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,13 @@
2222
import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR;
2323
import static org.apache.datasketches.kll.KllPreambleUtil.PREAMBLE_INTS_FULL;
2424
import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_UPDATABLE;
25-
import static org.apache.datasketches.kll.KllPreambleUtil.UPDATABLE_BIT_MASK;
2625
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryK;
2726
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryLevelZeroSortedFlag;
2827
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryM;
2928
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryMinK;
3029
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryN;
3130
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryNumLevels;
3231
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryFamilyID;
33-
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryFlags;
3432
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryK;
3533
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryLevelZeroSortedFlag;
3634
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryM;
@@ -59,13 +57,12 @@
5957
class KllDirectDoublesSketch extends KllDoublesSketch {
6058

6159
/**
62-
* The constructor with Memory that can be off-heap.
60+
* The constructor with WritableMemory that can be off-heap.
6361
* @param wmem the current WritableMemory
6462
* @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory
6563
* @param memVal the MemoryValadate object
6664
*/
67-
KllDirectDoublesSketch(final WritableMemory wmem, final MemoryRequestServer memReqSvr,
68-
final KllMemoryValidate memVal) {
65+
KllDirectDoublesSketch(final WritableMemory wmem, final MemoryRequestServer memReqSvr, final KllMemoryValidate memVal) {
6966
super(wmem, memReqSvr);
7067
levelsArr = memVal.levelsArr;
7168
}
@@ -83,7 +80,6 @@ static KllDirectDoublesSketch newDirectInstance(final int k, final int m, final
8380
setMemoryPreInts(dstMem, PREAMBLE_INTS_FULL);
8481
setMemorySerVer(dstMem, SERIAL_VERSION_UPDATABLE);
8582
setMemoryFamilyID(dstMem, Family.KLL.getID());
86-
setMemoryFlags(dstMem, UPDATABLE_BIT_MASK);
8783
setMemoryK(dstMem, k);
8884
setMemoryM(dstMem, m);
8985
setMemoryN(dstMem, 0);

‎src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,13 @@
2222
import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR;
2323
import static org.apache.datasketches.kll.KllPreambleUtil.PREAMBLE_INTS_FULL;
2424
import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_UPDATABLE;
25-
import static org.apache.datasketches.kll.KllPreambleUtil.UPDATABLE_BIT_MASK;
2625
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryK;
2726
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryLevelZeroSortedFlag;
2827
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryM;
2928
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryMinK;
3029
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryN;
3130
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryNumLevels;
3231
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryFamilyID;
33-
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryFlags;
3432
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryK;
3533
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryLevelZeroSortedFlag;
3634
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryM;
@@ -59,13 +57,12 @@
5957
class KllDirectFloatsSketch extends KllFloatsSketch {
6058

6159
/**
62-
* The constructor with Memory that can be off-heap.
60+
* The constructor with WritableMemory that can be off-heap.
6361
* @param wmem the current WritableMemory
6462
* @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory
6563
* @param memVal the MemoryValadate object
6664
*/
67-
KllDirectFloatsSketch(final WritableMemory wmem, final MemoryRequestServer memReqSvr,
68-
final KllMemoryValidate memVal) {
65+
KllDirectFloatsSketch(final WritableMemory wmem, final MemoryRequestServer memReqSvr, final KllMemoryValidate memVal) {
6966
super(wmem, memReqSvr);
7067
levelsArr = memVal.levelsArr;
7168
}
@@ -83,7 +80,6 @@ static KllDirectFloatsSketch newDirectInstance(final int k, final int m, final W
8380
setMemoryPreInts(dstMem, PREAMBLE_INTS_FULL);
8481
setMemorySerVer(dstMem, SERIAL_VERSION_UPDATABLE);
8582
setMemoryFamilyID(dstMem, Family.KLL.getID());
86-
setMemoryFlags(dstMem, UPDATABLE_BIT_MASK);
8783
setMemoryK(dstMem, k);
8884
setMemoryM(dstMem, m);
8985
setMemoryN(dstMem, 0);

‎src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,13 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllSketch oth
4949
final int myMinK = mySketch.getMinK();
5050

5151
//update this sketch with level0 items from the other sketch
52-
5352
if (otherDblSk.isCompactSingleItem()) {
5453
updateDouble(mySketch, otherDblSk.getDoubleSingleItem());
5554
otherDoubleItemsArr = new double[0];
5655
} else {
5756
otherDoubleItemsArr = otherDblSk.getDoubleItemsArray();
5857
for (int i = otherLevelsArr[0]; i < otherLevelsArr[1]; i++) {
59-
KllDoublesHelper.updateDouble(mySketch, otherDoubleItemsArr[i]);
58+
updateDouble(mySketch, otherDoubleItemsArr[i]);
6059
}
6160
}
6261
// after the level 0 update, we capture the state of levels and items arrays
@@ -68,7 +67,7 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllSketch oth
6867
int[] myNewLevelsArr = myCurLevelsArr;
6968
double[] myNewDoubleItemsArr = myCurDoubleItemsArr;
7069

71-
if (otherNumLevels > 1 && !otherDblSk.isCompactSingleItem()) { //now merge other levels if they exist
70+
if (otherNumLevels > 1 && !otherDblSk.isCompactSingleItem()) { //now merge higher levels if they exist
7271
final int tmpSpaceNeeded = mySketch.getNumRetained()
7372
+ KllHelper.getNumRetainedAboveLevelZero(otherNumLevels, otherLevelsArr);
7473
final double[] workbuf = new double[tmpSpaceNeeded];
@@ -115,7 +114,7 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllSketch oth
115114
}
116115

117116
//MEMORY SPACE MANAGEMENT
118-
if (mySketch.updatableMemFormat) {
117+
if (mySketch.serialVersionUpdatable) {
119118
mySketch.wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length);
120119
}
121120
}
@@ -358,7 +357,7 @@ private static void populateDoubleWorkArrays(
358357
worklevels[0] = 0;
359358

360359
// Note: the level zero data from "other" was already inserted into "self"
361-
final int selfPopZero = KllHelper.currentLevelSize(0, myCurNumLevels,myCurLevelsArr);
360+
final int selfPopZero = KllHelper.currentLevelSize(0, myCurNumLevels, myCurLevelsArr);
362361
System.arraycopy(myCurDoubleItemsArr, myCurLevelsArr[0], workbuf, worklevels[0], selfPopZero);
363362
worklevels[1] = worklevels[0] + selfPopZero;
364363

‎src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java‎

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@
2121

2222
import static java.lang.Math.max;
2323
import static java.lang.Math.min;
24-
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryUpdatableFormatFlag;
25-
import static org.apache.datasketches.kll.KllSketch.Error.MUST_NOT_BE_UPDATABLE_FORMAT;
24+
import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_UPDATABLE;
25+
import static org.apache.datasketches.kll.KllPreambleUtil.getMemorySerVer;
2626
import static org.apache.datasketches.kll.KllSketch.Error.TGT_IS_READ_ONLY;
2727
import static org.apache.datasketches.kll.KllSketch.Error.kllSketchThrow;
2828
import static org.apache.datasketches.kll.KllSketch.SketchType.DOUBLES_SKETCH;
@@ -61,60 +61,56 @@ public abstract class KllDoublesSketch extends KllSketch implements QuantilesDou
6161
*/
6262
public static KllDoublesSketch heapify(final Memory srcMem) {
6363
Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null");
64-
if (getMemoryUpdatableFormatFlag(srcMem)) { Error.kllSketchThrow(MUST_NOT_BE_UPDATABLE_FORMAT); }
6564
return KllHeapDoublesSketch.heapifyImpl(srcMem);
6665
}
6766

6867
/**
69-
* Create a new direct instance of this sketch with a given <em>k</em>.
70-
* @param k parameter that controls size of the sketch and accuracy of estimates.
68+
* Create a new direct instance of this sketch with the default <em>k</em>.
69+
* The default <em>k</em> = 200 results in a normalized rank error of about
70+
* 1.65%. Larger <em>k</em> will have smaller error but the sketch will be larger (and slower).
7171
* @param dstMem the given destination WritableMemory object for use by the sketch
7272
* @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory
7373
* @return a new direct instance of this sketch
7474
*/
7575
public static KllDoublesSketch newDirectInstance(
76-
final int k,
7776
final WritableMemory dstMem,
7877
final MemoryRequestServer memReqSvr) {
79-
Objects.requireNonNull(dstMem, "Parameter 'dstMem' must not be null");
80-
Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null");
81-
return KllDirectDoublesSketch.newDirectInstance(k, DEFAULT_M, dstMem, memReqSvr);
78+
return newDirectInstance(DEFAULT_K, dstMem, memReqSvr);
8279
}
83-
80+
8481
/**
85-
* Create a new direct instance of this sketch with the default <em>k</em>.
86-
* The default <em>k</em> = 200 results in a normalized rank error of about
87-
* 1.65%. Larger <em>k</em> will have smaller error but the sketch will be larger (and slower).
82+
* Create a new direct instance of this sketch with a given <em>k</em>.
83+
* @param k parameter that controls size of the sketch and accuracy of estimates.
8884
* @param dstMem the given destination WritableMemory object for use by the sketch
8985
* @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory
9086
* @return a new direct instance of this sketch
9187
*/
9288
public static KllDoublesSketch newDirectInstance(
89+
final int k,
9390
final WritableMemory dstMem,
9491
final MemoryRequestServer memReqSvr) {
9592
Objects.requireNonNull(dstMem, "Parameter 'dstMem' must not be null");
9693
Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null");
97-
return KllDirectDoublesSketch.newDirectInstance(DEFAULT_K, DEFAULT_M, dstMem, memReqSvr);
94+
return KllDirectDoublesSketch.newDirectInstance(k, DEFAULT_M, dstMem, memReqSvr);
9895
}
9996

10097
/**
10198
* Create a new heap instance of this sketch with the default <em>k = 200</em>.
10299
* The default <em>k</em> = 200 results in a normalized rank error of about
103100
* 1.65%. Larger K will have smaller error but the sketch will be larger (and slower).
104-
* This will have a rank error of about 1.65%.
105-
* @return new KllDoublesSketch on the heap.
101+
* @return new KllDoublesSketch on the Java heap.
106102
*/
107-
public static KllDoublesSketch newHeapInstance() {
108-
return new KllHeapDoublesSketch(DEFAULT_K, DEFAULT_M);
103+
public static KllDoublesSketch newHeapInstance() {
104+
return newHeapInstance(DEFAULT_K);
109105
}
110106

111107
/**
112108
* Create a new heap instance of this sketch with a given parameter <em>k</em>.
113-
* <em>k</em> can be between DEFAULT_M and 65535, inclusive.
109+
* <em>k</em> can be between 8, inclusive, and 65535, inclusive.
114110
* The default <em>k</em> = 200 results in a normalized rank error of about
115111
* 1.65%. Larger K will have smaller error but the sketch will be larger (and slower).
116112
* @param k parameter that controls size of the sketch and accuracy of estimates.
117-
* @return new KllDoublesSketch on the heap.
113+
* @return new KllDoublesSketch on the Java heap.
118114
*/
119115
public static KllDoublesSketch newHeapInstance(final int k) {
120116
return new KllHeapDoublesSketch(k, DEFAULT_M);
@@ -129,7 +125,7 @@ public static KllDoublesSketch newHeapInstance(final int k) {
129125
public static KllDoublesSketch wrap(final Memory srcMem) {
130126
Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null");
131127
final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, DOUBLES_SKETCH);
132-
if (memVal.updatableMemFormat) {
128+
if (getMemorySerVer(srcMem) == SERIAL_VERSION_UPDATABLE) {
133129
return new KllDirectDoublesSketch((WritableMemory) srcMem, null, memVal);
134130
} else {
135131
return new KllDirectCompactDoublesSketch(srcMem, memVal);
@@ -148,10 +144,8 @@ public static KllDoublesSketch writableWrap(
148144
final MemoryRequestServer memReqSvr) {
149145
Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null");
150146
final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, DOUBLES_SKETCH);
151-
if (memVal.updatableMemFormat) {
152-
if (!memVal.readOnly) {
153-
Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null");
154-
}
147+
if (getMemorySerVer(srcMem) == SERIAL_VERSION_UPDATABLE && !srcMem.isReadOnly()) {
148+
Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null");
155149
return new KllDirectDoublesSketch(srcMem, memReqSvr, memVal);
156150
} else {
157151
return new KllDirectCompactDoublesSketch(srcMem, memVal);

‎src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllSketch other
114114
}
115115

116116
//MEMORY SPACE MANAGEMENT
117-
if (mySketch.updatableMemFormat) {
117+
if (mySketch.serialVersionUpdatable) {
118118
mySketch.wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length);
119119
}
120120
}

‎src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java‎

Lines changed: 17 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@
2121

2222
import static java.lang.Math.max;
2323
import static java.lang.Math.min;
24-
import static org.apache.datasketches.kll.KllPreambleUtil.getMemoryUpdatableFormatFlag;
25-
import static org.apache.datasketches.kll.KllSketch.Error.MUST_NOT_BE_UPDATABLE_FORMAT;
24+
import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_UPDATABLE;
25+
import static org.apache.datasketches.kll.KllPreambleUtil.getMemorySerVer;
2626
import static org.apache.datasketches.kll.KllSketch.Error.TGT_IS_READ_ONLY;
2727
import static org.apache.datasketches.kll.KllSketch.Error.kllSketchThrow;
2828
import static org.apache.datasketches.kll.KllSketch.SketchType.FLOATS_SKETCH;
@@ -61,60 +61,56 @@ public abstract class KllFloatsSketch extends KllSketch implements QuantilesFloa
6161
*/
6262
public static KllFloatsSketch heapify(final Memory srcMem) {
6363
Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null");
64-
if (getMemoryUpdatableFormatFlag(srcMem)) { Error.kllSketchThrow(MUST_NOT_BE_UPDATABLE_FORMAT); }
6564
return KllHeapFloatsSketch.heapifyImpl(srcMem);
6665
}
6766

6867
/**
69-
* Create a new direct instance of this sketch with a given <em>k</em>.
70-
* @param k parameter that controls size of the sketch and accuracy of estimates.
68+
* Create a new direct instance of this sketch with the default <em>k</em>.
69+
* The default <em>k</em> = 200 results in a normalized rank error of about
70+
* 1.65%. Larger <em>k</em> will have smaller error but the sketch will be larger (and slower).
7171
* @param dstMem the given destination WritableMemory object for use by the sketch
7272
* @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory
7373
* @return a new direct instance of this sketch
7474
*/
7575
public static KllFloatsSketch newDirectInstance(
76-
final int k,
7776
final WritableMemory dstMem,
7877
final MemoryRequestServer memReqSvr) {
79-
Objects.requireNonNull(dstMem, "Parameter 'dstMem' must not be null");
80-
Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null");
81-
return KllDirectFloatsSketch.newDirectInstance(k, DEFAULT_M, dstMem, memReqSvr);
78+
return newDirectInstance(DEFAULT_K, dstMem, memReqSvr);
8279
}
83-
80+
8481
/**
85-
* Create a new direct instance of this sketch with the default <em>k</em>.
86-
* The default <em>k</em> = 200 results in a normalized rank error of about
87-
* 1.65%. Larger <em>k</em> will have smaller error but the sketch will be larger (and slower).
82+
* Create a new direct instance of this sketch with a given <em>k</em>.
83+
* @param k parameter that controls size of the sketch and accuracy of estimates.
8884
* @param dstMem the given destination WritableMemory object for use by the sketch
8985
* @param memReqSvr the given MemoryRequestServer to request a larger WritableMemory
9086
* @return a new direct instance of this sketch
9187
*/
9288
public static KllFloatsSketch newDirectInstance(
89+
final int k,
9390
final WritableMemory dstMem,
9491
final MemoryRequestServer memReqSvr) {
9592
Objects.requireNonNull(dstMem, "Parameter 'dstMem' must not be null");
9693
Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null");
97-
return KllDirectFloatsSketch.newDirectInstance(DEFAULT_K, DEFAULT_M, dstMem, memReqSvr);
94+
return KllDirectFloatsSketch.newDirectInstance(k, DEFAULT_M, dstMem, memReqSvr);
9895
}
9996

10097
/**
10198
* Create a new heap instance of this sketch with the default <em>k = 200</em>.
10299
* The default <em>k</em> = 200 results in a normalized rank error of about
103100
* 1.65%. Larger K will have smaller error but the sketch will be larger (and slower).
104-
* This will have a rank error of about 1.65%.
105-
* @return new KllFloatsSketch on the heap.
101+
* @return new KllFloatsSketch on the Java heap.
106102
*/
107103
public static KllFloatsSketch newHeapInstance() {
108-
return new KllHeapFloatsSketch(DEFAULT_K, DEFAULT_M);
104+
return newHeapInstance(DEFAULT_K);
109105
}
110106

111107
/**
112108
* Create a new heap instance of this sketch with a given parameter <em>k</em>.
113-
* <em>k</em> can be between DEFAULT_M and 65535, inclusive.
109+
* <em>k</em> can be between 8, inclusive, and 65535, inclusive.
114110
* The default <em>k</em> = 200 results in a normalized rank error of about
115111
* 1.65%. Larger K will have smaller error but the sketch will be larger (and slower).
116112
* @param k parameter that controls size of the sketch and accuracy of estimates.
117-
* @return new KllFloatsSketch on the heap.
113+
* @return new KllFloatsSketch on the Java heap.
118114
*/
119115
public static KllFloatsSketch newHeapInstance(final int k) {
120116
return new KllHeapFloatsSketch(k, DEFAULT_M);
@@ -129,7 +125,7 @@ public static KllFloatsSketch newHeapInstance(final int k) {
129125
public static KllFloatsSketch wrap(final Memory srcMem) {
130126
Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null");
131127
final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, FLOATS_SKETCH);
132-
if (memVal.updatableMemFormat) {
128+
if (getMemorySerVer(srcMem) == SERIAL_VERSION_UPDATABLE) {
133129
return new KllDirectFloatsSketch((WritableMemory) srcMem, null, memVal);
134130
} else {
135131
return new KllDirectCompactFloatsSketch(srcMem, memVal);
@@ -148,10 +144,8 @@ public static KllFloatsSketch writableWrap(
148144
final MemoryRequestServer memReqSvr) {
149145
Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null");
150146
final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, FLOATS_SKETCH);
151-
if (memVal.updatableMemFormat) {
152-
if (!memVal.readOnly) {
147+
if (getMemorySerVer(srcMem) == SERIAL_VERSION_UPDATABLE && !srcMem.isReadOnly()) {
153148
Objects.requireNonNull(memReqSvr, "Parameter 'memReqSvr' must not be null");
154-
}
155149
return new KllDirectFloatsSketch(srcMem, memReqSvr, memVal);
156150
} else {
157151
return new KllDirectCompactFloatsSketch(srcMem, memVal);

0 commit comments

Comments
 (0)