Skip to content

Commit 5c3746e

Browse files
authored
Merge pull request #448 from apache/revert_items_changes_prepare4.1.0
Revert items changes prepare4.1.0
2 parents a2599dc + 20e884c commit 5c3746e

31 files changed

Lines changed: 603 additions & 645 deletions

src/main/java/org/apache/datasketches/hll/DirectCouponList.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ HllSketchImpl couponUpdate(final int coupon) {
122122
insertInt(wmem, LIST_INT_ARR_START + (i << 2), coupon);
123123
int couponCount = extractListCount(mem);
124124
insertListCount(wmem, ++couponCount);
125-
insertEmptyFlag(wmem, false); //TODO only first time
125+
insertEmptyFlag(wmem, false); //only first time
126126
if (couponCount >= len) { //array full
127127
if (lgConfigK < 8) {
128128
return promoteListOrSetToHll(this);//oooFlag = false

src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
package org.apache.datasketches.kll;
2121

2222
import static org.apache.datasketches.kll.KllPreambleUtil.DATA_START_ADR;
23-
import static org.apache.datasketches.kll.KllPreambleUtil.DOUBLES_SKETCH_BIT_MASK;
2423
import static org.apache.datasketches.kll.KllPreambleUtil.PREAMBLE_INTS_FULL;
2524
import static org.apache.datasketches.kll.KllPreambleUtil.SERIAL_VERSION_UPDATABLE;
2625
import static org.apache.datasketches.kll.KllPreambleUtil.UPDATABLE_BIT_MASK;
@@ -40,10 +39,10 @@
4039
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryNumLevels;
4140
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryPreInts;
4241
import static org.apache.datasketches.kll.KllPreambleUtil.setMemorySerVer;
43-
import static org.apache.datasketches.kll.KllSketch.Error.MUST_NOT_CALL;
4442
import static org.apache.datasketches.kll.KllSketch.Error.NOT_SINGLE_ITEM;
4543
import static org.apache.datasketches.kll.KllSketch.Error.TGT_IS_READ_ONLY;
4644
import static org.apache.datasketches.kll.KllSketch.Error.kllSketchThrow;
45+
import static org.apache.datasketches.kll.KllSketch.SketchType.DOUBLES_SKETCH;
4746

4847
import org.apache.datasketches.common.Family;
4948
import org.apache.datasketches.memory.MemoryRequestServer;
@@ -84,7 +83,7 @@ static KllDirectDoublesSketch newDirectInstance(final int k, final int m, final
8483
setMemoryPreInts(dstMem, PREAMBLE_INTS_FULL);
8584
setMemorySerVer(dstMem, SERIAL_VERSION_UPDATABLE);
8685
setMemoryFamilyID(dstMem, Family.KLL.getID());
87-
setMemoryFlags(dstMem, DOUBLES_SKETCH_BIT_MASK | UPDATABLE_BIT_MASK);
86+
setMemoryFlags(dstMem, UPDATABLE_BIT_MASK);
8887
setMemoryK(dstMem, k);
8988
setMemoryM(dstMem, m);
9089
setMemoryN(dstMem, 0);
@@ -96,7 +95,7 @@ static KllDirectDoublesSketch newDirectInstance(final int k, final int m, final
9695
dstMem.putDoubleArray(offset, new double[] {Double.NaN, Double.NaN}, 0, 2);
9796
offset += 2 * Double.BYTES;
9897
dstMem.putDoubleArray(offset, new double[k], 0, k);
99-
final KllMemoryValidate memVal = new KllMemoryValidate(dstMem);
98+
final KllMemoryValidate memVal = new KllMemoryValidate(dstMem, DOUBLES_SKETCH);
10099
return new KllDirectDoublesSketch(dstMem, memReqSvr, memVal);
101100
}
102101

@@ -128,9 +127,6 @@ public long getN() {
128127
return wmem.getDouble(offset);
129128
}
130129

131-
@Override
132-
float getFloatSingleItem() { kllSketchThrow(MUST_NOT_CALL); return Float.NaN; }
133-
134130
@Override
135131
int getM() {
136132
return getMemoryM(wmem);
@@ -183,7 +179,7 @@ void setDoubleItemsArray(final double[] doubleItems) {
183179
void setDoubleItemsArrayAt(final int index, final double item) {
184180
if (readOnly) { kllSketchThrow(TGT_IS_READ_ONLY); }
185181
final int offset =
186-
DATA_START_ADR + getLevelsArray().length * Integer.BYTES + 2 * Double.BYTES + index * Double.BYTES;
182+
DATA_START_ADR + getLevelsArray().length * Integer.BYTES + (index + 2) * Double.BYTES;
187183
wmem.putDouble(offset, item);
188184
}
189185

src/main/java/org/apache/datasketches/kll/KllDirectFloatsSketch.java

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,16 +39,15 @@
3939
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryNumLevels;
4040
import static org.apache.datasketches.kll.KllPreambleUtil.setMemoryPreInts;
4141
import static org.apache.datasketches.kll.KllPreambleUtil.setMemorySerVer;
42-
import static org.apache.datasketches.kll.KllSketch.Error.MUST_NOT_CALL;
4342
import static org.apache.datasketches.kll.KllSketch.Error.NOT_SINGLE_ITEM;
4443
import static org.apache.datasketches.kll.KllSketch.Error.TGT_IS_READ_ONLY;
4544
import static org.apache.datasketches.kll.KllSketch.Error.kllSketchThrow;
45+
import static org.apache.datasketches.kll.KllSketch.SketchType.FLOATS_SKETCH;
4646

4747
import org.apache.datasketches.common.Family;
4848
import org.apache.datasketches.memory.MemoryRequestServer;
4949
import org.apache.datasketches.memory.WritableMemory;
5050

51-
//intentional extra line
5251
/**
5352
* This class implements an off-heap floats KllSketch via a WritableMemory instance of the sketch.
5453
*
@@ -96,7 +95,7 @@ static KllDirectFloatsSketch newDirectInstance(final int k, final int m, final W
9695
dstMem.putFloatArray(offset, new float[] {Float.NaN, Float.NaN}, 0, 2);
9796
offset += 2 * Float.BYTES;
9897
dstMem.putFloatArray(offset, new float[k], 0, k);
99-
final KllMemoryValidate memVal = new KllMemoryValidate(dstMem);
98+
final KllMemoryValidate memVal = new KllMemoryValidate(dstMem, FLOATS_SKETCH);
10099
return new KllDirectFloatsSketch(dstMem, memReqSvr, memVal);
101100
}
102101

@@ -110,9 +109,6 @@ public long getN() {
110109
return getMemoryN(wmem);
111110
}
112111

113-
@Override
114-
double getDoubleSingleItem() { kllSketchThrow(MUST_NOT_CALL); return Double.NaN; }
115-
116112
@Override //returns entire array including empty space at bottom
117113
float[] getFloatItemsArray() {
118114
final int capacityItems = levelsArr[getNumLevels()];

src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java

Lines changed: 51 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -35,39 +35,41 @@
3535
final class KllDoublesHelper {
3636

3737
//Called from KllSketch
38-
static void mergeDoubleImpl(final KllDoublesSketch sketch, final KllSketch other) {
39-
if (other.isEmpty()) { return; }
40-
sketch.nullSortedView();
41-
final long finalN = sketch.getN() + other.getN();
42-
final int otherNumLevels = other.getNumLevels();
43-
final int[] otherLevelsArr = other.getLevelsArray();
38+
static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllSketch other) {
39+
final KllDoublesSketch otherDblSk = (KllDoublesSketch) other;
40+
if (otherDblSk.isEmpty()) { return; }
41+
mySketch.nullSortedView();
42+
final long finalN = mySketch.getN() + otherDblSk.getN();
43+
final int otherNumLevels = otherDblSk.getNumLevels();
44+
final int[] otherLevelsArr = otherDblSk.getLevelsArray();
4445
final double[] otherDoubleItemsArr;
4546
//capture my min & max, minK
46-
final double myMin = sketch.isEmpty() ? Double.NaN : sketch.getMinDoubleItem();
47-
final double myMax = sketch.isEmpty() ? Double.NaN : sketch.getMaxDoubleItem();
48-
final int myMinK = sketch.getMinK();
47+
final double myMin = mySketch.isEmpty() ? Double.NaN : mySketch.getMinDoubleItem();
48+
final double myMax = mySketch.isEmpty() ? Double.NaN : mySketch.getMaxDoubleItem();
49+
final int myMinK = mySketch.getMinK();
4950

5051
//update this sketch with level0 items from the other sketch
51-
if (other.isCompactSingleItem()) {
52-
updateDouble(sketch, other.getDoubleSingleItem());
52+
53+
if (otherDblSk.isCompactSingleItem()) {
54+
updateDouble(mySketch, otherDblSk.getDoubleSingleItem());
5355
otherDoubleItemsArr = new double[0];
5456
} else {
55-
otherDoubleItemsArr = other.getDoubleItemsArray();
57+
otherDoubleItemsArr = otherDblSk.getDoubleItemsArray();
5658
for (int i = otherLevelsArr[0]; i < otherLevelsArr[1]; i++) {
57-
KllDoublesHelper.updateDouble(sketch, otherDoubleItemsArr[i]);
59+
KllDoublesHelper.updateDouble(mySketch, otherDoubleItemsArr[i]);
5860
}
5961
}
6062
// after the level 0 update, we capture the state of levels and items arrays
61-
final int myCurNumLevels = sketch.getNumLevels();
62-
final int[] myCurLevelsArr = sketch.getLevelsArray();
63-
final double[] myCurDoubleItemsArr = sketch.getDoubleItemsArray();
63+
final int myCurNumLevels = mySketch.getNumLevels();
64+
final int[] myCurLevelsArr = mySketch.getLevelsArray();
65+
final double[] myCurDoubleItemsArr = mySketch.getDoubleItemsArray();
6466

6567
int myNewNumLevels = myCurNumLevels;
6668
int[] myNewLevelsArr = myCurLevelsArr;
6769
double[] myNewDoubleItemsArr = myCurDoubleItemsArr;
6870

69-
if (otherNumLevels > 1 && !other.isCompactSingleItem()) { //now merge other levels if they exist
70-
final int tmpSpaceNeeded = sketch.getNumRetained()
71+
if (otherNumLevels > 1 && !otherDblSk.isCompactSingleItem()) { //now merge other levels if they exist
72+
final int tmpSpaceNeeded = mySketch.getNumRetained()
7173
+ KllHelper.getNumRetainedAboveLevelZero(otherNumLevels, otherLevelsArr);
7274
final double[] workbuf = new double[tmpSpaceNeeded];
7375
final int ub = KllHelper.ubOnNumLevels(finalN);
@@ -81,8 +83,8 @@ static void mergeDoubleImpl(final KllDoublesSketch sketch, final KllSketch other
8183
otherNumLevels, otherLevelsArr, otherDoubleItemsArr);
8284

8385
// notice that workbuf is being used as both the input and output
84-
final int[] result = generalDoublesCompress(sketch.getK(), sketch.getM(), provisionalNumLevels,
85-
workbuf, worklevels, workbuf, outlevels, sketch.isLevelZeroSorted(), KllSketch.random);
86+
final int[] result = generalDoublesCompress(mySketch.getK(), mySketch.getM(), provisionalNumLevels,
87+
workbuf, worklevels, workbuf, outlevels, mySketch.isLevelZeroSorted(), KllSketch.random);
8688
final int targetItemCount = result[1]; //was finalCapacity. Max size given k, m, numLevels
8789
final int curItemCount = result[2]; //was finalPop
8890

@@ -113,28 +115,28 @@ static void mergeDoubleImpl(final KllDoublesSketch sketch, final KllSketch other
113115
}
114116

115117
//MEMORY SPACE MANAGEMENT
116-
if (sketch.updatableMemFormat) {
117-
sketch.wmem = KllHelper.memorySpaceMgmt(sketch, myNewLevelsArr.length, myNewDoubleItemsArr.length);
118+
if (mySketch.updatableMemFormat) {
119+
mySketch.wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length);
118120
}
119121
}
120122

121123
//Update Preamble:
122-
sketch.setN(finalN);
123-
if (other.isEstimationMode()) { //otherwise the merge brings over exact items.
124-
sketch.setMinK(min(myMinK, other.getMinK()));
124+
mySketch.setN(finalN);
125+
if (otherDblSk.isEstimationMode()) { //otherwise the merge brings over exact items.
126+
mySketch.setMinK(min(myMinK, otherDblSk.getMinK()));
125127
}
126128

127129
//Update numLevels, levelsArray, items
128-
sketch.setNumLevels(myNewNumLevels);
129-
sketch.setLevelsArray(myNewLevelsArr);
130-
sketch.setDoubleItemsArray(myNewDoubleItemsArr);
130+
mySketch.setNumLevels(myNewNumLevels);
131+
mySketch.setLevelsArray(myNewLevelsArr);
132+
mySketch.setDoubleItemsArray(myNewDoubleItemsArr);
131133

132134
//Update min, max items
133-
final double otherMin = other.getMinDoubleItem();
134-
final double otherMax = other.getMaxDoubleItem();
135-
sketch.setMinDoubleItem(resolveDoubleMinItem(myMin, otherMin));
136-
sketch.setMaxDoubleItem(resolveDoubleMaxItem(myMax, otherMax));
137-
assert KllHelper.sumTheSampleWeights(sketch.getNumLevels(), sketch.getLevelsArray()) == sketch.getN();
135+
final double otherMin = otherDblSk.getMinDoubleItem();
136+
final double otherMax = otherDblSk.getMaxDoubleItem();
137+
mySketch.setMinDoubleItem(resolveDoubleMinItem(myMin, otherMin));
138+
mySketch.setMaxDoubleItem(resolveDoubleMaxItem(myMax, otherMax));
139+
assert KllHelper.sumTheSampleWeights(mySketch.getNumLevels(), mySketch.getLevelsArray()) == mySketch.getN();
138140
}
139141

140142
//Called from KllHelper and this.generalDoublesCompress(...), this.populateDoubleWorkArrays(...)
@@ -170,13 +172,13 @@ static void mergeSortedDoubleArrays(
170172
}
171173

172174
/**
173-
* Validation Method. This must be modified to test validation
175+
* Validation Method. This must be modified to use the validation test
174176
* @param buf the items array
175177
* @param start data start
176178
* @param length items array length
177179
* @param random instance of Random
178180
*/
179-
//NOTE Validation Method: Need to modify.
181+
//NOTE Validation Method: Need to modify to run.
180182
//Called from KllHelper, this.generalDoublesCompress(...)
181183
static void randomlyHalveDownDoubles(final double[] buf, final int start, final int length, final Random random) {
182184
assert isEven(length);
@@ -191,13 +193,13 @@ static void randomlyHalveDownDoubles(final double[] buf, final int start, final
191193
}
192194

193195
/**
194-
* Validation Method. This must be modified to test validation
196+
* Validation Method. This must be modified to use the validation test
195197
* @param buf the items array
196198
* @param start data start
197199
* @param length items array length
198200
* @param random instance of Random
199201
*/
200-
//NOTE Validation Method: Need to modify.
202+
//NOTE Validation Method: Need to modify to run.
201203
//Called from KllHelper, this.generalDoublesCompress(...)
202204
static void randomlyHalveUpDoubles(final double[] buf, final int start, final int length, final Random random) {
203205
assert isEven(length);
@@ -212,20 +214,20 @@ static void randomlyHalveUpDoubles(final double[] buf, final int start, final in
212214
}
213215

214216
//Called from KllDoublesSketch, this.mergeDoubleImpl(...)
215-
static void updateDouble(final KllSketch sketch, final double item) {
217+
static void updateDouble(final KllDoublesSketch dblSk, final double item) {
216218
if (Double.isNaN(item)) { return; }
217-
final double prevMin = sketch.getMinDoubleItem();
218-
final double prevMax = sketch.getMaxDoubleItem();
219-
sketch.setMinDoubleItem(resolveDoubleMinItem(prevMin, item));
220-
sketch.setMaxDoubleItem(resolveDoubleMaxItem(prevMax, item));
221-
if (sketch.getLevelsArray()[0] == 0) { KllHelper.compressWhileUpdatingSketch(sketch); }
222-
final int myLevelsArrAtZero = sketch.getLevelsArray()[0]; //LevelsArr could be expanded
223-
sketch.incN();
224-
sketch.setLevelZeroSorted(false);
219+
final double prevMin = dblSk.getMinDoubleItem();
220+
final double prevMax = dblSk.getMaxDoubleItem();
221+
dblSk.setMinDoubleItem(resolveDoubleMinItem(prevMin, item));
222+
dblSk.setMaxDoubleItem(resolveDoubleMaxItem(prevMax, item));
223+
if (dblSk.getLevelsArray()[0] == 0) { KllHelper.compressWhileUpdatingSketch(dblSk); }
224+
final int myLevelsArrAtZero = dblSk.getLevelsArray()[0]; //LevelsArr could be expanded
225+
dblSk.incN();
226+
dblSk.setLevelZeroSorted(false);
225227
final int nextPos = myLevelsArrAtZero - 1;
226228
assert myLevelsArrAtZero >= 0;
227-
sketch.setLevelsArrayAt(0, nextPos);
228-
sketch.setDoubleItemsArrayAt(nextPos, item);
229+
dblSk.setLevelsArrayAt(0, nextPos);
230+
dblSk.setDoubleItemsArrayAt(nextPos, item);
229231
}
230232

231233
/**
@@ -395,7 +397,7 @@ private static double resolveDoubleMinItem(final double myMin, final double othe
395397
* The following must be enabled for use with the KllDoublesValidationTest,
396398
* which is only enabled for manual testing. In addition, two Validation Methods
397399
* above need to be modified.
398-
*/ //NOTE Validation Method: Need to uncomment
400+
*/ //NOTE Validation Method: Need to uncomment to use
399401
// static int nextOffset = 0;
400402
//
401403
// private static int deterministicOffset() {

0 commit comments

Comments
 (0)