Skip to content

Commit b0e40fa

Browse files
author
lrhodes
committed
Fixed several bugs in cross-serialization.
1 parent cea3f74 commit b0e40fa

12 files changed

Lines changed: 186 additions & 50 deletions

‎src/main/java/com/yahoo/sketches/hll/AbstractHllArray.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,11 +110,9 @@ int getPreInts() {
110110

111111
abstract int getSlot(int slotNo);
112112

113-
@Override
113+
@Override //used by HLL6 and HLL8, Overridden by HLL4
114114
int getUpdatableSerializationBytes() {
115-
final AuxHashMap auxHashMap = getAuxHashMap();
116-
final int auxBytes = (auxHashMap == null) ? 0 : 4 << auxHashMap.getLgAuxArrInts();
117-
return HLL_BYTE_ARR_START + getHllByteArrBytes() + auxBytes;
115+
return HLL_BYTE_ARR_START + getHllByteArrBytes();
118116
}
119117

120118
@Override

‎src/main/java/com/yahoo/sketches/hll/DirectHll4Array.java‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import static com.yahoo.memory.UnsafeUtil.unsafe;
99
import static com.yahoo.sketches.hll.HllUtil.AUX_TOKEN;
10+
import static com.yahoo.sketches.hll.HllUtil.LG_AUX_ARR_INTS;
1011
import static com.yahoo.sketches.hll.HllUtil.hiNibbleMask;
1112
import static com.yahoo.sketches.hll.HllUtil.loNibbleMask;
1213
import static com.yahoo.sketches.hll.HllUtil.noWriteAccess;
@@ -88,6 +89,18 @@ final int getSlot(final int slotNo) {
8889
return theByte & loNibbleMask;
8990
}
9091

92+
@Override
93+
int getUpdatableSerializationBytes() {
94+
final AuxHashMap auxHashMap = getAuxHashMap();
95+
final int auxBytes;
96+
if (auxHashMap == null) {
97+
auxBytes = 4 << LG_AUX_ARR_INTS[lgConfigK];
98+
} else {
99+
auxBytes = 4 << auxHashMap.getLgAuxArrInts();
100+
}
101+
return HLL_BYTE_ARR_START + getHllByteArrBytes() + auxBytes;
102+
}
103+
91104
@Override
92105
final void putSlot(final int slotNo, final int newValue) {
93106
final long unsafeOffset = memAdd + HLL_BYTE_ARR_START + (slotNo >>> 1);
@@ -134,18 +147,14 @@ byte[] toUpdatableByteArray() {
134147
final int totBytes = getUpdatableSerializationBytes();
135148
final byte[] byteArr = new byte[totBytes];
136149
final WritableMemory memOut = WritableMemory.wrap(byteArr);
137-
if (!memIsCompact) { //mem is already consistent with result
150+
151+
if (!memIsCompact) { //both mem and target are updatable
138152
mem.copyTo(0, memOut, 0, totBytes);
139153
return byteArr;
140154
}
141-
//everything but perhaps aux array is consistent
142-
if (auxHashMap != null) {
143-
final HllSketch heapSk = HllSketch.heapify(mem);
144-
return heapSk.toUpdatableByteArray();
145-
}
146-
//no aux array
147-
mem.copyTo(0, memOut, 0, totBytes);
148-
return byteArr;
155+
//mem is compact, need to handle auxArr. Easiest way:
156+
final HllSketch heapSk = HllSketch.heapify(mem);
157+
return heapSk.toUpdatableByteArray();
149158
}
150159

151160
//ITERATOR

‎src/main/java/com/yahoo/sketches/hll/Hll4Array.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@
66
package com.yahoo.sketches.hll;
77

88
import static com.yahoo.sketches.hll.HllUtil.AUX_TOKEN;
9+
import static com.yahoo.sketches.hll.HllUtil.LG_AUX_ARR_INTS;
910
import static com.yahoo.sketches.hll.HllUtil.hiNibbleMask;
1011
import static com.yahoo.sketches.hll.HllUtil.loNibbleMask;
12+
import static com.yahoo.sketches.hll.PreambleUtil.HLL_BYTE_ARR_START;
1113
import static com.yahoo.sketches.hll.PreambleUtil.extractAuxCount;
1214
import static com.yahoo.sketches.hll.PreambleUtil.extractCompactFlag;
1315
import static com.yahoo.sketches.hll.PreambleUtil.extractLgK;
@@ -88,6 +90,18 @@ int getSlot(final int slotNo) {
8890
return theByte & loNibbleMask;
8991
}
9092

93+
@Override
94+
int getUpdatableSerializationBytes() {
95+
final AuxHashMap auxHashMap = getAuxHashMap();
96+
final int auxBytes;
97+
if (auxHashMap == null) {
98+
auxBytes = 4 << LG_AUX_ARR_INTS[lgConfigK];
99+
} else {
100+
auxBytes = 4 << auxHashMap.getLgAuxArrInts();
101+
}
102+
return HLL_BYTE_ARR_START + getHllByteArrBytes() + auxBytes;
103+
}
104+
91105
@Override
92106
void putSlot(final int slotNo, final int newValue) {
93107
final int byteno = slotNo >>> 1;

‎src/main/java/com/yahoo/sketches/hll/HllSketch.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -159,16 +159,16 @@ public static final HllSketch heapify(final Memory srcMem) {
159159
public static final HllSketch writableWrap(final WritableMemory wmem) {
160160
final Object memObj = wmem.getArray();
161161
final long memAdd = wmem.getCumulativeOffset(0);
162-
final int lgConfigK = extractLgK(memObj, memAdd);
163-
final TgtHllType tgtHllType = extractTgtHllType(memObj, memAdd);
164-
final long minBytes = getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
165-
final long capBytes = wmem.getCapacity();
166-
HllUtil.checkMemSize(minBytes, capBytes);
167162
final boolean compact = extractCompactFlag(memObj, memAdd);
168163
if (compact) {
169164
throw new SketchesArgumentException(
170165
"Cannot perform a writableWrap of a writable sketch image that is in compact form.");
171166
}
167+
final int lgConfigK = extractLgK(memObj, memAdd);
168+
final TgtHllType tgtHllType = extractTgtHllType(memObj, memAdd);
169+
final long minBytes = getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
170+
final long capBytes = wmem.getCapacity();
171+
HllUtil.checkMemSize(minBytes, capBytes);
172172

173173
final CurMode curMode = checkPreamble(wmem);
174174
final HllSketch directSketch;

‎src/main/java/com/yahoo/sketches/hll/HllUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ final class HllUtil {
5050

5151
/**
5252
* Log2 table sizes for exceptions based on lgK from 0 to 26.
53-
* However, only lgK from 7 to 21 are used.
53+
* However, only lgK from 4 to 21 are used.
5454
*/
5555
static final int[] LG_AUX_ARR_INTS = new int[] {
5656
0, 2, 2, 2, 2, 2, 2, 3, 3, 3, //0 - 9

‎src/main/java/com/yahoo/sketches/hll/ToByteArrayImpl.java‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@
66
package com.yahoo.sketches.hll;
77

88
import static com.yahoo.sketches.hll.AbstractCoupons.find;
9+
import static com.yahoo.sketches.hll.HllUtil.LG_AUX_ARR_INTS;
910
import static com.yahoo.sketches.hll.PreambleUtil.AUX_COUNT_INT;
1011
import static com.yahoo.sketches.hll.PreambleUtil.HLL_BYTE_ARR_START;
11-
import static com.yahoo.sketches.hll.PreambleUtil.HLL_PREINTS;
1212
import static com.yahoo.sketches.hll.PreambleUtil.insertAuxCount;
1313
import static com.yahoo.sketches.hll.PreambleUtil.insertCompactFlag;
1414
import static com.yahoo.sketches.hll.PreambleUtil.insertCurMin;
@@ -41,11 +41,15 @@ class ToByteArrayImpl {
4141
// To byte array used by the heap HLL types.
4242
static final byte[] toHllByteArray(final AbstractHllArray impl, final boolean compact) {
4343
int auxBytes = 0;
44-
final AuxHashMap auxHashMap = impl.getAuxHashMap();
45-
if (auxHashMap != null) { //only relevant for HLL_4
46-
auxBytes = (compact)
47-
? auxHashMap.getCompactSizeBytes()
48-
: auxHashMap.getUpdatableSizeBytes();
44+
if (impl.tgtHllType == TgtHllType.HLL_4) {
45+
final AuxHashMap auxHashMap = impl.getAuxHashMap();
46+
if (auxHashMap != null) {
47+
auxBytes = (compact)
48+
? auxHashMap.getCompactSizeBytes()
49+
: auxHashMap.getUpdatableSizeBytes();
50+
} else {
51+
auxBytes = (compact) ? 0 : 4 << LG_AUX_ARR_INTS[impl.lgConfigK];
52+
}
4953
}
5054
final int totBytes = HLL_BYTE_ARR_START + impl.getHllByteArrBytes() + auxBytes;
5155
final byte[] byteArr = new byte[totBytes];
@@ -70,7 +74,7 @@ private static final void insertCommonHll(final AbstractHllArray srcImpl,
7074
final WritableMemory tgtWmem, final boolean compact) {
7175
final Object tgtMemObj = tgtWmem.getArray();
7276
final long tgtMemAdd = tgtWmem.getCumulativeOffset(0L);
73-
insertPreInts(tgtMemObj, tgtMemAdd, HLL_PREINTS);
77+
insertPreInts(tgtMemObj, tgtMemAdd, srcImpl.getPreInts());
7478
insertSerVer(tgtMemObj, tgtMemAdd);
7579
insertFamilyId(tgtMemObj, tgtMemAdd);
7680
insertLgK(tgtMemObj, tgtMemAdd, srcImpl.getLgConfigK());

‎src/test/java/com/yahoo/sketches/hll/CouponListTest.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,13 @@ private static void toByteArrayHeapify(int lgK) {
125125
assertEquals(est2, est1, 0.0);
126126
}
127127

128+
@Test
129+
public void checkGetMemory() {
130+
HllSketch sk1 = new HllSketch(4);
131+
AbstractCoupons absCoup = (AbstractCoupons) sk1.hllSketchImpl;
132+
assertNull(absCoup.getMemory());
133+
}
134+
128135
@Test
129136
public void printlnTest() {
130137
println("PRINTING: "+this.getClass().getName());

‎src/test/java/com/yahoo/sketches/hll/DirectCouponListTest.java‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import static org.testng.Assert.assertEquals;
99
import static org.testng.Assert.assertFalse;
10+
import static org.testng.Assert.assertNull;
1011
import static org.testng.Assert.assertTrue;
1112

1213
import org.testng.annotations.Test;
@@ -109,27 +110,53 @@ private static void printDiffs(byte[] arr1, byte[] arr2) {
109110
}
110111

111112
@Test
112-
public void checkToByteArray() {
113+
public void checkCouponToByteArray() {
113114
int lgK = 8;
114115
TgtHllType type = TgtHllType.HLL_8;
115116
int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, type);
116117
WritableMemory wmem = WritableMemory.allocate(bytes);
117118
HllSketch sk = new HllSketch(lgK, type, wmem);
118-
for (int i = 0; i < 7; i++) { sk.update(i); }
119+
int i;
120+
for (i = 0; i < 7; i++) { sk.update(i); } //LIST
119121

120122
//toCompactMemArr from compact mem
121123
byte[] compactByteArr = sk.toCompactByteArray();
122124
Memory compactMem = Memory.wrap(compactByteArr);
123-
HllSketch sk2 = HllSketch.wrap(compactMem);
124-
byte[] compactByteArr2 = sk2.toCompactByteArray();
125+
HllSketch skCompact = HllSketch.wrap(compactMem);
126+
byte[] compactByteArr2 = skCompact.toCompactByteArray();
125127
assertEquals(compactByteArr2, compactByteArr);
126128

127129
//now check to UpdatableByteArr from compact mem
128130
byte[] updatableByteArr = sk.toUpdatableByteArray();
129-
byte[] updatableByteArr2 = sk2.toUpdatableByteArray();
131+
byte[] updatableByteArr2 = skCompact.toUpdatableByteArray();
130132
assertEquals(updatableByteArr2.length, updatableByteArr.length);
131-
assertEquals(sk2.getEstimate(), sk.getEstimate());
133+
assertEquals(skCompact.getEstimate(), sk.getEstimate());
132134

135+
136+
sk.update(i); //SET
137+
//toCompactMemArr from compact mem
138+
compactByteArr = sk.toCompactByteArray();
139+
compactMem = Memory.wrap(compactByteArr);
140+
skCompact = HllSketch.wrap(compactMem);
141+
compactByteArr2 = skCompact.toCompactByteArray();
142+
assertEquals(compactByteArr2, compactByteArr);
143+
144+
//now check to UpdatableByteArr from compact mem
145+
updatableByteArr = sk.toUpdatableByteArray();
146+
updatableByteArr2 = skCompact.toUpdatableByteArray();
147+
assertEquals(updatableByteArr2.length, updatableByteArr.length);
148+
assertEquals(skCompact.getEstimate(), sk.getEstimate());
149+
}
150+
151+
@Test
152+
public void checkDirectGetCouponIntArr() {
153+
int lgK = 8;
154+
TgtHllType type = TgtHllType.HLL_8;
155+
int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, type);
156+
WritableMemory wmem = WritableMemory.allocate(bytes);
157+
HllSketch sk = new HllSketch(lgK, type, wmem);
158+
AbstractCoupons absCoup = (AbstractCoupons)sk.hllSketchImpl;
159+
assertNull(absCoup.getCouponIntArr());
133160
}
134161

135162
@Test

‎src/test/java/com/yahoo/sketches/hll/DirectHllSketchTest.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package com.yahoo.sketches.hll;
77

88
import static org.testng.Assert.assertEquals;
9+
import static org.testng.Assert.assertTrue;
910
import static org.testng.Assert.fail;
1011

1112
import java.util.HashSet;
@@ -94,6 +95,55 @@ public void checkCompactToUpdatable() {
9495
assertEquals(validCount, validCount2);
9596
}
9697

98+
@Test
99+
public void checkPutKxQ1_Misc() {
100+
int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4);
101+
WritableMemory wmem = WritableMemory.allocate(bytes);
102+
HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem);
103+
for (int i = 0; i < 8; i++) { sk.update(i); }
104+
assertTrue(sk.getCurMode() == CurMode.HLL);
105+
AbstractHllArray absArr = (AbstractHllArray)sk.hllSketchImpl;
106+
absArr.putKxQ1(1.0);
107+
assertEquals(absArr.getKxQ1(), 1.0);
108+
absArr.putKxQ1(0.0);
109+
110+
Memory mem = wmem;
111+
HllSketch sk2 = HllSketch.wrap(mem);
112+
try {
113+
sk2.reset();
114+
fail();
115+
} catch (SketchesArgumentException e) {
116+
//expected
117+
}
118+
}
119+
120+
@Test
121+
public void checkToCompactByteArr() {
122+
int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4);
123+
WritableMemory wmem = WritableMemory.allocate(bytes);
124+
HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem);
125+
for (int i = 0; i < 8; i++) { sk.update(i); }
126+
byte[] compByteArr = sk.toCompactByteArray();
127+
Memory compMem = Memory.wrap(compByteArr);
128+
HllSketch sk2 = HllSketch.wrap(compMem);
129+
byte[] compByteArr2 = sk2.toCompactByteArray();
130+
assertEquals(compByteArr2, compByteArr);
131+
}
132+
133+
@Test
134+
public void checkToUpdatableByteArr() {
135+
int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4);
136+
WritableMemory wmem = WritableMemory.allocate(bytes);
137+
HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem);
138+
for (int i = 0; i < 8; i++) { sk.update(i); }
139+
byte[] udByteArr = sk.toUpdatableByteArray();
140+
byte[] compByteArr = sk.toCompactByteArray();
141+
Memory compMem = Memory.wrap(compByteArr);
142+
HllSketch sk2 = HllSketch.wrap(compMem);
143+
byte[] udByteArr2 = sk2.toUpdatableByteArray();
144+
assertEquals(udByteArr2, udByteArr);
145+
}
146+
97147
@Test
98148
public void printlnTest() {
99149
println("PRINTING: "+this.getClass().getName());

‎src/test/java/com/yahoo/sketches/hll/HllArrayTest.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,13 @@ private static void toByteArrayHeapify(int lgK, TgtHllType tgtHllType, int u, bo
109109
assertEquals(sk1.getEstimate(), 0.0, 0.0);
110110
}
111111

112+
@Test
113+
public void checkIsCompact() {
114+
HllSketch sk = new HllSketch(4);
115+
for (int i = 0; i < 8; i++) { sk.update(i); }
116+
assertFalse(sk.isCompact());
117+
}
118+
112119
@Test
113120
public void printlnTest() {
114121
println("PRINTING: "+this.getClass().getName());

0 commit comments

Comments
 (0)