The integer values produced are only as random as the MurmurHash3 algorithm, which may be - * adequate for many applications. However, if you are looking for high guarantees of randomness - * you should turn to more sophisticated random generators such as Mersenne Twister or Well19937c - * algorithms. - * - * @param data The input data (key) - * @param n The upper exclusive bound of the integers produced. Must be > 1. - * @return deterministic uniform random integer - */ - private static int asInteger(final long[] data, final int n) { - int t; - int cnt = 0; - long seed = 0; - if (n < 2) { - throw new SketchesArgumentException("Given value of n must be > 1."); - } - if (n > (1 << 30)) { - while (++cnt < 10000) { - final long[] h = MurmurHash3.hash(data, seed); - t = (int) (h[0] & INT_MASK); - if (t < n) { - return t; - } - t = (int) ((h[0] >>> 33)); - if (t < n) { - return t; - } - t = (int) (h[1] & INT_MASK); - if (t < n) { - return t; - } - t = (int) ((h[1] >>> 33)); - if (t < n) { - return t; - } - seed += PRIME; - } // end while - throw new SketchesStateException( - "Internal Error: Failed to find integer < n within 10000 iterations."); - } - final long mask = ceilingPowerOf2(n) - 1; - while (++cnt < 10000) { - final long[] h = MurmurHash3.hash(data, seed); - t = (int) (h[0] & mask); - if (t < n) { - return t; - } - t = (int) ((h[0] >>> 33) & mask); - if (t < n) { - return t; - } - t = (int) (h[1] & mask); - if (t < n) { - return t; - } - t = (int) ((h[1] >>> 33) & mask); - if (t < n) { - return t; - } - seed += PRIME; - } // end while - throw new SketchesStateException( - "Internal Error: Failed to find integer < n within 10000 iterations."); - } - - /** - * Returns a uniform random double with a minimum inclusive value of zero and a maximum exclusive - * value of 1.0. - * - *
The double values produced are only as random as the MurmurHash3 algorithm, which may be - * adequate for many applications. However, if you are looking for high guarantees of randomness - * you should turn to more sophisticated random generators such as Mersenne Twister or Well - * algorithms. - * - * @param hash The output of the MurmurHash3. - * @return the uniform random double. - */ - public static double asDouble(final long[] hash) { - return (hash[0] >>> 12) * 0x1.0p-52d; - } - - /** - * Returns the remainder from the modulo division of the 128-bit output of the murmurHash3 by the - * divisor. - * - * @param h0 The lower 64-bits of the 128-bit MurmurHash3 hash. - * @param h1 The upper 64-bits of the 128-bit MurmurHash3 hash. - * @param divisor Must be positive and greater than zero. - * @return the modulo result. - */ - public static int modulo(final long h0, final long h1, final int divisor) { - final long d = divisor; - final long modH0 = (h0 < 0L) ? addRule(mulRule(BIT62, 2L, d), (h0 & MAX_LONG), d) : h0 % d; - final long modH1 = (h1 < 0L) ? addRule(mulRule(BIT62, 2L, d), (h1 & MAX_LONG), d) : h1 % d; - final long modTop = mulRule(mulRule(BIT62, 4L, d), modH1, d); - return (int) addRule(modTop, modH0, d); - } - - /** - * Returns the remainder from the modulo division of the 128-bit output of the murmurHash3 by the - * divisor. - * - * @param hash The size 2 long array from the MurmurHash3. - * @param divisor Must be positive and greater than zero. - * @return the modulo result - */ - public static int modulo(final long[] hash, final int divisor) { - return modulo(hash[0], hash[1], divisor); - } - - private static long addRule(final long a, final long b, final long d) { - return ((a % d) + (b % d)) % d; - } - - private static long mulRule(final long a, final long b, final long d) { - return ((a % d) * (b % d)) % d; - } - - private static byte[] toByteArray(final long[] hash) { //Assumes Big Endian - final byte[] bArr = new byte[16]; - final ByteBuffer bb = ByteBuffer.wrap(bArr); - bb.putLong(hash[0]); - bb.putLong(hash[1]); - return bArr; - } - - private static long[] toLongArray(final byte[] data) { - final int dataLen = data.length; - final int longLen = (dataLen + 7) / 8; - final long[] longArr = new long[longLen]; - for (int bi = 0; bi < dataLen; bi++) { - final int li = bi / 8; - longArr[li] |= (((long)data[bi]) << ((bi * 8) % 64)); - } - return longArr; - } - - private static long[] toLongArray(final int[] data) { - final int dataLen = data.length; - final int longLen = (dataLen + 1) / 2; - final long[] longArr = new long[longLen]; - for (int ii = 0; ii < dataLen; ii++) { - final int li = ii / 2; - longArr[li] |= (((long)data[ii]) << ((ii * 32) % 64)); - } - return longArr; - } - -} diff --git a/src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java b/src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java new file mode 100644 index 000000000..a94fd3fd9 --- /dev/null +++ b/src/main/java/org/apache/datasketches/hash/MurmurHash3FFM21.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.hash; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Objects; + +import org.apache.datasketches.memory.Memory; + +/** + * The MurmurHash3 is a fast, non-cryptographic, 128-bit hash function that has + * excellent avalanche and 2-way bit independence properties. + * + *
Austin Appleby's C++ + * + * MurmurHash3_x64_128(...), final revision 150, + * which is in the Public Domain, was the inspiration for this implementation in Java.
+ * + *This implementation of the MurmurHash3 allows hashing of a block of on-heap Memory defined by an offset + * and length. The calling API also allows the user to supply the small output array of two longs, + * so that the entire hash function is static and free of object allocations.
+ * + *This implementation produces exactly the same hash result as the + * MurmurHash3 function in datasketches-java given compatible inputs.
+ * + *This FFM21 version of the implementation leverages the java.lang.foreign package (FFM) of JDK-21 in place of + * the Unsafe class. + * + * @author Lee Rhodes + */ +public final class MurmurHash3FFM21 { + private static final long C1 = 0x87c37b91114253d5L; + private static final long C2 = 0x4cf5ad432745937fL; + + /** + * Returns a 128-bit hash of the input. + * Provided for compatibility with older version of MurmurHash3, + * but empty or null input now throws IllegalArgumentException. + * @param in long array + * @param seed A long valued seed. + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final long[] in, final long seed) { + if ((in == null) || (in.length == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + return hash(MemorySegment.ofArray(in), 0L, in.length << 3, seed, new long[2]); + } + + /** + * Returns a 128-bit hash of the input. + * Provided for compatibility with older version of MurmurHash3, + * but empty or null input now throws IllegalArgumentException. + * @param in int array + * @param seed A long valued seed. + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final int[] in, final long seed) { + if ((in == null) || (in.length == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + return hash(MemorySegment.ofArray(in), 0L, in.length << 2, seed, new long[2]); + } + + /** + * Returns a 128-bit hash of the input. + * Provided for compatibility with older version of MurmurHash3, + * but empty or null input now throws IllegalArgumentException. + * @param in char array + * @param seed A long valued seed. + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final char[] in, final long seed) { + if ((in == null) || (in.length == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + return hash(MemorySegment.ofArray(in), 0L, in.length << 1, seed, new long[2]); + } + + /** + * Returns a 128-bit hash of the input. + * Provided for compatibility with older version of MurmurHash3, + * but empty or null input now throws IllegalArgumentException. + * @param in byte array + * @param seed A long valued seed. + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final byte[] in, final long seed) { + if ((in == null) || (in.length == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + return hash(MemorySegment.ofArray(in), 0L, in.length, seed, new long[2]); + } + + //Single primitive inputs + + /** + * Returns a 128-bit hash of the input. + * Note the entropy of the resulting hash cannot be more than 64 bits. + * @param in a long + * @param seed A long valued seed. + * @param hashOut A long array of size 2 + * @return the hash + */ + public static long[] hash(final long in, final long seed, final long[] hashOut) { + final long h1 = seed ^ mixK1(in); + final long h2 = seed; + return finalMix128(h1, h2, 8, hashOut); + } + + /** + * Returns a 128-bit hash of the input. + * Note the entropy of the resulting hash cannot be more than 64 bits. + * @param in a double + * @param seed A long valued seed. + * @param hashOut A long array of size 2 + * @return the hash + */ + public static long[] hash(final double in, final long seed, final long[] hashOut) { + final double d = (in == 0.0) ? 0.0 : in; // canonicalize -0.0, 0.0 + final long k1 = Double.doubleToLongBits(d); // canonicalize all NaN forms + final long h1 = seed ^ mixK1(k1); + final long h2 = seed; + return finalMix128(h1, h2, 8, hashOut); + } + + /** + * Returns a 128-bit hash of the input. + * An empty or null input throws IllegalArgumentException. + * @param in a String + * @param seed A long valued seed. + * @param hashOut A long array of size 2 + * @return the hash + * @throws IllegalArgumentException if input is empty or null + */ + public static long[] hash(final String in, final long seed, final long[] hashOut) { + if ((in == null) || (in.length() == 0)) { + throw new IllegalArgumentException("Input in is empty or null."); + } + final byte[] byteArr = in.getBytes(UTF_8); + return hash(MemorySegment.ofArray(byteArr), 0L, byteArr.length, seed, hashOut); + } + + //The main API calls + + /** + * Returns a 128-bit hash of the input as a long array of size 2. + * + * @param mem The input Memory. Must be non-null and non-empty, + * otherwise throws IllegalArgumentException. + * @param offsetBytes the starting point within Memory. + * @param lengthBytes the total number of bytes to be hashed. + * @param seed A long valued seed. + * @param hashOut the size 2 long array for the resulting 128-bit hash + * @return the hash. + */ + public static long[] hash(final Memory mem, final long offsetBytes, final long lengthBytes, + final long seed, final long[] hashOut) { + Objects.requireNonNull(mem, "Input Memory is null"); + final MemorySegment seg = mem.getMemorySegment(); + return hash(seg, offsetBytes, lengthBytes, seed, hashOut); + } + + /** + * Returns a 128-bit hash of the input as a long array of size 2. + * + * @param seg The input MemorySegment. Must be non-null and non-empty, + * otherwise throws IllegalArgumentException. + * @param offsetBytes the starting point within Memory. + * @param lengthBytes the total number of bytes to be hashed. + * @param seed A long valued seed. + * @param hashOut the size 2 long array for the resulting 128-bit hash + * @return the hash. + * @throws IllegalArgumentException if input MemorySegment is empty + */ + public static long[] hash(final MemorySegment seg, final long offsetBytes, final long lengthBytes, + final long seed, final long[] hashOut) { + Objects.requireNonNull(seg, "Input MemorySegment is null"); + if (seg.byteSize() == 0L) { throw new IllegalArgumentException("Input MemorySegment is empty."); } + + long cumOff = offsetBytes; + + long h1 = seed; + long h2 = seed; + long rem = lengthBytes; + + // Process the 128-bit blocks (the body) into the hash + while (rem >= 16L) { + final long k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); //0, 16, 32, ... + final long k2 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff + 8); //8, 24, 40, ... + cumOff += 16L; + rem -= 16L; + + h1 ^= mixK1(k1); + h1 = Long.rotateLeft(h1, 27); + h1 += h2; + h1 = (h1 * 5) + 0x52dce729L; + + h2 ^= mixK2(k2); + h2 = Long.rotateLeft(h2, 31); + h2 += h1; + h2 = (h2 * 5) + 0x38495ab5L; + } + + // Get the tail (if any): 1 to 15 bytes + if (rem > 0L) { + long k1 = 0; + long k2 = 0; + switch ((int) rem) { + case 15: { + k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 14) & 0xFFL) << 48; + } + //$FALL-THROUGH$ + case 14: { + k2 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 12) & 0xFFFFL) << 32; + k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; + k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); + break; + } + + case 13: { + k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 12) & 0xFFL) << 32; + } + //$FALL-THROUGH$ + case 12: { + k2 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff + 8) & 0xFFFFFFFFL; + k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); + break; + } + + case 11: { + k2 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 10) & 0xFFL) << 16; + } + //$FALL-THROUGH$ + case 10: { + k2 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 8) & 0xFFFFL; + k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); + break; + } + + case 9: { + k2 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff + 8) & 0xFFL; + } + //$FALL-THROUGH$ + case 8: { + k1 = seg.get(ValueLayout.JAVA_LONG_UNALIGNED, cumOff); + break; + } + + case 7: { + k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 6) & 0xFFL) << 48; + } + //$FALL-THROUGH$ + case 6: { + k1 ^= (seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff + 4) & 0xFFFFL) << 32; + k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL; + break; + } + + case 5: { + k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 4) & 0xFFL) << 32; + } + //$FALL-THROUGH$ + case 4: { + k1 ^= seg.get(ValueLayout.JAVA_INT_UNALIGNED, cumOff) & 0xFFFFFFFFL; + break; + } + + case 3: { + k1 ^= (seg.get(ValueLayout.JAVA_BYTE, cumOff + 2) & 0xFFL) << 16; + } + //$FALL-THROUGH$ + case 2: { + k1 ^= seg.get(ValueLayout.JAVA_SHORT_UNALIGNED, cumOff) & 0xFFFFL; + break; + } + + case 1: { + k1 ^= seg.get(ValueLayout.JAVA_BYTE, cumOff) & 0xFFL; + break; + } + default: break; //can't happen + } + + h1 ^= mixK1(k1); + h2 ^= mixK2(k2); + } + return finalMix128(h1, h2, lengthBytes, hashOut); + } + + //--Helper methods---------------------------------------------------- + + /** + * Self mix of k1 + * + * @param k1 input argument + * @return mix + */ + private static long mixK1(long k1) { + k1 *= C1; + k1 = Long.rotateLeft(k1, 31); + k1 *= C2; + return k1; + } + + /** + * Self mix of k2 + * + * @param k2 input argument + * @return mix + */ + private static long mixK2(long k2) { + k2 *= C2; + k2 = Long.rotateLeft(k2, 33); + k2 *= C1; + return k2; + } + + /** + * Final self mix of h*. + * + * @param h input to final mix + * @return mix + */ + private static long finalMix64(long h) { + h ^= h >>> 33; + h *= 0xff51afd7ed558ccdL; + h ^= h >>> 33; + h *= 0xc4ceb9fe1a85ec53L; + h ^= h >>> 33; + return h; + } + + /** + * Finalization: Add the length into the hash and mix + * @param h1 intermediate hash + * @param h2 intermediate hash + * @param lengthBytes the length in bytes + * @param hashOut the output array of 2 longs + * @return hashOut + */ + private static long[] finalMix128(long h1, long h2, final long lengthBytes, final long[] hashOut) { + h1 ^= lengthBytes; + h2 ^= lengthBytes; + + h1 += h2; + h2 += h1; + + h1 = finalMix64(h1); + h2 = finalMix64(h2); + + h1 += h2; + h2 += h1; + + hashOut[0] = h1; + hashOut[1] = h2; + return hashOut; + } + + private MurmurHash3FFM21() { } + +} diff --git a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java index 98884f5ea..e6c408f35 100644 --- a/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java +++ b/src/main/java/org/apache/datasketches/hll/DirectAuxHashMap.java @@ -195,7 +195,7 @@ private static final void grow(final DirectHllArray host, final int oldLgAuxArrI final WritableMemory newWmem = svr.request(host.wmem, requestBytes); host.wmem.copyTo(0, newWmem, 0, host.auxStart); newWmem.clear(host.auxStart, newAuxBytes); //clear space for new aux data - svr.requestClose(host.wmem, newWmem); //old host.wmem is now invalid + svr.requestClose(host.wmem); //old host.wmem is now invalid host.updateMemory(newWmem); } //rehash into larger aux array diff --git a/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java b/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java index 52ddf60ba..683c8be96 100644 --- a/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java +++ b/src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java @@ -35,6 +35,7 @@ import static org.apache.datasketches.hll.PreambleUtil.insertLgArr; import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.common.SketchesException; import org.apache.datasketches.common.SketchesStateException; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; @@ -105,13 +106,14 @@ private boolean checkGrowOrPromote() { if (lgCouponArrInts == (getLgConfigK() - 3)) { return true; // promote } - //TODO if direct, ask for more memory insertLgArr(wmem, ++lgCouponArrInts); growHashSet(wmem, lgCouponArrInts); } return false; } + //This could fail if the user has undersized the given WritableMemory + // and not used the public methods for sizing the Memory. See exception. private static final void growHashSet(final WritableMemory wmem, final int tgtLgCouponArrSize) { final int tgtArrSize = 1 << tgtLgCouponArrSize; final int[] tgtCouponIntArr = new int[tgtArrSize]; @@ -128,7 +130,11 @@ private static final void growHashSet(final WritableMemory wmem, final int tgtLg } } wmem.clear(HASH_SET_INT_ARR_START, tgtArrSize << 2); - wmem.putIntArray(HASH_SET_INT_ARR_START, tgtCouponIntArr, 0, tgtArrSize); + try { wmem.putIntArray(HASH_SET_INT_ARR_START, tgtCouponIntArr, 0, tgtArrSize); } + catch (final IndexOutOfBoundsException e) { + throw new SketchesException( + "The WritableMemory is undersized. Use the public methods for properly sizing Memory.", e); + } } //Searches the Coupon hash table (embedded in Memory) for an empty slot diff --git a/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java b/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java index 374dbd624..5724b563c 100644 --- a/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java +++ b/src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java @@ -205,7 +205,6 @@ else if (slotNo == (arrVal & configKmask)) { //Compares only on slotNo private void checkGrow() { if ((RESIZE_DENOM * auxCount) > (RESIZE_NUMER * auxIntArr.length)) { growAuxSpace(); - //TODO if direct, ask for more memory } } diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java index acbecdf07..cb5788a75 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java @@ -207,8 +207,11 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke //MEMORY SPACE MANAGEMENT if (mySketch.getWritableMemory() != null) { - final WritableMemory wmem = - KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length); + final WritableMemory oldWmem = mySketch.getWritableMemory(); + final WritableMemory wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewDoubleItemsArr.length); + if (!wmem.isSameResource(oldWmem)) { + mySketch.getMemoryRequestServer().requestClose(oldWmem); + } mySketch.setWritableMemory(wmem); } } //end of updating levels above level 0 diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java index fbe9dbb36..01608ca02 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java @@ -137,7 +137,7 @@ public static KllDoublesSketch wrap(final Memory srcMem) { Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, DOUBLES_SKETCH, null); if (memVal.sketchStructure == UPDATABLE) { - final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); //dummy + final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); return new KllDirectDoublesSketch(memVal.sketchStructure, (WritableMemory)srcMem, memReqSvr, memVal); } else { return new KllDirectCompactDoublesSketch(memVal.sketchStructure, srcMem, memVal); diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java index 69045f78c..1796cac4d 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java @@ -207,8 +207,11 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch //MEMORY SPACE MANAGEMENT if (mySketch.getWritableMemory() != null) { - final WritableMemory wmem = - KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length); + final WritableMemory oldWmem = mySketch.getWritableMemory(); + final WritableMemory wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length); + if (!wmem.isSameResource(oldWmem)) { + mySketch.getMemoryRequestServer().requestClose(oldWmem); + } mySketch.setWritableMemory(wmem); } } //end of updating levels above level 0 diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java index b993f9998..cf282c872 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java @@ -137,7 +137,7 @@ public static KllFloatsSketch wrap(final Memory srcMem) { Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, FLOATS_SKETCH, null); if (memVal.sketchStructure == UPDATABLE) { - final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); //dummy + final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); return new KllDirectFloatsSketch(memVal.sketchStructure, (WritableMemory)srcMem, memReqSvr, memVal); } else { return new KllDirectCompactFloatsSketch(memVal.sketchStructure, srcMem, memVal); diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java index 21188255c..8dd729da7 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java @@ -52,6 +52,7 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.kll.KllSketch.SketchStructure; import org.apache.datasketches.kll.KllSketch.SketchType; +import org.apache.datasketches.memory.MemoryRequestServer; import org.apache.datasketches.memory.WritableBuffer; import org.apache.datasketches.memory.WritableMemory; @@ -353,7 +354,8 @@ static WritableMemory memorySpaceMgmt( final WritableMemory newWmem; if (requiredSketchBytes > oldWmem.getCapacity()) { //Acquire new WritableMemory - newWmem = sketch.getMemoryRequestServer().request(oldWmem, requiredSketchBytes); + final MemoryRequestServer memReqSvr = sketch.getMemoryRequestServer(); + newWmem = memReqSvr.request(oldWmem, requiredSketchBytes); oldWmem.copyTo(0, newWmem, 0, DATA_START_ADR); //copy preamble (first 20 bytes) } else { //Expand or contract in current memory @@ -617,7 +619,7 @@ else if (sketchType == FLOATS_SKETCH) { maxFloat = fltSk.getMaxItem(); //assert we are following a certain growth scheme assert myCurFloatItemsArr.length == myCurTotalItemsCapacity; - } + } else if (sketchType == LONGS_SKETCH) { final KllLongsSketch lngSk = (KllLongsSketch) sketch; myCurLongItemsArr = lngSk.getLongItemsArray(); @@ -668,7 +670,7 @@ else if (sketchType == FLOATS_SKETCH) { myNewFloatItemsArr = new float[myNewTotalItemsCapacity]; // copy and shift the current items data into the new array System.arraycopy(myCurFloatItemsArr, 0, myNewFloatItemsArr, deltaItemsCap, myCurTotalItemsCapacity); - } + } else if (sketchType == LONGS_SKETCH) { myNewLongItemsArr = new long[myNewTotalItemsCapacity]; // copy and shift the current items data into the new array @@ -682,7 +684,11 @@ else if (sketchType == LONGS_SKETCH) { //MEMORY SPACE MANAGEMENT if (sketch.getWritableMemory() != null) { + final WritableMemory oldWmem = sketch.getWritableMemory(); final WritableMemory wmem = memorySpaceMgmt(sketch, myNewLevelsArr.length, myNewTotalItemsCapacity); + if (!wmem.isSameResource(oldWmem)) { + sketch.getMemoryRequestServer().requestClose(oldWmem); + } sketch.setWritableMemory(wmem); } @@ -700,7 +706,7 @@ else if (sketchType == FLOATS_SKETCH) { fltSk.setMinItem(minFloat); fltSk.setMaxItem(maxFloat); fltSk.setFloatItemsArray(myNewFloatItemsArr); - } + } else if (sketchType == LONGS_SKETCH) { final KllLongsSketch lngSk = (KllLongsSketch) sketch; lngSk.setMinItem(minLong); diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java index 04fe2cc08..2dc146103 100644 --- a/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllLongsHelper.java @@ -207,8 +207,11 @@ static void mergeLongsImpl(final KllLongsSketch mySketch, final KllLongsSketch o //MEMORY SPACE MANAGEMENT if (mySketch.getWritableMemory() != null) { - final WritableMemory wmem = - KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewLongItemsArray.length); + final WritableMemory oldWmem = mySketch.getWritableMemory(); + final WritableMemory wmem = KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewLongItemsArray.length); + if (!wmem.isSameResource(oldWmem)) { + mySketch.getMemoryRequestServer().requestClose(oldWmem); + } mySketch.setWritableMemory(wmem); } } //end of updating levels above level 0 diff --git a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java index f5688ad70..827f825f8 100644 --- a/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllLongsSketch.java @@ -137,7 +137,7 @@ public static KllLongsSketch wrap(final Memory srcMem) { Objects.requireNonNull(srcMem, "Parameter 'srcMem' must not be null"); final KllMemoryValidate memVal = new KllMemoryValidate(srcMem, LONGS_SKETCH, null); if (memVal.sketchStructure == UPDATABLE) { - final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); //dummy + final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); return new KllDirectLongsSketch(memVal.sketchStructure, (WritableMemory)srcMem, memReqSvr, memVal); } else { return new KllDirectCompactLongsSketch(memVal.sketchStructure, srcMem, memVal); diff --git a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java index 7a7a7a850..12604ecb3 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java @@ -266,9 +266,8 @@ private WritableMemory growCombinedMemBuffer(final int itemSpaceNeeded) { } final WritableMemory newMem = memReqSvr.request(mem_, needBytes); - mem_.copyTo(0, newMem, 0, memBytes); - memReqSvr.requestClose(mem_, newMem); + memReqSvr.requestClose(mem_); return newMem; } diff --git a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketchR.java b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketchR.java index 3e80b3f0f..234170331 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketchR.java +++ b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketchR.java @@ -146,7 +146,7 @@ int getBaseBufferCount() { @Override int getCombinedBufferItemCapacity() { - return ((int)mem_.getCapacity() - COMBINED_BUFFER) / 8; + return Math.max(0, (int)mem_.getCapacity() - COMBINED_BUFFER) / 8; } @Override diff --git a/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java index 2fc399f9b..4c9f74286 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/DoublesSketch.java @@ -460,9 +460,9 @@ public static int getUpdatableStorageBytes(final int k, final long n) { final int totLevels = computeNumLevelsNeeded(k, n); if (n <= k) { final int ceil = Math.max(ceilingPowerOf2((int)n), MIN_K * 2); - return metaPre + ceil << 3; + return (metaPre + ceil) << 3; } - return metaPre + (2 + totLevels) * k << 3; + return (metaPre + (2 + totLevels) * k) << 3; } /** diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java index a082fc27a..471ee5570 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java @@ -202,7 +202,7 @@ * @author Kevin Lang * @author Alexander Saydakov */ -@SuppressWarnings("javadoc") + public interface QuantilesAPI { /** The sketch must not be empty for this operation. */ diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java index 946603194..dbdedebd5 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentDirectQuickSelectSketch.java @@ -199,7 +199,7 @@ public void awaitBgPropagationTermination() { @Override public final void initBgPropagationService() { - executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().getId()); + executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().threadId()); } @Override @@ -258,7 +258,7 @@ public boolean validateEpoch(final long epoch) { private void advanceEpoch() { awaitBgPropagationTermination(); startEagerPropagation(); - ConcurrentPropagationService.resetExecutorService(Thread.currentThread().getId()); + ConcurrentPropagationService.resetExecutorService(Thread.currentThread().threadId()); //no inspection NonAtomicOperationOnVolatileField // this increment of a volatile field is done within the scope of the propagation // synchronization and hence is done by a single thread. diff --git a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java index 1ce3b4ecc..e4cc3a157 100644 --- a/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/ConcurrentHeapQuickSelectSketch.java @@ -194,7 +194,7 @@ public void awaitBgPropagationTermination() { @Override public void initBgPropagationService() { - executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().getId()); + executorService_ = ConcurrentPropagationService.getExecutorService(Thread.currentThread().threadId()); } @Override @@ -253,7 +253,7 @@ public boolean validateEpoch(final long epoch) { private void advanceEpoch() { awaitBgPropagationTermination(); startEagerPropagation(); - ConcurrentPropagationService.resetExecutorService(Thread.currentThread().getId()); + ConcurrentPropagationService.resetExecutorService(Thread.currentThread().threadId()); //no inspection NonAtomicOperationOnVolatileField // this increment of a volatile field is done within the scope of the propagation // synchronization and hence is done by a single thread diff --git a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java index ad9051a08..af073a5ee 100644 --- a/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java +++ b/src/main/java/org/apache/datasketches/theta/DirectQuickSelectSketch.java @@ -327,11 +327,11 @@ UpdateReturnState hashUpdate(final long hash) { throw new SketchesArgumentException("Out of Memory, MemoryRequestServer is null, cannot expand."); } - final WritableMemory newDstMem = memReqSvr_.request(wmem_,reqBytes); + final WritableMemory newDstMem = memReqSvr_.request(wmem_, reqBytes); moveAndResize(wmem_, preambleLongs, lgArrLongs, newDstMem, tgtLgArrLongs, thetaLong); - memReqSvr_.requestClose(wmem_, newDstMem); + memReqSvr_.requestClose(wmem_); wmem_ = newDstMem; hashTableThreshold_ = getOffHeapHashTableThreshold(lgNomLongs, tgtLgArrLongs); diff --git a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java index d9fda48bb..2457b37fe 100644 --- a/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java +++ b/src/main/java/org/apache/datasketches/thetacommon/SetOperationCornerCases.java @@ -28,7 +28,7 @@ * Simplifies and speeds up set operations by resolving specific corner cases. * @author Lee Rhodes */ -@SuppressWarnings("javadoc") + public class SetOperationCornerCases { private static final long MAX = Long.MAX_VALUE; diff --git a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java index 9b0ca33cb..a30d47edf 100644 --- a/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java +++ b/src/main/java/org/apache/datasketches/tuple/SerializerDeserializer.java @@ -31,7 +31,6 @@ public final class SerializerDeserializer { /** * Defines the sketch classes that this SerializerDeserializer can handle. */ - @SuppressWarnings("javadoc") public static enum SketchType { /** QuickSelectSketch */ QuickSelectSketch, diff --git a/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java b/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java index 9a33aa71f..c5ec12399 100644 --- a/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java +++ b/src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java @@ -56,7 +56,7 @@ public void checkWriteReadUnary() { for (int i = 0; i < 100; i++) { - //TODO Inline WriteUnary + // Inline WriteUnary ptrArr[NEXT_WORD_IDX] = nextWordIndex; ptrArr[BIT_BUF] = bitBuf; ptrArr[BUF_BITS] = bufBits; @@ -92,7 +92,7 @@ public void checkWriteReadUnary() { for (int i = 0; i < 100; i++) { - //TODO Inline ReadUnary + // Inline ReadUnary ptrArr[NEXT_WORD_IDX] = nextWordIndex; ptrArr[BIT_BUF] = bitBuf; ptrArr[BUF_BITS] = bufBits; diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java index 7d72fae2f..2d71805ac 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java @@ -31,7 +31,7 @@ import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; -import jdk.incubator.foreign.ResourceScope; +import java.lang.foreign.Arena; public class BloomFilterTest { @@ -52,8 +52,9 @@ public void createNewFilterTest() throws Exception { assertFalse(bf1.isDirect()); assertFalse(bf1.isReadOnly()); - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(sizeBytes)).scope()) { + + try (Arena arena = Arena.ofConfined()) { + WritableMemory wmem = WritableMemory.allocateDirect(sizeBytes, arena); final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem); assertTrue(bf2.isEmpty()); assertTrue(bf2.hasMemory()); @@ -158,8 +159,9 @@ public void basicFilterOperationsTest() { int numFound = 0; for (long i = 0; i < 2 * n; ++i) { - if (bf.query(i)) + if (bf.query(i)) { ++numFound; + } } assertTrue(numFound >= n); assertTrue(numFound < 1.1 * n); @@ -339,8 +341,12 @@ public void nonEmptySerializationTest() { int fromBytesCount = 0; for (int i = 0; i < numBits; ++i) { boolean val = fromBytes.query(0.5 + i); - if (val) ++fromBytesCount; - if (i < n) assertTrue(val); + if (val) { + ++fromBytesCount; + } + if (i < n) { + assertTrue(val); + } } assertEquals(fromBytesCount, n + count); // same numbers of items should match @@ -354,8 +360,12 @@ public void nonEmptySerializationTest() { int fromLongsCount = 0; for (int i = 0; i < numBits; ++i) { boolean val = fromLongs.query(0.5 + i); - if (val) ++fromLongsCount; - if (i < n) assertTrue(val); + if (val) { + ++fromLongsCount; + } + if (i < n) { + assertTrue(val); + } } assertEquals(fromLongsCount, n + count); // same numbers of items should match diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java deleted file mode 100644 index 16a8e6652..000000000 --- a/src/test/java/org/apache/datasketches/hash/MurmurHash3AdaptorTest.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.hash; - -import static org.apache.datasketches.hash.MurmurHash3Adaptor.asDouble; -import static org.apache.datasketches.hash.MurmurHash3Adaptor.asInt; -import static org.apache.datasketches.hash.MurmurHash3Adaptor.hashToBytes; -import static org.apache.datasketches.hash.MurmurHash3Adaptor.hashToLongs; -import static org.apache.datasketches.hash.MurmurHash3Adaptor.modulo; - -import org.apache.datasketches.common.SketchesArgumentException; -import org.testng.Assert; -import org.testng.annotations.Test; - -/** - * @author Lee Rhodes - */ -public class MurmurHash3AdaptorTest { - - @Test - public void checkToBytesLong() { - byte[] result = hashToBytes(2L, 0L); - for (int i = 8; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - } - - @Test - public void checkToBytesLongArr() { - long[] arr = { 1L, 2L }; - byte[] result = hashToBytes(arr, 0L); - for (int i = 8; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - arr = null; - result = hashToBytes(arr, 0L); - Assert.assertEquals(result, null); - - arr = new long[0]; - result = hashToBytes(arr, 0L); - Assert.assertEquals(result, null); - } - - @Test - public void checkToBytesIntArr() { - int[] arr = { 1, 2 }; - byte[] result = hashToBytes(arr, 0L); - for (int i = 8; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - arr = null; - result = hashToBytes(arr, 0L); - Assert.assertEquals(result, null); - - arr = new int[0]; - result = hashToBytes(arr, 0L); - Assert.assertEquals(result, null); - } - - @Test - public void checkToBytesCharArr() { - char[] arr = { 1, 2 }; - byte[] result = hashToBytes(arr, 0L); - for (int i = 8; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - arr = null; - result = hashToBytes(arr, 0L); - Assert.assertEquals(result, null); - - arr = new char[0]; - result = hashToBytes(arr, 0L); - Assert.assertEquals(result, null); - } - - @Test - public void checkToBytesByteArr() { - byte[] arr = { 1, 2 }; - byte[] result = hashToBytes(arr, 0L); - for (int i = 8; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - arr = null; - result = hashToBytes(arr, 0L); - Assert.assertEquals(result, null); - - arr = new byte[0]; - result = hashToBytes(arr, 0L); - Assert.assertEquals(result, null); - - } - - @Test - public void checkToBytesDouble() { - byte[] result = hashToBytes(1.0, 0L); - for (int i = 8; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - result = hashToBytes(0.0, 0L); - for (int i = 8; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - result = hashToBytes( -0.0, 0L); - for (int i = 8; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - } - - @Test - public void checkToBytesString() { - byte[] result = hashToBytes("1", 0L); - for (int i = 8; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - result = hashToBytes("", 0L); - Assert.assertEquals(result, null); - - String s = null; - result = hashToBytes(s, 0L); - Assert.assertEquals(result, null); - } - - /************/ - - @Test - public void checkToLongsLong() { - long[] result = hashToLongs(2L, 0L); - for (int i = 2; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - } - - @Test - public void checkToLongsLongArr() { - long[] arr = { 1L, 2L }; - long[] result = hashToLongs(arr, 0L); - for (int i = 2; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - arr = null; - result = hashToLongs(arr, 0L); - Assert.assertEquals(result, null); - - arr = new long[0]; - result = hashToLongs(arr, 0L); - Assert.assertEquals(result, null); - } - - @Test - public void checkToLongsIntArr() { - int[] arr = { 1, 2 }; - long[] result = hashToLongs(arr, 0L); - for (int i = 2; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - arr = null; - result = hashToLongs(arr, 0L); - Assert.assertEquals(result, null); - - arr = new int[0]; - result = hashToLongs(arr, 0L); - Assert.assertEquals(result, null); - } - - @Test - public void checkToLongsCharArr() { - char[] arr = { 1, 2 }; - long[] result = hashToLongs(arr, 0L); - for (int i = 2; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - arr = null; - result = hashToLongs(arr, 0L); - Assert.assertEquals(result, null); - - arr = new char[0]; - result = hashToLongs(arr, 0L); - Assert.assertEquals(result, null); - } - - @Test - public void checkToLongsByteArr() { - byte[] arr = { 1, 2 }; - long[] result = hashToLongs(arr, 0L); - for (int i = 2; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - arr = null; - result = hashToLongs(arr, 0L); - Assert.assertEquals(result, null); - - arr = new byte[0]; - result = hashToLongs(arr, 0L); - Assert.assertEquals(result, null); - - } - - @Test - public void checkToLongsDouble() { - long[] result = hashToLongs(1.0, 0L); - for (int i = 2; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - result = hashToLongs(0.0, 0L); - for (int i = 2; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - result = hashToLongs( -0.0, 0L); - for (int i = 2; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - } - - @Test - public void checkToLongsString() { - long[] result = hashToLongs("1", 0L); - for (int i = 2; i-- > 0;) { - Assert.assertNotEquals(result[i], 0); - } - result = hashToLongs("", 0L); - Assert.assertEquals(result, null); - String s = null; - result = hashToLongs(s, 0L); - Assert.assertEquals(result, null); - } - - /*************/ - - @Test - public void checkModulo() { - int div = 7; - for (int i = 20; i-- > 0;) { - long[] out = hashToLongs(i, 9001); - int mod = modulo(out[0], out[1], div); - Assert.assertTrue((mod < div) && (mod >= 0)); - mod = modulo(out, div); - Assert.assertTrue((mod < div) && (mod >= 0)); - } - } - - @Test - public void checkAsDouble() { - for (int i = 0; i < 10000; i++ ) { - double result = asDouble(hashToLongs(i, 0)); - Assert.assertTrue((result >= 0) && (result < 1.0)); - } - } - - //Check asInt() functions - - @Test - public void checkAsInt() { - int lo = (3 << 28); - int hi = (1 << 30) + 1; - for (byte i = 0; i < 126; i++ ) { - long[] longArr = {i, i+1}; //long[] - int result = asInt(longArr, lo); - Assert.assertTrue((result >= 0) && (result < lo)); - result = asInt(longArr, hi); - Assert.assertTrue((result >= 0) && (result < hi)); - - int[] intArr = {i, i+1}; //int[] - result = asInt(intArr, lo); - Assert.assertTrue((result >= 0) && (result < lo)); - result = asInt(intArr, hi); - Assert.assertTrue((result >= 0) && (result < hi)); - - byte[] byteArr = {i, (byte)(i+1)}; //byte[] - result = asInt(byteArr, lo); - Assert.assertTrue((result >= 0) && (result < lo)); - result = asInt(byteArr, hi); - Assert.assertTrue((result >= 0) && (result < hi)); - - long longV = i; //long - result = asInt(longV, lo); - Assert.assertTrue((result >= 0) && (result < lo)); - result = asInt(longV, hi); - Assert.assertTrue((result >= 0) && (result < hi)); - - double v = i; //double - result = asInt(v, lo); - Assert.assertTrue((result >= 0) && (result < lo)); - result = asInt(v, hi); - Assert.assertTrue((result >= 0) && (result < hi)); - - String s = Integer.toString(i); //String - result = asInt(s, lo); - Assert.assertTrue((result >= 0) && (result < lo)); - result = asInt(s, hi); - Assert.assertTrue((result >= 0) && (result < hi)); - } - } - - @Test (expectedExceptions = SketchesArgumentException.class) - public void checkAsIntCornerCaseLongNull() { - long[] arr = null; - asInt(arr, 1000); - } - - @Test (expectedExceptions = SketchesArgumentException.class) - public void checkAsIntCornerCaseLongEmpty() { - long[] arr = new long[0]; - asInt(arr, 1000); - } - - @Test (expectedExceptions = SketchesArgumentException.class) - public void checkAsIntCornerCaseIntNull() { - int[] arr = null; - asInt(arr, 1000); - } - - @Test (expectedExceptions = SketchesArgumentException.class) - public void checkAsIntCornerCaseIntEmpty() { - int[] arr = new int[0]; - asInt(arr, 1000); - } - - @Test (expectedExceptions = SketchesArgumentException.class) - public void checkAsIntCornerCaseByteNull() { - byte[] arr = null; - asInt(arr, 1000); - } - - @Test (expectedExceptions = SketchesArgumentException.class) - public void checkAsIntCornerCaseByteEmpty() { - byte[] arr = new byte[0]; - asInt(arr, 1000); - } - - @Test (expectedExceptions = SketchesArgumentException.class) - public void checkAsIntCornerCaseStringNull() { - String s = null; - asInt(s, 1000); - } - - @Test (expectedExceptions = SketchesArgumentException.class) - public void checkAsIntCornerCaseStringEmpty() { - String s = ""; - asInt(s, 1000); - } - - @Test (expectedExceptions = SketchesArgumentException.class) - public void checkAsIntCornerCaseNTooSmall() { - String s = "abc"; - asInt(s, 1); - } - - @Test - public void printlnTest() { - println("PRINTING: "+this.getClass().getName()); - } - - /** - * @param s value to print - */ - static void println(String s) { - //System.out.println(s); //disable here - } - -} diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21Test.java similarity index 83% rename from src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java rename to src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21Test.java index 8699a091a..d9771d304 100644 --- a/src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java +++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21Test.java @@ -28,13 +28,12 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.internal.MurmurHash3v3; import org.apache.datasketches.memory.WritableMemory; /** * @author Lee Rhodes */ -public class MurmurHash3v3Test { +public class MurmurHash3FFM21Test { private Random rand = new Random(); private static final int trials = 1 << 20; @@ -154,37 +153,35 @@ private static final long[] hashV1(byte[] key, long seed) { } private static final long[] hashV2(long[] key, long seed) { - return MurmurHash3v3.hash(key, seed); + return MurmurHash3FFM21.hash(key, seed); } private static final long[] hashV2(int[] key2, long seed) { - return MurmurHash3v3.hash(key2, seed); + return MurmurHash3FFM21.hash(key2, seed); } private static final long[] hashV2(char[] key, long seed) { - return MurmurHash3v3.hash(key, seed); + return MurmurHash3FFM21.hash(key, seed); } private static final long[] hashV2(byte[] key, long seed) { - return MurmurHash3v3.hash(key, seed); + return MurmurHash3FFM21.hash(key, seed); } //V2 single primitives private static final long[] hashV2(long key, long seed, long[] out) { - return MurmurHash3v3.hash(key, seed, out); + return MurmurHash3FFM21.hash(key, seed, out); } // private static final long[] hashV2(double key, long seed, long[] out) { -// return MurmurHash3v3.hash(key, seed, out); +// return MurmurHash3v4.hash(key, seed, out); // } // private static final long[] hashV2(String key, long seed, long[] out) { -// return MurmurHash3v3.hash(key, seed, out); +// return MurmurHash3v4.hash(key, seed, out); // } - - @Test public void offsetChecks() { long seed = 12345; @@ -199,7 +196,7 @@ public void offsetChecks() { for (int offset = 0; offset < 16; offset++) { int arrLen = cap - offset; - hash1 = MurmurHash3v3.hash(wmem, offset, arrLen, seed, hash1); + hash1 = MurmurHash3FFM21.hash(wmem, offset, arrLen, seed, hash1); byte[] byteArr2 = new byte[arrLen]; wmem.getByteArray(offset, byteArr2, 0, arrLen); hash2 = MurmurHash3.hash(byteArr2, seed); @@ -222,8 +219,8 @@ public void byteArrChecks() { for (int i = 0; i < j; i++) { wmem.putByte(i, (byte) (-128 + i)); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v3.hash(in, seed); + hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3FFM21.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -246,8 +243,8 @@ public void charArrChecks() { for (int i = 0; i < j; i++) { wmem.putInt(i, i); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v3.hash(in, seed); + hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3FFM21.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -270,8 +267,8 @@ public void intArrChecks() { for (int i = 0; i < j; i++) { wmem.putInt(i, i); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v3.hash(in, seed); + hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3FFM21.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -294,8 +291,8 @@ public void longArrChecks() { for (int i = 0; i < j; i++) { wmem.putLong(i, i); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v3.hash(in, seed); + hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3FFM21.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -313,8 +310,8 @@ public void longCheck() { WritableMemory wmem = WritableMemory.writableWrap(in); long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v3.hash(in, seed); + hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3FFM21.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -325,57 +322,57 @@ public void checkEmptiesNulls() { long seed = 123; long[] hashOut = new long[2]; try { - MurmurHash3v3.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty + MurmurHash3FFM21.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty fail(); } catch (final IllegalArgumentException e) { } //OK try { String s = ""; - MurmurHash3v3.hash(s, seed, hashOut); //string empty + MurmurHash3FFM21.hash(s, seed, hashOut); //string empty fail(); } catch (final IllegalArgumentException e) { } //OK try { String s = null; - MurmurHash3v3.hash(s, seed, hashOut); //string null + MurmurHash3FFM21.hash(s, seed, hashOut); //string null fail(); } catch (final IllegalArgumentException e) { } //OK try { byte[] barr = new byte[0]; - MurmurHash3v3.hash(barr, seed); //byte[] empty + MurmurHash3FFM21.hash(barr, seed); //byte[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { byte[] barr = null; - MurmurHash3v3.hash(barr, seed); //byte[] null + MurmurHash3FFM21.hash(barr, seed); //byte[] null fail(); } catch (final IllegalArgumentException e) { } //OK try { char[] carr = new char[0]; - MurmurHash3v3.hash(carr, seed); //char[] empty + MurmurHash3FFM21.hash(carr, seed); //char[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { char[] carr = null; - MurmurHash3v3.hash(carr, seed); //char[] null + MurmurHash3FFM21.hash(carr, seed); //char[] null fail(); } catch (final IllegalArgumentException e) { } //OK try { int[] iarr = new int[0]; - MurmurHash3v3.hash(iarr, seed); //int[] empty + MurmurHash3FFM21.hash(iarr, seed); //int[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { int[] iarr = null; - MurmurHash3v3.hash(iarr, seed); //int[] null + MurmurHash3FFM21.hash(iarr, seed); //int[] null fail(); } catch (final IllegalArgumentException e) { } //OK try { long[] larr = new long[0]; - MurmurHash3v3.hash(larr, seed); //long[] empty + MurmurHash3FFM21.hash(larr, seed); //long[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { long[] larr = null; - MurmurHash3v3.hash(larr, seed); //long[] null + MurmurHash3FFM21.hash(larr, seed); //long[] null fail(); } catch (final IllegalArgumentException e) { } //OK } @@ -385,9 +382,9 @@ public void checkStringLong() { long seed = 123; long[] hashOut = new long[2]; String s = "123"; - assertTrue(MurmurHash3v3.hash(s, seed, hashOut)[0] != 0); + assertTrue(MurmurHash3FFM21.hash(s, seed, hashOut)[0] != 0); long v = 123; - assertTrue(MurmurHash3v3.hash(v, seed, hashOut)[0] != 0); + assertTrue(MurmurHash3FFM21.hash(v, seed, hashOut)[0] != 0); } @Test @@ -415,8 +412,8 @@ private static long[] checkDouble(double dbl) { WritableMemory wmem = WritableMemory.writableWrap(dataArr); long[] hash1 = MurmurHash3.hash(dataArr, 0); - hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v3.hash(dbl, seed, hash2); + hash2 = MurmurHash3FFM21.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3FFM21.hash(dbl, seed, hash2); assertEquals(hash1, hash2); assertEquals(hash1, hash3); diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java new file mode 100644 index 000000000..1e23772b5 --- /dev/null +++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3FFM21bTest.java @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.hash; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.datasketches.hash.MurmurHash3FFM21.hash; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.ByteOrder; + +import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.Resource; +import org.apache.datasketches.memory.WritableMemory; +import org.testng.Assert; +import org.testng.annotations.Test; + +/** + * Tests the MurmurHash3 against specific, known hash results given known + * inputs obtained from the public domain C++ version 150. + * + * @author Lee Rhodes + */ +public class MurmurHash3FFM21bTest { + private static final MemoryRequestServer memReqSvr = Resource.defaultMemReqSvr; + + @Test + public void checkByteArrRemainderGT8() { //byte[], remainder > 8 + String keyStr = "The quick brown fox jumps over the lazy dog"; + byte[] key = keyStr.getBytes(UTF_8); + long[] result = hash(key, 0); + //Should be: + long h1 = 0xe34bbc7bbc071b6cL; + long h2 = 0x7a433ca9c49a9347L; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + @Test + public void checkByteArrRemainderGT8withSegment() { //byte[], remainder > 8 + String keyStr = "The quick brown fox jumps over the lazy dog"; + byte[] key = keyStr.getBytes(UTF_8); + long[] out = new long[2]; + MemorySegment seg = MemorySegment.ofArray(key); + long[] result = hash(seg, 0, seg.byteSize(), 0, out); + //Should be: + long h1 = 0xe34bbc7bbc071b6cL; + long h2 = 0x7a433ca9c49a9347L; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + @Test + public void checkByteArrChange1bit() { //byte[], change one bit + String keyStr = "The quick brown fox jumps over the lazy eog"; + byte[] key = keyStr.getBytes(UTF_8); + long[] result = hash(key, 0); + //Should be: + long h1 = 0x362108102c62d1c9L; + long h2 = 0x3285cd100292b305L; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + @Test + public void checkByteArrRemainderLt8() { //byte[], test a remainder < 8 + String keyStr = "The quick brown fox jumps over the lazy dogdogdog"; + byte[] key = keyStr.getBytes(UTF_8); + long[] result = hash(key, 0); + //Should be; + long h1 = 0x9c8205300e612fc4L; + long h2 = 0xcbc0af6136aa3df9L; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + @Test + public void checkByteArrReaminderEQ8() { //byte[], test a remainder = 8 + String keyStr = "The quick brown fox jumps over the lazy1"; + byte[] key = keyStr.getBytes(UTF_8); + long[] result = hash(key, 0); + //Should be: + long h1 = 0xe3301a827e5cdfe3L; + long h2 = 0xbdbf05f8da0f0392L; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + /** + * This test should have the exact same output as Test4 + */ + @Test + public void checkLongArrRemainderEQ8() { //long[], test a remainder = 8 + String keyStr = "The quick brown fox jumps over the lazy1"; + long[] key = stringToLongs(keyStr); + long[] result = hash(key, 0); + //Should be: + long h1 = 0xe3301a827e5cdfe3L; + long h2 = 0xbdbf05f8da0f0392L; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + /** + * This test should have the exact same output as Test4 + */ + @Test + public void checkIntArrRemainderEQ8() { //int[], test a remainder = 8 + String keyStr = "The quick brown fox jumps over the lazy1"; //40B + int[] key = stringToInts(keyStr); + long[] result = hash(key, 0); + //Should be: + long h1 = 0xe3301a827e5cdfe3L; + long h2 = 0xbdbf05f8da0f0392L; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + @Test + public void checkIntArrRemainderEQ0() { //int[], test a remainder = 0 + String keyStr = "The quick brown fox jumps over t"; //32B + int[] key = stringToInts(keyStr); + long[] result = hash(key, 0); + //Should be: + long h1 = 0xdf6af91bb29bdacfL; + long h2 = 0x91a341c58df1f3a6L; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + /** + * Tests an odd remainder of int[]. + */ + @Test + public void checkIntArrOddRemainder() { //int[], odd remainder + String keyStr = "The quick brown fox jumps over the lazy dog"; //43B + int[] key = stringToInts(keyStr); + long[] result = hash(key, 0); + //Should be: + long h1 = 0x1eb232b0087543f5L; + long h2 = 0xfc4c1383c3ace40fL; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + /** + * Tests an odd remainder of int[]. + */ + @Test + public void checkCharArrOddRemainder() { //char[], odd remainder + String keyStr = "The quick brown fox jumps over the lazy dog.."; //45B + char[] key = keyStr.toCharArray(); + long[] result = hash(key, 0); + //Should be: + long h1 = 0xca77b498ea9ed953L; + long h2 = 0x8b8f8ec3a8f4657eL; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + /** + * Tests an odd remainder of int[]. + */ + @Test + public void checkCharArrRemainderEQ0() { //char[], remainder of 0 + String keyStr = "The quick brown fox jumps over the lazy "; //40B + char[] key = keyStr.toCharArray(); + long[] result = hash(key, 0); + //Should be: + long h1 = 0x51b15e9d0887f9f1L; + long h2 = 0x8106d226786511ebL; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + @Test + public void checkByteArrAllOnesZeros() { //byte[], test a ones byte and a zeros byte + byte[] key = { + 0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69, 0x63, 0x6b, 0x20, 0x62, 0x72, 0x6f, 0x77, 0x6e, + 0x20, 0x66, 0x6f, 0x78, 0x20, 0x6a, 0x75, 0x6d, 0x70, 0x73, 0x20, 0x6f, 0x76, 0x65, + 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x6c, 0x61, 0x7a, 0x79, 0x20, 0x64, 0x6f, 0x67, + (byte) 0xff, 0x64, 0x6f, 0x67, 0x00 + }; + long[] result = MurmurHash3FFM21.hash(key, 0); + //Should be: + long h1 = 0xe88abda785929c9eL; + long h2 = 0x96b98587cacc83d6L; + Assert.assertEquals(result[0], h1); + Assert.assertEquals(result[1], h2); + } + + /** + * This test demonstrates that the hash of byte[], char[], int[], or long[] will produce the + * same hash result if, and only if, all the arrays have the same exact length in bytes, and if + * the contents of the values in the arrays have the same byte endianness and overall order. + */ + @Test + public void checkCrossTypeHashConsistency() { + long[] out; + println("Bytes"); + byte[] bArr = {1,2,3,4,5,6,7,8, 9,10,11,12,13,14,15,16, 17,18,19,20,21,22,23,24}; + long[] out1 = hash(bArr, 0L); + println(longToHexBytes(out1[0])); + println(longToHexBytes(out1[1])); + + println("Chars"); + char[] cArr = {0X0201, 0X0403, 0X0605, 0X0807, 0X0a09, 0X0c0b, 0X0e0d, 0X100f, + 0X1211, 0X1413, 0X1615, 0X1817}; + out = hash(cArr, 0L); + Assert.assertEquals(out, out1); + println(longToHexBytes(out[0])); + println(longToHexBytes(out[1])); + + println("Ints"); + int[] iArr = {0X04030201, 0X08070605, 0X0c0b0a09, 0X100f0e0d, 0X14131211, 0X18171615}; + out = hash(iArr, 0L); + Assert.assertEquals(out, out1); + println(longToHexBytes(out[0])); + println(longToHexBytes(out[1])); + + println("Longs"); + long[] lArr = {0X0807060504030201L, 0X100f0e0d0c0b0a09L, 0X1817161514131211L}; + out = hash(lArr, 0L); + Assert.assertEquals(out, out1); + println(longToHexBytes(out[0])); + println(longToHexBytes(out[1])); + } + + @Test + public void checkEmptyOrNullExceptions() { + try { + long[] arr = null; + hash(arr, 1L); + fail(); + } + catch (final IllegalArgumentException e) { } + try { + int[] arr = null; hash(arr, 1L); fail(); + } catch (final IllegalArgumentException e) { } + try { + char[] arr = null; hash(arr, 1L); fail(); + } catch (final IllegalArgumentException e) { } + try { + byte[] arr = null; hash(arr, 1L); fail(); + } catch (final IllegalArgumentException e) { } + + long[] out = new long[2]; + try { + String in = null; hash(in, 1L, out); fail(); + } catch (final IllegalArgumentException e) { } + try { + Memory mem = Memory.wrap(new byte[0]); + hash(mem, 0L, 4L, 1L, out); + } catch (final IllegalArgumentException e) { } + try (Arena arena = Arena.ofConfined()) { + Memory mem = WritableMemory.allocateDirect(8, 1, ByteOrder.nativeOrder(), memReqSvr, arena); + out = hash(mem, 0L, 4L, 1L, out); + } + assertTrue((out[0] != 0) && (out[1] != 0)); + } + + @Test + public void checkHashTails() { + long[] out = new long[2]; + WritableMemory mem = WritableMemory.allocate(32); + mem.fill((byte)85); + + for (int i = 16; i <= 32; i++) { + out = hash(mem, 0, i, 1L, out); + } + } + + @Test + public void checkSinglePrimitives() { + long[] out = new long[2]; + out = hash(1L, 1L, out); + out = hash(0.0, 1L, out); + out = hash("123", 1L, out); + } + + //Helper methods + + private static long[] stringToLongs(String in) { + byte[] bArr = in.getBytes(UTF_8); + int inLen = bArr.length; + int outLen = (inLen / 8) + (((inLen % 8) != 0) ? 1 : 0); + long[] out = new long[outLen]; + + for (int i = 0; i < (outLen - 1); i++ ) { + for (int j = 0; j < 8; j++ ) { + out[i] |= ((bArr[(i * 8) + j] & 0xFFL) << (j * 8)); + } + } + int inTail = 8 * (outLen - 1); + int rem = inLen - inTail; + for (int j = 0; j < rem; j++ ) { + out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8)); + } + return out; + } + + private static int[] stringToInts(String in) { + byte[] bArr = in.getBytes(UTF_8); + int inLen = bArr.length; + int outLen = (inLen / 4) + (((inLen % 4) != 0) ? 1 : 0); + int[] out = new int[outLen]; + + for (int i = 0; i < (outLen - 1); i++ ) { + for (int j = 0; j < 4; j++ ) { + out[i] |= ((bArr[(i * 4) + j] & 0xFFL) << (j * 8)); + } + } + int inTail = 4 * (outLen - 1); + int rem = inLen - inTail; + for (int j = 0; j < rem; j++ ) { + out[outLen - 1] |= ((bArr[inTail + j] & 0xFFL) << (j * 8)); + } + return out; + } + + /** + * Returns a string of spaced hex bytes in Big-Endian order. + * @param v the given long + * @return string of spaced hex bytes in Big-Endian order. + */ + private static String longToHexBytes(final long v) { + final long mask = 0XFFL; + final StringBuilder sb = new StringBuilder(); + for (int i = 8; i-- > 0; ) { + final String s = Long.toHexString((v >>> (i * 8)) & mask); + sb.append(zeroPad(s, 2)).append(" "); + } + return sb.toString(); + } + + /** + * Prepend the given string with zeros. If the given string is equal or greater than the given + * field length, it will be returned without modification. + * @param s the given string + * @param fieldLength desired total field length including the given string + * @return the given string prepended with zeros. + */ + private static final String zeroPad(final String s, final int fieldLength) { + return characterPad(s, fieldLength, '0', false); + } + + /** + * Prepend or postpend the given string with the given character to fill the given field length. + * If the given string is equal or greater than the given field length, it will be returned + * without modification. + * @param s the given string + * @param fieldLength the desired field length + * @param padChar the desired pad character + * @param postpend if true append the padCharacters to the end of the string. + * @return prepended or postpended given string with the given character to fill the given field + * length. + */ + private static final String characterPad(final String s, final int fieldLength, final char padChar, + final boolean postpend) { + final char[] chArr = s.toCharArray(); + final int sLen = chArr.length; + if (sLen < fieldLength) { + final char[] out = new char[fieldLength]; + final int blanks = fieldLength - sLen; + + if (postpend) { + for (int i = 0; i < sLen; i++) { + out[i] = chArr[i]; + } + for (int i = sLen; i < fieldLength; i++) { + out[i] = padChar; + } + } else { //prepend + for (int i = 0; i < blanks; i++) { + out[i] = padChar; + } + for (int i = blanks; i < fieldLength; i++) { + out[i] = chArr[i - blanks]; + } + } + + return String.valueOf(out); + } + return s; + } + + @Test + public void printlnTest() { + println("PRINTING: "+this.getClass().getName()); + } + + /** + * @param s value to print + */ + static void println(String s) { + //System.out.println(s); //disable here + } + +} diff --git a/src/test/java/org/apache/datasketches/hll/CouponListTest.java b/src/test/java/org/apache/datasketches/hll/CouponListTest.java index dbd5bfb0b..774e2c304 100644 --- a/src/test/java/org/apache/datasketches/hll/CouponListTest.java +++ b/src/test/java/org/apache/datasketches/hll/CouponListTest.java @@ -42,7 +42,7 @@ private static void checkIterator(boolean direct, int lgK, int n) { TgtHllType tgtHllType = TgtHllType.HLL_4; int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType); WritableMemory wmem = WritableMemory.allocate(bytes); - HllSketch sk = (direct) ? new HllSketch(lgK, tgtHllType, wmem) : new HllSketch(lgK); + HllSketch sk = (direct) ? new HllSketch(lgK, tgtHllType, wmem) : new HllSketch(lgK, tgtHllType); for (int i = 0; i < n; i++) { sk.update(i); } String store = direct ? "Memory" : "Heap"; println("CurMode: " + sk.getCurMode().toString() + "; Store: " + store); diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java index 75504bc83..ecebe4539 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java @@ -26,11 +26,10 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import java.nio.ByteOrder; +import java.lang.foreign.Arena; import java.util.HashMap; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; @@ -47,7 +46,7 @@ public void checkGrow() { int n = 8; //put lgConfigK == 4 into HLL mode long bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType); HllSketch hllSketch; - WritableMemory wmem = WritableMemory.allocateDirect(bytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmem = WritableMemory.allocateDirect(bytes, Arena.ofConfined()); hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); for (int i = 0; i < n; i++) { @@ -56,7 +55,7 @@ public void checkGrow() { hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values hllSketch.couponUpdate(HllUtil.pair(8, 15)); hllSketch.couponUpdate(HllUtil.pair(9, 15)); - //println(hllSketch.toString(true, true, true, true)); + println(hllSketch.toString(true, true, true, true)); DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl; assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2); assertTrue(hllSketch.isMemory()); @@ -74,14 +73,14 @@ public void checkGrow() { byteArray = hllSketch.toUpdatableByteArray(); WritableMemory wmem2 = WritableMemory.writableWrap(byteArray); hllSketch2 = HllSketch.writableWrap(wmem2); - //println(hllSketch2.toString(true, true, true, true)); + println(hllSketch2.toString(true, true, true, true)); DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl; assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2); assertEquals(dha2.getAuxHashMap().getAuxCount(), 3); //Check grow to on-heap hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow - //println(hllSketch.toString(true, true, true, true)); + println(hllSketch.toString(true, true, true, true)); dha = (DirectHllArray) hllSketch.hllSketchImpl; assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3); assertEquals(dha.getAuxHashMap().getAuxCount(), 4); diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java index 1b00e0301..6da2f05fe 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java @@ -25,12 +25,11 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; -import org.testng.annotations.Test; - -import jdk.incubator.foreign.ResourceScope; +import java.lang.foreign.Arena; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; /** * @author Lee Rhodes @@ -71,7 +70,8 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool //println("DIRECT"); byte[] barr1; WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes)).scope()) { + try (Arena arena = Arena.ofConfined()) { + wmem = WritableMemory.allocateDirect(bytes, arena); hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); assertTrue(hllSketch.isEmpty()); diff --git a/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java new file mode 100644 index 000000000..ad7a4e509 --- /dev/null +++ b/src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.hll; + +import static org.testng.Assert.assertEquals; + +import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; + +public class SizeAndModeTransitions { + static String hfmt = "%6s %7s %4s %10s %5s %10s %10s %10s %10s %10s %14s\n"; + static String dfmt = "%6s %7s %4d %,10d %5s %,10d %,10d %,10d %,10d %,10d %,14.3f\n"; + static String[] hdr = {"Type", "Store", "LgK", "N", "Mode", "ActCBytes", "CmpBytes", "ActUBytes", "UpdBytes", "MaxBytes", "Estimate"}; + + @Test + public void checkHLL8Heap() { + TgtHllType tgtHllType = TgtHllType.HLL_8; + boolean direct = false; + int lgK = 10; + int N = 97; + printf(hfmt, (Object[]) hdr); + int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType); + WritableMemory wmem = null;; + HllSketch sk; + if (direct) { + wmem = WritableMemory.allocate(maxBytes); + sk = new HllSketch(lgK, tgtHllType, wmem); + } else { + sk = new HllSketch(lgK, tgtHllType); + } + String type = tgtHllType.toString(); + String store = direct ? "Memory" : "Heap"; + for (int i = 1; i <= N; i++) { + sk.update(i); + if (i == 7) { checkAtN(sk, tgtHllType, store, lgK, i, "LIST", 36, 40, 1064); } + if (i == 8) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 44, 140, 1064); } + if (i == 24) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 108, 140, 1064); } + if (i == 25) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 112, 268, 1064); } + if (i == 48) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 204, 268, 1064); } + if (i == 49) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 208, 524, 1064); } + if (i == 96) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 396, 524, 1064); } + if (i == 97) { checkAtN(sk, tgtHllType, store, lgK, i, "HLL", 1064, 1064, 1064); } + } + println(""); + } + + private static void checkAtN(HllSketch sk, TgtHllType tgtHllType, String store, int lgK, int n, String trueMode, + int cmpTrueBytes, int updTrueBytes, int maxTrueBytes) { + int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType); + String type = tgtHllType.toString(); + int cmpBytes = sk.getCompactSerializationBytes(); + int updBytes = sk.getUpdatableSerializationBytes(); + byte[] actCBytes = sk.toCompactByteArray(); + byte[] actUBytes = sk.toUpdatableByteArray(); + String mode = sk.getCurMode().toString(); + double est = sk.getEstimate(); + printf(dfmt, type, store, lgK, n, mode, actCBytes.length, cmpBytes, actUBytes.length, updBytes, maxBytes, est); + assertEquals(mode, trueMode); + assertEquals(cmpBytes, actCBytes.length); + assertEquals(cmpBytes, cmpTrueBytes); + assertEquals(updBytes, actUBytes.length); + assertEquals(updBytes, updTrueBytes); + assertEquals(maxBytes, maxTrueBytes); + } + + @Test + public void printlnTest() { + println("PRINTING: "+this.getClass().getName()); + } + + /** + * @param s value to print + */ + static void println(String s) { + //System.out.println(s); //disable here + } + + /** + * @param fmt format + * @param args arguments + */ + static void printf(String fmt, Object...args) { + //System.out.printf(fmt, args); //disable here + } + +} diff --git a/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java index 9fc74d97b..2e6cc1ea1 100644 --- a/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllItemsSketchTest.java @@ -599,15 +599,18 @@ public void checkReadOnlyExceptions() { @Test public void checkIsSameResource() { int cap = 128; - WritableMemory wmem = WritableMemory.allocate(cap); + WritableMemory wmem = WritableMemory.allocate(cap); //heap WritableMemory reg1 = wmem.writableRegion(0, 64); WritableMemory reg2 = wmem.writableRegion(64, 64); assertFalse(reg1 == reg2); - assertFalse(reg1.isSameResource(reg2)); + assertFalse(reg1.isSameResource(reg2)); //same original resource, but different offsets WritableMemory reg3 = wmem.writableRegion(0, 64); assertFalse(reg1 == reg3); - assertTrue(reg1.isSameResource(reg3)); + assertTrue(reg1.isSameResource(reg3)); //same original resource, same offsets, different views. + reg1.putInt(0, -1); + assertEquals(-1, reg3.getInt(0)); //proof + reg1.putInt(0, 0); byte[] byteArr1 = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe).toByteArray(); reg1.putByteArray(0, byteArr1, 0, byteArr1.length); @@ -615,11 +618,11 @@ public void checkIsSameResource() { byte[] byteArr2 = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe).toByteArray(); reg2.putByteArray(0, byteArr2, 0, byteArr2.length); - assertFalse(sk1.isSameResource(reg2)); + assertFalse(sk1.isSameResource(reg2)); //same original resource, but different offsets byte[] byteArr3 = KllItemsSketch.newHeapInstance(20, Comparator.naturalOrder(), serDe).toByteArray(); reg3.putByteArray(0, byteArr3, 0, byteArr3.length); - assertTrue(sk1.isSameResource(reg3)); + assertTrue(sk1.isSameResource(reg3)); //same original resource, same offsets, different views. } // New added tests specially for KllItemsSketch diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java index e51be4b20..2b67f9772 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java @@ -27,7 +27,8 @@ import org.testng.annotations.Test; -import jdk.incubator.foreign.ResourceScope; +import java.lang.foreign.Arena; +import java.nio.ByteOrder; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIterator; @@ -63,8 +64,8 @@ public void test() { DoublesSketch.setRandom(1); //make deterministic for test DoublesUnion dUnion; DoublesSketch dSketch; - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(10_000_000)).scope()) { + try (Arena arena = Arena.ofConfined()) { + WritableMemory wmem = WritableMemory.allocateDirect(10_000_000, arena); dUnion = DoublesUnion.builder().setMaxK(8).build(wmem); for (int s = 0; s < numSketches; s++) { dUnion.union(sketchArr[s]); } dSketch = dUnion.getResult(); //result is on heap diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index 5195f4187..b61171ca1 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -24,6 +24,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import java.lang.foreign.Arena; import java.nio.ByteOrder; import org.apache.datasketches.memory.DefaultMemoryRequestServer; @@ -47,7 +48,7 @@ public void checkLimitedMemoryScenarios() { //Requesting application //########## Owning Implementation // This part would actually be part of the Memory owning implementation so it is faked here - WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined()); WritableMemory wmem2 = wmem; println("Initial mem size: " + wmem.getCapacity()); @@ -80,7 +81,7 @@ public void checkGrowBaseBuf() { final int u = 32; // don't need the BB to fill here final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything - WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined()); WritableMemory wmem2 = wmem; println("Initial mem size: " + wmem.getCapacity()); final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); @@ -99,7 +100,7 @@ public void checkGrowCombBuf() { final int u = (2 * k) - 1; //just to fill the BB final int initBytes = ((2 * k) + 4) << 3; //just room for BB - WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined()); WritableMemory wmem2 = wmem; println("Initial mem size: " + wmem.getCapacity()); final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); @@ -115,15 +116,29 @@ public void checkGrowCombBuf() { assertFalse(wmem2.isAlive()); } + @Test + public void checkUpdatableStorageBytes() { + final int k = 16; + final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, 1); + println("Predicted Updatable Storage Bytes: " + initBytes); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); + usk1.update(1.0); + byte[] uarr = usk1.toByteArray(); + println("Actual Storage Bytes " + uarr.length); + assertEquals(initBytes, uarr.length); + assertEquals(initBytes, 64); + } + + @Test public void checkGrowFromWrappedEmptySketch() { final int k = 16; final int n = 0; - final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n); //8 bytes + final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n); //empty: 8 bytes final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); - final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); + final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); //on heap - WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, Arena.ofConfined()); //off heap WritableMemory wmem2 = wmem; origSketchMem.copyTo(0, wmem, 0, initBytes); UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); @@ -133,8 +148,8 @@ public void checkGrowFromWrappedEmptySketch() { assertTrue(usk2.isEmpty()); //update the sketch forcing it to grow on-heap - for (int i = 1; i <= 5; i++) { usk2.update(i); } - assertEquals(usk2.getN(), 5); + usk2.update(1.0); + assertEquals(usk2.getN(), 1); WritableMemory mem2 = usk2.getMemory(); assertFalse(wmem.isSameResource(mem2)); assertFalse(mem2.isDirect()); //should now be on-heap diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index 8cdc7bf71..762cfc057 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -34,7 +34,7 @@ import org.testng.Assert; import org.testng.annotations.Test; -import jdk.incubator.foreign.ResourceScope; +import java.lang.foreign.Arena; public class DoublesSketchTest { @@ -141,7 +141,7 @@ public void checkEmptyExceptions() { @Test public void directSketchShouldMoveOntoHeapEventually() { - WritableMemory wmem = WritableMemory.allocateDirect(1000, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmem = WritableMemory.allocateDirect(1000, Arena.ofConfined()); WritableMemory wmem2 = wmem; UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); Assert.assertTrue(sketch.isSameResource(wmem)); @@ -155,7 +155,7 @@ public void directSketchShouldMoveOntoHeapEventually() { @Test public void directSketchShouldMoveOntoHeapEventually2() { int i = 0; - WritableMemory wmem = WritableMemory.allocateDirect(50, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + WritableMemory wmem = WritableMemory.allocateDirect(50, Arena.ofConfined()); WritableMemory wmem2 = wmem; UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); Assert.assertTrue(sketch.isSameResource(wmem)); @@ -172,10 +172,8 @@ public void directSketchShouldMoveOntoHeapEventually2() { @Test public void checkEmptyDirect() { - WritableMemory wmem ; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - + try (Arena arena = Arena.ofConfined()) { + WritableMemory wmem = WritableMemory.allocateDirect(1000, Arena.ofConfined()); UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); sketch.toByteArray(); //exercises a specific path } catch (final Exception e) { diff --git a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java index adf916ef3..765d4d037 100644 --- a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java @@ -37,24 +37,19 @@ import static org.apache.datasketches.quantiles.PreambleUtil.insertSerVer; import static org.testng.Assert.assertEquals; -import java.nio.ByteOrder; +import java.lang.foreign.Arena; -import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; -import jdk.incubator.foreign.ResourceScope; - public class PreambleUtilTest { @Test public void checkInsertsAndExtracts() { final int bytes = 32; - WritableMemory offHeapMem; - try (ResourceScope scope = (offHeapMem = WritableMemory.allocateDirect(bytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - + try (Arena arena = Arena.ofConfined()) { + WritableMemory offHeapMem = WritableMemory.allocateDirect(bytes, arena); final WritableMemory onHeapMem = WritableMemory.writableWrap(new byte[bytes]); onHeapMem.clear(); diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java index e0f79087b..cc5be6a85 100644 --- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java @@ -32,7 +32,7 @@ import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; -import jdk.incubator.foreign.ResourceScope; +import java.lang.foreign.Arena; /** * @author Lee Rhodes @@ -79,8 +79,8 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) { //Prepare Memory for direct int bytes = usk.getCompactBytes(); //for Compact - WritableMemory directMem; - try (ResourceScope scope = (directMem = WritableMemory.allocateDirect(bytes)).scope()) { + try (Arena arena = Arena.ofConfined()) { + WritableMemory directMem = WritableMemory.allocateDirect(bytes, arena); /**Via CompactSketch.compact**/ refSk = usk.compact(ordered, directMem); diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index f36597b7c..5f608113a 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -38,6 +38,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.foreign.Arena; import java.nio.ByteOrder; import java.util.Arrays; @@ -52,8 +53,6 @@ import org.apache.datasketches.thetacommon.ThetaUtil; import org.testng.annotations.Test; -import jdk.incubator.foreign.ResourceScope; - /** * @author Lee Rhodes */ @@ -62,8 +61,8 @@ public class DirectQuickSelectSketchTest { @Test//(expectedExceptions = SketchesArgumentException.class) public void checkBadSerVer() { int k = 512; - WritableMemory wmem; - try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) { + try (Arena arena = Arena.ofConfined()) { + WritableMemory wmem = makeNativeMemory(k, arena); UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(wmem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks @@ -88,8 +87,8 @@ public void checkBadSerVer() { @Test public void checkConstructorKtooSmall() { int k = 8; - WritableMemory wmem; - try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) { + try (Arena arena = Arena.ofConfined()) { + WritableMemory wmem = makeNativeMemory(k, arena); UpdateSketch.builder().setNominalEntries(k).build(wmem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} @@ -100,8 +99,8 @@ public void checkConstructorKtooSmall() { @Test public void checkConstructorMemTooSmall() { int k = 16; - WritableMemory wmem; - try (ResourceScope scope = (wmem = makeNativeMemory(k/2)).scope()) { + try (Arena arena = Arena.ofConfined()) { + WritableMemory wmem = makeNativeMemory(k/2, arena); UpdateSketch.builder().setNominalEntries(k).build(wmem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} @@ -126,10 +125,8 @@ public void checkHeapifyIllegalFamilyID_heapify() { public void checkHeapifyMemoryEstimating() { int k = 512; int u = 2*k; //thus estimating - - WritableMemory wmem; - try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) { - + try (Arena arena = Arena.ofConfined()) { + WritableMemory wmem = makeNativeMemory(k, arena); UpdateSketch sk1 = UpdateSketch.builder().setNominalEntries(k).build(wmem); for (int i=0; i