HADOOP-19875. Further improve CrcComposer performance. (#8497)

Author: szetszwo

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java

@@ -26,7 +26,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
-import java.util.function.ToIntFunction;
+import java.util.function.LongToIntFunction;
 
 /**
  * Encapsulates logic for composing multiple CRCs into one or more combined CRCs
@@ -40,7 +40,7 @@ public final class CrcComposer {
   private static final int CRC_SIZE_BYTES = 4;
   private static final Logger LOG = LoggerFactory.getLogger(CrcComposer.class);
 
-  private final ToIntFunction<Long> mod;
+  private final LongToIntFunction mod;
   private final int precomputedMonomialForHint;
   private final long bytesPerCrcHint;
   private final long stripeLength;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java

@@ -22,7 +22,7 @@
 import org.apache.hadoop.classification.InterfaceStability;
 
 import java.util.Arrays;
-import java.util.function.ToIntFunction;
+import java.util.function.LongToIntFunction;
 
 /**
  * This class provides utilities for working with CRCs.
@@ -39,7 +39,7 @@ public final class CrcUtil {
    * @return a * b (mod p),
    *         where mod p is computed by the given mod function.
    */
-  static int multiplyMod(int a, int b, ToIntFunction<Long> mod) {
+  static int multiplyMod(int a, int b, LongToIntFunction mod) {
     final long left  = ((long)a) << 32;
     final long right = ((long)b) << 32;
 
@@ -98,7 +98,7 @@ private CrcUtil() {
    * @param mod mod.
    * @return monomial.
    */
-  public static int getMonomial(long lengthBytes, ToIntFunction<Long> mod) {
+  public static int getMonomial(long lengthBytes, LongToIntFunction mod) {
     if (lengthBytes == 0) {
       return MULTIPLICATIVE_IDENTITY;
     } else if (lengthBytes < 0) {
@@ -135,7 +135,7 @@ public static int getMonomial(long lengthBytes, ToIntFunction<Long> mod) {
    * @return compose with monomial.
    */
   public static int composeWithMonomial(
-      int crcA, int crcB, int monomial, ToIntFunction<Long> mod) {
+      int crcA, int crcB, int monomial, LongToIntFunction mod) {
     return multiplyMod(crcA, monomial, mod) ^ crcB;
   }
 
@@ -148,7 +148,7 @@ public static int composeWithMonomial(
    * @param mod mod.
    * @return compose result.
    */
-  public static int compose(int crcA, int crcB, long lengthB, ToIntFunction<Long> mod) {
+  public static int compose(int crcA, int crcB, long lengthB, LongToIntFunction mod) {
     int monomial = getMonomial(lengthB, mod);
     return composeWithMonomial(crcA, crcB, monomial, mod);
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java

@@ -22,7 +22,7 @@
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.function.ToIntFunction;
+import java.util.function.LongToIntFunction;
 import java.util.zip.CRC32;
 import java.util.zip.CRC32C;
 import java.util.zip.Checksum;
@@ -97,7 +97,7 @@ static Checksum newCrc32C() {
    * @return the int representation of the polynomial associated with the
    *     CRC {@code type}, suitable for use with further CRC arithmetic.
    */
-  static ToIntFunction<Long> getModFunction(Type type) {
+  static LongToIntFunction getModFunction(Type type) {
     switch (type) {
     case CRC32:
       return PureJavaCrc32::mod;

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCrcUtil.java

@@ -19,7 +19,7 @@
 
 import java.util.Objects;
 import java.util.Random;
-import java.util.function.ToIntFunction;
+import java.util.function.LongToIntFunction;
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
@@ -91,7 +91,7 @@ public void testComposeCrc32CZeroLength() {
    */
   private static void doTestComposeCrc(
       byte[] data, DataChecksum.Type type, int chunkSize, boolean useMonomial) {
-    final ToIntFunction<Long> mod = DataChecksum.getModFunction(type);
+    final LongToIntFunction mod = DataChecksum.getModFunction(type);
 
     // Get full end-to-end CRC in a single shot first.
     DataChecksum checksum = DataChecksum.newDataChecksum(
@@ -142,7 +142,7 @@ private static void doTestComposeCrcZerolength(DataChecksum.Type type) {
     // Without loss of generality, we can pick any integer as our fake crcA
     // even if we don't happen to know the preimage.
     int crcA = 0xCAFEBEEF;
-    final ToIntFunction<Long> mod = DataChecksum.getModFunction(type);
+    final LongToIntFunction mod = DataChecksum.getModFunction(type);
     DataChecksum checksum = DataChecksum.newDataChecksum(
         type, Integer.MAX_VALUE);
     Objects.requireNonNull(checksum, "checksum");
@@ -230,7 +230,7 @@ public void testMultiplyMod() {
   private static long[] runTestMultiplyMod(int n, DataChecksum.Type type) {
     System.out.printf("Run %s with %d computations%n", type, n);
     final int polynomial = getCrcPolynomialForType(type);
-    final ToIntFunction<Long> mod = DataChecksum.getModFunction(type);
+    final LongToIntFunction mod = DataChecksum.getModFunction(type);
 
     final int[] p = new int[n];
     final int[] q = new int[n];
@@ -256,8 +256,8 @@ private static long[] runTestMultiplyMod(int n, DataChecksum.Type type) {
     }
     times[1] = System.currentTimeMillis() - t1;
     final double ops1 = n * 1000.0 / times[1];
-    System.out.printf("multiplyCrc32      : %.3fs (%.2f ops)%n", times[1] / 1000.0, ops1);
-    System.out.printf("multiplyCrc32 is %.2f%% faster%n", (ops1 - ops0) * 100.0 / ops0);
+    System.out.printf("tableMultiply      : %.3fs (%.2f ops)%n", times[1] / 1000.0, ops1);
+    System.out.printf("tableMultiply ops is %.2f%% faster%n", (ops1 - ops0) * 100.0 / ops0);
 
     for (int i = 0; i < n; i++) {
       if (expected[i] != computed[i]) {
@@ -354,11 +354,11 @@ public static void main(String[] args) throws Exception {
       }
 
       System.out.printf("%nResult) %d x %d computations:%n", m, n);
-      final double ops0 = n * 1000.0 / times[0];
+      final double ops0 = m * n * 1000.0 / times[0];
       System.out.printf("galoisFieldMultiply: %.3fs (%.2f ops)%n", times[0] / 1000.0, ops0);
-      final double ops1 = n * 1000.0 / times[1];
-      System.out.printf("multiplyCrc32      : %.3fs (%.2f ops)%n", times[1] / 1000.0, ops1);
-      System.out.printf("multiplyCrc32 is %.2f%% faster%n", (ops1 - ops0) * 100.0 / ops0);
+      final double ops1 = m * n * 1000.0 / times[1];
+      System.out.printf("tableMultiply      : %.3fs (%.2f ops)%n", times[1] / 1000.0, ops1);
+      System.out.printf("tableMultiply ops is %.2f%% faster%n", (ops1 - ops0) * 100.0 / ops0);
     }
   }
 }