[CALCITE-6821] Support crc32 function for Hive and Spark library

apache · Feb 13, 2025 · 2ddfb0e · 2ddfb0e
1 parent 929d9b4
commit 2ddfb0e
Show file tree

Hide file tree

Showing 6 changed files with 56 additions and 2 deletions.
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java
@@ -188,6 +188,7 @@
 import static org.apache.calcite.sql.fun.SqlLibraryOperators.COSD;
 import static org.apache.calcite.sql.fun.SqlLibraryOperators.COSH;
 import static org.apache.calcite.sql.fun.SqlLibraryOperators.COTH;
+import static org.apache.calcite.sql.fun.SqlLibraryOperators.CRC32;
 import static org.apache.calcite.sql.fun.SqlLibraryOperators.CSC;
 import static org.apache.calcite.sql.fun.SqlLibraryOperators.CSCH;
 import static org.apache.calcite.sql.fun.SqlLibraryOperators.CURRENT_DATETIME;
@@ -697,6 +698,7 @@ void populate1() {
       defineMethod(FROM_HEX, BuiltInMethod.FROM_HEX.method, NullPolicy.STRICT);
       defineMethod(BIN, BuiltInMethod.BIN.method, NullPolicy.STRICT);
       defineMethod(MD5, BuiltInMethod.MD5.method, NullPolicy.STRICT);
+      defineMethod(CRC32, BuiltInMethod.CRC32.method, NullPolicy.STRICT);
       defineMethod(SHA1, BuiltInMethod.SHA1.method, NullPolicy.STRICT);
       defineMethod(SHA256, BuiltInMethod.SHA256.method, NullPolicy.STRICT);
       defineMethod(SHA512, BuiltInMethod.SHA512.method, NullPolicy.STRICT);

diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -133,6 +133,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
+import java.util.zip.CRC32;
 
 import static org.apache.calcite.config.CalciteSystemProperty.FUNCTION_LEVEL_CACHE_MAX_SIZE;
 import static org.apache.calcite.linq4j.Nullness.castNonNull;
@@ -386,6 +387,24 @@ public static String bin(long value) {
     }
   }
 
+  /** SQL CRC32(string) function. */
+  public static long crc32(String value)  {
+    final CRC32 crc32 = new CRC32();
+    crc32.reset();
+    byte[] bytes = value.getBytes(UTF_8);
+    crc32.update(bytes, 0, bytes.length);
+    return crc32.getValue();
+  }
+
+  /** SQL CRC32(string) function for binary string. */
+  public static long crc32(ByteString value)  {
+    final CRC32 crc32 = new CRC32();
+    crc32.reset();
+    byte[] bytes = value.getBytes();
+    crc32.update(bytes, 0, bytes.length);
+    return crc32.getValue();
+  }
+
   /** SQL MD5(string) function. */
   public static String md5(String string)  {
     return DigestUtils.md5Hex(string.getBytes(UTF_8));

diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
@@ -2518,6 +2518,13 @@ private static RelDataType deriveTypeMapFromEntries(SqlOperatorBinding opBinding
           OperandTypes.STRING.or(OperandTypes.BINARY),
           SqlFunctionCategory.STRING);
 
+  @LibraryOperator(libraries = {SPARK, HIVE})
+  public static final SqlFunction CRC32 =
+      SqlBasicFunction.create("CRC32",
+          ReturnTypes.BIGINT_NULLABLE,
+          OperandTypes.STRING.or(OperandTypes.BINARY),
+          SqlFunctionCategory.STRING);
+
   @LibraryOperator(libraries = {BIG_QUERY, MYSQL, POSTGRESQL, SPARK, HIVE})
   public static final SqlFunction SHA1 =
       SqlBasicFunction.create("SHA1",

diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
@@ -440,6 +440,7 @@ public enum BuiltInMethod {
   FROM_HEX(SqlFunctions.class, "fromHex", String.class),
   BIN(SqlFunctions.class, "bin", long.class),
   MD5(SqlFunctions.class, "md5", String.class),
+  CRC32(SqlFunctions.class, "crc32", String.class),
   SHA1(SqlFunctions.class, "sha1", String.class),
   SHA256(SqlFunctions.class, "sha256", String.class),
   SHA512(SqlFunctions.class, "sha512", String.class),

diff --git a/site/_docs/reference.md b/site/_docs/reference.md
@@ -2842,6 +2842,7 @@ In the following:
 | p | COSD(numeric)                                  | Returns the cosine of *numeric* in degrees as a double. Returns NaN if *numeric* is NaN. Fails if *numeric* is greater than the maximum double value.
 | * | COSH(numeric)                                  | Returns the hyperbolic cosine of *numeric*
 | * | COTH(numeric)                                  | Returns the hyperbolic cotangent of *numeric*
+| s h | CRC32(string)                                | Calculates a cyclic redundancy check value for string or binary argument and returns bigint value
 | * | CSC(numeric)                                   | Returns the cosecant of *numeric* in radians
 | * | CSCH(numeric)                                  | Returns the hyperbolic cosecant of *numeric*
 | b | CURRENT_DATETIME([ timeZone ])                 | Returns the current time as a TIMESTAMP from *timezone*

diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
@@ -5760,8 +5760,7 @@ void testBitGetFunc(SqlOperatorFixture f, String functionName) {
    * <a href="https://issues.apache.org/jira/browse/CALCITE-6815">[CALCITE-6815]
    * Add bin function (enabled in Hive and Spark library)</a>. */
   @Test void testBin() {
-//    final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.BIN);
-    final SqlOperatorFixture f0 = Fixtures.forOperators(true).setFor(SqlLibraryOperators.BIN);
+    final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.BIN);
     f0.checkFails("^bin(x'')^",
         "No match found for function signature BIN\\(<BINARY>\\)",
         false);
@@ -5797,6 +5796,31 @@ void testBitGetFunc(SqlOperatorFixture f, String functionName) {
     f0.forEachLibrary(libraries, consumer);
   }
 
+  /** Test case for
+   * <a href="https://issues.apache.org/jira/browse/CALCITE-6821">[CALCITE-6821]
+   * Add crc32 function (enabled in Hive and Spark library)</a>. */
+  @Test void testCRC32() {
+    final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.CRC32);
+    final List<SqlLibrary> libraries =
+        ImmutableList.of(SqlLibrary.SPARK, SqlLibrary.HIVE);
+    final Consumer<SqlOperatorFixture> consumer = f -> {
+      f.checkString("crc32('ABC')",
+          "2743272264",
+          "BIGINT NOT NULL");
+      f.checkString("crc32(x'414243')",
+          "2743272264",
+          "BIGINT NOT NULL");
+      f.checkString("crc32('')",
+          "0",
+          "BIGINT NOT NULL");
+      f.checkString("crc32(x'')",
+          "0",
+          "BIGINT NOT NULL");
+      f.checkNull("crc32(null)");
+    };
+    f0.forEachLibrary(libraries, consumer);
+  }
+
   @Test void testMd5() {
     final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.MD5);
     f0.checkFails("^md5(x'')^",