Fix regex replacement parser Java spec gaps

Signed-off-by: Allen Xu <allxu@nvidia.com>

Author: wjxiz1992

integration_tests/src/main/python/regexp_test.py

@@ -449,6 +449,59 @@ def test_regexp_replace():
                 'regexp_replace(a, "a|b|c", "A")'),
         conf=_regexp_conf)
 
+
+# https://github.com/NVIDIA/spark-rapids/issues/14742
+# Replacement-string parser must match java.util.regex.Matcher#appendReplacement.
+# Use the DataFrame API rather than selectExpr because Spark SQL variable substitution
+# expands ${...} inside SQL string literals before regexp_replace sees it.
+def test_regexp_replace_subbug1_backslash_digit_is_literal_14742():
+    from pyspark.sql.functions import regexp_replace, col
+    assert_gpu_and_cpu_are_equal_collect(
+        lambda spark: spark.createDataFrame([("abc",)], ["a"]).select(
+            regexp_replace(col("a"), "(a)", "\\1")),
+        conf=_regexp_conf)
+
+
+@allow_non_gpu('ProjectExec', 'RegExpReplace')
+def test_regexp_replace_subbug2_trailing_backslash_throws_14742():
+    from pyspark.sql.functions import regexp_replace, col
+    assert_gpu_and_cpu_error(
+        lambda spark: spark.createDataFrame([("a",)], ["a"]).select(
+            regexp_replace(col("a"), "a", "\\")).collect(),
+        conf=_regexp_conf,
+        error_message="character to be escaped is missing")
+
+
+@allow_non_gpu('ProjectExec', 'RegExpReplace')
+def test_regexp_replace_subbug3_dollar_non_digit_throws_14742():
+    from pyspark.sql.functions import regexp_replace, col
+    assert_gpu_and_cpu_error(
+        lambda spark: spark.createDataFrame([("a",)], ["a"]).select(
+            regexp_replace(col("a"), "a", "$x")).collect(),
+        conf=_regexp_conf,
+        error_message="Illegal group reference")
+
+
+@allow_non_gpu('ProjectExec', 'RegExpReplace')
+def test_regexp_replace_subbug4_digit_leading_named_group_throws_14742():
+    from pyspark.sql.functions import regexp_replace, col
+    assert_gpu_and_cpu_error(
+        lambda spark: spark.createDataFrame([("a",)], ["a"]).select(
+            regexp_replace(col("a"), "(a)", "${1}")).collect(),
+        conf=_regexp_conf,
+        error_message="capturing group name {1} starts with digit character")
+
+
+@allow_non_gpu('ProjectExec', 'RegExpReplace')
+def test_regexp_replace_subbug5_unknown_named_group_throws_14742():
+    from pyspark.sql.functions import regexp_replace, col
+    assert_gpu_and_cpu_error(
+        lambda spark: spark.createDataFrame([("a",)], ["a"]).select(
+            regexp_replace(col("a"), "(a)", "${name}")).collect(),
+        conf=_regexp_conf,
+        error_message="No group with name")
+
+
 @pytest.mark.skipif(is_before_spark_320(), reason='regexp is synonym for RLike starting in Spark 3.2.0')
 def test_regexp():
     gen = mk_str_gen('[abcd]{1,3}')

sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala

@@ -364,56 +364,57 @@ class RegexParser(pattern: String) {
   }
 
   private def parseBackrefOrEscaped(): RegexAST = {
-    val start = pos
-
-    consumeInt match {
-      case Some(refNum) =>
-        RegexBackref(refNum)
+    peek() match {
       case None =>
-        pos = start
-        RegexChar('\\')
+        throw new RegexUnsupportedException(
+          "character to be escaped is missing", Some(pos))
+      case Some(_) =>
+        RegexSequence(ListBuffer(RegexChar('\\'), RegexChar(consume())))
     }
   }
 
   private def parseBackrefOrLiteralDollar(): RegexAST = {
-    val start = pos
-
-    def treatAsLiteralDollar() = {
-      pos = start
-      RegexChar('$')
-    }
-
     peek() match {
       case Some('{') =>
         consumeExpected('{')
-        val num = consumeInt()
-        if (peek().contains('}')) {
-          consumeExpected('}')
-          num match {
-            case Some(_) =>
+        peek() match {
+          case Some(ch) if isAsciiDigit(ch) =>
+            throw new RegexUnsupportedException(
+              "Illegal group reference: group name starts with digit character",
+              Some(pos))
+          case Some(ch) if isLetter(ch) =>
+            val nameStart = pos
+            while (!eof() && peek().exists(c => isLetter(c) || isAsciiDigit(c))) {
+              skip()
+            }
+            val name = pattern.substring(nameStart, pos)
+            if (!peek().contains('}')) {
               throw new RegexUnsupportedException(
-                "Numeric `${N}` backref in replacement string is not supported on GPU " +
-                  "(Java's Matcher.appendReplacement rejects this syntax)",
-                Some(start))
-            case _ =>
-              treatAsLiteralDollar()
-          }
-        } else {
-          treatAsLiteralDollar()
-        }
-      case Some(ch) if ch >= '1' && ch <= '9' =>
-        val num = consumeInt()
-        num match {
-          case Some(n) =>
-            RegexBackref(n)
+                "Illegal group reference: malformed " + "$" + "{name} reference",
+                Some(pos))
+            }
+            consumeExpected('}')
+            throw new RegexUnsupportedException(
+              s"named-group reference $${$name} is not supported on the GPU",
+              Some(pos))
           case _ =>
-            treatAsLiteralDollar()
+            throw new RegexUnsupportedException(
+              "Illegal group reference: empty or malformed " + "$" + "{name}",
+              Some(pos))
         }
+      case Some(ch) if isAsciiDigit(ch) =>
+        RegexBackref(consumeInt().get)
       case _ =>
-        treatAsLiteralDollar()
+        throw new RegexUnsupportedException(
+          "Illegal group reference", Some(pos))
     }
   }
 
+  private def isLetter(ch: Char): Boolean =
+    (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z')
+
+  private def isAsciiDigit(ch: Char): Boolean = ch >= '0' && ch <= '9'
+
   private def parseEscapedCharacter(): RegexAST = {
     peek() match {
       case None =>
@@ -620,7 +621,8 @@ class RegexParser(pattern: String) {
 
   private def consumeInt(): Option[Int] = {
     val start = pos
-    while (!eof() && peek().exists(_.isDigit)) {
+    // ASCII only: `substring.toInt` rejects Unicode digit codepoints.
+    while (!eof() && peek().exists(isAsciiDigit)) {
       skip()
     }
     if (start == pos) {

sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala

@@ -1073,13 +1073,12 @@ object GpuRegExpUtils {
   }
 
   /**
-   * Convert symbols of back-references if input string contains any.
-   * In spark's regex rule, there are two patterns of back-references:
-   * \group_index and \$group_index
-   * This method transforms above two patterns into cuDF pattern \${group_index}, except they are
-   * preceded by escape character.
+   * Convert numbered `$group_index` back-reference symbols into cuDF's `${group_index}` form.
+   * Java `Matcher#appendReplacement` treats `\digit` as the literal digit character, not as a
+   * back-reference. Escaped pairs are kept verbatim here and are later normalized by
+   * `unescapeReplaceString`.
    *
-   * Java's `Matcher.appendReplacement` reads the digits after `$` or `\` one at a time and
+   * Java's `Matcher.appendReplacement` reads the digits after `$` one at a time and
    * stops as soon as adding the next digit would make the running group index exceed the
    * actual capture-group count; any further digits are treated as literal characters
    * (greedy-with-backoff). When `numCaptureGroups` is negative the caller is opting out of the
@@ -1114,8 +1113,7 @@ object GpuRegExpUtils {
           b.append(rep.charAt(i))
           i += 1
         }
-      } else if (Seq('$', '\\').contains(rep.charAt(i))
-        && i + 1 < rep.length && rep.charAt(i + 1).isDigit) {
+      } else if (rep.charAt(i) == '$' && i + 1 < rep.length && rep.charAt(i + 1).isDigit) {
 
         // Consume digits one at a time. If the running group index would exceed the actual
         // capture-group count, stop and leave the remaining digits as literals. When no digit
@@ -1153,8 +1151,7 @@ object GpuRegExpUtils {
           i = k
         }
       } else if (rep.charAt(i) == '\\' && i + 1 < rep.length) {
-        // skip potential escape sequences like `\$` or `\\`; `\digit` is handled by the
-        // greedy-with-backoff branch above.
+        // Keep `\X` pairs verbatim; `unescapeReplaceString` strips the leading backslash.
         b.append('\\').append(rep.charAt(i + 1))
         i += 2
       } else {

tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionParserSuite.scala

@@ -283,15 +283,97 @@ class RegularExpressionParserSuite extends AnyFunSuite {
     RegexChar('$'))))
   }
 
-  test("replacement: numeric braced backref rejected (Java spec)") {
-    val brace = "$" + "{"
-    val cases = Seq(s"[${brace}2}]", s"${brace}1}", s"${brace}12}",
-        s"a${brace}3}b", s"${brace}0}")
-    for (rep <- cases) {
+  test("issue-14742-subbug1: \\N in replacement is the literal character N, not a backref") {
+    val repl = new RegexParser("\\1").parseReplacement(numCaptureGroups = 1)
+    assert(repl.parts.toList === List(RegexChar('\\'), RegexChar('1')))
+  }
+
+  test("issue-14742-subbug1: \\a in replacement is the literal character a") {
+    val repl = new RegexParser("\\a").parseReplacement(numCaptureGroups = 0)
+    assert(repl.parts.toList === List(RegexChar('\\'), RegexChar('a')))
+  }
+
+  test("issue-14742-subbug2: trailing \\ in replacement throws") {
+    val ex = intercept[RegexUnsupportedException] {
+      new RegexParser("\\").parseReplacement(numCaptureGroups = 0)
+    }
+    assert(ex.getMessage.contains("character to be escaped is missing"))
+  }
+
+  test("issue-14742-subbug3: bare $X for non-digit X throws") {
+    val ex = intercept[RegexUnsupportedException] {
+      new RegexParser("$x").parseReplacement(numCaptureGroups = 0)
+    }
+    assert(ex.getMessage.contains("Illegal group reference"))
+  }
+
+  test("issue-14742-subbug3: trailing bare $ throws") {
+    val ex = intercept[RegexUnsupportedException] {
+      new RegexParser("$").parseReplacement(numCaptureGroups = 0)
+    }
+    assert(ex.getMessage.contains("Illegal group reference"))
+  }
+
+  test("issue-14742-subbug4: dollar-brace-digit-brace throws") {
+    val ex = intercept[RegexUnsupportedException] {
+      new RegexParser("$" + "{1}").parseReplacement(numCaptureGroups = 1)
+    }
+    assert(ex.getMessage.contains("Illegal group reference"))
+    assert(ex.getMessage.contains("digit"))
+  }
+
+  test("issue-14742-subbug5: dollar-brace-name-brace for named group is not supported on GPU") {
+    val ex = intercept[RegexUnsupportedException] {
+      new RegexParser("$" + "{name}").parseReplacement(numCaptureGroups = 1)
+    }
+    assert(ex.getMessage.contains("named-group reference"))
+  }
+
+  test("issue-14742-subbug5: dollar-brace-name with missing closing brace throws") {
+    val ex = intercept[RegexUnsupportedException] {
+      new RegexParser("$" + "{name").parseReplacement(numCaptureGroups = 0)
+    }
+    assert(ex.getMessage.contains("Illegal group reference"))
+  }
+
+  test("issue-14742: dollar-brace with empty body throws") {
+    val ex = intercept[RegexUnsupportedException] {
+      new RegexParser("$" + "{}").parseReplacement(numCaptureGroups = 0)
+    }
+    assert(ex.getMessage.contains("Illegal group reference"))
+  }
+
+  test("issue-14742: numbered backref $0 still works") {
+    val repl = new RegexParser("$0").parseReplacement(numCaptureGroups = 0)
+    assert(repl.parts.toList === List(RegexBackref(0)))
+  }
+
+  test("issue-14742: numbered backref $1 still works") {
+    val repl = new RegexParser("$1").parseReplacement(numCaptureGroups = 1)
+    assert(repl.parts.toList === List(RegexBackref(1)))
+  }
+
+  test("issue-14742: numbered backref $12 still consumes max digits") {
+    val repl = new RegexParser("$12").parseReplacement(numCaptureGroups = 12)
+    assert(repl.parts.toList === List(RegexBackref(12)))
+  }
+
+  test("issue-14742: escaped metachar \\$ in replacement keeps the \\ pair") {
+    val repl = new RegexParser("\\$").parseReplacement(numCaptureGroups = 0)
+    assert(repl.parts.toList === List(RegexChar('\\'), RegexChar('$')))
+  }
+
+  test("issue-14742: escaped backslash \\\\ in replacement keeps the \\ pair") {
+    val repl = new RegexParser("\\\\").parseReplacement(numCaptureGroups = 0)
+    assert(repl.parts.toList === List(RegexChar('\\'), RegexChar('\\')))
+  }
+
+  test("issue-14742: non-ASCII Unicode digit after `$` triggers GPU fallback") {
+    for (rep <- Seq("$٢", "$१", "$۱")) {
       val e = intercept[RegexUnsupportedException] {
-        new RegexParser(rep).parseReplacement(4)
+        new RegexParser(rep).parseReplacement(numCaptureGroups = 4)
       }
-      assert(e.getMessage.contains("backref in replacement string is not supported"),
+      assert(e.getMessage.startsWith("Illegal group reference"),
         s"unexpected message for replacement '$rep': ${e.getMessage}")
     }
   }

tests/src/test/scala/com/nvidia/spark/rapids/StringFunctionSuite.scala

@@ -239,8 +239,8 @@ class RegExpUtilsSuite extends AnyFunSuite {
       (2, "$2", true, open + "2}"),
       // 0 groups, "$1": legacy path -- emit ${1} so cuDF surfaces the error.
       (0, "$1", true, open + "1}"),
-      // Same shape with backslash backref.
-      (2, "\\12", true, open + "1}2"),
+      // Java replacement strings treat `\digit` as the literal digit, not a backref.
+      (2, "\\12", false, "\\12"),
       // No digits after `$` -- literal `$`.
       (2, "$a", false, "$a"),
       // `$0` is the whole-match backref and is always valid (cuDF supports group 0).