fix: fall back to Spark for str_to_map when legacy regex split truncation is enabled

Spark 4.1.1 added spark.sql.legacy.truncateForEmptyRegexSplit, which makes
StringToMap truncate trailing empty entries from the split result. Comet's
native str_to_map always behaves as if the flag were false, producing
incorrect results in legacy mode.

Downgrade CometStrToMap to Incompatible when the flag is enabled so it falls
back to Spark unless the user explicitly opts in via allowIncompatible. The
config is read by string key so it resolves on Spark versions where it is not
registered.

Closes #4477

Author: andygrove

spark/src/main/scala/org/apache/comet/serde/maps.scala

@@ -20,6 +20,7 @@
 package org.apache.comet.serde
 
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 import org.apache.comet.serde.QueryPlanSerde.{createBinaryExpr, exprToProtoInternal, optExprWithFallbackReason, scalarFunctionExprToProto}
@@ -164,6 +165,21 @@ object CometMapFromEntries
   }
 }
 
-object CometStrToMap extends CometScalarFunction[StringToMap]("str_to_map")
+object CometStrToMap extends CometScalarFunction[StringToMap]("str_to_map") {
+
+  // Spark 4.1.1+ honours spark.sql.legacy.truncateForEmptyRegexSplit by truncating trailing
+  // empty entries from the split result. Comet's native str_to_map always behaves as if the flag
+  // were false, so it is incompatible when legacy truncation is enabled. Read by string key so it
+  // resolves on older Spark versions where the config is not registered.
+  private val legacyTruncateConfig = "spark.sql.legacy.truncateForEmptyRegexSplit"
+
+  override def getSupportLevel(expr: StringToMap): SupportLevel = {
+    if (SQLConf.get.getConfString(legacyTruncateConfig, "false").toBoolean) {
+      Incompatible(Some(s"$legacyTruncateConfig is enabled"))
+    } else {
+      Compatible(None)
+    }
+  }
+}
 
 object CometMapConcat extends CometCodegenDispatch[MapConcat]

spark/src/test/resources/sql-tests/expressions/map/str_to_map_legacy_truncate.sql

@@ -0,0 +1,41 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Tests that str_to_map falls back to Spark when
+-- spark.sql.legacy.truncateForEmptyRegexSplit is enabled. In legacy mode Spark truncates trailing
+-- empty entries from the split result, which Comet's native str_to_map does not honour.
+-- See https://github.com/apache/datafusion-comet/issues/4477
+
+-- Config: spark.sql.legacy.truncateForEmptyRegexSplit=true
+
+-- trailing pair delimiter: legacy mode truncates the trailing empty entry, so Comet must fall
+-- back to Spark
+query expect_fallback(truncateForEmptyRegexSplit)
+SELECT str_to_map('a:1,b:2,', ',', ':')
+
+-- column input also falls back
+statement
+CREATE TABLE test_str_to_map_legacy(s STRING, pair_delim STRING, key_value_delim STRING) USING parquet
+
+statement
+INSERT INTO test_str_to_map_legacy VALUES
+  ('a:1,b:2,', ',', ':'),
+  ('x:1;y:2;', ';', ':'),
+  (NULL, ',', ':')
+
+query expect_fallback(truncateForEmptyRegexSplit)
+SELECT str_to_map(s, pair_delim, key_value_delim) FROM test_str_to_map_legacy