[SPARK-59019][SQL] Make RewriteWithExpression non-excludable

### What changes were proposed in this pull request?

Add `RewriteWithExpression` to the `nonExcludableRules` list in `Optimizer.scala`.

### Why are the changes needed?

`With` expressions are `Unevaluable` — they are internal tree constructs for common subexpression elimination that must be rewritten before physical planning. `FinishAnalysis` replaces `RuntimeReplaceable` expressions (e.g., `Between`) with `With` nodes; `RewriteWithExpression` must then rewrite those nodes into executable form. If excluded via `spark.sql.optimizer.excludedRules`, the `With` nodes survive into codegen and throw `INTERNAL_ERROR: Cannot generate code for expression: with(...)`.

### Does this PR introduce _any_ user-facing change?

Yes. Users who previously set `spark.sql.optimizer.excludedRules=org.apache.spark.sql.catalyst.optimizer.RewriteWithExpression` will no longer see the rule excluded (it becomes a no-op config entry). This is strictly better — the previous behavior was a crash.

### How was this patch tested?

Added a regression test in `SQLQuerySuite` that runs a `BETWEEN` query with `RewriteWithExpression` in `excludedRules` and asserts correct results.

### Was this patch authored or co-authored using generative AI tooling?

Yes. Co-Authored using Claude Opus 4.8

Closes #58323 from shrirangmhalgi/SPARK-59019-rewrite-with-non-excludable.

Authored-by: Shrirang Mhalgi <shrirangmhalgi@gmail.com>
Signed-off-by: Peter Toth <peter.toth@gmail.com>

Author: shrirangmhalgi

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala

@@ -321,6 +321,9 @@ abstract class Optimizer(catalogManager: CatalogManager)
       // execution, so it must never be excludable.
       ConvertToCatalyst.ruleName,
       FinishAnalysis.ruleName,
+      // ReplaceExpressions (in FinishAnalysis) turns Between/NullIf into the Unevaluable
+      // With expression; excluding this rule leaks it into codegen and fails with INTERNAL_ERROR.
+      RewriteWithExpression.ruleName,
       RewriteDistinctAggregates.ruleName,
       ReplaceDeduplicateWithAggregate.ruleName,
       ReplaceIntersectWithSemiJoin.ruleName,

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

@@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.ExtendedAnalysisException
 import org.apache.spark.sql.catalyst.expressions.{CodegenObjectFactoryMode, GenericRow, Hex}
 import org.apache.spark.sql.catalyst.expressions.Cast._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{Complete, Partial}
-import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, NestedColumnAliasingSuite}
+import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, NestedColumnAliasingSuite, RewriteWithExpression}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.plans.logical.{LocalLimit, Project, RepartitionByExpression, Sort}
 import org.apache.spark.sql.connector.catalog.CatalogManager
@@ -5315,6 +5315,20 @@ class SQLQuerySuite extends SharedSparkSession with AdaptiveSparkPlanHelper
         checkToRDD = false)
     }
   }
+
+  test("SPARK-59019: BETWEEN succeeds when RewriteWithExpression is in excludedRules") {
+    // RewriteWithExpression is non-excludable, so adding it to excludedRules has no effect.
+    // Before the fix, this threw INTERNAL_ERROR because With nodes reached codegen.
+    withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key ->
+      RewriteWithExpression.ruleName) {
+      checkAnswer(
+        sql("SELECT x BETWEEN 1 AND 2 AS in_range FROM (VALUES (1)) AS t(x)"),
+        Row(true))
+      checkAnswer(
+        sql("SELECT x BETWEEN 1 AND 2 AS in_range FROM (VALUES (3)) AS t(x)"),
+        Row(false))
+    }
+  }
 }
 
 case class Foo(bar: Option[String])