Fix and streamline release branch compatibility checks

## Summary
Run release compatibility checks for both GitHub target-branch formats and replace redundant compile, setup, credential, and per-package SBT tasks with one cached, project-scoped validation process.

## Prompting Intent
The engineer asked to fix the silently skipped ReleaseBranchCompat job and simplify it before enabling it so the check is both reliable and efficient.

## Linked Sources
- Base CI hardening PR: #2581
- Evidence build with skipped phase: ADO build 229176406
- Parent stack layer: ci/databricks-impact-gating

## Rationale
The target condition accepts both values observed across Azure Repos and GitHub PR providers. A single SBT process retains full test compilation and the intended core, VW, and OpenCV compatibility suites while removing repeated build loading, root-wide IntelliJ setup, unnecessary Key Vault access, and Azure CLI authentication.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: ranadeepsingh

pipeline.yaml

@@ -912,7 +912,15 @@ jobs:
   cancelTimeoutInMinutes: 0
   timeoutInMinutes: 60
   continueOnError: true
-  condition: and(succeeded(), eq(variables.isPR, true), eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/master'))
+  condition: >-
+    and(
+      succeeded(),
+      eq(variables.isPR, true),
+      or(
+        eq(variables['System.PullRequest.TargetBranch'], 'master'),
+        eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/master')
+      )
+    )
   pool:
     vmImage: $(UBUNTU_VERSION)
   strategy:
@@ -968,56 +976,34 @@ jobs:
       displayName: 'Rebase $(RELEASE_BRANCH) onto PR HEAD'
 
     - template: templates/sbt_cache.yml
-    - task: AzureCLI@2
-      displayName: 'Compile $(RELEASE_BRANCH) after rebase'
-      timeoutInMinutes: 20
-      inputs:
-        azureSubscription: 'SynapseML Build'
-        scriptLocation: inlineScript
-        scriptType: bash
-        inlineScript: |
-          set -e
-          export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
-          echo "=== Compiling $(RELEASE_BRANCH) rebased onto PR HEAD ==="
-          sbt $(SBT_JAVA_OPTS) compile test:compile
-          echo "$(RELEASE_BRANCH) compiles successfully after rebase"
-
-    - task: AzureCLI@2
-      displayName: 'Setup repo for tests'
-      inputs:
-        azureSubscription: 'SynapseML Build'
-        scriptLocation: inlineScript
-        scriptType: bash
-        inlineScript: |
-          (timeout 30s pip install requests) || (echo "retrying" && timeout 30s pip install requests)
-          bash tools/ci/sbt_retry.sh $(SBT_JAVA_OPTS) setup
-
-    - template: templates/kv.yml
+    - bash: |
+        set -e
+        export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
+        CORE_TESTS="com.microsoft.azure.synapse.ml.core.** \
+        com.microsoft.azure.synapse.ml.automl.** \
+        com.microsoft.azure.synapse.ml.causal.** \
+        com.microsoft.azure.synapse.ml.featurize.** \
+        com.microsoft.azure.synapse.ml.image.** \
+        com.microsoft.azure.synapse.ml.isolationforest.** \
+        com.microsoft.azure.synapse.ml.stages.** \
+        com.microsoft.azure.synapse.ml.recommendation.** \
+        com.microsoft.azure.synapse.ml.nn.** \
+        com.microsoft.azure.synapse.ml.train.** \
+        com.microsoft.azure.synapse.ml.exploratory.**"
 
-    - task: AzureCLI@2
-      displayName: 'Unit tests on $(RELEASE_BRANCH) after rebase'
-      timeoutInMinutes: 60
-      inputs:
-        azureSubscription: 'SynapseML Build'
-        scriptLocation: inlineScript
-        scriptType: bash
-        inlineScript: |
-          set -e
-          export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
-          echo "=== Running unit tests on $(RELEASE_BRANCH) rebased onto PR HEAD ==="
-          FAILURES=0
-          for pkg in core automl causal featurize image isolationforest stages recommendation nn train vw opencv exploratory; do
-            echo "=== Testing $pkg ==="
-            if ! timeout 10m sbt $(SBT_JAVA_OPTS) "testOnly com.microsoft.azure.synapse.ml.$pkg.**"; then
-              echo "##vso[task.logissue type=warning]$pkg tests failed on $(RELEASE_BRANCH)"
-              FAILURES=$((FAILURES + 1))
-            fi
-          done
-          if [ $FAILURES -gt 0 ]; then
-            echo "##vso[task.logissue type=warning]$FAILURES package(s) failed on $(RELEASE_BRANCH)"
-            exit 1
-          fi
-          echo "All unit tests passed on $(RELEASE_BRANCH)"
+        echo "=== Compiling and testing $(RELEASE_BRANCH) rebased onto PR HEAD ==="
+        timeout 50m sbt $(SBT_JAVA_OPTS) \
+          test:compile \
+          getDatasets \
+          "project core" \
+          "testOnly $CORE_TESTS" \
+          "project vw" \
+          "testOnly com.microsoft.azure.synapse.ml.vw.**" \
+          "project opencv" \
+          "testOnly com.microsoft.azure.synapse.ml.opencv.**"
+        echo "$(RELEASE_BRANCH) compiles and passes compatibility tests"
+      displayName: 'Validate $(RELEASE_BRANCH) after rebase'
+      timeoutInMinutes: 55
 
     - task: PublishTestResults@2
       displayName: 'Publish $(RELEASE_BRANCH) Test Results'

tools/ci/tests/test_pipeline_yaml.py

@@ -157,6 +157,37 @@ def test_databricks_e2e_uses_fail_open_pr_impact_detection():
     assert any(step.get("displayName") == "Publish Test Results" for step in steps)
 
 
+def test_release_compat_accepts_github_target_and_uses_one_sbt_process():
+    data = yaml.safe_load(_pipeline_text())
+    jobs = {j.get("job"): j for j in _jobs(data["jobs"])}
+    release_compat = jobs["ReleaseBranchCompat"]
+
+    condition = release_compat["condition"]
+    assert "System.PullRequest.TargetBranch" in condition
+    assert "'master'" in condition
+    assert "'refs/heads/master'" in condition
+
+    steps = release_compat["steps"]
+    assert not any(step.get("task") == "AzureCLI@2" for step in steps)
+    assert not any(step.get("template") == "templates/kv.yml" for step in steps)
+
+    validation_steps = [
+        step
+        for step in steps
+        if isinstance(step, dict)
+        and step.get("displayName") == "Validate $(RELEASE_BRANCH) after rebase"
+    ]
+    assert len(validation_steps) == 1
+    script = validation_steps[0]["bash"]
+    assert script.count("sbt $(SBT_JAVA_OPTS)") == 1
+    assert "test:compile" in script
+    assert "getDatasets" in script
+    for project in ("core", "vw", "opencv"):
+        assert f'"project {project}"' in script
+    assert "sbt_retry.sh" not in script
+    assert "for pkg in" not in script
+
+
 def test_every_sbt_running_job_waits_for_the_prewarm_cache():
     """Each sbt job must restore the cache after the required prewarm job."""
     data = yaml.safe_load(_pipeline_text())