fix: platform status stays in_progress

scripts/receive_therock/tests/therock_summary_test.py

@@ -218,12 +218,23 @@ def test_in_progress_dominates_cancelled_and_success_on_platform() -> None:
     assert doc.summary.linux.status is Status.in_progress
 
 
-def test_failure_beats_cancelled_and_success() -> None:
+def test_failure_beats_cancelled_and_success_within_one_pipeline() -> None:
+    # Within one pipeline (matrix cells / build+test), a terminal failure
+    # still outranks a terminal cancelled and a terminal success -- that
+    # precedence is unaffected by the cross-pipeline sibling fix below. Every
+    # sibling pipeline also reports terminally here, isolating the
+    # within-pipeline precedence from the cross-pipeline
+    # in_progress-outranks-failure rule (see
+    # test_multiple_pipelines_aggregate_into_platform_status).
     doc = StatusDocument()
     _freeze(doc, "linux", ["gfx942", "gfx1100"])
     doc.upsert_leaf("linux", "", "rocm", "build", _leaf(status=Status.success))
     doc.upsert_leaf("linux", "gfx942", "rocm", "test", _leaf(status=Status.cancelled))
     doc.upsert_leaf("linux", "gfx1100", "rocm", "test", _leaf(status=Status.failure))
+    doc.upsert_leaf("linux", "", "pytorch", "build", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "jax", "build", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "native_packages", "rpm", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "native_packages", "deb", _leaf(status=Status.success))
     rebuild_summary(doc)
     assert doc.summary.linux.status is Status.failure
 
@@ -431,7 +442,25 @@ def test_native_packages_counts_rpm_and_deb_into_platform_status() -> None:
     assert native is not None
     assert native.rpm.status is Status.success
     assert native.deb.status is Status.failure
-    # deb failure drags the platform rollup.
+    # deb failure sets native_packages' own status to failure, but rocm/
+    # pytorch/jax are still pending (never reported) on this platform, so the
+    # platform itself must not crystallize to failure yet.
+    assert doc.summary.linux.status is Status.in_progress
+
+
+def test_native_packages_failure_drags_platform_once_siblings_are_terminal() -> None:
+    # Same deb failure, but every sibling pipeline has also reported
+    # terminally: nothing is left pending, so the worst-of (native's failure)
+    # now applies.
+    doc = StatusDocument()
+    _freeze(doc, "linux", ["gfx942"])
+    doc.upsert_leaf("linux", "", "rocm", "build", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "gfx942", "rocm", "test", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "pytorch", "build", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "jax", "build", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "native_packages", "rpm", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "native_packages", "deb", _leaf(status=Status.failure))
+    rebuild_summary(doc)
     assert doc.summary.linux.status is Status.failure
 
 
@@ -560,8 +589,10 @@ def test_running_test_leaf_not_masked_by_terminal_cells() -> None:
 
 def test_test_full_phase_rolls_up_separately_and_feeds_platform() -> None:
     # The full-suite `test-full` phase renders as its own counts block and its
-    # status still feeds the platform worst-of (a failing full suite fails the
-    # platform even when the smoke `test` passed).
+    # status still feeds pytorch's own rollup (a failing full suite fails
+    # pytorch even when the smoke `test` passed). jax/native_packages are
+    # still pending on this platform, so the platform itself stays
+    # in_progress until they report too.
     doc = StatusDocument()
     _freeze(doc, "linux", ["gfx942"])
     doc.upsert_leaf("linux", "", "rocm", "build", _leaf(status=Status.success))
@@ -574,6 +605,24 @@ def test_test_full_phase_rolls_up_separately_and_feeds_platform() -> None:
     assert pytorch.test.success == 1
     assert pytorch.test_full is not None
     assert pytorch.test_full.failure == 1
+    assert doc.summary.linux.status is Status.in_progress
+
+
+def test_test_full_phase_failure_drags_platform_once_siblings_are_terminal() -> None:
+    # Same failing full suite, but jax/native_packages have also reported
+    # terminally: nothing is left pending, so the worst-of (pytorch's
+    # test-full failure) now applies.
+    doc = StatusDocument()
+    _freeze(doc, "linux", ["gfx942"])
+    doc.upsert_leaf("linux", "", "rocm", "build", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "gfx942", "pytorch", "test", _leaf(status=Status.success))
+    doc.upsert_leaf(
+        "linux", "gfx942", "pytorch", "test-full", _leaf(status=Status.failure)
+    )
+    doc.upsert_leaf("linux", "", "jax", "build", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "native_packages", "rpm", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "native_packages", "deb", _leaf(status=Status.success))
+    rebuild_summary(doc)
     assert doc.summary.linux.status is Status.failure
 
 
@@ -695,9 +744,12 @@ def test_test_only_pipeline_rollup_does_not_leave_build_in_progress() -> None:
 
 
 def test_multiple_pipelines_aggregate_into_platform_status() -> None:
-    # rocm and pytorch both run on linux. The platform status is worst-of across
-    # pipelines, not within one: rocm all-success + pytorch build failure -> the
-    # platform rolls up to failure while each pipeline keeps its own status.
+    # rocm and pytorch both run on linux, but jax/native_packages are
+    # independent sibling pipelines that have not reported anything yet.
+    # rocm succeeds and pytorch's build fails -- each pipeline keeps its own
+    # status -- but jax/native_packages might still succeed or fail, so the
+    # platform must not crystallize to failure while they are still pending:
+    # it stays in_progress (see rollup_sibling_statuses).
     doc = StatusDocument()
     _freeze(doc, "linux", ["gfx942"])
     doc.upsert_leaf("linux", "", "rocm", "build", _leaf(status=Status.success))
@@ -707,6 +759,22 @@ def test_multiple_pipelines_aggregate_into_platform_status() -> None:
     assert doc.summary.linux.rocm.build.status is Status.success
     assert doc.summary.linux.rocm.test.success == 1
     assert doc.summary.linux.pytorch.build.status is Status.failure
+    assert doc.summary.linux.status is Status.in_progress
+
+
+def test_multiple_pipelines_failure_wins_once_every_sibling_is_terminal() -> None:
+    # Same rocm success + pytorch build failure, but jax and native_packages
+    # have now also reported terminally: nothing on the platform is left
+    # pending, so the worst-of (pytorch's failure) applies.
+    doc = StatusDocument()
+    _freeze(doc, "linux", ["gfx942"])
+    doc.upsert_leaf("linux", "", "rocm", "build", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "gfx942", "rocm", "test", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "pytorch", "build", _leaf(status=Status.failure))
+    doc.upsert_leaf("linux", "", "jax", "build", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "native_packages", "rpm", _leaf(status=Status.success))
+    doc.upsert_leaf("linux", "", "native_packages", "deb", _leaf(status=Status.success))
+    rebuild_summary(doc)
     assert doc.summary.linux.status is Status.failure
 
 

scripts/receive_therock/therock_status_document.py

@@ -129,6 +129,41 @@ def rollup_statuses(statuses: Iterable[Status], fallback: Status) -> Status:
     return Status.skipped
 
 
+def rollup_sibling_statuses(statuses: Iterable[Status], fallback: Status) -> Status:
+    """Collapse the statuses of independent sibling pipelines (rocm / pytorch /
+    jax / native_packages) on the same platform into one platform status.
+
+    This is deliberately a different precedence than `rollup_statuses`, which
+    combines matrix cells / build+test *within* one pipeline: there, a
+    terminal failure in one cell is irreversible and must not be masked by a
+    still-running sibling *cell* of the same phase. Sibling *pipelines* are
+    separate, independently-dispatched units of work, so one of them failing
+    says nothing about whether another, still-running one is done -- it may
+    yet also fail, or may still succeed. Crystallizing the platform to
+    `failure` while a sibling pipeline has not reported a terminal status
+    would report a verdict the platform has not actually reached yet.
+
+    Precedence: in_progress > failure > cancelled > success > skipped. This
+    also makes `failure` behave consistently with `cancelled` here (which
+    already lost to `in_progress` even under the old precedence): both are
+    terminal-negative, but neither can override still-pending sibling work.
+    Once every sibling has reported a terminal status, the worst one applies.
+    Returns `fallback` when there is nothing to roll up.
+    """
+    seen = set(statuses)
+    if not seen:
+        return fallback
+    for status in (
+        Status.in_progress,
+        Status.failure,
+        Status.cancelled,
+        Status.success,
+    ):
+        if status in seen:
+            return status
+    return Status.skipped
+
+
 class Variant(BaseModel):
     """One matrix cell of a fan-out pipeline (e.g. pytorch py x torch).
 

scripts/receive_therock/therock_summary.py

@@ -16,6 +16,7 @@
     StatusDocument,
     Summary,
     TestRollup,
+    rollup_sibling_statuses,
     rollup_statuses,
 )
 from therock_types import EXPECTED_PIPELINE_TYPES
@@ -111,20 +112,31 @@ def _build_platform_summary(
         cancelled      cancelled                     cancelled
         failure        failure                       skipped
 
-    A cancelled/failed rocm *test* gates nothing downstream, so the children stay
-    `in_progress` and the platform stays `in_progress` (in_progress outranks
-    cancelled in the worst-of).
+    Each pipeline (rocm, pytorch, jax, native_packages) first rolls its own
+    build/test/test_full leaves up to a single status with `rollup_statuses`
+    (failure beats a still-running cell of the *same* pipeline -- that cell
+    cannot undo a real failure). The platform status then combines those
+    per-pipeline statuses -- plus a placeholder for any pipeline that has not
+    reported at all -- with `rollup_sibling_statuses`, which lets in_progress
+    outrank a sibling pipeline's failure: independent pipelines are separate
+    units of work, so one having already failed does not mean another,
+    still-running one is done. This makes a failed rocm *test* behave like a
+    cancelled one (see below) instead of prematurely deciding the platform's
+    verdict while pytorch/jax/native_packages are still pending.
     """
     architectures = (
         doc.linux_architectures if platform == "linux" else doc.windows_architectures
     )
     urls = doc.linux_urls if platform == "linux" else doc.windows_urls
 
-    seen_statuses: list[Status] = []
-    rocm = _pipeline_rollup(doc.pipelines.rocm, platform, seen_statuses)
-    pytorch = _pipeline_rollup(doc.pipelines.pytorch, platform, seen_statuses)
-    jax = _pipeline_rollup(doc.pipelines.jax, platform, seen_statuses)
-    native_packages = _native_rollup(doc, platform, seen_statuses)
+    rocm_seen: list[Status] = []
+    pytorch_seen: list[Status] = []
+    jax_seen: list[Status] = []
+    native_seen: list[Status] = []
+    rocm = _pipeline_rollup(doc.pipelines.rocm, platform, rocm_seen)
+    pytorch = _pipeline_rollup(doc.pipelines.pytorch, platform, pytorch_seen)
+    jax = _pipeline_rollup(doc.pipelines.jax, platform, jax_seen)
+    native_packages = _native_rollup(doc, platform, native_seen)
 
     empty_platform_status = Status.in_progress if architectures else Status.skipped
 
@@ -136,20 +148,30 @@ def _build_platform_summary(
     )
 
     rollups = {
-        "rocm": rocm,
-        "pytorch": pytorch,
-        "jax": jax,
-        "native_packages": native_packages,
+        "rocm": (rocm, rocm_seen),
+        "pytorch": (pytorch, pytorch_seen),
+        "jax": (jax, jax_seen),
+        "native_packages": (native_packages, native_seen),
     }
-    status_inputs = list(seen_statuses)
-    if doc.completed_at is None and any(
-        rollups[pipeline_type] is None
-        for pipeline_type in EXPECTED_PIPELINE_TYPES[platform]
-    ):
-        status_inputs.append(unstarted_status)
+
+    sibling_statuses: list[Status] = []
+    has_data = bool(architectures)
+    for pipeline_type in EXPECTED_PIPELINE_TYPES[platform]:
+        rollup_obj, seen = rollups[pipeline_type]
+        if rollup_obj is None:
+            # A pipeline that never reported feeds the sibling worst-of as
+            # its eventual (gate-derived) status while the release is live,
+            # so it keeps holding the platform at in_progress until it does.
+            # Once finalized it is dropped instead: it did not run this
+            # release, so it must not drag a finished platform.
+            if doc.completed_at is None:
+                sibling_statuses.append(unstarted_status)
+            continue
+        has_data = True
+        sibling_statuses.append(rollup_statuses(seen, empty_platform_status))
 
     fields: dict[str, object] = {
-        "status": rollup_statuses(status_inputs, empty_platform_status),
+        "status": rollup_sibling_statuses(sibling_statuses, empty_platform_status),
         "architectures": list(architectures),
         "urls": dict(urls),
         "rocm": rocm or placeholder,
@@ -161,7 +183,6 @@ def _build_platform_summary(
             native_packages if native_packages is not None else native_placeholder
         )
 
-    has_data = bool(architectures) or bool(seen_statuses)
     return PlatformSummary.for_platform(platform, **fields), has_data