🤖 update CI test matrix (#299)

Author: danielgafni

scripts/generate_test_matrix.py

@@ -59,9 +59,9 @@ class TestConfig(NamedTuple):
 # Component versions
 OS_VERSIONS = ["Ubuntu"]
 PYTHON_VERSIONS = ["3.12", "3.11", "3.10"]
-RAY_VERSIONS = ["2.49.2", "2.46.0"]
-DAGSTER_VERSIONS = ["1.11.13", "1.11.6", "1.11.1", "1.10.21"]
-KUBERAY_VERSIONS = ["1.4.0", "1.3.0", "1.2.2"]
+RAY_VERSIONS = ["2.53.0", "2.46.0"]
+DAGSTER_VERSIONS = ["1.12.12", "1.11.16", "1.10.21"]
+KUBERAY_VERSIONS = ["1.5.1", "1.4.2", "1.3.0", "1.2.2"]
 
 # Component registry for DRY iteration
 COMPONENTS = {

src/dagster_ray/kuberay/client/rayjob/client.py

@@ -26,9 +26,9 @@ class RayJobStatus(TypedDict):
     jobId: NotRequired[str]
     rayJobInfo: NotRequired[dict[str, Any]]
     jobDeploymentStatus: str
-    rayClusterName: str
-    rayClusterStatus: RayClusterStatus
-    startTime: str
+    rayClusterName: NotRequired[str]
+    rayClusterStatus: NotRequired[RayClusterStatus]
+    startTime: NotRequired[str]
 
     dashboardURL: NotRequired[str]
     endTime: NotRequired[str]
@@ -55,7 +55,25 @@ def __init__(
         super().__init__(group=GROUP, version=VERSION, kind=KIND, plural=PLURAL, api_client=api_client)
 
     def get_ray_cluster_name(self, name: str, namespace: str, timeout: float, poll_interval: float = 1.0) -> str:
-        return self.get_status(name, namespace, timeout=timeout, poll_interval=poll_interval)["rayClusterName"]
+        start_time = time.time()
+        status = None
+
+        while not time.time() - start_time > timeout:
+            status = self.get_status(name, namespace, timeout=timeout, poll_interval=poll_interval)
+            if ray_cluster_name := status.get("rayClusterName"):
+                return ray_cluster_name
+
+            logger.debug(
+                f"RayJob {namespace}/{name} status does not yet contain rayClusterName. "
+                f"Current status: {status}. Waiting..."
+            )
+
+            time.sleep(poll_interval)
+
+        raise TimeoutError(
+            f"Timed out ({timeout:.1f}s) waiting for rayClusterName in RayJob {namespace}/{name} status. "
+            f"Last status: {status}"
+        )
 
     def get_job_submission_id(
         self, name: str, namespace: str, timeout: float, poll_interval: float = 1.0

tests/kuberay/conftest.py

@@ -134,9 +134,19 @@ def k8s_with_kuberay(
         args.extend(
             [
                 "--set",
-                "featureGates\\[0\\].name=RayClusterStatusConditions",
+                "featureGates[0].name=RayClusterStatusConditions",
                 "--set",
-                "featureGates\\[0\\].enabled=true",
+                "featureGates[0].enabled=true",
+            ]
+        )
+
+    if Version(kuberay_version) >= Version("1.5.0"):
+        args.extend(
+            [
+                "--set",
+                "featureGates[1].name=RayJobDeletionPolicy",
+                "--set",
+                "featureGates[1].enabled=true",
             ]
         )