Add test steps in test

Signed-off-by: Milind waykole <mwaykole@redhat.com>

Author: mwaykole

tests/model_serving/model_server/kserve/authentication/test_kserve_token_authentication_raw.py

@@ -21,6 +21,16 @@
     indirect=True,
 )
 class TestKserveTokenAuthenticationRawForRest:
+    """Validate KServe raw deployment token-based authentication for REST inference.
+
+    Steps:
+        1. Deploy an OVMS model with authentication enabled in a raw deployment namespace.
+        2. Query the model with a valid token and verify a successful REST inference response.
+        3. Disable authentication and verify the model is still queryable without a token.
+        4. Re-enable authentication and verify the model requires a valid token again.
+        5. Attempt cross-model authentication using another model's token and verify access is denied.
+    """
+
     @pytest.mark.smoke
     @pytest.mark.ocp_interop
     @pytest.mark.dependency(name="test_model_authentication_using_rest_raw")

tests/model_serving/model_server/kserve/authentication/test_non_admin_users.py

@@ -21,6 +21,14 @@
 @pytest.mark.smoke
 @pytest.mark.rawdeployment
 class TestRawUnprivilegedUser:
+    """Validate that a non-admin user can deploy and query a KServe raw deployment model.
+
+    Steps:
+        1. Create a namespace with unprivileged user credentials.
+        2. Deploy an OVMS model as a raw deployment using the non-admin user.
+        3. Query the deployed model via REST and verify a successful inference response.
+    """
+
     def test_non_admin_deploy_raw_and_query_model(
         self,
         unprivileged_s3_ovms_raw_inference_service,

tests/model_serving/model_server/kserve/inference_graph/test_inference_graph_raw.py

@@ -19,12 +19,22 @@
     indirect=True,
 )
 class TestInferenceGraphRaw:
+    """Validate KServe InferenceGraph functionality in raw deployment mode.
+
+    Steps:
+        1. Deploy an OVMS serving runtime with ONNX support in the test namespace.
+        2. Create inference graphs with various configurations (public, private, auth-enabled).
+        3. Send inference requests through each graph and verify correct routing and responses.
+        4. Verify authentication enforcement by testing with privileged and unprivileged tokens.
+    """
+
     @pytest.mark.parametrize(
         "dog_breed_inference_graph",
         [{"name": "dog-breed-raw-pipeline", "deployment-mode": KServeDeploymentType.RAW_DEPLOYMENT}],
         indirect=True,
     )
     def test_inference_graph_raw_deployment(self, dog_breed_inference_graph):
+        """Verify inference through a public raw deployment inference graph."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -46,6 +56,7 @@ def test_inference_graph_raw_deployment(self, dog_breed_inference_graph):
         indirect=True,
     )
     def test_private_inference_graph_raw_deployment(self, dog_breed_inference_graph):
+        """Verify inference through a private (no external route) raw deployment inference graph."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -69,6 +80,7 @@ def test_private_inference_graph_raw_deployment(self, dog_breed_inference_graph)
         indirect=True,
     )
     def test_inference_graph_raw_authentication(self, dog_breed_inference_graph, inference_graph_sa_token_with_access):
+        """Verify inference through an auth-enabled inference graph using an authorized service account token."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -94,6 +106,7 @@ def test_inference_graph_raw_authentication(self, dog_breed_inference_graph, inf
     def test_private_inference_graph_raw_authentication(
         self, dog_breed_inference_graph, inference_graph_sa_token_with_access
     ):
+        """Verify inference through a private auth-enabled inference graph using an authorized token."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -119,6 +132,7 @@ def test_private_inference_graph_raw_authentication(
     def test_inference_graph_raw_authentication_without_privileges(
         self, dog_breed_inference_graph, inference_graph_unprivileged_sa_token
     ):
+        """Verify that an unprivileged token is denied access to an auth-enabled inference graph."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -145,6 +159,7 @@ def test_inference_graph_raw_authentication_without_privileges(
     def test_private_inference_graph_raw_authentication_without_privileges(
         self, dog_breed_inference_graph, inference_graph_unprivileged_sa_token
     ):
+        """Verify that an unprivileged token is denied access to a private auth-enabled inference graph."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,

tests/model_serving/model_server/kserve/inference_service_lifecycle/test_isvc_env_vars_updates.py

@@ -43,6 +43,15 @@
     indirect=True,
 )
 class TestRawISVCEnvVarsUpdates:
+    """Validate adding and removing environment variables on a KServe raw deployment ISVC.
+
+    Steps:
+        1. Deploy an OVMS inference service with custom environment variables.
+        2. Verify the environment variables are present in the predictor pods.
+        3. Remove the environment variables from the inference service.
+        4. Verify the environment variables are no longer present in the predictor pods.
+    """
+
     def test_raw_with_isvc_env_vars(self, ovms_kserve_inference_service):
         """Test adding environment variables to the inference service"""
         verify_env_vars_in_isvc_pods(isvc=ovms_kserve_inference_service, env_vars=ISVC_ENV_VARS, vars_exist=True)

tests/model_serving/model_server/kserve/inference_service_lifecycle/test_isvc_pull_secret_updates.py

@@ -24,6 +24,15 @@
     indirect=True,
 )
 class TestISVCPullSecretUpdate:
+    """Validate pull secret lifecycle operations on a KServe model car inference service.
+
+    Steps:
+        1. Deploy a model car ISVC with an initial pull secret attached.
+        2. Verify the initial pull secret is correctly set in the predictor pod.
+        3. Update the pull secret to a new value and verify it is reflected in the new pod.
+        4. Remove the pull secret and verify it is no longer present in the pod.
+    """
+
     @pytest.mark.tier1
     def test_initial_pull_secret_set(self, model_car_raw_inference_service_with_pull_secret):
         """Ensure initial pull secret is correctly set in the pod"""
@@ -36,4 +45,5 @@ def test_update_pull_secret(self, updated_isvc_pull_secret):
         verify_pull_secret(isvc=updated_isvc_pull_secret, pull_secret=UPDATED_PULL_SECRET, secret_exists=True)
 
     def test_remove_pull_secret(self, updated_isvc_remove_pull_secret):
+        """Remove the pull secret and verify it is no longer present in the pod."""
         verify_pull_secret(isvc=updated_isvc_remove_pull_secret, pull_secret=UPDATED_PULL_SECRET, secret_exists=False)

tests/model_serving/model_server/kserve/inference_service_lifecycle/test_isvc_replicas_update.py

@@ -39,6 +39,16 @@
     indirect=True,
 )
 class TestRawISVCReplicasUpdates:
+    """Validate scaling replica count up and down on a KServe raw deployment ISVC.
+
+    Steps:
+        1. Deploy an OVMS inference service with 2 min-replicas and 4 max-replicas.
+        2. Verify that 2 predictor pods are running after deployment.
+        3. Run inference to confirm the model responds correctly with multiple replicas.
+        4. Patch the ISVC to scale down to 1 replica and verify only 1 pod remains.
+        5. Run inference again to confirm the model responds correctly after scale-down.
+    """
+
     @pytest.mark.dependency(name="test_raw_increase_isvc_replicas")
     def test_raw_increase_isvc_replicas(self, isvc_pods, ovms_kserve_inference_service):
         """Test replicas increase"""

tests/model_serving/model_server/kserve/inference_service_lifecycle/test_raw_stop_resume_model.py

@@ -33,6 +33,15 @@
     indirect=True,
 )
 class TestStopRaw:
+    """Validate stopping a running KServe raw deployment model via the stop annotation.
+
+    Steps:
+        1. Deploy an OVMS ONNX model as a raw deployment with stop set to false.
+        2. Verify the model can be queried via REST.
+        3. Patch the ISVC stop annotation to true.
+        4. Verify all predictor pods are deleted and remain absent.
+    """
+
     def test_raw_onnx_rest_inference(
         self, unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_raw_inference_service
     ):
@@ -58,8 +67,7 @@ def test_stop_and_update_to_true_delete_pod_rollout(
         ovms_raw_inference_service,
         patched_raw_inference_service_stop_annotation,
     ):
-        """Verify pod rollout is deleted when the stop annotation updated to true"""
-        """Verify pods do not exist"""
+        """Verify pod rollout is deleted when the stop annotation is updated to true."""
         result = consistently_verify_no_pods_exist(
             client=unprivileged_client,
             isvc=patched_raw_inference_service_stop_annotation,
@@ -86,15 +94,23 @@ def test_stop_and_update_to_true_delete_pod_rollout(
     indirect=True,
 )
 class TestStoppedResumeRaw:
+    """Validate resuming a stopped KServe raw deployment model by clearing the stop annotation.
+
+    Steps:
+        1. Deploy an OVMS ONNX model as a raw deployment with stop set to true.
+        2. Verify no predictor pods are created while the stop annotation is true.
+        3. Patch the ISVC stop annotation to false.
+        4. Verify predictor pods are rolled out and the model can be queried via REST.
+    """
+
     def test_stop_and_true_no_pod_rollout(
         self,
         unprivileged_client,
         unprivileged_model_namespace,
         ovms_kserve_serving_runtime,
         ovms_raw_inference_service,
     ):
-        """Verify no pod rollout when the stop annotation is true"""
-        """Verify pods do not exist"""
+        """Verify no pod rollout when the stop annotation is true."""
         result = consistently_verify_no_pods_exist(
             client=unprivileged_client,
             isvc=ovms_raw_inference_service,
@@ -114,8 +130,7 @@ def test_stop_and_update_to_false_pod_rollout(
         ovms_raw_inference_service,
         patched_raw_inference_service_stop_annotation,
     ):
-        """Verify pod rollout when the stop annotation is updated to false"""
-        """Verify that kserve Raw ONNX model can be queried using REST"""
+        """Verify pod rollout and REST inference after the stop annotation is set to false."""
         verify_inference_response(
             inference_service=patched_raw_inference_service_stop_annotation,
             inference_config=ONNX_INFERENCE_CONFIG,

tests/model_serving/model_server/kserve/ingress/test_route_reconciliation.py

@@ -39,11 +39,11 @@ def test_raw_onnx_rout_reconciliation(self, ovms_raw_inference_service):
         )
 
     def test_route_value_before_and_after_deletion(self, unprivileged_client, ovms_raw_inference_service):
-        # Validate ingress status before and after route deletion
+        """Verify that the ingress status changes after the route is deleted and recreated."""
         assert_ingress_status_changed(client=unprivileged_client, inference_service=ovms_raw_inference_service)
 
     def test_model_works_after_route_is_recreated(self, ovms_raw_inference_service):
-        # Final inference validation after route update
+        """Verify that the model is still queryable via REST after the route is recreated."""
         verify_inference_response(
             inference_service=ovms_raw_inference_service,
             inference_config=ONNX_INFERENCE_CONFIG,

tests/model_serving/model_server/kserve/ingress/test_route_visibility.py

@@ -44,6 +44,16 @@
     indirect=True,
 )
 class TestRestRawDeploymentRoutes:
+    """Validate REST route visibility transitions for KServe raw deployment.
+
+    Steps:
+        1. Deploy a Caikit-TGIS model as a raw deployment with HTTP enabled.
+        2. Verify the default route visibility label is not set.
+        3. Query the model via the internal route and confirm a successful response.
+        4. Patch the ISVC to expose the route externally and verify inference over HTTPS.
+        5. Revert the route to local-cluster visibility and verify external access is disabled.
+    """
+
     def test_default_visibility_value(self, s3_models_inference_service):
         """Test default route visibility value"""
         if labels := s3_models_inference_service.labels:
@@ -126,6 +136,15 @@ def test_disabled_rest_raw_deployment_exposed_route(self, patched_s3_caikit_kser
     indirect=True,
 )
 class TestRestRawDeploymentRoutesTimeout:
+    """Validate REST route timeout behavior for KServe raw deployment.
+
+    Steps:
+        1. Deploy a Caikit-TGIS model as a raw deployment with an external route.
+        2. Verify inference succeeds over the exposed HTTPS route.
+        3. Patch the route with an extremely low timeout annotation.
+        4. Verify inference fails with a 504 Gateway Time-out error.
+    """
+
     @pytest.mark.dependency(name="test_rest_raw_deployment_exposed_route")
     def test_rest_raw_deployment_exposed_route(self, s3_models_inference_service):
         """Test HTTP inference using exposed (external) route"""
@@ -191,6 +210,14 @@ def test_rest_raw_deployment_exposed_route_with_timeout(self, s3_models_inferenc
 )
 @pytest.mark.skip(reason="skipping grpc raw for tgis-caikit")
 class TestGrpcRawDeployment:
+    """Validate gRPC route visibility transitions for KServe raw deployment.
+
+    Steps:
+        1. Deploy a Caikit-TGIS model as a raw deployment with gRPC enabled.
+        2. Query the model via the internal gRPC route and confirm a successful streaming response.
+        3. Patch the ISVC to expose the route externally and verify gRPC inference over the exposed route.
+    """
+
     def test_grpc_raw_deployment_internal_route(self, s3_models_inference_service):
         """Test GRPC inference using internal route"""
         verify_inference_response(
@@ -247,6 +274,15 @@ def test_grpc_raw_deployment_exposed_route(self, patched_s3_caikit_kserve_isvc_v
 )
 @pytest.mark.skip(reason="skipping grpc raw for tgis-caikit")
 class TestGrpcRawDeploymentTimeout:
+    """Validate gRPC route timeout behavior for KServe raw deployment.
+
+    Steps:
+        1. Deploy a Caikit-TGIS model as a raw deployment with an external gRPC route.
+        2. Verify gRPC inference succeeds over the exposed route.
+        3. Patch the route with an extremely low timeout annotation.
+        4. Verify gRPC inference fails with a 504 Gateway Time-out error.
+    """
+
     @pytest.mark.dependency(name="test_grpc_raw_deployment_exposed_route")
     def test_grpc_raw_deployment_exposed_route(self, s3_models_inference_service):
         """Test GRPC inference using exposed (external) route"""

tests/model_serving/model_server/kserve/multi_node/test_nvidia_multi_node.py

@@ -47,6 +47,19 @@
     indirect=True,
 )
 class TestMultiNode:
+    """Validate multi-node GPU inference with Ray-based vLLM serving on KServe.
+
+    Steps:
+        1. Deploy a Granite-8B model on a multi-node vLLM inference service backed by PVC storage.
+        2. Verify Ray cluster health and NVIDIA GPU status across head and worker pods.
+        3. Validate default runtime worker spec (tensorParallelSize=1, pipelineParallelSize=2).
+        4. Confirm pods are distributed across GPU nodes and TLS certificates are provisioned.
+        5. Test pod resilience by deleting head and worker pods and verifying inference recovery.
+        6. Validate TLS secret reconciliation, runtime deletion cleanup, and runtime re-creation.
+        7. Test external route inference and dynamic updates to tensor/pipeline parallel sizes.
+        8. Verify model args propagation to and removal from the vLLM command spec.
+    """
+
     def test_multi_node_ray_status(self, multi_node_predictor_pods_scope_class):
         """Test multi node ray status"""
         verify_ray_status(pods=multi_node_predictor_pods_scope_class)