Skip to content

Commit 5e8f03c

Browse files
brettmthompsonpre-commit-ci[bot]
brettmthompson
and
pre-commit-ci[bot]
authored
adding e2e tests for openshift route timeout (#332)
* adding e2e tests for openshift route timeout Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> * adding pytests dependency markers Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> * removing unused fixture Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> * fixing wrong test deps Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> * adding comment to sleep statements for pre-commit to ignore Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * creating fixture to update isvc with annotations and removing use of sleep statements in rout timeout tests Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> * code rabbit suggestions Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> * updating fixture and small changes Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> --------- Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent 453d98c commit 5e8f03c

File tree

5 files changed

+269
-0
lines changed

5 files changed

+269
-0
lines changed

‎tests/model_serving/model_server/conftest.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from ocp_resources.inference_service import InferenceService
1010
from ocp_resources.namespace import Namespace
1111
from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
12+
from ocp_resources.resource import ResourceEditor
1213
from ocp_resources.secret import Secret
1314
from ocp_resources.service_account import ServiceAccount
1415
from ocp_resources.serving_runtime import ServingRuntime
@@ -138,6 +139,23 @@ def s3_models_inference_service(
138139
yield isvc
139140

140141

142+
@pytest.fixture(scope="function")
143+
def s3_models_inference_service_patched_annotations(
144+
request: FixtureRequest, s3_models_inference_service: InferenceService
145+
) -> InferenceService:
146+
if hasattr(request, "param"):
147+
with ResourceEditor(
148+
patches={
149+
s3_models_inference_service: {
150+
"metadata": {
151+
"annotations": request.param["annotations"],
152+
}
153+
}
154+
}
155+
):
156+
yield s3_models_inference_service
157+
158+
141159
@pytest.fixture(scope="class")
142160
def model_pvc(
143161
request: FixtureRequest,

‎tests/model_serving/model_server/routes/test_raw_deployment.py‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,21 @@
22

33
from tests.model_serving.model_server.utils import verify_inference_response
44
from utilities.constants import (
5+
Annotations,
56
KServeDeploymentType,
67
Labels,
78
ModelFormat,
89
ModelStoragePath,
10+
OpenshiftRouteTimeout,
911
Protocols,
1012
ModelInferenceRuntime,
1113
RuntimeTemplates,
1214
)
15+
from utilities.exceptions import (
16+
InferenceResponseError,
17+
)
1318
from utilities.inference_utils import Inference
19+
from utilities.infra import wait_for_route_timeout
1420
from utilities.manifests.caikit_tgis import CAIKIT_TGIS_INFERENCE_CONFIG
1521

1622
pytestmark = [pytest.mark.usefixtures("valid_aws_config"), pytest.mark.rawdeployment]
@@ -98,6 +104,71 @@ def test_disabled_rest_raw_deployment_exposed_route(self, patched_s3_caikit_kser
98104
)
99105

100106

107+
@pytest.mark.parametrize(
108+
"unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service",
109+
[
110+
pytest.param(
111+
{"name": "raw-deployment-caikit-flan-rest-timeout"},
112+
{
113+
"name": f"{Protocols.HTTP}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}",
114+
"template-name": RuntimeTemplates.CAIKIT_TGIS_SERVING,
115+
"multi-model": False,
116+
"enable-http": True,
117+
"enable-grpc": False,
118+
},
119+
{
120+
"name": f"{Protocols.HTTP}-{ModelFormat.CAIKIT}",
121+
"deployment-mode": KServeDeploymentType.RAW_DEPLOYMENT,
122+
"model-dir": ModelStoragePath.FLAN_T5_SMALL_CAIKIT,
123+
"external-route": True,
124+
},
125+
)
126+
],
127+
indirect=True,
128+
)
129+
class TestRestRawDeploymentRoutesTimeout:
130+
@pytest.mark.dependency(name="test_rest_raw_deployment_exposed_route")
131+
def test_rest_raw_deployment_exposed_route(self, s3_models_inference_service):
132+
"""Test HTTP inference using exposed (external) route"""
133+
verify_inference_response(
134+
inference_service=s3_models_inference_service,
135+
inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
136+
inference_type=Inference.ALL_TOKENS,
137+
protocol=Protocols.HTTPS,
138+
model_name=ModelFormat.CAIKIT,
139+
use_default_query=True,
140+
)
141+
142+
@pytest.mark.parametrize(
143+
"s3_models_inference_service_patched_annotations",
144+
[
145+
pytest.param({
146+
"annotations": {Annotations.HaproxyRouterOpenshiftIo.TIMEOUT: OpenshiftRouteTimeout.TIMEOUT_1MICROSEC}
147+
})
148+
],
149+
indirect=True,
150+
)
151+
@pytest.mark.dependency(depends=["test_rest_raw_deployment_exposed_route"])
152+
def test_rest_raw_deployment_exposed_route_with_timeout(self, s3_models_inference_service_patched_annotations):
153+
"""Test HTTP inference using exposed (external) route fails when timeout is set too low"""
154+
wait_for_route_timeout(
155+
name=s3_models_inference_service_patched_annotations.name,
156+
namespace=s3_models_inference_service_patched_annotations.namespace,
157+
route_timeout=OpenshiftRouteTimeout.TIMEOUT_1MICROSEC,
158+
)
159+
160+
with pytest.raises(InferenceResponseError) as ire:
161+
verify_inference_response(
162+
inference_service=s3_models_inference_service_patched_annotations,
163+
inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
164+
inference_type=Inference.ALL_TOKENS,
165+
protocol=Protocols.HTTPS,
166+
model_name=ModelFormat.CAIKIT,
167+
use_default_query=True,
168+
)
169+
assert "504 Gateway Time-out" in str(ire)
170+
171+
101172
@pytest.mark.parametrize(
102173
"unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service",
103174
[
@@ -151,3 +222,69 @@ def test_grpc_raw_deployment_exposed_route(self, patched_s3_caikit_kserve_isvc_v
151222
model_name=ModelFormat.CAIKIT,
152223
use_default_query=True,
153224
)
225+
226+
227+
@pytest.mark.parametrize(
228+
"unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service",
229+
[
230+
pytest.param(
231+
{"name": "raw-deployment-caikit-flan-grpc-timeout"},
232+
{
233+
"name": f"{Protocols.HTTP}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}",
234+
"template-name": RuntimeTemplates.CAIKIT_TGIS_SERVING,
235+
"multi-model": False,
236+
"enable-grpc": True,
237+
"enable-http": False,
238+
},
239+
{
240+
"name": f"{Protocols.GRPC}-{ModelFormat.CAIKIT}",
241+
"deployment-mode": KServeDeploymentType.RAW_DEPLOYMENT,
242+
"model-dir": ModelStoragePath.FLAN_T5_SMALL_CAIKIT,
243+
"external-route": True,
244+
},
245+
)
246+
],
247+
indirect=True,
248+
)
249+
@pytest.mark.jira("RHOAIENG-17783", run=False)
250+
class TestGrpcRawDeploymentTimeout:
251+
@pytest.mark.dependency(name="test_grpc_raw_deployment_exposed_route")
252+
def test_grpc_raw_deployment_exposed_route(self, s3_models_inference_service):
253+
"""Test GRPC inference using exposed (external) route"""
254+
verify_inference_response(
255+
inference_service=s3_models_inference_service,
256+
inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
257+
inference_type=Inference.STREAMING,
258+
protocol=Protocols.GRPC,
259+
model_name=ModelFormat.CAIKIT,
260+
use_default_query=True,
261+
)
262+
263+
@pytest.mark.parametrize(
264+
"s3_models_inference_service_patched_annotations",
265+
[
266+
pytest.param({
267+
"annotations": {Annotations.HaproxyRouterOpenshiftIo.TIMEOUT: OpenshiftRouteTimeout.TIMEOUT_1MICROSEC}
268+
})
269+
],
270+
indirect=True,
271+
)
272+
@pytest.mark.dependency(depends=["test_grpc_raw_deployment_exposed_route"])
273+
def test_grpc_raw_deployment_exposed_route_with_timeout(self, s3_models_inference_service_patched_annotations):
274+
"""Test GRPC inference using exposed (external) route fails when timeout is set too low"""
275+
wait_for_route_timeout(
276+
name=s3_models_inference_service_patched_annotations.name,
277+
namespace=s3_models_inference_service_patched_annotations.namespace,
278+
route_timeout=OpenshiftRouteTimeout.TIMEOUT_1MICROSEC,
279+
)
280+
281+
with pytest.raises(InferenceResponseError) as ire:
282+
verify_inference_response(
283+
inference_service=s3_models_inference_service_patched_annotations,
284+
inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
285+
inference_type=Inference.STREAMING,
286+
protocol=Protocols.GRPC,
287+
model_name=ModelFormat.CAIKIT,
288+
use_default_query=True,
289+
)
290+
assert "504 Gateway Time-out" in str(ire)

‎tests/model_serving/model_server/routes/test_serverless.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
import pytest
22

33
from tests.model_serving.model_server.utils import verify_inference_response
4+
from timeout_sampler import TimeoutExpiredError
45
from utilities.constants import (
6+
Annotations,
57
KServeDeploymentType,
68
Labels,
79
ModelFormat,
810
ModelStoragePath,
11+
OpenshiftRouteTimeout,
912
Protocols,
1013
ModelInferenceRuntime,
1114
RuntimeTemplates,
1215
)
1316
from utilities.inference_utils import Inference
17+
from utilities.infra import wait_for_route_timeout
1418
from utilities.manifests.caikit_tgis import CAIKIT_TGIS_INFERENCE_CONFIG
1519

1620
pytestmark = [pytest.mark.usefixtures("valid_aws_config"), pytest.mark.serverless, pytest.mark.sanity]
@@ -94,3 +98,69 @@ def test_rest_serverless_exposed_label_route(self, patched_s3_caikit_kserve_isvc
9498
model_name=ModelFormat.CAIKIT,
9599
use_default_query=True,
96100
)
101+
102+
103+
@pytest.mark.parametrize(
104+
"unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service",
105+
[
106+
pytest.param(
107+
{"name": "serverless-rcaikit-routes-timeout"},
108+
{
109+
"name": f"{Protocols.HTTP}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}",
110+
"template-name": RuntimeTemplates.CAIKIT_TGIS_SERVING,
111+
"multi-model": False,
112+
"enable-http": True,
113+
},
114+
{
115+
"name": f"{Protocols.HTTP}-{ModelFormat.CAIKIT}",
116+
"deployment-mode": KServeDeploymentType.SERVERLESS,
117+
"model-dir": ModelStoragePath.FLAN_T5_SMALL_CAIKIT,
118+
"external-route": True,
119+
},
120+
)
121+
],
122+
indirect=True,
123+
)
124+
class TestRestServerlessRoutesTimeout:
125+
@pytest.mark.dependency(name="test_rest_serverless_external_route")
126+
def test_rest_serverless_external_route(self, s3_models_inference_service):
127+
"""Test HTTP inference using exposed (external) route"""
128+
verify_inference_response(
129+
inference_service=s3_models_inference_service,
130+
inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
131+
inference_type=Inference.ALL_TOKENS,
132+
protocol=Protocols.HTTPS,
133+
model_name=ModelFormat.CAIKIT,
134+
use_default_query=True,
135+
)
136+
137+
@pytest.mark.parametrize(
138+
"s3_models_inference_service_patched_annotations",
139+
[
140+
pytest.param({
141+
"annotations": {Annotations.HaproxyRouterOpenshiftIo.TIMEOUT: OpenshiftRouteTimeout.TIMEOUT_1MICROSEC}
142+
})
143+
],
144+
indirect=True,
145+
)
146+
@pytest.mark.dependency(depends=["test_rest_serverless_external_route"])
147+
def test_rest_serverless_external_route_with_timeout(self, s3_models_inference_service_patched_annotations):
148+
"""Test HTTP inference using external route fails when timeout is set too low"""
149+
wait_for_route_timeout(
150+
name=(
151+
f"{s3_models_inference_service_patched_annotations.name}-"
152+
f"{s3_models_inference_service_patched_annotations.namespace}"
153+
),
154+
namespace="istio-system",
155+
route_timeout=OpenshiftRouteTimeout.TIMEOUT_1MICROSEC,
156+
)
157+
158+
with pytest.raises(TimeoutExpiredError):
159+
verify_inference_response(
160+
inference_service=s3_models_inference_service_patched_annotations,
161+
inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
162+
inference_type=Inference.ALL_TOKENS,
163+
protocol=Protocols.HTTPS,
164+
model_name=ModelFormat.CAIKIT,
165+
use_default_query=True,
166+
)

‎utilities/constants.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ class AcceleratorType:
118118

119119

120120
class ApiGroups:
121+
HAPROXY_ROUTER_OPENSHIFT_IO: str = "haproxy.router.openshift.io"
121122
OPENDATAHUB_IO: str = "opendatahub.io"
122123

123124

@@ -138,6 +139,9 @@ class OpenDataHubIo:
138139
MANAGED: str = f"{ApiGroups.OPENDATAHUB_IO}/managed"
139140
SERVICE_MESH: str = f"{ApiGroups.OPENDATAHUB_IO}/service-mesh"
140141

142+
class HaproxyRouterOpenshiftIo:
143+
TIMEOUT: str = f"{ApiGroups.HAPROXY_ROUTER_OPENSHIFT_IO}/timeout"
144+
141145

142146
class StorageClassName:
143147
NFS: str = "nfs"
@@ -207,6 +211,10 @@ class Timeout:
207211
TIMEOUT_20MIN: int = 20 * TIMEOUT_1MIN
208212

209213

214+
class OpenshiftRouteTimeout:
215+
TIMEOUT_1MICROSEC: str = "1us"
216+
217+
210218
class Containers:
211219
KSERVE_CONTAINER_NAME: str = "kserve-container"
212220

‎utilities/infra.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -905,6 +905,42 @@ def get_operator_distribution(client: DynamicClient, dsc_name: str = "default-ds
905905
raise ValueError("DSC release name not found in {dsc_name}")
906906

907907

908+
def wait_for_route_timeout(name: str, namespace: str, route_timeout: str) -> None:
909+
"""
910+
Wait for route to be annotated with timeout value.
911+
Given that there is a delay between the openshift route timeout annotation being set
912+
and the timeout being applied to the route, a counter is instituted to wait until the
913+
annotation is found in the route twice. This allows for the TimeoutSampler sleep time
914+
to be executed and the route timeout to be successfully applied.
915+
916+
Args:
917+
name (str): Name of the route.
918+
namespace (str): Namespace the route is located in.
919+
route_timeout (str): The expected value of the openshift route timeout annotation.
920+
921+
Raises:
922+
TimeoutExpiredError: If route annotation is not set to the expected value before timeout expires.
923+
"""
924+
annotation_found_count = 0
925+
for route in TimeoutSampler(
926+
wait_timeout=Timeout.TIMEOUT_30SEC,
927+
sleep=10,
928+
exceptions_dict={ResourceNotFoundError: []},
929+
func=Route,
930+
name=name,
931+
namespace=namespace,
932+
ensure_exists=True,
933+
):
934+
if (
935+
route.instance.metadata.get("annotations", {}).get(Annotations.HaproxyRouterOpenshiftIo.TIMEOUT)
936+
!= route_timeout
937+
):
938+
continue
939+
annotation_found_count += 1
940+
if annotation_found_count == 2:
941+
return
942+
943+
908944
def wait_for_serverless_pods_deletion(resource: Project | Namespace, admin_client: DynamicClient | None) -> None:
909945
"""
910946
Wait for serverless pods deletion.

0 commit comments

Comments
 (0)