Skip to content

Commit 6b722d9

Browse files
Raghul-MRaghul-M
authored
Merge branch 'main' into model-validation-v2
2 parents f0710ba + 50e3681 commit 6b722d9

File tree

21 files changed

+353
-55
lines changed

21 files changed

+353
-55
lines changed

‎tests/model_registry/async_job/__init__.py‎

Whitespace-only changes.

‎tests/model_registry/async_job/conftest.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import pytest
2+
from typing import Any, Generator
3+
import shortuuid
4+
from pytest import FixtureRequest
5+
6+
from ocp_resources.namespace import Namespace
7+
from ocp_resources.pod import Pod
8+
from ocp_resources.route import Route
9+
from ocp_resources.service import Service
10+
11+
from utilities.infra import create_ns
12+
from utilities.constants import OCIRegistry, MinIo, Protocols, Labels
13+
14+
from kubernetes.dynamic import DynamicClient
15+
16+
17+
# OCI Registry
18+
@pytest.fixture(scope="class")
19+
def oci_namespace(admin_client: DynamicClient) -> Generator[Namespace, Any, Any]:
20+
with create_ns(
21+
name=f"{OCIRegistry.Metadata.NAME}-{shortuuid.uuid().lower()}",
22+
admin_client=admin_client,
23+
) as ns:
24+
yield ns
25+
26+
27+
@pytest.fixture(scope="class")
28+
def oci_registry_pod_with_minio(
29+
request: FixtureRequest,
30+
admin_client: DynamicClient,
31+
oci_namespace: Namespace,
32+
minio_service: Service,
33+
) -> Generator[Pod, Any, Any]:
34+
pod_labels = {Labels.Openshift.APP: OCIRegistry.Metadata.NAME}
35+
36+
if labels := request.param.get("labels"):
37+
pod_labels.update(labels)
38+
39+
minio_fqdn = f"{minio_service.name}.{minio_service.namespace}.svc.cluster.local"
40+
minio_endpoint = f"{minio_fqdn}:{MinIo.Metadata.DEFAULT_PORT}"
41+
42+
with Pod(
43+
client=admin_client,
44+
name=OCIRegistry.Metadata.NAME,
45+
namespace=oci_namespace.name,
46+
containers=[
47+
{
48+
"args": request.param.get("args"),
49+
"env": [
50+
{"name": "ZOT_STORAGE_STORAGEDRIVER_NAME", "value": OCIRegistry.Storage.STORAGE_DRIVER},
51+
{
52+
"name": "ZOT_STORAGE_STORAGEDRIVER_ROOTDIRECTORY",
53+
"value": OCIRegistry.Storage.STORAGE_DRIVER_ROOT_DIRECTORY,
54+
},
55+
{"name": "ZOT_STORAGE_STORAGEDRIVER_BUCKET", "value": MinIo.Buckets.MODELMESH_EXAMPLE_MODELS},
56+
{"name": "ZOT_STORAGE_STORAGEDRIVER_REGION", "value": OCIRegistry.Storage.STORAGE_DRIVER_REGION},
57+
{"name": "ZOT_STORAGE_STORAGEDRIVER_REGIONENDPOINT", "value": f"http://{minio_endpoint}"},
58+
{"name": "ZOT_STORAGE_STORAGEDRIVER_ACCESSKEY", "value": MinIo.Credentials.ACCESS_KEY_VALUE},
59+
{"name": "ZOT_STORAGE_STORAGEDRIVER_SECRETKEY", "value": MinIo.Credentials.SECRET_KEY_VALUE},
60+
{
61+
"name": "ZOT_STORAGE_STORAGEDRIVER_SECURE",
62+
"value": OCIRegistry.Storage.STORAGE_STORAGEDRIVER_SECURE,
63+
},
64+
{
65+
"name": "ZOT_STORAGE_STORAGEDRIVER_FORCEPATHSTYLE",
66+
"value": OCIRegistry.Storage.STORAGE_STORAGEDRIVER_FORCEPATHSTYLE,
67+
},
68+
{"name": "ZOT_HTTP_ADDRESS", "value": OCIRegistry.Metadata.DEFAULT_HTTP_ADDRESS},
69+
{"name": "ZOT_HTTP_PORT", "value": str(OCIRegistry.Metadata.DEFAULT_PORT)},
70+
{"name": "ZOT_LOG_LEVEL", "value": "info"},
71+
],
72+
"image": request.param.get("image", OCIRegistry.PodConfig.REGISTRY_IMAGE),
73+
"name": OCIRegistry.Metadata.NAME,
74+
"securityContext": {
75+
"allowPrivilegeEscalation": False,
76+
"capabilities": {"drop": ["ALL"]},
77+
"runAsNonRoot": True,
78+
"seccompProfile": {"type": "RuntimeDefault"},
79+
},
80+
"volumeMounts": [
81+
{
82+
"name": "zot-data",
83+
"mountPath": "/var/lib/registry",
84+
}
85+
],
86+
}
87+
],
88+
volumes=[
89+
{
90+
"name": "zot-data",
91+
"emptyDir": {},
92+
}
93+
],
94+
label=pod_labels,
95+
annotations=request.param.get("annotations"),
96+
) as oci_pod:
97+
oci_pod.wait_for_condition(condition="Ready", status="True")
98+
yield oci_pod
99+
100+
101+
@pytest.fixture(scope="class")
102+
def oci_registry_service(admin_client: DynamicClient, oci_namespace: Namespace) -> Generator[Service, Any, Any]:
103+
with Service(
104+
client=admin_client,
105+
name=OCIRegistry.Metadata.NAME,
106+
namespace=oci_namespace.name,
107+
ports=[
108+
{
109+
"name": f"{OCIRegistry.Metadata.NAME}-port",
110+
"port": OCIRegistry.Metadata.DEFAULT_PORT,
111+
"protocol": Protocols.TCP,
112+
"targetPort": OCIRegistry.Metadata.DEFAULT_PORT,
113+
}
114+
],
115+
selector={
116+
Labels.Openshift.APP: OCIRegistry.Metadata.NAME,
117+
},
118+
session_affinity="ClientIP",
119+
) as oci_service:
120+
yield oci_service
121+
122+
123+
@pytest.fixture(scope="class")
124+
def oci_registry_route(admin_client: DynamicClient, oci_registry_service: Service) -> Generator[Route, Any, Any]:
125+
with Route(
126+
client=admin_client,
127+
name=OCIRegistry.Metadata.NAME,
128+
namespace=oci_registry_service.namespace,
129+
service=oci_registry_service.name,
130+
) as oci_route:
131+
yield oci_route

‎tests/model_registry/async_job/test_basic_oci_registry.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import pytest
2+
import json
3+
4+
from ocp_resources.route import Route
5+
6+
from utilities.constants import OCIRegistry, MinIo
7+
from simple_logger.logger import get_logger
8+
from tests.model_registry.async_job.utils import (
9+
push_blob_to_oci_registry,
10+
create_manifest,
11+
push_manifest_to_oci_registry,
12+
pull_manifest_from_oci_registry,
13+
)
14+
15+
LOGGER = get_logger(name=__name__)
16+
17+
18+
@pytest.mark.parametrize(
19+
"minio_pod, oci_registry_pod_with_minio",
20+
[
21+
pytest.param(
22+
MinIo.PodConfig.MODEL_MESH_MINIO_CONFIG,
23+
OCIRegistry.PodConfig.REGISTRY_BASE_CONFIG,
24+
)
25+
],
26+
indirect=True,
27+
)
28+
@pytest.mark.usefixtures("minio_pod", "oci_registry_pod_with_minio", "oci_registry_route")
29+
class TestOciRegistry:
30+
"""
31+
Temporary test for OCI registry deployment functionality using MinIO backend
32+
It will be replaced with a more comprehensive e2e test as part of https://issues.redhat.com/browse/RHOAISTRAT-456
33+
"""
34+
35+
def test_oci_registry_push_and_pull_operations(
36+
self,
37+
oci_registry_route: Route,
38+
) -> None:
39+
"""Test pushing and pulling content to/from the OCI registry with MinIO backend."""
40+
41+
registry_host = oci_registry_route.instance.spec.host
42+
registry_url = f"http://{registry_host}"
43+
44+
LOGGER.info(f"Testing OCI registry at: {registry_url}")
45+
test_repo = "test/simple-artifact"
46+
test_tag = "v1.0"
47+
test_data = b"Hello from OCI registry test! This could be model data stored in MinIO."
48+
49+
blob_digest = push_blob_to_oci_registry(registry_url=registry_url, data=test_data, repo=test_repo)
50+
51+
config_data = {"architecture": "amd64", "os": "linux"}
52+
config_json = json.dumps(config_data, separators=(",", ":")).encode("utf-8")
53+
config_digest = push_blob_to_oci_registry(registry_url=registry_url, data=config_json, repo=test_repo)
54+
55+
manifest = create_manifest(
56+
blob_digest=blob_digest, config_json=config_json, config_digest=config_digest, data=test_data
57+
)
58+
59+
push_manifest_to_oci_registry(registry_url=registry_url, manifest=manifest, repo=test_repo, tag=test_tag)
60+
61+
manifest_get = pull_manifest_from_oci_registry(registry_url=registry_url, repo=test_repo, tag=test_tag)
62+
63+
assert manifest_get["schemaVersion"] == 2
64+
assert manifest_get["config"]["digest"] == config_digest
65+
assert len(manifest_get["layers"]) == 1
66+
assert manifest_get["layers"][0]["digest"] == blob_digest

‎tests/model_registry/async_job/utils.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import hashlib
2+
import requests
3+
import json
4+
5+
from simple_logger.logger import get_logger
6+
7+
LOGGER = get_logger(name=__name__)
8+
9+
10+
def push_blob_to_oci_registry(registry_url: str, data: bytes, repo: str = "test/simple-artifact") -> str:
11+
"""
12+
Push a blob to an OCI registry.
13+
https://specs.opencontainers.org/distribution-spec/?v=v1.0.0#pushing-blobs
14+
POST to /v2/<repo>/blobs/uploads/ in order to initiate the upload
15+
The response will contain a Location header that contains the upload URL
16+
PUT to the Location URL with the data to be uploaded
17+
"""
18+
19+
blob_digest = f"sha256:{hashlib.sha256(data).hexdigest()}"
20+
21+
LOGGER.info(f"Pushing blob with digest: {blob_digest}")
22+
23+
upload_response = requests.post(f"{registry_url}/v2/{repo}/blobs/uploads/", timeout=10)
24+
LOGGER.info(f"Blob upload initiation: {upload_response.status_code}")
25+
assert upload_response.status_code == 202, f"Failed to initiate blob upload: {upload_response.status_code}"
26+
27+
upload_location = upload_response.headers.get("Location")
28+
LOGGER.info(f"Upload location: {upload_location}")
29+
base_url = f"{registry_url}{upload_location}"
30+
upload_url = f"{base_url}?digest={blob_digest}"
31+
response = requests.put(url=upload_url, data=data, headers={"Content-Type": "application/octet-stream"}, timeout=10)
32+
assert response.status_code == 201, f"Failed to upload blob: {response.status_code}"
33+
return blob_digest
34+
35+
36+
def create_manifest(blob_digest: str, config_json: str, config_digest: str, data: bytes) -> bytes:
37+
"""Create a manifest for an OCI registry."""
38+
39+
manifest = {
40+
"schemaVersion": 2,
41+
"mediaType": "application/vnd.oci.image.manifest.v1+json",
42+
"config": {
43+
"mediaType": "application/vnd.oci.image.config.v1+json",
44+
"size": len(config_json),
45+
"digest": config_digest,
46+
},
47+
"layers": [{"mediaType": "application/vnd.oci.image.layer.v1.tar", "size": len(data), "digest": blob_digest}],
48+
}
49+
50+
return json.dumps(manifest, separators=(",", ":")).encode("utf-8")
51+
52+
53+
def push_manifest_to_oci_registry(registry_url: str, manifest: bytes, repo: str, tag: str) -> None:
54+
"""Push a manifest to an OCI registry."""
55+
response = requests.put(
56+
f"{registry_url}/v2/{repo}/manifests/{tag}",
57+
data=manifest,
58+
headers={"Content-Type": "application/vnd.oci.image.manifest.v1+json"},
59+
timeout=10,
60+
)
61+
assert response.status_code == 201, f"Failed to push manifest: {response.status_code}"
62+
63+
64+
def pull_manifest_from_oci_registry(registry_url: str, repo: str, tag: str) -> dict:
65+
"""Pull a manifest from an OCI registry."""
66+
response = requests.get(
67+
f"{registry_url}/v2/{repo}/manifests/{tag}",
68+
headers={"Accept": "application/vnd.oci.image.manifest.v1+json"},
69+
timeout=10,
70+
)
71+
LOGGER.info(f"Manifest pull: {response.status_code}")
72+
assert response.status_code == 200, f"Failed to pull manifest: {response.status_code}"
73+
return response.json()

‎tests/model_registry/conftest.py‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@ def model_registry_db_service(
6262
admin_client: DynamicClient,
6363
teardown_resources: bool,
6464
model_registry_namespace: str,
65-
is_model_registry_oauth: bool,
6665
) -> Generator[list[Service], Any, Any]:
6766
num_resources = getattr(request, "param", {}).get("num_resources", 1)
6867
if pytestconfig.option.post_upgrade:
@@ -89,7 +88,6 @@ def model_registry_db_pvc(
8988
admin_client: DynamicClient,
9089
teardown_resources: bool,
9190
model_registry_namespace: str,
92-
is_model_registry_oauth: bool,
9391
) -> Generator[list[PersistentVolumeClaim], Any, Any]:
9492
num_resources = getattr(request, "param", {}).get("num_resources", 1)
9593
if pytestconfig.option.post_upgrade:
@@ -116,7 +114,6 @@ def model_registry_db_secret(
116114
admin_client: DynamicClient,
117115
teardown_resources: bool,
118116
model_registry_namespace: str,
119-
is_model_registry_oauth: bool,
120117
) -> Generator[list[Secret], Any, Any]:
121118
num_resources = getattr(request, "param", {}).get("num_resources", 1)
122119
if pytestconfig.option.post_upgrade:
@@ -143,7 +140,6 @@ def model_registry_db_deployment(
143140
admin_client: DynamicClient,
144141
teardown_resources: bool,
145142
model_registry_namespace: str,
146-
is_model_registry_oauth: bool,
147143
) -> Generator[list[Deployment], Any, Any]:
148144
num_resources = getattr(request, "param", {}).get("num_resources", 1)
149145
if pytestconfig.option.post_upgrade:
@@ -182,7 +178,6 @@ def model_registry_instance_mysql(
182178
admin_client: DynamicClient,
183179
teardown_resources: bool,
184180
model_registry_namespace: str,
185-
is_model_registry_oauth: bool,
186181
) -> Generator[list[Any], Any, Any]:
187182
"""Creates a model registry instance with oauth proxy configuration."""
188183
param = getattr(request, "param", {})
@@ -257,7 +252,7 @@ def updated_dsc_component_state_scope_class(
257252
dsc_resource.wait_for_condition(
258253
condition=DscComponents.COMPONENT_MAPPING[component_name], status="True"
259254
)
260-
namespace = Namespace(name=py_config["model_registry_namespace"], ensure_exists=True)
255+
namespace = Namespace(name=py_config["model_registry_namespace"], wait_for_resource=True)
261256
namespace.wait_for_status(status=Namespace.Status.ACTIVE)
262257
wait_for_pods_running(
263258
admin_client=admin_client,
@@ -455,11 +450,6 @@ def delete_mr_deployment() -> None:
455450
mr_deployment.delete(wait=True)
456451

457452

458-
@pytest.fixture(scope="class")
459-
def is_model_registry_oauth(request: FixtureRequest) -> bool:
460-
return getattr(request, "param", {}).get("use_oauth_proxy", True)
461-
462-
463453
@pytest.fixture(scope="session")
464454
def api_server_url(admin_client: DynamicClient) -> str:
465455
infrastructure = Infrastructure(client=admin_client, name="cluster", ensure_exists=True)

‎tests/model_registry/constants.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,6 @@ class ModelRegistryEndpoints:
2828
MR_INSTANCE_BASE_NAME: str = "model-registry"
2929
MR_INSTANCE_NAME: str = f"{MR_INSTANCE_BASE_NAME}0"
3030
SECURE_MR_NAME: str = "secure-db-mr"
31-
ISTIO_CONFIG_DICT: dict[str, Any] = {
32-
"gateway": {"grpc": {"tls": {}}, "rest": {"tls": {}}},
33-
}
3431
OAUTH_PROXY_CONFIG_DICT: dict[str, Any] = {
3532
"port": 8443,
3633
"routePort": 443,

‎tests/model_registry/image_validation/test_verify_rhoai_images.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313

1414
@pytest.mark.usefixtures(
1515
"updated_dsc_component_state_scope_class",
16-
"is_model_registry_oauth",
1716
"mysql_metadata_resources",
1817
"model_registry_instance_mysql",
1918
)

‎tests/model_registry/python_client/test_model_registry_creation.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
)
3232
@pytest.mark.usefixtures(
3333
"updated_dsc_component_state_scope_class",
34-
"is_model_registry_oauth",
3534
"model_registry_namespace",
3635
"model_registry_db_secret",
3736
"model_registry_db_pvc",

‎tests/model_registry/rbac/conftest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,7 @@ def updated_dsc_component_state_parametrized(
451451
dsc_resource.wait_for_condition(
452452
condition=DscComponents.COMPONENT_MAPPING[DscComponents.MODELREGISTRY], status="True"
453453
)
454-
namespace = Namespace(name=namespace_name, ensure_exists=True)
454+
namespace = Namespace(name=namespace_name, wait_for_resource=True)
455455
namespace.wait_for_status(status=Namespace.Status.ACTIVE)
456456
wait_for_pods_running(
457457
admin_client=admin_client,

‎tests/model_registry/rbac/test_mr_rbac.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040

4141
@pytest.mark.usefixtures(
4242
"updated_dsc_component_state_scope_class",
43-
"is_model_registry_oauth",
4443
"mysql_metadata_resources",
4544
"model_registry_instance_mysql",
4645
)

0 commit comments

Comments
 (0)