Add api tests stateless

.github/workflows/python-tests.yml

@@ -13,7 +13,7 @@ on:
       - ".github/ISSUE_TEMPLATE/**"
       - ".github/dependabot.yml"
       - "docs/**"
-
+    types: [opened, synchronize, reopened, labeled]
 jobs:
   lint:
     name: ${{ matrix.session }}
@@ -214,13 +214,23 @@ jobs:
       - name: Deploy OCI Registry using manifests
         run: ./scripts/deploy_local_kind_registry.sh
       - name: Nox test end-to-end
+        if: github.event.label.name != 'test-fuzz'
         working-directory: clients/python
         run: |
           kubectl port-forward -n kubeflow service/model-registry-service 8080:8080 &
           kubectl port-forward -n minio svc/minio 9000:9000 &
           kubectl port-forward service/distribution-registry-test-service 5001:5001 &
           sleep 2
           nox --python=${{ matrix.python }} --session=e2e -- --cov-report=xml
+      - name: Nox test fuzz (main only or PR with /test-fuzz comment)
+        if: github.ref == 'refs/heads/main' || github.event.label.name == 'test-fuzz'
+        working-directory: clients/python
+        run: |
+          kubectl port-forward -n kubeflow service/model-registry-service 8080:8080 &
+          kubectl port-forward -n minio svc/minio 9000:9000 &
+          kubectl port-forward service/distribution-registry-test-service 5001:5001 &
+          sleep 2
+          nox --python=${{ matrix.python }} --session=fuzz
 
   docs-build:
     name: ${{ matrix.session }}

clients/python/.gitignore

@@ -7,3 +7,4 @@
 __pycache__/
 venv/
 .port-forwards.pid
+.hypothesis/

clients/python/Makefile

@@ -36,6 +36,16 @@ test-e2e: deploy-latest-mr deploy-local-registry deploy-test-minio
 	$(MAKE) test-e2e-cleanup
 	@exit $$STATUS
 
+.PHONY: test-fuzz
+test-fuzz: deploy-latest-mr deploy-local-registry deploy-test-minio
+	@echo "Starting test-fuzz"
+	poetry install --all-extras
+	@set -a; . ../../scripts/manifests/minio/.env; set +a; \
+	poetry run pytest --fuzz -v -s --hypothesis-show-statistics
+	@rm -f ../../scripts/manifests/minio/.env
+	$(MAKE) test-e2e-cleanup
+	@exit $$STATUS
+
 .PHONY: test-e2e-run
 test-e2e-run:
 	@echo "Ensuring all extras are installed..."

clients/python/README.md

@@ -345,6 +345,12 @@ Then you can run tests:
 make test test-e2e
 ```
 
+Then you can run fuzz tests:
+
+```bash
+make test-fuzz
+```
+
 ### Using Nox
 
 Common tasks, such as building documentation and running tests, can be executed using [`nox`](https://github.com/wntrblm/nox) sessions.

clients/python/noxfile.py

@@ -60,6 +60,7 @@ def tests(session: Session) -> None:
         "pytest-asyncio",
         "uvloop",
         "olot",
+        "schemathesis",
     )
     session.run(
         "pytest",
@@ -83,6 +84,7 @@ def e2e_tests(session: Session) -> None:
         "boto3",
         "olot",
         "uvloop",
+        "schemathesis",
     )
     try:
         session.run(
@@ -99,6 +101,22 @@ def e2e_tests(session: Session) -> None:
             session.notify("coverage", posargs=[])
 
 
+@session(name="fuzz", python=python_versions)
+def fuzz_tests(session: Session) -> None:
+    """Run the fuzzing tests."""
+    session.install(
+        ".",
+        "requests",
+        "pytest",
+        "uvloop",
+        "olot",
+        "schemathesis",
+    )
+    session.run(
+        "pytest",
+        "--fuzz",
+        "-rA",
+    )
 @session(python=python_versions[0])
 def coverage(session: Session) -> None:
     """Produce the coverage report."""

clients/python/pyproject.toml

@@ -29,6 +29,7 @@ eval-type-backport = "^0.2.0"
 huggingface-hub = { version = ">=0.20.1,<0.34.0", optional = true }
 olot = { version = "^0.1.6", optional = true }
 boto3 = { version = "^1.37.34", optional = true }
+urllib3 = ">=1.25.4,<1.27"
 
 [tool.poetry.extras]
 hf = ["huggingface-hub"]
@@ -60,6 +61,7 @@ requests = "^2.32.2"
 black = ">=24.4.2,<26.0.0"
 types-python-dateutil = "^2.9.0.20240906"
 pytest-html = "^4.1.1"
+schemathesis = ">=4.0.3"
 
 [tool.coverage.run]
 branch = true
@@ -81,7 +83,10 @@ line-length = 119
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
-markers = ["e2e: end-to-end testing"]
+markers = [
+    "e2e: end-to-end testing",
+    "fuzz: mark a test as a fuzzing (property-based or randomized) test"
+]
 
 [tool.ruff]
 target-version = "py39"

clients/python/schemathesis.toml

@@ -0,0 +1,5 @@
+base-url = "${API_HOST}"
+
+[generation]
+# Don't shrink failing examples to save time
+no-shrink = true

clients/python/tests/conftest.py

@@ -1,5 +1,6 @@
 import asyncio
 import base64
+import contextlib
 import inspect
 import json
 import os
@@ -8,32 +9,50 @@
 import subprocess
 import tempfile
 import time
+from collections.abc import Generator
 from contextlib import asynccontextmanager
 from pathlib import Path
 from unittest.mock import Mock, patch
 from urllib.parse import urlparse
 
 import pytest
 import requests
+import schemathesis
 import uvloop
+from schemathesis.specs.openapi.schemas import BaseOpenAPISchema
 
 from model_registry import ModelRegistry
 from model_registry.utils import BackendDefinition, _get_skopeo_backend
 
+from .constants import DEFAULT_API_TIMEOUT
+
 
 def pytest_addoption(parser):
     parser.addoption("--e2e", action="store_true", help="run end-to-end tests")
+    parser.addoption("--fuzz", action="store_true", help="run fuzzing tests")
 
 
 def pytest_collection_modifyitems(config, items):
+    skip_reasons = {
+        "e2e": pytest.mark.skip(reason="this is an end-to-end test, requires explicit opt-in --e2e option to run."),
+        "fuzz": pytest.mark.skip(reason="this is a fuzzing test, requires explicit opt-in --fuzz option to run."),
+        "skip": pytest.mark.skip(reason="skipping non-e2e and non-fuzz tests"),
+    }
+    e2e = config.getoption("--e2e")
+    fuzz = config.getoption("--fuzz")
+
     for item in items:
-        if "e2e" in item.keywords:
-            skip_e2e = pytest.mark.skip(
-                reason="this is an end-to-end test, requires explicit opt-in --e2e option to run."
-            )
-            if not config.getoption("--e2e"):
-                item.add_marker(skip_e2e)
-            continue
+        if e2e:
+            if "e2e" not in item.keywords:
+                item.add_marker(skip_reasons["skip"])
+        elif fuzz:
+            if "fuzz" not in item.keywords:
+                item.add_marker(skip_reasons["skip"])
+        else:
+            if "e2e" in item.keywords:
+                item.add_marker(skip_reasons["e2e"])
+            if "fuzz" in item.keywords:
+                item.add_marker(skip_reasons["fuzz"])
 
 
 def pytest_report_teststatus(report, config):
@@ -317,3 +336,52 @@ def mock_override(base_image, dest_dir, params):
         skopeo_pull_mock.side_effect = mock_override
         skopeo_push_mock.side_effect = mock_override
         yield backend, skopeo_pull_mock, skopeo_push_mock, generic_auth_vars
+
+@pytest.fixture(scope="session")
+def generated_schema(pytestconfig: pytest.Config ) -> BaseOpenAPISchema:
+    """Generate the schema for the API"""
+
+    os.environ["API_HOST"] = REGISTRY_URL
+    config = schemathesis.config.SchemathesisConfig.from_path(f"{pytestconfig.rootpath}/schemathesis.toml")
+    local_schema_path = f"{pytestconfig.rootpath}/../../api/openapi/model-registry.yaml"
+    schema = schemathesis.openapi.from_path(
+        path=local_schema_path,
+        config=config,
+    )
+    schema.config.output.sanitization.update(enabled=False)
+    return schema
+
+@pytest.fixture
+def cleanup_artifacts(request: pytest.FixtureRequest, auth_headers: dict):
+    """Cleanup artifacts created during the test."""
+    created_ids = []
+    def register(artifact_id):
+        created_ids.append(artifact_id)
+
+    yield register
+
+    for artifact_id in created_ids:
+        del_url = f"{REGISTRY_URL}/api/model_registry/v1alpha3/artifacts/{artifact_id}"
+        try:
+            requests.delete(del_url, headers=auth_headers, timeout=DEFAULT_API_TIMEOUT)
+        except Exception as e:
+            print(f"Failed to delete artifact {artifact_id}: {e}")
+
+@pytest.fixture
+def artifact_resource():
+    """Create an artifact resource for the test."""
+    @contextlib.contextmanager
+    def _artifact_resource(auth_headers: dict, payload: dict) -> Generator[str, None, None]:
+        create_endpoint = f"{REGISTRY_URL}/api/model_registry/v1alpha3/artifacts"
+        resp = requests.post(create_endpoint, headers=auth_headers, json=payload, timeout=DEFAULT_API_TIMEOUT)
+        resp.raise_for_status()
+        artifact_id = resp.json()["id"]
+        try:
+            yield artifact_id
+        finally:
+            del_url = f"{REGISTRY_URL}/api/model_registry/v1alpha3/artifacts/{artifact_id}"
+            try:
+                requests.delete(del_url, headers=auth_headers, timeout=DEFAULT_API_TIMEOUT)
+            except Exception as e:
+                print(f"Failed to delete artifact {artifact_id}: {e}")
+    return _artifact_resource

clients/python/tests/constants.py

@@ -0,0 +1,15 @@
+ARTIFACT_STATES = [
+   "LIVE",
+   "PENDING",
+   "MARKED_FOR_DELETION",
+   "DELETED",
+   "ABANDONED",
+   "REFERENCE",
+   "UNKNOWN",
+]
+ARTIFACT_TYPE_PARAMS = [
+    ("model-artifact", "s3://test-bucket/models/"),
+    ("doc-artifact", "https://docs.example.com/docs/"),
+]
+DEFAULT_API_TIMEOUT = 5.0
+

clients/python/tests/test_api.py

@@ -0,0 +1,117 @@
+import secrets
+from typing import Callable
+
+import pytest
+import requests
+import schemathesis
+from hypothesis import HealthCheck, settings
+
+from .conftest import REGISTRY_URL
+from .constants import ARTIFACT_STATES, ARTIFACT_TYPE_PARAMS, DEFAULT_API_TIMEOUT
+
+schema = schemathesis.pytest.from_fixture("generated_schema")
+
+
+@pytest.fixture
+def auth_headers(setup_env_user_token):
+    """Provides authorization headers for API requests."""
+    return {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {setup_env_user_token}"
+    }
+
+schema = (
+    schema
+    .exclude(
+        path="/api/model_registry/v1alpha3/artifacts/{id}",
+        method="PATCH"
+    )
+    .exclude(
+        path="/api/model_registry/v1alpha3/model_versions/{modelversionId}/artifacts",
+        method="POST"
+    )
+)
+@schema.parametrize()
+@settings(
+    max_examples=100,
+    deadline=None,
+    suppress_health_check=[
+        HealthCheck.filter_too_much,
+        HealthCheck.too_slow,
+        HealthCheck.data_too_large,
+    ],
+)
+@pytest.mark.fuzz
+def test_mr_api_stateless(auth_headers: dict, case: schemathesis.Case):
+    """Test the Model Registry API endpoints.
+
+    This test uses schemathesis to generate and validate API requests
+    """
+
+    case.call_and_validate(headers=auth_headers)
+
+@pytest.mark.fuzz
+@pytest.mark.parametrize(("artifact_type", "uri_prefix"), ARTIFACT_TYPE_PARAMS)
+@pytest.mark.parametrize("state", ARTIFACT_STATES)
+def test_post_model_version_artifacts(auth_headers: dict, artifact_type: str, uri_prefix: str, state: str, cleanup_artifacts: Callable):
+    """
+    Direct test for POST /api/model_registry/v1alpha3/model_versions/{modelversionId}/artifacts.
+    """
+    model_version_id = str(secrets.randbelow(2000000000 - 100000 + 1) + 100000)
+
+    endpoint = f"{REGISTRY_URL}/api/model_registry/v1alpha3/model_versions/{model_version_id}/artifacts"
+
+    payload = {
+        "artifactType": artifact_type,
+        "name": "my-test-model-artifact-post",
+        "uri": f"{uri_prefix}my-test-model.pkl",
+        "state": state,
+        "description": "A test model artifact created via direct POST test.",
+        "externalId": str(secrets.randbelow(2000000000 - 100000 + 1) + 100000)
+    }
+
+    response = requests.post(endpoint, headers=auth_headers, json=payload, timeout=DEFAULT_API_TIMEOUT)
+    artifact_id = response.json()["id"]
+    cleanup_artifacts(artifact_id)
+
+    assert response.status_code in {200, 201}, f"Expected 200 or 201, got {response.status_code}: {response.text}"
+    response_json = response.json()
+    assert response_json.get("id"), "Response body should contain 'id'"
+    assert response_json.get("name") == payload["name"], "Response name should match payload name"
+    assert response_json.get("artifactType") == payload["artifactType"], "Response artifactType should match payload"
+
+
+@pytest.mark.fuzz
+@pytest.mark.parametrize(("artifact_type", "uri_prefix"), ARTIFACT_TYPE_PARAMS)
+def test_patch_artifact(auth_headers: dict, artifact_resource: Callable, artifact_type: str, uri_prefix: str):
+    """
+    Direct test for PATCH /api/model_registry/v1alpha3/artifacts/{id}.
+    """
+    initial_state = "PENDING"
+    target_state = "LIVE"
+
+    create_payload = {
+        "artifactType": artifact_type,
+        "name": "test-create-for-patch",
+        "uri": "s3://my-test-bucket/models/initial-model.pkl",
+        "state": initial_state,
+    }
+    if artifact_type == "model-artifact":
+        create_payload["modelFormatName"] = "tensorflow"
+        create_payload["modelFormatVersion"] = "1.0"
+
+
+    with artifact_resource(auth_headers, create_payload) as artifact_id:
+        patch_endpoint = f"{REGISTRY_URL}/api/model_registry/v1alpha3/artifacts/{artifact_id}"
+        patch_payload = {
+            "artifactType": artifact_type,
+            "description": f"Updated description for {artifact_type} ({target_state})",
+            "state": target_state,
+        }
+        patch_response = requests.patch(patch_endpoint, headers=auth_headers, json=patch_payload, timeout=DEFAULT_API_TIMEOUT)
+        assert patch_response.status_code == 200
+        patch_response_json = patch_response.json()
+        assert patch_response_json.get("id") == artifact_id
+        assert patch_response_json.get("description") == patch_payload["description"]
+        assert patch_response_json.get("state") == patch_payload["state"]
+        assert patch_response_json.get("artifactType") == artifact_type