refactor: move KServe tests from apps to tests/gh-actions

.github/workflows/full_kubeflow_integration_test.yaml

@@ -166,11 +166,7 @@ jobs:
 
     - name: Run KServe Test
       run: |
-        kubectl apply -f tests/gh-actions/kf-objects/kserve_test.yaml
-        sleep 30
-        kubectl get inferenceservice -n $KF_PROFILE
-        kubectl wait --for=condition=Ready inferenceservice.serving.kserve.io/sklearn-iris -n $KF_PROFILE --timeout=300s
-# TODO the individual KServe tests is currently being restructured. Afterwards we can also test inferencing
+        ./tests/gh-actions/test_kserve.sh ${KF_PROFILE}
 
     - name: Run Spark Test
       run: chmod u+x tests/gh-actions/*.sh && ./tests/gh-actions/test_spark.sh "${KF_PROFILE}"

.github/workflows/kserve_test.yaml

@@ -5,6 +5,8 @@
     - tests/gh-actions/install_KinD_create_KinD_cluster_install_kustomize.sh
     - .github/workflows/kserve_m2m_test.yaml
     - apps/kserve/**
+    - tests/gh-actions/kserve/**
+    - tests/gh-actions/test_kserve.sh
     - tests/gh-actions/install_kserve.sh
     - common/istio*/**
     - tests/gh-actions/install_istio*.sh
@@ -55,107 +57,16 @@
     - name: Create KF Profile
       run: ./tests/gh-actions/install_kubeflow_profile.sh
 
-    - name: Diagnose KServe Service Labels
-      run: |
-        echo "=== KServe Predictor Service Labels ==="
-        kubectl get pods -n kubeflow-user-example-com -l serving.knative.dev/service=isvc-sklearn-predictor-default --show-labels
-
-    # TODO for follow up PR
-    #- name: Apply KServe predictor AuthorizationPolicy
-    #  run: |
-    #      cat <<EOF | kubectl apply -f -
-    #      apiVersion: security.istio.io/v1beta1
-    #      kind: AuthorizationPolicy
-    #      metadata:
-    #        name: sklearn-iris-predictor-allow
-    #        namespace: kubeflow-user-example-com
-    #      spec:
-    #        selector:
-    #          matchLabels:
-    #            serving.knative.dev/service: isvc-sklearn-predictor
-    #        action: ALLOW
-    #        rules:
-    #        - from:
-    #            - source:
-    #                namespaces:
-    #                - "istio-system"
-    #                - "knative-serving"
-    #               - "kubeflow"
-    #              - "kubeflow-user-example-com"
-    #            - principals:
-    #              - "cluster.local/ns/kubeflow-user-example-com/sa/default-editor"
-    #              - "cluster.local/ns/kubeflow-user-example-com/sa/default"
-    #              - "cluster.local/ns/kubeflow-user-example-com/sa/default-viewer"
-    #          to:
-    #            - operation:
-    #                paths:
-    #                - "/v1/models/*"
-    #                - "/v2/models/*"
-    #     EOF
-
-    - name: Apply INSECURE KServe AuthorizationPolicy
-      run: |
-          cat <<EOF | kubectl apply -f -
-          apiVersion: security.istio.io/v1beta1
-          kind: AuthorizationPolicy
-          metadata:
-            name: allow-in-cluster-kserve
-            namespace: kubeflow-user-example-com
-          spec:
-            rules:
-              - to:
-                  - operation:
-                      paths:
-                        - /v1/models/*
-                        - /v2/models/*
-          EOF
-
-    - name: Add KServe path-based routing for external access
-      run: |
-        cat <<EOF | kubectl apply -f -
-        apiVersion: networking.istio.io/v1beta1
-        kind: VirtualService
-        metadata:
-          name: isvc-sklearn-external
-          namespace: kubeflow-user-example-com
-        spec:
-          gateways:
-            - kubeflow/kubeflow-gateway
-          hosts:
-            - '*'
-          http:
-            - match:
-                - uri:
-                    prefix: /kserve/kubeflow-user-example-com/isvc-sklearn/
-              rewrite:
-                uri: /
-              route:
-                - destination:
-                    host: knative-local-gateway.istio-system.svc.cluster.local
-                  headers:
-                    request:
-                      set:
-                        Host: isvc-sklearn-predictor-default.kubeflow-user-example-com.svc.cluster.local
-                  weight: 100
-              timeout: 300s
-        EOF
-
     - name: Setup python 3.12
       uses: actions/setup-python@v4
       with:
         python-version: 3.12
 
-    - name: Install test dependencies
-      run: pip install -r ./apps/kserve/tests/requirements.txt
-
     - name: Port forward
       run: ./tests/gh-actions/port_forward_gateway.sh
 
-    - name: Run kserve tests with m2m token from SA kubeflow-user-example-com/default-editor
-      run: |
-        export KSERVE_INGRESS_HOST_PORT=localhost:8080
-        export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)"
-        cd ./apps/kserve/tests && pytest . -vs --log-level info
+    - name: Run KServe tests
+      run: ./tests/gh-actions/test_kserve.sh kubeflow-user-example-com
 
     - name: Detailed KServe Access Diagnostics
       run: |
@@ -174,10 +85,10 @@
           -d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}'
 
     # TODO FOR FOLLOW UP PR
     #- name: Run and fail kserve tests without kserve m2m token
       #run: |
       #  export KSERVE_INGRESS_HOST_PORT=localhost:8080
-      #  cd ./apps/kserve/tests
+      #  cd ./tests/gh-actions/kserve
       #  if pytest . -vs --log-level info; then
       #    echo "This test should fail with an HTTP redirect to oauth2-proxy/dex auth."; exit 1
       #  else
@@ -186,7 +97,7 @@
       #  fi
 
     # TODO FOR FOLLOW UP PR
     #- name: Test that token from attacker namespace is rejected
     #  run: |
     #    export KSERVE_INGRESS_HOST_PORT=localhost:8080
     #    kubectl create ns kubeflow-user-example-com-attacker
@@ -204,18 +115,6 @@
     #      exit 1
     #    fi
 
-    - name: Test path-based external access
-      run: |
-        export KSERVE_INGRESS_HOST_PORT=localhost:8080
-        export KSERVE_M2M_TOKEN="$(kubectl -n kubeflow-user-example-com create token default-editor)"
-
-        # Test external path-based access
-        curl -v -H "Host: isvc-sklearn.kubeflow-user-example-com.example.com" \
-            -H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
-            -H "Content-Type: application/json" \
-            "http://${KSERVE_INGRESS_HOST_PORT}/kserve/kubeflow-user-example-com/isvc-sklearn/v1/models/isvc-sklearn:predict" \
-            -d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}'
-
     - name: Run kserve models webapp test
       run: |
         kubectl wait --for=condition=Available --timeout=300s -n kubeflow deployment/kserve-models-web-app

tests/gh-actions/install_katib.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -euxo pipefail
 
+sudo apt-get update
 sudo apt-get install -y apparmor-profiles
 sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
 

tests/gh-actions/test_kserve.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+set -euxo pipefail
+
+NAMESPACE=${1:-kubeflow-user-example-com}
+SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TEST_DIRECTORY="${SCRIPT_DIRECTORY}/kserve"
+
+echo "=== KServe Predictor Service Labels ==="
+kubectl get pods -n ${NAMESPACE} -l serving.knative.dev/service=isvc-sklearn-predictor-default --show-labels
+
+cat <<EOF | kubectl apply -f -
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: allow-in-cluster-kserve
+  namespace: ${NAMESPACE}
+spec:
+  rules:
+    - to:
+        - operation:
+            paths:
+              - /v1/models/*
+              - /v2/models/*
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: networking.istio.io/v1beta1
+kind: VirtualService
+metadata:
+  name: isvc-sklearn-external
+  namespace: ${NAMESPACE}
+spec:
+  gateways:
+    - kubeflow/kubeflow-gateway
+  hosts:
+    - '*'
+  http:
+    - match:
+        - uri:
+            prefix: /kserve/${NAMESPACE}/isvc-sklearn/
+      rewrite:
+        uri: /
+      route:
+        - destination:
+            host: knative-local-gateway.istio-system.svc.cluster.local
+          headers:
+            request:
+              set:
+                Host: isvc-sklearn-predictor-default.${NAMESPACE}.svc.cluster.local
+          weight: 100
+      timeout: 300s
+EOF
+
+if ! command -v pytest &> /dev/null; then
+  echo "Installing test dependencies..."
+  pip install -r ${TEST_DIRECTORY}/requirements.txt
+fi
+
+export KSERVE_INGRESS_HOST_PORT=${KSERVE_INGRESS_HOST_PORT:-localhost:8080}
+export KSERVE_M2M_TOKEN="$(kubectl -n ${NAMESPACE} create token default-editor)"
+cd ${TEST_DIRECTORY} && pytest . -vs --log-level info
+
+echo "=== Testing path-based external access ==="
+curl -v -H "Host: isvc-sklearn.${NAMESPACE}.example.com" \
+    -H "Authorization: Bearer ${KSERVE_M2M_TOKEN}" \
+    -H "Content-Type: application/json" \
+    "http://${KSERVE_INGRESS_HOST_PORT}/kserve/${NAMESPACE}/isvc-sklearn/v1/models/isvc-sklearn:predict" \
+    -d '{"instances": [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}'
+
+# TODO FOR FOLLOW-UP PR: Implement proper security with AuthorizationPolicy that restricts access