Implement TLS option for pod-to-pod communication.

Signed-off-by: agoins <alyssacgoins@gmail.com>

Author: alyssacgoins

.github/actions/deploy/action.yml

@@ -38,6 +38,10 @@ inputs:
     required: false
     default: 'true'
     description: "If you want to forward API server port to localhost:8888"
+  pod_to_pod_tls_enabled:
+    description: "If KFP should be deployed with TLS pod-to-pod communication."
+    required: false
+    default: 'false'
 
 runs:
   using: "composite"
@@ -99,6 +103,9 @@ runs:
           echo "Deploying with argo version ${{ inputs.argo_version }}"
           ARGS="${ARGS} --argo-version ${{ inputs.argo_version }}"
         fi
+        if [ "${{inputs.pod_to_pod_tls_enabled }}" = "true" ]; then
+          ARGS="${ARGS} --tls-enabled"
+        fi
         echo "ARGS=$ARGS" >> $GITHUB_OUTPUT
 
     - name: Deploy KFP

.github/actions/test-and-report/action.yml

@@ -48,6 +48,14 @@ inputs:
     required: false
     default: ""
     description: "Override it if you want a custom name for your test report file"
+  tls_enabled:
+    description: "If KFP should be deployed with TLS pod-to-pod communication."
+    required: false
+    default: 'false'
+  ca_cert_path:
+    description: "Path to the CA certificate file."
+    required: false
+    default: ""
 
 
 runs:

.github/resources/manifests/standalone/tls-enabled/apiserver-env.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: V2_DRIVER_IMAGE
+              value: kind-registry:5000/driver:latest
+            - name: V2_LAUNCHER_IMAGE
+              value: kind-registry:5000/launcher:latest
+            - name: LOG_LEVEL
+              value: "debug"
+            - name: TLS_ENABLED
+              value: "true"

.github/resources/manifests/standalone/tls-enabled/kustomization.yaml

@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../../../manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls
+
+images:
+  - name: ghcr.io/kubeflow/kfp-api-server
+    newName: kind-registry:5000/apiserver
+    newTag: latest
+  - name: ghcr.io/kubeflow/kfp-persistence-agent
+    newName: kind-registry:5000/persistenceagent
+    newTag: latest
+  - name: ghcr.io/kubeflow/kfp-scheduled-workflow-controller
+    newName: kind-registry:5000/scheduledworkflow
+    newTag: latest
+
+patches:
+  - path: apiserver-env.yaml

.github/resources/scripts/deploy-kfp.sh

@@ -31,6 +31,7 @@ CACHE_DISABLED=false
 MULTI_USER=false
 STORAGE_BACKEND="seaweedfs"
 AWF_VERSION=""
+POD_TO_POD_TLS_ENABLED=false
 
 # Loop over script arguments passed. This uses a single switch-case
 # block with default value in case we want to make alternative deployments
@@ -67,6 +68,10 @@ while [ "$#" -gt 0 ]; do
         exit 1
       fi
       ;;
+    --tls-enabled)
+      POD_TO_POD_TLS_ENABLED=true
+      shift
+      ;;
   esac
 done
 
@@ -100,8 +105,8 @@ if [[ $EXIT_CODE -ne 0 ]]; then
   exit $EXIT_CODE
 fi
 
-# If pipelines store is set to 'kubernetes', cert-manager must be deployed
-if [ "${PIPELINES_STORE}" == "kubernetes" ]; then
+# If pipelines store is set to 'kubernetes' or pod-to-pod TLS is set to 'true', cert-manager must be deployed
+if [ "${PIPELINES_STORE}" == "kubernetes" ] || [ "${POD_TO_POD_TLS_ENABLED}" == "true" ]; then
   #Install cert-manager
   make -C ./backend install-cert-manager || EXIT_CODE=$?
   if [[ $EXIT_CODE -ne 0 ]]
@@ -156,6 +161,8 @@ if [ "${MULTI_USER}" == "false" ] && [ "${PIPELINES_STORE}" != "kubernetes" ]; t
     TEST_MANIFESTS="${TEST_MANIFESTS}/proxy-minio"
   elif $CACHE_DISABLED && $USE_PROXY && [ "${STORAGE_BACKEND}" == "minio" ]; then
     TEST_MANIFESTS="${TEST_MANIFESTS}/cache-disabled-proxy-minio"
+  elif $POD_TO_POD_TLS_ENABLED; then
+    TEST_MANIFESTS="${TEST_MANIFESTS}/tls-enabled"
   fi
 elif [ "${MULTI_USER}" == "false" ] && [ "${PIPELINES_STORE}" == "kubernetes" ]; then
   TEST_MANIFESTS="${TEST_MANIFESTS}/kubernetes-native"

.github/workflows/api-server-tests.yml

@@ -8,6 +8,7 @@ env:
   NAMESPACE: "kubeflow"
   USER_NAMESPACE: "kubeflow-user-example-com"
   PYTHON_VERSION: "3.9"
+  CA_CERT_PATH: ""
 
 on:
   push:
@@ -58,15 +59,19 @@ jobs:
         proxy: [ "true", "false" ]
         argo_version: [ "v3.7.1", "v3.6.10" ]
         pipeline_store: [ "database" ]
+        pod_to_pod_tls_enabled: [ "false" ]
         include:
           - k8s_version: "v1.29.2"
             cache_enabled: "true"
             argo_version: "v3.6.10"
           - k8s_version: "v1.29.2"
             cache_enabled: "true"
             argo_version: "v3.5.15"
+          - k8s_version: "v1.31.0"
+            cache_enabled: "true"
+            pod_to_pod_tls_enabled: "true"
       fail-fast: false # So that failure in 1 type of parameterized job does not cause other jobs to terminate prematurely
-    name: KFP API Server tests Standalone - K8sVersion=${{ matrix.k8s_version }} argo_version=${{ matrix.argo_version }} cacheEnabled=${{ matrix.cache_enabled }} proxyEnabled=${{ matrix.proxy }}
+    name: KFP API Server tests Standalone - K8sVersion=${{ matrix.k8s_version }} argo_version=${{ matrix.argo_version }} cacheEnabled=${{ matrix.cache_enabled }} proxyEnabled=${{ matrix.proxy }} pod_to_pod_tls_enabled=${{ matrix.pod_to_pod_tls_enabled }}
 
     steps:
       - name: Checkout code
@@ -88,10 +93,18 @@ jobs:
           cache_enabled: ${{ matrix.cache_enabled }}
           proxy: ${{ matrix.proxy }}
           argo_version: ${{ matrix.argo_version }}
+          pod_to_pod_tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }}
           image_path: ${{ needs.build.outputs.IMAGE_PATH }}
           image_tag: ${{ needs.build.outputs.IMAGE_TAG }}
           image_registry: ${{ needs.build.outputs.IMAGE_REGISTRY }}
 
+      - name: Configure Cluster CA Cert for TLS-Enabled Tests
+        shell: bash
+        id: configure-cluster-ca-cert
+        if: ${{ matrix.pod_to_pod_tls_enabled == "true"}}
+        run: |
+          kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > ca.crt
+
       - name: Configure Input Variables
         shell: bash
         id: configure
@@ -110,7 +123,12 @@ jobs:
           if [ -z "${PROXY:-}" ]; then
             PROXY=false
           fi
-
+          
+          if [ "${{ matrix.pod_to_pod_tls_enabled }}" == "true" ]; then
+            CA_CERT_PATH="../../../../ca.crt"
+          fi
+          
+          echo "CA_CERT_PATH=$CA_CERT_PATH" >> $GITHUB_OUTPUT
           echo "NUMBER_OF_NODES=$NUMBER_OF_NODES" >> $GITHUB_OUTPUT
           echo "TEST_LABEL=$TEST_LABEL" >> $GITHUB_OUTPUT
           echo "NAMESPACE=$NAMESPACE" >> $GITHUB_OUTPUT
@@ -129,6 +147,8 @@ jobs:
           default_namespace: ${{ steps.configure.outputs.NAMESPACE }}
           python_version: ${{ env.PYTHON_VERSION }}
           report_name: "Standalone_k8sVersion=${{ matrix.k8s_version }}_argoVersion=${{ matrix.argo_version }}_cacheEnabled=${{ matrix.cache_enabled }}_proxyEnabled=${{ matrix.proxy }}"
+          tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }}
+          ca_cert_path: ${{ steps.configure-cluster-ca-cert.outputs.CA_CERT_PATH}}
 
       - name: Notify test reports
         shell: bash
@@ -148,6 +168,13 @@ jobs:
         uploadPipelinesWithKubernetesClient: [ "true", "false" ]
         argo_version: [ "v3.7.1", "v3.6.10" ]
         pipeline_store: [ "kubernetes" ]
+        pod_to_pod_tls_enabled: [ "false" ]
+        include:
+          - k8s_version: "v1.31.0"
+            cache_enabled: "true"
+            pipeline_store: "kubernetes"
+            pod_to_pod_tls_enabled: "true"
+            uploadPipelinesWithKubernetesClient: "true"
       fail-fast: false # So that failure in 1 type of parameterized job does not cause other jobs to terminate prematurely
     name: KFP API Server tests K8s Native API - K8sVersion=${{ matrix.k8s_version }} cacheEnabled=${{ matrix.cache_enabled }} argoVersion=${{ matrix.argo_version }} uploadPipelinesWithKubernetesClient=${{ matrix.uploadPipelinesWithKubernetesClient }}
 

.github/workflows/e2e-test-frontend.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
     paths:
       - '.github/workflows/e2e-test-frontend.yml'
-      - '.github/actions/create-cluster/**'
       - 'frontend/**'
       - 'manifests/kustomize/**'
       - '!**/*.md'
@@ -23,7 +22,8 @@ jobs:
     strategy:
       matrix:
         k8s_version: [ "v1.29.2", "v1.31.0" ]
-    name: Frontend Integration Tests - K8s ${{ matrix.k8s_version }}
+        pod_to_pod_tls_enabled: [ "true", "false" ]
+    name: Frontend Integration Tests - K8s ${{ matrix.k8s_version }} pod_to_pod_tls_enabled=${{ matrix.pod_to_pod_tls_enabled }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
@@ -49,6 +49,7 @@ jobs:
           image_path: ${{ needs.build.outputs.IMAGE_PATH }}
           image_tag: ${{ needs.build.outputs.IMAGE_TAG }}
           image_registry: ${{ needs.build.outputs.IMAGE_REGISTRY }}
+          pod_to_pod_tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }}
 
       - name: Forward Frontend port
         id: forward-frontend-port

.github/workflows/e2e-test.yml

@@ -6,6 +6,7 @@ env:
   NAMESPACE: "kubeflow"
   PYTHON_VERSION: "3.9"
   USER_NAMESPACE: "kubeflow-user-example-com"
+  CA_CERT_PATH: ""
 
 on:
   push:
@@ -59,6 +60,7 @@ jobs:
         storage: [ "seaweedfs", "minio"]
         proxy: [ "false" ]
         test_label: [ "E2ECritical" ]
+        pod_to_pod_tls_enabled: [ "false" ]
         include:
           - k8s_version: "v1.29.2"
             cache_enabled: "false"
@@ -68,8 +70,12 @@ jobs:
             cache_enabled: "false"
             proxy: "true"
             test_label: "E2EProxy"
+          - k8s_version: "v1.31.0"
+            cache_enabled: "true"
+            pod_to_pod_tls_enabled: "true"
+            test_label: "E2ECritical"
       fail-fast: false
-    name: End to End Critical Scenario Tests - K8s ${{ matrix.k8s_version }} cacheEnabled=${{ matrix.cache_enabled }} argoVersion=${{ matrix.argo_version}} proxy=${{ matrix.proxy}} storage=${{ matrix.storage }}
+    name: End to End Critical Scenario Tests - K8s ${{ matrix.k8s_version }} cacheEnabled=${{ matrix.cache_enabled }} argoVersion=${{ matrix.argo_version}} proxy=${{ matrix.proxy}} storage=${{ matrix.storage }} pod_to_pod_tls_enabled=${{ matrix.pod_to_pod_tls_enabled }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
@@ -92,10 +98,20 @@ jobs:
           cache_enabled: ${{ matrix.cache_enabled }}
           argo_version: ${{ matrix.argo_version }}
           storage_backend: ${{ matrix.storage }}
+          pod_to_pod_tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }}
           image_path: ${{ needs.build.outputs.IMAGE_PATH }}
           image_tag: ${{ needs.build.outputs.IMAGE_TAG }}
           image_registry: ${{ needs.build.outputs.IMAGE_REGISTRY }}
 
+      - name: Configure Cluster CA Cert for TLS-Enabled Tests
+        shell: bash
+        id: configure-cluster-ca-cert
+        if: ${{ matrix.pod_to_pod_tls_enabled == "true"}}
+        env:
+          SCHEME: "https"
+        run: |
+          kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ env.E2E_TESTS_DIR }}"/ca.crt
+
       - name: Configure Input Variables
         shell: bash
         id: configure
@@ -109,6 +125,12 @@ jobs:
             TEST_LABEL=${{ inputs.test_label }}
             NAMESPACE=${{ inputs.namespace }}
           fi
+          
+          if [ "${{ matrix.pod_to_pod_tls_enabled }}" == "true" ]; then
+            CA_CERT_PATH="ca.crt"
+          fi
+          
+          echo "CA_CERT_PATH=$CA_CERT_PATH" >> $GITHUB_OUTPUT
           echo "NUMBER_OF_NODES=$NUMBER_OF_NODES" >> $GITHUB_OUTPUT
           echo "TEST_LABEL=$TEST_LABEL" >> $GITHUB_OUTPUT
           echo "NAMESPACE=$NAMESPACE" >> $GITHUB_OUTPUT
@@ -133,6 +155,8 @@ jobs:
           default_namespace: ${{ steps.configure.outputs.NAMESPACE }}
           python_version: ${{ env.PYTHON_VERSION }}
           report_name: "E2ECriticalTests_K8s=${{ matrix.k8s_version }}_cacheEnabled=${{ matrix.cache_enabled }}_argoVersion=${{ matrix.argo_version}}_proxy=${{ matrix.proxy }}_storage=${{ matrix.storage }}"
+          tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }}
+          ca_cert_path: ${{ steps.configure.outputs.CA_CERT_PATH}}
 
       - name: Notify test reports
         shell: bash
@@ -209,6 +233,7 @@ jobs:
         id: test-run
         env:
           LOCAL_API_SERVER: "true"
+          MULTI_USER: ${{ matrix.multi_user }}
         with:
           multi_user: ${{ matrix.multi_user }}
           cache_enabled: ${{ matrix.cache_enabled }}

.github/workflows/legacy-v2-api-integration-tests.yml

@@ -28,8 +28,10 @@ jobs:
     strategy:
       matrix:
         pipeline_store: [ "database", "kubernetes" ]
-        k8s_version: [ "v1.29.2", "v1.31.0" ]
-    name: API integration tests v2 - K8s with ${{ matrix.pipeline_store }} ${{ matrix.k8s_version }}
+        k8s_version: [ "v1.31.0" ]
+        pod_to_pod_tls_enabled: [ true, false ]
+
+    name: API integration tests v2 - K8s with ${{ matrix.pipeline_store }} ${{ matrix.k8s_version }} pod_to_pod_tls_enabled=${{ matrix.pod_to_pod_tls_enabled }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
@@ -58,6 +60,26 @@ jobs:
           image_tag: ${{ needs.build.outputs.IMAGE_TAG }}
           image_registry: ${{ needs.build.outputs.IMAGE_REGISTRY }}
           pipeline_store: ${{ matrix.pipeline_store }}
+          pod_to_pod_tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }}
+
+      - name: Configure Cluster CA Cert for TLS-Enabled Tests
+        shell: bash
+        id: configure-cluster-ca-cert
+        if: ${{ matrix.pod_to_pod_tls_enabled }}
+        run: |
+          kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > ca.crt
+          CA_CERT_PATH="ca.crt"
+          echo "CA_CERT_PATH=CA_CERT_PATH" >> $GITHUB_OUTPUT
+
+      - name: Configure Input Variables
+        shell: bash
+        id: configure
+        if: ${{ steps.deploy.outcome == 'success' &&  matrix.pod_to_pod_tls_enabled == 'true'}}
+        run: |
+          if [ "${{ matrix.pod_to_pod_tls_enabled }}" == "true" ]; then
+            CA_CERT_PATH="/ca.crt"
+          fi
+          echo "CA_CERT_PATH=$CA_CERT_PATH" >> $GITHUB_OUTPUT
 
       - name: Forward MLMD port
         id: forward-mlmd-port
@@ -70,6 +92,9 @@ jobs:
         if: ${{ steps.forward-mlmd-port.outcome == 'success' }}
         working-directory: ./backend/test/v2/integration
         run: go test -v ./... -args -runIntegrationTests=true -namespace=kubeflow
+        with:
+          pod_to_pod_tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }}
+          ca_cert_path: ${{ steps.configure.outputs.CA_CERT_PATH}}
         env:
           PULL_NUMBER: ${{ github.event.pull_request.number }}
           PIPELINE_STORE: ${{ matrix.pipeline_store }}

backend/Makefile

@@ -1,6 +1,7 @@
 BUILD=build
 MOD_ROOT=..
 KIND_NAME ?= dev-pipelines-api
+TLS_ENABLED ?= "false"
 
 CERT_MANAGER_VERSION ?= v1.16.2
 
@@ -86,17 +87,29 @@ kind-cluster-agnostic:
 	kind create cluster --name $(KIND_NAME)
 	kubectl config use-context kind-$(KIND_NAME)
 	kind get kubeconfig --name $(KIND_NAME) > $(CURDIR)/../kubeconfig_$(KIND_NAME)
+	@if [ "${TLS_ENABLED}" = "true" ]; then \
+    		$(MAKE) install-cert-manager; \
+    fi
 	# Deploy cluster resources required by KFP
 	kubectl apply -k $(CURDIR)/../manifests/kustomize/cluster-scoped-resources
 	kubectl wait --for condition=established --timeout=1m crd/applications.app.k8s.io
 	# Deploy KFP
-	kubectl apply -k $(CURDIR)/../manifests/kustomize/env/platform-agnostic
+	@if [ "${TLS_ENABLED}" = "true" ]; then \
+    		kubectl apply -k $(CURDIR)/../manifests/kustomize/env/cert-manager/platform-agnostic-standalone-tls; \
+    	else \
+    		kubectl apply -k $(CURDIR)/../manifests/kustomize/env/platform-agnostic; \
+     fi
 	kubectl -n kubeflow wait --for condition=Available --timeout=10m deployment/mysql
 	kubectl -n kubeflow wait --for condition=Available --timeout=3m deployment/metadata-grpc-deployment
 	kubectl -n kubeflow wait --for condition=Available --timeout=3m deployment/ml-pipeline
 	# Switch to Kubeflow namespace context
 	kubectl config set-context --current --namespace=kubeflow
 
+# Creates a Kind cluster and deploys a standalone KFP instance in the Kubeflow namespace with pod-to-pod TLS enabled.
+.PHONY: kind-cluster-agnostic-tls
+kind-cluster-agnostic-tls:
+	$(MAKE) TLS_ENABLED="true" kind-cluster-agnostic
+
 .PHONY: dev-kind-cluster
 dev-kind-cluster:
 	${CONTAINER_ENGINE} build -t ${IMG_TAG_WEBHOOK_PROXY} -f $(CURDIR)/../tools/kind/Dockerfile.webhook-proxy $(CURDIR)/../tools/kind