feat(backend): Add the ability to set a proxy for accessing external resources

.github/actions/kfp-cluster/action.yml

@@ -5,6 +5,10 @@ inputs:
   k8s_version:
     description: "The Kubernetes version to use for the Kind cluster"
     required: true
+  proxy:
+    description: "If KFP should be deployed with proxy configuration"
+    required: false
+    default: false
 
 runs:
   using: "composite"
@@ -17,10 +21,26 @@ runs:
         version: v0.25.0
         node_image: kindest/node:${{ inputs.k8s_version }}
 
+    - name: Deploy Squid
+      id: deploy-squid
+      if: ${{ (inputs.proxy == 'true' )}}
+      shell: bash
+      run: ./.github/resources/squid/deploy-squid.sh
+
     - name: Build images
       shell: bash
-      run: ./.github/resources/scripts/build-images.sh
+      run: |
+        if [ "${{ inputs.proxy }}" = "true" ]; then
+          ./.github/resources/scripts/build-images.sh --proxy
+        else
+          ./.github/resources/scripts/build-images.sh
+        fi
 
     - name: Deploy KFP
       shell: bash
-      run: ./.github/resources/scripts/deploy-kfp.sh
+      run: |
+        if [ "${{ inputs.proxy }}" = "true" ]; then
+          ./.github/resources/scripts/deploy-kfp.sh --proxy
+        else
+          ./.github/resources/scripts/deploy-kfp.sh
+        fi

.github/resources/manifests/argo/kustomization.yaml → .github/resources/manifests/argo/overlays/no-proxy/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../../../../manifests/kustomize/env/platform-agnostic
+- ../../../../../../manifests/kustomize/env/platform-agnostic
 
 images:
 - name: ghcr.io/kubeflow/kfp-api-server
@@ -16,4 +16,4 @@ images:
   newTag: latest
 
 patchesStrategicMerge:
-- overlays/apiserver-env.yaml
+- apiserver-env.yaml

.github/resources/manifests/argo/overlays/proxy/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../no-proxy
+
+patches:
+  - path: proxy-env.yaml
+    target:
+      kind: Deployment
+      name: ml-pipeline

.github/resources/manifests/argo/overlays/proxy/proxy-env.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: HTTP_PROXY
+              value: "http://squid.squid.svc.cluster.local:3128"
+            - name: HTTPS_PROXY
+              value: "http://squid.squid.svc.cluster.local:3128"
+            - name: NO_PROXY
+              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,metadata-grpc-service,0,1,2,3,4,5,6,7,8,9"

.github/resources/manifests/kubernetes-native/kustomization.yaml → .github/resources/manifests/kubernetes-native/overlays/no-proxy/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../../../../manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native
+- ../../../../../../manifests/kustomize/env/cert-manager/platform-agnostic-k8s-native
 
 images:
 - name: ghcr.io/kubeflow/kfp-api-server
@@ -16,4 +16,4 @@ images:
   newTag: latest
 
 patchesStrategicMerge:
-- overlays/apiserver-env.yaml
+- apiserver-env.yaml

.github/resources/manifests/kubernetes-native/overlays/proxy/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../no-proxy
+
+patches:
+  - path: proxy-env.yaml
+    target:
+      kind: Deployment
+      name: ml-pipeline

.github/resources/manifests/kubernetes-native/overlays/proxy/proxy-env.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: HTTP_PROXY
+              value: "http://squid.squid.svc.cluster.local:3128"
+            - name: HTTPS_PROXY
+              value: "http://squid.squid.svc.cluster.local:3128"
+            - name: NO_PROXY
+              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,metadata-grpc-service,0,1,2,3,4,5,6,7,8,9"

.github/resources/scripts/deploy-kfp.sh

@@ -24,6 +24,17 @@ C_DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi
 source "${C_DIR}/helper-functions.sh"
 
+USE_PROXY=false
+
+while getopts ":p-:" OPT; do
+    case $OPT in
+        -) [ "$OPTARG" = "proxy" ] && USE_PROXY=true || { echo "Unknown option --$OPTARG"; exit 1; };;
+        \?) echo "Invalid option: -$OPTARG" >&2; exit 1;;
+    esac
+done
+
+shift $((OPTIND-1))
+
 kubectl apply -k "manifests/kustomize/cluster-scoped-resources/"
 kubectl apply -k "manifests/kustomize/base/crds"
 kubectl wait crd/applications.app.k8s.io --for condition=established --timeout=60s || EXIT_CODE=$?
@@ -48,6 +59,12 @@ if [[ "$PIPELINE_STORE" == "kubernetes" ]]; then
   TEST_MANIFESTS=".github/resources/manifests/kubernetes-native"
 fi
 
+if $USE_PROXY; then
+  TEST_MANIFESTS="${TEST_MANIFESTS}/overlays/proxy"
+else
+  TEST_MANIFESTS="${TEST_MANIFESTS}/overlays/no-proxy"
+fi
+
 kubectl apply -k "${TEST_MANIFESTS}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then

.github/resources/squid/Containerfile

@@ -0,0 +1,10 @@
+FROM quay.io/fedora/fedora:41
+
+RUN dnf install -y squid && \
+    dnf clean all
+
+COPY squid.conf /etc/squid/squid.conf
+
+EXPOSE 3128
+
+CMD ["squid", "-N", "-d", "1"]

.github/resources/squid/deploy-squid.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+C_DIR="${BASH_SOURCE%/*}"
+NAMESPACE="squid"
+
+docker build --progress=plain -t "registry.domain.local/squid:test" -f ${C_DIR}/Containerfile ${C_DIR}
+kind --name kfp load docker-image registry.domain.local/squid:test
+
+kubectl apply -k ${C_DIR}/manifests
+
+if ! kubectl -n ${NAMESPACE} wait --for=condition=available deployment/squid --timeout=60s; then
+    echo "Timeout occurred while waiting for the Squid deployment."
+    exit 1
+fi

.github/resources/squid/manifests/deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: squid
+  namespace: squid
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: squid
+  template:
+    metadata:
+      labels:
+        app: squid
+    spec:
+      containers:
+        - name: squid
+          image: registry.domain.local/squid:test
+          ports:
+            - containerPort: 3128
+          volumeMounts:
+            - name: squid-cache
+              mountPath: /var/cache/squid
+            - name: squid-log
+              mountPath: /var/log/squid
+      volumes:
+        - name: squid-cache
+          emptyDir: { }
+        - name: squid-log
+          emptyDir: { }

.github/resources/squid/manifests/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - deployment.yaml
+  - service.yaml
+  - namespace.yaml

.github/resources/squid/manifests/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: squid

.github/resources/squid/manifests/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: squid
+  namespace: squid
+spec:
+  selector:
+    app: squid
+  ports:
+    - protocol: TCP
+      port: 3128
+      targetPort: 3128

.github/resources/squid/squid.conf

@@ -0,0 +1,8 @@
+# Define an access control list (ACL) for all source IP addresses
+acl all src all
+
+# Allow HTTP access from all sources
+http_access allow all
+
+# Define the port Squid will listen on
+http_port 3128

.github/workflows/e2e-test.yml

@@ -188,6 +188,56 @@ jobs:
           name: kfp-api-integration-tests-v2-artifacts-k8s-${{ matrix.k8s_version }}
           path: /tmp/tmp*/*
 
+  api-integration-tests-v2-with-proxy:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k8s_version: [ "v1.31.0" ]
+    name: API integration tests v2 with proxy - K8s ${{ matrix.k8s_version }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Create KFP cluster
+        id: create-kfp-cluster
+        uses: ./.github/actions/kfp-cluster
+        with:
+          k8s_version: ${{ matrix.k8s_version }}
+          proxy: 'true'
+        continue-on-error: true
+
+      - name: Forward API port
+        id: forward-api-port
+        if: ${{ (steps.create-kfp-cluster.outcome == 'success' )}}
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        continue-on-error: true
+
+      - name: API integration tests v2
+        id: tests
+        if: ${{ (steps.forward-api-port.outcome == 'success' )}}
+        working-directory: ./backend/test/v2/integration
+        run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true -useProxy=true
+        continue-on-error: true
+
+      - name: Collect failed logs
+        if: ${{ (steps.create-kfp-cluster.outcome != 'success' ) || ( steps.forward-api-port.outcome != 'success' ) || ( steps.tests.outcome != 'success' )}}
+        run: |
+          ./.github/resources/scripts/collect-logs.sh --ns squid --output /tmp/tmp_squid_pod_log.txt
+          ./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
+          exit 1
+
+      - name: Collect test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: kfp-api-integration-tests-v2-with-proxy-artifacts-k8s-${{ matrix.k8s_version }}
+          path: /tmp/tmp*/*
+
   frontend-integration-test:
     runs-on: ubuntu-latest
     strategy: