upgrade-test.yml

name: KFP upgrade tests

env:
  TESTS_DIR: "./backend/test/v2/api"
  TESTS_LABEL: "ApiServerTests"
  NUMBER_OF_PARALLEL_NODES: 15
  CLUSTER_NAME: "kfp"
  NAMESPACE: "kubeflow"
  PYTHON_VERSION: "3.9"

on:
  push:
    branches: [master]

  pull_request:
    paths:
      - '.github/workflows/upgrade-test.yml'
      - '.github/workflows/image-builds.yml'
      - '.github/actions/create-cluster/**'
      - '.github/actions/deploy/**'
      - '.github/resources/**'
      - 'backend/api/v2beta1/**'
      - 'backend/src/**'
      - 'backend/metadata_writer/**'
      - 'backend/test/v2/api/**'
      - 'manifests/kustomize/**'
      - 'test_data/sdk_compiled_pipelines/**'
      - '!**/*.md'
      - '!**/OWNERS'

jobs:
  build:
    uses: ./.github/workflows/image-builds.yml

  upgrade-test:
    needs: build
    runs-on: ubuntu-latest
    strategy:
      matrix:
        k8s_version: [ "v1.34.0" ]
    name: KFP upgrade tests - K8s ${{ matrix.k8s_version }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version-file: go.mod

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: 3.9

      - name: Get last release tag
        shell: bash
        id: get-tag
        run: |
          lastRelease=""
          for attempt in 1 2 3; do
            lastRelease=$(curl -sSL --fail-with-body --connect-timeout 10 --max-time 60 \
              -H "Accept: application/vnd.github+json" \
              -H "Authorization: Bearer ${{ github.token }}" \
              "https://api.github.com/repos/kubeflow/pipelines/releases/latest" | jq -r .tag_name 2>/dev/null || echo "")
            if [[ -n "$lastRelease" && "$lastRelease" != "null" ]]; then
              break
            fi
            if [[ "$attempt" -lt 3 ]]; then
              echo "Attempt $attempt: Failed to fetch release tag. Retrying in 5s..."
              sleep 5
            fi
          done

          if [[ -z "$lastRelease" || "$lastRelease" == "null" ]]; then
            echo "::error::Could not fetch latest release tag from GitHub API after 3 attempts."
            exit 1
          fi

          echo "Fetched last release tag: $lastRelease"
          echo "lastRelease=$lastRelease" >> "$GITHUB_OUTPUT"

      - name: Create cluster
        uses: ./.github/actions/create-cluster
        id: create-cluster
        with:
          k8s_version: ${{ matrix.k8s_version }}

      - name: Deploy Last Release
        shell: bash
        id: deploy-release
        run: |
          kubectl apply -k https://github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=${{ steps.get-tag.outputs.lastRelease }}
          kubectl apply -k https://github.com/kubeflow/pipelines/manifests/kustomize/env/platform-agnostic?ref=${{ steps.get-tag.outputs.lastRelease }}
          source "./.github/resources/scripts/helper-functions.sh"
          wait_for_pods || EXIT_CODE=$?
          if [[ $EXIT_CODE -ne 0 ]]; then
            echo "Deploy unsuccessful. Not all pods running."
            exit 1
          fi

      - name: Forward API port
        id: forward-api-port
        if: ${{ steps.deploy-release.outcome == 'success' }}
        shell: bash
        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
        continue-on-error: true

      - name: Prepare for Upgrade
        id: prepare-upgrade
        if: ${{ steps.forward-api-port.outcome == 'success' }}
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          go run github.com/onsi/ginkgo/v2/ginkgo -r -v --cover -p --keep-going --github-output=true --nodes=${{ env.NUMBER_OF_PARALLEL_NODES }} -v --label-filter="UpgradePreparation"
        continue-on-error: true

      - name: Stop port forwarding
        run: |
          pkill -f "port-forward"
        continue-on-error: true

      - name: Deploy from Branch
        uses: ./.github/actions/deploy
        if: ${{ steps.create-cluster.outcome == 'success' }}
        id: deploy
        with:
          image_path: ${{ needs.build.outputs.IMAGE_PATH }}
          image_tag: ${{ needs.build.outputs.IMAGE_TAG }}
          image_registry: ${{ needs.build.outputs.IMAGE_REGISTRY }}
          forward_port: 'true'

      - name: Verify Upgrade
        uses: ./.github/actions/test-and-report
        if: ${{ steps.deploy.outcome == 'success' }}
        with:
          test_directory: ${{ env.TESTS_DIR }}
          test_label: "UpgradeVerification"
          num_parallel_nodes: ${{ env.NUMBER_OF_PARALLEL_NODES }}
          default_namespace: ${{ env.NAMESPACE }}
          python_version: ${{ env.PYTHON_VERSION }}
          report_name: "Upgrade Verification"