build-x86-image.yaml

name: Build x86 Image

on:
  pull_request:
    branches:
    - master
    - release-*
    paths-ignore:
    - 'docs/**'
    - '**.md'
  push:
    branches:
    - master
    - release-*
    paths-ignore:
    - 'docs/**'
    - '**.md'

concurrency:
  group: "${{ github.workflow }}-${{ github.ref }}"
  cancel-in-progress: true

env:
  # renovate: datasource=github-releases depName=kind packageName=kubernetes-sigs/kind
  KIND_VERSION: v0.31.0
  # renovate: datasource=github-releases depName=golangci-lint packageName=golangci/golangci-lint
  GOLANGCI_LINT_VERSION: v2.8.0
  # renovate: datasource=github-releases depName=helm packageName=helm/helm
  HELM_VERSION: v4.0.5
  # renovate: datasource=github-releases depName=submariner packageName=submariner-io/submariner
  SUBMARINER_VERSION: 0.22.0
  # renovate: datasource=github-releases depName=talos packageName=siderolabs/talos
  TALOS_VERSION: v1.12.1

jobs:
  build-kube-ovn-base:
    name: Build kube-ovn-base
    runs-on: ubuntu-24.04
    outputs:
      build-base: ${{ steps.check.outputs.build-base }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 2

      - id: check
        run: |
          if [ ${{ github.event_name }} != 'pull_request' ]; then
            exit
          fi
          tmp_dir=`mktemp -d`
          cat > "$tmp_dir/on_changes.txt" <<EOF
          dist/images/Dockerfile.base
          dist/images/OpenBFDD-compile.patch
          dist/images/go-deps/download-go-deps.sh
          dist/images/go-deps/rebuild-go-deps.sh
          EOF
          if git diff --name-only HEAD^ HEAD | grep -Ff "$tmp_dir/on_changes.txt"; then
            echo build-base=1 >> "$GITHUB_OUTPUT"
          fi
          rm -frv "$tmp_dir"

      - uses: jlumbroso/free-disk-space@v1.3.1
        if: steps.check.outputs.build-base == 1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: docker/setup-buildx-action@v3
        if: steps.check.outputs.build-base == 1
        with:
          driver-opts: env.BUILDKIT_STEP_LOG_MAX_SIZE=50000000

      - uses: actions/setup-go@v6
        if: steps.check.outputs.build-base == 1
        id: setup-go
        with:
          go-version-file: go.mod
          check-latest: true
          cache: false

      - name: Build kube-ovn-base image
        id: build
        if: steps.check.outputs.build-base == 1
        env:
          GO_VERSION: ${{ steps.setup-go.outputs.go-version }}
        run: |
            make base-amd64
            make base-tar-amd64

      - name: Upload base images to artifact
        if: steps.check.outputs.build-base == 1
        uses: actions/upload-artifact@v6
        with:
          name: kube-ovn-base
          path: image-amd64.tar

  build-kube-ovn-dpdk-base:
    name: Build kube-ovn-dpdk-base
    runs-on: ubuntu-24.04
    outputs:
      build-dpdk-base: ${{ steps.check.outputs.build-dpdk-base }}
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 2

      - id: check
        run: |
          if [ ${{ github.event_name }} != 'pull_request' ]; then
            exit
          fi
          if ! git diff --exit-code HEAD^...HEAD -- dist/images/Dockerfile.base-dpdk; then
            echo build-dpdk-base=1 >> "$GITHUB_OUTPUT"
          fi

      - uses: jlumbroso/free-disk-space@v1.3.1
        if: steps.check.outputs.build-dpdk-base == 1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: docker/setup-buildx-action@v3
        if: steps.check.outputs.build-dpdk-base == 1

      - name: Build kube-ovn-dpdk-base image
        id: build
        if: steps.check.outputs.build-dpdk-base == 1
        run: |
          make base-amd64-dpdk
          make base-tar-amd64-dpdk

      - name: Upload dpdk base images to artifact
        if: steps.check.outputs.build-dpdk-base == 1
        uses: actions/upload-artifact@v6
        with:
          name: kube-ovn-dpdk-base
          path: image-amd64-dpdk.tar

  build-kube-ovn:
    name: Build kube-ovn
    runs-on: ubuntu-24.04
    needs:
      - build-kube-ovn-base
      - build-kube-ovn-dpdk-base
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6
      - uses: docker/setup-buildx-action@v3
      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: go.mod
          check-latest: true
          cache: false

      - name: Setup environment variables
        run: |
          echo "TAG=$(cat VERSION)" >> "$GITHUB_ENV"
          echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Unit test
        run: make ut

      - uses: shogo82148/actions-goveralls@v1
        continue-on-error: true
        with:
          path-to-profile: profile.cov

      - name: Install golangci-lint
        run: |
          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin $GOLANGCI_LINT_VERSION

      - name: Download base images
        if: needs.build-kube-ovn-base.outputs.build-base == 1
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn-base

      - name: Load base images
        if: needs.build-kube-ovn-base.outputs.build-base == 1
        run: |
          docker load --input image-amd64.tar
          docker tag kubeovn/kube-ovn-base:$TAG-amd64 kubeovn/kube-ovn-base:$TAG
          docker tag kubeovn/kube-ovn-base:$TAG-debug-amd64 kubeovn/kube-ovn-base:$TAG-debug

      - name: Download dpdk base images
        if: needs.build-kube-ovn-dpdk-base.outputs.build-dpdk-base == 1
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn-dpdk-base

      - name: Load dpdk base images
        if: needs.build-kube-ovn-dpdk-base.outputs.build-dpdk-base == 1
        run: |
          docker load --input image-amd64-dpdk.tar
          docker tag kubeovn/kube-ovn-base:$TAG-amd64-dpdk kubeovn/kube-ovn-base:$TAG-dpdk

      - name: Scan base image
        uses: aquasecurity/trivy-action@0.33.1
        env:
          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2
        with:
          scan-type: image
          scanners: vuln
          image-ref: docker.io/kubeovn/kube-ovn-base:${{ env.TAG }}
          format: json
          output: trivy-result.json
          ignore-unfixed: true
          trivyignores: .trivyignore
          vuln-type: library

      - name: Build kubectl and CNI plugins from source
        env:
          CGO_ENABLED: "0"
          GO_INSTALL: "go install -v -mod=mod -trimpath"
        run: |
          go mod tidy
          cat trivy-result.json
          dockerfile=${{ github.workspace }}/dist/images/Dockerfile
          export GOBIN=`dirname "$dockerfile"`
          cni_plugins_version=`go list -m -f '{{.Version}}' github.com/containernetworking/plugins`
          cni_plugins_build_flags="-ldflags '-extldflags -static -X github.com/containernetworking/plugins/pkg/utils/buildversion.BuildVersion=$cni_plugins_version'"
          jq -r '.Results[] | select((.Type=="gobinary") and (.Vulnerabilities!=null)) | .Target' trivy-result.json | while read f; do
            bin=`basename $f`
            case $bin in
              loopback|macvlan)
                echo "Building $bin@$cni_plugins_version from source..."
                sh -c "cd /tmp && $GO_INSTALL $cni_plugins_build_flags github.com/containernetworking/plugins/plugins/main/$bin@$cni_plugins_version"
                echo "COPY $bin /$f" >> "$dockerfile"
                ;;
              portmap)
                echo "Building $bin@$cni_plugins_version from source..."
                sh -c "cd /tmp && $GO_INSTALL $cni_plugins_build_flags github.com/containernetworking/plugins/plugins/meta/$bin@$cni_plugins_version"
                echo "COPY $bin /$f" >> "$dockerfile"
                ;;
              kubectl)
                version=`go list -m -f '{{.Version}}' k8s.io/kubernetes`
                mod_dir=`go list -m -f '{{.Dir}}' k8s.io/kubernetes`
                source "$mod_dir/hack/lib/util.sh"
                source "$mod_dir/hack/lib/logging.sh"
                source "$mod_dir/hack/lib/version.sh"
                repo=kubernetes/kubernetes
                commit=unknown
                read type tag_sha < <(echo $(curl -s "https://api.github.com/repos/$repo/git/ref/tags/$version" |
                  jq -r '.object.type,.object.sha'))
                if [ $type = "commit" ]; then
                  commit=$tag_sha
                else
                  commit=$(curl -s "https://api.github.com/repos/$repo/git/tags/$tag_sha" | jq -r '.object.sha')
                fi
                export KUBE_GIT_COMMIT="${commit}"
                export KUBE_GIT_TREE_STATE='clean'
                export KUBE_GIT_VERSION="${version}"
                export KUBE_GIT_MAJOR=`echo $KUBE_GIT_VERSION | cut -d. -f1 | sed 's/$v//'`
                export KUBE_GIT_MINOR=`echo $KUBE_GIT_VERSION | cut -d. -f2`
                goldflags="all=$(kube::version::ldflags) -s -w"
                echo "Building $bin@$version from source..."
                $GO_INSTALL -ldflags="${goldflags}" k8s.io/kubernetes/cmd/kubectl
                echo "COPY $bin /$f" >> "$dockerfile"
                ;;
              gobgp)
                version=`go list -m -f '{{.Version}}' github.com/osrg/gobgp/v4`
                echo "Building $bin@$version from source..."
                $GO_INSTALL github.com/osrg/gobgp/v4/cmd/$bin
                echo "COPY $bin /$f" >> "$dockerfile"
                ;;
              *)
                ;;
            esac
          done

      - name: Build
        run: |
          go mod tidy
          git diff --exit-code -- go.mod go.sum
          make lint
          if [ ${{ needs.build-kube-ovn-base.outputs.build-base || 0 }} = 1 ]; then
            make build-kube-ovn
          else
            make image-kube-ovn
          fi
          make tar-kube-ovn

      - name: Build dpdk
        run: |
          if [ ${{ needs.build-kube-ovn-dpdk-base.outputs.build-dpdk-base || 0 }} = 1 ]; then
            make build-kube-ovn-dpdk
          else
            make image-kube-ovn-dpdk
          fi
          make tar-kube-ovn-dpdk

      - name: Upload images to artifact
        uses: actions/upload-artifact@v6
        with:
          name: kube-ovn
          path: kube-ovn.tar

      - name: Upload dpdk images to artifact
        uses: actions/upload-artifact@v6
        if: github.event_name != 'pull_request'
        with:
          name: kube-ovn-dpdk
          path: kube-ovn-dpdk.tar

  build-vpc-nat-gateway:
    name: Build vpc-nat-gateway
    runs-on: ubuntu-24.04
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Build
        run: |
          make image-vpc-nat-gateway
          make tar-vpc-nat-gateway

      - name: Upload image to artifact
        uses: actions/upload-artifact@v6
        with:
          name: vpc-nat-gateway
          path: vpc-nat-gateway.tar

  build-e2e-binaries:
    name: Build E2E Binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Lookup Go cache
        id: lookup-go-cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-
          lookup-only: true

      - uses: jlumbroso/free-disk-space@v1.3.1
        if: steps.lookup-go-cache.outputs.cache-hit != 'true'
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - name: Go cache
        if: steps.lookup-go-cache.outputs.cache-hit != 'true'
        uses: actions/cache@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - run: make e2e-build
        if: steps.lookup-go-cache.outputs.cache-hit != 'true'
        working-directory: ${{ env.E2E_DIR }}

  netpol-path-filter:
    name: Network Policy Path Filter
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-24.04
    outputs:
      test-netpol: ${{ steps.filter.outputs.kube-ovn-controller }}
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: go.mod
          check-latest: true
          cache: false

      - name: Generate path filter
        run: |
          filter=".github/path-filters.yaml"
          cat > $filter <<EOF
          kube-ovn-controller:
          - go.mod
          - go.sum
          EOF

          sh hack/go-list.sh pkg/controller | while read f; do
            echo "- $f" | tee -a $filter
          done

      - uses: dorny/paths-filter@v3
        id: filter
        with:
          base: ${{ github.base_ref || github.ref_name }}
          filters: .github/path-filters.yaml
          list-files: csv

  k8s-conformance-e2e:
    name: Kubernetes Conformance E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 25
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          - ipv6
          - dual
        mode:
          - overlay
          - underlay
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - name: Remove DNS search domain
        run: |
          sudo sed -i '/^search/d' /etc/resolv.conf
          sudo systemctl restart docker

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Set environment variables
        run: |
          if [ $(($RANDOM%2)) -ne 0 ]; then
            # run as root and use valgrind to debug memory leak
            echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV"
            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
          fi

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        id: install
        run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }}

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
          E2E_NETWORK_MODE: ${{ matrix.mode }}
        run: make k8s-conformance-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-events.yaml
          tar zcf k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-events.tar.gz k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-events
          path: k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-audit-log
          path: k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-ko-log
          path: k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

      - name: Check valgrind result
        run: |
          if [ "x$DEBUG_WRAPPER" != "xvalgrind" ]; then
            exit
          fi

          kubectl -n kube-system rollout restart ds ovs-ovn
          kubectl -n kube-system rollout status ds ovs-ovn
          sleep 10
          kubectl -n kube-system rollout restart deploy ovn-central
          kubectl -n kube-system rollout status deploy ovn-central
          while true; do
            if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then
              break
            fi
            sleep 1
          done
          kubectl ko log ovn
          kubectl ko log ovs

          for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
            echo "Checking if valgrind log file for $daemon exists..."
            find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
          done

          find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
            if grep -qw 'definitely lost' "$f"; then
              echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
              echo $f
              cat "$f"
              exit 1
            fi;
          done

  k8s-netpol-e2e:
    name: Kubernetes Network Policy E2E
    if: |
      always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') &&
      (needs.netpol-path-filter.outputs.test-netpol == 1 || contains(github.event.pull_request.labels.*.name, 'network policy'))
    needs:
      - build-kube-ovn
      - build-e2e-binaries
      - netpol-path-filter
    runs-on: ubuntu-24.04
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          - ipv6
          - dual
        np-enforcement:
          - standard
          - lax
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - name: Remove DNS search domain
        run: |
          sudo sed -i '/^search/d' /etc/resolv.conf
          sudo systemctl restart docker

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        env:
          NP_ENFORCEMENT: "${{ matrix.np-enforcement }}"
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Set environment variables
        run: |
          if [ $(($RANDOM%2)) -ne 0 ]; then
            # run as root and use valgrind to debug memory leak
            echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV"
            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
          fi

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        id: install
        run: make kind-install-${{ matrix.ip-family }}

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        run: make k8s-netpol-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > k8s-netpol-e2e-${{ matrix.ip-family }}-events.yaml
          tar zcf k8s-netpol-e2e-${{ matrix.ip-family }}-events.tar.gz k8s-netpol-e2e-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: k8s-netpol-e2e-${{ matrix.ip-family }}-events
          path: k8s-netpol-e2e-${{ matrix.ip-family }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf k8s-netpol-e2e-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: k8s-netpol-e2e-${{ matrix.ip-family }}-audit-log
          path: k8s-netpol-e2e-${{ matrix.ip-family }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log
          path: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

      - name: Check valgrind result
        run: |
          if [ "x$DEBUG_WRAPPER" != "xvalgrind" ]; then
            exit
          fi

          kubectl -n kube-system rollout restart ds ovs-ovn
          kubectl -n kube-system rollout status ds ovs-ovn
          sleep 10
          kubectl -n kube-system rollout restart deploy ovn-central
          kubectl -n kube-system rollout status deploy ovn-central
          while true; do
            if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then
              break
            fi
            sleep 1
          done
          kubectl ko log ovn
          kubectl ko log ovs

          for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
            echo "Checking if valgrind log file for $daemon exists..."
            find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
          done

          find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
            if grep -qw 'definitely lost' "$f"; then
              echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
              echo $f
              cat "$f"
              exit 1
            fi;
          done

  cyclonus-netpol-e2e:
    name: Cyclonus Network Policy E2E
    if: |
      always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') &&
      (needs.netpol-path-filter.outputs.test-netpol == 1 || contains(github.event.pull_request.labels.*.name, 'network policy'))
    needs:
      - build-kube-ovn
      - netpol-path-filter
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          - ipv6
          - dual
        np-enforcement:
          - standard
          - lax
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Set environment variables
        run: |
          if [ $(($RANDOM%2)) -ne 0 ]; then
            # run as root and use valgrind to debug memory leak
            echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV"
            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
          fi

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        env:
          NP_ENFORCEMENT: "${{ matrix.np-enforcement }}"
        id: install
        run: make kind-install-${{ matrix.ip-family }}

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        run: make cyclonus-netpol-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > cyclonus-netpol-e2e-${{ matrix.ip-family }}-events.yaml
          tar zcf cyclonus-netpol-e2e-${{ matrix.ip-family }}-events.tar.gz cyclonus-netpol-e2e-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: cyclonus-netpol-e2e-${{ matrix.ip-family }}-events
          path: cyclonus-netpol-e2e-${{ matrix.ip-family }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf cyclonus-netpol-e2e-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: cyclonus-netpol-e2e-${{ matrix.ip-family }}-audit-log
          path: cyclonus-netpol-e2e-${{ matrix.ip-family }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log
          path: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

      - name: Check valgrind result
        run: |
          if [ "x$DEBUG_WRAPPER" != "xvalgrind" ]; then
            exit
          fi

          kubectl -n kube-system rollout restart ds ovs-ovn
          kubectl -n kube-system rollout status ds ovs-ovn
          sleep 10
          kubectl -n kube-system rollout restart deploy ovn-central
          kubectl -n kube-system rollout status deploy ovn-central
          while true; do
            if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then
              break
            fi
            sleep 1
          done
          kubectl ko log ovn
          kubectl ko log ovs

          for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
            echo "Checking if valgrind log file for $daemon exists..."
            find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
          done

          find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
            if grep -qw 'definitely lost' "$f"; then
              echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
              echo $f
              cat "$f"
              exit 1
            fi;
          done

  kube-ovn-conformance-e2e:
    name: Kube-OVN Conformance E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          - ipv6
          - dual
        mode:
          - overlay
          - underlay
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Set environment variables
        run: |
          if [ $(($RANDOM%2)) -ne 0 ]; then
            # run as root and use valgrind to debug memory leak
            echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV"
            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
          fi

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        id: install
        run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }}

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
          E2E_NETWORK_MODE: ${{ matrix.mode }}
        run: |
          make kube-ovn-conformance-e2e
          make kind-install-kubevirt
          make kube-ovn-kubevirt-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml
          tar zcf kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events
          path: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-audit-log
          path: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log
          path: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

      - name: Check valgrind result
        run: |
          if [ "x$DEBUG_WRAPPER" != "xvalgrind" ]; then
            exit
          fi

          kubectl -n kube-system rollout restart ds ovs-ovn
          kubectl -n kube-system rollout status ds ovs-ovn
          sleep 10
          kubectl -n kube-system rollout restart deploy ovn-central
          kubectl -n kube-system rollout status deploy ovn-central
          while true; do
            if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then
              break
            fi
            sleep 1
          done
          kubectl ko log ovn
          kubectl ko log ovs

          for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
            echo "Checking if valgrind log file for $daemon exists..."
            find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
          done

          find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
            if grep -qw 'definitely lost' "$f"; then
              echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
              echo $f
              cat "$f"
              exit 1
            fi;
          done

      - name: Cleanup
        run: timeout -k 10 180 sh -x dist/images/cleanup.sh

  kube-ovn-ic-conformance-e2e:
    name: Kube-OVN IC Conformance E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          - ipv6
          - dual
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind clusters
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-init-ovn-ic-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        id: install
        run: make kind-install-ovn-ic-${{ matrix.ip-family }}

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
        run: make kube-ovn-ic-conformance-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          for cluster in `kind get clusters`; do
            kubectl config use-context kind-$cluster
            kubectl get events -A -o yaml > kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-$cluster-events.yaml
          done
          tar zcf kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz \
            `kind get clusters | xargs -I {} echo kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-{}-events.yaml`

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-events
          path: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          for cluster in `kind get clusters`; do
            docker cp $cluster-control-plane:/var/log/kubernetes/kube-apiserver-audit.log \
              ./kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-$cluster-kube-apiserver-audit.log
          done
          tar zcvf kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz \
            `kind get clusters | xargs -I {} echo kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-{}-kube-apiserver-audit.log`

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-audit-log
          path: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          for cluster in `kind get clusters`; do
            kubectl config use-context kind-$cluster
            make kubectl-ko-log
            mv kubectl-ko-log.tar.gz kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-$cluster-ko-log.tar.gz
          done
          tar zcvf kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz \
            `kind get clusters | xargs -I {} echo kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-{}-ko-log.tar.gz`

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log
          path: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  multus-conformance-e2e:
    name: Multus Conformance E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          - ipv6
          - dual
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load images
        run: docker load -i kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        id: install
        run: make kind-install-${{ matrix.ip-family }}

      - name: Install Multus
        run: make kind-install-multus

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
        run: make kube-ovn-multus-conformance-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > multus-conformance-e2e-${{ matrix.ip-family }}-events.yaml
          tar zcf multus-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz multus-conformance-e2e-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: multus-conformance-e2e-${{ matrix.ip-family }}-events
          path: multus-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf multus-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: multus-conformance-e2e-${{ matrix.ip-family }}-audit-log
          path: multus-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz multus-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: multus-conformance-e2e-${{ matrix.ip-family }}-ko-log
          path: multus-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  non-primary-cni-e2e:
    name: Non-Primary CNI E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - name: Remove DNS search domain
        run: |
          sudo sed -i '/^search/d' /etc/resolv.conf
          sudo systemctl restart docker

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Download vpc-nat-gateway image
        uses: actions/download-artifact@v7
        with:
          name: vpc-nat-gateway

      - name: Load image
        run: |
          docker load --input kube-ovn.tar
          docker load --input vpc-nat-gateway.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Load vpc-nat-gateway image into kind cluster
        run: make kind-load-image-vpc-nat-gateway

      - name: Install Mults-CNI, Cilium, and Kube-OVN
        id: install
        run: make kind-install-multus-cilium-kubeovn-non-primary-${{ matrix.ip-family }}

      - name: Run non-primary CNI e2e
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
          E2E_NETWORK_MODE: overlay
          KUBE_OVN_PRIMARY_CNI: "false"
        run: make kube-ovn-non-primary-cni-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > non-primary-cni-e2e-${{ matrix.ip-family }}-events.yaml
          tar zcf non-primary-cni-e2e-${{ matrix.ip-family }}-events.tar.gz non-primary-cni-e2e-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: non-primary-cni-e2e-${{ matrix.ip-family }}-events
          path: non-primary-cni-e2e-${{ matrix.ip-family }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf non-primary-cni-e2e-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: non-primary-cni-e2e-${{ matrix.ip-family }}-audit-log
          path: non-primary-cni-e2e-${{ matrix.ip-family }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        working-directory: ${{ env.E2E_DIR }}
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz non-primary-cni-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: non-primary-cni-e2e-${{ matrix.ip-family }}-ko-log
          path: non-primary-cni-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  chart-test:
    name: Chart Installation/Uninstallation Test
    needs: build-kube-ovn
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        ssl:
          - "true"
          - "false"
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN
        id: install
        env:
          ENABLE_SSL: "${{ matrix.ssl }}"
        run: make kind-install-chart

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
        run: make check-kube-ovn-pod-restarts

      - name: Uninstall Kube-OVN
        run: make uninstall-chart

  underlay-logical-gateway-installation-test:
    name: Underlay Logical Gateway Installation Test
    needs: build-kube-ovn
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-dual

      - name: Install Kube-OVN
        id: install
        run: make kind-install-underlay-logical-gateway-dual

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
        run: make check-kube-ovn-pod-restarts

      - name: Cleanup
        run: timeout -k 10 180 sh -x dist/images/cleanup.sh

  no-ovn-lb-test:
    name: Disable OVN LB Test
    needs: build-kube-ovn
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN without LoadBalancer
        id: install
        env:
          ENABLE_LB: "false"
        run: make kind-install

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
        run: make check-kube-ovn-pod-restarts

      - name: Cleanup
        run: timeout -k 10 180 sh -x dist/images/cleanup.sh

  no-np-test:
    name: Disable Network Policy Test
    needs: build-kube-ovn
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN
        id: install
        env:
          ENABLE_NP: "false"
        run: make kind-install

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
        run: make check-kube-ovn-pod-restarts

      - name: Cleanup
        run: timeout -k 10 180 sh -x dist/images/cleanup.sh

  lb-svc-e2e:
    name: LB Service E2E
    needs:
      - build-kube-ovn
      - build-vpc-nat-gateway
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Download vpc-nat-gateway image
        uses: actions/download-artifact@v7
        with:
          name: vpc-nat-gateway

      - name: Load images
        run: |
          docker load -i kube-ovn.tar
          docker load -i vpc-nat-gateway.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Multus and Kube-OVN
        id: install
        run: make kind-install-lb-svc

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
        run: make kube-ovn-lb-svc-conformance-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > lb-svc-e2e-events.yaml
          tar zcf lb-svc-e2e-events.tar.gz lb-svc-e2e-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: lb-svc-e2e-events
          path: lb-svc-e2e-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf lb-svc-e2e-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: lb-svc-e2e-audit-log
          path: lb-svc-e2e-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz lb-svc-e2e-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: lb-svc-e2e-ko-log
          path: lb-svc-e2e-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  webhook-e2e:
    name: Webhook E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load images
        run: |
          docker load -i kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN with webhook
        id: install
        run: make kind-install-webhook

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
        run: make kube-ovn-webhook-e2e

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz webhook-e2e-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: webhook-e2e-ko-log
          path: webhook-e2e-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  installation-compatibility-test:
    name: Installation Compatibility Test
    needs: build-kube-ovn
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    strategy:
      fail-fast: false
      matrix:
        k8s-version:
          - v1.29.14
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          K8S_VERSION: ${{ matrix.k8s-version }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN
        id: install
        run: make kind-install

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
        run: make check-kube-ovn-pod-restarts

      - name: kubectl ko log
        if: failure() && steps.install.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz installation-compatibility-test-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.install.conclusion == 'failure'
        with:
          name: installation-compatibility-test-ko-log
          path: installation-compatibility-test-ko-log.tar.gz

      - name: Cleanup
        run: timeout -k 10 180 sh -x dist/images/cleanup.sh

  talos-installation-test:
    name: Talos Installation Test
    needs: build-kube-ovn
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          - ipv6
          - dual
        mode:
          - overlay
          - underlay
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Install talosctl
        run: |
          curl -sLo install-talosctl.sh https://talos.dev/install
          sed -i "s|/latest/download/|/download/${{ env.TALOS_VERSION }}/|g" install-talosctl.sh
          sed -i "s|/latest/|/${{ env.TALOS_VERSION }}/|g" install-talosctl.sh
          sh install-talosctl.sh

      - name: Install KVM and libvirt
        run: |
          sudo apt update
          sudo apt install qemu-kvm libvirt-daemon-system

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create Talos cluster
        run: |
          pipx install jinjanator
          make talos-init-${{ matrix.ip-family }}
          kubectl describe node

      - name: Install Kube-OVN
        id: install
        run: |
          make talos-install-${{ matrix.mode }}-${{ matrix.ip-family }}
          kubectl -n kube-system rollout status ds/kube-ovn-pinger
          kubectl -n kube-system rollout status deploy/coredns
          kubectl get pod -o wide -A
          kubectl get endpointslice -A
          kubectl get subnet -o wide
          if [ "${{ matrix.mode }}" = "underlay" ]; then
            kubectl get provider-network -o wide
            kubectl get vlan -o wide
          fi

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
        run: make check-kube-ovn-pod-restarts || make check-kube-ovn-pod-restarts

  kube-ovn-cnp-domain-e2e:
    name: Kube-OVN CNP Domain E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
        mode:
          - overlay
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Set environment variables
        run: |
          if [ $(($RANDOM%2)) -ne 0 ]; then
            # run as root and use valgrind to debug memory leak
            echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV"
            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
          fi

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Clone DNSNameResolver repository
        run: |
          git clone https://github.com/kubeovn/dnsnameresolver.git
          cd dnsnameresolver
          make build
          docker save kubeovn/dnsnameresolver:dev -o dnsnameresolver.tar
          docker load --input dnsnameresolver.tar
          make install

      - name: Install Kube-OVN
        id: install
        run: make kind-install-anp-dns-resolver

      - name: Run CNP Domain E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
          E2E_NETWORK_MODE: ${{ matrix.mode }}
        run: make kube-ovn-cnp-domain-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > kube-ovn-anp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml
          tar zcf kube-ovn-anp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz kube-ovn-anp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-cnp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events
          path: kube-ovn-cnp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  kube-ovn-anp-domain-e2e:
    name: Kube-OVN ANP Domain E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
        mode:
          - overlay
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Set environment variables
        run: |
          if [ $(($RANDOM%2)) -ne 0 ]; then
            # run as root and use valgrind to debug memory leak
            echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV"
            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
          fi

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Clone DNSNameResolver repository
        run: |
          git clone https://github.com/kubeovn/dnsnameresolver.git
          cd dnsnameresolver
          make build
          docker save kubeovn/dnsnameresolver:dev -o dnsnameresolver.tar
          docker load --input dnsnameresolver.tar
          make install

      - name: Install Kube-OVN
        id: install
        run: make kind-install-anp-dns-resolver

      - name: Run ANP Domain E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
          E2E_NETWORK_MODE: ${{ matrix.mode }}
        run: make kube-ovn-anp-domain-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > kube-ovn-anp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml
          tar zcf kube-ovn-anp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz kube-ovn-anp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-anp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events
          path: kube-ovn-anp-domain-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  cilium-chaining-e2e:
    name: Cilium Chaining E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          # - ipv6
          # - dual
        mode:
          - overlay
          - underlay
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6
      - uses: azure/setup-helm@v4.3.1
        with:
          version: '${{ env.HELM_VERSION }}'

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - name: Configure docker
        run: |
          # enable addition of ip6tables rules
          sudo sh -c "echo '{\"experimental\": true, \"ip6tables\": true}' > /etc/docker/daemon.json"
          # remove DNS search domain
          sudo sed -i '/^search/d' /etc/resolv.conf
          # restart docker
          sudo systemctl restart docker

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-cilium-chaining-${{ matrix.ip-family }}

      - name: Install Kube-OVN with Cilium chaining
        id: install
        run: make kind-install-cilium-chaining-${{ matrix.mode }}-${{ matrix.ip-family }}

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_CILIUM_CHAINING: "true"
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
          E2E_NETWORK_MODE: ${{ matrix.mode }}
        run: make k8s-conformance-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml
          tar zcf cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events
          path: cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-audit-log
          path: cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: cilium-chaining-e2e-ko-log
          path: cilium-chaining-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

      - name: Cleanup
        run: timeout -k 10 180 sh -x dist/images/cleanup.sh

  kube-ovn-ha-e2e:
    name: Kube-OVN HA E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ssl:
          - "true"
          - "false"
        bind-local:
          - "true"
          - "false"
        ip-family:
          - ipv4
          - ipv6
          - dual
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-ha-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        id: install
        env:
          NET_STACK: "${{ matrix.ip-family }}"
          ENABLE_SSL: "${{ matrix.ssl }}"
          ENABLE_BIND_LOCAL_IP: "${{ matrix.bind-local }}"
        run: make kind-install-chart

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
        run: |
          make kube-ovn-security-e2e
          make kube-ovn-ha-e2e

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log
          path: kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

      - name: Cleanup
        run: timeout -k 10 180 sh -x dist/images/cleanup.sh

  kube-ovn-submariner-conformance-e2e:
    name: Kube-OVN Submariner Conformance E2E
    needs:
      - build-kube-ovn
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Install submariner subctl
        env:
          VERSION: v${{ env.SUBMARINER_VERSION }}
          DESTDIR: /usr/local/bin
        run: curl -Ls https://get.submariner.io | bash

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-ovn-submariner

      - name: Install Kube-OVN and Submariner
        id: install
        run: make kind-install-ovn-submariner

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        run: make kube-ovn-submariner-conformance-e2e

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz kube-ovn-submariner-conformance-e2e-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-submariner-conformance-e2e-ko-log
          path: kube-ovn-submariner-conformance-e2e-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

      - name: Cleanup
        run: timeout -k 10 180 sh -x dist/images/cleanup.sh

  vpc-egress-gateway-e2e:
    name: VPC Egress Gateway E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          - ipv6
          - dual
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          n_worker=2 make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        id: install
        env:
          UNTAINT_CONTROL_PLANE: "false"
        run: make kind-install-debug-valgrind-${{ matrix.ip-family }}

      - name: Install Multus
        run: make kind-install-multus

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
        run: make vpc-egress-gateway-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > vpc-egress-gateway-e2e-${{ matrix.ip-family }}-events.yaml
          tar zcf vpc-egress-gateway-e2e-${{ matrix.ip-family }}-events.tar.gz vpc-egress-gateway-e2e-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: vpc-egress-gateway-e2e-${{ matrix.ip-family }}-events
          path: vpc-egress-gateway-e2e-${{ matrix.ip-family }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf vpc-egress-gateway-e2e-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: vpc-egress-gateway-e2e-${{ matrix.ip-family }}-audit-log
          path: vpc-egress-gateway-e2e-${{ matrix.ip-family }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz vpc-egress-gateway-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: vpc-egress-gateway-e2e-${{ matrix.ip-family }}-ko-log
          path: vpc-egress-gateway-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

      - name: Check valgrind result
        run: |
          kubectl -n kube-system rollout restart ds ovs-ovn
          kubectl -n kube-system rollout status ds ovs-ovn
          sleep 10
          kubectl -n kube-system rollout restart deploy ovn-central
          kubectl -n kube-system rollout status deploy ovn-central
          while true; do
            if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then
              break
            fi
            sleep 1
          done
          kubectl ko log ovn
          kubectl ko log ovs

          for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
            echo "Checking if valgrind log file for $daemon exists..."
            find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
          done

          find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
            if grep -qw 'definitely lost' "$f"; then
              echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
              echo $f
              cat "$f"
              exit 1
            fi;
          done

      - name: Cleanup
        run: timeout -k 10 180 sh -x dist/images/cleanup.sh

  iptables-vpc-nat-gw-conformance-e2e:
    name: Iptables VPC NAT Gateway E2E
    needs:
      - build-kube-ovn
      - build-vpc-nat-gateway
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Download vpc-nat-gateway image
        uses: actions/download-artifact@v7
        with:
          name: vpc-nat-gateway

      - name: Load images
        run: |
          docker load -i kube-ovn.tar
          docker load -i vpc-nat-gateway.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN with VPC NAT gateway enabled
        id: install
        run: make kind-install-vpc-nat-gw

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
        run: make iptables-vpc-nat-gw-conformance-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > iptables-vpc-nat-gw-conformance-e2e-events.yaml
          tar zcf iptables-vpc-nat-gw-conformance-e2e-events.tar.gz iptables-vpc-nat-gw-conformance-e2e-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: iptables-vpc-nat-gw-conformance-e2e-events
          path: iptables-vpc-nat-gw-conformance-e2e-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf iptables-vpc-nat-gw-conformance-e2e-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: iptables-vpc-nat-gw-conformance-e2e-audit-log
          path: iptables-vpc-nat-gw-conformance-e2e-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz iptables-vpc-nat-gw-conformance-e2e-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: iptables-vpc-nat-gw-conformance-e2e-ko-log
          path: iptables-vpc-nat-gw-conformance-e2e-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  ovn-vpc-nat-gw-conformance-e2e:
    name: OVN VPC NAT Gateway E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load images
        run: docker load -i kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN
        id: install
        run: make kind-install

      - name: Run Vip E2E
        id: vip-e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
        run: make vip-conformance-e2e

      - name: Run Ovn VPC NAT GW E2E
        id: vpc-e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
        run: make ovn-vpc-nat-gw-conformance-e2e

      - name: Collect k8s events
        if: failure() && (steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')
        run: |
          kubectl get events -A -o yaml > ovn-vpc-nat-gw-conformance-e2e-events.yaml
          tar zcf ovn-vpc-nat-gw-conformance-e2e-events.tar.gz ovn-vpc-nat-gw-conformance-e2e-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && (steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')
        with:
          name: ovn-vpc-nat-gw-conformance-e2e-events
          path: ovn-vpc-nat-gw-conformance-e2e-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && (steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf ovn-vpc-nat-gw-conformance-e2e-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && (steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')
        with:
          name: ovn-vpc-nat-gw-conformance-e2e-audit-log
          path: ovn-vpc-nat-gw-conformance-e2e-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && (steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz ovn-vpc-nat-gw-conformance-e2e-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && (steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')
        with:
          name: ovn-vpc-nat-gw-conformance-e2e-ko-log
          path: ovn-vpc-nat-gw-conformance-e2e-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  kube-ovn-ipsec-e2e:
    name: OVN IPSEC E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load images
        run: docker load -i kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN
        id: install
        run: make kind-install-ovn-ipsec

      - name: Run Ovn IPSEC E2E
        id: kube-ovn-ipsec-e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
        run: make kube-ovn-ipsec-e2e

      - name: Collect k8s events
        if: failure() && ( steps.ovn-ipsec-e2e.conclusion == 'failure')
        run: |
          kubectl get events -A -o yaml > kube-ovn-ipsec-e2e-events.yaml
          tar zcf kube-ovn-ipsec-e2e-events.tar.gz kube-ovn-ipsec-e2e-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-ipsec-e2e-events
          path: kube-ovn-ipsec-e2e-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf kube-ovn-ipsec-e2e-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-ipsec-e2e-audit-log
          path: kube-ovn-ipsec-e2e-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz kube-ovn-ipsec-e2e-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-ipsec-e2e-ko-log
          path: kube-ovn-ipsec-e2e-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-ipsec-e2e.conclusion == 'failure')) }}
        env:
          IGNORABLE_PODS: app=kube-ovn-pinger
        run: make check-kube-ovn-pod-restarts

  kube-ovn-ipsec-cert-mgr-e2e:
    name: OVN IPSEC E2E CERT MANAGER
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load images
        run: docker load -i kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN
        id: install
        run: make kind-install-ovn-ipsec-cert-manager

      - name: Run Ovn IPSEC cert-manager E2E
        id: kube-ovn-ipsec-cert-mgr-e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
        run: make kube-ovn-ipsec-cert-mgr-e2e

      - name: Collect k8s events
        if: failure() && ( steps.ovn-ipsec-cert-mgr-e2e.conclusion == 'failure')
        run: |
          kubectl get events -A -o yaml > kube-ovn-ipsec-cert-mgr-e2e-events.yaml
          tar zcf kube-ovn-ipsec-cert-mgr-e2e-events.tar.gz kube-ovn-ipsec-cert-mgr-e2e-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-ipsec-cert-mgr-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-ipsec-cert-mgr-e2e-events
          path: kube-ovn-ipsec-cert-mgr-e2e-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && (steps.kube-ovn-ipsec-cert-mgr-e2e.conclusion == 'failure')
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf kube-ovn-ipsec-cert-mgr-e2e-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-ipse-cert-mgrc-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-ipsec-cert-mgr-e2e-audit-log
          path: kube-ovn-ipsec-cert-mgr-e2e-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && (steps.kube-ovn-ipsec-cert-mgr-e2e.conclusion == 'failure')
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz kube-ovn-ipsec-cert-mgr-e2e-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-ipsec-cert-mgr-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-ipsec-cert-mgr-e2e-ko-log
          path: kube-ovn-ipsec-cert-mgr-e2e-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-ipsec-cert-mgr-e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  kube-ovn-connectivity-test:
    name: Kube-OVN Connectivity E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        mode:
          - overlay
          - underlay
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load images
        run: docker load -i kube-ovn.tar

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init

      - name: Install Kube-OVN
        id: install
        run: make kind-install-${{ matrix.mode }}

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
        run: make kube-ovn-connectivity-e2e

      - name: kubectl ko log
        if: failure() && (steps.e2e.conclusion == 'failure')
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz kube-ovn-connectivity-e2e-${{ matrix.mode }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-connectivity-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-connectivity-e2e-${{ matrix.mode }}-ko-log
          path: kube-ovn-connectivity-e2e-${{ matrix.mode }}-ko-log.tar.gz


  kube-ovn-underlay-metallb-e2e:
    name: OVN METALLB E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          - ipv4
          - ipv6
          - dual
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v6
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v6
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: |
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.ref_name }}-
            ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ github.base_ref }}-

      - name: Install kind
        uses: helm/kind-action@v1.13.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Load images
        run: docker load -i kube-ovn.tar

      - name: Set environment variables
        run: |
          if [ $(($RANDOM%2)) -ne 0 ]; then
            # run as root and use valgrind to debug memory leak
            echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV"
            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
          fi

      - name: Create kind cluster
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pipx install jinjanator
          make kind-ghcr-pull kind-init-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        id: install
        run: make kind-install-metallb-pool-from-underlay-${{ matrix.ip-family }}

      - name: Run Ovn Metallb and Kube-OVN Combine E2E
        id: kube-ovn-underlay-metallb-e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
        run: make kube-ovn-underlay-metallb-e2e

      - name: Collect k8s events
        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
        run: |
          kubectl get events -A -o yaml > kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-events.yaml
          tar zcf kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-events.tar.gz kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-events
          path: kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-audit-log
          path: kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v6
        if: failure() && (steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')
        with:
          name: kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-ko-log
          path: kube-ovn-underlay-metallb-${{ matrix.ip-family }}-e2e-ko-log.tar.gz

      - name: Check kube ovn pod restarts
        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-underlay-metallb-e2e.conclusion == 'failure')) }}
        run: make check-kube-ovn-pod-restarts

  push:
    name: Push Images
    needs:
      - k8s-conformance-e2e
      - k8s-netpol-e2e
      - cyclonus-netpol-e2e
      - kube-ovn-conformance-e2e
      - kube-ovn-ic-conformance-e2e
      - kube-ovn-ipsec-e2e
      - kube-ovn-ipsec-cert-mgr-e2e
      - kube-ovn-underlay-metallb-e2e
      - multus-conformance-e2e
      - non-primary-cni-e2e
      - vpc-egress-gateway-e2e
      - ovn-vpc-nat-gw-conformance-e2e
      - iptables-vpc-nat-gw-conformance-e2e
      - webhook-e2e
      - lb-svc-e2e
      - underlay-logical-gateway-installation-test
      - chart-test
      - installation-compatibility-test
      - talos-installation-test
      - no-ovn-lb-test
      - no-np-test
      - cilium-chaining-e2e
      - kube-ovn-ha-e2e
      - kube-ovn-submariner-conformance-e2e
    if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
    runs-on: ubuntu-24.04
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v6

      - name: Download kube-ovn image
        uses: actions/download-artifact@v7
        with:
          name: kube-ovn

      - name: Download kube-ovn-dpdk image
        uses: actions/download-artifact@v7
        if: github.event_name != 'pull_request'
        with:
          name: kube-ovn-dpdk

      - name: Download vpc-nat-gateway image
        uses: actions/download-artifact@v7
        with:
          name: vpc-nat-gateway

      - name: Load image
        run: |
          docker load --input kube-ovn.tar
          docker load --input vpc-nat-gateway.tar
          if [ '${{ github.event_name }}' != 'pull_request' ]; then
            docker load --input kube-ovn-dpdk.tar
          fi

      - name: Security Scan
        env:
          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2
        run: |
          sudo apt-get install wget apt-transport-https gnupg lsb-release
          wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
          echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list
          sudo apt-get update
          sudo apt-get install trivy
          make scan

      - name: Push
        if: github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, 'release-')
        env:
          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
          COMMIT: ${{ github.sha }}
        run: |
          cat VERSION
          TAG=$(cat VERSION)
          echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
          docker tag kubeovn/kube-ovn:$TAG kubeovn/kube-ovn-dev:$COMMIT-x86
          docker tag kubeovn/kube-ovn:$TAG kubeovn/kube-ovn:$TAG-x86
          docker tag kubeovn/kube-ovn:$TAG-debug kubeovn/kube-ovn:$TAG-debug-x86
          docker tag kubeovn/kube-ovn:$TAG-dpdk kubeovn/kube-ovn:$TAG-dpdk-x86
          docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway-dev:$COMMIT-x86
          docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway:$TAG-x86
          docker images
          docker push kubeovn/kube-ovn:$TAG-x86
          docker push kubeovn/kube-ovn:$TAG-amd64-legacy
          docker push kubeovn/kube-ovn-dev:$COMMIT-x86
          docker push kubeovn/kube-ovn:$TAG-debug-x86
          docker push kubeovn/kube-ovn:$TAG-dpdk-x86
          docker push kubeovn/vpc-nat-gateway:$TAG-x86
          docker push kubeovn/vpc-nat-gateway-dev:$COMMIT-x86