build-x86-image.yaml

name: Build x86 Image

on:
  pull_request:
    branches:
    - master
    - release-*
    paths-ignore:
    - 'docs/**'
    - '**.md'
  push:
    branches:
    - master
    - release-*
    paths-ignore:
    - 'docs/**'
    - '**.md'

concurrency:
  group: "${{ github.workflow }}-${{ github.ref }}"
  cancel-in-progress: true

env:
  KIND_VERSION: v0.27.0
  GOLANGCI_LINT_VERSION: 'v2.1.1'
  HELM_VERSION: v3.17.3
  SUBMARINER_VERSION: '0.19.3'

jobs:
  build-kube-ovn-base:
    name: Build kube-ovn-base
    runs-on: ubuntu-24.04
    outputs:
      build-base: ${{ steps.check.outputs.build-base }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - id: check
        run: |
          if [ ${{ github.event_name }} != 'pull_request' ]; then
            exit
          fi
          tmp_dir=`mktemp -d`
          cat > "$tmp_dir/on_changes.txt" <<EOF
          dist/images/Dockerfile.base
          dist/images/OpenBFDD-compile.patch
          dist/images/go-deps/download-go-deps.sh
          dist/images/go-deps/rebuild-go-deps.sh
          EOF
          if git diff --name-only HEAD^ HEAD | grep -Ff "$tmp_dir/on_changes.txt"; then
            echo build-base=1 >> "$GITHUB_OUTPUT"
          fi
          rm -frv "$tmp_dir"

      - uses: jlumbroso/free-disk-space@v1.3.1
        if: steps.check.outputs.build-base == 1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: docker/setup-buildx-action@v3
        if: steps.check.outputs.build-base == 1

      - uses: actions/setup-go@v5
        if: steps.check.outputs.build-base == 1
        id: setup-go
        with:
          go-version-file: go.mod
          check-latest: true
          cache: false

      - name: Build kube-ovn-base image
        id: build
        if: steps.check.outputs.build-base == 1
        env:
          GO_VERSION: ${{ steps.setup-go.outputs.go-version }}
        run: |
            make base-amd64
            make base-tar-amd64

      - name: Upload base images to artifact
        if: steps.check.outputs.build-base == 1
        uses: actions/upload-artifact@v4
        with:
          name: kube-ovn-base
          path: image-amd64.tar

  build-kube-ovn-dpdk-base:
    name: Build kube-ovn-dpdk-base
    runs-on: ubuntu-24.04
    outputs:
      build-dpdk-base: ${{ steps.check.outputs.build-dpdk-base }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - id: check
        run: |
          if [ ${{ github.event_name }} != 'pull_request' ]; then
            exit
          fi
          if ! git diff --exit-code HEAD^...HEAD -- dist/images/Dockerfile.base-dpdk; then
            echo build-dpdk-base=1 >> "$GITHUB_OUTPUT"
          fi

      - uses: jlumbroso/free-disk-space@v1.3.1
        if: steps.check.outputs.build-dpdk-base == 1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: docker/setup-buildx-action@v3
        if: steps.check.outputs.build-dpdk-base == 1

      - name: Build kube-ovn-dpdk-base image
        id: build
        if: steps.check.outputs.build-dpdk-base == 1
        run: |
          make base-amd64-dpdk
          make base-tar-amd64-dpdk

      - name: Upload dpdk base images to artifact
        if: steps.check.outputs.build-dpdk-base == 1
        uses: actions/upload-artifact@v4
        with:
          name: kube-ovn-dpdk-base
          path: image-amd64-dpdk.tar

  build-kube-ovn:
    name: Build kube-ovn
    runs-on: ubuntu-24.04
    needs:
      - build-kube-ovn-base
      - build-kube-ovn-dpdk-base
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: actions/setup-go@v5
        id: setup-go
        with:
          go-version-file: go.mod
          check-latest: true
          cache: false

      - name: Setup environment variables
        run: |
          echo "TAG=$(cat VERSION)" >> "$GITHUB_ENV"
          echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-x86-${{ hashFiles('**/go.sum') }}
          restore-keys: ${{ runner.os }}-go-${{ env.GO_VERSION }}-x86-

      - name: Unit test
        run: |
          go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
          # make ut

      # - uses: shogo82148/actions-goveralls@v1
      #   with:
      #     path-to-profile: profile.cov

      - name: Install golangci-lint
        run: |
          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin $GOLANGCI_LINT_VERSION

      - name: Download base images
        if: needs.build-kube-ovn-base.outputs.build-base == 1
        uses: actions/download-artifact@v4
        with:
          name: kube-ovn-base

      - name: Load base images
        if: needs.build-kube-ovn-base.outputs.build-base == 1
        run: |
          docker load --input image-amd64.tar
          docker tag kubeovn/kube-ovn-base:$TAG-amd64 kubeovn/kube-ovn-base:$TAG
          docker tag kubeovn/kube-ovn-base:$TAG-debug-amd64 kubeovn/kube-ovn-base:$TAG-debug

      - name: Download dpdk base images
        if: needs.build-kube-ovn-dpdk-base.outputs.build-dpdk-base == 1
        uses: actions/download-artifact@v4
        with:
          name: kube-ovn-dpdk-base

      - name: Load dpdk base images
        if: needs.build-kube-ovn-dpdk-base.outputs.build-dpdk-base == 1
        run: |
          docker load --input image-amd64-dpdk.tar
          docker tag kubeovn/kube-ovn-base:$TAG-amd64-dpdk kubeovn/kube-ovn-base:$TAG-dpdk

      - name: Scan base image
        uses: aquasecurity/trivy-action@0.30.0
        env:
          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2
        with:
          scan-type: image
          scanners: vuln
          image-ref: docker.io/kubeovn/kube-ovn-base:${{ env.TAG }}
          format: json
          output: trivy-result.json
          ignore-unfixed: true
          trivyignores: .trivyignore
          vuln-type: library

      - name: Build kubectl and CNI plugins from source
        env:
          CGO_ENABLED: "0"
          GO_INSTALL: "go install -v -mod=mod -trimpath"
        run: |
          cat trivy-result.json
          dockerfile=${{ github.workspace }}/dist/images/Dockerfile
          export GOBIN=`dirname "$dockerfile"`
          cni_plugins_version=`go list -m -f '{{.Version}}' github.com/containernetworking/plugins`
          cni_plugins_build_flags="-ldflags '-extldflags -static -X github.com/containernetworking/plugins/pkg/utils/buildversion.BuildVersion=$cni_plugins_version'"
          jq -r '.Results[] | select((.Type=="gobinary") and (.Vulnerabilities!=null)) | .Target' trivy-result.json | while read f; do
            bin=`basename $f`
            case $bin in
              loopback|macvlan)
                echo "Building $bin@$cni_plugins_version from source..."
                sh -c "cd /tmp && $GO_INSTALL $cni_plugins_build_flags github.com/containernetworking/plugins/plugins/main/$bin@$cni_plugins_version"
                echo "COPY $bin /$f" >> "$dockerfile"
                ;;
              portmap)
                echo "Building $bin@$cni_plugins_version from source..."
                sh -c "cd /tmp && $GO_INSTALL $cni_plugins_build_flags github.com/containernetworking/plugins/plugins/meta/$bin@$cni_plugins_version"
                echo "COPY $bin /$f" >> "$dockerfile"
                ;;
              kubectl)
                go mod tidy
                version=`go list -m -f '{{.Version}}' k8s.io/kubernetes`
                mod_dir=`go list -m -f '{{.Dir}}' k8s.io/kubernetes`
                source "$mod_dir/hack/lib/util.sh"
                source "$mod_dir/hack/lib/logging.sh"
                source "$mod_dir/hack/lib/version.sh"
                repo=kubernetes/kubernetes
                commit=unknown
                read type tag_sha < <(echo $(curl -s "https://api.github.com/repos/$repo/git/ref/tags/$version" |
                  jq -r '.object.type,.object.sha'))
                if [ $type = "commit" ]; then
                  commit=$tag_sha
                else
                  commit=$(curl -s "https://api.github.com/repos/$repo/git/tags/$tag_sha" | jq -r '.object.sha')
                fi
                export KUBE_GIT_COMMIT="${commit}"
                export KUBE_GIT_TREE_STATE='clean'
                export KUBE_GIT_VERSION="${version}"
                export KUBE_GIT_MAJOR=`echo $KUBE_GIT_VERSION | cut -d. -f1 | sed 's/$v//'`
                export KUBE_GIT_MINOR=`echo $KUBE_GIT_VERSION | cut -d. -f2`
                goldflags="all=$(kube::version::ldflags) -s -w"
                echo "Building $bin@$version from source..."
                $GO_INSTALL -ldflags="${goldflags}" k8s.io/kubernetes/cmd/kubectl
                echo "COPY $bin /$f" >> "$dockerfile"
                ;;
              gobgp)
                $GO_INSTALL github.com/osrg/gobgp/v3/cmd/$bin
                echo "COPY $bin /$f" >> "$dockerfile"
                ;;
              *)
                ;;
            esac
          done

      - name: Build
        run: |
          go mod tidy
          git diff --exit-code -- go.mod go.sum
          # make lint
          if [ ${{ needs.build-kube-ovn-base.outputs.build-base || 0 }} = 1 ]; then
            make build-kube-ovn
          else
            make image-kube-ovn
          fi
          make tar-kube-ovn

      - name: Build dpdk
        run: |
          if [ ${{ needs.build-kube-ovn-dpdk-base.outputs.build-dpdk-base || 0 }} = 1 ]; then
            make build-kube-ovn-dpdk
          else
            make image-kube-ovn-dpdk
          fi
          make tar-kube-ovn-dpdk

      - name: Upload images to artifact
        uses: actions/upload-artifact@v4
        with:
          name: kube-ovn
          path: kube-ovn.tar

      - name: Upload dpdk images to artifact
        uses: actions/upload-artifact@v4
        if: github.event_name != 'pull_request'
        with:
          name: kube-ovn-dpdk
          path: kube-ovn-dpdk.tar

  build-vpc-nat-gateway:
    name: Build vpc-nat-gateway
    runs-on: ubuntu-24.04
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v4

      - name: Build
        run: |
          make image-vpc-nat-gateway
          make tar-vpc-nat-gateway

      - name: Upload image to artifact
        uses: actions/upload-artifact@v4
        with:
          name: vpc-nat-gateway
          path: vpc-nat-gateway.tar

  build-e2e-binaries:
    name: Build E2E Binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v5
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Lookup Go cache
        id: lookup-go-cache
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-
          lookup-only: true

      - uses: jlumbroso/free-disk-space@v1.3.1
        if: steps.lookup-go-cache.outputs.cache-hit != 'true'
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - name: Go cache
        if: steps.lookup-go-cache.outputs.cache-hit != 'true'
        uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-

      - name: Install ginkgo
        if: steps.lookup-go-cache.outputs.cache-hit != 'true'
        working-directory: ${{ env.E2E_DIR }}
        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo

      - run: make e2e-build
        if: steps.lookup-go-cache.outputs.cache-hit != 'true'
        working-directory: ${{ env.E2E_DIR }}

  kube-ovn-conformance-e2e:
    name: Kube-OVN Conformance E2E
    needs:
      - build-kube-ovn
      - build-e2e-binaries
    runs-on: ubuntu-24.04
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        ip-family:
          # - ipv4
          - ipv6
          # - dual
        mode:
          - overlay
          # - underlay
        index:
          - 0
          - 1
          - 2
          - 3
          - 4
          - 5
          - 6
          - 7
          - 8
          - 9
        index2:
          - 0
          - 1
          - 2
          - 3
          - 4
          - 5
          - 6
          - 7
          - 8
          - 9
    steps:
      - uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          docker-images: false
          large-packages: false
          tool-cache: false
          swap-storage: false

      - uses: actions/checkout@v4

      - name: Create the default branch directory
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        run: mkdir -p test/e2e/source

      - name: Check out the default branch
        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.repository.default_branch }}
          fetch-depth: 1
          path: test/e2e/source

      - name: Export E2E directory
        run: |
          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
            echo "E2E_DIR=." >> "$GITHUB_ENV"
          else
            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
          fi

      - uses: actions/setup-go@v5
        id: setup-go
        with:
          go-version-file: ${{ env.E2E_DIR }}/go.mod
          check-latest: true
          cache: false

      - name: Export Go full version
        run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV"

      - name: Go cache
        uses: actions/cache/restore@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
          restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-

      - name: Install kind
        uses: helm/kind-action@v1.12.0
        with:
          version: ${{ env.KIND_VERSION }}
          install_only: true

      - name: Install ginkgo
        working-directory: ${{ env.E2E_DIR }}
        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo

      - name: Download image
        uses: actions/download-artifact@v4
        with:
          name: kube-ovn

      - name: Load image
        run: docker load --input kube-ovn.tar

      - name: Set environment variables
        run: |
          if [ $(($RANDOM%2)) -ne 0 ]; then
            # run as root and use valgrind to debug memory leak
            echo "VERSION=$(cat VERSION)-debug" >> "$GITHUB_ENV"
            echo "DEBUG_WRAPPER=valgrind" >> "$GITHUB_ENV"
          fi

      - name: Create kind cluster
        run: |
          pipx install jinjanator
          make kind-init-${{ matrix.ip-family }}

      - name: Install Kube-OVN
        id: install
        run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }}

      - name: Run E2E
        id: e2e
        working-directory: ${{ env.E2E_DIR }}
        env:
          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
          E2E_IP_FAMILY: ${{ matrix.ip-family }}
          E2E_NETWORK_MODE: ${{ matrix.mode }}
        run: |
          make kube-ovn-conformance-e2e
          make kube-ovn-conformance-e2e
          make kube-ovn-conformance-e2e
          make kube-ovn-conformance-e2e
          make kube-ovn-conformance-e2e
          make kube-ovn-conformance-e2e
          make kube-ovn-conformance-e2e
          make kube-ovn-conformance-e2e
          make kube-ovn-conformance-e2e
          make kube-ovn-conformance-e2e
          # make kind-install-kubevirt
          # make kube-ovn-kubevirt-e2e

      - name: Collect k8s events
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          kubectl get events -A -o yaml > kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml
          tar zcf kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.yaml

      - name: Upload k8s events
        uses: actions/upload-artifact@v4
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events
          path: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-events.tar.gz

      - name: Collect apiserver audit logs
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
          tar zcf kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log

      - name: Upload apiserver audit logs
        uses: actions/upload-artifact@v4
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-audit-log
          path: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-audit-log.tar.gz

      - name: kubectl ko log
        if: failure() && steps.e2e.conclusion == 'failure'
        run: |
          make kubectl-ko-log
          mv kubectl-ko-log.tar.gz kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log.tar.gz

      - name: upload kubectl ko log
        uses: actions/upload-artifact@v4
        if: failure() && steps.e2e.conclusion == 'failure'
        with:
          name: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log
          path: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log.tar.gz

      # - name: Check kube ovn pod restarts
      #   if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
      #   run: make check-kube-ovn-pod-restarts

      # - name: Check valgrind result
      #   run: |
      #     if [ "x$DEBUG_WRAPPER" != "xvalgrind" ]; then
      #       exit
      #     fi

      #     kubectl -n kube-system rollout restart ds ovs-ovn
      #     kubectl -n kube-system rollout status ds ovs-ovn
      #     sleep 10
      #     kubectl -n kube-system rollout restart deploy ovn-central
      #     kubectl -n kube-system rollout status deploy ovn-central
      #     while true; do
      #       if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then
      #         break
      #       fi
      #       sleep 1
      #     done
      #     kubectl ko log ovn
      #     kubectl ko log ovs

      #     for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
      #       echo "Checking if valgrind log file for $daemon exists..."
      #       find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
      #     done

      #     find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
      #       if grep -qw 'definitely lost' "$f"; then
      #         echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
      #         echo $f
      #         cat "$f"
      #         exit 1
      #       fi;
      #     done

      # - name: Cleanup
      #   run: timeout -k 10 180 sh -x dist/images/cleanup.sh