release.yml

# yamllint disable rule:line-length rule:truthy
name: Release
# Generated by: gromit policy

# Distribution channels covered by this workflow
# - Ubuntu and Debian
# - RHEL and AL
# - docker hub
# - devenv ECR
# - Cloudsmith
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
on:
  # Trigger release every monday at midnight for master CI images
  schedule:
    - cron: "0 0 * * 1"
  pull_request:
  push:
    branches:
      - master
      - release-**
    tags:
      - 'v*'

permissions:
  contents: read

env:
  GOPRIVATE: github.com/TykTechnologies
  VARIATION: prod-variation
  DOCKER_BUILD_SUMMARY: false
  DOCKER_BUILD_RECORD_UPLOAD: false
  # startsWith covers pull_request_target too
  BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
jobs:
  goreleaser:
    if: github.event.pull_request.draft == false
    name: '${{ matrix.golang_cross }}'
    runs-on: warp-ubuntu-latest-x64-4x
    permissions:
      id-token: write # AWS OIDC JWT
      contents: read # actions/checkout
    strategy:
      fail-fast: false
      matrix:
        golang_cross:
          - 1.25-bookworm
        include:
          - golang_cross: 1.25-bookworm
            goreleaser: 'ci/goreleaser/goreleaser.yml'
            cgo: 0
            rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023'
            debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy ubuntu/noble debian/jessie debian/buster debian/bullseye debian/bookworm debian/trixie'
    outputs:
      fips_tags: ${{ steps.ci_metadata_fips.outputs.tags }}
      std_tags: ${{ steps.ci_metadata_std.outputs.tags }}
      commit_author: ${{ steps.set_outputs.outputs.commit_author}}
    steps:
      - name: Checkout of tyk-pump
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 1
      - name: Set some outputs for later
        id: set_outputs
        shell: bash
        env:
          HEAD_REF: ${{github.head_ref}}
        run: |
          echo "commit_author=$(git show -s --format='%ae' HEAD)" >> $GITHUB_OUTPUT
          echo "branch=${HEAD_REF##*/}" >> $GITHUB_OUTPUT
      - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
      - name: Login to DockerHub
        if: startsWith(github.ref, 'refs/tags')
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Login to Cloudsmith
        if: startsWith(github.ref, 'refs/tags')
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          registry: docker.tyk.io
          username: ${{ secrets.CLOUDSMITH_USERNAME }}
          password: ${{ secrets.CLOUDSMITH_API_KEY }}
      - uses: WarpBuilds/cache@f643a1ba29942d56621d07fc2d4284c7219868ad # v1
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
      - name: Build
        env:
          NFPM_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }}
          PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
        run: |
          echo '#!/bin/sh
          ci/bin/unlock-agent.sh
          git config --global url."https://${{ secrets.ORG_GH_TOKEN }}@github.com".insteadOf "https://github.com"
          git config --global --add safe.directory /go/src/github.com/TykTechnologies/tyk-pump
          goreleaser release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot --skip=sign,docker' || '--skip=docker' }}' | tee /tmp/build.sh
          chmod +x /tmp/build.sh
          mkdir -p ~/.docker && echo "{}" > ~/.docker/config.json
          docker run --rm --privileged -e GITHUB_TOKEN=${{ github.token }} \
          -e GOPRIVATE=github.com/TykTechnologies                                \
          -e DEBVERS='${{ matrix.debvers }}'                               \
          -e RPMVERS='${{ matrix.rpmvers }}'                               \
          -e CGO_ENABLED=${{ matrix.cgo }}                                 \
          -e NFPM_PASSPHRASE="$NFPM_PASSPHRASE"                          \
          -e GPG_FINGERPRINT=12B5D62C28F57592D1575BD51ED14C59E37DAC20            \
          -e PKG_SIGNING_KEY="$PKG_SIGNING_KEY"                                  \
          -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN                              \
          -v ${{github.workspace}}:/go/src/github.com/TykTechnologies/tyk-pump \
          -v /var/run/docker.sock:/var/run/docker.sock                           \
          -v ~/.docker/config.json:/root/.docker/config.json                     \
          -e GOCACHE=/cache/go-build                                             \
          -e GOMODCACHE=/go/pkg/mod                                              \
          -v ~/go/pkg/mod:/go/pkg/mod                                            \
          -v ~/.cache/go-build:/cache/go-build                                   \
          -v /tmp/build.sh:/tmp/build.sh                                         \
          -w /go/src/github.com/TykTechnologies/tyk-pump                      \
          tykio/golang-cross:${{ matrix.golang_cross }} /tmp/build.sh
      - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
          # Don't mask to pass it across job boundaries
          mask-aws-account-id: false
      - uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2
        id: ecr
        if: ${{ matrix.golang_cross == '1.25-bookworm' }}
        with:
          mask-password: 'true'
      - name: Docker metadata for fips CI
        id: ci_metadata_fips
        if: ${{ matrix.golang_cross == '1.25-bookworm' }}
        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
        with:
          images: |
            ${{ steps.ecr.outputs.registry }}/tyk-pump-fips
          flavor: |
            latest=false
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=sha,format=long
            type=semver,pattern={{major}},prefix=v
            type=semver,pattern={{major}}.{{minor}},prefix=v
            type=semver,pattern={{version}},prefix=v
      - name: push fips image to CI
        if: ${{ matrix.golang_cross == '1.25-bookworm' }}
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          push: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          tags: ${{ steps.ci_metadata_fips.outputs.tags }}
          labels: ${{ steps.ci_metadata_fips.outputs.labels }}
          build-args: |
            BUILD_PACKAGE_NAME=tyk-pump-fips
      - name: Docker metadata for fips tag push
        id: tag_metadata_fips
        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
        with:
          images: |
            tykio/tyk-pump-fips
          flavor: |
            latest=false
            prefix=v
          tags: |
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{version}}
          labels: |
            org.opencontainers.image.title=Tyk Analytics Pump FIPS
            org.opencontainers.image.description=Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once). This version is compiled with boringssl.
            org.opencontainers.image.vendor=tyk.io
            org.opencontainers.image.version=${{ github.ref_name }}
      - name: push fips image to prod
        if: ${{ matrix.golang_cross == '1.25-bookworm' }}
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: ${{ startsWith(github.ref, 'refs/tags') }}
          tags: ${{ steps.tag_metadata_fips.outputs.tags }}
          labels: ${{ steps.tag_metadata_fips.outputs.labels }}
          build-args: |
            BUILD_PACKAGE_NAME=tyk-pump-fips
      - name: Docker metadata for std CI
        id: ci_metadata_std
        if: ${{ matrix.golang_cross == '1.25-bookworm' }}
        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
        with:
          images: |
            ${{ steps.ecr.outputs.registry }}/tyk-pump
          flavor: |
            latest=false
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=sha,format=long
            type=semver,pattern={{major}},prefix=v
            type=semver,pattern={{major}}.{{minor}},prefix=v
            type=semver,pattern={{version}},prefix=v
      - name: push std image to CI
        if: ${{ matrix.golang_cross == '1.25-bookworm' }}
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64,linux/s390x
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          push: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          tags: ${{ steps.ci_metadata_std.outputs.tags }}
          labels: ${{ steps.ci_metadata_std.outputs.labels }}
          build-args: |
            BUILD_PACKAGE_NAME=tyk-pump
      - name: Docker metadata for std tag push
        id: tag_metadata_std
        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
        with:
          images: |
            docker.tyk.io/tyk-pump/tyk-pump
            tykio/tyk-pump-docker-pub
          flavor: |
            latest=false
            prefix=v
          tags: |
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{version}}
          labels: |
            org.opencontainers.image.title=Tyk Analytics Pump
            org.opencontainers.image.description=Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once).
            org.opencontainers.image.vendor=tyk.io
            org.opencontainers.image.version=${{ github.ref_name }}
      - name: push std image to prod
        if: ${{ matrix.golang_cross == '1.25-bookworm' }}
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64,linux/s390x
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: ${{ startsWith(github.ref, 'refs/tags') }}
          tags: ${{ steps.tag_metadata_std.outputs.tags }}
          labels: ${{ steps.tag_metadata_std.outputs.labels }}
          build-args: |
            BUILD_PACKAGE_NAME=tyk-pump
      - name: save deb
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        if: ${{ matrix.golang_cross == '1.25-bookworm' }}
        with:
          name: deb
          retention-days: 1
          path: |
            dist/*.deb
            !dist/*PAYG*.deb
            !dist/*fips*.deb
      - name: save rpm
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        if: ${{ matrix.golang_cross == '1.25-bookworm' }}
        with:
          name: rpm
          retention-days: 1
          path: |
            dist/*.rpm
            !dist/*PAYG*.rpm
            !dist/*fips*.rpm
  test-controller-api:
    if: github.event.pull_request.draft == false
    needs:
      - goreleaser
    runs-on: warp-ubuntu-latest-x64-2x
    outputs:
      envfiles: ${{ steps.params.outputs.envfiles }}
      pump: ${{ steps.params.outputs.pump }}
      sink: ${{ steps.params.outputs.sink }}
    steps:
      - name: Set test parameters
        uses: TykTechnologies/github-actions/.github/actions/tests/test-controller@d3fa20888fa2878e877e22bb7702141217290e7c # main
        id: params
        with:
          variation: ${{ env.VARIATION }}
          base_ref: ${{ env.BASE_REF }}
          test_type: api
  api-tests:
    needs:
      - test-controller-api
      - goreleaser
    runs-on: warp-ubuntu-latest-x64-4x
    env:
      XUNIT_REPORT_PATH: ${{ github.workspace}}/test-results.xml
    permissions:
      id-token: write # This is required for requesting the Github JWT
      contents: read # This is required for actions/checkout
    strategy:
      fail-fast: false
      matrix:
        envfiles: ${{ fromJson(needs.test-controller-api.outputs.envfiles) }}
        sink: ${{ fromJson(needs.test-controller-api.outputs.sink) }}
    steps:
      - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
      - id: ecr
        uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2
        with:
          mask-password: 'true'
      - name: Setup tmate session only in debug mode
        uses: mxschmitt/action-tmate@c0afd6f790e3a5564914980036ebf83216678101 # v3
        if: runner.debug == '1'
        with:
          detached: true
          limit-access-to-actor: true
          # Only ${{ github.actor }} has access
          # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
      - name: Fetch environment from tyk-pro
        uses: TykTechnologies/github-actions/.github/actions/tests/checkout-tyk-pro@d3fa20888fa2878e877e22bb7702141217290e7c # main
        with:
          org_gh_token: ${{ github.token }}
      - name: Set up test environment
        uses: TykTechnologies/github-actions/.github/actions/tests/env-up@d3fa20888fa2878e877e22bb7702141217290e7c # main
        timeout-minutes: 5
        id: env_up
        with:
          base_ref: ${{ env.BASE_REF }}
          tags: ${{ needs.goreleaser.outputs.ee_tags || needs.goreleaser.outputs.std_tags || format('{0}/tyk-ee:master', steps.ecr.outputs.registry) }}
          github_token: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
      - name: Choose test code branch
        uses: TykTechnologies/github-actions/.github/actions/tests/choose-test-branch@d3fa20888fa2878e877e22bb7702141217290e7c # main
        with:
          test_folder: api
          org_gh_token: ${{ secrets.ORG_GH_TOKEN }}
      - name: Run API tests
        uses: TykTechnologies/github-actions/.github/actions/tests/api-tests@d3fa20888fa2878e877e22bb7702141217290e7c # main
        timeout-minutes: 30
        id: test_execution
        with:
          user_api_secret: ${{ steps.env_up.outputs.USER_API_SECRET }}
      - name: Generate test reports and collect logs
        uses: TykTechnologies/github-actions/.github/actions/tests/reporting@d3fa20888fa2878e877e22bb7702141217290e7c # main
        if: always() && (steps.test_execution.conclusion != 'skipped')
        with:
          report_xml: 'true'
          execution_status: ${{ steps.test_execution.outcome }}
  aggregator-ci-test:
    name: Aggregated CI Status
    runs-on: warp-ubuntu-latest-x64-2x
    # Dynamically determine which jobs to depend on based on repository configuration
    needs: [goreleaser, api-tests]
    if: ${{ always() && github.event_name == 'pull_request' }}
    steps:
      - name: Aggregate results
        run: |
          failed=()
          # Get the needs context as JSON once
          needs_json='${{ toJSON(needs) }}'

          # Loop through all jobs in the needs context
          for job in $(echo "$needs_json" | jq -r 'keys[]'); do
            job_result=$(echo "$needs_json" | jq -r --arg job "$job" '.[$job].result')

            if [[ "$job_result" != "success" ]]; then
              failed+=("$job")
            fi
          done

          if (( ${#failed[@]} )); then
            # Join the failed job names with commas
            failed_jobs=$(IFS=", "; echo "${failed[*]}")
            echo "Failed jobs ----- : $failed_jobs"
            exit 1
          fi

          echo "All required jobs succeeded"
  test-controller-distros:
    if: github.event.pull_request.draft == false
    needs:
      - goreleaser
    runs-on: warp-ubuntu-latest-x64-2x
    outputs:
      deb: ${{ steps.params.outputs.deb }}
      rpm: ${{ steps.params.outputs.rpm }}
    steps:
      - name: set params
        id: params
        shell: bash
        env:
          # startsWith covers pull_request_target too
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
        run: |
          set -eo pipefail
          # WARNING: curl piped to shell - consider downloading and verifying before executing
          curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk-pump/$BASE_REF/${{ github.event_name}}/api/Distros.gho" | tee -a "$GITHUB_OUTPUT"
          if ! [[ $VARIATION =~ prod ]];then
            echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using test variation"
          fi
  upgrade-deb:
    services:
      httpbin.org:
        image: kennethreitz/httpbin
    runs-on: warp-ubuntu-latest-x64-2x
    needs:
      - test-controller-distros
    strategy:
      fail-fast: true
      matrix:
        arch:
          - amd64
          - arm64
        distro: ${{ fromJson(needs.test-controller-distros.outputs.deb) }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 1
          sparse-checkout: ci
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: deb
      - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
      - name: generate dockerfile
        run: |
          echo 'FROM ${{ matrix.distro }}
          ARG TARGETARCH
          COPY tyk-pump*_${TARGETARCH}.deb /tyk-pump.deb
          RUN apt-get update && apt-get install -y curl
          RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.deb.sh | bash || echo "Repository setup failed, but continuing" # WARNING: curl piped to shell - pinning not possible for packagecloud install script
          RUN apt-get install -y tyk-pump=1.6.0 || echo "Previous version not found, testing fresh install"
          RUN dpkg -i /tyk-pump.deb

          ' | tee Dockerfile
      - name: install on ${{ matrix.distro }}
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: "."
          platforms: linux/${{ matrix.arch }}
          build-args: |
            RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          file: Dockerfile
          push: false
  upgrade-rpm:
    services:
      httpbin.org:
        image: kennethreitz/httpbin
    runs-on: warp-ubuntu-latest-x64-2x
    needs:
      - test-controller-distros
    strategy:
      fail-fast: true
      matrix:
        arch:
          - amd64
          - arm64
        distro: ${{ fromJson(needs.test-controller-distros.outputs.rpm) }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 1
          sparse-checkout: ci
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: rpm
      - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
      - name: generate dockerfile
        run: |
          echo 'FROM ${{ matrix.distro }}
          ARG RHELARCH
          COPY tyk-pump*.${RHELARCH}.rpm /tyk-pump.rpm
          RUN command -v curl || yum install -y curl
          RUN command -v useradd || yum install -y shadow-utils
          RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash || echo "Repository setup failed, but continuing" # WARNING: curl piped to shell - pinning not possible for packagecloud install script
          RUN yum install -y tyk-pump-1.6.0-1 || echo "Previous version not found, testing fresh install"
          RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o tyk-pump.key && rpm --import tyk-pump.key
          RUN rpm --checksig /tyk-pump.rpm
          RUN rpm -Uvh --force /tyk-pump.rpm

          ' | tee Dockerfile
      - name: install on ${{ matrix.distro }}
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
        with:
          context: "."
          platforms: linux/${{ matrix.arch }}
          build-args: |
            RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          file: Dockerfile
          push: false