system-tests.yml

name: System Tests

on:
  workflow_call: # allows to reuse this workflow
    inputs:
      branch_ref:
        description: 'The branch to run the workflow on'
        required: true
        type: string
  push:
    branches:
      - release-v*
      - mq-working-branch-**
    tags-ignore:
      - 'contrib/**'
      - 'instrumentation/**'
      - 'internal/**'
      - 'orchestrion/**'
      - 'scripts/**'
  pull_request:
  workflow_dispatch:
    inputs:
        ref:
            description: 'System Tests ref/tag/branch'
            required: true
            default: main
            type: string
  schedule:
    - cron:  '00 04 * * *'

permissions:
  id-token: write
  contents: read
  packages: write

jobs:
  warm-repo-cache:
    runs-on: ubuntu-latest
    outputs:
      sha: ${{ steps.pin.outputs.sha }}
    steps:
      - name: Checkout system-tests
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          repository: 'DataDog/system-tests'
          ref: ${{ inputs.ref }}
      - name: Pin exact commit SHA for system-tests
        id: pin
        run: |
          echo "sha=$(git rev-parse FETCH_HEAD)" >> "$GITHUB_OUTPUT"
      - name: Cache
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .git
          key: gitdb-system-tests-${{ steps.pin.outputs.sha }}
  build-weblog-images:
    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go')
    runs-on: ubuntu-latest
    needs:
      - warm-repo-cache
    strategy:
      fail-fast: false
      matrix:
        weblog-variant:
          - net-http
          - echo
          - chi
          - gin
          - uds-echo
          - graph-gophers
          - graphql-go
          - gqlgen
          - net-http-orchestrion
    name: Build weblog (${{ matrix.weblog-variant }})
    steps:
      - name: Restore repo cache
        id: restore-repo-cache
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .git
          key: gitdb-system-tests-${{ needs.warm-repo-cache.outputs.sha }}

      - name: Checkout system tests (using cache)
        if: steps.restore-repo-cache.outcome == 'success' && steps.restore-repo-cache.outputs.cache-hit == 'true'
        shell: bash
        run: |
          git config safe.directory "$GITHUB_WORKSPACE"
          git checkout -f ${{ needs.warm-repo-cache.outputs.sha }}

      # Fall back in case of a cache miss, which can occur when retrying old runs.
      - name: Checkout system tests (cache miss)
        if: steps.restore-repo-cache.outcome == 'failure' || steps.restore-repo-cache.outputs.cache-hit != 'true'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          repository: 'DataDog/system-tests'
          ref: ${{ inputs.ref }}

      - name: Checkout dd-trace-go
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          ref: ${{ inputs.branch_ref || github.ref }}
          path: 'binaries/dd-trace-go'

      - name: Build and save weblog image
        run: ./build.sh golang -i weblog -w ${{ matrix.weblog-variant }} --save-to-binaries

      - name: Upload weblog image artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: weblog-image-${{ matrix.weblog-variant }}
          path: binaries/golang-${{ matrix.weblog-variant }}-weblog.tar.zst
          retention-days: 1

  system-tests:
    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go')
    # Note: Not using large runners because the jobs spawned by this pipeline
    # don't seem to get a noticable speedup from using larger runners.
    runs-on: ubuntu-latest
    needs:
      - warm-repo-cache
      - build-weblog-images
      - build-service-extensions-callout
      - build-haproxy
    strategy:
      matrix:
        weblog-variant:
          - net-http
          - echo
          - chi
          - gin
          - uds-echo
        scenario:
          - DEFAULT
          - INTEGRATIONS
          - CROSSED_TRACING_LIBRARIES
          - APPSEC_DISABLED
          - APPSEC_BLOCKING
          - APPSEC_BLOCKING_FULL_DENYLIST
          - APPSEC_API_SECURITY
          - APPSEC_RASP
          - APPSEC_RASP_NON_BLOCKING
          - APPSEC_RUNTIME_ACTIVATION
          - APM_TRACING_E2E_SINGLE_SPAN
          - APM_TRACING_E2E_OTEL
          - TRACING_CONFIG_SCENARIOS
        include:
          - weblog-variant: net-http
            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
          - weblog-variant: net-http
            scenario: REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING
          - weblog-variant: net-http
            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
          # AppSec scenarios that don't depend on the integrations, so we just run on the net/http variant
          - weblog-variant: net-http
            scenario: APPSEC_RATE_LIMITER
          - weblog-variant: net-http
            scenario: APPSEC_CUSTOM_RULES
          - weblog-variant: net-http
            scenario: APPSEC_MISSING_RULES
          - weblog-variant: net-http
            scenario: APPSEC_CORRUPTED_RULES
          - weblog-variant: net-http
            scenario: APPSEC_LOW_WAF_TIMEOUT
          - weblog-variant: net-http
            scenario: APPSEC_STANDALONE
          - weblog-variant: net-http
            scenario: APPSEC_STANDALONE_V2
          - weblog-variant: net-http
            scenario: APPSEC_META_STRUCT_DISABLED
          - weblog-variant: net-http
            scenario: APPSEC_CUSTOM_OBFUSCATION
          - weblog-variant: net-http
            scenario: APPSEC_WAF_TELEMETRY
          # APM scenarios requiring specific environment settings
          - scenario: APM_TRACING_E2E_SINGLE_SPAN
            env:
              DD_API_KEY=$DD_API_KEY
              DD_APPLICATION_KEY=$DD_APP_KEY
              DD_SITE="datadoghq.com"
          - scenario: APM_TRACING_E2E_OTEL
            env:
              DD_API_KEY=$DD_API_KEY
              DD_APPLICATION_KEY=$DD_APP_KEY
              DD_SITE="datadoghq.com"
          # GraphQL System Tests
          - scenario: GRAPHQL_APPSEC
            weblog-variant: graph-gophers
          - scenario: GRAPHQL_APPSEC
            weblog-variant: graphql-go
          - scenario: GRAPHQL_APPSEC
            weblog-variant: gqlgen
          # Telemetry
          - weblog-variant: net-http
            scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED
          - weblog-variant: net-http
            scenario: TELEMETRY_LOG_GENERATION_DISABLED
          - weblog-variant: net-http
            # TELEMETRY_METRIC_GENERATION_ENABLED scenario merged into AGENT_SUPPORTING_SPAN_EVENTS
            scenario: AGENT_SUPPORTING_SPAN_EVENTS
          - weblog-variant: net-http
            scenario: TELEMETRY_METRIC_GENERATION_DISABLED
          - weblog-variant: net-http
            scenario: APM_TRACING_EFFICIENT_PAYLOAD
          - weblog-variant: net-http
            scenario: FEATURE_FLAGGING_AND_EXPERIMENTATION
          # Client-side stats — run across all weblogs, plus the client_drop_p0s variant on net-http
          - weblog-variant: net-http
            scenario: TRACE_STATS_COMPUTATION
          - weblog-variant: echo
            scenario: TRACE_STATS_COMPUTATION
          - weblog-variant: chi
            scenario: TRACE_STATS_COMPUTATION
          - weblog-variant: gin
            scenario: TRACE_STATS_COMPUTATION
          - weblog-variant: uds-echo
            scenario: TRACE_STATS_COMPUTATION
          - weblog-variant: net-http
            scenario: TRACE_STATS_COMPUTATION_CLIENT_DROP_P0S_FALSE
          # RASP scenarios run on net-http-orchestrion because LFI and CMDi detection require orchestrion
          - weblog-variant: net-http-orchestrion
            scenario: APPSEC_RASP
          - weblog-variant: net-http-orchestrion
            scenario: APPSEC_RASP_NON_BLOCKING
          - weblog-variant: net-http-orchestrion
            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
          - weblog-variant: envoy
            scenario: DEFAULT
            image_artifact: service-extensions-image-linux-amd64
            image_name: service-extensions-callout
          - weblog-variant: envoy
            scenario: APPSEC_BLOCKING
            image_artifact: service-extensions-image-linux-amd64
            image_name: service-extensions-callout
          - weblog-variant: haproxy
            scenario: DEFAULT
            image_artifact: haproxy-spoa-image-linux-amd64
            image_name: haproxy-spoa
          - weblog-variant: haproxy
            scenario: APPSEC_BLOCKING
            image_artifact: haproxy-spoa-image-linux-amd64
            image_name: haproxy-spoa
      fail-fast: false
    env:
      SYSTEM_TESTS_WEBLOG: ${{ matrix.weblog-variant }}
    name: Test (${{ matrix.weblog-variant }}, ${{ matrix.scenario }})
    steps:
      - name: Restore repo cache
        id: restore-repo-cache
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .git
          key: gitdb-system-tests-${{ needs.warm-repo-cache.outputs.sha }}

      - name: Checkout system tests (using cache)
        if: steps.restore-repo-cache.outcome == 'success' && steps.restore-repo-cache.outputs.cache-hit == 'true'
        shell: bash
        run: |
          git config safe.directory "$GITHUB_WORKSPACE"
          git checkout -f ${{ needs.warm-repo-cache.outputs.sha }}
      
      # Fall back in case of a cache miss, which can occur when retrying old runs.
      - name: Checkout system tests (cache miss)
        if: steps.restore-repo-cache.outcome == 'failure' || steps.restore-repo-cache.outputs.cache-hit != 'true'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          repository: 'DataDog/system-tests'
          ref: ${{ inputs.ref }}

      - name: Checkout dd-trace-go
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          ref: ${{ inputs.branch_ref || github.ref }}
          path: 'binaries/dd-trace-go'
      
      # Use the app key policy to obtain DD_APP_KEY
      - name: Get Datadog credentials
        id: dd-sts
        uses: DataDog/dd-sts-action@639d841c72f15e4e77747bd726ef8105ce971da2
        with:
          policy: dd-trace-go-app-key
          
      - name: Set credentials
        run: |
          echo "DD_API_KEY=${{ steps.dd-sts.outputs.api_key }}" >> "$GITHUB_ENV"
          echo "DD_APP_KEY=${{ steps.dd-sts.outputs.app_key }}" >> "$GITHUB_ENV"

      - name: Cache npm packages for datadog-ci
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.npm
          key: npm-datadog-ci-${{ runner.os }}-v1
          restore-keys: |
            npm-datadog-ci-${{ runner.os }}-

      # Pre-install datadog-ci early so the action's npm install is instant.
      # continue-on-error: the action has its own retry; this is best-effort.
      - name: Pre-install datadog-ci
        continue-on-error: true
        run: npm install -g @datadog/datadog-ci

      - name: Download pre-built weblog image
        if: ${{ matrix.weblog-variant != 'envoy' && matrix.weblog-variant != 'haproxy' }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: weblog-image-${{ matrix.weblog-variant }}
          path: binaries/

      - name: Display the content of binaries folder
        run: ls -la binaries

      - name: Build weblog
        if: ${{ matrix.weblog-variant != 'envoy' && matrix.weblog-variant != 'haproxy' }}
        run: ./build.sh golang -i weblog -w ${{ matrix.weblog-variant }}

      - name: Download ${{ matrix.weblog-variant }} artifacts
        if: ${{ github.ref != 'refs/heads/main'  && (matrix.weblog-variant == 'envoy' || matrix.weblog-variant == 'haproxy') }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ${{ matrix.image_artifact }}

      - name: Load ${{ matrix.weblog-variant }} image
        if: ${{ github.ref != 'refs/heads/main'  && (matrix.weblog-variant == 'envoy' || matrix.weblog-variant == 'haproxy') }}
        run: docker load -i ${{ matrix.image_artifact }}.tar
      
      - name: Set ${{ matrix.image_name }} image name
        if: ${{ matrix.weblog-variant == 'envoy' || matrix.weblog-variant == 'haproxy' }}
        run: echo "ghcr.io/datadog/dd-trace-go/${{ matrix.image_name }}:dev" > binaries/golang-${{ matrix.image_name }}-image

      - name: Build runner
        uses: ./.github/actions/install_runner

      - name: Run
        shell: bash
        run: |
          env ${{ matrix.env }} ./run.sh ${{ matrix.scenario }}

      - name: Upload test results to Test Optimization
        if: ${{ always() }}
        uses: DataDog/system-tests/.github/actions/push_to_test_optim@6fa83580dfdb27e6932c7498d0dbda7b9363ed62

      - name: Compress artifact
        if: ${{ always() }}
        run: |
          if compgen -G "logs*/" > /dev/null; then
            tar -czvf artifact.tar.gz logs*/
          else
            echo "No logs*/ found; skipping tar."
          fi

      - name: Upload artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: ${{ always() }}
        with:
          name: logs_${{ matrix.weblog-variant }}_${{ matrix.scenario }}
          path: artifact.tar.gz

  # Pushing an image tagged with "dev" only on commit to main,
  # otherwise build and use the image from the artifact.
  build-service-extensions-callout:
    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go')
    uses: ./.github/workflows/docker-build-and-push.yml
    with:
      image: ghcr.io/datadog/dd-trace-go/service-extensions-callout
      dockerfile: ./contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile
      artifact_prefix: service-extensions-image
      commit_sha: ${{ github.sha }}
      tags: >-
        dev
      push: ${{ github.ref == 'refs/heads/main' }}
      platforms: '["linux/amd64"]'

  # Pushing an image tagged with "dev" only on commit to main,
  # otherwise build and use the image from the artifact.
  build-haproxy:
    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go')
    uses: ./.github/workflows/docker-build-and-push.yml
    with:
      image: ghcr.io/datadog/dd-trace-go/haproxy-spoa
      dockerfile: ./contrib/haproxy/stream-processing-offload/cmd/spoa/Dockerfile
      artifact_prefix: haproxy-spoa-image
      commit_sha: ${{ github.sha }}
      tags: >-
        dev
      push: ${{ github.ref == 'refs/heads/main' }}
      platforms: '["linux/amd64"]'

  # Pushing an image tagged with "dev" only on commit to main,
  # otherwise build and use the image from the artifact.
  build-apim-callout:
    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go')
    uses: ./.github/workflows/docker-build-and-push.yml
    with:
      image: ghcr.io/datadog/dd-trace-go/apim-callout
      dockerfile: ./contrib/azure/apim-callout/cmd/apim-callout/Dockerfile
      artifact_prefix: apim-callout-image
      commit_sha: ${{ github.sha }}
      tags: >-
        dev
      push: ${{ github.ref == 'refs/heads/main' }}
      platforms: '["linux/amd64"]'

  tracer-release:
    if: github.event_name == 'schedule'
    uses: DataDog/system-tests/.github/workflows/system-tests.yml@main
    permissions:
      contents: read
      id-token: write
    with:
      library: golang
      scenarios_groups: tracer-release
      _system_tests_dev_mode: true
      push_to_test_optimization: true
      desired_execution_time: 300

  system-tests-done:
    name: System Tests
    needs:
      - build-weblog-images
      - system-tests
      - tracer-release
    runs-on: ubuntu-latest
    if: '!cancelled()'
    steps:
      - name: Success
        if: |
          needs.build-weblog-images.result == 'success'
          && needs.system-tests.result == 'success'
          && (needs.tracer-release.result == 'success' || needs.tracer-release.result == 'skipped')
        run: echo "Success!"
      - name: Failure
        if: |
          needs.build-weblog-images.result != 'success'
          || needs.system-tests.result != 'success'
          || (needs.tracer-release.result != 'success' && needs.tracer-release.result != 'skipped')
        run: echo "Failure!" && exit 1