test-run-enos-scenario-matrix.yml

---
name: enos

on:
  # Only trigger this working using workflow_call. This workflow requires many
  # secrets that must be inherited from the caller workflow.
  workflow_call:
    inputs:
      # The name of the artifact that we're going to use for testing. This should
      # match exactly to build artifacts uploaded to Github and Artifactory.
      build-artifact-name:
        required: true
        type: string
      # The maximum number of scenarios to include in the test sample.
      sample-max:
        default: 1
        type: number
      # The name of the enos scenario sample that defines compatible scenarios we can
      # can test with.
      sample-name:
        required: true
        type: string
      runs-on:
        # NOTE: The value should be JSON encoded as that's the only way we can
        # pass arrays with workflow_call.
        type: string
        required: false
        default: '"ubuntu-latest"'
      ssh-key-name:
        type: string
        default: ${{ github.event.repository.name }}-ci-ssh-key
      vault-edition:
        required: false
        type: string
        default: ce
      # The Git commit SHA used as the revision when building vault
      vault-revision:
        required: true
        type: string
      vault-version:
        required: true
        type: string

jobs:
  metadata:
    runs-on: ${{ fromJSON(inputs.runs-on) }}
    permissions:
      id-token: write # vault-auth
      contents: read
    outputs:
      build-date: ${{ steps.metadata.outputs.build-date }}
      is-enterprise: ${{ steps.metadata.outputs.is-enterprise }}
      sample: ${{ steps.metadata.outputs.sample }}
      vault-version: ${{ steps.metadata.outputs.vault-version }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ inputs.vault-revision }}
      - if: inputs.vault-edition != 'ce'
        id: vault-auth
        name: Vault Authenticate
        run: vault-auth
      - if: inputs.vault-edition != 'ce'
        id: vault-secrets
        name: Fetch Vault Secrets
        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
        with:
          url: ${{ steps.vault-auth.outputs.addr }}
          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
          token: ${{ steps.vault-auth.outputs.token }}
          secrets: |
            kv/data/github/${{ github.repository }}/github-token token | ELEVATED_GITHUB_TOKEN;
      - uses: hashicorp/action-setup-enos@b9fa53484a1e8fdcc7b02a118bcf01d65b9414c9 # v1.37.0
        with:
          github-token: ${{ inputs.vault-edition == 'ce' && secrets.ELEVATED_GITHUB_TOKEN || steps.vault-secrets.outputs.ELEVATED_GITHUB_TOKEN }}
      - uses: ./.github/actions/create-dynamic-config
        with:
          github-token: ${{ inputs.vault-edition == 'ce' && secrets.ELEVATED_GITHUB_TOKEN || steps.vault-secrets.outputs.ELEVATED_GITHUB_TOKEN }}
          vault-version: ${{ inputs.vault-version }}
          vault-edition: ${{ inputs.vault-edition }}
      - id: metadata
        run: |
          build_date=$(make ci-get-date)
          sample_seed=$(date +%s)
          if ! sample=$(enos scenario sample observe "${{ inputs.sample-name }}" --chdir ./enos --min 1 --max "${{ inputs.sample-max }}" --seed "${sample_seed}" --format json | jq -c ".observation.elements"); then
            echo "failed to do sample observation: $sample" 2>&1
            exit 1
          fi
          if [[ "${{ inputs.vault-edition }}" == "ce" ]]; then
            vault_version="${{ inputs.vault-version }}"
          else
            # shellcheck disable=2001
            vault_version="$(sed 's/+ent/+${{ inputs.vault-edition }}/g' <<< '${{ inputs.vault-version }}')"
          fi
          sample_seed=$(date +%s)
          sample=$(enos scenario sample observe "${{ inputs.sample-name }}" --chdir ./enos --min 1 --max "${{ inputs.sample-max }}" --seed "${sample_seed}" --format json | jq -c ".observation.elements")
          {
            echo "build-date=${build_date}"
            echo "is-enterprise=${{ inputs.vault-edition != 'ce' }}"
            echo "sample=${sample}"
            echo "sample-seed=${sample_seed}" # This isn't used outside of here but is nice to know for duplicating observations
            echo "vault-version=${vault_version}"
          } | tee -a "$GITHUB_OUTPUT"

  # Run the Enos test scenario(s)
  run:
    needs: metadata
    name: run ${{ matrix.scenario.id.filter }}
    strategy:
      fail-fast: false # don't fail as that can skip required cleanup steps for jobs
      matrix:
        include: ${{ fromJSON(needs.metadata.outputs.sample) }}
    runs-on: ${{ fromJSON(inputs.runs-on) }}
    permissions:
      id-token: write # vault-auth
      contents: read
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ inputs.vault-revision }}
      - if: needs.metadata.outputs.is-enterprise == 'true'
        id: vault-auth
        name: Vault Authenticate
        run: vault-auth
      - if: needs.metadata.outputs.is-enterprise == 'true'
        id: vault-secrets
        name: Fetch Vault Secrets
        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
        with:
          url: ${{ steps.vault-auth.outputs.addr }}
          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
          token: ${{ steps.vault-auth.outputs.token }}
          secrets: |
            kv/data/github/${{ github.repository }}/artifactory token | ARTIFACTORY_TOKEN;
            kv/data/github/${{ github.repository }}/artifactory username | ARTIFACTORY_USER;
            kv/data/github/${{ github.repository }}/aws access-key-id | AWS_ACCESS_KEY_ID_CI;
            kv/data/github/${{ github.repository }}/aws secret-access-key | AWS_SECRET_ACCESS_KEY_CI;
            kv/data/github/${{ github.repository }}/aws role-arn | AWS_ROLE_ARN_CI;
            kv/data/github/${{ github.repository }}/consul license | CONSUL_LICENSE;
            kv/data/github/${{ github.repository }}/vault-radar license | RADAR_LICENSE;
            kv/data/github/${{ github.repository }}/enos slack-webhook-url | SLACK_WEBHOOK_URL;
            kv/data/github/${{ github.repository }}/enos ssh-key | SSH_KEY_PRIVATE_CI;
            kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE;
            kv/data/github/${{ github.repository }}/github-token token | ELEVATED_GITHUB_TOKEN;
      - id: secrets
        run: |
          if [[ "${{ needs.metadata.outputs.is-enterprise }}" != 'true' ]]; then
            {
              echo "artifactory-user=${{ secrets.ARTIFACTORY_USER }}"
              echo "artifactory-token=${{ secrets.ARTIFACTORY_TOKEN }}"
              echo "aws-access-key-id=${{ secrets.AWS_ACCESS_KEY_ID_CI }}"
              echo "aws-secret-access-key=${{ secrets.AWS_SECRET_ACCESS_KEY_CI }}"
              echo "aws-role-arn=${{ secrets.AWS_ROLE_ARN_CI }}"
              echo "consul-license=${{ secrets.CONSUL_LICENSE }}"
              echo "github-token=${{ secrets.ELEVATED_GITHUB_TOKEN }}"
              echo "radar-license=${{ secrets.RADAR_LICENSE }}"
              echo "slack-webhook-url=${{ secrets.SLACK_WEBHOOK_URL }}"
              echo 'ssh-key<<EOFSSHKEYCE'
              echo "${{ secrets.SSH_KEY_PRIVATE_CI }}"
              echo EOFSSHKEYCE
              echo "vault-license=${{ secrets.VAULT_LICENSE }}"
           } | tee -a "$GITHUB_OUTPUT"
          else
            {
              echo "artifactory-user=${{ steps.vault-secrets.outputs.ARTIFACTORY_USER }}"
              echo "artifactory-token=${{ steps.vault-secrets.outputs.ARTIFACTORY_TOKEN }}"
              echo "aws-access-key-id=${{ steps.vault-secrets.outputs.AWS_ACCESS_KEY_ID_CI }}"
              echo "aws-secret-access-key=${{ steps.vault-secrets.outputs.AWS_SECRET_ACCESS_KEY_CI }}"
              echo "aws-role-arn=${{ steps.vault-secrets.outputs.AWS_ROLE_ARN_CI }}"
              echo "consul-license=${{ steps.vault-secrets.outputs.CONSUL_LICENSE }}"
              echo "github-token=${{ steps.vault-secrets.outputs.ELEVATED_GITHUB_TOKEN }}"
              echo "radar-license=${{ steps.vault-secrets.outputs.RADAR_LICENSE }}"
              echo "slack-webhook-url=${{ steps.vault-secrets.outputs.SLACK_WEBHOOK_URL }}"
              echo 'ssh-key<<EOFSSHKEYENT'
              echo "${{ steps.vault-secrets.outputs.SSH_KEY_PRIVATE_CI }}"
              echo EOFSSHKEYENT
              echo "vault-license=${{ steps.vault-secrets.outputs.VAULT_LICENSE }}"
            } | tee -a "$GITHUB_OUTPUT"
          fi
      - id: env
        run: |
          # Configure input environment variables.
          {
            echo "GITHUB_TOKEN=${{ steps.secrets.outputs.github-token }}"
            echo "ENOS_DEBUG_DATA_ROOT_DIR=/tmp/enos-debug-data"
            echo "ENOS_VAR_artifactory_username=${{ steps.secrets.outputs.artifactory-user }}"
            echo "ENOS_VAR_artifactory_token=${{ steps.secrets.outputs.artifactory-token }}"
            echo "ENOS_VAR_aws_region=${{ matrix.attributes.aws_region }}"
            echo "ENOS_VAR_aws_ssh_keypair_name=${{ inputs.ssh-key-name }}"
            echo "ENOS_VAR_aws_ssh_private_key_path=./support/private_key.pem"
            echo "ENOS_VAR_consul_license_path=./support/consul.hclic"
            echo "ENOS_VAR_distro_version_amzn=${{ matrix.attributes.distro_version_amzn }}"
            echo "ENOS_VAR_distro_version_leap=${{ matrix.attributes.distro_version_leap }}"
            echo "ENOS_VAR_distro_version_rhel=${{ matrix.attributes.distro_version_rhel }}"
            echo "ENOS_VAR_distro_version_sles=${{ matrix.attributes.distro_version_sles }}"
            echo "ENOS_VAR_distro_version_ubuntu=${{ matrix.attributes.distro_version_ubuntu }}"
            echo "ENOS_VAR_terraform_plugin_cache_dir=./support/terraform-plugin-cache"
            echo "ENOS_VAR_vault_artifact_path=./support/downloads/${{ inputs.build-artifact-name }}"
            echo "ENOS_VAR_vault_build_date=${{ needs.metadata.outputs.build-date }}"
            echo "ENOS_VAR_vault_license_path=./support/vault.hclic"
            echo "ENOS_VAR_vault_product_version=${{ needs.metadata.outputs.vault-version }}"
            echo "ENOS_VAR_vault_radar_license_path=./support/vault-radar.hclic"
            echo "ENOS_VAR_vault_revision=${{ inputs.vault-revision }}"
            echo "ENOS_VAR_vault_upgrade_initial_version=${{ matrix.attributes.upgrade_initial_version }}"
            echo "ENOS_VAR_verify_aws_secrets_engine=false"
            echo "ENOS_VAR_verify_log_secrets=true"
          } | tee -a "$GITHUB_ENV"
      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
        with:
          # the Terraform wrapper will break Terraform execution in Enos because
          # it changes the output to text when we expect it to be JSON.
          terraform_wrapper: false
      - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
        with:
          aws-access-key-id: ${{ steps.secrets.outputs.aws-access-key-id }}
          aws-secret-access-key: ${{ steps.secrets.outputs.aws-secret-access-key }}
          aws-region: ${{ matrix.attributes.aws_region }}
          role-to-assume: ${{ steps.secrets.outputs.aws-role-arn }}
          role-skip-session-tagging: true
          role-duration-seconds: 3600
      - uses: hashicorp/action-setup-enos@b9fa53484a1e8fdcc7b02a118bcf01d65b9414c9 # v1.37.0
        with:
          github-token: ${{ steps.secrets.outputs.github-token }}
      - uses: ./.github/actions/create-dynamic-config
        with:
          github-token: ${{ steps.secrets.outputs.github-token }}
          vault-version: ${{ inputs.vault-version }}
          vault-edition: ${{ inputs.vault-edition }}
      - name: Prepare scenario dependencies
        id: prepare_scenario
        run: |
          mkdir -p "./enos/support/terraform-plugin-cache"
          echo "${{ steps.secrets.outputs.ssh-key }}" > "./enos/support/private_key.pem"
          chmod 600 "./enos/support/private_key.pem"
          sha256sum "./enos/support/private_key.pem"
          du -h "./enos/support/private_key.pem"
          echo "debug_data_artifact_name=enos-debug-data_$(echo "${{ matrix.scenario }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')" >> "$GITHUB_OUTPUT"
      - if: contains(inputs.sample-name, 'build')
        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
        with:
          name: ${{ inputs.build-artifact-name }}
          path: ./enos/support/downloads
      - if: contains(inputs.sample-name, 'ent')
        name: Configure Vault license
        run: echo "${{ steps.secrets.outputs.vault-license }}" > ./enos/support/vault.hclic || true
      - if: contains(matrix.scenario.id.filter, 'consul_edition:ent')
        name: Configure Consul license
        run: |
          echo "${{ steps.secrets.outputs.consul-license }}" > ./enos/support/consul.hclic || true
      - name: Configure Vault Radar license
        run: |
          echo "${{ steps.secrets.outputs.radar-license }}" > ./enos/support/vault-radar.hclic || true
      - id: launch
        name: enos scenario launch ${{ matrix.scenario.id.filter }}
        # Continue once and retry to handle occasional blips when creating infrastructure.
        continue-on-error: true
        run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
      - if: steps.launch.outcome == 'failure'
        id: launch_retry
        name: Retry enos scenario launch ${{ matrix.scenario.id.filter }}
        run: enos scenario launch --timeout 45m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
      - name: Upload Debug Data
        if: failure()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          # The name of the artifact is the same as the matrix scenario name with the spaces replaced with underscores and colons replaced by equals.
          name: ${{ steps.prepare_scenario.outputs.debug_data_artifact_name }}
          path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }}
          retention-days: 30
        continue-on-error: true
      - if: ${{ always() }}
        id: destroy
        name: enos scenario destroy ${{ matrix.scenario.id.filter }}
        continue-on-error: true
        run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
      - if: steps.destroy.outcome == 'failure'
        id: destroy_retry
        name: Retry enos scenario destroy ${{ matrix.scenario.id.filter }}
        continue-on-error: true
        run: enos scenario destroy --timeout 10m0s --chdir ./enos ${{ matrix.scenario.id.filter }}
      - name: Clean up Enos runtime directories
        id: cleanup
        if: ${{ always() }}
        continue-on-error: true
        run: |
          rm -rf /tmp/enos*
          rm -rf ./enos/support
          rm -rf ./enos/.enos
      # Send slack notifications to #feed-vault-enos-failures any of our enos scenario commands fail.
      # There is an incoming webhook set up on the "Enos Vault Failure Bot" Slackbot:
      # https://api.slack.com/apps/A05E31CH1LG/incoming-webhooks
      - if: ${{ always() && ! cancelled() }}
        name: Notify launch failed
        uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
        with:
          failure-message: "enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
          status: ${{ steps.launch.outcome }}
          slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
      - if: ${{ always() && ! cancelled() }}
        name: Notify retry launch failed
        uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
        with:
          failure-message: "retry enos scenario launch ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
          status: ${{ steps.launch_retry.outcome }}
          slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
      - if: ${{ always() && ! cancelled() }}
        name: Notify destroy failed
        uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
        with:
          failure-message: "enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
          status: ${{ steps.destroy.outcome }}
          slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}
      - if: ${{ always() && ! cancelled() }}
        name: Notify retry destroy failed
        uses: hashicorp/actions-slack-status@1a3f63b30bd476aee1f3bd6f9d8f2aacc4f14d81 # v2.0.1
        with:
          failure-message: "retry enos scenario destroy ${{ matrix.scenario.id.filter}} failed. \nTriggering event: `${{ github.event_name }}` \nActor: `${{ github.actor }}`"
          status: ${{ steps.destroy_retry.outcome }}
          slack-webhook-url: ${{ steps.secrets.outputs.slack-webhook-url }}