Try using snyk container test instead of older approach #678
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This is the main build pipeline that verifies and tags the software | |
| name: CICD Pipeline | |
| on: | |
| # Triggers the workflow on push events | |
| push: | |
| branches: | |
| - main | |
| - develop | |
| - 'release/**' | |
| - 'feature/**' | |
| - 'issue/**' | |
| - 'issues/**' | |
| - 'dependabot/**' | |
| tags-ignore: | |
| - '*' | |
| # Do not trigger build if pyproject.toml was the only thing changed | |
| paths-ignore: | |
| - 'pyproject.toml' | |
| - 'poetry.lock' | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| venue: | |
| type: choice | |
| description: Venue to deploy to | |
| options: | |
| - SIT | |
| - UAT | |
| env: | |
| POETRY_VERSION: "2.1.3" | |
| PYTHON_VERSION: "3.12" | |
| TERRAFORM_VERSION: "1.14.0" | |
| REGISTRY: ghcr.io | |
| jobs: | |
| build: | |
| name: Build python and tf module | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| outputs: | |
| deploy_env: ${{ steps.poetry-build.outputs.deploy_env }} | |
| version: ${{ steps.poetry-build.outputs.the_version }} | |
| pyproject_name: ${{ steps.poetry-build.outputs.pyproject_name }} | |
| tf_module_artifact_name: ${{ steps.poetry-build.outputs.tf_module_artifact_name }} | |
| steps: | |
| - uses: getsentry/action-github-app-token@v4.0.0 | |
| name: CICD Token | |
| id: cicd-app | |
| with: | |
| app_id: ${{ secrets.CICD_APP_ID }} | |
| private_key: ${{ secrets.CICD_APP_PRIVATE_KEY }} | |
| - uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ github.repository }} | |
| token: ${{ steps.cicd-app.outputs.token }} | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Poetry | |
| uses: abatilo/actions-poetry@v3 | |
| with: | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| - name: Configure Poetry to install into the active environment | |
| # GDAL (libgdal.so) is provided by the conda/micromamba environment, so we must | |
| # NOT create a separate Poetry venv. Installing into the active micromamba env keeps | |
| # the osgeo python bindings and libgdal.so co-located on the same loader path. | |
| run: | | |
| poetry config virtualenvs.create false --local | |
| - name: Get pre-build version | |
| id: get-version | |
| run: | | |
| echo "current_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT | |
| echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV | |
| - name: Manual Build | |
| # If triggered by workflow dispatch, no version bump | |
| if: github.event_name == 'workflow_dispatch' | |
| id: manual | |
| run: | | |
| echo "TARGET_ENV_UPPERCASE=${{ github.event.inputs.venue }}" >> $GITHUB_ENV | |
| - name: Bump pre-alpha version | |
| # If triggered by push to a non-tracked branch | |
| if: | | |
| github.ref != 'refs/heads/develop' && | |
| github.ref != 'refs/heads/main' | |
| run: | | |
| new_ver="${{ steps.get-version.outputs.current_version }}+$(git rev-parse --short ${GITHUB_SHA})" | |
| poetry version $new_ver | |
| echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV | |
| - name: Bump alpha version | |
| # If triggered by push to the develop branch | |
| if: | | |
| github.ref == 'refs/heads/develop' && | |
| steps.manual.conclusion == 'skipped' | |
| id: alpha | |
| run: | | |
| poetry version prerelease | |
| echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV | |
| - name: Bump rc version | |
| # If triggered by push to a release branch | |
| if: | | |
| startsWith(github.ref, 'refs/heads/release/') && | |
| steps.manual.conclusion == 'skipped' | |
| id: rc | |
| env: | |
| # True if the version already has a 'rc' pre-release identifier | |
| BUMP_RC: ${{ contains(steps.get-version.outputs.current_version, 'rc') }} | |
| run: | | |
| if [ "$BUMP_RC" = true ]; then | |
| poetry version prerelease | |
| else | |
| poetry version ${GITHUB_REF#refs/heads/release/}rc1 | |
| fi | |
| echo "TARGET_ENV_UPPERCASE=UAT" >> $GITHUB_ENV | |
| - name: Release version | |
| # If triggered by push to the main branch | |
| if: | | |
| startsWith(github.ref, 'refs/heads/main') && | |
| steps.manual.conclusion == 'skipped' | |
| id: release | |
| env: | |
| CURRENT_VERSION: ${{ steps.get-version.outputs.current_version }} | |
| # Remove rc* from end of version string | |
| # The ${string%%substring} syntax below deletes the longest match of $substring from back of $string. | |
| run: | | |
| poetry version ${CURRENT_VERSION%%rc*} | |
| echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV | |
| echo "venue=ops" >> $GITHUB_ENV | |
| echo "TARGET_ENV_UPPERCASE=OPS" >> $GITHUB_ENV | |
| - name: Get install version | |
| # Get the version of the software being installed and save it as an ENV var | |
| run: | | |
| echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV | |
| - name: Install micromamba | |
| uses: mamba-org/setup-micromamba@v3 | |
| with: | |
| environment-file: conda-environment.yaml | |
| cache-environment: true | |
| - name: Install package | |
| run: poetry install | |
| - name: Lint | |
| run: | | |
| poetry run pylint bignbit | |
| poetry run flake8 bignbit | |
| - name: Test and coverage | |
| run: | | |
| poetry run pytest --junitxml=build/reports/pytest.xml --cov=bignbit --cov-report=xml:build/reports/coverage.xml tests/ | |
| - uses: hashicorp/setup-terraform@v4 | |
| with: | |
| terraform_version: ${{ env.TERRAFORM_VERSION }} | |
| terraform_wrapper: false | |
| - name: Validate Terraform | |
| working-directory: terraform | |
| run: | | |
| terraform init -backend=false -upgrade | |
| terraform validate -no-color | |
| - name: SonarCloud Scan | |
| uses: sonarsource/sonarqube-scan-action@v8.2.0 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| with: | |
| args: > | |
| -Dsonar.organization=${{ github.repository_owner }} | |
| -Dsonar.projectKey=${{ github.repository_owner }}_${{ github.event.repository.name }} | |
| -Dsonar.python.coverage.reportPaths=build/reports/coverage.xml | |
| -Dsonar.sources=bignbit/ | |
| -Dsonar.tests=tests/ | |
| -Dsonar.projectName=${{ github.repository }} | |
| -Dsonar.projectVersion=${{ env.software_version }} | |
| -Dsonar.python.version=3.12 | |
| - name: Run Snyk as a blocking step | |
| uses: snyk/actions/python-3.12@master | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| with: | |
| command: test | |
| args: > | |
| --org=${{ secrets.SNYK_ORG_ID }} | |
| --project-name=${{ github.repository }} | |
| --severity-threshold=high | |
| --fail-on=all | |
| - name: Run Snyk on Python | |
| uses: snyk/actions/python-3.12@master | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| with: | |
| command: monitor | |
| args: > | |
| --org=${{ secrets.SNYK_ORG_ID }} | |
| --project-name=${{ github.repository }} | |
| - name: Build Python Artifact | |
| id: poetry-build | |
| run: | | |
| poetry build | |
| echo "deploy_env=${{ env.TARGET_ENV_UPPERCASE }}" >> $GITHUB_OUTPUT | |
| echo "the_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT | |
| echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_OUTPUT | |
| echo "tf_module_artifact_name=$(poetry version | awk '{print $1}')-${{ env.software_version }}-cumulus-tf" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ steps.poetry-build.outputs.pyproject_name }}-dist | |
| path: dist/* | |
| - name: Zip artifact for deployment | |
| run: | | |
| mkdir -p build/${{ steps.poetry-build.outputs.tf_module_artifact_name }} | |
| cp terraform/* build/${{ steps.poetry-build.outputs.tf_module_artifact_name }}/ | |
| cd build | |
| zip ${{ steps.poetry-build.outputs.tf_module_artifact_name }}.zip ${{ steps.poetry-build.outputs.tf_module_artifact_name }}/* -r -j | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ steps.poetry-build.outputs.tf_module_artifact_name }} | |
| path: build/${{ steps.poetry-build.outputs.tf_module_artifact_name }}.zip | |
| - name: Commit Version Bump | |
| # If building an alpha, release candidate, or release then we commit the version bump back to the repo | |
| if: | | |
| steps.alpha.conclusion == 'success' || | |
| steps.rc.conclusion == 'success' || | |
| steps.release.conclusion == 'success' | |
| run: | | |
| git config user.name "${GITHUB_ACTOR}" | |
| git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| git commit -am "/version ${{ env.software_version }}" | |
| git push | |
| - name: Push Tag | |
| if: | | |
| steps.alpha.conclusion == 'success' || | |
| steps.rc.conclusion == 'success' || | |
| steps.release.conclusion == 'success' | |
| run: | | |
| git config user.name "${GITHUB_ACTOR}" | |
| git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| git tag -a "${{ env.software_version }}" -m "Version ${{ env.software_version }}" | |
| git push origin "${{ env.software_version }}" | |
| - name: Create GH release | |
| if: | | |
| steps.alpha.conclusion == 'success' || | |
| steps.rc.conclusion == 'success' || | |
| steps.release.conclusion == 'success' | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| artifacts: build/${{ steps.poetry-build.outputs.tf_module_artifact_name }}.zip | |
| generateReleaseNotes: true | |
| name: ${{ env.software_version }} | |
| prerelease: ${{ steps.alpha.conclusion == 'success' || steps.rc.conclusion == 'success'}} | |
| tag: ${{ env.software_version }} | |
| docker: | |
| name: Build & Publish Docker Image | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| security-events: write | |
| needs: build | |
| outputs: | |
| container_image_uri: ${{ steps.set-outputs.outputs.container_image_uri }} | |
| env: | |
| THE_VERSION: ${{ needs.build.outputs.version }} | |
| PYPROJECT_NAME: ${{ needs.build.outputs.pyproject_name }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ github.repository }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v6 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ github.repository }}/${{env.PYPROJECT_NAME}} | |
| tags: | | |
| type=pep440,pattern={{version}},value=${{ env.THE_VERSION }} | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: docker/Dockerfile | |
| push: true | |
| pull: true | |
| platforms: linux/amd64 | |
| provenance: false | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Set up Snyk CLI | |
| uses: snyk/actions/setup@master | |
| - name: Run Snyk on Docker Image | |
| # Use the modern `snyk container test` subcommand directly instead of the | |
| # snyk/actions/docker action. That action runs legacy `snyk test --docker`, | |
| # which (a) executes inside a container with no `python`, tripping over the | |
| # repo's pyproject.toml, and (b) mishandles `--file`, treating the Dockerfile | |
| # as an open-source manifest ("Could not detect package manager"). | |
| # `snyk container test` targets only the image, reads the Dockerfile via | |
| # --file for base-image upgrade advice, and reliably writes SARIF for upload | |
| # to GitHub Code Scanning. continue-on-error keeps a vulnerable image (or a | |
| # scan error) from failing the build. | |
| continue-on-error: true | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| IMAGE_URI: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} | |
| run: | | |
| snyk container test "$IMAGE_URI" \ | |
| --file=docker/Dockerfile \ | |
| --severity-threshold=high \ | |
| --sarif-file-output=snyk.sarif | |
| - name: Upload Snyk result to GitHub Code Scanning | |
| # Always attempt to upload, even if the Snyk scan found vulnerabilities and | |
| # exited non-zero. Guarded on the SARIF file existing so that a scan which | |
| # fails to produce output does not fail the whole job. | |
| if: ${{ always() && hashFiles('snyk.sarif') != '' }} | |
| uses: github/codeql-action/upload-sarif@v4 | |
| with: | |
| sarif_file: snyk.sarif | |
| - name: Set output | |
| id: set-outputs | |
| run: | | |
| echo "container_image_uri=${{ fromJSON(steps.meta.outputs.json).tags[0] }}" >> $GITHUB_OUTPUT | |
| deploy: | |
| name: Deploy | |
| needs: [build, docker] | |
| runs-on: ubuntu-latest | |
| environment: ${{ needs.build.outputs.deploy_env }} | |
| env: | |
| THE_VERSION: ${{ needs.build.outputs.version }} | |
| CONTAINER_IMAGE_URI: ${{ needs.docker.outputs.container_image_uri }} | |
| TF_MODULE_ARTIFACT_NAME: ${{ needs.build.outputs.tf_module_artifact_name }} | |
| TF_VAR_gibs_account_id: ${{ secrets.GIBS_ACCOUNT_ID }} | |
| if: | | |
| github.ref == 'refs/heads/develop' || | |
| startsWith(github.ref, 'refs/heads/release') || | |
| github.event_name == 'workflow_dispatch' | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ github.repository }} | |
| - uses: hashicorp/setup-terraform@v4 | |
| with: | |
| terraform_version: ${{ env.TERRAFORM_VERSION }} | |
| terraform_wrapper: false | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v6.1.0 | |
| with: | |
| aws-region: us-west-2 | |
| role-session-name: GitHubActions | |
| aws-access-key-id: ${{ secrets[vars.AWS_ACCESS_KEY_ID_SECRET_NAME] }} | |
| aws-secret-access-key: ${{ secrets[vars.AWS_SECRET_ACCESS_KEY_SECRET_NAME] }} | |
| mask-aws-account-id: true | |
| - name: Validate Terraform | |
| working-directory: examples/cumulus-tf | |
| run: | | |
| terraform init -backend=false -upgrade | |
| terraform validate -no-color | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Deploy to venue | |
| id: terraform-deploy | |
| working-directory: examples/cumulus-tf | |
| env: | |
| AWS_DEFAULT_REGION: us-west-2 | |
| run: | | |
| ./bin/deploy.sh --app-version ${{ env.THE_VERSION }} --tf-venue ${{ vars.TF_VENUE }} --lambda_container_image_uri ${{ env.CONTAINER_IMAGE_URI }} |