python.yml

name: Python

on:
  push:
  pull_request:
  workflow_dispatch:

permissions: {}

jobs:
  detect-packages:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      packages: ${{ steps.find-packages.outputs.packages }}
      changed-directories: ${{ steps.find-changed-directories.outputs.changed-directories }}
      changed-files: ${{ steps.find-changed-directories.outputs.changed-files }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
      - name: Fetch base branch
        run: git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}
      - name: find changed directories
        id: find-changed-directories
        env:
          EVENT_NAME: ${{ github.event_name }}
          EVENT_BEFORE: ${{ github.event.before }}
          BASE_REF: ${{ github.base_ref }}
        run: |
          # Push         against last commit
          # Pull Request against target branch sha
          # Otherwise    against latest release
          if [ "$EVENT_NAME" == "push" ]; then
            # Handle null SHA case for new branches
            if [ "$EVENT_BEFORE" == "0000000000000000000000000000000000000000" ]; then
              SINCE="$(git merge-base HEAD origin/main)"
            else
              SINCE="$EVENT_BEFORE"
            fi
          elif [ "$EVENT_NAME" == "pull_request" ]; then
            SINCE="$BASE_REF"
          else
            SINCE="$(gh release list --exclude-drafts --exclude-pre-releases --limit 1 --json tagName | jq -r '.[].tagName')"
          fi;
          if [ -z "$SINCE" ]; then SINCE="$(git rev-list --max-parents=0 HEAD)"; fi;
          echo "$SINCE"
          CHANGEDFILES="$(git diff --name-only "$SINCE" HEAD | sed 's/^\.\///' | jq -R -s -c 'split("\n")[:-1]')"
          CHANGEDDIRECTORIES="$(echo $CHANGEDFILES | jq -r '.[] | select(. | startswith("src\/"))' | cut -d'/' -f2 | sort -u | sed 's/^\.\///' | jq -R -s -c 'split("\n")[:-1]')"
          echo "$CHANGEDDIRECTORIES"
          echo "$CHANGEDFILES"
          echo "changed-files=$CHANGEDFILES" >> $GITHUB_OUTPUT
          echo "changed-directories=$CHANGEDDIRECTORIES" >> $GITHUB_OUTPUT

      - name: Find Python packages
        id: find-packages
        working-directory: src
        run: |
          PACKAGES=$(find . -name pyproject.toml -exec dirname {} \; | sed 's/^\.\///' | jq -R -s -c 'split("\n")[:-1]')
          echo "packages=$PACKAGES" >> $GITHUB_OUTPUT

  build:
    needs: [detect-packages]
    if: ${{ needs.detect-packages.outputs.packages != '[]' && needs.detect-packages.outputs.packages != '' }}
    strategy:
      fail-fast: false
      matrix:
        package: ${{ fromJson(needs.detect-packages.outputs.packages) }}
    name: Build ${{ matrix.package }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
      security-events: write
      actions: read
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install uv
        uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version-file: "src/${{ matrix.package }}/.python-version"
          # cache: uv (not supported)

      - name: Cache GraphViz
        uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 #v5.0.3
        id: cache-graphviz
        with:
          path: "~/graphviz"
          key: graphviz

      - name: Install Graphviz
        env:
          CACHE_HIT: ${{steps.cache-graphviz.outputs.cache-hit}}
        run: |
          if [[ "$CACHE_HIT" == 'true' ]]; then
            sudo cp --verbose --force --recursive ~/graphviz/* /
          else
            sudo apt-get update && sudo apt-get install -y graphviz
            mkdir -p ~/graphviz
            sudo dpkg -L graphviz | while IFS= read -r f; do if test -f $f; then echo $f; fi; done | xargs cp --parents --target-directory ~/graphviz/
          fi

      - name: Install Bandit
        run: |
          pip install --require-hashes --requirement .github/workflows/bandit-requirements.txt

      - name: Security check - Bandit
        id: bandit-check
        working-directory: src/${{ matrix.package }}
        run: bandit -r --severity-level medium --confidence-level medium -f html -o bandit-report-${{ matrix.package }}.html -c "pyproject.toml" . || echo "status=failure" >> $GITHUB_OUTPUT

      - name: Store Bandit as Artifact
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: bandit-report-${{ matrix.package }}.html
          path: src/${{ matrix.package }}/bandit-report-${{ matrix.package }}.html

      - name: Stop on Bandit failure
        if: steps.bandit-check.outputs.status == 'failure'
        run: exit 1

      - name: Install dependencies
        working-directory: src/${{ matrix.package }}
        run: uv sync --frozen --all-extras --dev

      - name: Verify package name consistency
        run: |
          python3 scripts/verify_package_name.py src/${{ matrix.package }}
          uv run --script scripts/verify_awslabs_init.py src/${{ matrix.package }}
          python3 scripts/verify_tool_names.py src/${{ matrix.package }} || true

      - name: Run tests
        working-directory: src/${{ matrix.package }}
        run: |
          if [ -d "tests" ]; then
            uv run --frozen pytest --cov --cov-branch --cov-report=term-missing --cov-report=xml:${{ matrix.package }}-coverage.xml --junitxml=${{ matrix.package}}-junit.xml -o junit-family=legacy
          else
            echo "No tests directory found, skipping tests"
          fi

      - name: Upload coverage reports to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de #v5.5.2
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ${{ matrix.package }}-coverage.xml
          report_type: "coverage"
          handle_no_reports_found: true

      - name: Upload test reports to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de #v5.5.2
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ${{ matrix.package }}-junit.xml
          report_type: "test_results"
          handle_no_reports_found: true

      - name: Run pyright
        working-directory: src/${{ matrix.package }}
        run: uv run --frozen pyright

      - name: Run ruff format
        working-directory: src/${{ matrix.package }}
        run: uv run --frozen ruff format .

      - name: Run ruff check
        working-directory: src/${{ matrix.package }}
        run: uv run --frozen ruff check .

      - name: Build package
        working-directory: src/${{ matrix.package }}
        run: uv build

      - name: Upload distribution
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: dist-${{ matrix.package }}
          path: src/${{ matrix.package }}/dist/

      - name: Generate Software Bill of Materials (SBOM)
        working-directory: src/${{ matrix.package }}
        run: |
          source .venv/bin/activate
          echo "Attempt to convert to proper UTF-8 files https://github.com/CycloneDX/cyclonedx-python/issues/868"
          find .venv -type f -path '*/*.dist-info/*' > .venv/FILES
          # because grep with xargs returns 123 have to do this the long and hard way...
          while IFS= read -r line; do
            (grep -s -q -axv '.*' $line &&
              if [[ "$(file -b --mime-encoding $line)" != "binary" ]]; then
                echo "illegal utf-8 characters in $line...converting...";
                iconv -f $(file -b --mime-encoding $line) -t utf-8 $line > $line.utf8;
                mv $line.utf8 $line;
              fi;
            ) || echo "good $line"
          done < .venv/FILES;
          uv tool run --from cyclonedx-bom==6.1.3 cyclonedx-py environment $VIRTUAL_ENV --PEP-639 --gather-license-texts --pyproject pyproject.toml --mc-type library --output-format JSON > sbom.json
      - name: Display SBOM
        working-directory: src/${{ matrix.package }}
        run: |
          cat <<EOT |
          import re
          import json
          import importlib.metadata as metadata

          def parse_bom(json_file):
              # Parse the JSON file
              with open(json_file, 'r') as file:
                  data = json.load(file)

              # Extract components
              components = []
              for component in data['components']:
                  comp_info = {}

                  # Get name, version, description, and purl
                  comp_info['name'] = component.get('name', 'Unknown')
                  comp_info['version'] = component.get('version', 'Unknown')
                  comp_info['description'] = component.get('description', 'Unknown')
                  comp_info['purl'] = component.get('purl', 'Unknown')

                  # Get licenses
                  comp_info['licenses'] = []
                  licenses = component.get('licenses', [])
                  for license in licenses:
                      if license.get('license', {}).get('id'):
                          comp_info['licenses'].append(license.get('license').get('id'))
                  if len(comp_info['licenses']) == 0:
                      comp_info['licenses'].append("No licenses")

                  # Extract additional information (copyright, etc.)
                  copyright_info = extract_copyright_from_metadata(comp_info['name'])
                  comp_info['copyright'] = copyright_info if copyright_info else "No copyright information"

                  components.append(comp_info)

              return components

          def extract_copyright_from_metadata(package_name):
              try:
                  # Use importlib.metadata to retrieve metadata from the installed package
                  dist = metadata.distribution(package_name)
                  metadata_info = dist.metadata

                  # Extract relevant metadata
                  copyright_info = []
                  author = metadata_info.get('Author')
                  author_email = metadata_info.get('Author-email')
                  license_info = metadata_info.get('License')

                  if author:
                      copyright_info.append(f"Author: {author}")
                  if author_email:
                      copyright_info.append(f"Author Email: {author_email}")
                  if license_info:
                      copyright_info.append(f"License: {license_info}")

                  # Check for classifiers or any extra metadata fields
                  if 'Classifier' in metadata_info:
                      for classifier in metadata_info.get_all('Classifier'):
                          if 'copyright' in classifier.lower():
                              copyright_info.append(classifier)

                  return ', '.join(copyright_info) if copyright_info else None

              except metadata.PackageNotFoundError:
                  return None


          def main():
              bom_file = 'sbom.json'  # Replace with your BOM file path
              components = parse_bom(bom_file)

              for component in components:
                  print(f"Name: {component['name']}")
                  print(f"Version: {component['version']}")
                  print(f"Description: {component['description']}")
                  print(f"PURL: {component['purl']}")
                  print(f"Licenses: {', '.join(component['licenses'])}")
                  print(f"Copyright: {component['copyright']}")
                  print("-" * 40)

          if __name__ == "__main__":
              main()
          EOT
           python -

      - name: Upload Software Bill of Materials
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: sbom-${{ matrix.package }}
          path: src/${{ matrix.package }}/sbom.json