Merge pull request #19 from scality/gh_refacto

♻️ refactor all ghaction

Author: m4nch0t

.devcontainer/Dockerfile

@@ -8,6 +8,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         bash-completion \
         curl \
         git \
+        libmagic1 \
         libsqlite3-dev \
         python3 \
         python3-pip \

.devcontainer/devcontainer.json

@@ -30,7 +30,9 @@
         //Git history
         "donjayamanne.githistory",
         //Git blame
-        "solomonkinard.git-blame"
+        "solomonkinard.git-blame",
+        //Mermaid support
+        "bierner.markdown-mermaid"
       ],
       "settings": {
         "terminal.integrated.profiles.linux": {

.devcontainer/requirements.txt

@@ -2,4 +2,7 @@ requests==2.32.3
 GitPython==3.1.44
 pyunpack==0.3
 patool==4.0.0
-pre-commit==4.1.0
+pre-commit==4.1.0
+click==8.1.8
+python-magic==0.4.27
+pydantic==2.11.2

.github/scripts/update_scanners.py

@@ -1,35 +1,80 @@
 #!/usr/bin/env python3
+""""
+Update scanner versions in the install.py file.
+This script fetches the latest release versions of the specified scanners
+from their GitHub repositories and updates the version strings in the
+install.py file.
+"""
 
 import re
 import requests
 
 # Define the scanners and their GitHub repositories
-scanners = {
+SCANNERS_REPOSITORIES = {
     "syft": "anchore/syft",
     "grype": "anchore/grype",
-    "trivy": "aquasecurity/trivy"
 }
 
 def get_latest_release(repo):
+    """Fetch the latest release version from a GitHub repository."""
     url = f"https://api.github.com/repos/{repo}/releases/latest"
-    response = requests.get(url)
+    response = requests.get(url, timeout=10)
     response.raise_for_status()
     return response.json()["tag_name"].lstrip("v")
 
 def update_versions(file_path):
-    with open(file_path, "r") as file:
+    """Update the scanner versions in the specified file."""
+    content = None
+    with open(file_path, "r", encoding="utf8") as file:
         content = file.read()
 
-    for scanner, repo in scanners.items():
+        # Find the scanners dictionary block
+        scanners_pattern = r"SCANNERS_VERSION = \{(.*?)\}"
+        scanners_match = re.search(scanners_pattern, content, re.DOTALL)
+
+    if not scanners_match:
+        raise ValueError("Could not find scanners dictionary in the file")
+
+    scanners_block = scanners_match.group(0)
+    updated_block = scanners_block
+    updates_made = False
+
+    for scanner, repo in SCANNERS_REPOSITORIES.items():
+        # Extract current version from the file
+        current_version_pattern = rf'"{scanner}": "([0-9]+\.[0-9]+\.[0-9]+)"'
+        current_version_match = re.search(current_version_pattern, scanners_block)
+        current_version = current_version_match.group(1) if current_version_match else "unknown"
+
+        # Get latest version from GitHub
         latest_version = get_latest_release(repo)
-        content = re.sub(
-            f'("{scanner}": ")([^"]+)',
-            lambda match: f'{match.group(1)}{latest_version}',
-            content
-        )
 
-    with open(file_path, "w") as file:
+        # Compare versions
+        if current_version == latest_version:
+            print(f"✓ {scanner} is already at latest version {latest_version}")
+            continue
+
+        # Version is different, update needed
+        print(f"⬆ Updating {scanner} from {current_version} to {latest_version}")
+        updates_made = True
+
+        # Build pattern and replacement
+        scanner_pattern = rf'("{scanner}": ")([0-9]+\.[0-9]+\.[0-9]+)'
+
+        # Use lambda for replacement
+        updated_block = re.sub(
+            scanner_pattern,
+            lambda match, version=latest_version: match.group(1) + version, updated_block)
+
+    # If no updates were made, print and return
+    if not updates_made:
+        print("✓ All scanners are already at their latest versions. No updates needed.")
+        return
+
+    # Write to file if changes were made
+    content = content.replace(scanners_block, updated_block)
+    with open(file_path, "w", encoding="utf8") as file:
         file.write(content)
+    print("✓ Updates applied to", file_path)
 
 if __name__ == "__main__":
-    update_versions("src/lib/install.py")
+    update_versions("src/modules/install.py")

.github/workflows/nightly.yaml

@@ -11,46 +11,6 @@ permissions:
   contents: read
 
 jobs:
-  update-scanners:
-    runs-on: ubuntu-24.04
-    steps:
-
-      - name: Create github token
-        uses: actions/create-github-app-token@v2
-        id: app-token
-        with:
-          app-id: ${{ vars.ACTIONS_APP_ID }}
-          private-key: ${{ secrets.ACTIONS_APP_PRIVATE_KEY }}
-          owner: ${{ github.repository_owner }}
-
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          token: ${{ steps.app-token.outputs.token }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.12
-
-      - name: Install dependencies
-        run: pip install requests
-
-      - name: Update scanner versions
-        run: python .github/scripts/update_scanners.py
-
-      - name: Create pull request  
-        uses: actions/github-script@v7  
-        with:  
-          script: |
-            const pr = await github.rest.pulls.create({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              head: "feature/deps-update",
-              base: "main",
-              title: ":arrow_up: Update scanner versions"
-            })
-
   vuln-scan:
     permissions:
       contents: read # for actions/checkout to fetch code
@@ -65,23 +25,17 @@ jobs:
           fetch-tags: true
 
       - name: Create SBOM
-        uses: anchore/sbom-action@v0
-        with:
-          path: ./
-          format: cyclonedx-json
-          output-file: "${{ github.event.repository.name }}-sbom.cdx.json"
-
-      - name: Scan SBOM
-        uses: anchore/scan-action@v6
-        id: scan
+        uses: ./
         with:
-          sbom: "${{ github.event.repository.name }}-sbom.cdx.json"
-          output-format: sarif
-          fail-build: false
-          add-cpes-if-none: true
-          by-cve: true
+          target: ./
+          target_type: file
+          output_format: cyclonedx-json
+          output_file: "/tmp/sbom/sbom.cdx.json"
+          vuln: true
+          vuln_output_format: sarif
+          vuln_output_file: "/tmp/sbom/sbom.sarif"
 
       - name: Upload SARIF report
         uses: github/codeql-action/upload-sarif@v3
         with:
-          sarif_file: ${{ steps.scan.outputs.sarif }}
+          sarif_file: "/tmp/sbom/sbom.sarif"

.github/workflows/tests.yaml

@@ -1,88 +1,143 @@
-name: "action-test"
+name: "SBOM Action Tests"
 on:
   push:
   pull_request:
 
 jobs:
-  test-as-action:
+  test-repository:
     runs-on: ubuntu-24.04
     steps:
-      - name: Checkout
+      - name: Checkout repository
         uses: actions/checkout@v4
+
+      - name: Test file target type (repository)
+        id: repo-test
+        uses: ./
         with:
-          path: ./
-          fetch-depth: 0
-          fetch-tags: true
+          target: ${{ github.workspace }}
+          target_type: file
+          output_dir: "/tmp/sbom"
+          name: "repo"
+          output_file: "repository_sbom.json"
+          vuln: true
+          vuln_output_format: "sarif"
+          vuln_output_file: "repository_vuln.sarif"
 
-      - name: Download artifact
+      - name: Validate Repository SBOM
         shell: bash
-        run: curl -o /tmp/Core-15.0.iso https://distro.ibiblio.org/tinycorelinux/15.x/x86/release/Core-15.0.iso
+        run: |
+          echo "## File Target (Repository)" >> $GITHUB_STEP_SUMMARY
+          if jq -e '.components[] | select(.name == "lodash" or .name == "yaml")' /tmp/sbom/repository_sbom.json > /dev/null 2>&1; then
+            echo "✅ Repository SBOM contains expected packages" >> $GITHUB_STEP_SUMMARY
+            echo "RESULT=PASS" >> $GITHUB_ENV
+          else
+            echo "❌ Repository SBOM missing expected packages" >> $GITHUB_STEP_SUMMARY
+            echo "RESULT=FAIL" >> $GITHUB_ENV
+            exit 1
+          fi
 
-      - name: Scan repo
-        uses: ./
-        with:
-          target: ./
-          output-dir: "/tmp/test/sbom"
-          vuln-report: True
+  test-directory:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-      - name: Scan directory
+      - name: Test directory target type
+        id: dir-test
         uses: ./
         with:
           target: /usr/local/bin
-          output-dir: "/tmp/test/sbom"
-          syft-version: "1.10.0"
+          target_type: file
           name: "usrlocalbin"
+          output_format: "spdx-json"
+          syft_version: "1.21.0"
+
+      - name: Validate Directory SBOM
+        shell: bash
+        run: |
+          echo "## Directory Target" >> $GITHUB_STEP_SUMMARY
+          if jq -e '(.spdxVersion != null) and (.name | contains("usrlocalbin"))' /tmp/sbom/usrlocalbin_sbom.json > /dev/null 2>&1; then
+            echo "✅ Directory SBOM contains correct metadata" >> $GITHUB_STEP_SUMMARY
+            echo "RESULT=PASS" >> $GITHUB_ENV
+          else
+            echo "❌ Directory SBOM has incorrect metadata" >> $GITHUB_STEP_SUMMARY
+            echo "RESULT=FAIL" >> $GITHUB_ENV
+            exit 1
+          fi
+
+      - name: List files
+        shell: bash
+        run: |
+          find /tmp/sbom -type f -name "*.json" | sort
+
+  test-iso:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download test iso
+        shell: bash
+        run: |
+          # Download ISO for testing
+          curl -o /tmp/Core-15.0.iso https://distro.ibiblio.org/tinycorelinux/15.x/x86/release/Core-15.0.iso
 
-      - name: Scan iso
+      - name: Test ISO target type
+        id: iso-test
         uses: ./
         with:
           target: /tmp/Core-15.0.iso
-          output-dir: "/tmp/test/sbom"
+          target_type: iso
           version: "15.0"
+          output_dir: "/tmp/sbom"
           name: "tinycorelinux"
-          vuln-report: False
 
-      - name: Ensure generated sbom file for repo contains the expected content
+      - name: Validate ISO SBOM
         shell: bash
         run: |
-          if jq -e '.components[] | select(.name == "lodash")' /tmp/test/sbom/repo_sbom_*.json > /dev/null; then
-            echo "lodash is present in the JSON file."
-            exit 0
+          echo "## ISO Target" >> $GITHUB_STEP_SUMMARY
+          if jq -e '.metadata.component.name == "tinycorelinux"' /tmp/sbom/tinycorelinux_15.0_sbom.json > /dev/null 2>&1; then
+            echo "✅ ISO SBOM contains correct metadata" >> $GITHUB_STEP_SUMMARY
+            echo "RESULT=PASS" >> $GITHUB_ENV
           else
-            echo "lodash is NOT present in the JSON file."
+            echo "❌ ISO SBOM missing expected components" >> $GITHUB_STEP_SUMMARY
+            echo "RESULT=FAIL" >> $GITHUB_ENV
             exit 1
           fi
 
-      - name: Ensure generated sbom file for iso contains the expected content
+  test-image:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download test image
         shell: bash
         run: |
-          if jq -e '.components[] | select(.version == "6.6.8-tinycore")' /tmp/test/sbom/iso_tinycorelinux_15.0.json > /dev/null; then
-            echo "tinycore is present in the JSON file."
-            exit 0
-          else
-            echo "tinycore is NOT present in the JSON file."
-            exit 1
-          fi
+          docker pull alpine:latest
+          docker save alpine:latest > /tmp/alpine.tar
 
-      - name: Ensure generated sbom file for dir contains the expected content
-        shell: bash
+      - name: Test image target type (tarball)
+        id: image-test
+        uses: ./
+        with:
+          target: /tmp/alpine.tar
+          target_type: image
+          output_dir: "/tmp/sbom"
+
+      - name: Validate container image SBOM
         run: |
-          if jq -e '.components[] | select(.purl == "pkg:golang/github.com/anchore/syft@v1.10.0")' /tmp/test/sbom/dir_bin_undefined.json > /dev/null; then
-            echo "syft is present in the JSON file."
-            exit 0
+          if jq -e '.metadata.component.type == "container"' /tmp/sbom/alpine_latest_sbom.json > /dev/null 2>&1; then
+            echo "✅ Container image SBOM has correct type"
           else
-            echo "syft is NOT present in the JSON file."
+            echo "❌ Container image SBOM missing expected type"
             exit 1
           fi
 
-      - name: Print the content of generated sbom file
-        shell: bash
-        run: |
-          for sbom in /tmp/test/sbom/*.json; do
-            echo "Content of $sbom"
-            cat $sbom
-          done
-          for sbom in /tmp/test/sbom/reports/*.html; do
-            echo "Content of vulnerability result for SBOM: $sbom"
-            cat $sbom
-          done
+  test-summary:
+    needs: [test-repository, test-directory, test-iso, test-image]
+    if: always()
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Tests summary
+        run: echo "All tests completed"