Update secfixes tracker #5020
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update secfixes tracker | |
| on: | |
| schedule: | |
| - cron: "0 */6 * * *" | |
| workflow_dispatch: | |
| jobs: | |
| update: | |
| name: Update | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:3.9-alpine | |
| env: | |
| FLASK_APP: secfixes_tracker | |
| SECFIXES_TRACKER_CONFIG: application.cfg | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v2 | |
| - name: Install system dependencies | |
| run: | | |
| apk add --no-cache alpine-sdk gcc musl-dev libc-dev apk-tools | |
| # Find actual libapk file and create unversioned symlink for Python ctypes | |
| LIBAPK_FILE=$(find /usr/lib /lib -name "libapk.so.*" 2>/dev/null | head -1) | |
| if [ -n "$LIBAPK_FILE" ]; then | |
| LIBAPK_DIR=$(dirname "$LIBAPK_FILE") | |
| ln -sf "$LIBAPK_FILE" "$LIBAPK_DIR/libapk.so" | |
| echo "Created libapk.so symlink: $LIBAPK_DIR/libapk.so -> $LIBAPK_FILE" | |
| else | |
| echo "ERROR: No libapk.so.* file found!" | |
| exit 1 | |
| fi | |
| # Set library path for Python ctypes | |
| echo "LD_LIBRARY_PATH=/usr/lib:/lib" >> $GITHUB_ENV | |
| - name: Install Python dependencies | |
| run: pip3 install -r requirements.txt | |
| - name: Initialize DB | |
| run: flask init-db | |
| - name: Import APKINDEX | |
| run: flask import-apkindex | |
| - name: Import secfixes | |
| run: flask import-secfixes | |
| - name: Clone vuln-list-nvd and cleanup unwanted data | |
| run: | | |
| echo "=== Cloning aquasecurity/vuln-list-nvd repository ===" | |
| git clone https://github.com/aquasecurity/vuln-list-nvd.git --depth 1 | |
| echo "✓ Repository cloned" | |
| echo "=== Cleaning up unwanted folders to save disk space ===" | |
| cd vuln-list-nvd | |
| # Keep only the years we need (2018-current) | |
| current_year=$(date "+%Y") | |
| for dir in api/*/; do | |
| year=$(basename "$dir") | |
| if [[ "$year" =~ ^[0-9]{4}$ ]] && [ "$year" -lt 2018 ]; then | |
| echo "Deleting pre-2018 data: $dir" | |
| rm -rf "$dir" | |
| fi | |
| done | |
| # Remove other non-API directories to save space | |
| rm -rf .git docs README.md || true | |
| cd .. | |
| echo "✓ Cleanup completed" | |
| - name: Import NVD from local files with progressive cleanup | |
| run: | | |
| # Import NVD CVEs from local files with disk space management | |
| echo "=== Importing ALL years with progressive cleanup ===" | |
| for year in $(seq 2018 $(date "+%Y")); do | |
| if [ -d "vuln-list-nvd/api/$year" ]; then | |
| echo "=== Processing year $year ($(find vuln-list-nvd/api/$year -name '*.json' | wc -l) files) ===" | |
| # Import the year's data | |
| timeout 60m flask import-nvd-files vuln-list-nvd/api/$year || echo "Year $year timed out, continuing..." | |
| # Immediately delete processed files to free disk space | |
| echo "Cleaning up $year files to free disk space..." | |
| rm -rf vuln-list-nvd/api/$year | |
| # Force garbage collection between years | |
| python3 -c "import gc; gc.collect()" | |
| # Show remaining disk space | |
| df -h . | tail -1 | awk '{print "Disk usage: " $3 " used, " $4 " available"}' | |
| echo "=== Completed and cleaned up year $year ===" | |
| else | |
| echo "=== Skipping year $year (directory not found) ===" | |
| fi | |
| done | |
| echo "=== Final cleanup: removing vuln-list-nvd directory ===" | |
| rm -rf vuln-list-nvd | |
| env: | |
| NVD_API_KEY: ${{ secrets.NVD_API_KEY }} | |
| - name: Update states | |
| run: flask update-states | |
| - name: Clean existing files | |
| run: rm -rf data && mkdir data | |
| - name: Export JSON files | |
| run: flask export | |
| - name: Compress | |
| run: | | |
| tar cvfz all.tar.gz data | |
| mkdir -p public | |
| mv all.tar.gz public | |
| - name: Deploy | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./public |