feat: CI/CD pipeline optimization and health monitoring

.github/workflows/advanced-metrics.yml

@@ -0,0 +1,162 @@
+name: Advanced Metrics Badges
+
+permissions:
+  contents: read
+  actions: read
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - 'pyproject.toml'
+  workflow_dispatch:
+
+jobs:
+  config:
+    name: Configuration
+    uses: ./.github/workflows/shared-config.yml
+
+  metrics:
+    name: Generate Advanced Metrics
+    needs: config
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      actions: read
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v6.0.1
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ needs.config.outputs.default-python-version }}
+
+    - name: Cache management
+      uses: ./.github/workflows/cache-management.yml
+      with:
+        python-version: ${{ needs.config.outputs.default-python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install coverage pytest cloc
+
+    - name: Run tests with coverage
+      run: |
+        coverage run -m pytest tests/ --tb=short
+        coverage report --format=total > coverage.txt
+        coverage xml
+
+    - name: Calculate lines of code
+      run: |
+        # Install cloc if not available
+        sudo apt-get update && sudo apt-get install -y cloc
+
+        # Count lines of code (excluding tests, docs, config)
+        cloc src/ --json --out=cloc.json
+
+        # Extract metrics
+        total_lines=$(jq -r '.SUM.code // 0' cloc.json)
+        comment_lines=$(jq -r '.SUM.comment // 0' cloc.json)
+
+        # Calculate comment percentage
+        if [ "$total_lines" -gt 0 ]; then
+          comment_percent=$(echo "scale=1; $comment_lines * 100 / $total_lines" | bc)
+        else
+          comment_percent="0"
+        fi
+
+        # Format for badges
+        if [ "$total_lines" -ge 1000 ]; then
+          loc_display=$(echo "scale=1; $total_lines / 1000" | bc)k
+        else
+          loc_display="$total_lines"
+        fi
+
+        echo "TOTAL_LOC=$loc_display" >> "$GITHUB_ENV"
+        echo "COMMENT_PERCENT=$comment_percent" >> "$GITHUB_ENV"
+
+    - name: Run performance test
+      run: |
+        start_time=$(date +%s.%N)
+        python -m pytest tests/ -x --tb=no -q
+        end_time=$(date +%s.%N)
+
+        # Calculate duration in seconds
+        duration=$(echo "$end_time - $start_time" | bc)
+        duration_formatted=$(printf "%.1fs" "$duration")
+
+        echo "TEST_DURATION=$duration_formatted" >> "$GITHUB_ENV"
+
+    - name: Extract coverage percentage
+      run: |
+        coverage_percent=$(cat coverage.txt)
+        echo "COVERAGE_PERCENT=$coverage_percent" >> "$GITHUB_ENV"
+
+        # Set coverage color
+        if [ "$coverage_percent" -ge 90 ]; then
+          coverage_color="brightgreen"
+        elif [ "$coverage_percent" -ge 75 ]; then
+          coverage_color="yellow"
+        elif [ "$coverage_percent" -ge 60 ]; then
+          coverage_color="orange"
+        else
+          coverage_color="red"
+        fi
+
+        echo "COVERAGE_COLOR=$coverage_color" >> "$GITHUB_ENV"
+
+    - name: Create coverage badge
+      uses: schneegans/dynamic-badges-action@v1.7.0
+      with:
+        auth: ${{ secrets.GITHUB_TOKEN }}
+        gistID: ${{ secrets.METRICS_GIST_ID }}
+        filename: coverage.json
+        label: Coverage
+        message: ${{ env.COVERAGE_PERCENT }}%
+        color: ${{ env.COVERAGE_COLOR }}
+
+    - name: Create lines of code badge
+      uses: schneegans/dynamic-badges-action@v1.7.0
+      with:
+        auth: ${{ secrets.GITHUB_TOKEN }}
+        gistID: ${{ secrets.METRICS_GIST_ID }}
+        filename: lines-of-code.json
+        label: Lines of Code
+        message: ${{ env.TOTAL_LOC }}
+        color: lightgrey
+
+    - name: Create comment percentage badge
+      uses: schneegans/dynamic-badges-action@v1.7.0
+      with:
+        auth: ${{ secrets.GITHUB_TOKEN }}
+        gistID: ${{ secrets.METRICS_GIST_ID }}
+        filename: comments.json
+        label: Comments
+        message: ${{ env.COMMENT_PERCENT }}%
+        valColorRange: ${{ env.COMMENT_PERCENT }}
+        maxColorRange: 30
+        minColorRange: 0
+
+    - name: Create test duration badge
+      uses: schneegans/dynamic-badges-action@v1.7.0
+      with:
+        auth: ${{ secrets.GITHUB_TOKEN }}
+        gistID: ${{ secrets.METRICS_GIST_ID }}
+        filename: test-duration.json
+        label: Test Duration
+        message: ${{ env.TEST_DURATION }}
+        color: blue
+
+    - name: Upload coverage report
+      uses: actions/upload-artifact@v4.4.3
+      with:
+        name: coverage-report
+        retention-days: 30
+        path: |
+          coverage.xml
+          cloc.json

.github/workflows/cache-management.yml

@@ -13,9 +13,8 @@ on:
         type: string
       python-version:
         description: 'Python version for cache key'
-        required: false
+        required: true
         type: string
-        default: '3.13'
     outputs:
       cache-key:
         description: 'Generated cache key'
@@ -59,7 +58,7 @@ jobs:
 
     - name: Restore cache
       id: cache
-      uses: actions/cache@v5.0.0
+      uses: actions/cache@v5
       with:
         path: |
           ~/.cache/uv

.github/workflows/changelog.yml → .github/workflows/changelog-validation.yml

@@ -7,19 +7,37 @@
       - '.git-changelog.toml'
       - '.changelog-template.md'
       - 'dev-tools/release/changelog_manager.py'
-  push:
-    branches: [main]
-    paths:
-      - 'CHANGELOG.md'
+      - '.github/workflows/changelog-validation.yml'
   workflow_dispatch:
 
-env:
-  PYTHON_VERSION: '3.11'
-
 jobs:
+  get-config:
+    name: Get Configuration
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      default-python-version: ${{ steps.config.outputs.default-python-version }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v6.0.1
+    - name: Get project configuration
+      id: config
+      uses: ./.github/actions/get-config
+
+  setup-cache:
+    name: Setup Cache
+    needs: get-config
+    uses: ./.github/workflows/cache-management.yml
+    with:
+      cache-type: dependencies
+      cache-key-base: changelog-validation
+      python-version: ${{ needs.get-config.outputs.default-python-version }}
+
   validate-changelog:
-    name: Validate Changelog
+    name: Validate Changelog Format
     runs-on: ubuntu-latest
+    needs: [get-config, setup-cache]
     permissions:
       contents: read
 
@@ -32,7 +50,7 @@
     - name: Setup Python and UV
       uses: ./.github/actions/setup-uv-cached
       with:
-        cache-key: changelog-${{ env.PYTHON_VERSION }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+        cache-key: ${{ needs.setup-cache.outputs.cache-key }}
         fail-on-cache-miss: false
 
     - name: Install changelog dependencies
@@ -82,23 +100,3 @@
         } >> "$GITHUB_STEP_SUMMARY"
         make changelog-preview --from-commit="${{ github.event.pull_request.base.sha }}" >> "$GITHUB_STEP_SUMMARY" || echo "No changes to preview" >> "$GITHUB_STEP_SUMMARY"
         echo '```' >> "$GITHUB_STEP_SUMMARY"
-
-  check-changelog-sync:
-    name: Check Changelog Sync
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v6.0.1
-      with:
-        fetch-depth: 0
-
-    - name: Setup Python and UV
-      uses: ./.github/actions/setup-uv-cached
-      with:
-        cache-key: changelog-sync-${{ env.PYTHON_VERSION }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
-        fail-on-cache-miss: false
-
-    - name: Validate changelog
-      run: make changelog-validate

.github/workflows/ci-quality.yml

@@ -1,4 +1,4 @@
-name: CI Quality
+name: Quality Checks
 
 on:
   push:
@@ -8,6 +8,7 @@ on:
       - 'tests/**'
       - 'pyproject.toml'
       - 'requirements*.txt'
+      - 'uv.lock'
       - '.ruff.toml'
       - 'mypy.ini'
       - '.github/workflows/ci-quality.yml'
@@ -18,33 +19,33 @@ on:
       - 'tests/**'
       - 'pyproject.toml'
       - 'requirements*.txt'
+      - 'uv.lock'
       - '.ruff.toml'
       - 'mypy.ini'
       - '.github/workflows/ci-quality.yml'
 
 permissions:
   contents: read
-  pull-requests: read
-
-env:
-  AWS_DEFAULT_REGION: us-east-1
-  AWS_ACCESS_KEY_ID: testing
-  AWS_SECRET_ACCESS_KEY: testing
-  ENVIRONMENT: testing
-  TESTING: true
 
 jobs:
   config:
     name: Configuration
     uses: ./.github/workflows/shared-config.yml
 
   quality-check:
-    name: Professional Quality Standards
+    name: Quality Standards
     runs-on: ubuntu-latest
     needs: config
     permissions:
       contents: read
 
+    env:
+      AWS_DEFAULT_REGION: ${{ needs.config.outputs.aws-region }}
+      AWS_ACCESS_KEY_ID: ${{ needs.config.outputs.aws-access-key }}
+      AWS_SECRET_ACCESS_KEY: ${{ needs.config.outputs.aws-secret-key }}
+      ENVIRONMENT: ${{ needs.config.outputs.environment }}
+      TESTING: ${{ needs.config.outputs.testing-flag }}
+
     steps:
     - uses: actions/checkout@v6.0.1
       with:
@@ -56,7 +57,7 @@ jobs:
         python-version: ${{ needs.config.outputs.default-python-version }}
         cache-key-suffix: quality
 
-    - name: Run professional quality checks
+    - name: Run quality checks
       run: |
         if [ "${{ github.event_name }}" = "schedule" ]; then
           make quality-check-all
@@ -145,7 +146,7 @@ jobs:
         continue-on-error: true
 
   lint-mypy:
-    name: mypy (Type Checking)
+    name: Type Checking (mypy) - Optional
     runs-on: ubuntu-latest
     needs: [config, setup-cache, lint-ruff]
     permissions:
@@ -161,18 +162,27 @@ jobs:
         fail-on-cache-miss: false
 
     - name: Run mypy type check
+      continue-on-error: true  # TODO: Remove once type issues are fixed
       run: make ci-quality-mypy
-      continue-on-error: true
 
   arch-validation:
-    name: Architecture Validation
+    name: Architecture Validation - Optional
     runs-on: ubuntu-latest
     needs: [config, setup-cache, lint-ruff]
     permissions:
       contents: read
     strategy:
       matrix:
         check: [cqrs, clean, imports, file-sizes]
+        include:
+          - check: cqrs
+            description: "CQRS Pattern Validation"
+          - check: clean  
+            description: "Clean Architecture Dependencies"
+          - check: imports
+            description: "Import Validation"
+          - check: file-sizes
+            description: "File Size Compliance"
     steps:
     - name: Checkout code
       uses: actions/checkout@v6.0.1
@@ -183,6 +193,6 @@ jobs:
         cache-key: ${{ needs.setup-cache.outputs.cache-key }}
         fail-on-cache-miss: false
 
-    - name: Run architecture validation
+    - name: Run architecture validation (${{ matrix.description }})
+      continue-on-error: true  # TODO: Remove once architectural issues are fixed
       run: make ci-arch-${{ matrix.check }}
-      continue-on-error: true