Test Matrix #63
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Matrix | |
| on: | |
| schedule: | |
| # Run nightly comprehensive testing at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| # Allow manual triggering | |
| push: | |
| branches: [main] | |
| tags: ['v*'] | |
| # Only run on main branch and releases for comprehensive testing | |
| env: | |
| AWS_DEFAULT_REGION: us-east-1 | |
| AWS_ACCESS_KEY_ID: testing | |
| AWS_SECRET_ACCESS_KEY: testing | |
| ENVIRONMENT: testing | |
| TESTING: true | |
| jobs: | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "requirements*.txt" | |
| - name: Install dependencies with uv (Unix) | |
| if: runner.os != 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Install dependencies with uv (Windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Run unit tests | |
| run: | | |
| make ci-tests-unit | |
| # TODO: Remove continue-on-error once unit tests are fixed and stable across all Python versions | |
| continue-on-error: true | |
| - name: Upload unit test results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: unit-test-results-${{ matrix.os }}-${{ matrix.python-version }} | |
| path: | | |
| junit-*.xml | |
| coverage-*.xml | |
| integration-tests: | |
| name: Integration Tests | |
| runs-on: ${{ matrix.os }} | |
| needs: [unit-tests] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "requirements*.txt" | |
| - name: Install dependencies with uv (Unix) | |
| if: runner.os != 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Install dependencies with uv (Windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Run integration tests | |
| run: | | |
| make ci-tests-integration | |
| # TODO: Remove continue-on-error once integration tests are fixed and stable across all Python versions | |
| continue-on-error: true | |
| - name: Upload integration test results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: integration-test-results-${{ matrix.os }}-${{ matrix.python-version }} | |
| path: | | |
| junit-*.xml | |
| coverage-*.xml | |
| e2e-tests: | |
| name: End-to-End Tests | |
| runs-on: ${{ matrix.os }} | |
| needs: [unit-tests, integration-tests] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "requirements*.txt" | |
| - name: Install dependencies with uv (Unix) | |
| if: runner.os != 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Install dependencies with uv (Windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Run e2e tests | |
| run: | | |
| make ci-tests-e2e | |
| # TODO: Remove continue-on-error once e2e tests are fixed and stable across all Python versions | |
| continue-on-error: true | |
| - name: Upload e2e test results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: e2e-test-results-${{ matrix.os }}-${{ matrix.python-version }} | |
| path: | | |
| junit-*.xml | |
| test-summary: | |
| name: Test Summary | |
| runs-on: ubuntu-latest | |
| needs: [unit-tests, integration-tests, e2e-tests] | |
| if: always() | |
| steps: | |
| - name: Download all test results | |
| uses: actions/download-artifact@v5 | |
| with: | |
| path: test-results/ | |
| - name: Display test summary | |
| run: | | |
| echo "## Test Matrix Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Count test result files | |
| UNIT_RESULTS=$(find test-results/ -name "*unit.xml" | wc -l) | |
| INTEGRATION_RESULTS=$(find test-results/ -name "*integration.xml" | wc -l) | |
| echo "- Unit test configurations: $UNIT_RESULTS" >> $GITHUB_STEP_SUMMARY | |
| echo "- Integration test configurations: $INTEGRATION_RESULTS" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # List all test result files | |
| echo "### Test Result Files:" >> $GITHUB_STEP_SUMMARY | |
| find test-results/ -name "*.xml" | sort >> $GITHUB_STEP_SUMMARY |