Test Matrix #87
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Matrix | |
| on: | |
| schedule: | |
| # Run nightly comprehensive testing at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| # Allow manual triggering | |
| push: | |
| branches: [main] | |
| tags: ['v*'] | |
| # Only run on main branch and releases for comprehensive testing | |
| env: | |
| AWS_DEFAULT_REGION: us-east-1 | |
| AWS_ACCESS_KEY_ID: testing | |
| AWS_SECRET_ACCESS_KEY: testing | |
| ENVIRONMENT: testing | |
| TESTING: true | |
| jobs: | |
| get-config: | |
| name: Get Configuration from Makefile | |
| runs-on: ubuntu-latest | |
| outputs: | |
| python-versions: ${{ steps.config.outputs.python-versions }} | |
| default-python-version: ${{ steps.config.outputs.default-python-version }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Install yq | |
| uses: mikefarah/yq@master | |
| - name: Get configuration from project config | |
| id: config | |
| run: | | |
| DEFAULT_PYTHON_VERSION=$(make print-DEFAULT_PYTHON_VERSION) | |
| PYTHON_VERSIONS=$(make print-json-PYTHON_VERSIONS) | |
| echo "default-python-version=$DEFAULT_PYTHON_VERSION" >> "$GITHUB_OUTPUT" | |
| echo "python-versions=$PYTHON_VERSIONS" >> "$GITHUB_OUTPUT" | |
| echo "Found Python versions: $PYTHON_VERSIONS" | |
| echo "Default Python version: $DEFAULT_PYTHON_VERSION" | |
| unit-tests: | |
| name: Unit Tests | |
| needs: get-config | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| python-version: ${{ fromJSON(needs.get-config.outputs.python-versions) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "requirements*.txt" | |
| - name: Install dependencies with uv (Unix) | |
| if: runner.os != 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Install dependencies with uv (Windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Run unit tests | |
| run: | | |
| make ci-tests-unit | |
| # TODO: Remove continue-on-error once unit tests are fixed and stable across all Python versions | |
| continue-on-error: true | |
| - name: Upload unit test results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: unit-test-results-${{ matrix.os }}-${{ matrix.python-version }} | |
| path: | | |
| junit-*.xml | |
| coverage-*.xml | |
| integration-tests: | |
| name: Integration Tests | |
| runs-on: ${{ matrix.os }} | |
| needs: [get-config, unit-tests] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| python-version: ${{ fromJSON(needs.get-config.outputs.python-versions) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "requirements*.txt" | |
| - name: Install dependencies with uv (Unix) | |
| if: runner.os != 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Install dependencies with uv (Windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Run integration tests | |
| run: | | |
| make ci-tests-integration | |
| # TODO: Remove continue-on-error once integration tests are fixed and stable across all Python versions | |
| continue-on-error: true | |
| - name: Upload integration test results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: integration-test-results-${{ matrix.os }}-${{ matrix.python-version }} | |
| path: | | |
| junit-*.xml | |
| coverage-*.xml | |
| e2e-tests: | |
| name: End-to-End Tests | |
| runs-on: ${{ matrix.os }} | |
| needs: [get-config, unit-tests, integration-tests] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| python-version: ${{ fromJSON(needs.get-config.outputs.python-versions) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "requirements*.txt" | |
| - name: Install dependencies with uv (Unix) | |
| if: runner.os != 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Install dependencies with uv (Windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| make ci-install | |
| - name: Run e2e tests | |
| run: | | |
| make ci-tests-e2e | |
| # TODO: Remove continue-on-error once e2e tests are fixed and stable across all Python versions | |
| continue-on-error: true | |
| - name: Upload e2e test results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: e2e-test-results-${{ matrix.os }}-${{ matrix.python-version }} | |
| path: | | |
| junit-*.xml | |
| test-summary: | |
| name: Test Summary | |
| runs-on: ubuntu-latest | |
| needs: [unit-tests, integration-tests, e2e-tests] | |
| if: always() | |
| steps: | |
| - name: Download all test results | |
| uses: actions/download-artifact@v5 | |
| with: | |
| path: test-results/ | |
| - name: Display test summary | |
| run: | | |
| # Count test result files | |
| UNIT_RESULTS=$(find test-results/ -name "*unit.xml" | wc -l) | |
| INTEGRATION_RESULTS=$(find test-results/ -name "*integration.xml" | wc -l) | |
| { | |
| echo "## Test Matrix Results" | |
| echo "" | |
| echo "- Unit test configurations: $UNIT_RESULTS" | |
| echo "- Integration test configurations: $INTEGRATION_RESULTS" | |
| echo "" | |
| echo "### Test Result Files:" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| find test-results/ -name "*.xml" | sort >> "$GITHUB_STEP_SUMMARY" |