Test Matrix #407
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Matrix | |
| on: | |
| schedule: | |
| # Run nightly comprehensive testing at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| # Allow manual triggering | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'src/**' | |
| - 'tests/**' | |
| - 'pyproject.toml' | |
| - 'requirements*.txt' | |
| - 'uv.lock' | |
| - 'Makefile' | |
| - 'makefiles/**' | |
| - '.github/workflows/test-matrix.yml' | |
| # Only run on main branch for comprehensive testing | |
| jobs: | |
| config: | |
| name: Get Configuration | |
| uses: ./.github/workflows/shared-config.yml | |
| unit-tests: | |
| name: Unit Tests | |
| needs: config | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| # Dynamic matrix from config with static fallback for act/dry-run | |
| # Update static list when adding new Python versions (currently 3.10-3.14) | |
| python-version: ${{ fromJSON(needs.config.outputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }} | |
| uses: ./.github/workflows/reusable-test.yml | |
| with: | |
| test-type: unit | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| default-python-version: ${{ needs.config.outputs.default-python-version }} | |
| continue-on-error: true # TODO: Remove once tests are stable | |
| aws-region: ${{ needs.config.outputs.aws-region }} | |
| aws-access-key: ${{ needs.config.outputs.aws-access-key }} | |
| aws-secret-key: ${{ needs.config.outputs.aws-secret-key }} | |
| environment: ${{ needs.config.outputs.environment }} | |
| testing-flag: ${{ needs.config.outputs.testing-flag }} | |
| integration-tests: | |
| name: Integration Tests | |
| needs: [config, unit-tests] | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| # Dynamic matrix from config with static fallback for act/dry-run | |
| # Update static list when adding new Python versions (currently 3.10-3.14) | |
| python-version: ${{ fromJSON(needs.config.outputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }} | |
| uses: ./.github/workflows/reusable-test.yml | |
| with: | |
| test-type: integration | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| default-python-version: ${{ needs.config.outputs.default-python-version }} | |
| continue-on-error: true # TODO: Remove once tests are stable | |
| aws-region: ${{ needs.config.outputs.aws-region }} | |
| aws-access-key: ${{ needs.config.outputs.aws-access-key }} | |
| aws-secret-key: ${{ needs.config.outputs.aws-secret-key }} | |
| environment: ${{ needs.config.outputs.environment }} | |
| testing-flag: ${{ needs.config.outputs.testing-flag }} | |
| package-variant-tests: | |
| name: Package Variant Tests | |
| needs: [config, unit-tests] | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| # Dynamic matrix from config with static fallback | |
| python-version: ${{ fromJSON(needs.config.outputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }} | |
| package-variant: | |
| - "." | |
| - ".[cli]" | |
| - ".[api]" | |
| - ".[monitoring]" | |
| - ".[all]" | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install package variant | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e "${{ matrix.package-variant }}" | |
| - name: Test import guards | |
| run: | | |
| python -c " | |
| import sys | |
| # Test basic imports (should always work) | |
| try: | |
| import orb | |
| print('✓ Basic orb import works') | |
| except ImportError as e: | |
| print(f'✗ Basic orb import failed: {e}') | |
| sys.exit(1) | |
| # Test CLI imports | |
| try: | |
| from src.cli.formatters import format_table | |
| print('✓ CLI formatters import works') | |
| except ImportError as e: | |
| if '${{ matrix.package-variant }}' in ['orb-py', 'orb-py[api]', 'orb-py[monitoring]']: | |
| print(f'✓ Expected CLI import failure for ${{ matrix.package-variant }}: {e}') | |
| else: | |
| print(f'✗ Unexpected CLI import failure: {e}') | |
| sys.exit(1) | |
| # Test API imports | |
| try: | |
| from src.interface.serve_command_handler import handle_serve_command | |
| print('✓ API handler import works') | |
| except ImportError as e: | |
| if '${{ matrix.package-variant }}' in ['orb-py', 'orb-py[cli]', 'orb-py[monitoring]']: | |
| print(f'✓ Expected API import failure for ${{ matrix.package-variant }}: {e}') | |
| else: | |
| print(f'✗ Unexpected API import failure: {e}') | |
| sys.exit(1) | |
| print('✓ All import tests passed for ${{ matrix.package-variant }}') | |
| " | |
| e2e-tests: | |
| name: End-to-End Tests | |
| needs: [config, unit-tests, integration-tests] | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| # Dynamic matrix from config with static fallback for act/dry-run | |
| # Update static list when adding new Python versions (currently 3.10-3.14) | |
| python-version: ${{ fromJSON(needs.config.outputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }} | |
| uses: ./.github/workflows/reusable-test.yml | |
| with: | |
| test-type: e2e | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| default-python-version: ${{ needs.config.outputs.default-python-version }} | |
| continue-on-error: true # TODO: Remove once tests are stable | |
| aws-region: ${{ needs.config.outputs.aws-region }} | |
| aws-access-key: ${{ needs.config.outputs.aws-access-key }} | |
| aws-secret-key: ${{ needs.config.outputs.aws-secret-key }} | |
| environment: ${{ needs.config.outputs.environment }} | |
| testing-flag: ${{ needs.config.outputs.testing-flag }} | |
| test-summary: | |
| name: Test Summary | |
| runs-on: ubuntu-latest | |
| needs: [unit-tests, integration-tests, package-variant-tests, e2e-tests] | |
| if: always() | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Download all test results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| path: test-results/ | |
| - name: Display test summary | |
| run: | | |
| # Count test result files | |
| UNIT_RESULTS=$(find test-results/ -name "*unit.xml" | wc -l) | |
| INTEGRATION_RESULTS=$(find test-results/ -name "*integration.xml" | wc -l) | |
| { | |
| echo "## Test Matrix Results" | |
| echo "" | |
| echo "- Unit test configurations: $UNIT_RESULTS" | |
| echo "- Integration test configurations: $INTEGRATION_RESULTS" | |
| echo "" | |
| echo "### Test Result Files:" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| find test-results/ -name "*.xml" | sort >> "$GITHUB_STEP_SUMMARY" |