1.5.0 #486
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Matrix | |
| on: | |
| schedule: | |
| # Run nightly comprehensive testing at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| # Allow manual triggering | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'src/**' | |
| - 'tests/**' | |
| - 'pyproject.toml' | |
| - 'requirements*.txt' | |
| - 'uv.lock' | |
| - 'Makefile' | |
| - 'makefiles/**' | |
| - '.github/workflows/test-matrix.yml' | |
| # Only run on main branch for comprehensive testing | |
| jobs: | |
| config: | |
| name: Get Configuration | |
| uses: ./.github/workflows/shared-config.yml | |
| unit-tests: | |
| name: Unit Tests | |
| needs: config | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| # Dynamic matrix from config with static fallback for act/dry-run | |
| # Update static list when adding new Python versions (currently 3.10-3.14) | |
| python-version: ${{ fromJSON(needs.config.outputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }} | |
| uses: ./.github/workflows/reusable-test.yml | |
| with: | |
| test-type: unit | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| default-python-version: ${{ needs.config.outputs.default-python-version }} | |
| continue-on-error: true # TODO: Remove once tests are stable | |
| aws-region: ${{ needs.config.outputs.aws-region }} | |
| aws-access-key: ${{ needs.config.outputs.aws-access-key }} | |
| aws-secret-key: ${{ needs.config.outputs.aws-secret-key }} | |
| environment: ${{ needs.config.outputs.environment }} | |
| testing-flag: ${{ needs.config.outputs.testing-flag }} | |
| integration-tests: | |
| name: Integration Tests | |
| needs: [config, unit-tests] | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| # Dynamic matrix from config with static fallback for act/dry-run | |
| # Update static list when adding new Python versions (currently 3.10-3.14) | |
| python-version: ${{ fromJSON(needs.config.outputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }} | |
| uses: ./.github/workflows/reusable-test.yml | |
| with: | |
| test-type: integration | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| default-python-version: ${{ needs.config.outputs.default-python-version }} | |
| continue-on-error: true # TODO: Remove once tests are stable | |
| aws-region: ${{ needs.config.outputs.aws-region }} | |
| aws-access-key: ${{ needs.config.outputs.aws-access-key }} | |
| aws-secret-key: ${{ needs.config.outputs.aws-secret-key }} | |
| environment: ${{ needs.config.outputs.environment }} | |
| testing-flag: ${{ needs.config.outputs.testing-flag }} | |
| package-variant-tests: | |
| name: Package Variant Tests | |
| needs: [config, unit-tests] | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| # Dynamic matrix from config with static fallback | |
| python-version: ${{ fromJSON(needs.config.outputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }} | |
| package-variant: | |
| - "." | |
| - ".[cli]" | |
| - ".[api]" | |
| - ".[monitoring]" | |
| - ".[all]" | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install package variant | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| pip install -e "${{ matrix.package-variant }}" | |
| - name: Test import guards | |
| run: | | |
| python -c " | |
| import sys | |
| variant = '${{ matrix.package-variant }}' | |
| # Test basic imports (should always work) | |
| try: | |
| import orb | |
| print('✓ Basic orb import works') | |
| except ImportError as e: | |
| print(f'✗ Basic orb import failed: {e}') | |
| sys.exit(1) | |
| # Test CLI flag: RICH_AVAILABLE reflects whether rich is installed | |
| from orb.cli.console import RICH_AVAILABLE | |
| if variant in ['.[cli]', '.[all]']: | |
| if not RICH_AVAILABLE: | |
| print(f'✗ Expected RICH_AVAILABLE=True for {variant} but got False') | |
| sys.exit(1) | |
| print(f'✓ RICH_AVAILABLE=True as expected for {variant}') | |
| else: | |
| print(f'✓ RICH_AVAILABLE={RICH_AVAILABLE} (rich not required for {variant})') | |
| # Test API imports: fastapi is only available with .[api] or .[all] | |
| try: | |
| from orb.api.routers.machines import router | |
| if variant in ['.', '.[cli]', '.[monitoring]']: | |
| print(f'✗ Expected API import to fail for {variant} but it succeeded') | |
| sys.exit(1) | |
| print(f'✓ API router import works for {variant}') | |
| except ImportError as e: | |
| if variant in ['.[api]', '.[all]']: | |
| print(f'✗ Unexpected API import failure for {variant}: {e}') | |
| sys.exit(1) | |
| print(f'✓ Expected API import failure for {variant}') | |
| print(f'✓ All import tests passed for {variant}') | |
| " | |
| e2e-tests: | |
| name: End-to-End Tests | |
| needs: [config, unit-tests, integration-tests] | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] # Disabled windows-latest, macos-latest for now | |
| # Dynamic matrix from config with static fallback for act/dry-run | |
| # Update static list when adding new Python versions (currently 3.10-3.14) | |
| python-version: ${{ fromJSON(needs.config.outputs.python-versions || '["3.10", "3.11", "3.12", "3.13", "3.14"]') }} | |
| uses: ./.github/workflows/reusable-test.yml | |
| with: | |
| test-type: e2e | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| default-python-version: ${{ needs.config.outputs.default-python-version }} | |
| continue-on-error: true # TODO: Remove once tests are stable | |
| aws-region: ${{ needs.config.outputs.aws-region }} | |
| aws-access-key: ${{ needs.config.outputs.aws-access-key }} | |
| aws-secret-key: ${{ needs.config.outputs.aws-secret-key }} | |
| environment: ${{ needs.config.outputs.environment }} | |
| testing-flag: ${{ needs.config.outputs.testing-flag }} | |
| test-summary: | |
| name: Test Summary | |
| runs-on: ubuntu-latest | |
| needs: [unit-tests, integration-tests, package-variant-tests, e2e-tests] | |
| if: always() | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Download all test results | |
| uses: actions/download-artifact@v8 | |
| with: | |
| path: test-results/ | |
| - name: Display test summary | |
| run: | | |
| # Count test result files | |
| UNIT_RESULTS=$(find test-results/ -name "*unit.xml" | wc -l) | |
| INTEGRATION_RESULTS=$(find test-results/ -name "*integration.xml" | wc -l) | |
| { | |
| echo "## Test Matrix Results" | |
| echo "" | |
| echo "- Unit test configurations: $UNIT_RESULTS" | |
| echo "- Integration test configurations: $INTEGRATION_RESULTS" | |
| echo "" | |
| echo "### Test Result Files:" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| find test-results/ -name "*.xml" | sort >> "$GITHUB_STEP_SUMMARY" |