Robot Framework Tests #414
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Robot Framework Tests | |
| # One suite, run against a service profile (tests/profiles.yml). | |
| # | |
| # There is deliberately no "with API keys" / "without API keys" split. Every test | |
| # runs in every profile; the stub profile replays recorded real provider | |
| # responses (tests/cassettes/) so content assertions mean the same thing either | |
| # way. What varies between profiles is which backing services are real, not which | |
| # tests execute. | |
| # | |
| # pull_request -> mock. Needs no secrets, so it works unchanged on forks. | |
| # (A fork PR cannot read secrets under `pull_request` no | |
| # matter what label it carries, so the old | |
| # label-triggered workflow could never have worked for | |
| # the fork case it was added for. Making it work would | |
| # need `pull_request_target`, which runs untrusted code | |
| # with secrets in scope -- not worth it.) | |
| # push dev/main -> mock. | |
| # workflow_dispatch -> any profile, on demand. | |
| # | |
| # No cloud provider runs automatically. Automatic runs against a paid API cost | |
| # money on every push and pin CI to whichever vendor happens to be listed here, | |
| # which goes stale the moment the deployment switches providers. Testing a | |
| # specific provider's integration is a deliberate act: dispatch the profile you | |
| # want. | |
| on: | |
| pull_request: | |
| paths: | |
| - 'tests/**' | |
| - 'backends/advanced/src/**' | |
| - '.github/workflows/robot-tests.yml' | |
| push: | |
| branches: [dev, main] | |
| paths: | |
| - 'tests/**' | |
| - 'backends/advanced/src/**' | |
| - '.github/workflows/robot-tests.yml' | |
| workflow_dispatch: | |
| inputs: | |
| profile: | |
| description: 'Service profile to run against' | |
| required: true | |
| default: 'mock' | |
| type: choice | |
| options: | |
| - mock | |
| - smallest-openai | |
| - deepgram-openai | |
| - deepgram-openai-speaker | |
| - parakeet-ollama | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| select-profiles: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| profiles: ${{ steps.pick.outputs.profiles }} | |
| steps: | |
| - id: pick | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo 'profiles=["${{ inputs.profile }}"]' >> "$GITHUB_OUTPUT" | |
| else | |
| echo 'profiles=["mock"]' >> "$GITHUB_OUTPUT" | |
| fi | |
| robot-tests: | |
| needs: select-profiles | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| profile: ${{ fromJSON(needs.select-profiles.outputs.profiles) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver-opts: | | |
| image=moby/buildkit:latest | |
| network=host | |
| - uses: actions/cache@v4 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ hashFiles('backends/advanced/Dockerfile', 'backends/advanced/pyproject.toml') }} | |
| restore-keys: ${{ runner.os }}-buildx- | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - uses: astral-sh/setup-uv@v4 | |
| with: | |
| version: "latest" | |
| - name: Create plugins.yml from template | |
| run: cp config/plugins.yml.template config/plugins.yml | |
| - name: Run Robot Framework tests (profile=${{ matrix.profile }}) | |
| working-directory: tests | |
| env: | |
| PROFILE: ${{ matrix.profile }} | |
| # Every provider credential the dispatchable profiles might ask for. | |
| # All are absent for the mock profile and on fork PRs, which is fine -- | |
| # resolve_profile.py fails fast with an actionable message when the | |
| # chosen profile needs one and it is missing, rather than letting auth | |
| # errors surface as confusing test failures. | |
| SMALLEST_API_KEY: ${{ secrets.SMALLEST_API_KEY }} | |
| DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| CLEANUP_CONTAINERS: "false" | |
| run: | | |
| TEST_EXIT_CODE=0 | |
| make test OUTPUTDIR=results || TEST_EXIT_CODE=$? | |
| echo "test_exit_code=$TEST_EXIT_CODE" >> "$GITHUB_ENV" | |
| exit 0 | |
| - name: Save service logs | |
| if: always() | |
| working-directory: tests | |
| run: | | |
| mkdir -p logs | |
| for svc in chronicle-backend-test workers-test mongo-test redis-test; do | |
| docker logs "backend-test-$svc-1" > "logs/$svc.log" 2>&1 || true | |
| done | |
| - name: Upload service logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: robot-service-logs-${{ matrix.profile }} | |
| path: tests/logs/ | |
| retention-days: 7 | |
| - name: Upload Robot reports | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: robot-reports-${{ matrix.profile }} | |
| path: | | |
| tests/results/report.html | |
| tests/results/log.html | |
| tests/results/output.xml | |
| retention-days: 14 | |
| - name: Summarize | |
| if: always() | |
| working-directory: tests | |
| run: | | |
| { | |
| echo "## Robot Framework — profile \`${{ matrix.profile }}\`" | |
| echo "" | |
| if [ -f results/output.xml ]; then | |
| uv run --with-requirements test-requirements.txt python show_results.py || true | |
| else | |
| echo "No results produced." | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Fail if tests failed | |
| if: always() | |
| run: | | |
| if [ "${test_exit_code:-0}" != "0" ]; then | |
| echo "Robot Framework tests failed (exit ${test_exit_code})" | |
| exit 1 | |
| fi |