CLI Integration Tests #523
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CLI Integration Tests | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| schedule: | |
| # Run daily at 06:00 UTC | |
| - cron: "0 6 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: "Test mode: 'verify' to check against fixtures, 'record' to update fixtures" | |
| required: true | |
| default: "verify" | |
| type: choice | |
| options: | |
| - verify | |
| - record | |
| permissions: | |
| contents: read | |
| jobs: | |
| cli-verify: | |
| name: CLI integration tests (verify) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| outputs: | |
| small_mismatches: ${{ steps.check_results.outputs.small_mismatches }} | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.11" | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| version: "latest" | |
| enable-cache: true | |
| - name: Install dependencies | |
| run: | | |
| make requirements | |
| - name: Verify wikipedia-api command is available | |
| run: | | |
| uv run wikipedia-api --version | |
| - name: Verify CLI output against fixtures | |
| id: verify_step | |
| run: | | |
| uv run tests/cli/test_cli.sh verify | tee verify_output.txt || true | |
| - name: Check for small mismatches and failures | |
| id: check_results | |
| run: | | |
| MISMATCH_COUNT=$(grep 'Small Mismatch:' verify_output.txt | awk '{print $3}' || echo "0") | |
| FAILED_COUNT=$(grep 'Failed:' verify_output.txt | awk '{print $2}' || echo "0") | |
| echo "small_mismatches=$MISMATCH_COUNT" >> $GITHUB_OUTPUT | |
| echo "failed_count=$FAILED_COUNT" >> $GITHUB_OUTPUT | |
| - name: Fail job if there were test failures | |
| if: steps.check_results.outputs.failed_count > 0 | |
| run: | | |
| echo "Failing job due to test failures." | |
| exit 1 | |
| cli-record: | |
| name: CLI integration tests (record) | |
| needs: cli-verify | |
| if: > | |
| (github.event_name == 'workflow_dispatch' && inputs.mode == 'record') || | |
| (needs.cli-verify.outputs.small_mismatches > 1 && github.event_name != 'pull_request') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| steps: | |
| - name: Harden the runner (Audit all outbound calls) | |
| uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.11" | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| version: "latest" | |
| enable-cache: true | |
| - name: Install dependencies | |
| run: | | |
| make requirements | |
| - name: Verify wikipedia-api command is available | |
| run: | | |
| uv run wikipedia-api --version | |
| - name: Record CLI fixtures | |
| run: | | |
| uv run bash tests/cli/test_cli.sh record | |
| - name: Verify freshly recorded fixtures | |
| run: | | |
| uv run bash tests/cli/test_cli.sh verify | |
| - name: Commit updated fixtures | |
| id: commit | |
| env: | |
| GH_TOKEN: ${{ secrets.BOT_PAT }} | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add tests/cli/expected/ | |
| if git diff --cached --quiet; then | |
| echo "No fixture changes detected." | |
| echo "branch_name=" >> $GITHUB_OUTPUT | |
| echo "existing_pr=false" >> $GITHUB_OUTPUT | |
| else | |
| # Always create a new branch first | |
| BRANCH_NAME="update-cli-fixtures-$(date +%Y%m%d-%H%M%S)" | |
| echo "Creating new branch: $BRANCH_NAME" | |
| git checkout -b $BRANCH_NAME | |
| # Commit the fixture changes to the new branch | |
| echo "Committing fixture changes to new branch" | |
| git commit -m "Update CLI test fixtures [automated]" | |
| # Check if there's an existing open PR for fixture updates | |
| echo "Searching for existing PRs with 'Update CLI test fixtures'..." | |
| EXISTING_PR=$(gh pr list --state open --search "Update CLI test fixtures" --json number --jq '.[0].number' || echo "") | |
| echo "Found existing PR: $EXISTING_PR" | |
| if [ -n "$EXISTING_PR" ]; then | |
| # Update existing PR by overwriting its branch | |
| echo "Updating existing PR #$EXISTING_PR" | |
| EXISTING_BRANCH=$(gh pr view $EXISTING_PR --json headRefName --jq '.headRefName') | |
| echo "Existing branch name: $EXISTING_BRANCH" | |
| # Force push to overwrite the existing branch | |
| echo "Force pushing to overwrite existing branch: origin/$EXISTING_BRANCH" | |
| git push origin $BRANCH_NAME:$EXISTING_BRANCH --force | |
| echo "branch_name=$EXISTING_BRANCH" >> $GITHUB_OUTPUT | |
| echo "existing_pr=true" >> $GITHUB_OUTPUT | |
| else | |
| # Push the new branch | |
| echo "Pushing new branch to origin" | |
| git push origin $BRANCH_NAME | |
| echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT | |
| echo "existing_pr=false" >> $GITHUB_OUTPUT | |
| fi | |
| fi | |
| - name: Create Pull Request | |
| if: steps.commit.outputs.branch_name && steps.commit.outputs.existing_pr == 'false' | |
| env: | |
| GH_TOKEN: ${{ secrets.BOT_PAT }} | |
| run: | | |
| echo "Creating new pull request for branch: ${{ steps.commit.outputs.branch_name }}" | |
| gh pr create \ | |
| --base master \ | |
| --head ${{ steps.commit.outputs.branch_name }} \ | |
| --title "Update CLI test fixtures [automated]" \ | |
| --body "This PR updates CLI test fixtures automatically. Please review and merge if the changes look correct." | |
| echo "Pull request created successfully" |