Fix submit_eval_jobs.py for olmo-eval-internal runs #4812
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docs | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - main | |
| - benchmark | |
| pull_request: | |
| merge_group: | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| check-diff: | |
| name: Check diff | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -euo pipefail {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.x | |
| - name: Build docs from PR branch | |
| run: | | |
| uv sync --group dev --no-install-project | |
| uv run mkdocs build | |
| cp -r site site-pr | |
| - name: Build docs from base branch | |
| run: | | |
| git fetch origin ${{ github.base_ref }} | |
| git checkout origin/${{ github.base_ref }} | |
| uv sync --group dev --no-install-project | |
| uv run mkdocs build | |
| cp -r site site-base | |
| - name: Compare docs and comment if different | |
| run: | | |
| if ! diff -rq site-pr site-base > /dev/null 2>&1; then | |
| echo "DOCS_CHANGED=true" >> $GITHUB_ENV | |
| echo "## Documentation Changes Detected" > comment.md | |
| echo "" >> comment.md | |
| # Get list of changed files, excluding verbose files | |
| # diff returns exit code 1 when differences exist; || true prevents pipefail from failing the script | |
| changed_files=$(diff -rq site-pr site-base | grep "^Files" | grep "differ$" | grep -v "search_index.json" | head -5 || true) | |
| if [ -n "$changed_files" ]; then | |
| file_count=0 | |
| echo "$changed_files" | while IFS= read -r line; do | |
| file_count=$((file_count + 1)) | |
| # Extract file paths from "Files site-pr/path and site-base/path differ" | |
| file1=$(echo "$line" | awk '{print $2}') | |
| file2=$(echo "$line" | awk '{print $4}') | |
| filename=$(echo "$file1" | sed 's|site-pr/||') | |
| echo "<details>" >> comment.md | |
| echo "<summary>📄 <code>$filename</code></summary>" >> comment.md | |
| echo "" >> comment.md | |
| echo "\`\`\`diff" >> comment.md | |
| diff -u "$file2" "$file1" | head -10 >> comment.md || true | |
| echo "\`\`\`" >> comment.md | |
| echo "</details>" >> comment.md | |
| echo "" >> comment.md | |
| # Check comment size and break if too large | |
| if [ $(wc -c < comment.md) -gt 60000 ]; then | |
| echo "" >> comment.md | |
| echo "_Comment truncated due to size limits. Review remaining changes locally._" >> comment.md | |
| break | |
| fi | |
| done | |
| fi | |
| echo "" >> comment.md | |
| echo "_Showing first 10 lines of diff for each changed file (up to 5 files, excluding search indices)._" >> comment.md | |
| else | |
| echo "DOCS_CHANGED=false" >> $GITHUB_ENV | |
| fi | |
| - name: Post comment | |
| if: env.DOCS_CHANGED == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| let comment = fs.readFileSync('comment.md', 'utf8'); | |
| // GitHub comment size limit is 65536 characters | |
| const MAX_SIZE = 65000; | |
| if (comment.length > MAX_SIZE) { | |
| comment = comment.substring(0, MAX_SIZE) + '\n\n_[Comment truncated due to size limits]_'; | |
| } | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); | |
| deploy: | |
| name: Deploy | |
| if: github.event_name == 'push' || github.event_name == 'merge_group' | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -euo pipefail {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Configure Git Credentials | |
| run: | | |
| git config user.name github-actions[bot] | |
| git config user.email 41898282+github-actions[bot]@users.noreply.github.com | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.x | |
| - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV | |
| - uses: actions/cache@v4 | |
| with: | |
| key: mkdocs-material-${{ env.cache_id }} | |
| path: .cache | |
| restore-keys: | | |
| mkdocs-material- | |
| - run: uv sync --group dev --no-install-project | |
| - run: uv run mkdocs gh-deploy --force |