notebook-pr.yaml

name: notebook-pr

on:
  pull_request:
    branches: main
    paths: "**/*.ipynb"

env:
  NB_KERNEL: python
  ORG: NeuromatchAcademy
  NMA_REPO: course-content-dl
  NMA_MAIN_BRANCH: main

jobs:

  setup:
    runs-on: ubuntu-latest
    outputs:
      notebooks: ${{ steps.set-matrix.outputs.notebooks }}
      has_notebooks: ${{ steps.set-matrix.outputs.has_notebooks }}
      commit_message: ${{ steps.get-commit.outputs.message }}
      materials_changed: ${{ steps.materials-check.outputs.materials_changed }}
      skip_ci: ${{ steps.check-skip.outputs.skip }}
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          persist-credentials: false
          fetch-depth: 0
          ref: ${{ github.head_ref }}

      - name: Get commit message
        id: get-commit
        run: |
          msg=$(git log -1 --pretty=format:"%s")
          echo "message=$msg" >> $GITHUB_OUTPUT

      - name: Check skip ci
        id: check-skip
        env:
          COMMIT_MESSAGE: ${{ steps.get-commit.outputs.message }}
        run: |
          if [[ "$COMMIT_MESSAGE" == *"skip ci"* ]]; then
            echo "skip=true" >> $GITHUB_OUTPUT
          else
            echo "skip=false" >> $GITHUB_OUTPUT
          fi

      - name: Setup Python environment
        if: steps.check-skip.outputs.skip != 'true'
        uses: ./.github/actions/setup-python-env

      - name: Setup CI tools
        if: steps.check-skip.outputs.skip != 'true'
        uses: ./.github/actions/setup-ci-tools
        with:
          commit-message: ${{ steps.get-commit.outputs.message }}

      - name: Setup rendering dependencies
        if: steps.check-skip.outputs.skip != 'true'
        uses: ./.github/actions/setup-rendering-deps

      - name: Get changed files
        if: steps.check-skip.outputs.skip != 'true'
        id: changed-files
        uses: tj-actions/changed-files@v47.0.6

      - name: Check if materials.yml changed
        id: materials-check
        if: steps.check-skip.outputs.skip != 'true'
        run: |
          if [[ "${{ steps.changed-files.outputs.all_changed_files }}" =~ materials\.yml ]]; then
            echo "materials_changed=true" >> $GITHUB_OUTPUT
          else
            echo "materials_changed=false" >> $GITHUB_OUTPUT
          fi

      - name: Set notebook matrix
        id: set-matrix
        if: steps.check-skip.outputs.skip != 'true'
        run: |
          nbs=$(nmaci select-notebooks ${{ steps.changed-files.outputs.all_changed_files }})

          if [ -z "$nbs" ]; then
            echo "No notebooks to process"
            echo "has_notebooks=false" >> $GITHUB_OUTPUT
            echo "notebooks=[]" >> $GITHUB_OUTPUT
          else
            echo "Notebooks to process: $nbs"
            echo "has_notebooks=true" >> $GITHUB_OUTPUT
            json_array=$(echo "$nbs" | tr ' ' '\n' | jq -R . | jq -s -c .)
            echo "notebooks=$json_array" >> $GITHUB_OUTPUT
          fi

  process:
    needs: setup
    if: needs.setup.outputs.skip_ci != 'true' && needs.setup.outputs.has_notebooks == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 180
    strategy:
      fail-fast: false
      matrix:
        notebook: ${{ fromJson(needs.setup.outputs.notebooks) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          persist-credentials: false
          fetch-depth: 0
          ref: ${{ github.head_ref }}

      - name: Setup Python environment
        uses: ./.github/actions/setup-python-env

      - name: Setup CI tools
        uses: ./.github/actions/setup-ci-tools
        with:
          commit-message: ${{ needs.setup.outputs.commit_message }}

      - name: Setup rendering dependencies
        uses: ./.github/actions/setup-rendering-deps

      - name: Process notebook
        env:
          COMMIT_MESSAGE: ${{ needs.setup.outputs.commit_message }}
          PYTHONUNBUFFERED: "1"
        run: |
          echo "::group::Processing ${{ matrix.notebook }}"
          echo "Started at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          echo "Job timeout: 120 minutes"

          if [[ "$COMMIT_MESSAGE" == *"ci:check"* ]]; then
            execflag="--check-execution"
            echo "Mode: Check execution (no changes)"
          else
            execflag="--execute"
            echo "Mode: Execute and process"
          fi
          echo "::endgroup::"

          echo "::group::Notebook execution output"
          nmaci process-notebooks "${{ matrix.notebook }}" $execflag
          exit_code=$?
          echo "::endgroup::"

          echo "Completed at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          exit $exit_code

      - name: Get tutorial directory
        id: get-dir
        run: |
          nb="${{ matrix.notebook }}"
          dir=$(dirname "$nb")
          nb_name=$(basename "$nb" .ipynb)
          echo "dir=$dir" >> $GITHUB_OUTPUT
          echo "nb_name=$nb_name" >> $GITHUB_OUTPUT
          safe_dir=$(echo "$dir" | sed 's|/|___|g')
          echo "artifact_name=${safe_dir}___${nb_name}" >> $GITHUB_OUTPUT

      - name: Upload processed tutorial directory
        uses: actions/upload-artifact@v5
        with:
          name: tutorial-${{ steps.get-dir.outputs.artifact_name }}
          path: |
            ${{ matrix.notebook }}
            ${{ steps.get-dir.outputs.dir }}/static/${{ steps.get-dir.outputs.nb_name }}*
            ${{ steps.get-dir.outputs.dir }}/solutions/${{ steps.get-dir.outputs.nb_name }}*
            ${{ steps.get-dir.outputs.dir }}/student/${{ steps.get-dir.outputs.nb_name }}*
            ${{ steps.get-dir.outputs.dir }}/instructor/${{ steps.get-dir.outputs.nb_name }}*
          if-no-files-found: ignore
          retention-days: 1

  finalize:
    needs: [setup, process]
    if: always() && needs.setup.outputs.skip_ci != 'true' && needs.setup.outputs.has_notebooks == 'true'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          persist-credentials: false
          fetch-depth: 0
          ref: ${{ github.head_ref }}

      - name: Setup Python environment
        uses: ./.github/actions/setup-python-env

      - name: Setup CI tools
        uses: ./.github/actions/setup-ci-tools
        with:
          commit-message: ${{ needs.setup.outputs.commit_message }}

      - name: Download all artifacts
        uses: actions/download-artifact@v5
        with:
          path: artifacts/

      - name: Detect failed notebooks
        id: detect-failures
        run: |
          expected='${{ needs.setup.outputs.notebooks }}'
          failed_nbs=""
          success_nbs=""

          echo "Checking which notebooks were successfully processed..."
          for nb in $(echo "$expected" | jq -r '.[]'); do
            nb_name=$(basename "$nb" .ipynb)
            if ls artifacts/tutorial-*"___${nb_name}" 1>/dev/null 2>&1; then
              echo "  ✓ $nb - SUCCESS"
              success_nbs="$success_nbs $nb"
            else
              echo "  ✗ $nb - FAILED (no artifact)"
              failed_nbs="$failed_nbs $nb"
            fi
          done

          success_nbs="${success_nbs# }"
          failed_nbs="${failed_nbs# }"

          echo "success_notebooks=$success_nbs" >> $GITHUB_OUTPUT
          echo "failed_notebooks=$failed_nbs" >> $GITHUB_OUTPUT

          if [ -n "$failed_nbs" ]; then
            echo "has_failures=true" >> $GITHUB_OUTPUT
            echo ""
            echo "⚠️  Some notebooks failed to process: $failed_nbs"
          else
            echo "has_failures=false" >> $GITHUB_OUTPUT
            echo ""
            echo "✅ All notebooks processed successfully"
          fi

      - name: Collect failure context
        if: steps.detect-failures.outputs.has_failures == 'true'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          run_id=${{ github.run_id }}
          echo "## Notebook Processing Failure Context" > failure_context.md
          echo "" >> failure_context.md
          jobs_json=$(gh api repos/${{ github.repository }}/actions/runs/${run_id}/jobs)
          for nb in ${{ steps.detect-failures.outputs.failed_notebooks }}; do
            nb_name=$(basename "$nb" .ipynb)
            echo "### $nb" >> failure_context.md
            echo '```' >> failure_context.md
            job_id=$(echo "$jobs_json" | jq -r --arg nb "$nb" '.jobs[] | select(.name | contains($nb)) | .id')
            if [ -n "$job_id" ] && [ "$job_id" != "null" ]; then
              gh api repos/${{ github.repository }}/actions/jobs/${job_id}/logs 2>/dev/null | \
                awk '
                  /##\[group\]Notebook execution output/{capturing=1; next}
                  capturing && /##\[endgroup\]/{exit}
                  capturing && /##\[error\]/{print; exit}
                  capturing && /##\[group\]/{exit}
                  capturing && /Post job cleanup/{exit}
                  capturing{print}
                ' | \
                tail -100 >> failure_context.md || echo "Could not retrieve logs" >> failure_context.md
            else
              echo "Job not found for $nb" >> failure_context.md
            fi
            echo '```' >> failure_context.md
            echo "" >> failure_context.md
          done

      - name: Post failure context comment
        if: steps.detect-failures.outputs.has_failures == 'true'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if [ -f failure_context.md ]; then
            gh pr comment ${{ github.event.pull_request.number }} --body-file failure_context.md
          fi

      - name: Restore processed files
        run: |
          echo "Restoring processed files from artifacts..."

          for dir in artifacts/tutorial-*; do
            if [ -d "$dir" ]; then
              artifact_name=$(basename "$dir" | sed 's/^tutorial-//')
              full_path=$(echo "$artifact_name" | sed 's|___|/|g')
              tutorial_dir=$(dirname "$full_path")
              echo "Restoring from artifact $(basename "$dir") to: $tutorial_dir"
              mkdir -p "$tutorial_dir"
              cp -rv "$dir"/* "$tutorial_dir/" 2>/dev/null || true
            fi
          done

          echo "Restore complete."

      - name: Verify exercises
        env:
          COMMIT_MESSAGE: ${{ needs.setup.outputs.commit_message }}
        run: |
          nbs='${{ needs.setup.outputs.notebooks }}'
          nb_list=$(echo "$nbs" | jq -r '.[]' | tr '\n' ' ')
          nmaci verify-exercises $nb_list --c "$COMMIT_MESSAGE"

      - name: Make PR comment
        run: |
          nbs='${{ needs.setup.outputs.notebooks }}'
          nb_list=$(echo "$nbs" | jq -r '.[]' | tr '\n' ' ')
          branch=${{ github.event.pull_request.head.ref }}
          nmaci make-pr-comment $nb_list --branch $branch --o comment.txt

      - name: Update READMEs
        if: needs.setup.outputs.materials_changed == 'true'
        run: nmaci generate-readmes

      - name: Remove unreferenced derivatives
        run: |
          nmaci find-unreferenced > to_remove.txt
          if [ -s to_remove.txt ]; then git rm --pathspec-from-file=to_remove.txt; fi

      - name: Clean up artifacts directory
        run: rm -rf artifacts/

      - name: Revert failed notebooks to main
        if: steps.detect-failures.outputs.has_failures == 'true' && steps.detect-failures.outputs.success_notebooks != ''
        run: |
          failed_nbs="${{ steps.detect-failures.outputs.failed_notebooks }}"
          echo "Reverting failed notebooks to main branch version: $failed_nbs"

          for nb in $failed_nbs; do
            if git show origin/main:"$nb" > /dev/null 2>&1; then
              git checkout origin/main -- "$nb"
              echo "  Reverted: $nb"
            else
              git rm -f "$nb" 2>/dev/null || rm -f "$nb"
              echo "  Removed (new file): $nb"
            fi
          done

      - name: Create follow-up PR for failed notebooks
        if: steps.detect-failures.outputs.has_failures == 'true' && steps.detect-failures.outputs.success_notebooks != ''
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"

          failed_nbs="${{ steps.detect-failures.outputs.failed_notebooks }}"
          pr_number=${{ github.event.pull_request.number }}
          original_branch=${{ github.head_ref }}
          new_branch="reprocess-pr${pr_number}-$(date +%Y%m%d%H%M%S)"

          git stash --include-untracked
          git fetch origin main
          git checkout -b "$new_branch" origin/main

          for nb in $failed_nbs; do
            git checkout "origin/$original_branch" -- "$nb" 2>/dev/null || echo "Note: $nb may be new in this PR"
          done

          git add $failed_nbs
          git commit -m "Reprocess notebooks from PR #${pr_number}

          These notebooks failed to process in the original PR and need reprocessing:
          $(echo "$failed_nbs" | tr ' ' '\n' | sed 's/^/- /')"

          gh auth setup-git
          git push origin "$new_branch"

          new_pr_url=$(gh pr create \
            --title "Reprocess: Failed notebooks from PR #${pr_number}" \
            --body "## Auto-generated PR

          The following notebooks failed to process in PR #${pr_number} and need reprocessing:

          $(echo "$failed_nbs" | tr ' ' '\n' | sed 's/^/- /')

          This PR was automatically created to allow these notebooks to be processed independently.

          Original PR: #${pr_number}" \
            --base "$original_branch" \
            --head "$new_branch")

          echo "Created follow-up PR: $new_pr_url"

          gh pr comment "$pr_number" --body "## Follow-up PR Created

          A follow-up PR has been created for the notebooks that failed to process:

          **Follow-up PR:** $new_pr_url
          **Branch:** \`$new_branch\`

          The failed notebooks have been reverted to their \`main\` branch versions in this PR.
          Once this PR is merged, the follow-up PR can be used to reprocess the failed notebooks independently."

          git checkout "$original_branch"
          git stash pop || true

      - name: Commit post-processed files
        if: steps.detect-failures.outputs.success_notebooks != ''
        run: |
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          git add '**/*.ipynb'
          git add '**/static/*.png'
          git add '**/solutions/*.py'
          git add '**/student/*.ipynb'
          git add '**/instructor/*.ipynb'
          git add '**/README.md'

          if [ "${{ steps.detect-failures.outputs.has_failures }}" == "true" ]; then
            msg="Process tutorial notebooks (some notebooks moved to follow-up PR)"
          else
            msg="Process tutorial notebooks"
          fi

          git diff-index --quiet HEAD || git commit -m "$msg"

      - name: Push post-processed files
        if: steps.detect-failures.outputs.success_notebooks != ''
        uses: ad-m/github-push-action@v1.3.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          branch: ${{ github.head_ref }}