check.yml

name: iCourse Check

on:
  schedule:
    # Every day at 19:36 BTC (UTC + 8)
    # 设计这个奇怪的时间是为了避免整点时的github队列峰值，避免排队过久。
    - cron: "36 11 * * *"
  workflow_dispatch:

permissions:
  contents: write

jobs:
  check:
    runs-on: ubuntu-latest
    timeout-minutes: 360

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Cache Python packages
        id: cache-pip
        uses: actions/cache@v4
        with:
          path: ${{ env.pythonLocation }}/lib/python*/site-packages
          key: pip-${{ hashFiles('requirements.txt') }}

      - name: Install Python dependencies
        if: steps.cache-pip.outputs.cache-hit != 'true'
        run: pip install -r requirements.txt

      - name: Install ffmpeg
        run: |
          sudo apt-get update
          sudo apt-get install -y ffmpeg

      - name: Cache ASR models
        id: cache-models
        uses: actions/cache@v4
        with:
          path: |
            sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/
            silero_vad.onnx
          key: asr-models-sense-voice-2024-07-17

      - name: Download ASR models
        if: steps.cache-models.outputs.cache-hit != 'true'
        run: |
          wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
          tar xf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
          rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
          wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx

      - name: Cache RapidOCR models
        uses: actions/cache@v4
        with:
          path: ~/.cache/rapidocr
          key: rapidocr-onnx-default-v1

      - name: Fetch database from data branch
        run: |
          mkdir -p data
          git fetch origin data --depth=1 2>/dev/null || echo "No data branch yet."
          if ! git rev-parse origin/data >/dev/null 2>&1; then
            echo "No data branch, starting fresh."
            exit 0
          fi
          # Prefer the sharded layout (current).
          if git cat-file -e origin/data:data/icourse-index.enc 2>/dev/null; then
            mkdir -p data/sharded/shards
            git show origin/data:data/icourse-index.enc > data/sharded/icourse-index.enc
            shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true)
            for path in $shard_paths; do
              base=$(basename "$path")
              git show "origin/data:$path" > "data/sharded/shards/$base"
            done
            echo "Loaded $(ls data/sharded/shards 2>/dev/null | wc -l) shard(s)."
          elif git cat-file -e origin/data:data/icourse.db.gz.enc 2>/dev/null; then
            git show origin/data:data/icourse.db.gz.enc > data/icourse.db.gz.enc
            echo "Loaded legacy compressed database."
          elif git cat-file -e origin/data:data/icourse.db.enc 2>/dev/null; then
            git show origin/data:data/icourse.db.enc > data/icourse.db.enc
            echo "Loaded legacy uncompressed database."
          else
            echo "No database found on data branch, starting fresh."
          fi

      - name: Decode database
        if: hashFiles('data/sharded/icourse-index.enc', 'data/icourse.db.gz.enc', 'data/icourse.db.enc') != ''
        env:
          STUID: ${{ secrets.STUID }}
          UISPSW: ${{ secrets.UISPSW }}
          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
        run: |
          if [ -f data/sharded/icourse-index.enc ]; then
            # Sharded format — always v2.
            python scripts/db_shard.py reassemble data/sharded data/icourse.db
            echo "Reassembled sharded database."
            exit 0
          fi

          # Legacy single-file path (one-time migration on first sharded run).
          NEW_KEY=$(printf '%s' "ICSv2:${STUID}:${UISPSW}" | sha256sum | cut -d' ' -f1)
          LEGACY_KEY="${STUID}${UISPSW}${DASHSCOPE_API_KEY}${SMTP_PASSWORD}"

          decrypt_blob() {
            local in_path="$1"
            local out_path="$2"
            if openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \
                 -in "$in_path" -out "$out_path" \
                 -pass pass:"$NEW_KEY" 2>/dev/null; then
              echo "v2"
              return 0
            fi
            if openssl enc -aes-256-cbc -d -pbkdf2 \
                 -in "$in_path" -out "$out_path" \
                 -pass pass:"$LEGACY_KEY" 2>/dev/null; then
              echo "legacy"
              return 0
            fi
            return 1
          }

          if [ -f data/icourse.db.gz.enc ]; then
            if VER=$(decrypt_blob data/icourse.db.gz.enc data/icourse.db.gz); then
              gzip -d data/icourse.db.gz
              echo "Legacy compressed DB decrypted ($VER)."
            else
              echo "::warning::Decryption failed (both keys). Starting with fresh database."
              rm -f data/icourse.db data/icourse.db.gz
            fi
          elif [ -f data/icourse.db.enc ]; then
            if VER=$(decrypt_blob data/icourse.db.enc data/icourse.db); then
              echo "Legacy uncompressed DB decrypted ($VER)."
            else
              echo "::warning::Decryption failed (both keys). Starting with fresh database."
              rm -f data/icourse.db
            fi
          fi

      - name: Record database state
        id: db-before
        run: |
          if [ -f data/icourse.db ]; then
            echo "hash=$(md5sum data/icourse.db | cut -d' ' -f1)" >> $GITHUB_OUTPUT
          else
            echo "hash=none" >> $GITHUB_OUTPUT
          fi

      - name: Run
        timeout-minutes: 350
        env:
          StuId: ${{ secrets.STUID }}
          UISPsw: ${{ secrets.UISPSW }}
          COURSE_IDS: ${{ secrets.COURSE_IDS }}
          CRAWL_TERM: ${{ secrets.CRAWL_TERM }}
          RESUMMARIZE_OLD: "1"
          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          SMTP_EMAIL: ${{ secrets.SMTP_EMAIL }}
          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
          RECEIVER_EMAIL: ${{ secrets.RECEIVER_EMAIL }}
        run: python -u main.py

      - name: Deploy database to data branch
        if: always()
        env:
          STUID: ${{ secrets.STUID }}
          UISPSW: ${{ secrets.UISPSW }}
          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if [ ! -f data/icourse.db ]; then
            echo "No database file, skipping."
            exit 0
          fi

          AFTER=$(md5sum data/icourse.db | cut -d' ' -f1)
          if [ "$AFTER" = "${{ steps.db-before.outputs.hash }}" ]; then
            echo "Database unchanged, skipping commit."
            exit 0
          fi

          # Pull the latest remote shards (a concurrent run may have written
          # new data) and merge our local changes in before re-sharding.
          git fetch origin data --depth=1 2>/dev/null || echo "No data branch yet."
          DEPLOY_DB=data/icourse.db
          if git rev-parse origin/data >/dev/null 2>&1 && \
             git cat-file -e origin/data:data/icourse-index.enc 2>/dev/null; then
            rm -rf data/remote_sharded
            mkdir -p data/remote_sharded/shards
            git show origin/data:data/icourse-index.enc > data/remote_sharded/icourse-index.enc
            shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true)
            for path in $shard_paths; do
              base=$(basename "$path")
              git show "origin/data:$path" > "data/remote_sharded/shards/$base"
            done
            python scripts/db_shard.py reassemble data/remote_sharded data/remote.db
            echo "Merging local changes into latest remote..."
            python scripts/merge_db.py data/icourse.db data/remote.db
            DEPLOY_DB=data/remote.db
          fi

          # Re-shard the merged DB and stage to a clean tree for force-push.
          rm -rf data/sharded_out
          mkdir -p data/sharded_out
          python scripts/db_shard.py shard "$DEPLOY_DB" data/sharded_out

          rm -rf /tmp/db_deploy
          mkdir -p /tmp/db_deploy/data/shards
          cp data/sharded_out/icourse-index.enc /tmp/db_deploy/data/
          cp data/sharded_out/shards/*.db.gz.enc /tmp/db_deploy/data/shards/
          cd /tmp/db_deploy

          git init -q
          git checkout -q -b data
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add data/
          git commit -q -m "chore: update encrypted database (sharded v2)"
          git push --force "https://${{ github.actor }}:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" data