iCourse Check #166
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: iCourse Check | |
| on: | |
| schedule: | |
| # Every day at 19:36 BTC (UTC + 8) | |
| # 设计这个奇怪的时间是为了避免整点时的github队列峰值，避免排队过久。 | |
| - cron: "36 11 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Cache Python packages | |
| id: cache-pip | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.pythonLocation }}/lib/python*/site-packages | |
| key: pip-${{ hashFiles('requirements.txt') }} | |
| - name: Install Python dependencies | |
| if: steps.cache-pip.outputs.cache-hit != 'true' | |
| run: pip install -r requirements.txt | |
| - name: Install ffmpeg | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y ffmpeg | |
| - name: Cache ASR models | |
| id: cache-models | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/ | |
| silero_vad.onnx | |
| key: asr-models-sense-voice-2024-07-17 | |
| - name: Download ASR models | |
| if: steps.cache-models.outputs.cache-hit != 'true' | |
| run: | | |
| wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 | |
| tar xf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 | |
| rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 | |
| wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx | |
| - name: Cache RapidOCR models | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/rapidocr | |
| key: rapidocr-onnx-default-v1 | |
| - name: Fetch database from data branch | |
| run: | | |
| mkdir -p data | |
| # Force-fetch (+refspec) so non-fast-forward updates from | |
| # force-pushed data branches are picked up (shallow clone | |
| # otherwise refuses to overwrite the stale tracking ref). | |
| git fetch --depth=1 origin "+refs/heads/data:refs/remotes/origin/data" 2>/dev/null \ | |
| || { echo "No data branch, starting fresh."; exit 0; } | |
| # Use ls-tree (reads tree directly) instead of cat-file | |
| # (requires fetched blobs) in case the shallow clone hasn't | |
| # fetched every blob object yet. | |
| git ls-tree origin/data --name-only data/ | grep -q . \ | |
| || { echo "No database found on data branch, starting fresh."; exit 0; } | |
| # Determine format from tree listing (ls-tree reads tree | |
| # objects directly without needing the blobs fetched). | |
| files=$(git ls-tree origin/data --name-only data/ | tr '\n' ' ') | |
| echo "Data branch files: $files" | |
| if echo "$files" | grep -q "icourse-index.enc"; then | |
| mkdir -p data/sharded/shards | |
| git show origin/data:data/icourse-index.enc > data/sharded/icourse-index.enc | |
| shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true) | |
| for path in $shard_paths; do | |
| base=$(basename "$path") | |
| git show "origin/data:$path" > "data/sharded/shards/$base" | |
| done | |
| echo "Loaded $(ls data/sharded/shards 2>/dev/null | wc -l) shard(s)." | |
| elif echo "$files" | grep -q "icourse.db.gz.enc"; then | |
| git show origin/data:data/icourse.db.gz.enc > data/icourse.db.gz.enc | |
| echo "Loaded legacy compressed database." | |
| elif echo "$files" | grep -q "icourse.db.enc"; then | |
| git show origin/data:data/icourse.db.enc > data/icourse.db.enc | |
| echo "Loaded legacy uncompressed database." | |
| else | |
| echo "No database found on data branch, starting fresh." | |
| fi | |
| - name: Decode database | |
| if: hashFiles('data/sharded/icourse-index.enc', 'data/icourse.db.gz.enc', 'data/icourse.db.enc') != '' | |
| env: | |
| STUID: ${{ secrets.STUID }} | |
| UISPSW: ${{ secrets.UISPSW }} | |
| DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} | |
| SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} | |
| run: | | |
| if [ -f data/sharded/icourse-index.enc ]; then | |
| # Sharded format — always v2. | |
| python scripts/db_shard.py reassemble data/sharded data/icourse.db | |
| echo "Reassembled sharded database." | |
| exit 0 | |
| fi | |
| # Legacy single-file path (one-time migration on first sharded run). | |
| NEW_KEY=$(printf '%s' "ICSv2:${STUID}:${UISPSW}" | sha256sum | cut -d' ' -f1) | |
| LEGACY_KEY="${STUID}${UISPSW}${DASHSCOPE_API_KEY}${SMTP_PASSWORD}" | |
| decrypt_blob() { | |
| local in_path="$1" | |
| local out_path="$2" | |
| if openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \ | |
| -in "$in_path" -out "$out_path" \ | |
| -pass pass:"$NEW_KEY" 2>/dev/null; then | |
| echo "v2" | |
| return 0 | |
| fi | |
| if openssl enc -aes-256-cbc -d -pbkdf2 \ | |
| -in "$in_path" -out "$out_path" \ | |
| -pass pass:"$LEGACY_KEY" 2>/dev/null; then | |
| echo "legacy" | |
| return 0 | |
| fi | |
| return 1 | |
| } | |
| if [ -f data/icourse.db.gz.enc ]; then | |
| if VER=$(decrypt_blob data/icourse.db.gz.enc data/icourse.db.gz); then | |
| gzip -d data/icourse.db.gz | |
| echo "Legacy compressed DB decrypted ($VER)." | |
| else | |
| echo "::warning::Decryption failed (both keys). Starting with fresh database." | |
| rm -f data/icourse.db data/icourse.db.gz | |
| fi | |
| elif [ -f data/icourse.db.enc ]; then | |
| if VER=$(decrypt_blob data/icourse.db.enc data/icourse.db); then | |
| echo "Legacy uncompressed DB decrypted ($VER)." | |
| else | |
| echo "::warning::Decryption failed (both keys). Starting with fresh database." | |
| rm -f data/icourse.db | |
| fi | |
| fi | |
| - name: Record database state | |
| id: db-before | |
| run: | | |
| if [ -f data/icourse.db ]; then | |
| echo "hash=$(md5sum data/icourse.db | cut -d' ' -f1)" >> $GITHUB_OUTPUT | |
| else | |
| echo "hash=none" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Run | |
| timeout-minutes: 350 | |
| env: | |
| StuId: ${{ secrets.STUID }} | |
| UISPsw: ${{ secrets.UISPSW }} | |
| COURSE_IDS: ${{ secrets.COURSE_IDS }} | |
| CRAWL_TERM: ${{ secrets.CRAWL_TERM }} | |
| RESUMMARIZE_OLD: "1" | |
| DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| SMTP_EMAIL: ${{ secrets.SMTP_EMAIL }} | |
| SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} | |
| RECEIVER_EMAIL: ${{ secrets.RECEIVER_EMAIL }} | |
| run: python -u main.py | |
| - name: Deploy database to data branch | |
| if: always() | |
| env: | |
| STUID: ${{ secrets.STUID }} | |
| UISPSW: ${{ secrets.UISPSW }} | |
| COURSE_IDS: ${{ secrets.COURSE_IDS }} | |
| # Persisted by the sharder into meta('subscribed_course_ids') | |
| # so single_run.yml / reshard.yml (which run with a different | |
| # COURSE_IDS) still know the user's real subscription and | |
| # don't reclassify everything as cold. | |
| SUBSCRIBED_COURSE_IDS: ${{ secrets.COURSE_IDS }} | |
| DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} | |
| SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [ ! -f data/icourse.db ]; then | |
| echo "No database file, skipping." | |
| exit 0 | |
| fi | |
| AFTER=$(md5sum data/icourse.db | cut -d' ' -f1) | |
| if [ "$AFTER" = "${{ steps.db-before.outputs.hash }}" ]; then | |
| echo "Database unchanged, skipping commit." | |
| exit 0 | |
| fi | |
| # Pull the latest remote shards (a concurrent run may have written | |
| # new data) and merge our local changes in before re-sharding. | |
| git fetch origin data --depth=1 2>/dev/null || echo "No data branch yet." | |
| DEPLOY_DB=data/icourse.db | |
| if git rev-parse origin/data >/dev/null 2>&1 && \ | |
| git cat-file -e origin/data:data/icourse-index.enc 2>/dev/null; then | |
| rm -rf data/remote_sharded | |
| mkdir -p data/remote_sharded/shards | |
| git show origin/data:data/icourse-index.enc > data/remote_sharded/icourse-index.enc | |
| shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true) | |
| for path in $shard_paths; do | |
| base=$(basename "$path") | |
| git show "origin/data:$path" > "data/remote_sharded/shards/$base" | |
| done | |
| python scripts/db_shard.py reassemble data/remote_sharded data/remote.db | |
| echo "Merging local changes into latest remote..." | |
| python scripts/merge_db.py data/icourse.db data/remote.db | |
| DEPLOY_DB=data/remote.db | |
| fi | |
| # Re-shard the merged DB and stage to a clean tree for force-push. | |
| rm -rf data/sharded_out | |
| mkdir -p data/sharded_out | |
| python scripts/db_shard.py shard "$DEPLOY_DB" data/sharded_out | |
| rm -rf /tmp/db_deploy | |
| mkdir -p /tmp/db_deploy/data/shards | |
| cp data/sharded_out/icourse-index.enc /tmp/db_deploy/data/ | |
| cp data/sharded_out/shards/*.db.gz.enc /tmp/db_deploy/data/shards/ | |
| cd /tmp/db_deploy | |
| git init -q | |
| git checkout -q -b data | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add data/ | |
| git commit -q -m "chore: update encrypted database (sharded v2)" | |
| git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" | |
| git push --force origin data |