iCourse Check #135
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: iCourse Check | |
| on: | |
| schedule: | |
| # Every day at 19:36 BTC (UTC + 8) | |
| # 设计这个奇怪的时间是为了避免整点时的github队列峰值，避免排队过久。 | |
| - cron: "36 11 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Cache Python packages | |
| id: cache-pip | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.pythonLocation }}/lib/python*/site-packages | |
| key: pip-${{ hashFiles('requirements.txt') }} | |
| - name: Install Python dependencies | |
| if: steps.cache-pip.outputs.cache-hit != 'true' | |
| run: pip install -r requirements.txt | |
| - name: Install ffmpeg | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y ffmpeg | |
| - name: Cache ASR models | |
| id: cache-models | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/ | |
| silero_vad.onnx | |
| key: asr-models-sense-voice-2024-07-17 | |
| - name: Download ASR models | |
| if: steps.cache-models.outputs.cache-hit != 'true' | |
| run: | | |
| wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 | |
| tar xf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 | |
| rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2 | |
| wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx | |
| - name: Cache RapidOCR models | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/rapidocr | |
| key: rapidocr-onnx-default-v1 | |
| - name: Fetch database from data branch | |
| run: | | |
| mkdir -p data | |
| git fetch origin data --depth=1 2>/dev/null || echo "No data branch yet." | |
| if ! git rev-parse origin/data >/dev/null 2>&1; then | |
| echo "No data branch, starting fresh." | |
| exit 0 | |
| fi | |
| # Prefer the sharded layout (current). | |
| if git cat-file -e origin/data:data/icourse-index.enc 2>/dev/null; then | |
| mkdir -p data/sharded/shards | |
| git show origin/data:data/icourse-index.enc > data/sharded/icourse-index.enc | |
| shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true) | |
| for path in $shard_paths; do | |
| base=$(basename "$path") | |
| git show "origin/data:$path" > "data/sharded/shards/$base" | |
| done | |
| echo "Loaded $(ls data/sharded/shards 2>/dev/null | wc -l) shard(s)." | |
| elif git cat-file -e origin/data:data/icourse.db.gz.enc 2>/dev/null; then | |
| git show origin/data:data/icourse.db.gz.enc > data/icourse.db.gz.enc | |
| echo "Loaded legacy compressed database." | |
| elif git cat-file -e origin/data:data/icourse.db.enc 2>/dev/null; then | |
| git show origin/data:data/icourse.db.enc > data/icourse.db.enc | |
| echo "Loaded legacy uncompressed database." | |
| else | |
| echo "No database found on data branch, starting fresh." | |
| fi | |
| - name: Decode database | |
| if: hashFiles('data/sharded/icourse-index.enc', 'data/icourse.db.gz.enc', 'data/icourse.db.enc') != '' | |
| env: | |
| STUID: ${{ secrets.STUID }} | |
| UISPSW: ${{ secrets.UISPSW }} | |
| DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} | |
| SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} | |
| run: | | |
| if [ -f data/sharded/icourse-index.enc ]; then | |
| # Sharded format — always v2. | |
| python scripts/db_shard.py reassemble data/sharded data/icourse.db | |
| echo "Reassembled sharded database." | |
| exit 0 | |
| fi | |
| # Legacy single-file path (one-time migration on first sharded run). | |
| NEW_KEY=$(printf '%s' "ICSv2:${STUID}:${UISPSW}" | sha256sum | cut -d' ' -f1) | |
| LEGACY_KEY="${STUID}${UISPSW}${DASHSCOPE_API_KEY}${SMTP_PASSWORD}" | |
| decrypt_blob() { | |
| local in_path="$1" | |
| local out_path="$2" | |
| if openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \ | |
| -in "$in_path" -out "$out_path" \ | |
| -pass pass:"$NEW_KEY" 2>/dev/null; then | |
| echo "v2" | |
| return 0 | |
| fi | |
| if openssl enc -aes-256-cbc -d -pbkdf2 \ | |
| -in "$in_path" -out "$out_path" \ | |
| -pass pass:"$LEGACY_KEY" 2>/dev/null; then | |
| echo "legacy" | |
| return 0 | |
| fi | |
| return 1 | |
| } | |
| if [ -f data/icourse.db.gz.enc ]; then | |
| if VER=$(decrypt_blob data/icourse.db.gz.enc data/icourse.db.gz); then | |
| gzip -d data/icourse.db.gz | |
| echo "Legacy compressed DB decrypted ($VER)." | |
| else | |
| echo "::warning::Decryption failed (both keys). Starting with fresh database." | |
| rm -f data/icourse.db data/icourse.db.gz | |
| fi | |
| elif [ -f data/icourse.db.enc ]; then | |
| if VER=$(decrypt_blob data/icourse.db.enc data/icourse.db); then | |
| echo "Legacy uncompressed DB decrypted ($VER)." | |
| else | |
| echo "::warning::Decryption failed (both keys). Starting with fresh database." | |
| rm -f data/icourse.db | |
| fi | |
| fi | |
| - name: Record database state | |
| id: db-before | |
| run: | | |
| if [ -f data/icourse.db ]; then | |
| echo "hash=$(md5sum data/icourse.db | cut -d' ' -f1)" >> $GITHUB_OUTPUT | |
| else | |
| echo "hash=none" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Run | |
| timeout-minutes: 350 | |
| env: | |
| StuId: ${{ secrets.STUID }} | |
| UISPsw: ${{ secrets.UISPSW }} | |
| COURSE_IDS: ${{ secrets.COURSE_IDS }} | |
| CRAWL_TERM: ${{ secrets.CRAWL_TERM }} | |
| RESUMMARIZE_OLD: "1" | |
| DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| SMTP_EMAIL: ${{ secrets.SMTP_EMAIL }} | |
| SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} | |
| RECEIVER_EMAIL: ${{ secrets.RECEIVER_EMAIL }} | |
| run: python -u main.py | |
| - name: Deploy database to data branch | |
| if: always() | |
| env: | |
| STUID: ${{ secrets.STUID }} | |
| UISPSW: ${{ secrets.UISPSW }} | |
| DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} | |
| SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [ ! -f data/icourse.db ]; then | |
| echo "No database file, skipping." | |
| exit 0 | |
| fi | |
| AFTER=$(md5sum data/icourse.db | cut -d' ' -f1) | |
| if [ "$AFTER" = "${{ steps.db-before.outputs.hash }}" ]; then | |
| echo "Database unchanged, skipping commit." | |
| exit 0 | |
| fi | |
| # Pull the latest remote shards (a concurrent run may have written | |
| # new data) and merge our local changes in before re-sharding. | |
| git fetch origin data --depth=1 2>/dev/null || echo "No data branch yet." | |
| DEPLOY_DB=data/icourse.db | |
| if git rev-parse origin/data >/dev/null 2>&1 && \ | |
| git cat-file -e origin/data:data/icourse-index.enc 2>/dev/null; then | |
| rm -rf data/remote_sharded | |
| mkdir -p data/remote_sharded/shards | |
| git show origin/data:data/icourse-index.enc > data/remote_sharded/icourse-index.enc | |
| shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true) | |
| for path in $shard_paths; do | |
| base=$(basename "$path") | |
| git show "origin/data:$path" > "data/remote_sharded/shards/$base" | |
| done | |
| python scripts/db_shard.py reassemble data/remote_sharded data/remote.db | |
| echo "Merging local changes into latest remote..." | |
| python scripts/merge_db.py data/icourse.db data/remote.db | |
| DEPLOY_DB=data/remote.db | |
| fi | |
| # Re-shard the merged DB and stage to a clean tree for force-push. | |
| rm -rf data/sharded_out | |
| mkdir -p data/sharded_out | |
| python scripts/db_shard.py shard "$DEPLOY_DB" data/sharded_out | |
| rm -rf /tmp/db_deploy | |
| mkdir -p /tmp/db_deploy/data/shards | |
| cp data/sharded_out/icourse-index.enc /tmp/db_deploy/data/ | |
| cp data/sharded_out/shards/*.db.gz.enc /tmp/db_deploy/data/shards/ | |
| cd /tmp/db_deploy | |
| git init -q | |
| git checkout -q -b data | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add data/ | |
| git commit -q -m "chore: update encrypted database (sharded v2)" | |
| git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" | |
| git push --force origin data |