iCourse Subscriber V2

.github/workflows/check.yml

@@ -46,7 +46,7 @@ jobs:
           path: |
             sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/
             silero_vad.onnx
-          key: asr-models-sensevoice-2024-07-17
+          key: asr-models-sense-voice-2024-07-17
 
       - name: Download ASR models
         if: steps.cache-models.outputs.cache-hit != 'true'
@@ -56,46 +56,98 @@ jobs:
           rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
           wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
 
+      - name: Cache RapidOCR models
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/rapidocr
+          key: rapidocr-onnx-default-v1
+
       - name: Fetch database from data branch
         run: |
           mkdir -p data
-          # Try new compressed format first, then fall back to legacy
-          if curl -fL -o data/icourse.db.gz.enc.tmp \
-               "https://raw.githubusercontent.com/${{ github.repository }}/data/data/icourse.db.gz.enc" 2>/dev/null; then
-            mv data/icourse.db.gz.enc.tmp data/icourse.db.gz.enc
-            echo "Loaded compressed database from data branch."
-          elif curl -fL -o data/icourse.db.enc.tmp \
-               "https://raw.githubusercontent.com/${{ github.repository }}/data/data/icourse.db.enc" 2>/dev/null; then
-            mv data/icourse.db.enc.tmp data/icourse.db.enc
-            echo "Loaded legacy (uncompressed) database from data branch."
+          # Force-fetch (+refspec) so non-fast-forward updates from
+          # force-pushed data branches are picked up (shallow clone
+          # otherwise refuses to overwrite the stale tracking ref).
+          git fetch --depth=1 origin "+refs/heads/data:refs/remotes/origin/data" 2>/dev/null \
+            || { echo "No data branch, starting fresh."; exit 0; }
+          # Use ls-tree (reads tree directly) instead of cat-file
+          # (requires fetched blobs) in case the shallow clone hasn't
+          # fetched every blob object yet.
+          git ls-tree origin/data --name-only data/ | grep -q . \
+            || { echo "No database found on data branch, starting fresh."; exit 0; }
+          # Determine format from tree listing (ls-tree reads tree
+          # objects directly without needing the blobs fetched).
+          files=$(git ls-tree origin/data --name-only data/ | tr '\n' ' ')
+          echo "Data branch files: $files"
+          if echo "$files" | grep -q "icourse-index.enc"; then
+            mkdir -p data/sharded/shards
+            git show origin/data:data/icourse-index.enc > data/sharded/icourse-index.enc
+            shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true)
+            for path in $shard_paths; do
+              base=$(basename "$path")
+              git show "origin/data:$path" > "data/sharded/shards/$base"
+            done
+            echo "Loaded $(ls data/sharded/shards 2>/dev/null | wc -l) shard(s)."
+          elif echo "$files" | grep -q "icourse.db.gz.enc"; then
+            git show origin/data:data/icourse.db.gz.enc > data/icourse.db.gz.enc
+            echo "Loaded legacy compressed database."
+          elif echo "$files" | grep -q "icourse.db.enc"; then
+            git show origin/data:data/icourse.db.enc > data/icourse.db.enc
+            echo "Loaded legacy uncompressed database."
           else
             echo "No database found on data branch, starting fresh."
           fi
 
-      - name: Decrypt database
-        if: hashFiles('data/icourse.db.gz.enc') != '' || hashFiles('data/icourse.db.enc') != ''
+      - name: Decode database
+        if: hashFiles('data/sharded/icourse-index.enc', 'data/icourse.db.gz.enc', 'data/icourse.db.enc') != ''
         env:
-          DB_KEY: ${{ secrets.STUID }}${{ secrets.UISPSW }}${{ secrets.DASHSCOPE_API_KEY }}${{ secrets.SMTP_PASSWORD }}
+          STUID: ${{ secrets.STUID }}
+          UISPSW: ${{ secrets.UISPSW }}
+          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
+          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
         run: |
-          if [ -f data/icourse.db.gz.enc ]; then
-            # New format: decrypt then decompress
+          if [ -f data/sharded/icourse-index.enc ]; then
+            # Sharded format — always v2.
+            python scripts/db_shard.py reassemble data/sharded data/icourse.db
+            echo "Reassembled sharded database."
+            exit 0
+          fi
+
+          # Legacy single-file path (one-time migration on first sharded run).
+          NEW_KEY=$(printf '%s' "ICSv2:${STUID}:${UISPSW}" | sha256sum | cut -d' ' -f1)
+          LEGACY_KEY="${STUID}${UISPSW}${DASHSCOPE_API_KEY}${SMTP_PASSWORD}"
+
+          decrypt_blob() {
+            local in_path="$1"
+            local out_path="$2"
+            if openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \
+                 -in "$in_path" -out "$out_path" \
+                 -pass pass:"$NEW_KEY" 2>/dev/null; then
+              echo "v2"
+              return 0
+            fi
             if openssl enc -aes-256-cbc -d -pbkdf2 \
-                 -in data/icourse.db.gz.enc -out data/icourse.db.gz \
-                 -pass env:DB_KEY 2>/dev/null && \
-               gzip -d data/icourse.db.gz; then
-              echo "Database decrypted and decompressed."
+                 -in "$in_path" -out "$out_path" \
+                 -pass pass:"$LEGACY_KEY" 2>/dev/null; then
+              echo "legacy"
+              return 0
+            fi
+            return 1
+          }
+
+          if [ -f data/icourse.db.gz.enc ]; then
+            if VER=$(decrypt_blob data/icourse.db.gz.enc data/icourse.db.gz); then
+              gzip -d data/icourse.db.gz
+              echo "Legacy compressed DB decrypted ($VER)."
             else
-              echo "::warning::Decryption failed. Starting with fresh database."
+              echo "::warning::Decryption failed (both keys). Starting with fresh database."
               rm -f data/icourse.db data/icourse.db.gz
             fi
           elif [ -f data/icourse.db.enc ]; then
-            # Legacy format: decrypt only
-            if openssl enc -aes-256-cbc -d -pbkdf2 \
-                 -in data/icourse.db.enc -out data/icourse.db \
-                 -pass env:DB_KEY 2>/dev/null; then
-              echo "Database decrypted (legacy format)."
+            if VER=$(decrypt_blob data/icourse.db.enc data/icourse.db); then
+              echo "Legacy uncompressed DB decrypted ($VER)."
             else
-              echo "::warning::Decryption failed. Starting with fresh database."
+              echo "::warning::Decryption failed (both keys). Starting with fresh database."
               rm -f data/icourse.db
             fi
           fi
@@ -115,7 +167,10 @@ jobs:
           StuId: ${{ secrets.STUID }}
           UISPsw: ${{ secrets.UISPSW }}
           COURSE_IDS: ${{ secrets.COURSE_IDS }}
+          CRAWL_TERM: ${{ secrets.CRAWL_TERM }}
+          RESUMMARIZE_OLD: "1"
           DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
+          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
           GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
           SMTP_EMAIL: ${{ secrets.SMTP_EMAIL }}
           SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
@@ -125,7 +180,10 @@ jobs:
       - name: Deploy database to data branch
         if: always()
         env:
-          DB_KEY: ${{ secrets.STUID }}${{ secrets.UISPSW }}${{ secrets.DASHSCOPE_API_KEY }}${{ secrets.SMTP_PASSWORD }}
+          STUID: ${{ secrets.STUID }}
+          UISPSW: ${{ secrets.UISPSW }}
+          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
+          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           if [ ! -f data/icourse.db ]; then
@@ -139,48 +197,42 @@ jobs:
             exit 0
           fi
 
-          # Re-fetch latest remote DB (may have been updated by concurrent workflow)
-          DEPLOY_DB=data/icourse.db
+          # Pull the latest remote shards (a concurrent run may have written
+          # new data) and merge our local changes in before re-sharding.
           git fetch origin data --depth=1 2>/dev/null || echo "No data branch yet."
-          if git rev-parse origin/data >/dev/null 2>&1; then
-            # Try new compressed format, then legacy
-            git show origin/data:data/icourse.db.gz.enc > data/remote.gz.enc 2>/dev/null || true
-            git show origin/data:data/icourse.db.enc > data/remote.enc 2>/dev/null || true
-            # Remove empty files from failed git show
-            find data -name 'remote.*' -empty -delete 2>/dev/null || true
-            if [ -s data/remote.gz.enc ] && \
-               openssl enc -aes-256-cbc -d -pbkdf2 \
-                 -in data/remote.gz.enc -out data/remote.db.gz \
-                 -pass env:DB_KEY 2>/dev/null && \
-               gzip -d data/remote.db.gz; then
-              echo "Merging local changes into latest remote database..."
-              python scripts/merge_db.py data/icourse.db data/remote.db
-              DEPLOY_DB=data/remote.db
-            elif [ -s data/remote.enc ] && \
-               openssl enc -aes-256-cbc -d -pbkdf2 \
-                 -in data/remote.enc -out data/remote.db \
-                 -pass env:DB_KEY 2>/dev/null; then
-              echo "Merging local changes into latest remote database (legacy)..."
-              python scripts/merge_db.py data/icourse.db data/remote.db
-              DEPLOY_DB=data/remote.db
-            fi
+          DEPLOY_DB=data/icourse.db
+          if git rev-parse origin/data >/dev/null 2>&1 && \
+             git cat-file -e origin/data:data/icourse-index.enc 2>/dev/null; then
+            rm -rf data/remote_sharded
+            mkdir -p data/remote_sharded/shards
+            git show origin/data:data/icourse-index.enc > data/remote_sharded/icourse-index.enc
+            shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true)
+            for path in $shard_paths; do
+              base=$(basename "$path")
+              git show "origin/data:$path" > "data/remote_sharded/shards/$base"
+            done
+            python scripts/db_shard.py reassemble data/remote_sharded data/remote.db
+            echo "Merging local changes into latest remote..."
+            python scripts/merge_db.py data/icourse.db data/remote.db
+            DEPLOY_DB=data/remote.db
           fi
 
-          # Compress then encrypt (new format)
-          gzip -c "$DEPLOY_DB" > data/icourse.db.gz
-          openssl enc -aes-256-cbc -salt -pbkdf2 \
-            -in data/icourse.db.gz -out data/icourse.db.gz.enc \
-            -pass env:DB_KEY
+          # Re-shard the merged DB and stage to a clean tree for force-push.
+          rm -rf data/sharded_out
+          mkdir -p data/sharded_out
+          python scripts/db_shard.py shard "$DEPLOY_DB" data/sharded_out
 
-          echo "Deploying to data branch..."
-          mkdir -p /tmp/db_deploy/data
-          cp data/icourse.db.gz.enc /tmp/db_deploy/data/
+          rm -rf /tmp/db_deploy
+          mkdir -p /tmp/db_deploy/data/shards
+          cp data/sharded_out/icourse-index.enc /tmp/db_deploy/data/
+          cp data/sharded_out/shards/*.db.gz.enc /tmp/db_deploy/data/shards/
           cd /tmp/db_deploy
 
-          git init
-          git checkout -b data
+          git init -q
+          git checkout -q -b data
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add data/icourse.db.gz.enc
-          git commit -m "chore: update encrypted database"
-          git push --force "https://${{ github.actor }}:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" data
+          git add data/
+          git commit -q -m "chore: update encrypted database (sharded v2)"
+          git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"
+          git push --force origin data

.github/workflows/deploy-frontend.yml

@@ -5,6 +5,12 @@ on:
     branches: [main]
     paths: ['frontend/**']
   workflow_dispatch:
+    inputs:
+      branch:
+        description: "Branch to deploy frontend from (e.g. main, dev)"
+        required: true
+        default: "main"
+        type: string
 
 permissions:
   contents: read
@@ -22,7 +28,12 @@ jobs:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
     steps:
+      # On manual dispatch we honour the ``branch`` input; for the auto
+      # push trigger this resolves to '' and actions/checkout falls back
+      # to the triggering ref (i.e. main per the push filter above).
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch || github.ref }}
 
       - name: Setup Pages
         uses: actions/configure-pages@v5

.github/workflows/export.yml

@@ -52,30 +52,61 @@ jobs:
       - name: Fetch database from data branch
         run: |
           mkdir -p data
-          if curl -fL -o data/icourse.db.gz.enc.tmp \
-               "https://raw.githubusercontent.com/${{ github.repository }}/data/data/icourse.db.gz.enc" 2>/dev/null; then
-            mv data/icourse.db.gz.enc.tmp data/icourse.db.gz.enc
-            echo "Loaded compressed database."
-          elif curl -fL -o data/icourse.db.enc.tmp \
-               "https://raw.githubusercontent.com/${{ github.repository }}/data/data/icourse.db.enc" 2>/dev/null; then
-            mv data/icourse.db.enc.tmp data/icourse.db.enc
-            echo "Loaded legacy database."
+          git fetch origin data --depth=1
+          if git cat-file -e origin/data:data/icourse-index.enc 2>/dev/null; then
+            mkdir -p data/sharded/shards
+            git show origin/data:data/icourse-index.enc > data/sharded/icourse-index.enc
+            shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true)
+            for path in $shard_paths; do
+              base=$(basename "$path")
+              git show "origin/data:$path" > "data/sharded/shards/$base"
+            done
+            echo "Loaded $(ls data/sharded/shards 2>/dev/null | wc -l) shard(s)."
+          elif git cat-file -e origin/data:data/icourse.db.gz.enc 2>/dev/null; then
+            git show origin/data:data/icourse.db.gz.enc > data/icourse.db.gz.enc
+            echo "Loaded legacy compressed database."
+          elif git cat-file -e origin/data:data/icourse.db.enc 2>/dev/null; then
+            git show origin/data:data/icourse.db.enc > data/icourse.db.enc
+            echo "Loaded legacy uncompressed database."
           else
             echo "::error::No database found on data branch."
             exit 1
           fi
 
-      - name: Decrypt database
+      - name: Decode database
         env:
-          DB_KEY: ${{ secrets.STUID }}${{ secrets.UISPSW }}${{ secrets.DASHSCOPE_API_KEY }}${{ secrets.SMTP_PASSWORD }}
+          STUID: ${{ secrets.STUID }}
+          UISPSW: ${{ secrets.UISPSW }}
+          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
+          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
         run: |
-          if [ -f data/icourse.db.gz.enc ]; then
+          if [ -f data/sharded/icourse-index.enc ]; then
+            python scripts/db_shard.py reassemble data/sharded data/icourse.db
+            echo "Reassembled sharded database."
+            exit 0
+          fi
+
+          NEW_KEY=$(printf '%s' "ICSv2:${STUID}:${UISPSW}" | sha256sum | cut -d' ' -f1)
+          LEGACY_KEY="${STUID}${UISPSW}${DASHSCOPE_API_KEY}${SMTP_PASSWORD}"
+
+          decrypt_blob() {
+            local in_path="$1"
+            local out_path="$2"
+            if openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \
+                 -in "$in_path" -out "$out_path" \
+                 -pass pass:"$NEW_KEY" 2>/dev/null; then
+              return 0
+            fi
             openssl enc -aes-256-cbc -d -pbkdf2 \
-              -in data/icourse.db.gz.enc -out data/icourse.db.gz -pass env:DB_KEY
+              -in "$in_path" -out "$out_path" \
+              -pass pass:"$LEGACY_KEY" 2>/dev/null
+          }
+
+          if [ -f data/icourse.db.gz.enc ]; then
+            decrypt_blob data/icourse.db.gz.enc data/icourse.db.gz
             gzip -d data/icourse.db.gz
           else
-            openssl enc -aes-256-cbc -d -pbkdf2 \
-              -in data/icourse.db.enc -out data/icourse.db -pass env:DB_KEY
+            decrypt_blob data/icourse.db.enc data/icourse.db
           fi
           echo "Database decrypted."