Skip to content

Commit 56f9946

Browse files
LeafCreeperLeafCreeper
authored
Merge pull request #25 from LeafCreeper/dev
iCourse Subscriber V2
2 parents 501008e + 9b36646 commit 56f9946

50 files changed

Lines changed: 6853 additions & 2118 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/check.yml‎

Lines changed: 116 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
path: |
4747
sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/
4848
silero_vad.onnx
49-
key: asr-models-sensevoice-2024-07-17
49+
key: asr-models-sense-voice-2024-07-17
5050

5151
- name: Download ASR models
5252
if: steps.cache-models.outputs.cache-hit != 'true'
@@ -56,46 +56,98 @@ jobs:
5656
rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17.tar.bz2
5757
wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
5858
59+
- name: Cache RapidOCR models
60+
uses: actions/cache@v4
61+
with:
62+
path: ~/.cache/rapidocr
63+
key: rapidocr-onnx-default-v1
64+
5965
- name: Fetch database from data branch
6066
run: |
6167
mkdir -p data
62-
# Try new compressed format first, then fall back to legacy
63-
if curl -fL -o data/icourse.db.gz.enc.tmp \
64-
"https://raw.githubusercontent.com/${{ github.repository }}/data/data/icourse.db.gz.enc" 2>/dev/null; then
65-
mv data/icourse.db.gz.enc.tmp data/icourse.db.gz.enc
66-
echo "Loaded compressed database from data branch."
67-
elif curl -fL -o data/icourse.db.enc.tmp \
68-
"https://raw.githubusercontent.com/${{ github.repository }}/data/data/icourse.db.enc" 2>/dev/null; then
69-
mv data/icourse.db.enc.tmp data/icourse.db.enc
70-
echo "Loaded legacy (uncompressed) database from data branch."
68+
# Force-fetch (+refspec) so non-fast-forward updates from
69+
# force-pushed data branches are picked up (shallow clone
70+
# otherwise refuses to overwrite the stale tracking ref).
71+
git fetch --depth=1 origin "+refs/heads/data:refs/remotes/origin/data" 2>/dev/null \
72+
|| { echo "No data branch, starting fresh."; exit 0; }
73+
# Use ls-tree (reads tree directly) instead of cat-file
74+
# (requires fetched blobs) in case the shallow clone hasn't
75+
# fetched every blob object yet.
76+
git ls-tree origin/data --name-only data/ | grep -q . \
77+
|| { echo "No database found on data branch, starting fresh."; exit 0; }
78+
# Determine format from tree listing (ls-tree reads tree
79+
# objects directly without needing the blobs fetched).
80+
files=$(git ls-tree origin/data --name-only data/ | tr '\n' ' ')
81+
echo "Data branch files: $files"
82+
if echo "$files" | grep -q "icourse-index.enc"; then
83+
mkdir -p data/sharded/shards
84+
git show origin/data:data/icourse-index.enc > data/sharded/icourse-index.enc
85+
shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true)
86+
for path in $shard_paths; do
87+
base=$(basename "$path")
88+
git show "origin/data:$path" > "data/sharded/shards/$base"
89+
done
90+
echo "Loaded $(ls data/sharded/shards 2>/dev/null | wc -l) shard(s)."
91+
elif echo "$files" | grep -q "icourse.db.gz.enc"; then
92+
git show origin/data:data/icourse.db.gz.enc > data/icourse.db.gz.enc
93+
echo "Loaded legacy compressed database."
94+
elif echo "$files" | grep -q "icourse.db.enc"; then
95+
git show origin/data:data/icourse.db.enc > data/icourse.db.enc
96+
echo "Loaded legacy uncompressed database."
7197
else
7298
echo "No database found on data branch, starting fresh."
7399
fi
74100
75-
- name: Decrypt database
76-
if: hashFiles('data/icourse.db.gz.enc') != '' || hashFiles('data/icourse.db.enc') != ''
101+
- name: Decode database
102+
if: hashFiles('data/sharded/icourse-index.enc', 'data/icourse.db.gz.enc', 'data/icourse.db.enc') != ''
77103
env:
78-
DB_KEY: ${{ secrets.STUID }}${{ secrets.UISPSW }}${{ secrets.DASHSCOPE_API_KEY }}${{ secrets.SMTP_PASSWORD }}
104+
STUID: ${{ secrets.STUID }}
105+
UISPSW: ${{ secrets.UISPSW }}
106+
DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
107+
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
79108
run: |
80-
if [ -f data/icourse.db.gz.enc ]; then
81-
# New format: decrypt then decompress
109+
if [ -f data/sharded/icourse-index.enc ]; then
110+
# Sharded format — always v2.
111+
python scripts/db_shard.py reassemble data/sharded data/icourse.db
112+
echo "Reassembled sharded database."
113+
exit 0
114+
fi
115+
116+
# Legacy single-file path (one-time migration on first sharded run).
117+
NEW_KEY=$(printf '%s' "ICSv2:${STUID}:${UISPSW}" | sha256sum | cut -d' ' -f1)
118+
LEGACY_KEY="${STUID}${UISPSW}${DASHSCOPE_API_KEY}${SMTP_PASSWORD}"
119+
120+
decrypt_blob() {
121+
local in_path="$1"
122+
local out_path="$2"
123+
if openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \
124+
-in "$in_path" -out "$out_path" \
125+
-pass pass:"$NEW_KEY" 2>/dev/null; then
126+
echo "v2"
127+
return 0
128+
fi
82129
if openssl enc -aes-256-cbc -d -pbkdf2 \
83-
-in data/icourse.db.gz.enc -out data/icourse.db.gz \
84-
-pass env:DB_KEY 2>/dev/null && \
85-
gzip -d data/icourse.db.gz; then
86-
echo "Database decrypted and decompressed."
130+
-in "$in_path" -out "$out_path" \
131+
-pass pass:"$LEGACY_KEY" 2>/dev/null; then
132+
echo "legacy"
133+
return 0
134+
fi
135+
return 1
136+
}
137+
138+
if [ -f data/icourse.db.gz.enc ]; then
139+
if VER=$(decrypt_blob data/icourse.db.gz.enc data/icourse.db.gz); then
140+
gzip -d data/icourse.db.gz
141+
echo "Legacy compressed DB decrypted ($VER)."
87142
else
88-
echo "::warning::Decryption failed. Starting with fresh database."
143+
echo "::warning::Decryption failed (both keys). Starting with fresh database."
89144
rm -f data/icourse.db data/icourse.db.gz
90145
fi
91146
elif [ -f data/icourse.db.enc ]; then
92-
# Legacy format: decrypt only
93-
if openssl enc -aes-256-cbc -d -pbkdf2 \
94-
-in data/icourse.db.enc -out data/icourse.db \
95-
-pass env:DB_KEY 2>/dev/null; then
96-
echo "Database decrypted (legacy format)."
147+
if VER=$(decrypt_blob data/icourse.db.enc data/icourse.db); then
148+
echo "Legacy uncompressed DB decrypted ($VER)."
97149
else
98-
echo "::warning::Decryption failed. Starting with fresh database."
150+
echo "::warning::Decryption failed (both keys). Starting with fresh database."
99151
rm -f data/icourse.db
100152
fi
101153
fi
@@ -115,7 +167,10 @@ jobs:
115167
StuId: ${{ secrets.STUID }}
116168
UISPsw: ${{ secrets.UISPSW }}
117169
COURSE_IDS: ${{ secrets.COURSE_IDS }}
170+
CRAWL_TERM: ${{ secrets.CRAWL_TERM }}
171+
RESUMMARIZE_OLD: "1"
118172
DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
173+
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
119174
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
120175
SMTP_EMAIL: ${{ secrets.SMTP_EMAIL }}
121176
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
@@ -125,7 +180,10 @@ jobs:
125180
- name: Deploy database to data branch
126181
if: always()
127182
env:
128-
DB_KEY: ${{ secrets.STUID }}${{ secrets.UISPSW }}${{ secrets.DASHSCOPE_API_KEY }}${{ secrets.SMTP_PASSWORD }}
183+
STUID: ${{ secrets.STUID }}
184+
UISPSW: ${{ secrets.UISPSW }}
185+
DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
186+
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
129187
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
130188
run: |
131189
if [ ! -f data/icourse.db ]; then
@@ -139,48 +197,42 @@ jobs:
139197
exit 0
140198
fi
141199
142-
# Re-fetch latest remote DB (may have been updated by concurrent workflow)
143-
DEPLOY_DB=data/icourse.db
200+
# Pull the latest remote shards (a concurrent run may have written
201+
# new data) and merge our local changes in before re-sharding.
144202
git fetch origin data --depth=1 2>/dev/null || echo "No data branch yet."
145-
if git rev-parse origin/data >/dev/null 2>&1; then
146-
# Try new compressed format, then legacy
147-
git show origin/data:data/icourse.db.gz.enc > data/remote.gz.enc 2>/dev/null || true
148-
git show origin/data:data/icourse.db.enc > data/remote.enc 2>/dev/null || true
149-
# Remove empty files from failed git show
150-
find data -name 'remote.*' -empty -delete 2>/dev/null || true
151-
if [ -s data/remote.gz.enc ] && \
152-
openssl enc -aes-256-cbc -d -pbkdf2 \
153-
-in data/remote.gz.enc -out data/remote.db.gz \
154-
-pass env:DB_KEY 2>/dev/null && \
155-
gzip -d data/remote.db.gz; then
156-
echo "Merging local changes into latest remote database..."
157-
python scripts/merge_db.py data/icourse.db data/remote.db
158-
DEPLOY_DB=data/remote.db
159-
elif [ -s data/remote.enc ] && \
160-
openssl enc -aes-256-cbc -d -pbkdf2 \
161-
-in data/remote.enc -out data/remote.db \
162-
-pass env:DB_KEY 2>/dev/null; then
163-
echo "Merging local changes into latest remote database (legacy)..."
164-
python scripts/merge_db.py data/icourse.db data/remote.db
165-
DEPLOY_DB=data/remote.db
166-
fi
203+
DEPLOY_DB=data/icourse.db
204+
if git rev-parse origin/data >/dev/null 2>&1 && \
205+
git cat-file -e origin/data:data/icourse-index.enc 2>/dev/null; then
206+
rm -rf data/remote_sharded
207+
mkdir -p data/remote_sharded/shards
208+
git show origin/data:data/icourse-index.enc > data/remote_sharded/icourse-index.enc
209+
shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true)
210+
for path in $shard_paths; do
211+
base=$(basename "$path")
212+
git show "origin/data:$path" > "data/remote_sharded/shards/$base"
213+
done
214+
python scripts/db_shard.py reassemble data/remote_sharded data/remote.db
215+
echo "Merging local changes into latest remote..."
216+
python scripts/merge_db.py data/icourse.db data/remote.db
217+
DEPLOY_DB=data/remote.db
167218
fi
168219
169-
# Compress then encrypt (new format)
170-
gzip -c "$DEPLOY_DB" > data/icourse.db.gz
171-
openssl enc -aes-256-cbc -salt -pbkdf2 \
172-
-in data/icourse.db.gz -out data/icourse.db.gz.enc \
173-
-pass env:DB_KEY
220+
# Re-shard the merged DB and stage to a clean tree for force-push.
221+
rm -rf data/sharded_out
222+
mkdir -p data/sharded_out
223+
python scripts/db_shard.py shard "$DEPLOY_DB" data/sharded_out
174224
175-
echo "Deploying to data branch..."
176-
mkdir -p /tmp/db_deploy/data
177-
cp data/icourse.db.gz.enc /tmp/db_deploy/data/
225+
rm -rf /tmp/db_deploy
226+
mkdir -p /tmp/db_deploy/data/shards
227+
cp data/sharded_out/icourse-index.enc /tmp/db_deploy/data/
228+
cp data/sharded_out/shards/*.db.gz.enc /tmp/db_deploy/data/shards/
178229
cd /tmp/db_deploy
179230
180-
git init
181-
git checkout -b data
231+
git init -q
232+
git checkout -q -b data
182233
git config user.name "github-actions[bot]"
183234
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
184-
git add data/icourse.db.gz.enc
185-
git commit -m "chore: update encrypted database"
186-
git push --force "https://${{ github.actor }}:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" data
235+
git add data/
236+
git commit -q -m "chore: update encrypted database (sharded v2)"
237+
git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"
238+
git push --force origin data

‎.github/workflows/deploy-frontend.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ on:
55
branches: [main]
66
paths: ['frontend/**']
77
workflow_dispatch:
8+
inputs:
9+
branch:
10+
description: "Branch to deploy frontend from (e.g. main, dev)"
11+
required: true
12+
default: "main"
13+
type: string
814

915
permissions:
1016
contents: read
@@ -22,7 +28,12 @@ jobs:
2228
name: github-pages
2329
url: ${{ steps.deployment.outputs.page_url }}
2430
steps:
31+
# On manual dispatch we honour the ``branch`` input; for the auto
32+
# push trigger this resolves to '' and actions/checkout falls back
33+
# to the triggering ref (i.e. main per the push filter above).
2534
- uses: actions/checkout@v4
35+
with:
36+
ref: ${{ inputs.branch || github.ref }}
2637

2738
- name: Setup Pages
2839
uses: actions/configure-pages@v5

‎.github/workflows/export.yml‎

Lines changed: 45 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -52,30 +52,61 @@ jobs:
5252
- name: Fetch database from data branch
5353
run: |
5454
mkdir -p data
55-
if curl -fL -o data/icourse.db.gz.enc.tmp \
56-
"https://raw.githubusercontent.com/${{ github.repository }}/data/data/icourse.db.gz.enc" 2>/dev/null; then
57-
mv data/icourse.db.gz.enc.tmp data/icourse.db.gz.enc
58-
echo "Loaded compressed database."
59-
elif curl -fL -o data/icourse.db.enc.tmp \
60-
"https://raw.githubusercontent.com/${{ github.repository }}/data/data/icourse.db.enc" 2>/dev/null; then
61-
mv data/icourse.db.enc.tmp data/icourse.db.enc
62-
echo "Loaded legacy database."
55+
git fetch origin data --depth=1
56+
if git cat-file -e origin/data:data/icourse-index.enc 2>/dev/null; then
57+
mkdir -p data/sharded/shards
58+
git show origin/data:data/icourse-index.enc > data/sharded/icourse-index.enc
59+
shard_paths=$(git ls-tree -r --name-only origin/data | grep '^data/shards/' || true)
60+
for path in $shard_paths; do
61+
base=$(basename "$path")
62+
git show "origin/data:$path" > "data/sharded/shards/$base"
63+
done
64+
echo "Loaded $(ls data/sharded/shards 2>/dev/null | wc -l) shard(s)."
65+
elif git cat-file -e origin/data:data/icourse.db.gz.enc 2>/dev/null; then
66+
git show origin/data:data/icourse.db.gz.enc > data/icourse.db.gz.enc
67+
echo "Loaded legacy compressed database."
68+
elif git cat-file -e origin/data:data/icourse.db.enc 2>/dev/null; then
69+
git show origin/data:data/icourse.db.enc > data/icourse.db.enc
70+
echo "Loaded legacy uncompressed database."
6371
else
6472
echo "::error::No database found on data branch."
6573
exit 1
6674
fi
6775
68-
- name: Decrypt database
76+
- name: Decode database
6977
env:
70-
DB_KEY: ${{ secrets.STUID }}${{ secrets.UISPSW }}${{ secrets.DASHSCOPE_API_KEY }}${{ secrets.SMTP_PASSWORD }}
78+
STUID: ${{ secrets.STUID }}
79+
UISPSW: ${{ secrets.UISPSW }}
80+
DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
81+
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
7182
run: |
72-
if [ -f data/icourse.db.gz.enc ]; then
83+
if [ -f data/sharded/icourse-index.enc ]; then
84+
python scripts/db_shard.py reassemble data/sharded data/icourse.db
85+
echo "Reassembled sharded database."
86+
exit 0
87+
fi
88+
89+
NEW_KEY=$(printf '%s' "ICSv2:${STUID}:${UISPSW}" | sha256sum | cut -d' ' -f1)
90+
LEGACY_KEY="${STUID}${UISPSW}${DASHSCOPE_API_KEY}${SMTP_PASSWORD}"
91+
92+
decrypt_blob() {
93+
local in_path="$1"
94+
local out_path="$2"
95+
if openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \
96+
-in "$in_path" -out "$out_path" \
97+
-pass pass:"$NEW_KEY" 2>/dev/null; then
98+
return 0
99+
fi
73100
openssl enc -aes-256-cbc -d -pbkdf2 \
74-
-in data/icourse.db.gz.enc -out data/icourse.db.gz -pass env:DB_KEY
101+
-in "$in_path" -out "$out_path" \
102+
-pass pass:"$LEGACY_KEY" 2>/dev/null
103+
}
104+
105+
if [ -f data/icourse.db.gz.enc ]; then
106+
decrypt_blob data/icourse.db.gz.enc data/icourse.db.gz
75107
gzip -d data/icourse.db.gz
76108
else
77-
openssl enc -aes-256-cbc -d -pbkdf2 \
78-
-in data/icourse.db.enc -out data/icourse.db -pass env:DB_KEY
109+
decrypt_blob data/icourse.db.enc data/icourse.db
79110
fi
80111
echo "Database decrypted."
81112

0 commit comments

Comments
 (0)