Merge pull request #25 from 5-xj/upstream-pr-safe-all

[update] 更新了多源，现在暴露了bioRxiv，更新了图中插图

Author: ziwenhahaha

.github/workflows/daily-paper-reader.yml

@@ -17,6 +17,10 @@ on:
         description: "抓取模式：auto/standard/skims（auto 按 fetch-days 阈值决定，standard/ skims 强制）"
         required: false
         default: "auto"
+      profile_tag:
+        description: "仅运行指定词条 tag；留空表示全量运行"
+        required: false
+        default: ""
 
 permissions:
   contents: write
@@ -31,10 +35,16 @@ jobs:
     timeout-minutes: 120
     env:
       BLT_API_KEY: ${{ secrets.BLT_API_KEY }}
-      BLT_REWRITE_MODEL: gemini-3-flash-preview
+      BLT_REWRITE_MODEL: ${{ secrets.BLT_REWRITE_MODEL }}
       BLT_RERANK_MODEL: qwen3-reranker-4b
-      BLT_FILTER_MODEL: gemini-3-flash-preview-nothinking
+      BLT_FILTER_MODEL: ${{ secrets.BLT_FILTER_MODEL }}
       DPR_EMBED_API_TIMEOUT: "60"
+      LLM_PRIMARY_BASE_URL: ${{ secrets.LLM_PRIMARY_BASE_URL }}
+      BLT_PRIMARY_BASE_URL: ${{ secrets.BLT_PRIMARY_BASE_URL }}
+      BLT_API_BASE: ${{ secrets.BLT_PRIMARY_BASE_URL }}
+      SUMMARY_API_KEY: ${{ secrets.SUMMARY_API_KEY }}
+      SUMMARY_BASE_URL: ${{ secrets.SUMMARY_BASE_URL }}
+      SUMMARY_MODEL: ${{ secrets.SUMMARY_MODEL }}
       PYTHONUNBUFFERED: "1"
 
     steps:
@@ -55,13 +65,23 @@ jobs:
         with:
           python-version: "3.11"
 
+      - name: Setup Java
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: "17"
+
       - name: Cache Python deps
         uses: actions/cache@v5
         with:
           path: |
             ~/.cache/pip
             ~/.cache/uv
             ~/.cache/torch
+            ~/.cache/dpr-tools/pdffigures2
+            ~/.cache/coursier
+            ~/.ivy2/cache
+            ~/.sbt
           key: ${{ runner.os }}-dpr-embed-deps-v1-${{ hashFiles('requirements.txt') }}
 
       - name: Install deps (skip sqlite3)
@@ -76,6 +96,27 @@ jobs:
           python -m pip install uv
           uv pip install --system -r /tmp/req.txt
 
+      - name: Prepare pdffigures2
+        run: |
+          set -euo pipefail
+
+          TOOL_DIR="$HOME/.cache/dpr-tools/pdffigures2"
+          SRC_DIR="$TOOL_DIR/src"
+          JAR_PATH="$TOOL_DIR/pdffigures2.jar"
+
+          mkdir -p "$TOOL_DIR"
+          if [ ! -f "$JAR_PATH" ]; then
+            rm -rf "$SRC_DIR"
+            git clone --depth 1 https://github.com/allenai/pdffigures2 "$SRC_DIR"
+            cd "$SRC_DIR"
+            curl -fsSL -o sbt.tgz https://github.com/sbt/sbt/releases/download/v1.10.1/sbt-1.10.1.tgz
+            tar -xzf sbt.tgz
+            ./sbt/bin/sbt assembly
+            cp pdffigures2.jar "$JAR_PATH"
+          fi
+
+          echo "PDFFIGURES2_JAR=$JAR_PATH" >> "$GITHUB_ENV"
+
       - name: Run Retrieval and Embedding
         env:
           HF_HOME: ${{ github.workspace }}/hf_cache
@@ -138,17 +179,21 @@ jobs:
           RUN_ENRICH="${{ github.event.inputs.run_enrich }}"
           FETCH_DAYS="${{ github.event.inputs.fetch_days }}"
           FETCH_MODE="${{ github.event.inputs.fetch_mode }}"
-          ARGS=""
+          PROFILE_TAG="${{ github.event.inputs.profile_tag }}"
+          ARGS=()
           if [ "$RUN_ENRICH" = "true" ]; then
-            ARGS="--run-enrich"
+            ARGS+=(--run-enrich)
           fi
           if [ -n "$FETCH_DAYS" ]; then
-            ARGS="$ARGS --fetch-days $FETCH_DAYS"
+            ARGS+=(--fetch-days "$FETCH_DAYS")
           fi
           if [ -n "$FETCH_MODE" ]; then
-            ARGS="$ARGS --fetch-mode $FETCH_MODE"
+            ARGS+=(--fetch-mode "$FETCH_MODE")
+          fi
+          if [ -n "$PROFILE_TAG" ]; then
+            ARGS+=(--profile-tag "$PROFILE_TAG")
           fi
-          python src/main.py $ARGS --embedding-device cpu --embedding-batch-size 8
+          python src/main.py "${ARGS[@]}" --embedding-device cpu --embedding-batch-size 8
 
           # 让前端可以在不配置 GitHub Token 的情况下直接读取配置（只读快照）
           cp -f config.yaml docs/config.yaml

.github/workflows/maintain-biorxiv.yml

@@ -0,0 +1,92 @@
+name: maintain-biorxiv
+
+on:
+  schedule:
+    - cron: "45 0 * * *" # 北京时间 08:45（UTC 00:45）
+    - cron: "45 8 * * *" # 北京时间 16:45（UTC 08:45）
+    - cron: "45 16 * * *" # 北京时间 00:45（UTC 16:45）
+  workflow_dispatch:
+    inputs:
+      fetch_days:
+        description: "回溯抓取天数（默认 30）"
+        required: false
+        default: "30"
+      force_full_window:
+        description: "是否忽略 seen 状态并全量回补当前窗口（true/false）"
+        required: false
+        default: "false"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: daily-paper-reader-maintain-biorxiv
+  cancel-in-progress: false
+
+jobs:
+  maintain_biorxiv:
+    if: github.repository == 'ziwenhahaha/daily-paper-reader' || github.repository == '5-xj/daily-paper-reader'
+    runs-on: ubuntu-latest
+    timeout-minutes: 240
+    env:
+      SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
+      SUPABASE_BACKEND_KEY: biorxiv
+      SUPABASE_PAPERS_TABLE: biorxiv_papers
+      SUPABASE_RETENTION_DAYS: "45"
+      DPR_ENABLE_BIORXIV_BACKEND: "1"
+      DPR_BIORXIV_ENABLED: "1"
+      DPR_BIORXIV_PAPERS_TABLE: biorxiv_papers
+      DPR_BIORXIV_VECTOR_RPC_EXACT: match_biorxiv_papers_exact
+      DPR_BIORXIV_BM25_RPC: match_biorxiv_papers_bm25
+      PYTHONUNBUFFERED: "1"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Cache pip + torch
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/torch
+          key: ${{ runner.os }}-dpr-biorxiv-hf-v1-${{ hashFiles('requirements.txt') }}
+
+      - name: Install deps (skip sqlite3)
+        run: |
+          python - <<'PY'
+          import re
+          lines = open("requirements.txt", "r", encoding="utf-8").read().splitlines()
+          lines = [l for l in lines if l.strip() and not re.match(r"^sqlite3\\b", l)]
+          open("/tmp/req.txt", "w", encoding="utf-8").write("\n".join(lines))
+          PY
+          python -m pip install --upgrade pip
+          python -m pip install uv
+          uv pip install --system -r /tmp/req.txt
+
+      - name: Run bioRxiv Maintain Pipeline
+        run: |
+          set -euo pipefail
+          set -x
+
+          if [ -z "${SUPABASE_SERVICE_KEY}" ]; then
+            echo "[WARN] 未配置 SUPABASE_SERVICE_KEY，已跳过 bioRxiv 维护同步。"
+            exit 0
+          fi
+
+          FETCH_DAYS="${{ github.event.inputs.fetch_days }}"
+          FORCE_FULL_WINDOW="${{ github.event.inputs.force_full_window }}"
+          if [ -z "$FETCH_DAYS" ]; then
+            FETCH_DAYS="30"
+          fi
+
+          ARGS=(--fetch-days "$FETCH_DAYS")
+          if [ "${FORCE_FULL_WINDOW:-false}" = "true" ]; then
+            ARGS+=(--force-full-window)
+          fi
+          python src/maintain/biorxiv.py "${ARGS[@]}"

.github/workflows/maintain-chemrxiv.yml

@@ -0,0 +1,90 @@
+name: maintain-chemrxiv
+
+on:
+  schedule:
+    - cron: "30 1 * * *" # 北京时间 09:30
+  workflow_dispatch:
+    inputs:
+      fetch_days:
+        description: "回溯抓取天数（默认 400；镜像源）"
+        required: false
+        default: "400"
+      force_full_window:
+        description: "是否忽略 seen 状态并全量回补当前窗口（true/false）"
+        required: false
+        default: "false"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: daily-paper-reader-maintain-chemrxiv
+  cancel-in-progress: false
+
+jobs:
+  maintain_chemrxiv:
+    if: github.repository == 'ziwenhahaha/daily-paper-reader' || github.repository == '5-xj/daily-paper-reader'
+    runs-on: ubuntu-latest
+    timeout-minutes: 240
+    env:
+      SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
+      SUPABASE_BACKEND_KEY: chemrxiv
+      SUPABASE_PAPERS_TABLE: chemrxiv_papers
+      SUPABASE_RETENTION_DAYS: "45"
+      DPR_ENABLE_CHEMRXIV_BACKEND: "1"
+      DPR_CHEMRXIV_ENABLED: "1"
+      DPR_CHEMRXIV_PAPERS_TABLE: chemrxiv_papers
+      DPR_CHEMRXIV_VECTOR_RPC_EXACT: match_chemrxiv_papers_exact
+      DPR_CHEMRXIV_BM25_RPC: match_chemrxiv_papers_bm25
+      PYTHONUNBUFFERED: "1"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Cache pip + torch
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/torch
+          key: ${{ runner.os }}-dpr-chemrxiv-hf-v1-${{ hashFiles('requirements.txt') }}
+
+      - name: Install deps (skip sqlite3)
+        run: |
+          python - <<'PY'
+          import re
+          lines = open("requirements.txt", "r", encoding="utf-8").read().splitlines()
+          lines = [l for l in lines if l.strip() and not re.match(r"^sqlite3\\b", l)]
+          open("/tmp/req.txt", "w", encoding="utf-8").write("\n".join(lines))
+          PY
+          python -m pip install --upgrade pip
+          python -m pip install uv
+          uv pip install --system -r /tmp/req.txt
+
+      - name: Run ChemRxiv Maintain Pipeline
+        run: |
+          set -euo pipefail
+          set -x
+
+          if [ -z "${SUPABASE_SERVICE_KEY}" ]; then
+            echo "[WARN] 未配置 SUPABASE_SERVICE_KEY，已跳过 ChemRxiv 维护同步。"
+            exit 0
+          fi
+
+          FETCH_DAYS="${{ github.event.inputs.fetch_days }}"
+          FORCE_FULL_WINDOW="${{ github.event.inputs.force_full_window }}"
+          if [ -z "$FETCH_DAYS" ]; then
+            FETCH_DAYS="400"
+          fi
+
+          ARGS=(--fetch-days "$FETCH_DAYS")
+          if [ "${FORCE_FULL_WINDOW:-false}" = "true" ]; then
+            ARGS+=(--force-full-window)
+          fi
+          python src/maintain/chemrxiv.py "${ARGS[@]}"

.github/workflows/maintain-medrxiv.yml

@@ -0,0 +1,90 @@
+name: maintain-medrxiv
+
+on:
+  schedule:
+    - cron: "15 1 * * *" # 北京时间 09:15
+  workflow_dispatch:
+    inputs:
+      fetch_days:
+        description: "回溯抓取天数（默认 30）"
+        required: false
+        default: "30"
+      force_full_window:
+        description: "是否忽略 seen 状态并全量回补当前窗口（true/false）"
+        required: false
+        default: "false"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: daily-paper-reader-maintain-medrxiv
+  cancel-in-progress: false
+
+jobs:
+  maintain_medrxiv:
+    if: github.repository == 'ziwenhahaha/daily-paper-reader' || github.repository == '5-xj/daily-paper-reader'
+    runs-on: ubuntu-latest
+    timeout-minutes: 240
+    env:
+      SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
+      SUPABASE_BACKEND_KEY: medrxiv
+      SUPABASE_PAPERS_TABLE: medrxiv_papers
+      SUPABASE_RETENTION_DAYS: "45"
+      DPR_ENABLE_MEDRXIV_BACKEND: "1"
+      DPR_MEDRXIV_ENABLED: "1"
+      DPR_MEDRXIV_PAPERS_TABLE: medrxiv_papers
+      DPR_MEDRXIV_VECTOR_RPC_EXACT: match_medrxiv_papers_exact
+      DPR_MEDRXIV_BM25_RPC: match_medrxiv_papers_bm25
+      PYTHONUNBUFFERED: "1"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Cache pip + torch
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/torch
+          key: ${{ runner.os }}-dpr-medrxiv-hf-v1-${{ hashFiles('requirements.txt') }}
+
+      - name: Install deps (skip sqlite3)
+        run: |
+          python - <<'PY'
+          import re
+          lines = open("requirements.txt", "r", encoding="utf-8").read().splitlines()
+          lines = [l for l in lines if l.strip() and not re.match(r"^sqlite3\\b", l)]
+          open("/tmp/req.txt", "w", encoding="utf-8").write("\n".join(lines))
+          PY
+          python -m pip install --upgrade pip
+          python -m pip install uv
+          uv pip install --system -r /tmp/req.txt
+
+      - name: Run medRxiv Maintain Pipeline
+        run: |
+          set -euo pipefail
+          set -x
+
+          if [ -z "${SUPABASE_SERVICE_KEY}" ]; then
+            echo "[WARN] 未配置 SUPABASE_SERVICE_KEY，已跳过 medRxiv 维护同步。"
+            exit 0
+          fi
+
+          FETCH_DAYS="${{ github.event.inputs.fetch_days }}"
+          FORCE_FULL_WINDOW="${{ github.event.inputs.force_full_window }}"
+          if [ -z "$FETCH_DAYS" ]; then
+            FETCH_DAYS="30"
+          fi
+
+          ARGS=(--fetch-days "$FETCH_DAYS")
+          if [ "${FORCE_FULL_WINDOW:-false}" = "true" ]; then
+            ARGS+=(--force-full-window)
+          fi
+          python src/maintain/medrxiv.py "${ARGS[@]}"