update

Author: TideDra

.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    services:
+      mailhog:
+        image: mailhog/mailhog:latest
+        ports:
+          - 1025:1025  # SMTP
+      openai:
+        image: tidedra/mock_openai:latest
+        ports:
+          - 30000:30000
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v7.1.4
+
+
+      - name: Run Pytest
+        env:
+          ZOTERO_ID: "0"
+          ZOTERO_KEY: "AbCdEfGhIjKlMnOpQrStUvWx"
+          SENDER: "test@example.com"
+          RECEIVER: "test@example.com"
+          SENDER_PASSWORD: "test"
+          OPENAI_API_KEY: "sk-xxx"
+          OPENAI_API_BASE: "http://openai:30000/v1"
+        run: |
+          uv run pytest

.github/workflows/main.yml

@@ -9,33 +9,27 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ vars.REPOSITORY }}
-          ref: ${{ vars.REF }}
+        uses: actions/checkout@v6
+      #  with:
+      #    repository: ${{ vars.REPOSITORY }}
+      #    ref: ${{ vars.REF }}
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v3
-        with:
-          version: '0.5.4'
+        uses: astral-sh/setup-uv@v7.1.4
+
 
       - name: Run script
         env:
           ZOTERO_ID: ${{ secrets.ZOTERO_ID }}
           ZOTERO_KEY: ${{ secrets.ZOTERO_KEY }}
-          ZOTERO_IGNORE: ${{ vars.ZOTERO_IGNORE }}
-          SEND_EMPTY: ${{ vars.SEND_EMPTY }}
-          ARXIV_QUERY: ${{ secrets.ARXIV_QUERY }}
-          SMTP_SERVER: ${{ secrets.SMTP_SERVER }}
-          SMTP_PORT: ${{ secrets.SMTP_PORT }}
           SENDER: ${{ secrets.SENDER }}
           RECEIVER: ${{ secrets.RECEIVER }}
           SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }}
-          MAX_PAPER_NUM: ${{ secrets.MAX_PAPER_NUM }}
-          USE_LLM_API: ${{ secrets.USE_LLM_API }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
-          MODEL_NAME: ${{ secrets.MODEL_NAME }}
-          LANGUAGE: ${{ vars.LANGUAGE }}
+          CUSTOM_CONFIG: ${{ vars.CUSTOM_CONFIG }}
         run: |
-          uv run main.py
+          printf "%b\n" "$CUSTOM_CONFIG" > config/custom.yaml
+          echo "Use custom config: "
+          cat config/custom.yaml
+          uv run src/zotero_arxiv_daily/main.py

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: Test workflow
+name: Test
 on:
   workflow_dispatch:
 
@@ -7,33 +7,28 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ vars.REPOSITORY }}
-          ref: ${{ vars.REF }}
+        uses: actions/checkout@v6
+      #  with:
+      #    repository: ${{ vars.REPOSITORY }}
+      #    ref: ${{ vars.REF }}
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v3
-        with:
-          version: '0.5.4'
+        uses: astral-sh/setup-uv@v7.1.4
+
 
       - name: Run script
         env:
           ZOTERO_ID: ${{ secrets.ZOTERO_ID }}
           ZOTERO_KEY: ${{ secrets.ZOTERO_KEY }}
-          ZOTERO_IGNORE: ${{ vars.ZOTERO_IGNORE }}
-          ARXIV_QUERY: ${{ secrets.ARXIV_QUERY }}
-          SEND_EMPTY: ${{ vars.SEND_EMPTY }}
-          SMTP_SERVER: ${{ secrets.SMTP_SERVER }}
-          SMTP_PORT: ${{ secrets.SMTP_PORT }}
           SENDER: ${{ secrets.SENDER }}
           RECEIVER: ${{ secrets.RECEIVER }}
           SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }}
-          MAX_PAPER_NUM: ${{ secrets.MAX_PAPER_NUM }}
-          USE_LLM_API: ${{ secrets.USE_LLM_API }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
-          MODEL_NAME: ${{ secrets.MODEL_NAME }}
-          LANGUAGE: ${{ vars.LANGUAGE }}
+          CUSTOM_CONFIG: ${{ vars.CUSTOM_CONFIG }}
         run: |
-          uv run main.py --debug
+          export DEBUG=true
+          printf "%b\n" "$CUSTOM_CONFIG" > config/custom.yaml
+          echo "Use custom config: "
+          cat config/custom.yaml
+          uv run src/zotero_arxiv_daily/main.py

config/base.yaml

@@ -22,7 +22,7 @@ llm:
   generation_kwargs:
     max_tokens: 16384
     temperature: 0.4
-    model: gpt-4o-mini
+    model: ???
   language: English
 
 reranker:

config/custom.yaml

@@ -0,0 +1,34 @@
+zotero:
+  user_id: ${oc.env:ZOTERO_ID}
+  api_key: ${oc.env:ZOTERO_KEY}
+  include_path: null
+
+email:
+  sender: ${oc.env:SENDER}
+  receiver: ${oc.env:RECEIVER}
+  smtp_server: smtp.qq.com
+  smtp_port: 465
+  sender_password: ${oc.env:SENDER_PASSWORD}
+
+llm:
+  api:
+    key: ${oc.env:OPENAI_API_KEY}
+    base_url: ${oc.env:OPENAI_API_BASE}
+  generation_kwargs:
+    model: gpt-4o-mini
+
+reranker:
+  local:
+    model: Qwen/Qwen3-Embedding-0.6B
+  api:
+    key: ${oc.env:OPENAI_API_KEY}
+    base_url: ${oc.env:OPENAI_API_BASE}
+    model: text-embedding-3-large
+
+source:
+  arxiv:
+    query: cs.AI+cs.CV+cs.LG+cs.CL
+
+executor:
+  debug: ${oc.env:DEBUG,false}
+  source: ['arxiv']

config/default.yaml

@@ -1,4 +1,3 @@
 defaults:
   - base
-  - private
-  - public
+  - custom

config/public.yaml

@@ -18,11 +18,21 @@ dependencies = [
     "pyzotero>=1.6.11",
     "scikit-learn>=1.7.1",
     "sentence-transformers>=5.0.0",
+    "torch",
     "openai>=1.99.1",
-    "llama-cpp-python>=0.3.2",
     "arxiv>=2.2.0",
+    "pymupdf4llm>=0.2.7",
+    "pymupdf-layout>=1.26.6",
 ]
 
+[tool.uv.sources]
+torch = { index = "pytorch" }
+
+[[tool.uv.index]]
+name = "pytorch"
+url = "https://download.pytorch.org/whl/cpu"
+explicit = true
+
 [dependency-groups]
 dev = [
     "ipykernel>=7.1.0",

pyproject.toml

@@ -52,13 +52,16 @@ def filter_corpus(self, corpus:list[CorpusPaper]) -> list[CorpusPaper]:
                 new_corpus.append(c)
         samples = random.sample(new_corpus, min(5, len(new_corpus)))
         samples = '\n'.join([c.title + ' - ' + '\n'.join(c.paths) for c in samples])
-        logger.info(f"Selected {len(new_corpus)} zotero papers:\n{samples}")
+        logger.info(f"Selected {len(new_corpus)} zotero papers:\n{samples}\n...")
         return new_corpus
 
 
     def run(self):
         corpus = self.fetch_zotero_corpus()
         corpus = self.filter_corpus(corpus)
+        if len(corpus) == 0:
+            logger.error(f"No zotero papers found. Please check your zotero settings:\n{self.config.zotero}")
+            return
         all_papers = []
         for source, retriever in self.retrievers.items():
             logger.info(f"Retrieving {source} papers...")