Cache tests requests to avoid connections errors in CI

.github/workflows/python-package.yml

@@ -52,6 +52,10 @@ jobs:
       run: uv python pin ${{ matrix.settings.python }}
     - name: Sync dependencies
       run: uv sync
+    - name: Setup cache
+      run: |
+        git submodule update --init
+        hatch run setup-cache
     - name: Setup mongo
       run: hatch run mongo:start
     - name: Test with pytest

.gitignore

@@ -20,4 +20,3 @@ requests-cache.sqlite
 .cursorignore
 /cache
 /data
-/tests/cache

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/cache"]
+	path = tests/cache
+	url = https://github.com/satyaog/paperoni-testscache.git

pyproject.toml

@@ -108,6 +108,14 @@ make-secrets = [
     'serieux patch -m "paperoni.config:gifnoc_model" -f "{env:PAPERONI_SECRETS}" -o "$(echo "{env:GIFNOC_FILE}" | cut -d"," -f1)" {args}',
     'cp $(dirname "{env:PAPERONI_SECRETS}")/*focuses*.yaml "$(dirname "$(echo "{env:GIFNOC_FILE}" | cut -d"," -f1)")"',
 ]
+setup-cache = [
+    "gzip --decompress --force --keep {root:real}/tests/cache/cache.tar.gz",
+    "tar -xf {root:real}/tests/cache/cache.tar -C {root:real}/tests/cache",
+]
+save-cache = [
+    "cd {root:real}/tests/cache && tar -cf cache.tar --exclude 'cache.tar*' --uid 0 --gid 0 $(find * -type f | sort)",
+    "gzip --force --keep --no-name -1 {root:real}/tests/cache/cache.tar",
+]
 
 [tool.hatch.envs.hatch-test]
 installer = "uv"
@@ -120,8 +128,8 @@ post-install-commands = [
 [tool.hatch.envs.hatch-test.extra-scripts]
 uv = 'UV_PROJECT_ENVIRONMENT="{env:VIRTUAL_ENV}" "$(which uv)" {args}'
 mongo = [
-    "uv sync --inexact",
     "hatch run mongo:restart",
+    "uv sync --inexact",
     "run {args}",
     "hatch run mongo:stop",
 ]

tests/config/test-config.yaml

@@ -5,15 +5,21 @@ paperoni:
   fetch:
     $class: RulesFetcher
     rules:
+      "^https://api.biorxiv.org/": cloudflare
       "^https://www.biorxiv.org/": cloudflare
       "^https://www.medrxiv.org/": cloudflare
-      ".*": cached
+      "^https://pmc.ncbi.nlm.nih.gov/": cached
+      ".*": cached_ua
     fetchers:
-      cached:
+      cached_ua:
         $class: CachedFetcher
         cache_path: ${paperoni.cache_path}/requests-cache.sql
         expire_after: 7d
         user_agent: chrome
+      cached:
+        $class: CachedFetcher
+        cache_path: ${paperoni.cache_path}/requests-cache.sql
+        expire_after: 7d
       cloudflare:
         $class: CloudFlareFetcher
         user_agent: chrome

tests/refinement/test_fetch.py

@@ -96,7 +96,7 @@ def test__test_tags(f_tags, tags, pass_):
 
 # biorxiv links sometimes fails with requests.exceptions.HTTPError: 421 Client
 # Error: Misdirected Request
-links_w_redirect_errors = {
+links_w_errors = {
     "10.1101/2020.10.29.359778",
     "10.1101/2023.05.17.541168",
     "10.1101/2023.10.27.564468",
@@ -116,7 +116,11 @@ def test_refine(func, link, data_regression):
         data = serialize(Paper, result)
         data_regression.check(data)
     except requests.exceptions.HTTPError as e:
-        if e.response.status_code == 421 and link in links_w_redirect_errors:
+        if e.response.status_code == 403 and link in links_w_errors:
+            pytest.skip(
+                f"{link} fails with a forbidden request ending in error {e.response.status_code}"
+            )
+        elif e.response.status_code == 421 and link in links_w_errors:
             pytest.skip(
                 f"{link} fails with a misdirected request ending in error {e.response.status_code}"
             )