Merge branch 'master' into refactor/remove-mixins-less

Author: jimchamp

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -20,6 +20,7 @@ body:
       label: Reproducing the bug
       description: Being as specific as possible, what steps result in triggering this bug?
       value: |
+
         1. Go to ...
         2. Do ...
 
@@ -61,4 +62,4 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thanks for taking the time to fill out this bug report! 👍
+        Thanks for taking the time to fill out this bug report! 👍

compose.production.yaml

@@ -27,7 +27,7 @@ services:
     restart: unless-stopped
     hostname: "$HOSTNAME"
     environment:
-      - GUNICORN_OPTS= --workers 1 --timeout 300 --max-requests 500
+      - GUNICORN_OPTS= --workers 2 --timeout 300 --max-requests 500
       - OL_CONFIG=/olsystem/etc/openlibrary.yml
     volumes:
       - ../booklending_utils:/booklending_utils

compose.staging.yaml

@@ -38,7 +38,7 @@ services:
     restart: unless-stopped
     hostname: "$HOSTNAME"
     environment:
-      - GUNICORN_OPTS= --workers 1 --timeout 180 --max-requests 500
+      - GUNICORN_OPTS= --workers 2 --timeout 180 --max-requests 500
       - OL_CONFIG=/olsystem/etc/openlibrary.yml
     volumes:
       - ../olsystem:/olsystem

compose.yaml

@@ -20,7 +20,7 @@ services:
     image: "${OLIMAGE:-oldev:latest}"
     environment:
       - OL_CONFIG=${OL_CONFIG:-/openlibrary/conf/openlibrary.yml}
-      - GUNICORN_OPTS=${GUNICORN_OPTS:- --reload --workers 1 --timeout 180 --max-requests 500}
+      - GUNICORN_OPTS=${GUNICORN_OPTS:- --reload --workers 2 --timeout 180 --max-requests 500}
     command: docker/ol-web-fastapi-start.sh
     ports:
       - ${FAST_WEB_PORT:-18080}:8080

docker/nginx.conf

@@ -32,6 +32,9 @@ http {
     include /olsystem/etc/nginx/logging.conf;
     access_log    /var/log/nginx/access.log iacombined;
 
+    js_shared_dict_zone zone=crawler_ips:10M type=number timeout=300s;
+    js_import /olsystem/etc/nginx/tagger.js;
+
     client_max_body_size 50m;
 
     sendfile        on;
@@ -52,20 +55,42 @@ http {
     # These rules only do anything if invoked, e.g., in web_nginx.conf.
     # TLDR: these rules can be disabled in `docker/web_nginx.conf`
     # and `docker/covers_nginx.conf`.
-    geo $should_apply_limit {
-      # No rate limit when IP obfuscation is not applied, as every IP is 255.0.0.0.
-      255.0.0.0 0;
-      # In cluster traffic
-      207.241.224.0/20 0;
-      # All other traffic
-      default 1;
+    geo $is_blessed_ip {
+      255.0.0.0 1;         # Internal
+      207.241.224.0/20 1;  # In cluster traffic
+      default 0;           # All other traffic
     }
 
-    map $should_apply_limit $rate_limit_key {
-      0 '';
-      1 $binary_remote_addr;
+    # Provides $is_blessed_ua
+    include /olsystem/etc/nginx/is_blessed_ua.map;
+
+    map "$is_blessed_ip:$is_blessed_ua" $rate_limit_key {
+      "0:0" $binary_remote_addr;  # Rate-limit by IP
+      default '';  # Don't rate-limit
     }
 
+    # check if user-agent provides a means of identification
+    map $http_user_agent $is_identifying_ua {
+        default 0;
+        "~*bot" 1;
+        "~*spider" 1;
+        "~*crawl" 1;
+        "~*google" 1; # sometimes just GoogleOther
+        "~*http" 1;   # Includes url
+        "~*@" 1;      # Includes email
+     }
+
+    js_set $has_hit_crawler_links tagger.check;
+
+    # The only crawlers we want to limit are the ones that don't identify themselves as such
+    map "$is_blessed_ip:$is_identifying_ua:$has_hit_crawler_links" $global_nonidentifying_crawler_rate_limit_key {
+        default '';  # No shared rate limiting
+        "0:0:1" '1'; # Shared rate limit
+    }
+
+    # Limit the crawlers that scrape links but don't ID themselves globally
+    limit_req_zone $global_nonidentifying_crawler_rate_limit_key zone=global_crawler_limit:5m rate=15r/s;
+
     # Matches other sites
     limit_req_zone $rate_limit_key zone=web_limit:10m rate=1r/s;
     # Higher rate for APIs since they are cheaper and we often hit them

docker/web_nginx.conf

@@ -105,8 +105,12 @@ server {
 
     location / {
         limit_req zone=web_limit burst=100 delay=10;
+        limit_req zone=global_crawler_limit nodelay;
         limit_req_status 429;
 
+        js_set $is_crawler_link tagger.tag_crawler;
+        add_header X-SPS $is_crawler_link;  # Need to reference the variable for the js method to be executed
+
         # For returning 200 when someone tries to randomly sort author results.
         if ($is_sus_random_sort) {
             return 200;
@@ -117,7 +121,7 @@ server {
         }
 
         if ($is_sus_referer) {
-            return 444;
+            return 403;
         }
 
         # Haproxy to better handle load/traffic
@@ -138,7 +142,7 @@ server {
         limit_req_status 429;
 
         if ($http_user_agent ~* (bytespider|meta-externalagent) ) {
-            return 444;
+            return 403;
         }
 
         # Haproxy to better handle load/traffic

openlibrary/asgi_app.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import re
+import sys
 from pathlib import Path
 
 import yaml
@@ -114,28 +115,29 @@ async def i18n_middleware(request: Request, call_next):
 
 
 def create_app() -> FastAPI:
-    _setup_env()
+    if "pytest" not in sys.modules:
+        _setup_env()
 
-    if os.environ.get("CI"):
-        import pytest
+        if os.environ.get("CI"):
+            import pytest
 
-        pytest.skip("Skipping in CI", allow_module_level=True)
+            pytest.skip("Skipping in CI", allow_module_level=True)
 
-    ol_config_path = Path(__file__).parent / "conf" / "openlibrary.yml"
-    ol_config = os.environ.get("OL_CONFIG", str(ol_config_path))
-    try:
-        # We still call this even though we don't use it because of the side effects
-        legacy_wsgi = _load_legacy_wsgi(ol_config)  # noqa: F841
+        ol_config_path = Path(__file__).parent / "conf" / "openlibrary.yml"
+        ol_config = os.environ.get("OL_CONFIG", str(ol_config_path))
+        try:
+            # We still call this even though we don't use it because of the side effects
+            legacy_wsgi = _load_legacy_wsgi(ol_config)  # noqa: F841
 
-        global sentry
-        if sentry is not None:
-            return
-        sentry = init_sentry(getattr(infogami.config, 'sentry', {}))
-        set_tag("fastapi", True)
+            global sentry
+            if sentry is not None:
+                return
+            sentry = init_sentry(getattr(infogami.config, 'sentry', {}))
+            set_tag("fastapi", True)
 
-    except Exception:
-        logger.exception("Failed to initialize legacy WSGI app")
-        raise
+        except Exception:
+            logger.exception("Failed to initialize legacy WSGI app")
+            raise
 
     app = FastAPI(title="OpenLibrary ASGI", version="0.0.1")
 
@@ -154,6 +156,13 @@ def create_app() -> FastAPI:
 
     setup_i18n(app)
 
+    @app.middleware("http")
+    async def add_fastapi_header(request: Request, call_next):
+        """Middleware to add a header indicating the response came from FastAPI."""
+        response = await call_next(request)
+        response.headers["X-Served-By"] = "FastAPI"
+        return response
+
     # --- Fast routes (mounted within this app) ---
     @app.get("/health")
     def health() -> dict[str, str]:

openlibrary/book_providers.py

@@ -381,7 +381,7 @@ def get_acquisitions(
                 access=access,
                 format='web',
                 price=None,
-                url=f'https://archive.org/details/{self.get_best_identifier(db_edition or ed_or_solr)}',
+                url=f'https://archive.org/details/{self.get_best_identifier(db_edition or ed_or_solr)}?view=theater&wrapper=false',
                 provider_name=self.short_name,
             )
         ]

openlibrary/core/cache.py

@@ -25,7 +25,6 @@
     "MemcacheCache",
     "MemoryCache",
     "RequestCache",
-    "cached_property",
     "get_memcache",
     "memcache_memoize",
     "memoize",
@@ -241,28 +240,6 @@ def memcache_get(self, args: tuple, kw: dict) -> tuple[T, float] | None:
 ####
 
 
-def cached_property(getter):
-    """Decorator like `property`, but the value is computed on first call and cached.
-
-    class Foo:
-
-        @cached_property
-        def memcache_client(self):
-            ...
-    """
-    name = getter.__name__
-
-    def g(self):
-        if name in self.__dict__:
-            return self.__dict__[name]
-
-        value = getter(self)
-        self.__dict__[name] = value
-        return value
-
-    return property(g)
-
-
 class Cache:
     """Cache interface."""
 
@@ -322,7 +299,7 @@ class MemcacheCache(Cache):
     Expects that the memcache servers are specified in web.config.memcache_servers.
     """
 
-    @cached_property
+    @functools.cached_property
     def memcache(self):
         if servers := config.get("memcache_servers", None):
             return olmemcache.Client(servers)

openlibrary/core/ia.py

@@ -42,6 +42,48 @@ def get_api_response(url: str, params: dict | None = None) -> dict:
     return api_response
 
 
+def save_page_now(
+    url: str, access_key: str | None = None, secret_key: str | None = None
+) -> str:
+    """Archive a URL using the Internet Archive Save Page Now API.
+
+    Returns job_id on success, or an error string like "NO_CREDENTIALS",
+    "ERROR_HTTP_<code>", or "ERROR_EXCEPTION_<msg>" on failure.
+    """
+    if not access_key or not secret_key:
+        access_key, secret_key = get_ia_s3_keys()
+
+    if not access_key or not secret_key:
+        return "NO_CREDENTIALS"
+
+    headers = {
+        "Authorization": f"LOW {access_key}:{secret_key}",
+        "Accept": "application/json",
+    }
+    data = {"url": url}
+
+    try:
+        r = session.post(
+            "https://web.archive.org/save", headers=headers, data=data, timeout=30
+        )
+        if r.status_code == 200:
+            try:
+                result = r.json()
+            except ValueError:
+                return f"ERROR_NO_JOB_ID_{r.status_code}"
+            return result.get('job_id', f"ERROR_NO_JOB_ID_{r.status_code}")
+        else:
+            return f"ERROR_HTTP_{r.status_code}"
+    except (httpx.RequestException, ValueError) as e:
+        return f"ERROR_EXCEPTION_{str(e)[:50]}"
+
+
+def get_ia_s3_keys() -> tuple[str | None, str | None]:
+    """Resolve IA S3 creds via infogami config."""
+    spn_config = config.get("ol_spn_api_s3", {})
+    return spn_config.get("s3_key"), spn_config.get("s3_secret")
+
+
 def get_metadata_direct(
     itemid: str, only_metadata: bool = True, cache: bool = True
 ) -> dict: