do not use standard view fallback

Author: jdkent

elsevier_coordinate_extraction/client.py

@@ -127,6 +127,26 @@ async def _request(
                     headers=request_headers,
                 )
             delay = rate_limits.get_retry_delay(response)
+            max_wait = self._settings.max_rate_limit_wait
+            if (
+                delay is not None
+                and response.status_code == 429
+                and max_wait is not None
+                and delay > max_wait
+            ):
+                snapshot = rate_limits.get_rate_limit_snapshot(response)
+                wait_seconds = snapshot.seconds_until_reset() or delay
+                message = (
+                    "Rate limit reset wait "
+                    f"({wait_seconds:g}s) exceeds configured maximum "
+                    f"({max_wait:g}s)."
+                )
+                raise httpx.HTTPStatusError(
+                    message
+                    + " Increase ELSEVIER_MAX_RATE_LIMIT_WAIT_SECONDS to allow longer waits.",
+                    request=response.request,
+                    response=response,
+                )
             if (
                 delay is not None
                 and response.status_code in {429, 500, 503}

elsevier_coordinate_extraction/download/api.py

@@ -9,6 +9,7 @@
 import httpx
 from lxml import etree
 
+from elsevier_coordinate_extraction import rate_limits
 from elsevier_coordinate_extraction.client import ScienceDirectClient
 from urllib.parse import urlparse
 
@@ -74,6 +75,8 @@ async def _runner() -> list[ArticleContent]:
                     cache=cache,
                     cache_namespace=cache_namespace,
                 )
+            except httpx.HTTPError:
+                raise
             except Exception:
                 continue
             if article is None:
@@ -154,17 +157,36 @@ async def _download_identifier(
             metadata["transport"] = "cache"
 
     view_used = initial_view
+    response_for_metadata: httpx.Response | None = None
     if payload is None:
-        params = {"httpAccept": "text/xml", "view": view_used}
         path = _endpoint_path_for_identifier(identifier, identifier_type)
-        response = await client.request(
-            "GET",
-            path,
-            params=params,
-            accept="application/xml",
-        )
+        params = {"httpAccept": "text/xml", "view": view_used}
+        try:
+            response = await client.request(
+                "GET",
+                path,
+                params=params,
+                accept="application/xml",
+            )
+        except httpx.HTTPStatusError as exc:
+            if (
+                view_used == "FULL"
+                and exc.response.status_code == 400
+                and _is_invalid_view_error(exc.response)
+            ):
+                message = (
+                    "ScienceDirect rejected FULL view for "
+                    f"{identifier_type}:{identifier}. Ensure your credentials grant full-text access."
+                )
+                raise httpx.HTTPStatusError(
+                    message,
+                    request=exc.request,
+                    response=exc.response,
+                ) from exc
+            raise
         payload = response.content
         content_type = response.headers.get("content-type", "application/xml")
+        response_for_metadata = response
         metadata.update(
             {
                 "transport": response.request.url.scheme,
@@ -176,14 +198,29 @@ async def _download_identifier(
                 "identifier_type": identifier_type,
             }
         )
+        snapshot = rate_limits.get_rate_limit_snapshot(response)
+        metadata.update(snapshot.to_metadata())
         if cache is not None:
             await cache.set(cache_namespace, cache_key, payload)
 
     full_text = _payload_contains_full_text(payload)
     inferred_view = "FULL" if full_text else "STANDARD"
-    metadata.setdefault("view_requested", initial_view)
-    metadata.setdefault("view_obtained", inferred_view)
-    metadata.setdefault("view", metadata.get("view", inferred_view))
+    if initial_view == "FULL" and not full_text:
+        message = (
+            "ScienceDirect returned metadata-only payload when FULL view was requested. "
+            "Confirm your entitlements allow full-text retrieval."
+        )
+        if response_for_metadata is not None:
+            raise httpx.HTTPStatusError(
+                message,
+                request=response_for_metadata.request,
+                response=response_for_metadata,
+            )
+        raise RuntimeError(message + " Cached payload violates requirement.")
+
+    metadata["view_requested"] = metadata.get("view_requested", initial_view)
+    metadata["view_obtained"] = inferred_view
+    metadata["view"] = inferred_view
     metadata["full_text_retrieved"] = full_text
 
     pii = _extract_pii(payload)
@@ -343,3 +380,16 @@ def _guess_cdn_url(api_url: str, extension: str | None) -> str | None:
         else:
             filename = f"{filename}.{extension}"
     return f"{_CDN_BASE}/{filename}"
+
+
+def _is_invalid_view_error(response: httpx.Response) -> bool:
+    """Detect Elsevier errors indicating the requested view is unsupported."""
+
+    status_header = response.headers.get("X-ELS-Status", "").lower()
+    if "view" in status_header and "invalid" in status_header:
+        return True
+    try:
+        body_text = response.text.lower()
+    except Exception:  # pragma: no cover - defensive fallback
+        return False
+    return "view" in body_text and "not valid" in body_text

elsevier_coordinate_extraction/rate_limits.py

@@ -2,12 +2,37 @@
 
 from __future__ import annotations
 
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from email.utils import parsedate_to_datetime
 
 import httpx
 
 
+@dataclass(frozen=True)
+class RateLimitSnapshot:
+    """Structured view over rate-limit response headers."""
+
+    limit: int | None
+    remaining: int | None
+    reset_epoch: float | None
+
+    def seconds_until_reset(self) -> float | None:
+        """Return seconds remaining until reset, if known."""
+        if self.reset_epoch is None:
+            return None
+        now = datetime.now(timezone.utc).timestamp()
+        return max(self.reset_epoch - now, 0.0)
+
+    def to_metadata(self) -> dict[str, float | int | None]:
+        """Convert snapshot into serializable metadata."""
+        return {
+            "rate_limit_limit": self.limit,
+            "rate_limit_remaining": self.remaining,
+            "rate_limit_reset_epoch": self.reset_epoch,
+        }
+
+
 def get_retry_delay(response: httpx.Response) -> float | None:
     """Return a suggested delay (seconds) before retrying a request.
 
@@ -16,6 +41,7 @@ def get_retry_delay(response: httpx.Response) -> float | None:
     provided we attempt to derive a delay from ``X-RateLimit-Reset``.
     """
 
+    snapshot = get_rate_limit_snapshot(response)
     retry_after = response.headers.get("Retry-After")
     if retry_after:
         try:
@@ -31,14 +57,33 @@ def get_retry_delay(response: httpx.Response) -> float | None:
             delta = (dt - now).total_seconds()
             return max(delta, 0.0)
 
-    reset = response.headers.get("X-RateLimit-Reset")
-    if reset:
+    if snapshot.reset_epoch is not None:
+        delay = snapshot.seconds_until_reset()
+        if delay and delay > 0:
+            return delay
+    return None
+
+
+def get_rate_limit_snapshot(response: httpx.Response) -> RateLimitSnapshot:
+    """Collect structured rate-limit header information from a response."""
+
+    def _parse_int(value: str | None) -> int | None:
+        if value is None:
+            return None
         try:
-            reset_epoch = float(reset)
+            return int(value)
         except ValueError:
             return None
-        now = datetime.now(timezone.utc).timestamp()
-        delay = reset_epoch - now
-        if delay > 0:
-            return delay
-    return None
+
+    def _parse_float(value: str | None) -> float | None:
+        if value is None:
+            return None
+        try:
+            return float(value)
+        except ValueError:
+            return None
+
+    limit = _parse_int(response.headers.get("X-RateLimit-Limit"))
+    remaining = _parse_int(response.headers.get("X-RateLimit-Remaining"))
+    reset_epoch = _parse_float(response.headers.get("X-RateLimit-Reset"))
+    return RateLimitSnapshot(limit=limit, remaining=remaining, reset_epoch=reset_epoch)

elsevier_coordinate_extraction/settings.py

@@ -14,6 +14,7 @@
 _DEFAULT_CONCURRENCY: Final[int] = 4
 _DEFAULT_CACHE_DIR: Final[str] = ".elsevier_cache"
 _DEFAULT_USER_AGENT: Final[str] = "elsevierCoordinateExtraction/0.1.0"
+_DEFAULT_MAX_RATE_LIMIT_WAIT: Final[float] = 3600.0  # 1 hour
 
 _CACHED_SETTINGS: Settings | None = None
 
@@ -32,6 +33,7 @@ class Settings:
     http_proxy: str | None
     https_proxy: str | None
     use_proxy: bool
+    max_rate_limit_wait: float | None
 
 
 _TRUE_VALUES: Final[set[str]] = {"1", "true", "yes", "on"}
@@ -86,6 +88,16 @@ def get_settings(*, force_reload: bool = False) -> Settings:
         default=default_use_proxy,
     )
 
+    max_wait_raw = os.getenv("ELSEVIER_MAX_RATE_LIMIT_WAIT_SECONDS")
+    if max_wait_raw is None or not max_wait_raw.strip():
+        max_rate_limit_wait: float | None = _DEFAULT_MAX_RATE_LIMIT_WAIT
+    else:
+        normalized = max_wait_raw.strip().lower()
+        if normalized in {"none", "infinite", "inf", "unlimited"}:
+            max_rate_limit_wait = None
+        else:
+            max_rate_limit_wait = max(float(max_wait_raw), 0.0)
+
     _CACHED_SETTINGS = Settings(
         api_key=api_key,
         base_url=base_url,
@@ -97,5 +109,6 @@ def get_settings(*, force_reload: bool = False) -> Settings:
         http_proxy=http_proxy,
         https_proxy=https_proxy,
         use_proxy=use_proxy,
+        max_rate_limit_wait=max_rate_limit_wait,
     )
     return _CACHED_SETTINGS