feat(github integration): Enhance GitHub provider with retry logic for transient errors and add GitHub rate limit probing. Update .env.example to include configuration for maximum retry attempts. Modify API health checks to report GitHub rate limit status. Introduce validation for organization handles against GitHub API to ensure accurate entity representation.

Author: caviri

.env.example

@@ -22,6 +22,12 @@
 # `src/index/github/` indexer (REST metadata + README fetches).
 GITHUB_TOKEN=your-github-token-here
 
+# Bound for retrying transient GitHub provider failures (gimie returning
+# empty/malformed JSON during 502s, rate-limit blips, etc.). Default 3.
+# Set to 0 to disable. Non-transient errors (NOT_FOUND, UnicodeDecodeError)
+# never retry — UnicodeDecodeError falls back to a minimal payload instead.
+# V2_GITHUB_REPO_MAX_RETRIES=3
+
 # ---------------------------------------------------------------------------
 # LLM provider credentials (only the one(s) referenced by your model config
 # need to be set — `src/v2/agents/llm/runtime.py` validates this at startup

src/api.py

@@ -75,10 +75,25 @@ def _resolve_package_version(name: str) -> str:
 logger = logging.getLogger(__name__)
 
 
-async def startup_event():
-    """Initialize resources on application startup"""
+async def startup_event(app: FastAPI | None = None):
+    """Initialize resources on application startup."""
     logger.info("🚀 Application startup - initializing resources")
 
+    # Pre-warm the v2 provider cache so the SQLite file exists in WAL mode
+    # before any request hits a worker. Without this, multi-worker uvicorn
+    # against a cold .cache/ races on first-request to create+init the
+    # file and the losing workers raise `database is locked`, surfacing
+    # as 500s for the first 1-2 jobs of a batch run.
+    if app is not None:
+        try:
+            from src.v2.dependencies import _resolve_provider_cache
+
+            cache = _resolve_provider_cache(app.state)
+            if cache is not None:
+                logger.info("✅ v2 provider cache initialized")
+        except Exception as exc:  # noqa: BLE001 — startup must not crash on cache issues
+            logger.warning(f"v2 provider cache pre-warm skipped: {exc}")
+
 
 async def shutdown_event():
     """Cleanup resources on application shutdown"""
@@ -119,8 +134,8 @@ async def shutdown_event():
 
 
 @asynccontextmanager
-async def lifespan(_: FastAPI):
-    await startup_event()
+async def lifespan(app: FastAPI):
+    await startup_event(app)
     try:
         yield
     finally:

src/v1/data_models/infoscience.py

@@ -24,6 +24,17 @@ class InfosciencePublication(BaseModel):
         description="List of author names",
         default_factory=list,
     )
+    author_authorities: List[Optional[str]] = Field(
+        description=(
+            "Parallel array to `authors` (same length, same order) holding "
+            "the DSpace `authority` UUID for each author when available "
+            "(EPFL-affiliated, authority-controlled). `None` for non-EPFL "
+            "authors that DSpace stores as bare names. Used by v2 to bind "
+            "publication authors directly to Infoscience person entities "
+            "without name-based fuzzy matching."
+        ),
+        default_factory=list,
+    )
     abstract: Optional[str] = Field(
         description="Publication abstract or description",
         default=None,

src/v2/agents/llm/refiners/membership/agent.py

@@ -51,7 +51,7 @@ def __init__(
         self,
         *,
         llm_runtime: V2LLMRuntime | None = None,
-        llm_call_timeout_seconds: float = 30.0,
+        llm_call_timeout_seconds: float = 120.0,
     ) -> None:
         if llm_call_timeout_seconds <= 0:
             message = "llm_call_timeout_seconds must be > 0"

src/v2/agents/llm/refiners/organization/agent.py

@@ -58,7 +58,7 @@ def __init__(
         self,
         *,
         llm_runtime: V2LLMRuntime | None = None,
-        llm_call_timeout_seconds: float = 60.0,
+        llm_call_timeout_seconds: float = 120.0,
     ) -> None:
         if llm_call_timeout_seconds <= 0:
             message = "llm_call_timeout_seconds must be > 0"

src/v2/agents/llm/refiners/person/agent.py

@@ -47,7 +47,7 @@ def __init__(
         self,
         *,
         llm_runtime: V2LLMRuntime | None = None,
-        llm_call_timeout_seconds: float = 60.0,
+        llm_call_timeout_seconds: float = 120.0,
     ) -> None:
         if llm_call_timeout_seconds <= 0:
             message = "llm_call_timeout_seconds must be > 0"

src/v2/agents/llm/refiners/repository/agent.py

@@ -60,7 +60,7 @@ def __init__(
         self,
         *,
         llm_runtime: V2LLMRuntime | None = None,
-        llm_call_timeout_seconds: float = 60.0,
+        llm_call_timeout_seconds: float = 120.0,
     ) -> None:
         if llm_call_timeout_seconds <= 0:
             message = "llm_call_timeout_seconds must be > 0"

src/v2/agents/rule_based/article_agent.py

@@ -274,8 +274,15 @@ def _build_person_lookup(
     persons: list[dict[str, Any]],
     *,
     person_derivations: Any = None,
-) -> dict[str, str]:
+) -> tuple[dict[str, str], dict[str, str]]:
+    """Build the alias→person_id lookup plus an infoscience-authority→person_id map.
+
+    The authority lookup is keyed on raw DSpace authority UUIDs (``pulse:infosciencePersonIdentifier``)
+    so article authors can be mapped directly without name fuzzy-matching when DSpace returned
+    an authority on the publication.
+    """
     lookup: dict[str, str] = {}
+    authority_lookup: dict[str, str] = {}
     derivation_by_id: dict[str, dict[str, Any]] = {}
     if isinstance(person_derivations, list):
         for derivation in person_derivations:
@@ -296,7 +303,17 @@ def _build_person_lookup(
             derivation_by_id=derivation_by_id,
         ):
             _register_lookup_token(lookup, alias_token, person_id)
-    return lookup
+
+        infoscience_id = _as_string(person.get("pulse:infosciencePersonIdentifier"))
+        if infoscience_id is None:
+            identifiers = person.get("identifiers")
+            if isinstance(identifiers, dict):
+                infoscience_id = _as_string(
+                    identifiers.get("pulse:infosciencePersonIdentifier"),
+                )
+        if infoscience_id:
+            authority_lookup.setdefault(infoscience_id, person_id)
+    return lookup, authority_lookup
 
 
 def _build_organization_lookup(organizations: list[dict[str, Any]]) -> dict[str, str]:  # noqa: C901
@@ -489,6 +506,7 @@ def _map_author_ids(
     publication: dict[str, Any],
     *,
     person_lookup: dict[str, str],
+    authority_lookup: dict[str, str],
     publication_reference: str,
 ) -> tuple[list[str], list[str], list[str], int, int]:
     warnings: list[str] = []
@@ -497,8 +515,22 @@ def _map_author_ids(
     matched_authors = 0
     unresolved_count = 0
     author_names = _as_string_list(publication.get("authors"))
-    for author_name in author_names:
-        resolved_author = _resolve_lookup_token(person_lookup, author_name)
+    raw_authorities = publication.get("author_authorities")
+    authority_values: list[str | None]
+    if isinstance(raw_authorities, list) and len(raw_authorities) == len(author_names):
+        authority_values = [
+            item if isinstance(item, str) and item.strip() else None
+            for item in raw_authorities
+        ]
+    else:
+        authority_values = [None] * len(author_names)
+
+    for author_name, authority in zip(author_names, authority_values, strict=False):
+        resolved_author: str | None = None
+        if authority is not None:
+            resolved_author = authority_lookup.get(authority)
+        if resolved_author is None:
+            resolved_author = _resolve_lookup_token(person_lookup, author_name)
         if isinstance(resolved_author, str):
             mapped_authors.append(resolved_author)
             matched_authors += 1
@@ -749,7 +781,7 @@ async def run(  # noqa: C901, PLR0912, PLR0915
                 stats={"queries": queries, "articles": []},
             )
 
-        person_lookup = _build_person_lookup(
+        person_lookup, authority_lookup = _build_person_lookup(
             _collect_known_persons(context),
             person_derivations=context.get("person_derivations"),
         )
@@ -773,6 +805,7 @@ async def run(  # noqa: C901, PLR0912, PLR0915
             ) = _map_author_ids(
                 candidate.publication,
                 person_lookup=person_lookup,
+                authority_lookup=authority_lookup,
                 publication_reference=publication_reference,
             )
             for unresolved_author in unresolved_authors:

src/v2/agents/rule_based/contribution_agent.py

@@ -237,6 +237,26 @@ def _build_contribution_payload(  # noqa: PLR0913
     }
 
 
+def _person_github_login(person: Any) -> str | None:
+    """Best-effort GitHub login for a person entity.
+
+    Mirrors the LLM contribution agent's helper: prefers the explicit
+    `pulse:githubUsername`, falls back to parsing `id` when shaped like
+    `https://github.com/{login}`.
+    """
+    if not isinstance(person, dict):
+        return None
+    handle = person.get("pulse:githubUsername")
+    if isinstance(handle, str) and handle.strip():
+        return handle.strip()
+    identifier = person.get("id")
+    if isinstance(identifier, str) and identifier.startswith("https://github.com/"):
+        candidate = identifier.removeprefix("https://github.com/").strip("/")
+        if candidate and "/" not in candidate:
+            return candidate
+    return None
+
+
 class ContributionAgentV2:
     """Deterministic contribution agent from repository contributor metadata."""
 
@@ -252,6 +272,17 @@ async def run(  # noqa: C901
         organization_lookup = _build_organization_lookup(_collect_known_organizations(context))
         repositories = _collect_known_repositories(context)
 
+        # The orchestrator fans out one work item per (person, repo) pair and
+        # passes both `target_person` and `target_repository`. Honour that
+        # scoping: only emit a contribution for the target person, never warn
+        # about other contributors that happen to share the repo's signal list
+        # (they have their own fanout work item).
+        target_person = context.get("target_person")
+        target_login = _person_github_login(target_person)
+        normalized_target_login = (
+            target_login.casefold() if isinstance(target_login, str) else None
+        )
+
         contribution_data_by_composite: dict[str, dict[str, Any]] = {}
         for repository in repositories:
             repository_id = _as_string(
@@ -268,6 +299,12 @@ async def run(  # noqa: C901
                 if normalized_login is None:
                     continue
 
+                if (
+                    normalized_target_login is not None
+                    and normalized_login != normalized_target_login
+                ):
+                    continue
+
                 person_id = person_lookup.get(normalized_login)
                 if person_id is None:
                     if normalized_login in organization_lookup:

src/v2/api.py

@@ -38,6 +38,10 @@
 from src.v2.ingest.cache import ProviderCache
 from src.v2.ingest.detection import UnsupportedGitHubURL, classify_github_url
 from src.v2.jobs import JobStore
+from src.v2.observation.github_rate_limit import (
+    GitHubRateLimitSummary,
+    probe_github_rate_limit,
+)
 from src.v2.observation.query_log import QueryLog, query_log_var
 from src.v2.pipeline import PipelineOrchestrator
 from src.v2.pipeline.stages import (
@@ -65,6 +69,9 @@
     validate_author_classes,
     validate_ownership,
 )
+from src.v2.pipeline.stages.validate_org_github_handles import (
+    validate_org_github_handles,
+)
 from src.v2.pipeline.stages.refine_with_llm import (
     is_enabled as _hybrid_refiner_is_enabled,
 )
@@ -706,6 +713,19 @@ async def extract(  # noqa: C901, PLR0911, PLR0912, PLR0913, PLR0915
     for warning in repo_author_warnings:
         _append_unique_warning(warnings, warning)
 
+    # Validate `@handle`-style org names against GitHub before strict
+    # validation so we either stamp the missing handle or drop the
+    # hallucinated entity (rather than just losing it to anyOf).
+    stage_started_at = perf_counter()
+    reconciled, org_handle_warnings = validate_org_github_handles(reconciled, providers)
+    logger.info(
+        "validate_org_github_handles: actions=%d in %.2fs",
+        len(org_handle_warnings),
+        perf_counter() - stage_started_at,
+    )
+    for warning in org_handle_warnings:
+        _append_unique_warning(warnings, warning)
+
     stage_started_at = perf_counter()
     strict_validation_entities = _iter_reconciled_entities(
         reconciled_entities=reconciled.entities,
@@ -1347,9 +1367,18 @@ async def health() -> V2HealthResponse:
     except ValueError:
         component_statuses["config"] = "unhealthy"
 
-    component_statuses["github_token"] = (
-        "healthy" if config and config.GITHUB_TOKEN else "degraded"
-    )
+    rate_limit_summary: GitHubRateLimitSummary | None = None
+    if config and config.GITHUB_TOKEN:
+        try:
+            rate_limit_summary = probe_github_rate_limit()
+        except Exception:  # noqa: BLE001 — probe must never crash health
+            logger.exception("github rate-limit probe failed")
+            rate_limit_summary = None
+        component_statuses["github_token"] = (
+            rate_limit_summary.status if rate_limit_summary is not None else "degraded"
+        )
+    else:
+        component_statuses["github_token"] = "degraded"
 
     overall_status: Literal["healthy", "degraded", "unhealthy"]
     if "unhealthy" in component_statuses.values():
@@ -1363,4 +1392,5 @@ async def health() -> V2HealthResponse:
         status=overall_status,
         components=component_statuses,
         version=PACKAGE_VERSION,
+        github_rate_limit=rate_limit_summary,
     )