source-archive: skip only complete captures on re-runs (#298)

ContentStore.lookup now treats a capture as a cache hit only when it has every
expected format (html + markdown + screenshot; PDFs need only markdown). A
partial capture (e.g. a failed screenshot encode) becomes a miss, so a re-run
retries the missing format instead of skipping it forever.

Author: probably-jaden

code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_content_store.py

@@ -159,3 +159,36 @@ def test_different_content_not_aliased(tmp_path):
     b = store.store(_result("https://b.test/y", "<p>two different</p>"))
     assert b.capture.content_alias_of is None
     assert b.capture.html_key != a.capture.html_key
+
+
+def test_incomplete_capture_is_not_a_cache_hit(tmp_path):
+    # A browser capture whose screenshot failed to encode (screenshot_key=None)
+    # is not "done" — the next run should retry it to fill the missing format.
+    store = _store(tmp_path, ttl_days=14)
+    store.store(
+        CaptureResult(
+            url="https://a.test",
+            final_url="https://a.test",
+            status_code=200,
+            html="<p>one</p>",
+            markdown="md " * 50,
+            screenshot=None,
+            fetcher="cloakbrowser",
+        )
+    )
+    assert store.lookup("https://a.test") is None
+
+
+def test_pdf_capture_without_screenshot_is_complete(tmp_path):
+    # PDFs have no screenshot by nature, so markdown alone counts as complete.
+    store = _store(tmp_path, ttl_days=14)
+    store.store(
+        CaptureResult(
+            url="https://a.test/x.pdf",
+            final_url="https://a.test/x.pdf",
+            status_code=200,
+            markdown="md " * 50,
+            fetcher="pdf",
+        )
+    )
+    assert store.lookup("https://a.test/x.pdf") is not None

forecasting_tools/agents_and_tools/source_archive/content_store.py

@@ -69,6 +69,20 @@ def _parse_iso(ts: str) -> datetime:
     return dt
 
 
+def _capture_is_complete(cap: dict) -> bool:
+    """Whether a stored capture has every format we expect for its type.
+
+    A browser capture is complete only with html + markdown + screenshot; a PDF
+    (which has no screenshot) only needs its markdown. Used by :meth:`lookup` so
+    an incomplete capture is re-fetched rather than treated as already done.
+    """
+    if (cap.get("fetcher") or "").lower() == "pdf":
+        return bool(cap.get("markdown_key"))
+    return bool(
+        cap.get("html_key") and cap.get("markdown_key") and cap.get("screenshot_key")
+    )
+
+
 class ContentStore:
     def __init__(self, blob_store: BlobStore, config: ArchiveConfig | None = None):
         self.blobs = blob_store
@@ -166,6 +180,12 @@ def lookup(self, url: str) -> StoredCapture | None:
         age = datetime.now(timezone.utc) - last_seen
         if age > timedelta(days=self.config.ttl_days):
             return None
+        # Skip only a COMPLETE capture. A partial one (e.g. a failed screenshot
+        # encode left screenshot_key=None) is treated as a miss so the next run
+        # retries the missing format instead of skipping it forever. PDFs have no
+        # screenshot by nature, so they only need their markdown.
+        if not _capture_is_complete(latest):
+            return None
         return StoredCapture.model_validate(latest)
 
     def store(self, result: CaptureResult) -> StoreResult: