fix: skip non-posted payments (pending/scheduled/processing) in allocation extraction

Generalizes the cancelled-date skip to detect all non-posted payment
statuses (pending, scheduled, processing, cancelled) so the scraper
no longer tries to click into non-clickable rows.

Single-row parse failures are now non-fatal (warn + continue) instead
of aborting the entire run. If ALL rows fail, a RuntimeError is still
raised as a safety net.

Closes #10

Made-with: Cursor

Author: UnexpectedFisting

src/studentaid_monarch_sync/portal/client.py

@@ -1852,13 +1852,13 @@ def _extract_payment_allocations(
         # Best-effort: switch the history filter from "Last 12 Months" to "All" so older payments
         # are visible when scanning. If this fails, we proceed with the default selection.
         self._try_select_payment_activity_show_all(page)
-        cancelled_payment_dates: set[date] = set()
+        non_posted_dates: dict[date, str] = {}
         try:
-            cancelled_payment_dates = self._cancelled_payment_dates_from_payment_activity_text(page.inner_text("body"))
+            non_posted_dates = self._non_posted_payment_dates_from_payment_activity_text(page.inner_text("body"))
         except Exception:
-            cancelled_payment_dates = set()
-        if cancelled_payment_dates:
-            logger.info("Detected %d cancelled payment entries; skipping them.", len(cancelled_payment_dates))
+            non_posted_dates = {}
+        if non_posted_dates:
+            logger.info("Detected %d non-posted payment entries; skipping them.", len(non_posted_dates))
 
         # Primary strategy: click the Payment Date links in the history table (they are the most stable entry point).
         # These appear as links like "11/26/2025".
@@ -1916,8 +1916,12 @@ def _collect_date_texts() -> list[str]:
                     )
                     break
 
-                if payment_dt in cancelled_payment_dates:
-                    logger.info("Skipping cancelled payment entry dated %s.", payment_dt.isoformat())
+                if payment_dt in non_posted_dates:
+                    logger.info(
+                        "Skipping non-posted payment entry dated %s (status=%s).",
+                        payment_dt.isoformat(),
+                        non_posted_dates[payment_dt],
+                    )
                     continue
 
                 if opened >= max_payments_to_scan:
@@ -1951,11 +1955,19 @@ def _collect_date_texts() -> list[str]:
                     )
                 except Exception:
                     self._save_debug(page, debug_dir=debug_dir, name_prefix=f"payment_detail_{idx}_error")
-                    raise
+                    logger.warning(
+                        "Failed to parse payment detail for date %s (idx=%d); skipping row.",
+                        dt_str, idx, exc_info=True,
+                    )
                 finally:
                     # Return to Payment Activity list without relying on browser history.
                     self._close_payment_detail(page)
 
+            if not allocations and opened > 0:
+                raise RuntimeError(
+                    f"All {opened} payment detail rows failed to parse; aborting. "
+                    "Check debug artifacts (payment_detail_*_error) for details."
+                )
             return allocations
 
         # Gather clickable \"View/Details\" elements.
@@ -2047,53 +2059,59 @@ def _try_select_payment_activity_show_all(self, page: Page) -> None:
         except Exception:
             pass
 
-    def _cancelled_payment_dates_from_payment_activity_text(self, body_text: str) -> set[date]:
+    _NON_POSTED_STATUS_RE = re.compile(
+        r"\b(cancel+l?ed|pending|scheduled|processing)\b", re.I
+    )
+
+    def _non_posted_payment_dates_from_payment_activity_text(
+        self, body_text: str
+    ) -> dict[date, str]:
         """
-        Parse the Payment Activity list view and find payment dates whose status is "Cancelled".
+        Parse the Payment Activity list view and find payment dates whose status is non-posted
+        (cancelled, pending, scheduled, processing).
 
-        This is used to avoid clicking into cancelled entries, since we only want posted/successful
-        payment allocations.
+        Returns a dict mapping each non-posted date to the matched status keyword so callers
+        can log which status caused the skip.
         """
         lines = [ln.strip() for ln in (body_text or "").splitlines() if ln.strip()]
 
         date_start_re = re.compile(r"^(\d{1,2}/\d{1,2}/\d{4})\b")
-        cancelled_word_re = re.compile(r"\bcancel+l?ed\b", re.I)
 
-        def _block_is_cancelled(block_lines: list[str]) -> bool:
-            # Most commonly the status is its own line ("Cancelled"), but some layouts may inline it.
-            for ln in block_lines:
-                if re.fullmatch(r"cancel+l?ed", ln, re.I):
-                    return True
+        def _non_posted_status(block_lines: list[str]) -> Optional[str]:
             for ln in block_lines:
-                if cancelled_word_re.search(ln) and "$" in ln:
-                    return True
-            return False
+                m = self._NON_POSTED_STATUS_RE.search(ln)
+                if m:
+                    return m.group(1).lower()
+            return None
 
-        out: set[date] = set()
+        out: dict[date, str] = {}
         current_date: Optional[str] = None
         current_block: list[str] = []
 
         for ln in lines:
             m = date_start_re.match(ln)
             if m:
-                # Finalize previous block.
-                if current_date and _block_is_cancelled(current_block):
-                    try:
-                        out.add(parse_us_date(current_date))
-                    except Exception:
-                        pass
+                if current_date:
+                    status = _non_posted_status(current_block)
+                    if status:
+                        try:
+                            out[parse_us_date(current_date)] = status
+                        except Exception:
+                            pass
                 current_date = m.group(1)
                 current_block = [ln]
                 continue
 
             if current_date is not None:
                 current_block.append(ln)
 
-        if current_date and _block_is_cancelled(current_block):
-            try:
-                out.add(parse_us_date(current_date))
-            except Exception:
-                pass
+        if current_date:
+            status = _non_posted_status(current_block)
+            if status:
+                try:
+                    out[parse_us_date(current_date)] = status
+                except Exception:
+                    pass
 
         return out
 

tests/test_parsing.py

@@ -147,7 +147,7 @@ def test_parse_payment_allocations_parses_row_with_prefix_text_when_expected_gro
     assert all(a.payment_total_cents == (2571 + 1699) for a in allocs)
 
 
-def test_payment_activity_cancelled_detection_finds_cancelled_dates() -> None:
+def test_non_posted_detection_finds_cancelled_dates() -> None:
     c = _client()
     body = """
     Payment History
@@ -159,5 +159,65 @@ def test_payment_activity_cancelled_detection_finds_cancelled_dates() -> None:
     02/13/2025 $34,632.04 $15,606.93 $19,025.11 Electronic
     """.strip()
 
-    got = c._cancelled_payment_dates_from_payment_activity_text(body)
-    assert got == {date(2025, 3, 1), date(2025, 2, 14)}
+    got = c._non_posted_payment_dates_from_payment_activity_text(body)
+    assert set(got.keys()) == {date(2025, 3, 1), date(2025, 2, 14)}
+    assert all("cancel" in v for v in got.values())
+
+
+def test_non_posted_detection_finds_pending_dates() -> None:
+    c = _client()
+    body = """
+    Payment Date Payment Amount Applied to Principal Applied to Interest Payment Type
+
+    01/15/2026 $500.00 $0.00 $0.00 Pending
+    01/10/2026 $278.52 $184.12 $94.40 Electronic
+    """.strip()
+
+    got = c._non_posted_payment_dates_from_payment_activity_text(body)
+    assert date(2026, 1, 15) in got
+    assert got[date(2026, 1, 15)] == "pending"
+    assert date(2026, 1, 10) not in got
+
+
+def test_non_posted_detection_finds_scheduled_dates() -> None:
+    c = _client()
+    body = """
+    02/01/2026 $250.00 $0.00 $0.00 Scheduled
+    01/15/2026 $278.52 $184.12 $94.40 Electronic
+    """.strip()
+
+    got = c._non_posted_payment_dates_from_payment_activity_text(body)
+    assert date(2026, 2, 1) in got
+    assert got[date(2026, 2, 1)] == "scheduled"
+    assert date(2026, 1, 15) not in got
+
+
+def test_non_posted_detection_finds_processing_dates() -> None:
+    c = _client()
+    body = """
+    02/15/2026 $300.00 $0.00 $0.00 Processing
+    02/01/2026 $278.52 $184.12 $94.40 Electronic
+    """.strip()
+
+    got = c._non_posted_payment_dates_from_payment_activity_text(body)
+    assert date(2026, 2, 15) in got
+    assert got[date(2026, 2, 15)] == "processing"
+    assert date(2026, 2, 1) not in got
+
+
+def test_non_posted_detection_mixed_statuses() -> None:
+    """Multiple non-posted statuses in one body text are all detected."""
+    c = _client()
+    body = """
+    03/01/2026 $100.00 $0.00 $0.00 Pending
+    02/15/2026 $200.00 $0.00 $0.00 Processing
+    02/01/2026 $150.00 $0.00 $0.00 Cancelled
+    01/15/2026 $278.52 $184.12 $94.40 Electronic
+    """.strip()
+
+    got = c._non_posted_payment_dates_from_payment_activity_text(body)
+    assert len(got) == 3
+    assert got[date(2026, 3, 1)] == "pending"
+    assert got[date(2026, 2, 15)] == "processing"
+    assert got[date(2026, 2, 1)] == "cancelled"
+    assert date(2026, 1, 15) not in got