feat: v1.6.0 skip already-processed WiGLE uploads

HiroAlleyCat · HiroAlleyCat · commit 611287f362b1 · 2026-06-15T15:58:40.000-04:00
Daily --from-wigle re-downloaded the same upload every night; WiGLE
regenerates each CSV server-side (minutes for a large upload), so this
wasted time and quota. Now records each successfully pushed transid in
~/.config/wigle-to-wdgwars/processed-transids.json and skips re-pulling
it. Recorded only on a real (non-dry-run) success; --reprocess forces.
Adds tests/test_processed_skip.py (7 cases).
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,26 @@ All notable changes to wigle-to-wdgwars are documented here. Format
 follows [Keep a Changelog](https://keepachangelog.com/) and the
 project uses [Semantic Versioning](https://semver.org/).
 
+## [1.6.0] - 2026-06-15 - Skip already-processed uploads
+
+Stops the daily `--from-wigle` pull from re-downloading uploads it has
+already pushed. WiGLE regenerates each CSV server-side (minutes for a
+large upload), so re-pulling one that's already on WDGoWars wasted both
+time and quota every night.
+
+### Added
+
+- Persistent processed-transid state at
+  `~/.config/wigle-to-wdgwars/processed-transids.json`. Each WiGLE upload
+  is recorded only after a real (non-dry-run) successful push, so failed
+  uploads and dry-runs are retried.
+- `--reprocess` flag to re-pull uploads even if already recorded.
+
+### Changed
+
+- `--from-wigle` now skips already-processed uploads before downloading;
+  if nothing is new it exits early without hitting WiGLE's slow CSV export.
+
 ## [1.5.1] - 2026-06-15 - Survive slow WiGLE CSV exports
 
 ### Fixed
diff --git a/tests/test_processed_skip.py b/tests/test_processed_skip.py
@@ -0,0 +1,96 @@
+"""Tests for the processed-transid skip gate introduced in v1.6.0.
+
+The daily --from-wigle pull records each successfully pushed WiGLE upload
+in a state file and skips re-downloading it on the next run (WiGLE
+regenerates each CSV server-side, so re-pulling one already on WDGoWars
+wastes minutes). A transid is recorded only on a real, non-dry-run
+success; --reprocess ignores the state entirely.
+"""
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import wigle_to_wdgwars as w2w
+
+
+class ProcessedTransidTests(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.state = Path(self._tmp.name) / "processed-transids.json"
+        self._patch = mock.patch.object(w2w, "PROCESSED_FILE", self.state)
+        self._patch.start()
+
+    def tearDown(self):
+        self._patch.stop()
+        self._tmp.cleanup()
+
+    def test_load_empty_when_missing(self):
+        self.assertEqual(w2w._load_processed_transids(), set())
+
+    def test_mark_then_load_roundtrip(self):
+        w2w._mark_transid_processed("20260612-01443")
+        w2w._mark_transid_processed("20260613-00001")
+        self.assertEqual(w2w._load_processed_transids(),
+                         {"20260612-01443", "20260613-00001"})
+        data = json.loads(self.state.read_text())
+        self.assertEqual(data["processed"],
+                         ["20260612-01443", "20260613-00001"])
+
+    def test_skips_already_processed_without_downloading(self):
+        w2w._mark_transid_processed("T1")
+        with mock.patch.object(w2w, "wigle_list_transactions", return_value=["T1"]), \
+             mock.patch.object(w2w, "wigle_download_csv") as dl, \
+             mock.patch.object(w2w, "upload_csv_bytes") as up:
+            rc = w2w.pull_from_wigle_push_to_wdgwars(
+                "tok", "key", "file", latest=1, dry_run=False,
+                chunk_rows=10000, cooldown_sec=0)
+        self.assertEqual(rc, 0)
+        dl.assert_not_called()
+        up.assert_not_called()
+
+    def test_marks_on_real_success(self):
+        with mock.patch.object(w2w, "wigle_list_transactions", return_value=["T2"]), \
+             mock.patch.object(w2w, "wigle_download_csv", return_value=b"x"), \
+             mock.patch.object(w2w, "upload_csv_bytes", return_value=0):
+            rc = w2w.pull_from_wigle_push_to_wdgwars(
+                "tok", "key", "file", latest=1, dry_run=False,
+                chunk_rows=10000, cooldown_sec=0)
+        self.assertEqual(rc, 0)
+        self.assertIn("T2", w2w._load_processed_transids())
+
+    def test_dry_run_does_not_mark(self):
+        with mock.patch.object(w2w, "wigle_list_transactions", return_value=["T3"]), \
+             mock.patch.object(w2w, "wigle_download_csv", return_value=b"x"), \
+             mock.patch.object(w2w, "upload_csv_bytes", return_value=0):
+            w2w.pull_from_wigle_push_to_wdgwars(
+                "tok", "key", "file", latest=1, dry_run=True,
+                chunk_rows=10000, cooldown_sec=0)
+        self.assertNotIn("T3", w2w._load_processed_transids())
+
+    def test_failed_upload_not_marked(self):
+        with mock.patch.object(w2w, "wigle_list_transactions", return_value=["T4"]), \
+             mock.patch.object(w2w, "wigle_download_csv", return_value=b"x"), \
+             mock.patch.object(w2w, "upload_csv_bytes", return_value=1):
+            rc = w2w.pull_from_wigle_push_to_wdgwars(
+                "tok", "key", "file", latest=1, dry_run=False,
+                chunk_rows=10000, cooldown_sec=0)
+        self.assertEqual(rc, 1)
+        self.assertNotIn("T4", w2w._load_processed_transids())
+
+    def test_reprocess_ignores_state(self):
+        w2w._mark_transid_processed("T5")
+        with mock.patch.object(w2w, "wigle_list_transactions", return_value=["T5"]), \
+             mock.patch.object(w2w, "wigle_download_csv", return_value=b"x") as dl, \
+             mock.patch.object(w2w, "upload_csv_bytes", return_value=0):
+            w2w.pull_from_wigle_push_to_wdgwars(
+                "tok", "key", "file", latest=1, dry_run=False,
+                chunk_rows=10000, cooldown_sec=0, reprocess=True)
+        dl.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/wigle_to_wdgwars.py b/wigle_to_wdgwars.py
@@ -34,7 +34,7 @@
 """
 from __future__ import annotations
 
-__version__ = "1.5.1"
+__version__ = "1.6.0"
 GITHUB_REPO = "HiroAlleyCat/wigle-to-wdgwars"
 GITHUB_URL = f"https://github.com/{GITHUB_REPO}"
 
@@ -102,6 +102,7 @@
 WIGLE_KEY_FILE = CONFIG_DIR / "wigle.key"
 COOLDOWN_FILE = CONFIG_DIR / "cooldown.json"
 HWM_FILE = CONFIG_DIR / "hwm.json"
+PROCESSED_FILE = CONFIG_DIR / "processed-transids.json"
 
 # ───────────────────────────── Self-update / version check ──────────────────
 #
@@ -1264,29 +1265,73 @@ def wigle_download_csv(token: str, transid: str) -> bytes:
     sys.exit(f"[wigle] CSV download for {transid} timed out after retries: {last_err}")
 
 
+def _load_processed_transids() -> set:
+    """Transids already pushed to WDGoWars (persisted in PROCESSED_FILE).
+
+    Lets the daily pull skip re-downloading uploads it has already sent.
+    WiGLE regenerates each CSV server-side (minutes for a big upload), so
+    re-pulling one that's already on WDGoWars is pure waste."""
+    try:
+        data = json.loads(PROCESSED_FILE.read_text())
+        return set(data.get("processed", []))
+    except Exception:
+        return set()
+
+
+def _mark_transid_processed(transid: str) -> None:
+    """Record a transid as successfully pushed. Best-effort; never raises."""
+    try:
+        seen = _load_processed_transids()
+        seen.add(transid)
+        PROCESSED_FILE.parent.mkdir(parents=True, exist_ok=True)
+        PROCESSED_FILE.write_text(json.dumps({"processed": sorted(seen)}, indent=2))
+    except Exception as e:
+        print(f"[wigle] warning: could not record {transid} as processed: {e}",
+              file=sys.stderr)
+
+
 def pull_from_wigle_push_to_wdgwars(wigle_token: str, wdg_key: str, field: str,
                                     latest: int, dry_run: bool, chunk_rows: int,
                                     cooldown_sec: float,
-                                    since_seconds: int = 0) -> int:
+                                    since_seconds: int = 0,
+                                    reprocess: bool = False) -> int:
     """Pull your latest WiGLE upload(s) and push each to WDGoWars.
 
     ``since_seconds > 0`` drops rows older than the cutoff from each
-    downloaded WiGLE CSV before uploading."""
+    downloaded WiGLE CSV before uploading.
+
+    Uploads already recorded in PROCESSED_FILE are skipped (no re-download)
+    unless ``reprocess`` is set. A transid is recorded only after a real
+    (non-dry-run) successful push, so failed uploads and dry-runs are retried."""
     transids = wigle_list_transactions(wigle_token, latest)
     if not transids:
         print("[wigle] no uploads found on your account", file=sys.stderr)
         return 0
-    print(f"[wigle] pulling {len(transids)} most-recent upload(s): "
-          f"{', '.join(transids)}", file=sys.stderr)
+
+    processed = set() if reprocess else _load_processed_transids()
+    pending = [t for t in transids if t not in processed]
+    skipped = [t for t in transids if t in processed]
+    if skipped:
+        print(f"[wigle] skipping {len(skipped)} already-processed upload(s): "
+              f"{', '.join(skipped)} (use --reprocess to force)", file=sys.stderr)
+    if not pending:
+        print("[wigle] nothing new to pull — all recent uploads already processed",
+              file=sys.stderr)
+        return 0
+
+    print(f"[wigle] pulling {len(pending)} upload(s): "
+          f"{', '.join(pending)}", file=sys.stderr)
     rc = 0
-    for tid in transids:
+    for tid in pending:
         csv_bytes = wigle_download_csv(wigle_token, tid)
         print(f"[wigle] {tid}: {len(csv_bytes) / 1024:.1f} KB -> WDGoWars",
               file=sys.stderr)
         r = upload_csv_bytes(csv_bytes, f"{tid}.csv", wdg_key, field,
                              dry_run, chunk_rows, cooldown_sec,
                              since_seconds=since_seconds)
         rc = rc or r
+        if r == 0 and not dry_run:
+            _mark_transid_processed(tid)
     return rc
 
 
@@ -1843,6 +1888,9 @@ def main() -> int:
     ap.add_argument("--wigle-latest", type=int, default=1, metavar="N",
                     help="with --from-wigle, how many most-recent WiGLE uploads to pull "
                          "(default: 1)")
+    ap.add_argument("--reprocess", action="store_true",
+                    help="with --from-wigle, re-pull uploads even if already recorded "
+                         "as processed (ignores the processed-transids state file)")
     ap.add_argument("--setup", action="store_true",
                     help="interactive first-time setup — prompts for your "
                          "WDGoWars and WiGLE keys, validates them, saves them "
@@ -1944,7 +1992,7 @@ def main() -> int:
         return pull_from_wigle_push_to_wdgwars(
             wigle_token, key, args.field, args.wigle_latest,
             args.dry_run, args.chunk_size, args.chunk_cooldown,
-            since_seconds=since_seconds)
+            since_seconds=since_seconds, reprocess=args.reprocess)
 
     if args.aircraft_json:
         return upload_aircraft_json(Path(args.aircraft_json), key,
