predup check for all urls

Author: Barbara Miller

brozzler/worker.py

@@ -316,6 +316,25 @@ def should_ytdlp(self, logger, site, page, page_status):
                     ytdlp_url,
                     str(e),
                 )
+        else:
+            # check all other urls, using default value for recent, currently 30 days
+            try:
+                recent_capture_exists = self._video_data.recent_video_captures(
+                    site, ytdlp_url
+                )
+                if recent_capture_exists:
+                    logger.info(
+                        "recent previous captures of %s found, skipping ytdlp",
+                        ytdlp_url,
+                    )
+                    return False
+            except Exception as e:
+                logger.warning(
+                    "exception querying for previous capture for %s: %s",
+                    ytdlp_url,
+                    str(e),
+                )
+
         return True
 
     @metrics.brozzler_page_processing_duration_seconds.time()