fix(scraper): handle optional list and filter discovered links

obeone · obeone · commit 70e610b2abe7 · 2025-07-13T18:00:26.000+02:00
diff --git a/crawler_to_md/scraper.py b/crawler_to_md/scraper.py
@@ -186,7 +186,7 @@ def scrape_page(self, html, url):
             logger.error(f"Error scraping {url}: {e}")
             return None, None
 
-    def start_scraping(self, url=None, urls_list=[]):
+    def start_scraping(self, url=None, urls_list=None):
         """
         Initiates the scraping process for a single URL or a list of URLs.
         It validates URLs, logs the scraping process, and manages the
@@ -197,10 +197,11 @@ def start_scraping(self, url=None, urls_list=[]):
             urls_list (list, optional): A list of URLs to scrape.
         """
         # Validate and insert the provided URLs into the database
-        if urls_list:
+        urls = urls_list or []
+        if urls:
             # Build a new list of valid URLs without modifying the original list
             validated_urls = []
-            for url_item in urls_list:
+            for url_item in urls:
                 if not self.is_valid_link(url_item):
                     logger.warning(f"Skipping invalid URL: {url_item}")
                     continue
diff --git a/tests/test_scraper.py b/tests/test_scraper.py
@@ -262,3 +262,47 @@ def close(self):
     scraper.start_scraping(urls_list=urls)
 
     assert 'http://example.com/exclude/page' not in db.links
+
+
+def test_start_scraping_filters_discovered_links(monkeypatch):
+    db = ListDB()
+    scraper = Scraper(
+        base_url='http://example.com',
+        exclude_patterns=['/exclude'],
+        db_manager=db,
+    )
+
+    html = (
+        '<html><body>'
+        '<a href="/page1">1</a>'
+        '<a href="/exclude/page">2</a>'
+        '<a href="/page2">3</a>'
+        '</body></html>'
+    )
+
+    class DummyResp:
+        status_code = 200
+        headers = {'content-type': 'text/html'}
+        text = html
+
+    monkeypatch.setattr(scraper.session, 'get', lambda url: DummyResp())
+
+    monkeypatch.setattr(
+        Scraper, 'scrape_page', lambda self, html, url: ('# MD', {'url': url})
+    )
+
+    class DummyTqdm:
+        def __init__(self, *a, **k):
+            self.total = k.get('total', 0)
+        def update(self, n):
+            pass
+        def refresh(self):
+            pass
+        def close(self):
+            pass
+
+    monkeypatch.setattr(tqdm, 'tqdm', lambda *a, **k: DummyTqdm(*a, **k))
+
+    scraper.start_scraping(url='http://example.com')
+
+    assert 'http://example.com/exclude/page' not in db.links
