feat(proxy): validate proxy and add short flag

Author: obeone

README.md

@@ -52,7 +52,7 @@ pip install .
 Start scraping with the following command:
 
 ```shell
-crawler-to-md --url <URL> [--output-folder ./output] [--cache-folder ./cache] [--base-url <BASE_URL>] [--exclude <KEYWORD_IN_URL>] [--title <TITLE>] [--urls-file <URLS_FILE>] [--proxy <PROXY_URL>]
+crawler-to-md --url <URL> [--output-folder ./output] [--cache-folder ./cache] [--base-url <BASE_URL>] [--exclude <KEYWORD_IN_URL>] [--title <TITLE>] [--urls-file <URLS_FILE>] [-p <PROXY_URL>]
 ```
 
 Options:
@@ -67,7 +67,7 @@ Options:
 - `--export-individual`, `-ei`: Export each page as an individual Markdown file. 📝
 - `--rate-limit`, `-rl`: Maximum number of requests per minute (default: 0, no rate limit). ⏱️
 - `--delay`, `-d`: Delay between requests in seconds (default: 0, no delay). 🕒
-- `--proxy`: Proxy URL for HTTP or SOCKS requests. 🌐
+- `--proxy`, `-p`: Proxy URL for HTTP or SOCKS requests. 🌐
 
 One of the `--url` or `--urls-file` options is required.
 

crawler_to_md/cli.py

@@ -90,6 +90,7 @@ def main():
     )
     parser.add_argument(
         "--proxy",
+        "-p",
         help="Proxy URL for HTTP or SOCKS requests",
         default=None,
     )
@@ -171,14 +172,17 @@ def main():
     db_manager = DatabaseManager(db_path)
     logger.info("DatabaseManager initialized.")
 
-    scraper = Scraper(
-        base_url=args.base_url,
-        exclude_patterns=args.exclude,
-        db_manager=db_manager,
-        rate_limit=args.rate_limit,
-        delay=args.delay,
-        proxy=args.proxy,
-    )
+    try:
+        scraper = Scraper(
+            base_url=args.base_url,
+            exclude_patterns=args.exclude,
+            db_manager=db_manager,
+            rate_limit=args.rate_limit,
+            delay=args.delay,
+            proxy=args.proxy,
+        )
+    except ValueError as exc:
+        parser.error(str(exc))
     logger.info("Scraper initialized.")
 
     # Start the scraping process

crawler_to_md/scraper.py

@@ -37,6 +37,9 @@ def __init__(
             rate_limit (int): Maximum number of requests per minute.
             delay (float): Delay between requests in seconds.
             proxy (str, optional): Proxy URL for HTTP or SOCKS requests.
+
+        Raises:
+            ValueError: If a proxy is provided but unreachable.
         """
         logger.debug(f"Initializing Scraper with base URL: {base_url}")
         self.base_url = base_url
@@ -49,6 +52,21 @@ def __init__(
             self.session.proxies.update({"http": proxy, "https": proxy})
         self.proxy = proxy
 
+        if proxy:
+            self._test_proxy()
+
+    def _test_proxy(self):
+        """
+        Ensure the configured proxy is reachable.
+
+        Raises:
+            ValueError: If the proxy cannot fetch the base URL.
+        """
+        try:
+            self.session.head(self.base_url, timeout=5)
+        except requests.RequestException as exc:
+            raise ValueError(f"Proxy unreachable: {exc}") from exc
+
     def is_valid_link(self, link):
         """
         Check if the given link is valid for scraping.

tests/test_cli.py

@@ -1,5 +1,7 @@
 import sys
 
+import pytest
+
 from crawler_to_md import cli
 from crawler_to_md.export_manager import ExportManager
 from crawler_to_md.scraper import Scraper
@@ -82,6 +84,42 @@ def fake_init(
     assert captured.get('proxy') == 'http://proxy:8080'
 
 
+def test_cli_proxy_short_option(monkeypatch, tmp_path):
+    captured = {}
+
+    def fake_init(
+        self,
+        base_url,
+        exclude_patterns,
+        db_manager,
+        rate_limit=0,
+        delay=0,
+        proxy=None,
+    ):
+        captured['proxy'] = proxy
+
+    monkeypatch.setattr(Scraper, '__init__', fake_init)
+    monkeypatch.setattr(Scraper, 'start_scraping', lambda *a, **k: None)
+    monkeypatch.setattr(ExportManager, 'export_to_markdown', lambda *a, **k: None)
+    monkeypatch.setattr(ExportManager, 'export_to_json', lambda *a, **k: None)
+
+    cache_folder = tmp_path / 'cache'
+    args = [
+        'prog',
+        '--url',
+        'http://example.com',
+        '--output-folder',
+        str(tmp_path),
+        '--cache-folder',
+        str(cache_folder),
+        '-p',
+        'http://proxy:8080',
+    ]
+    monkeypatch.setattr(sys, 'argv', args)
+    cli.main()
+    assert captured.get('proxy') == 'http://proxy:8080'
+
+
 def test_cli_socks_proxy(monkeypatch, tmp_path):
     captured = {}
 
@@ -117,3 +155,25 @@ def fake_init(
     cli.main()
     assert captured.get('proxy') == 'socks5://localhost:9050'
 
+
+def test_cli_proxy_error(monkeypatch, tmp_path):
+    def fake_init(*a, **k):
+        raise ValueError('Proxy unreachable')
+
+    monkeypatch.setattr(Scraper, '__init__', fake_init)
+    cache_folder = tmp_path / 'cache'
+    args = [
+        'prog',
+        '--url',
+        'http://example.com',
+        '--output-folder',
+        str(tmp_path),
+        '--cache-folder',
+        str(cache_folder),
+        '--proxy',
+        'http://proxy:8080',
+    ]
+    monkeypatch.setattr(sys, 'argv', args)
+    with pytest.raises(SystemExit):
+        cli.main()
+

tests/test_scraper.py

@@ -1,5 +1,7 @@
 from unittest.mock import MagicMock, patch
 
+import pytest
+import requests
 import tqdm
 
 from crawler_to_md.database_manager import DatabaseManager
@@ -172,20 +174,34 @@ def close(self):
     assert db.pages[0][0] == 'http://example.com/page'
 
 
-def test_scraper_proxy_initialization():
+def test_scraper_proxy_initialization(monkeypatch):
     db = DummyDB()
+    monkeypatch.setattr(Scraper, '_test_proxy', lambda self: None)
     scraper = Scraper(
         base_url='http://example.com', exclude_patterns=[], db_manager=db, proxy='http://proxy:8080'
     )
     assert scraper.session.proxies.get('http') == 'http://proxy:8080'
     assert scraper.session.proxies.get('https') == 'http://proxy:8080'
 
 
-def test_scraper_socks_proxy_initialization():
+def test_scraper_socks_proxy_initialization(monkeypatch):
     db = DummyDB()
     proxy = 'socks5://localhost:9050'
+    monkeypatch.setattr(Scraper, '_test_proxy', lambda self: None)
     scraper = Scraper(
         base_url='http://example.com', exclude_patterns=[], db_manager=db, proxy=proxy
     )
     assert scraper.session.proxies.get('http') == proxy
     assert scraper.session.proxies.get('https') == proxy
+
+
+def test_scraper_proxy_failure_detection(monkeypatch):
+    db = DummyDB()
+    def fake_head(self, url, timeout=5):
+        raise requests.exceptions.ProxyError("fail")
+
+    monkeypatch.setattr(requests.Session, 'head', fake_head)
+    with pytest.raises(ValueError):
+        Scraper(
+            base_url='http://example.com', exclude_patterns=[], db_manager=db, proxy='http://proxy:8080'
+        )