feat: implement site auditing command with crawler and PDF report generation capabilities

Author: vikasswebdev

results/festivaldryfruits_in_seo_report.pdf

@@ -409,7 +409,7 @@ def audit(
         max_pages: Annotated[int, typer.Option("--max-pages", help="Max pages to crawl for site audits (0 for unlimited)")] = 10,
         max_depth: Annotated[int, typer.Option("--depth", help="Maximum directory depth to crawl")] = 3,
         ignore_robots: Annotated[bool, typer.Option("--ignore-robots", help="Ignore robots.txt instructions")] = False,
-        report: Annotated[bool, typer.Option("--report", help="Generate a PDF report in results/ folder")] = False,
+        report: Annotated[bool, typer.Option("--report", help="Generate a PDF report in Downloads folder")] = False,
         sitemap_only: Annotated[bool, typer.Option("--sitemap-only", help="Only crawl URLs listed in sitemap.xml")] = False,
         interactive: Annotated[bool, typer.Option("--interactive", "-i", help="Run audit in interactive mode")] = False,
         verbose: Annotated[bool, typer.Option("--verbose", "-v", help="Verbose output")] = False,
@@ -444,7 +444,7 @@ def audit(
             no_llm = disable_llm
 
             # Ask about PDF report
-            report = typer.confirm("Generate a PDF report in results/ folder?", default=True)
+            report = typer.confirm("Generate a PDF report in Downloads folder?", default=True)
             typer.secho("\n🚀 Configuration locked! Starting audit...\n", fg=typer.colors.GREEN)
 
         asyncio.run(

results/site_seo_report.pdf

@@ -12,7 +12,7 @@
 from openseo.crawler.http import HttpCrawler
 from openseo.crawler.playwright_crawler import PlaywrightCrawler
 from openseo.models.page import Page
-from openseo.constants import USER_AGENT
+from openseo.constants import CRAWLER_TIMEOUT, USER_AGENT
 
 logger = logging.getLogger(__name__)
 
@@ -84,7 +84,7 @@ async def initialize(self) -> None:
             if page.status_code == 200 and page.body_text:
                 # Page extractor gets visible text. We need raw body, let's fetch it as text
                 import httpx
-                async with httpx.AsyncClient(headers={"User-Agent": USER_AGENT}) as client:
+                async with httpx.AsyncClient(headers={"User-Agent": USER_AGENT}, timeout=CRAWLER_TIMEOUT) as client:
                     resp = await client.get(robots_url, follow_redirects=True)
                     if resp.status_code == 200:
                         self.robots_txt_content = resp.text
@@ -109,7 +109,7 @@ async def initialize(self) -> None:
             for s_url in self.sitemap_urls:
                 try:
                     import httpx
-                    async with httpx.AsyncClient(headers={"User-Agent": USER_AGENT}) as client:
+                    async with httpx.AsyncClient(headers={"User-Agent": USER_AGENT}, timeout=CRAWLER_TIMEOUT) as client:
                         resp = await client.get(s_url, follow_redirects=True)
                         if resp.status_code == 200:
                             soup = BeautifulSoup(resp.text, "xml")

results/www_festivaldryfruits_in_seo_report.pdf

@@ -27,8 +27,8 @@ class PdfRenderer:
     """Generates beautiful, professional PDF reports for website audits."""
 
     def __init__(self, output_dir: Path | None = None) -> None:
-        # Default to "results" folder in the current working directory
-        self.output_dir = output_dir or Path.cwd() / "results"
+        # Default to "Downloads" folder in the user's home directory
+        self.output_dir = output_dir or Path.home() / "Downloads"
         self.output_dir.mkdir(parents=True, exist_ok=True)
 
     def generate_report(self, result: Any) -> str:

src/openseo/commands/audit.py

@@ -36,3 +36,11 @@ def test_pdf_report_generation(tmp_path):
 
     assert Path(file_path).exists()
     assert Path(file_path).name == "example_com_seo_report.pdf"
+
+
+def test_pdf_report_generation_default_dir(monkeypatch, tmp_path):
+    monkeypatch.setattr(Path, "home", lambda: tmp_path)
+    renderer = PdfRenderer()
+    assert renderer.output_dir == tmp_path / "Downloads"
+    assert renderer.output_dir.exists()
+