feat(tools): add URLReadTool for reading arbitrary URLs

lib/crewai-tools/src/crewai_tools/__init__.py

@@ -209,6 +209,7 @@
 )
 from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool
 from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool
+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
 from crewai_tools.tools.vision_tool.vision_tool import VisionTool
 from crewai_tools.tools.wait_tool.wait_tool import WaitTool
 from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool
@@ -327,6 +328,7 @@
     "TavilyGetResearchTool",
     "TavilyResearchTool",
     "TavilySearchTool",
+    "URLReadTool",
     "VisionTool",
     "WaitTool",
     "WeaviateVectorSearchTool",

lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py

@@ -2,13 +2,17 @@
 
 import os
 from pathlib import Path
-import tempfile
 from typing import Any
 from urllib.parse import urlparse
 
 from crewai_tools.rag.base_loader import BaseLoader, LoaderResult
 from crewai_tools.rag.source_content import SourceContent
-from crewai_tools.security.safe_requests import safe_get
+from crewai_tools.security.safe_requests import safe_get_bounded
+
+
+# Remote PDFs are held in memory for the whole extraction, so the download needs
+# a ceiling. Override per call with the ``max_bytes`` kwarg to ``load``.
+DEFAULT_MAX_PDF_BYTES = 50 * 1024 * 1024
 
 
 class PDFLoader(BaseLoader):
@@ -24,18 +28,24 @@ def _is_url(path: str) -> bool:
             return False
 
     @staticmethod
-    def _download_from_url(url: str, kwargs: dict[str, Any]) -> str:
-        """Download PDF from a URL to a temporary file and return its path.
+    def _fetch_from_url(url: str, kwargs: dict[str, Any]) -> bytes:
+        """Download a PDF from a URL and return its bytes.
+
+        The content stays in memory rather than going to a temporary file: the
+        whole body has to be buffered either way, and a temp file would need
+        unlinking on every error path to avoid leaving files behind. Because it
+        is held in memory, the download is capped.
 
         Args:
             url: The URL to download from.
-            kwargs: Optional dict that may contain custom headers.
+            kwargs: Optional dict that may contain custom ``headers`` and a
+                ``max_bytes`` ceiling for the download.
 
         Returns:
-            Path to the temporary file containing the PDF.
+            The raw PDF content.
 
         Raises:
-            ValueError: If the download fails.
+            ValueError: If the download fails or exceeds the size ceiling.
         """
         headers = kwargs.get(
             "headers",
@@ -46,12 +56,13 @@ def _download_from_url(url: str, kwargs: dict[str, Any]) -> str:
         )
 
         try:
-            response = safe_get(url, headers=headers, timeout=30)
-            response.raise_for_status()
-
-            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file:
-                temp_file.write(response.content)
-                return temp_file.name
+            body, _content_type, _final_url = safe_get_bounded(
+                url,
+                max_bytes=kwargs.get("max_bytes", DEFAULT_MAX_PDF_BYTES),
+                headers=headers,
+                timeout=30,
+            )
+            return body
         except Exception as e:
             raise ValueError(f"Failed to download PDF from {url}: {e!s}") from e
 
@@ -93,21 +104,25 @@ def load(self, source: SourceContent, **kwargs: Any) -> LoaderResult:  # type: i
 
         try:
             if is_url:
-                local_path = self._download_from_url(file_path, kwargs)
-                doc = pymupdf.open(local_path)
+                doc = pymupdf.open(
+                    stream=self._fetch_from_url(file_path, kwargs), filetype="pdf"
+                )
             else:
                 if not os.path.isfile(file_path):
                     raise FileNotFoundError(f"PDF file not found: {file_path}")
                 doc = pymupdf.open(file_path)
 
-            metadata["num_pages"] = len(doc)
-
-            for page_num, page in enumerate(doc, 1):
-                page_text = page.get_text()
-                if page_text.strip():
-                    text_content.append(f"Page {page_num}:\n{page_text}")
-
-            doc.close()
+            # Closed in a finally so a failure mid-extraction still releases the
+            # document handle.
+            try:
+                metadata["num_pages"] = len(doc)
+
+                for page_num, page in enumerate(doc, 1):
+                    page_text = page.get_text()
+                    if page_text.strip():
+                        text_content.append(f"Page {page_num}:\n{page_text}")
+            finally:
+                doc.close()
         except FileNotFoundError:
             raise
         except Exception as e:

lib/crewai-tools/src/crewai_tools/security/safe_requests.py

@@ -11,6 +11,7 @@
 
 
 _REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}
+_STREAM_CHUNK_SIZE = 65536
 _SENSITIVE_HEADER_NAMES = {
     "authorization",
     "cookie",
@@ -49,40 +50,124 @@ def _strip_cross_origin_credentials(request_kwargs: dict[str, Any]) -> dict[str,
 
 
 def safe_get(url: str, *, max_redirects: int = 10, **kwargs: Any) -> requests.Response:
-    """GET a URL while validating each redirect target before following it."""
+    """GET a URL while validating each redirect target before following it.
+
+    On success the hops are attached to the returned response's ``history`` and
+    are the caller's to close. On failure they are closed here: a caller given
+    an exception has no handle on them, and a streamed hop holds its connection
+    until its body is read or closed.
+    """
     current_url = validate_url(url)
     request_kwargs = {**kwargs, "allow_redirects": False}
     timeout = request_kwargs.pop("timeout", 30)
     history: list[requests.Response] = []
     redirects_followed = 0
 
-    while True:
-        response = requests.get(current_url, timeout=timeout, **request_kwargs)
-        if (
-            response.status_code not in _REDIRECT_STATUS_CODES
-            or "Location" not in response.headers
-        ):
-            response.history = history
-            return response
-
-        if redirects_followed >= max_redirects:
-            response.close()
-            raise ValueError(f"Too many redirects while fetching URL: {url}")
-
-        location = response.headers.get("Location")
-        if not location:
-            response.history = history
-            return response
-
-        try:
-            redirect_url = validate_url(urljoin(response.url, location))
-        except ValueError:
-            response.close()
-            raise
-
-        if not _same_origin(current_url, redirect_url):
-            request_kwargs = _strip_cross_origin_credentials(request_kwargs)
-
-        history.append(response)
-        current_url = redirect_url
-        redirects_followed += 1
+    try:
+        while True:
+            response = requests.get(current_url, timeout=timeout, **request_kwargs)
+            if (
+                response.status_code not in _REDIRECT_STATUS_CODES
+                or "Location" not in response.headers
+            ):
+                response.history = history
+                return response
+
+            if redirects_followed >= max_redirects:
+                response.close()
+                raise ValueError(f"Too many redirects while fetching URL: {url}")
+
+            location = response.headers.get("Location")
+            if not location:
+                response.history = history
+                return response
+
+            try:
+                redirect_url = validate_url(urljoin(response.url, location))
+            except ValueError:
+                response.close()
+                raise
+
+            if not _same_origin(current_url, redirect_url):
+                request_kwargs = _strip_cross_origin_credentials(request_kwargs)
+
+            history.append(response)
+            current_url = redirect_url
+            redirects_followed += 1
+    except BaseException:
+        for hop in history:
+            hop.close()
+        raise
+
+
+def safe_get_bounded(
+    url: str,
+    *,
+    max_bytes: int,
+    timeout: float | tuple[float, float] = 30,
+    headers: dict[str, str] | None = None,
+    max_redirects: int = 10,
+) -> tuple[bytes, str, str]:
+    """GET a URL through :func:`safe_get`, refusing bodies over *max_bytes*.
+
+    The body is streamed and abandoned as soon as it crosses the limit, so an
+    oversized response costs one chunk of memory instead of all of it. The cap
+    counts decoded bytes, which is what a compressed response expands into --
+    ``Content-Length`` describes the wire size and cannot bound that.
+
+    Args:
+        url: The URL to fetch.
+        max_bytes: Largest body to accept, in decoded bytes.
+        timeout: Request timeout, passed through to requests.
+        headers: Request headers.
+        max_redirects: Hops to follow before giving up.
+
+    Returns:
+        A ``(body, content_type, final_url)`` tuple, where *final_url* is the
+        last validated URL in the redirect chain.
+
+    Raises:
+        ValueError: If *max_bytes* is not positive, URL validation fails, the
+            redirect chain is too long, or the body exceeds *max_bytes*.
+        requests.RequestException: If the request fails or returns an error
+            status.
+    """
+    if max_bytes <= 0:
+        raise ValueError(f"max_bytes must be positive, got {max_bytes}.")
+
+    response = safe_get(
+        url,
+        max_redirects=max_redirects,
+        headers=headers,
+        timeout=timeout,
+        stream=True,
+    )
+    try:
+        response.raise_for_status()
+
+        chunks: list[bytes] = []
+        total = 0
+        for chunk in response.iter_content(chunk_size=_STREAM_CHUNK_SIZE):
+            if not chunk:
+                continue
+            total += len(chunk)
+            if total > max_bytes:
+                # Names the URL that served the body, which after a redirect is
+                # not the one that was requested.
+                raise ValueError(
+                    f"Response body from '{response.url}' exceeds the "
+                    f"{max_bytes} byte limit."
+                )
+            chunks.append(chunk)
+
+        return (
+            b"".join(chunks),
+            response.headers.get("Content-Type", ""),
+            response.url,
+        )
+    finally:
+        # Under stream=True each hop holds its connection until the body is read,
+        # so the redirects need closing too, not just the response we return.
+        for hop in response.history:
+            hop.close()
+        response.close()

lib/crewai-tools/src/crewai_tools/tools/__init__.py

@@ -196,6 +196,7 @@
 )
 from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool
 from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool
+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
 from crewai_tools.tools.vision_tool.vision_tool import VisionTool
 from crewai_tools.tools.wait_tool.wait_tool import WaitTool
 from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool
@@ -310,6 +311,7 @@
     "TavilyGetResearchTool",
     "TavilyResearchTool",
     "TavilySearchTool",
+    "URLReadTool",
     "VisionTool",
     "WaitTool",
     "WeaviateVectorSearchTool",

lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py

@@ -0,0 +1,4 @@
+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
+
+
+__all__ = ["URLReadTool"]