feat(tools): add URLReadTool for reading arbitrary URLs

FileReadTool is confined to the local filesystem, so there was no way for
an agent to read a document that lives behind an http(s) URL. Rather than
adding a flag to FileReadTool, this adds a separate tool: granting it
grants network egress to addresses an LLM picks at runtime, and that
should be a deliberate choice rather than a toggle on a filesystem tool.

URLReadTool fetches a URL and returns its content as text. PDF and DOCX
bodies have their text extracted, HTML is stripped to visible text, and
text-shaped types (plain text, Markdown, JSON, XML, YAML, CSV) are
decoded using the charset the server declares. Any other content type is
refused rather than returned as base64, keeping the output text-only.

Requests reuse the existing SSRF protections in security/safe_requests:
validate_url resolves every hostname and rejects private, loopback,
link-local and reserved addresses (covering cloud metadata endpoints),
and safe_get never auto-follows redirects, revalidating each hop and
dropping credentials on cross-origin ones. Resolving before validating
also normalizes encoded forms, so http://2130706433/ is rejected as
127.0.0.1 without needing a string blocklist.

Adds safe_get_bounded on top of that, which streams the body and
abandons it once it crosses max_bytes. The cap counts decoded bytes,
which is what a compressed response expands into -- Content-Length
describes the wire size and cannot bound that. It also closes the
redirect hops, which stream=True would otherwise leave holding their
connections.

Two risks are documented rather than closed. Validation resolves the
hostname and requests resolves it again to connect, so DNS rebinding
remains possible; closing it needs the connection pinned to the
validated address, which would change behavior for all existing
safe_get callers. And the returned text is untrusted remote content
entering an agent's context, which input validation cannot address.

Also fixes a temp file leak in PDFLoader, which reached the same
pymupdf-from-URL path. It wrote downloads to NamedTemporaryFile with
delete=False and never unlinked them, so every PDF ingested from a URL
left a file behind. It now opens from memory, the way URLReadTool does,
which removes the leak by construction instead of relying on cleanup on
each error path; its doc.close() also moves into a finally so a failure
mid-extraction still releases the handle. PDFLoader had no test file, so
this adds one covering both paths plus a regression test asserting no
temp file is created.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: joaomdmoura

lib/crewai-tools/src/crewai_tools/__init__.py

@@ -209,6 +209,7 @@
 )
 from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool
 from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool
+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
 from crewai_tools.tools.vision_tool.vision_tool import VisionTool
 from crewai_tools.tools.wait_tool.wait_tool import WaitTool
 from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool
@@ -327,6 +328,7 @@
     "TavilyGetResearchTool",
     "TavilyResearchTool",
     "TavilySearchTool",
+    "URLReadTool",
     "VisionTool",
     "WaitTool",
     "WeaviateVectorSearchTool",

lib/crewai-tools/src/crewai_tools/rag/loaders/pdf_loader.py

@@ -2,7 +2,6 @@
 
 import os
 from pathlib import Path
-import tempfile
 from typing import Any
 from urllib.parse import urlparse
 
@@ -24,15 +23,19 @@ def _is_url(path: str) -> bool:
             return False
 
     @staticmethod
-    def _download_from_url(url: str, kwargs: dict[str, Any]) -> str:
-        """Download PDF from a URL to a temporary file and return its path.
+    def _fetch_from_url(url: str, kwargs: dict[str, Any]) -> bytes:
+        """Download a PDF from a URL and return its bytes.
+
+        The content stays in memory rather than going to a temporary file: the
+        whole body has to be buffered either way, and a temp file would need
+        unlinking on every error path to avoid leaving files behind.
 
         Args:
             url: The URL to download from.
             kwargs: Optional dict that may contain custom headers.
 
         Returns:
-            Path to the temporary file containing the PDF.
+            The raw PDF content.
 
         Raises:
             ValueError: If the download fails.
@@ -48,10 +51,7 @@ def _download_from_url(url: str, kwargs: dict[str, Any]) -> str:
         try:
             response = safe_get(url, headers=headers, timeout=30)
             response.raise_for_status()
-
-            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file:
-                temp_file.write(response.content)
-                return temp_file.name
+            return response.content
         except Exception as e:
             raise ValueError(f"Failed to download PDF from {url}: {e!s}") from e
 
@@ -93,21 +93,25 @@ def load(self, source: SourceContent, **kwargs: Any) -> LoaderResult:  # type: i
 
         try:
             if is_url:
-                local_path = self._download_from_url(file_path, kwargs)
-                doc = pymupdf.open(local_path)
+                doc = pymupdf.open(
+                    stream=self._fetch_from_url(file_path, kwargs), filetype="pdf"
+                )
             else:
                 if not os.path.isfile(file_path):
                     raise FileNotFoundError(f"PDF file not found: {file_path}")
                 doc = pymupdf.open(file_path)
 
-            metadata["num_pages"] = len(doc)
-
-            for page_num, page in enumerate(doc, 1):
-                page_text = page.get_text()
-                if page_text.strip():
-                    text_content.append(f"Page {page_num}:\n{page_text}")
-
-            doc.close()
+            # Closed in a finally so a failure mid-extraction still releases the
+            # document handle.
+            try:
+                metadata["num_pages"] = len(doc)
+
+                for page_num, page in enumerate(doc, 1):
+                    page_text = page.get_text()
+                    if page_text.strip():
+                        text_content.append(f"Page {page_num}:\n{page_text}")
+            finally:
+                doc.close()
         except FileNotFoundError:
             raise
         except Exception as e:

lib/crewai-tools/src/crewai_tools/security/safe_requests.py

@@ -11,6 +11,7 @@
 
 
 _REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}
+_STREAM_CHUNK_SIZE = 65536
 _SENSITIVE_HEADER_NAMES = {
     "authorization",
     "cookie",
@@ -86,3 +87,70 @@ def safe_get(url: str, *, max_redirects: int = 10, **kwargs: Any) -> requests.Re
         history.append(response)
         current_url = redirect_url
         redirects_followed += 1
+
+
+def safe_get_bounded(
+    url: str,
+    *,
+    max_bytes: int,
+    timeout: float | tuple[float, float] = 30,
+    headers: dict[str, str] | None = None,
+    max_redirects: int = 10,
+) -> tuple[bytes, str, str]:
+    """GET a URL through :func:`safe_get`, refusing bodies over *max_bytes*.
+
+    The body is streamed and abandoned as soon as it crosses the limit, so an
+    oversized response costs one chunk of memory instead of all of it. The cap
+    counts decoded bytes, which is what a compressed response expands into --
+    ``Content-Length`` describes the wire size and cannot bound that.
+
+    Args:
+        url: The URL to fetch.
+        max_bytes: Largest body to accept, in decoded bytes.
+        timeout: Request timeout, passed through to requests.
+        headers: Request headers.
+        max_redirects: Hops to follow before giving up.
+
+    Returns:
+        A ``(body, content_type, final_url)`` tuple, where *final_url* is the
+        last validated URL in the redirect chain.
+
+    Raises:
+        ValueError: If URL validation fails, the redirect chain is too long, or
+            the body exceeds *max_bytes*.
+        requests.RequestException: If the request fails or returns an error
+            status.
+    """
+    response = safe_get(
+        url,
+        max_redirects=max_redirects,
+        headers=headers,
+        timeout=timeout,
+        stream=True,
+    )
+    try:
+        response.raise_for_status()
+
+        chunks: list[bytes] = []
+        total = 0
+        for chunk in response.iter_content(chunk_size=_STREAM_CHUNK_SIZE):
+            if not chunk:
+                continue
+            total += len(chunk)
+            if total > max_bytes:
+                raise ValueError(
+                    f"Response body from '{url}' exceeds the {max_bytes} byte limit."
+                )
+            chunks.append(chunk)
+
+        return (
+            b"".join(chunks),
+            response.headers.get("Content-Type", ""),
+            response.url,
+        )
+    finally:
+        # Under stream=True each hop holds its connection until the body is read,
+        # so the redirects need closing too, not just the response we return.
+        for hop in response.history:
+            hop.close()
+        response.close()

lib/crewai-tools/src/crewai_tools/tools/__init__.py

@@ -196,6 +196,7 @@
 )
 from crewai_tools.tools.tavily_search_tool.tavily_search_tool import TavilySearchTool
 from crewai_tools.tools.txt_search_tool.txt_search_tool import TXTSearchTool
+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
 from crewai_tools.tools.vision_tool.vision_tool import VisionTool
 from crewai_tools.tools.wait_tool.wait_tool import WaitTool
 from crewai_tools.tools.weaviate_tool.vector_search import WeaviateVectorSearchTool
@@ -310,6 +311,7 @@
     "TavilyGetResearchTool",
     "TavilyResearchTool",
     "TavilySearchTool",
+    "URLReadTool",
     "VisionTool",
     "WaitTool",
     "WeaviateVectorSearchTool",

lib/crewai-tools/src/crewai_tools/tools/url_read_tool/__init__.py

@@ -0,0 +1,4 @@
+from crewai_tools.tools.url_read_tool.url_read_tool import URLReadTool
+
+
+__all__ = ["URLReadTool"]