Fix: request /crawl with stream: true issue #1066

deploy/docker/server.py

@@ -470,9 +470,20 @@ async def crawl(
 ):
     """
     Crawl a list of URLs and return the results as JSON.
+    For streaming responses, use /crawl/stream endpoint.
     """
     if not crawl_request.urls:
         raise HTTPException(400, "At least one URL required")
+
+    # Check whether it is a redirection for a streaming request
+    crawler_config = CrawlerRunConfig.load(crawl_request.crawler_config)
+    if crawler_config.stream:
+        return RedirectResponse(
+            url="/crawl/stream",
+            status_code=307,
+            headers={"Content-Type": "application/json"}
+        )
+
     res = await handle_crawl_request(
         urls=crawl_request.urls,
         browser_config=crawl_request.browser_config,

deploy/docker/static/playground/index.html

@@ -503,6 +503,25 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
             } else {
                 // For /crawl endpoints, show the advanced config
                 advConfig.classList.remove('hidden');
+
+                // Update the stream configuration in CodeMirror
+                let currentConfig = cm.getValue();
+                if (endpoint === 'crawl_stream') {
+                    // Make sure that stream=True
+                    if (!currentConfig.includes('stream=')) {
+                        currentConfig = currentConfig.replace('CrawlerRunConfig(', 'CrawlerRunConfig(\n    stream=True,');
+                    } else {
+                        currentConfig = currentConfig.replace(/stream=False/, 'stream=True');
+                    }
+                } else {
+                    // Make sure that stream=False
+                    if (!currentConfig.includes('stream=')) {
+                        currentConfig = currentConfig.replace('CrawlerRunConfig(', 'CrawlerRunConfig(\n    stream=False,');
+                    } else {
+                        currentConfig = currentConfig.replace(/stream=True/, 'stream=False');
+                    }
+                }
+                cm.setValue(currentConfig);
             }
         });
 

tests/docker/test_server_requests.py

@@ -144,6 +144,42 @@ async def test_simple_crawl_single_url(self, async_client: httpx.AsyncClient):
         # We don't specify a markdown generator in this test, so don't make assumptions about markdown field
         # It might be null, missing, or populated depending on the server's default behavior
 
+    async def test_crawl_with_stream_redirects(self, async_client: httpx.AsyncClient):
+        """Test that when requesting /crawl, parameters with stream=True will be automatically redirected to /crawl/stream."""
+        payload = {
+            "urls": [SIMPLE_HTML_URL],
+            "browser_config": {
+                "type": "BrowserConfig",
+                "params": {
+                    "headless": True,
+                }
+            },
+            "crawler_config": {
+                "type": "CrawlerRunConfig", 
+                "params": {
+                    "stream": True,  # Set stream to True to trigger redirection
+                    "screenshot": False,
+                    "cache_mode": CacheMode.BYPASS.value
+                }
+            }
+        }
+
+        # Send a request to the /crawl endpoint
+        response = await async_client.post("/crawl", json=payload, follow_redirects=True)
+
+        assert response.status_code == 200
+        assert response.headers["content-type"] == "application/x-ndjson"
+        assert response.headers.get("x-stream-status") == "active"
+
+        results = await process_streaming_response(response)
+
+        assert len(results) == 1
+        result = results[0]
+        await assert_crawl_result_structure(result)
+        assert result["success"] is True
+        assert result["url"] == SIMPLE_HTML_URL
+        assert "<h1>Herman Melville - Moby-Dick</h1>" in result["html"]
+
     async def test_simple_crawl_single_url_streaming(self, async_client: httpx.AsyncClient):
         """Test /crawl/stream with a single URL and simple config values."""
         payload = {