|
12 | 12 | ScrapeRequest, |
13 | 13 | ScrapeResponse, |
14 | 14 | ScraperEngine, |
| 15 | + html_document_headers, |
| 16 | + utf8_normalize_html, |
15 | 17 | ) |
16 | 18 | from app.metadata import MetadataExtractor |
17 | 19 | from app.security import UrlGuard |
@@ -207,6 +209,33 @@ def test_html_response_sets_utf8_content_type_and_normalizes_body(self): |
207 | 209 | self.assertNotIn("Caffè", result["html"]) |
208 | 210 | result["html"].encode("utf-8") |
209 | 211 |
|
| 212 | + def test_utf8_normalize_leaves_correct_unicode_unchanged(self): |
| 213 | + html = "<html><body><h1>Caffè 日本語</h1></body></html>" |
| 214 | + self.assertEqual(utf8_normalize_html(html), html) |
| 215 | + |
| 216 | + normalized, headers = html_document_headers( |
| 217 | + html, {"content-type": "text/html"} |
| 218 | + ) |
| 219 | + self.assertEqual(normalized, html) |
| 220 | + self.assertEqual(headers["content-type"], "text/html; charset=utf-8") |
| 221 | + |
| 222 | + _FakeRequest.response = _FakeHttpResponse( |
| 223 | + text=html, |
| 224 | + status_code=200, |
| 225 | + headers={"content-type": "text/html"}, |
| 226 | + url="https://example.com/", |
| 227 | + ) |
| 228 | + payload = ScrapeRequest(url="https://example.com", execution_mode="request") |
| 229 | + with tempfile.TemporaryDirectory() as tmp: |
| 230 | + engine = ScraperEngine(runtime_root=Path(tmp)) |
| 231 | + with patch("app.engine.Request", _FakeRequest): |
| 232 | + result = engine.execute(payload) |
| 233 | + |
| 234 | + self.assertEqual(result["html"], html) |
| 235 | + self.assertEqual( |
| 236 | + result["headers"]["content-type"], "text/html; charset=utf-8" |
| 237 | + ) |
| 238 | + |
210 | 239 | def test_request_tier_blocked_status_escalates_to_browser(self): |
211 | 240 | payload = ScrapeRequest(url="https://example.com") |
212 | 241 | for status in (401, 403, 429): |
@@ -761,6 +790,41 @@ def test_schema_422_returns_scrape_envelope(self): |
761 | 790 | self.assertIn("host=example.com", log_text) |
762 | 791 | self.assertIn("field=window_size", log_text) |
763 | 792 |
|
| 793 | + def test_scrape_clamps_wait_timeout_instead_of_422(self): |
| 794 | + from fastapi.testclient import TestClient |
| 795 | + |
| 796 | + import app.main as main_mod |
| 797 | + from app.main import app |
| 798 | + |
| 799 | + captured: dict[str, int] = {} |
| 800 | + |
| 801 | + def fake_execute(payload, _deadline=None): |
| 802 | + captured["wait"] = payload.wait_timeout_seconds |
| 803 | + return ScrapeResponse.create_success( |
| 804 | + str(payload.url), |
| 805 | + request_id="req-wait-clamp", |
| 806 | + html="<html></html>", |
| 807 | + attempts=1, |
| 808 | + strategy_used="anti_detect_request", |
| 809 | + render_ms=1, |
| 810 | + execution_tier="http_request", |
| 811 | + ) |
| 812 | + |
| 813 | + with patch.object(main_mod._engine, "execute", side_effect=fake_execute): |
| 814 | + client = TestClient(app) |
| 815 | + response = client.post( |
| 816 | + "/scrape", |
| 817 | + json={ |
| 818 | + "url": "https://example.com", |
| 819 | + "wait_timeout_seconds": 28, |
| 820 | + }, |
| 821 | + ) |
| 822 | + |
| 823 | + self.assertNotEqual(response.status_code, 422) |
| 824 | + self.assertEqual(response.status_code, 200) |
| 825 | + self.assertEqual(captured["wait"], DEFAULT_SCRAPE_TIMEOUT_SECONDS) |
| 826 | + self.assertEqual(captured["wait"], 20) |
| 827 | + |
764 | 828 |
|
765 | 829 | if __name__ == "__main__": |
766 | 830 | unittest.main() |
0 commit comments