Cherry pick browser request validation improvements (#59045)

Cherry pick #58553 #58648 #59042

---------

Signed-off-by: Richo Healey <richo@anyscale.com>
Signed-off-by: Edward Oakes <ed.nmi.oakes@gmail.com>
Co-authored-by: richo-anyscale <richo@anyscale.com>
Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com>

Author: 3 people

python/ray/dashboard/http_server_head.py

@@ -183,8 +183,8 @@ async def path_clean_middleware(self, request, handler):
     @aiohttp.web.middleware
     async def browsers_no_post_put_middleware(self, request, handler):
         if (
-            # A best effort test for browser traffic. All common browsers
-            # start with Mozilla at the time of writing.
+            # Deny mutating requests from browsers.
+            # See `is_browser_request` for details of the check.
             dashboard_optional_utils.is_browser_request(request)
             and request.method in [hdrs.METH_POST, hdrs.METH_PUT]
         ):

python/ray/dashboard/optional_utils.py

@@ -128,18 +128,39 @@ def _update_cache(task):
 
 
 def is_browser_request(req: Request) -> bool:
-    """Checks if a request is made by a browser like user agent.
-
-    This heuristic is very weak, but hard for a browser to bypass- eg,
-    fetch/xhr and friends cannot alter the user-agent, but requests made with
-    an http library can stumble into this if they choose to user a browser like
-    user agent.
+    """Best-effort detection if the request was made by a browser.
+
+    Uses three heuristics:
+        1) If the `User-Agent` header starts with 'Mozilla'. This heuristic is weak,
+        but hard for a browser to bypass e.g., fetch/xhr and friends cannot alter the
+        user agent, but requests made with an HTTP library can stumble into this if
+        they choose to user a browser-like user agent. At the time of writing, all
+        common browsers' user agents start with 'Mozilla'.
+        2) If any of the `Sec-Fetch-*` headers are present.
+        3) If any of the various CORS headers are present
     """
-    return req.headers["User-Agent"].startswith("Mozilla")
+    return req.headers.get("User-Agent", "").startswith("Mozilla") or any(
+        h in req.headers
+        for h in (
+            # Origin and Referer are sent by browser user agents to give
+            # information about the requesting origin
+            "Referer",
+            "Origin",
+            # Sec-Fetch headers are sent with many but not all `fetch`
+            # requests, and will eventually be sent on all requests.
+            "Sec-Fetch-Mode",
+            "Sec-Fetch-Dest",
+            "Sec-Fetch-Site",
+            "Sec-Fetch-User",
+            # CORS headers specifying which other headers are modified
+            "Access-Control-Request-Method",
+            "Access-Control-Request-Headers",
+        )
+    )
 
 
 def deny_browser_requests() -> Callable:
-    """Reject any requests that appear to be made by a browser"""
+    """Reject any requests that appear to be made by a browser."""
 
     def decorator_factory(f: Callable) -> Callable:
         @functools.wraps(f)
@@ -149,6 +170,7 @@ async def decorator(self, req: Request):
                     text="Browser requests not allowed",
                     status=aiohttp.web.HTTPMethodNotAllowed.status_code,
                 )
+
             return await f(self, req)
 
         return decorator