Validate HTTP Range responses in subset downloads to prevent silent disk blowup

Author: jacob1299

src/herbie/core.py

@@ -1098,8 +1098,29 @@ def subset(search, outFile, verbose=True):
                             grib_source,
                             headers=headers,
                             timeout=30,
+                            stream=True,
                         )
                         response.raise_for_status()
+
+                        # Guard: if the server did not honor the Range
+                        # request, each group would silently download the
+                        # entire file and append it, ballooning disk usage.
+                        if response.status_code != 206:
+                            content_length = response.headers.get(
+                                "Content-Length", "unknown"
+                            )
+                            response.close()
+                            raise RuntimeError(
+                                f"Range request not honored: server returned "
+                                f"HTTP {response.status_code} "
+                                f"(Content-Length: {content_length}). "
+                                f"This can happen when a network proxy or "
+                                f"VPN strips the Range header. Try downloading "
+                                f"the full file first, then subset locally:\n"
+                                f"  full_file = Herbie(...).download()\n"
+                                f"  Herbie(...).download(search=...)"
+                            )
+
                         data = response.content
 
                     # Write or append to output file

tests/test_subset_range_validation.py

@@ -0,0 +1,43 @@
+"""Test that subset downloads validate HTTP Range request responses."""
+
+from datetime import datetime, timedelta
+from unittest.mock import Mock, patch
+
+import pytest
+
+from herbie import Herbie, config
+
+now = datetime.now()
+today = datetime(now.year, now.month, now.day, now.hour) - timedelta(hours=6)
+
+save_dir = config["default"]["save_dir"] / "Herbie-Tests-Data/"
+
+
+def test_subset_raises_on_non_206_response():
+    """RuntimeError should fire when the server returns 200 instead of 206.
+
+    This prevents the silent disk-space blowup described in issue #514:
+    without the guard, each subset group downloads the entire multi-GB
+    file and appends it, producing output many times larger than the
+    source.
+    """
+    H = Herbie(
+        today,
+        model="hrrr",
+        product="sfc",
+        save_dir=save_dir,
+        overwrite=True,
+    )
+
+    # Force the index to be fetched and cached before we patch requests.
+    _ = H.index_as_dataframe
+
+    mock_response = Mock()
+    mock_response.status_code = 200
+    mock_response.headers = {"Content-Length": "9999999999"}
+    mock_response.raise_for_status = Mock()
+    mock_response.close = Mock()
+
+    with patch("herbie.core.requests.get", return_value=mock_response):
+        with pytest.raises(RuntimeError, match="Range request not honored"):
+            H.download("TMP:2 m", overwrite=True)