Fix exception-threshold-bump and restore lost headroom

### What does this PR do?
Fix `fetch_main_headroom()` in tasks/static_quality_gates/metrics.py so
`quality-gates.exception-threshold-bump` actually restores headroom
instead of reproducing `main`'s own failure.

Widen its lookback window from 1 day to 7 so it doesn't miss `main`'s
last report when no commit lands over a weekend.

Floor the computed headroom at `BUFFER_SIZE` instead of 0 when main is
already over its own limit, since `max - current` is then negative and
previously clamped straight to 0.

### Motivation
`static_quality_gates` has been failing by a handful of KB on at least 3
[unrelated PRs last week](https://dd.slack.com/archives/C08SK4B0FK8/p1786035443536819)
and on `main` itself: `agent_rpm_arm64` and `agent_suse_arm64` both
exceeded their 735.07 MiB limit after #52372 and #54319 landed.

Main's own headroom was already at or below 0 for these gates, so
`exception-threshold-bump`'s formula, `new_threshold = PR_current +
main_headroom`, reduced to just PR_current: it silently produced a
threshold still below main's actual usage, which would not even have
unblocked `main`.

Separately, `fetch_main_headroom()`'s 1-day lookback returned no data
at all across the weekend gap between Friday's merges and Monday's
incident, since main only reports this metric when a commit lands.

### Describe how you validated your changes
Added `test_headroom_floored_when_main_over_its_own_limit` (renamed
from the now-inverted `test_headroom_never_negative`) asserting the
floor applies when main is over budget.

Reran `quality-gates.exception-threshold-bump` twice independently: both
runs produced identical thresholds (`agent_rpm_arm64/agent_suse_arm64`
-> 736.03 MiB, `iot_agent_deb/rpm/suse_amd64` -> 47.33 MiB), confirming
the fix is deterministic.

### Additional Notes
This restores headroom but does not address recurring growths that
disregard earlier efforts.

Author: rdesgroppes

tasks/static_quality_gates/metrics.py

@@ -109,7 +109,8 @@ def fetch_main_headroom(failing_gates: list[str]) -> dict[str, dict[str, int]]:
         f"avg:datadog.agent.static_quality_gate.{m}{{git_ref:main AND ({gate_filter})}} by {{gate_name}}"
         for m in metric_map
     )
-    result = query_metrics(queries, from_time="now-1d", to_time="now")
+    # 1-day window can miss main's last report (e.g. no commits over a weekend)
+    result = query_metrics(queries, from_time="now-7d", to_time="now")
 
     for series in result:
         gate_name = _extract_gate_name_from_scope(series.get("scope", ""))
@@ -127,13 +128,16 @@ def fetch_main_headroom(failing_gates: list[str]) -> dict[str, dict[str, int]]:
                     main_metrics[gate_name][key] = int(latest_value)
                 break
 
+    from tasks.static_quality_gates.thresholds import BUFFER_SIZE  # lazy: avoids a circular import
+
     headroom: dict[str, dict[str, int]] = {}
     for gate_name, metrics in main_metrics.items():
         disk_headroom = metrics.get("max_disk", 0) - metrics.get("current_disk", 0)
         wire_headroom = metrics.get("max_wire", 0) - metrics.get("current_wire", 0)
+        # Floor at BUFFER_SIZE, not 0: main over its own limit shouldn't clamp headroom to 0
         headroom[gate_name] = {
-            "disk_headroom": max(0, disk_headroom),
-            "wire_headroom": max(0, wire_headroom),
+            "disk_headroom": max(BUFFER_SIZE, disk_headroom),
+            "wire_headroom": max(BUFFER_SIZE, wire_headroom),
         }
 
     return headroom

tasks/unit_tests/static_quality_gates/metrics_tests.py

@@ -8,6 +8,7 @@
     fetch_main_headroom,
     fetch_pr_metrics,
 )
+from tasks.static_quality_gates.thresholds import BUFFER_SIZE
 
 
 class MockPoint:
@@ -248,8 +249,8 @@ def test_calculates_headroom_correctly(self, mock_query):
         self.assertEqual(headroom["wire_headroom"], 25 * 1024 * 1024)
 
     @patch("tasks.static_quality_gates.metrics.query_metrics")
-    def test_headroom_never_negative(self, mock_query):
-        """Headroom should never be negative (clamped to 0)."""
+    def test_headroom_floored_when_main_over_its_own_limit(self, mock_query):
+        """Headroom should floor at BUFFER_SIZE, not 0-, when main is over its own limit."""
         # Single API call with current > max
         mock_query.return_value = [
             {
@@ -277,8 +278,8 @@ def test_headroom_never_negative(self, mock_query):
         result = fetch_main_headroom(["static_quality_gate_agent_deb_amd64"])
 
         headroom = result["static_quality_gate_agent_deb_amd64"]
-        # disk_headroom = max(0, 150 - 200) = 0
-        self.assertEqual(headroom["disk_headroom"], 0)
+        # disk_headroom = max(BUFFER_SIZE, 150 - 200) = BUFFER_SIZE
+        self.assertEqual(headroom["disk_headroom"], BUFFER_SIZE)
 
     def test_returns_empty_for_no_gates(self):
         """Should return empty dict when no gates provided."""

test/static/static_quality_gates.yml

@@ -1,54 +1,54 @@
 static_quality_gate_agent_deb_amd64:
-  max_on_disk_size: 764.01 MiB
-  max_on_wire_size: 182.72 MiB
+  max_on_disk_size: 764.02 MiB
+  max_on_wire_size: 182.75 MiB
 static_quality_gate_agent_deb_amd64_fips:
   max_on_disk_size: 715.32 MiB
   max_on_wire_size: 176.9 MiB
 static_quality_gate_agent_heroku_amd64:
   max_on_disk_size: 318.88 MiB
   max_on_wire_size: 81.49 MiB
 static_quality_gate_agent_msi:
-  max_on_disk_size: 660.49 MiB
-  max_on_wire_size: 159.03 MiB
+  max_on_disk_size: 660.5 MiB
+  max_on_wire_size: 159.04 MiB
 static_quality_gate_agent_rpm_amd64:
-  max_on_disk_size: 763.98 MiB
+  max_on_disk_size: 763.99 MiB
   max_on_wire_size: 186.49 MiB
 static_quality_gate_agent_rpm_amd64_fips:
   max_on_disk_size: 715.32 MiB
   max_on_wire_size: 177.67 MiB
 static_quality_gate_agent_rpm_arm64:
-  max_on_disk_size: 735.07 MiB
-  max_on_wire_size: 167.02 MiB
+  max_on_disk_size: 736.03 MiB
+  max_on_wire_size: 167.01 MiB
 static_quality_gate_agent_rpm_arm64_fips:
   max_on_disk_size: 693.29 MiB
-  max_on_wire_size: 159.34 MiB
+  max_on_wire_size: 159.32 MiB
 static_quality_gate_agent_suse_amd64:
-  max_on_disk_size: 763.98 MiB
+  max_on_disk_size: 763.99 MiB
   max_on_wire_size: 186.46 MiB
 static_quality_gate_agent_suse_amd64_fips:
   max_on_disk_size: 715.32 MiB
   max_on_wire_size: 177.88 MiB
 static_quality_gate_agent_suse_arm64:
-  max_on_disk_size: 735.07 MiB
-  max_on_wire_size: 166.98 MiB
+  max_on_disk_size: 736.03 MiB
+  max_on_wire_size: 167.01 MiB
 static_quality_gate_agent_suse_arm64_fips:
   max_on_disk_size: 693.29 MiB
-  max_on_wire_size: 159.32 MiB
+  max_on_wire_size: 159.34 MiB
 static_quality_gate_docker_agent_amd64:
-  max_on_disk_size: 819.6 MiB
-  max_on_wire_size: 277.77 MiB
+  max_on_disk_size: 819.61 MiB
+  max_on_wire_size: 277.78 MiB
 static_quality_gate_docker_agent_arm64:
-  max_on_disk_size: 820.34 MiB
-  max_on_wire_size: 265.7 MiB
+  max_on_disk_size: 820.35 MiB
+  max_on_wire_size: 265.72 MiB
 static_quality_gate_docker_agent_jmx_amd64:
-  max_on_disk_size: 1010.36 MiB
+  max_on_disk_size: 1010.37 MiB
   max_on_wire_size: 346.39 MiB
 static_quality_gate_docker_agent_jmx_arm64:
-  max_on_disk_size: 1000.02 MiB
-  max_on_wire_size: 330.31 MiB
+  max_on_disk_size: 1000.03 MiB
+  max_on_wire_size: 330.32 MiB
 static_quality_gate_docker_cluster_agent_amd64:
   max_on_disk_size: 211.25 MiB
-  max_on_wire_size: 74.4 MiB
+  max_on_wire_size: 74.45 MiB
 static_quality_gate_docker_cluster_agent_arm64:
   max_on_disk_size: 223.75 MiB
   max_on_wire_size: 69.5 MiB
@@ -83,7 +83,7 @@ static_quality_gate_dogstatsd_suse_amd64:
   max_on_disk_size: 31.15 MiB
   max_on_wire_size: 8.94 MiB
 static_quality_gate_iot_agent_deb_amd64:
-  max_on_disk_size: 46.38 MiB
+  max_on_disk_size: 47.33 MiB
   max_on_wire_size: 13.64 MiB
 static_quality_gate_iot_agent_deb_arm64:
   max_on_disk_size: 43.72 MiB
@@ -92,8 +92,8 @@ static_quality_gate_iot_agent_deb_armhf:
   max_on_disk_size: 43.96 MiB
   max_on_wire_size: 12.19 MiB
 static_quality_gate_iot_agent_rpm_amd64:
-  max_on_disk_size: 46.38 MiB
+  max_on_disk_size: 47.33 MiB
   max_on_wire_size: 13.65 MiB
 static_quality_gate_iot_agent_suse_amd64:
-  max_on_disk_size: 46.38 MiB
+  max_on_disk_size: 47.33 MiB
   max_on_wire_size: 13.64 MiB