Skip to content

Commit dee19d1

Browse files
committed
Fix exception-threshold-bump's headroom calculation
### What does this PR do? Fix `fetch_main_headroom()` in tasks/static_quality_gates/metrics.py: widen its lookback from 1 day to 7 so it doesn't miss `main`'s last report over a weekend, and floor the computed headroom at `BUFFER_SIZE` instead of 0 when main is already over its own limit. ### Motivation `agent_rpm_arm64`/`agent_suse_arm64` exceeded their 735.07 MiB limit on `main`, driven mainly by #54204's Rust data-security check (+3.49 MiB). Restoring headroom via `quality-gates.exception-threshold-bump` should have been a one-line fix, but the script failed to produce a usable threshold, for two reasons. Its 1-day lookback missed `main`'s last report across the weekend gap between Friday's merges and the incident, since main only reports this metric when a commit lands. And it sets the new limit to the PR's own reported size plus main's headroom, so whenever main's headroom was at or below 0 (as it was here) the new limit collapsed to just the PR's own size: a threshold still below main's actual usage, one that would not even have unblocked `main`. ### Describe how you validated your changes Added `test_headroom_floored_when_main_over_its_own_limit` (renamed from the now-inverted `test_headroom_never_negative`) asserting the floor applies when main is over budget. Reran `quality-gates.exception-threshold-bump` twice independently: both runs produced identical thresholds, confirming the fix is deterministic.
1 parent 2ffb147 commit dee19d1

2 files changed

Lines changed: 10 additions & 7 deletions

File tree

tasks/static_quality_gates/metrics.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ def fetch_main_headroom(failing_gates: list[str]) -> dict[str, dict[str, int]]:
109109
f"avg:datadog.agent.static_quality_gate.{m}{{git_ref:main AND ({gate_filter})}} by {{gate_name}}"
110110
for m in metric_map
111111
)
112-
result = query_metrics(queries, from_time="now-1d", to_time="now")
112+
result = query_metrics(queries, from_time="now-7d", to_time="now") # accounts for weekends
113113

114114
for series in result:
115115
gate_name = _extract_gate_name_from_scope(series.get("scope", ""))
@@ -127,13 +127,15 @@ def fetch_main_headroom(failing_gates: list[str]) -> dict[str, dict[str, int]]:
127127
main_metrics[gate_name][key] = int(latest_value)
128128
break
129129

130+
from tasks.static_quality_gates.thresholds import BUFFER_SIZE # avoids a circular import
131+
130132
headroom: dict[str, dict[str, int]] = {}
131133
for gate_name, metrics in main_metrics.items():
132134
disk_headroom = metrics.get("max_disk", 0) - metrics.get("current_disk", 0)
133135
wire_headroom = metrics.get("max_wire", 0) - metrics.get("current_wire", 0)
134136
headroom[gate_name] = {
135-
"disk_headroom": max(0, disk_headroom),
136-
"wire_headroom": max(0, wire_headroom),
137+
"disk_headroom": max(BUFFER_SIZE, disk_headroom),
138+
"wire_headroom": max(BUFFER_SIZE, wire_headroom),
137139
}
138140

139141
return headroom

tasks/unit_tests/static_quality_gates/metrics_tests.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
fetch_main_headroom,
99
fetch_pr_metrics,
1010
)
11+
from tasks.static_quality_gates.thresholds import BUFFER_SIZE
1112

1213

1314
class MockPoint:
@@ -248,8 +249,8 @@ def test_calculates_headroom_correctly(self, mock_query):
248249
self.assertEqual(headroom["wire_headroom"], 25 * 1024 * 1024)
249250

250251
@patch("tasks.static_quality_gates.metrics.query_metrics")
251-
def test_headroom_never_negative(self, mock_query):
252-
"""Headroom should never be negative (clamped to 0)."""
252+
def test_headroom_floored_when_main_over_its_own_limit(self, mock_query):
253+
"""Headroom should floor at BUFFER_SIZE, not 0-, when main is over its own limit."""
253254
# Single API call with current > max
254255
mock_query.return_value = [
255256
{
@@ -277,8 +278,8 @@ def test_headroom_never_negative(self, mock_query):
277278
result = fetch_main_headroom(["static_quality_gate_agent_deb_amd64"])
278279

279280
headroom = result["static_quality_gate_agent_deb_amd64"]
280-
# disk_headroom = max(0, 150 - 200) = 0
281-
self.assertEqual(headroom["disk_headroom"], 0)
281+
# disk_headroom = max(BUFFER_SIZE, 150 - 200) = BUFFER_SIZE
282+
self.assertEqual(headroom["disk_headroom"], BUFFER_SIZE)
282283

283284
def test_returns_empty_for_no_gates(self):
284285
"""Should return empty dict when no gates provided."""

0 commit comments

Comments
 (0)