Skip to content

Commit 6dc8996

Browse files
committed
fix: better observed_seconds handling in history models
1 parent 4f173a3 commit 6dc8996

4 files changed

Lines changed: 147 additions & 25 deletions

File tree

packages/ai-horde-service-alerts/src/ai_horde_service_alerts/db/repositories/history.py

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -35,18 +35,21 @@ class DailyBucket:
3535
"""Daily roll-up used for the public 90-day history bars."""
3636

3737
date: str # ISO YYYY-MM-DD
38-
status_level: int # 0 ok, 1 minor (degraded/unknown), 2 major (partial/down), 3 maintenance
39-
observed_seconds: int # elapsed/observable seconds in this day (86400 for past days, partial for today)
38+
status_level: int # 0 ok, 1 minor (degraded), 2 major (partial/down), 3 maintenance
4039
operational_seconds: int
4140
degraded_seconds: int
4241
down_seconds: int
4342
maintenance_seconds: int
4443
unknown_seconds: int
4544

4645

46+
# Unknown == "no signal", not an outage: it must never raise the bar above OK. A
47+
# bucket is only minor/major/maintenance when it actually saw degraded/down/maint
48+
# time. (Real degradation outranks UNKNOWN in STATUS_RANK, so a day that mixes
49+
# unknown with a genuine problem still reports the problem's level.)
4750
_LEVEL_BY_STATUS: dict[ComponentStatusValue, int] = {
4851
ComponentStatusValue.OPERATIONAL: 0,
49-
ComponentStatusValue.UNKNOWN: 1,
52+
ComponentStatusValue.UNKNOWN: 0,
5053
ComponentStatusValue.DEGRADED: 1,
5154
ComponentStatusValue.PARTIAL: 2,
5255
ComponentStatusValue.DOWN: 2,
@@ -163,13 +166,10 @@ async def daily_buckets(
163166
if secs > 0 and STATUS_RANK[status] > STATUS_RANK[worst_status]:
164167
worst_status = status
165168
worst_level = _LEVEL_BY_STATUS[worst_status]
166-
observed_end = min(anchor, day_end)
167-
observed_seconds = max(0, int((observed_end - day_start).total_seconds()))
168169
buckets.append(
169170
DailyBucket(
170171
date=day_start.date().isoformat(),
171172
status_level=worst_level,
172-
observed_seconds=observed_seconds,
173173
operational_seconds=seconds[ComponentStatusValue.OPERATIONAL],
174174
degraded_seconds=seconds[ComponentStatusValue.DEGRADED],
175175
down_seconds=seconds[ComponentStatusValue.DOWN] + seconds[ComponentStatusValue.PARTIAL],
@@ -186,24 +186,26 @@ async def uptime_percent(
186186
days: int,
187187
now: datetime | None = None,
188188
) -> float | None:
189-
"""Return uptime% over the trailing window, excluding maintenance time.
189+
"""Return uptime% over the trailing window.
190190
191-
Returns ``None`` when no history exists at all (so callers can render
192-
``—`` instead of a misleading ``0%``).
191+
The denominator is time for which we have a real status signal
192+
(operational + degraded + down). Maintenance, unknown, and no-data days
193+
are excluded outright: counting them would conflate "we weren't watching"
194+
/ "scheduled maintenance" with downtime. The current day is therefore
195+
self-correcting too — its not-yet-elapsed remainder has no signal and so
196+
never enters the denominator.
197+
198+
Returns ``None`` when there is no signal at all in the window (so callers
199+
can render ``—`` instead of a misleading ``0%`` or ``100%``).
193200
"""
194201
buckets = await self.daily_buckets(component_id, days=days, now=now)
195202
operational = sum(b.operational_seconds for b in buckets)
196-
maintenance = sum(b.maintenance_seconds for b in buckets)
197-
# Use elapsed/observable time, not days * 86400: the current day is only
198-
# partially elapsed, so counting its future remainder would understate uptime.
199-
observable = sum(b.observed_seconds for b in buckets)
200-
non_maintenance = observable - maintenance
201-
if non_maintenance <= 0:
202-
return None
203-
observed = operational + sum((b.degraded_seconds + b.down_seconds + b.unknown_seconds) for b in buckets)
204-
if observed == 0:
203+
degraded = sum(b.degraded_seconds for b in buckets)
204+
down = sum(b.down_seconds for b in buckets)
205+
signal = operational + degraded + down
206+
if signal == 0:
205207
return None
206-
return round(operational / non_maintenance * 100.0, 4)
208+
return round(operational / signal * 100.0, 4)
207209

208210
async def close_open_slice_at(
209211
self,

packages/ai-horde-service-alerts/src/ai_horde_service_alerts/models/public.py

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -128,11 +128,8 @@ class PublicHistoryDay(BaseModel):
128128
status_level: int = Field(
129129
ge=0,
130130
le=3,
131-
description="0 ok | 1 minor (degraded/unknown) | 2 major (partial/down) | 3 maintenance.",
132-
)
133-
observed_seconds: int = Field(
134-
description="Elapsed/observable seconds in this day: 86400 for past days, partial for today. "
135-
"Normalize the bar against this rather than assuming a full 86400-second day.",
131+
description="0 ok | 1 minor (degraded) | 2 major (partial/down) | 3 maintenance. "
132+
"Unknown/no-signal time never raises the level above 0.",
136133
)
137134
operational_seconds: int
138135
degraded_seconds: int

packages/ai-horde-service-alerts/src/ai_horde_service_alerts/services/projections.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ def history_response(
148148
PublicHistoryDay(
149149
date=b.date,
150150
status_level=b.status_level,
151-
observed_seconds=b.observed_seconds,
152151
operational_seconds=b.operational_seconds,
153152
degraded_seconds=b.degraded_seconds,
154153
down_seconds=b.down_seconds,
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
"""Regression tests for the public 90-day history roll-up.
2+
3+
These reproduce the two production symptoms observed on the live status page:
4+
5+
* an always-yellow trailing bar caused by a short ``unknown`` scrape gap on the
6+
current (partial) day, even though the component never went degraded/down; and
7+
* uptime collapsing toward ~0% because historical no-data days were counted as
8+
downtime.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
from datetime import UTC, datetime, timedelta
14+
15+
import pytest
16+
from sqlalchemy.ext.asyncio import AsyncSession
17+
18+
from ai_horde_service_alerts.db.models import Component, ComponentStatusHistory
19+
from ai_horde_service_alerts.db.repositories.history import HistoryRepository
20+
from ai_horde_service_alerts.db.types import Audience, ComponentStatusValue, HistorySource
21+
22+
NOON_TODAY = datetime(2026, 6, 21, 12, 0, tzinfo=UTC)
23+
COMPONENT_ID = "api"
24+
25+
26+
async def _seed_component(session: AsyncSession) -> None:
27+
session.add(
28+
Component(
29+
id=COMPONENT_ID,
30+
name="API",
31+
description="",
32+
audience=Audience.PUBLIC,
33+
),
34+
)
35+
await session.flush()
36+
37+
38+
def _slice(
39+
status: ComponentStatusValue,
40+
started_at: datetime,
41+
ended_at: datetime | None,
42+
) -> ComponentStatusHistory:
43+
return ComponentStatusHistory(
44+
component_id=COMPONENT_ID,
45+
status=status,
46+
source=HistorySource.PROBER,
47+
started_at=started_at,
48+
ended_at=ended_at,
49+
)
50+
51+
52+
@pytest.mark.asyncio
53+
async def test_trailing_unknown_gap_does_not_paint_bar_minor(db_session: AsyncSession) -> None:
54+
"""A ~71-minute unknown gap today, with no degraded/down time, stays level 0."""
55+
await _seed_component(db_session)
56+
day_start = NOON_TODAY.replace(hour=0, minute=0, second=0, microsecond=0)
57+
gap_start = NOON_TODAY - timedelta(seconds=4254)
58+
# operational -> unknown (scrape gap) -> operational, all on the current day.
59+
db_session.add_all(
60+
[
61+
_slice(ComponentStatusValue.OPERATIONAL, day_start, gap_start),
62+
_slice(ComponentStatusValue.UNKNOWN, gap_start, NOON_TODAY - timedelta(seconds=600)),
63+
_slice(ComponentStatusValue.OPERATIONAL, NOON_TODAY - timedelta(seconds=600), None),
64+
],
65+
)
66+
await db_session.flush()
67+
68+
repo = HistoryRepository(db_session)
69+
buckets = await repo.daily_buckets(COMPONENT_ID, days=1, now=NOON_TODAY)
70+
71+
today = buckets[-1]
72+
assert today.degraded_seconds == 0
73+
assert today.down_seconds == 0
74+
assert today.maintenance_seconds == 0
75+
assert today.unknown_seconds > 0 # the gap is recorded...
76+
assert today.status_level == 0 # ...but it must NOT fold the bar to minor
77+
78+
79+
@pytest.mark.asyncio
80+
async def test_uptime_excludes_no_data_days(db_session: AsyncSession) -> None:
81+
"""No-data historical days must not count as downtime (was driving ~0.5%)."""
82+
await _seed_component(db_session)
83+
# The only signal in a 90-day window is one fully-operational slice covering
84+
# the last ~2 days. Everything before it is a genuine no-data gap.
85+
db_session.add(
86+
_slice(ComponentStatusValue.OPERATIONAL, NOON_TODAY - timedelta(days=2), None),
87+
)
88+
await db_session.flush()
89+
90+
repo = HistoryRepository(db_session)
91+
uptime = await repo.uptime_percent(COMPONENT_ID, days=90, now=NOON_TODAY)
92+
93+
assert uptime == 100.0
94+
95+
96+
@pytest.mark.asyncio
97+
async def test_uptime_none_when_no_signal_at_all(db_session: AsyncSession) -> None:
98+
"""A window with no operational/degraded/down signal returns None, not 0/100."""
99+
await _seed_component(db_session)
100+
repo = HistoryRepository(db_session)
101+
102+
uptime = await repo.uptime_percent(COMPONENT_ID, days=90, now=NOON_TODAY)
103+
104+
assert uptime is None
105+
106+
107+
@pytest.mark.asyncio
108+
async def test_uptime_counts_real_downtime(db_session: AsyncSession) -> None:
109+
"""Sanity: actual down time is reflected in the ratio (operational / signal)."""
110+
await _seed_component(db_session)
111+
start = NOON_TODAY - timedelta(days=1)
112+
midpoint = start + timedelta(hours=12)
113+
db_session.add_all(
114+
[
115+
_slice(ComponentStatusValue.OPERATIONAL, start, midpoint),
116+
_slice(ComponentStatusValue.DOWN, midpoint, NOON_TODAY),
117+
],
118+
)
119+
await db_session.flush()
120+
121+
repo = HistoryRepository(db_session)
122+
uptime = await repo.uptime_percent(COMPONENT_ID, days=2, now=NOON_TODAY)
123+
124+
assert uptime == pytest.approx(50.0, abs=0.5)

0 commit comments

Comments
 (0)