Skip to content

Commit 75f11a0

Browse files
dipannita08copybara-github
authored andcommitted
Fix occasional gaps in cumulative Goodput Monitor.
PiperOrigin-RevId: 787142345
1 parent e116e41 commit 75f11a0

File tree

3 files changed

+8
-4
lines changed

3 files changed

+8
-4
lines changed

ml_goodput_measurement/src/gcp_metrics.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ def send_metrics(self, metrics: list[Dict[str, Any]]):
100100
self.client.create_time_series(
101101
name=self.project_name, time_series=time_series_list
102102
)
103-
logger.info("Sent %d metrics to GCP Monitoring.", len(metrics))
103+
logger.info("Sent %d Goodput metrics to GCM Monitoring.", len(metrics))
104104

105105
except GoogleAPIError as e:
106-
logger.error("Failed to send metrics: %s", e)
106+
logger.error("Failed to send Goodput metrics to GCM Monitoring: %s", e)

ml_goodput_measurement/src/goodput.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1174,7 +1174,10 @@ def _fetch_new_entries(self, query_time: datetime.datetime) -> list[Any]:
11741174
cached_last_entry_info = self._goodput_cache.get_last_entry_info()
11751175
if cached_last_entry_info:
11761176
cached_last_entry_ts, _ = cached_last_entry_info
1177-
if query_time <= cached_last_entry_ts:
1177+
if (
1178+
cached_last_entry_ts is not None
1179+
and query_time <= cached_last_entry_ts
1180+
):
11781181
return []
11791182

11801183
new_entries, current_last_entry_info = (

ml_goodput_measurement/src/goodput_cache.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ def update_cached_entries(
3434
):
3535
"""Updated the cached entries."""
3636
self._cached_entries.extend(entries)
37-
self.update_last_entry_info(last_entry_info)
37+
if last_entry_info and last_entry_info[0] is not None:
38+
self.update_last_entry_info(last_entry_info)
3839
self.update_job_start_time()
3940
self.update_job_end_time()
4041
new_step_entries = [entry for entry in entries if _STEP_START_TIME in entry]

0 commit comments

Comments
 (0)