fix(output): stop load_energy_predicted's whole-day total drifting through the day (#4519)

load_today_comparison() applied load_scaling/load_scaling_dynamic/manual_load_adjust
to future minutes only (#4506, #4511), which correctly fixed today_remaining but
also fed load_total_pred - the accumulator the whole-day "today" total is read
from - a mix of unscaled past buckets and scaled future ones. As minutes_now
advanced, buckets kept flipping from the scaled group to the unscaled one, so
the published "today" total (and its chart) drifted by whatever load_scaling
wasn't 1.0, even though nothing about the underlying forecast changed.

Adds a separate, consistently-scaled accumulator (load_total_pred_day) used only
for load_energy_predicted's state/today/today_so_far/today_remaining/results,
leaving load_total_pred itself (and the actual-vs-predicted divergence ratio it
feeds) untouched.

Author: chalfontchubby

apps/predbat/output.py

@@ -2549,6 +2549,7 @@ def load_today_comparison(self, load_minutes, load_forecast, car_minutes, import
         """
         load_total_pred = 0
         load_total_pred_now = 0
+        load_total_pred_day = 0
         car_total_pred = 0
         car_total_actual = 0
         car_value_pred = 0
@@ -2557,7 +2558,7 @@ def load_today_comparison(self, load_minutes, load_forecast, car_minutes, import
         actual_total_today = 0
         import_ignored_load_pred = 0
         import_ignored_load_actual = 0
-        load_predict_stamp = {}
+        load_predict_day_stamp = {}
         load_actual_stamp = {}
         load_predict_data = {}
         total_forecast_value_pred = 0
@@ -2588,34 +2589,44 @@ def load_today_comparison(self, load_minutes, load_forecast, car_minutes, import
             load_value_pred += forecast_value_pred
             load_value_pred_raw += forecast_value_pred
 
+            # Consistently-scaled prediction for THIS minute, applying load_scaling,
+            # load_scaling_dynamic, and manual_load_adjust the same way whether the minute is
+            # in the past or future. This feeds load_total_pred_day below, which is the sole
+            # source for load_energy_predicted's state/today/today_so_far/today_remaining/results
+            # (batpred#4496 follow-up). Using it consistently across the whole day is what makes
+            # that "predicted" total (and its chart) stay flat as minutes_now advances - it isn't
+            # meant to reflect what actually happened today, only the model's day-ahead forecast
+            # under today's scaling settings, so every bucket needs the same treatment regardless
+            # of whether it has elapsed yet.
+            manual_adjust_day = 0.0
+            if self.manual_load_adjust:
+                manual_adjust_day = self.manual_load_adjust.get(minute, 0) * step / float(self.plan_interval_minutes)
+                manual_adjust_day = max(manual_adjust_day, -load_value_pred)
+            scaling_dynamic_day = self.load_scaling_dynamic.get(minute, 1.0) if self.load_scaling_dynamic else 1.0
+            load_value_pred_day = (load_value_pred + manual_adjust_day) * self.load_scaling * scaling_dynamic_day
+
             # For FUTURE minutes only, apply load_scaling, load_scaling_dynamic, and
-            # manual_load_adjust so the published predicted/adjusted curves (and their
-            # today_remaining attribute) match step_data_history() (fetch.py), which the plan
-            # itself uses to build load_minutes_step as
-            # (value + load_extra) * scaling_dynamic * scale_fixed, where load_extra includes
-            # manual_load_adjust, scaling_dynamic is load_scaling_dynamic, and scale_fixed
-            # includes the flat load_scaling. load_scaling_dynamic carries saving-session/
-            # free-electricity-event scaling as well as any per-window override from
-            # rates_import_override/the manual API (e.g. a "power up" event) - a first pass at
-            # this fix (#4506) only applied the flat load_scaling and missed both of these,
+            # manual_load_adjust so the published today_remaining attribute matches
+            # step_data_history() (fetch.py), which the plan itself uses to build
+            # load_minutes_step as (value + load_extra) * scaling_dynamic * scale_fixed, where
+            # load_extra includes manual_load_adjust, scaling_dynamic is load_scaling_dynamic,
+            # and scale_fixed includes the flat load_scaling. load_scaling_dynamic carries
+            # saving-session/free-electricity-event scaling as well as any per-window override
+            # from rates_import_override/the manual API (e.g. a "power up" event) - a first pass
+            # at this fix (#4506) only applied the flat load_scaling and missed both of these,
             # confirmed against a real follow-up report on issue #4496 where a 1.5x
             # load_scaling_dynamic override for a 2-hour power-up event wasn't reflected in
             # today_remaining at all.
             #
-            # Minutes already elapsed today are deliberately left untouched: load_total_pred_now
-            # below feeds the actual-vs-predicted divergence ratio, which compares actual
-            # consumption against the raw model, not an adjusted one.
+            # Minutes already elapsed today are deliberately left untouched here: load_total_pred
+            # and load_total_pred_now below feed the actual-vs-predicted divergence ratio, which
+            # compares actual consumption against the raw model, not an adjusted one.
             if minute >= minutes_now:
-                manual_adjust = 0.0
-                if self.manual_load_adjust:
-                    manual_adjust = self.manual_load_adjust.get(minute, 0) * step / float(self.plan_interval_minutes)
-                    manual_adjust = max(manual_adjust, -load_value_pred)
-                load_value_pred += manual_adjust
-                load_value_pred_raw += manual_adjust
+                load_value_pred += manual_adjust_day
+                load_value_pred_raw += manual_adjust_day
 
-                scaling_dynamic = self.load_scaling_dynamic.get(minute, 1.0) if self.load_scaling_dynamic else 1.0
-                load_value_pred *= self.load_scaling * scaling_dynamic
-                load_value_pred_raw *= self.load_scaling * scaling_dynamic
+                load_value_pred *= self.load_scaling * scaling_dynamic_day
+                load_value_pred_raw *= self.load_scaling * scaling_dynamic_day
 
             # Track (but no longer exclude) periods where import exceeds raw load, assumed to
             # include deliberate battery charging (overnight for example). The house's own load
@@ -2643,14 +2654,15 @@ def load_today_comparison(self, load_minutes, load_forecast, car_minutes, import
                 actual_total_today += load_value_pred
 
             load_total_pred += load_value_pred
+            load_total_pred_day += load_value_pred_day
             total_forecast_value_pred += forecast_value_pred
 
             load_predict_data[minute] = load_value_pred
 
             # Store for charts
             minute_timestamp = self.midnight_utc + timedelta(seconds=60 * minute)
             stamp = minute_timestamp.strftime(TIME_FORMAT)
-            load_predict_stamp[stamp] = dp3(load_total_pred)
+            load_predict_day_stamp[stamp] = dp3(load_total_pred_day)
             load_actual_stamp[stamp] = dp3(actual_total_today)
 
         # Fetch yesterday's in-day adjustment factor from history
@@ -2753,18 +2765,18 @@ def load_today_comparison(self, load_minutes, load_forecast, car_minutes, import
                     "icon": "mdi:percent",
                 },
             )
-        load_so_far = self.filtered_today(load_predict_stamp, stamp=self.now_utc)
-        load_today = self.filtered_today(load_predict_stamp)
+        load_so_far = self.filtered_today(load_predict_day_stamp, stamp=self.now_utc)
+        load_today = self.filtered_today(load_predict_day_stamp)
         load_today_remaining = None
         if (load_so_far is not None) and (load_today is not None):
             load_today_remaining = load_today - load_so_far
 
         if save:
             self.dashboard_item(
                 self.prefix + ".load_energy_predicted",
-                state=dp3(load_total_pred),
+                state=dp3(load_total_pred_day),
                 attributes={
-                    "results": self.filtered_times(load_predict_stamp),
+                    "results": self.filtered_times(load_predict_day_stamp),
                     "today": dp2(load_today) if load_today is not None else 0.0,
                     "today_so_far": dp2(load_so_far) if load_so_far is not None else 0.0,
                     "today_remaining": dp2(load_today_remaining) if load_today_remaining is not None else 0.0,

apps/predbat/tests/test_load_today_comparison.py

@@ -403,6 +403,97 @@ def _test_load_scaling_dynamic_and_manual_adjust_applied_to_predicted(my_predbat
     return failed
 
 
+def _test_predicted_today_stable_across_the_day(my_predbat, failed):
+    """
+    Follow-up regression test for issue #4496: load_energy_predicted's state/"today" total
+    (the whole midnight-to-midnight forecast, as opposed to "today_remaining") must not drift
+    as minutes_now advances through the day when load_scaling != 1.0. The two fixes above
+    (#4506, #4511) applied load_scaling/load_scaling_dynamic/manual_load_adjust to future
+    minutes only, correctly fixing today_remaining, but load_total_pred - the same accumulator
+    the whole-day "today" total was read from - summed those scaled future buckets alongside
+    unscaled past buckets. As minutes_now advanced, buckets kept flipping from the scaled group
+    to the unscaled one, so the published "today" total silently shrank (with load_scaling > 1)
+    or grew (with load_scaling < 1) even though nothing about the underlying forecast changed -
+    exactly the "prediction drops from 11.5kWh to 11.3kWh by 08:00" regression a user reported
+    after taking the #4511 fix. This asserts the whole-day total is identical whether computed
+    at the start of the day or partway through it, given the same underlying load model.
+    """
+    print("  test: load_energy_predicted's whole-day 'today' total is stable as minutes_now advances")
+
+    saved = {
+        "car_charging_hold": my_predbat.car_charging_hold,
+        "car_charging_energy": my_predbat.car_charging_energy,
+        "iboost_energy_subtract": my_predbat.iboost_energy_subtract,
+        "iboost_energy_today": my_predbat.iboost_energy_today,
+        "base_load": my_predbat.base_load,
+        "load_forecast_only": my_predbat.load_forecast_only,
+        "days_previous": my_predbat.days_previous,
+        "days_previous_weight": my_predbat.days_previous_weight,
+        "load_minutes_age": my_predbat.load_minutes_age,
+        "load_scaling": my_predbat.load_scaling,
+        "load_scaling_dynamic": my_predbat.load_scaling_dynamic,
+        "manual_load_adjust": my_predbat.manual_load_adjust,
+        "now_utc": my_predbat.now_utc,
+        "midnight_utc": my_predbat.midnight_utc,
+        "minutes_now": my_predbat.minutes_now,
+    }
+
+    try:
+        my_predbat.car_charging_hold = False
+        my_predbat.car_charging_energy = None
+        my_predbat.iboost_energy_subtract = False
+        my_predbat.iboost_energy_today = None
+        my_predbat.base_load = 0.0
+        my_predbat.load_forecast_only = False
+        my_predbat.days_previous = [1]
+        my_predbat.days_previous_weight = [1.0]
+        my_predbat.load_minutes_age = 1
+        my_predbat.load_scaling = 1.05
+        my_predbat.load_scaling_dynamic = {}
+        my_predbat.manual_load_adjust = {}
+
+        midnight_utc = datetime(2026, 1, 1, 0, 0, 0, tzinfo=UTC)
+        my_predbat.midnight_utc = midnight_utc
+
+        load_minutes = build_cumulative(0.02, 3000)  # 0.02 kWh/min -> 0.1 kWh per 5-min bucket
+        load_forecast = {}
+        import_minutes = build_cumulative(0.0, 3000)
+
+        # Same day, same underlying load model, only minutes_now differs: at midnight (nothing
+        # elapsed yet) and again mid-afternoon (most of the day elapsed).
+        today_at_midnight = None
+        today_mid_afternoon = None
+        for minutes_now in (0, 900):  # 00:00 and 15:00
+            my_predbat.now_utc = midnight_utc + timedelta(minutes=minutes_now)
+            my_predbat.minutes_now = minutes_now
+            my_predbat.load_today_comparison(load_minutes, load_forecast, {}, import_minutes, minutes_now=minutes_now, step=5, save=True)
+            attrs = my_predbat.dashboard_values[my_predbat.prefix + ".load_energy_predicted"]["attributes"]
+            state = my_predbat.dashboard_values[my_predbat.prefix + ".load_energy_predicted"]["state"]
+            if minutes_now == 0:
+                today_at_midnight = attrs["today"]
+                state_at_midnight = state
+            else:
+                today_mid_afternoon = attrs["today"]
+                state_mid_afternoon = state
+
+        if today_at_midnight <= 0:
+            print("  ERROR: today total at midnight should be positive for this to be a meaningful test, got {}".format(today_at_midnight))
+            failed = True
+        elif abs(today_mid_afternoon - today_at_midnight) > 0.02:
+            print("  ERROR: 'today' total drifted as minutes_now advanced - midnight {} vs 15:00 {} (load_scaling=1.05 constant throughout)".format(today_at_midnight, today_mid_afternoon))
+            failed = True
+        elif abs(state_mid_afternoon - state_at_midnight) > 0.02:
+            print("  ERROR: state drifted as minutes_now advanced - midnight {} vs 15:00 {} (load_scaling=1.05 constant throughout)".format(state_at_midnight, state_mid_afternoon))
+            failed = True
+        else:
+            print("  PASS: 'today' total and state stayed flat across the day ({} at 00:00, {} at 15:00)".format(today_at_midnight, today_mid_afternoon))
+    finally:
+        for key, value in saved.items():
+            setattr(my_predbat, key, value)
+
+    return failed
+
+
 # ---------------------------------------------------------------------------
 # Entry point
 # ---------------------------------------------------------------------------
@@ -412,9 +503,10 @@ def test_load_today_comparison(my_predbat):
     """
     Unit tests for load_today_comparison() covering the None-guard fix
     for dp2() calls when filtered_today() returns None, the
-    import-exceeds-load regression (batpred#4154, #2537), and the
+    import-exceeds-load regression (batpred#4154, #2537), the
     load_scaling/load_scaling_dynamic/manual_load_adjust-not-applied
-    regression (#4496).
+    regression (#4496), and the whole-day "today" total drifting as
+    minutes_now advances (#4496 follow-up).
     """
     failed = False
     print("**** Running load_today_comparison tests ****")
@@ -423,5 +515,6 @@ def test_load_today_comparison(my_predbat):
     failed = _test_import_exceeding_load_still_counted(my_predbat, failed) or failed
     failed = _test_load_scaling_applied_to_predicted(my_predbat, failed) or failed
     failed = _test_load_scaling_dynamic_and_manual_adjust_applied_to_predicted(my_predbat, failed) or failed
+    failed = _test_predicted_today_stable_across_the_day(my_predbat, failed) or failed
 
     return failed