fix(deye): source energy counters from the Daily registers

DEYE_ENERGY_KEYS mapped Predbat's *_today args onto the lifetime "Total*"
accumulators. Those are firmware-derived and drift: on a live capture
TotalConsumption read 14332.40 kWh where the other lifetime counters implied
15475.00 (buy 13579.20 + PV 2208.30 - sell 11.80 + discharge 5255.90 -
charge 5556.60), a 7.4% shortfall. That fed straight into Predbat's learned
load, so load_today climbed slower than the house actually consumed and every
charge window came out under-sized.

The Daily registers balance exactly on the same payload
(DailyConsumption 12.80 = DailyActiveProduction 4.50 + DailyEnergyPurchased
7.00 - DailyGridFeedIn 0.00 + DailyDischargingEnergy 4.10 -
DailyChargingEnergy 2.80), so all four counters now read from them.

The old comment claimed daily counters were unusable because Predbat needs an
incrementing series and a midnight reset would read as a large negative delta.
That is not the case: minute_data() smooths the near-midnight drop and
clean_incrementing_reverse() re-bases on any reset to <= 0, so the nightly
return to 0.00 is absorbed. Comment corrected rather than left to mislead.

Tests: energy counters now assert the Daily values, plus a new test pinning
the daily energy balance so a drifted key spelling breaks the sum - the same
failure that hid the original bug.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: springfall2008

apps/predbat/deye.py

@@ -812,7 +812,9 @@ async def publish_data(self):
             if rated_power > 0:
                 self.dashboard_item(self._sensor_name(sn, "inverter_limit"), state=rated_power, attributes={"unit_of_measurement": "W", "friendly_name": f"DEYE {sn} Inverter Limit"}, app="deye")
 
-            # Lifetime energy counters feed Predbat's load/import/export history learning.
+            # Daily energy counters feed Predbat's load/import/export history learning. They
+            # reset at midnight; minute_data/clean_incrementing_reverse absorb that (see
+            # DEYE_ENERGY_KEYS for why the Daily registers are used over the Total* ones).
             for leaf, value in self.device_energy.get(sn, {}).items():
                 self.dashboard_item(
                     self._sensor_name(sn, leaf),

apps/predbat/deye_const.py

@@ -182,19 +182,29 @@
 DEYE_TELEMETRY_NEGATE = ("grid_power",)
 
 # Cumulative energy counters Predbat needs for its history-based load/rate learning, from
-# the same device/latest dataList. The lifetime "Total*" counters are used rather than the
-# "Daily*" ones because Predbat requires an incrementing series and the daily counters
-# reset at midnight, which would read as a large negative delta every night.
+# the same device/latest dataList.
 #
 # The mapping is confirmed by DEYE's own daily figures balancing exactly:
 #   DailyConsumption 12.80 = DailyActiveProduction 4.50 + DailyEnergyPurchased 7.00
 #                            + (DailyDischargingEnergy 4.10 - DailyChargingEnergy 2.80)
 # which only holds if ActiveProduction is PV generation alone, excluding battery discharge.
+#
+# These are the DAILY registers, not the lifetime "Total*" ones. The lifetime accumulators
+# are firmware-derived and drift: on a live capture TotalConsumption read 14332.40 kWh where
+# the other lifetime counters implied 15475.00 (buy 13579.20 + PV 2208.30 - sell 11.80 +
+# discharge 5255.90 - charge 5556.60). That 7.4% shortfall fed straight into Predbat's
+# learned load and under-sized every charge window. The Daily registers balance exactly on
+# the same payload, so the whole set reads from them.
+#
+# A daily counter is safe despite resetting at midnight: minute_data() smooths the
+# near-midnight drop (utils.py, the near_midnight branch) and clean_incrementing_reverse()
+# re-bases on any reset to <= 0, so the nightly return to 0.00 is absorbed rather than read
+# as a negative delta. Predbat only ever needs today-so-far plus history from these args.
 DEYE_ENERGY_KEYS = {
-    "import_today": "TotalEnergyBuy",
-    "export_today": "TotalEnergySell",
-    "pv_today": "TotalActiveProduction",
-    "load_today": "TotalConsumption",
+    "import_today": "DailyEnergyPurchased",
+    "export_today": "DailyGridFeedIn",
+    "pv_today": "DailyActiveProduction",
+    "load_today": "DailyConsumption",
 }
 
 # Metrics that must be present in every device/latest response. A key missing here means

apps/predbat/predbat.py

@@ -35,7 +35,7 @@
 import pytz
 import asyncio
 
-THIS_VERSION = "v8.47.0"
+THIS_VERSION = "v8.47.1"
 
 from download import predbat_update_move, predbat_update_download, check_install, DEFAULT_PREDBAT_REPOSITORY
 from const import MINUTE_WATT

apps/predbat/tests/test_deye_api.py

@@ -390,8 +390,16 @@ async def fake_post_never(endpoint_key, body):
     assert not failed, "test_rated_power_survives_a_payload_that_omits_it"
 
 
-def test_fetch_device_data_captures_energy_counters():
-    """The lifetime energy counters are captured from device/latest for Predbat's history."""
+def test_energy_counters_use_the_daily_registers():
+    """The energy counters read the Daily registers, not the drifting lifetime accumulators.
+
+    The lifetime "Total*" accumulators are firmware-derived and drift. On a live capture
+    TotalConsumption read 14332.40 kWh where the other lifetime counters implied 15475.00
+    (buy 13579.20 + PV 2208.30 - sell 11.80 + discharge 5255.90 - charge 5556.60), a 7.4%
+    shortfall that made Predbat's learned load too low and under-sized every charge window.
+    The Daily registers balanced exactly on that same payload - see
+    test_daily_energy_registers_balance_on_the_live_payload.
+    """
     failed = False
     d = MockDeye()
 
@@ -403,12 +411,30 @@ async def fake_post(endpoint_key, body):
         run_async_local(d.fetch_device_data("INV1"))
 
     energy = d.device_energy.get("INV1", {})
-    expected = {"import_today": 13579.1, "export_today": 11.7, "pv_today": 2198.1, "load_today": 14326.4}
+    # The Daily* values from LIVE_DATA_LIST, not the Total* ones (14326.40, 13579.10, ...).
+    expected = {"import_today": 7.0, "export_today": 0.0, "pv_today": 4.5, "load_today": 12.8}
     for name, want in expected.items():
-        if abs(energy.get(name, 0.0) - want) > 0.01:
-            print(f"ERROR: {name} expected {want}, got {energy.get(name)}")
+        got = energy.get(name)
+        if got is None or abs(got - want) > 0.01:
+            print(f"ERROR: {name} expected {want} (daily register), got {got}")
             failed = True
-    assert not failed, "test_fetch_device_data_captures_energy_counters"
+    assert not failed, "test_energy_counters_use_the_daily_registers"
+
+
+def test_daily_energy_registers_balance_on_the_live_payload():
+    """The Daily registers are self-consistent, which is why load_today trusts them.
+
+    Guards the key map as a set: if any of these five names drifts to a wrong spelling the
+    balance breaks, which is the same failure that hid the original TotalConsumption bug.
+    """
+    failed = False
+    flat = {item["key"]: float(item["value"]) for item in LIVE_DATA_LIST}
+
+    derived = flat["DailyActiveProduction"] + flat["DailyEnergyPurchased"] - flat["DailyGridFeedIn"] + flat["DailyDischargingEnergy"] - flat["DailyChargingEnergy"]
+    if abs(derived - flat["DailyConsumption"]) > 0.01:
+        print(f"ERROR: daily energy balance {derived} != DailyConsumption {flat['DailyConsumption']}")
+        failed = True
+    assert not failed, "test_daily_energy_registers_balance_on_the_live_payload"
 
 
 def test_energy_counters_absent_are_not_invented():
@@ -586,7 +612,8 @@ def run_deye_api_tests(my_predbat):
         ("battery_rate_max", test_battery_rate_max_from_charge_current),
         ("rated_power_captured", test_rated_power_captured_for_inverter_limit),
         ("rated_power_not_clobbered", test_rated_power_survives_a_payload_that_omits_it),
-        ("energy_counters_captured", test_fetch_device_data_captures_energy_counters),
+        ("energy_counters_daily_registers", test_energy_counters_use_the_daily_registers),
+        ("daily_energy_balance", test_daily_energy_registers_balance_on_the_live_payload),
         ("energy_counters_absent", test_energy_counters_absent_are_not_invented),
         ("derive_capacity_no_rating", test_derive_battery_capacity_without_rating),
         ("fetch_battery_config_success", test_fetch_battery_config_caches_on_success),

apps/predbat/tests/test_deye_publish.py

@@ -328,20 +328,21 @@ def test_automatic_config_skips_missing_ratings():
 
 
 def test_automatic_config_maps_energy_counters():
-    """The lifetime energy counters are published and mapped to Predbat's history args."""
+    """The energy counters are published and mapped to Predbat's history args."""
     failed = False
     d = RecordingDeye()
     d.device_list = ["INV1"]
     d.device_values = {"INV1": {"soc": 100.0}}
-    d.device_energy = {"INV1": {"import_today": 13579.1, "export_today": 11.7, "pv_today": 2198.1, "load_today": 14326.4}}
+    # Daily-register magnitudes, matching DEYE_ENERGY_KEYS' Daily* sources.
+    d.device_energy = {"INV1": {"import_today": 7.0, "export_today": 0.0, "pv_today": 4.5, "load_today": 12.8}}
     d.set_args = {}
     d.set_arg = lambda k, v: d.set_args.__setitem__(k, v)
     import tests.test_infra as ti
 
     ti.run_async(d.publish_data())
     ti.run_async(d.automatic_config())
 
-    for leaf, value in (("import_today", 13579.1), ("export_today", 11.7), ("pv_today", 2198.1), ("load_today", 14326.4)):
+    for leaf, value in (("import_today", 7.0), ("export_today", 0.0), ("pv_today", 4.5), ("load_today", 12.8)):
         entity = f"sensor.predbat_deye_inv1_{leaf}"
         if d.published.get(entity) != value:
             print(f"ERROR: {entity} published as {d.published.get(entity)}, expected {value}")