predbat.py

# -----------------------------------------------------------------------------
# Predbat Home Battery System
# Copyright Trefor Southwell 2025-2026 - All Rights Reserved
# This application maybe used for personal use only and not for commercial use
# -----------------------------------------------------------------------------
# fmt off
# pylint: disable=consider-using-f-string
# pylint: disable=line-too-long
# pylint: disable=attribute-defined-outside-init


"""Main PredBat orchestrator module.

Entry point for the battery prediction and optimisation system. The PredBat class
uses multiple inheritance to combine Home Assistant interface, data fetching,
planning, execution, output publishing, and user interface capabilities into a
single orchestrator that runs the main prediction/optimisation loop every 5 minutes.
"""

import copy
import os
import sys
from datetime import datetime, timedelta, timezone
import traceback
import sys
import gc
import random
import time

# from memory_profiler import profile

IS_COMPILED = getattr(sys, "frozen", False)

import hass as hass
import pytz
import asyncio

THIS_VERSION = "v8.50.0"

from download import predbat_update_move, predbat_update_download, check_install, DEFAULT_PREDBAT_REPOSITORY
from const import MINUTE_WATT

# Only do the self-install/self-update logic if we are NOT compiled.
if not IS_COMPILED:
    # Sanity check the install and re-download if corrupted
    passed, modified = check_install(THIS_VERSION, repository=DEFAULT_PREDBAT_REPOSITORY)
    if not passed:
        print("Warn: Predbat files are not installed correctly, trying to download them")
        files = predbat_update_download(THIS_VERSION, repository=DEFAULT_PREDBAT_REPOSITORY)
        if files:
            predbat_update_move(THIS_VERSION, files)
        sys.exit(1)
    elif modified:
        print("Warn: Predbat files are installed but have modifications")
    else:
        print("Predbat files are installed correctly for version {}".format(THIS_VERSION))
else:
    # In compiled mode, we skip the entire self-update logic
    print("Running in compiled mode; skipping local file checks and auto-update.")

from const import (
    TIME_FORMAT,
    PREDICT_STEP,
    RUN_EVERY,
    INVERTER_TEST,
    CONFIG_ROOTS,
    CONFIG_REFRESH_PERIOD,
    INVERTER_QUICK_UPDATE_SECONDS,
)
from config import APPS_SCHEMA, CONFIG_ITEMS
from utils import minutes_since_yesterday, dp1, dp2, dp3
from predheat import PredHeat
from octopus import Octopus
from energydataservice import Energidataservice
from stromligning import Stromligning
from components import Components, COMPONENT_LIST
from execute import Execute
from marginal import Marginal
from plan import Plan
from fetch import Fetch
from output import Output
from userinterface import UserInterface
from compare import Compare
from plugin_system import PluginSystem
from github import GitHub
from ha import run_async
from control_ledger import ControlLedger


class PredBat(hass.Hass, Octopus, Energidataservice, Stromligning, Fetch, Plan, Marginal, Execute, Output, UserInterface, GitHub):
    """Main PredBat orchestrator combining all subsystems via multiple inheritance.

    Inherits from Hass (HA interface), Octopus (rate loading), Energidataservice, Stromligning,
    Fetch (data loading), Plan (optimisation), Execute (inverter control),
    Output (sensor publishing), and UserInterface (config management).
    Runs the main prediction/optimisation loop every 5 minutes via update_pred().
    """

    def unit_conversion(self, entity_id, state, units, required_unit, going_to=False):
        """
        Convert state to the required unit if necessary
        """

        if not required_unit:
            return state

        if not units:
            units = self.ha_interface.get_state(entity_id=entity_id, default="", attribute="unit_of_measurement")
        if not units:
            return state

        units = str(units).strip()
        required_unit = str(required_unit).strip()

        try:
            state = float(state)
        except (ValueError, TypeError):
            pass

        # Swap order for going to conversion
        if going_to:
            units, required_unit = required_unit, units

        if isinstance(state, float) and units and required_unit and units != required_unit:
            units_lhs = units[0]
            if units_lhs == "K":
                units_lhs = "k"
            if units_lhs not in ["k", "M", "m"]:
                units_lhs = ""

            required_lhs = required_unit[0]
            if required_lhs not in ["k", "M", "m"]:
                required_lhs = ""
            if required_lhs == "K":
                required_lhs = "k"

            if units_lhs == "M" and required_lhs == "k":
                # Convert MW to kW
                state *= 1000.0
                units = "k" + units[1:]
            elif units_lhs == "k" and required_lhs == "M":
                # Convert kW to MW
                state /= 1000.0
                units = "M" + units[1:]
            elif units_lhs == "k" and required_lhs == "":
                # Convert kWh to Wh
                state *= 1000.0
                units = units[1:]  # Remove 'k' from units
            elif units_lhs == "" and required_lhs == "k":
                # Convert Wh to kWh
                state /= 1000.0
                units = "k" + units  # Add 'k' to units
            elif units_lhs == "m" and required_lhs == "":
                # Convert mW to W
                state /= 1000.0
                units = units[1:]  # Remove 'm' from units
            elif units_lhs == "" and required_lhs == "m":
                # Convert W to mW
                state *= 1000.0
                units = "m" + units  # Add 'm' to units

            if units != required_unit:
                self.log("Warn: unit_conversion - Units mismatch for {}: expected {}, got {} after conversion".format(entity_id, required_unit, units))
        return state

    def get_state_wrapper(self, entity_id=None, default=None, attribute=None, refresh=False, required_unit=None, raw=False):
        """
        Wrapper function to get state from HA

        entity_id = The entity id to get
        default = Default value if not found
        attribute = Specific attribute to get, if not set defaults to the current state
        refresh = Force a refresh of the state from HA
        required_unit = Convert the state to this unit if different
        raw = Fetch the raw database information which includes state and all the attributes
        """
        if not self.ha_interface:
            self.log("Error: get_state_wrapper - No HA interface available")
            return None

        # Entity with coded attribute
        if entity_id and "$" in entity_id:
            entity_id, attribute = entity_id.split("$")

        state = self.ha_interface.get_state(entity_id=entity_id, default=default, attribute=attribute, refresh=refresh, raw=raw)
        if not raw and required_unit:
            state = self.unit_conversion(entity_id, state, None, required_unit)

        return state

    def set_state_wrapper(self, entity_id, state, attributes={}, required_unit=None):
        """
        Wrapper function to get state from HA
        """
        if not self.ha_interface:
            self.log("Error: set_state_wrapper - No HA interface available")
            return False

        state = self.unit_conversion(entity_id, state, None, required_unit, going_to=True)
        return self.ha_interface.set_state(entity_id, state, attributes=attributes)

    def fire_event_wrapper(self, domain, service):
        """
        Wrapper function to fire a HA event
        """
        if not self.ha_interface:
            self.log("Error: fire_event_wrapper - No HA interface available")
            return False
        return self.call_service_wrapper("fire_event/service_registered", event_domain=domain, event_service=service)

    def call_service_wrapper(self, service, **kwargs):
        """
        Wrapper function to call a HA service
        """
        if not self.ha_interface:
            self.log("Error: call_service_wrapper - No HA interface available")
            return False
        return self.ha_interface.call_service(service, **kwargs)

    def get_services_wrapper(self):
        """
        Wrapper function to get services from HA
        """
        if not self.ha_interface:
            self.log("Error: get_services_wrapper - No HA interface available")
            return False
        return self.ha_interface.get_services()

    def get_history_wrapper(self, entity_id, days=30, required=True, tracked=True):
        """
        Wrapper function to get history from HA
        """
        if not self.ha_interface:
            self.log("Error: get_history_wrapper - No HA interface available")
            return None

        # If history cache is enabled then use it
        ha_history = None
        if self.components:
            ha_history = self.components.get_component("ha_history")
        if ha_history:
            history = ha_history.get_history(entity_id, days=days, tracked=tracked)
        else:
            history = self.ha_interface.get_history(entity_id, days=days, now=self.now_utc)

        if history and isinstance(history, list):
            ## Get default units and patch it into missing entries in the history
            unit_of_measurement = self.get_state_wrapper(entity_id, attribute="unit_of_measurement")
            if unit_of_measurement:
                for record in history[0]:
                    if "attributes" not in record or not record["attributes"]:
                        record["attributes"] = {}
                    if "unit_of_measurement" not in record["attributes"]:
                        record["attributes"]["unit_of_measurement"] = unit_of_measurement

        if required and (history is None):
            self.log("Error: Failure to fetch history for {}".format(entity_id))
            raise ValueError
        else:
            return history

    def time_now_str(self):
        """
        Return time now as human string
        """
        return (self.midnight + timedelta(minutes=self.minutes_now)).strftime("%H:%M:%S")

    def time_abs_str(self, minute):
        """
        Return time absolute as human string
        """
        return (self.midnight + timedelta(minutes=minute)).strftime("%m-%d %H:%M:%S")

    def reset(self):
        """
        Init stub
        """
        self.text_plan = "Computing please wait..."
        self.prediction_cache_enable = True
        self.base_load = 0
        self.plan_interval_minutes = self.args.get("plan_interval_minutes", 30)
        self.db_manager = None
        self.plan_debug = False
        self.arg_errors = {}
        self.validate_config_retries_remaining = 0
        self.validate_config_next_retry_time = None
        self.ha_interface = None
        self.num_cars = 0
        self.fatal_error = False
        self.components = None
        self.CONFIG_ITEMS = copy.deepcopy(CONFIG_ITEMS)
        self.comparison = None
        self.predheat = None
        self.predbat_mode = "Monitor"
        self.soc_kwh_history = {}
        self.unmatched_args = {}
        self.define_service_list()
        self.stop_thread = False

        # Forecast.solar API request metrics for monitoring
        self.currency_symbols = self.args.get("currency_symbols", "£p")
        self.watch_list = []
        self.restart_active = False
        self.control_ledger = ControlLedger()
        self.control_ledger_restored = False
        self.inverter_needs_reset = False
        self.inverter_needs_reset_force = ""
        self.inverters = []
        self.manual_charge_times = []
        self.manual_export_times = []
        self.manual_freeze_charge_times = []
        self.manual_freeze_export_times = []
        self.manual_demand_times = []
        self.manual_all_times = []
        self.manual_api = []
        self.manual_import_rates = {}
        self.manual_export_rates = {}
        self.manual_load_adjust = {}
        self.config_index = {}
        self.dashboard_index = []
        self.dashboard_index_app = {}
        self.dashboard_values = {}
        self.prefix = self.args.get("prefix", "predbat")
        self.current_status = None
        self.previous_status = None
        self.had_errors = False
        self.plan_valid = False
        self.plan_preclip = None
        self.plan_last_updated = None
        self.plan_last_updated_minutes = 0
        self.plugin_system = None
        self.calculate_plan_every = 5
        self.prediction_started = False
        self.update_pending = True
        self.midnight_utc = None
        self.difference_minutes = 0
        self.minutes_to_midnight = 0
        self.days_previous = [7]
        self.days_previous_weight = [1]
        self.max_days_previous = max(self.days_previous) + 1
        self.forecast_days = 0
        self.forecast_minutes = 0
        self.soc_kw = 0
        self.soc_percent = 0
        self.soc_max = 10.0
        self.battery_temperature = 20
        self.end_record = 24 * 60 * 2
        self.predict_soc = {}
        self.predict_soc_best = {}
        self.predict_iboost_best = {}
        self.predict_metric_best = {}
        self.metric_min_improvement = 0.0
        self.metric_min_improvement_export = 0.1
        self.metric_min_improvement_export_freeze = 0.1
        self.metric_min_improvement_plan = 2.0
        self.export_more_solar = False
        self.export_more_solar_threshold = 1.0
        self.metric_battery_cycle = 0.0
        self.metric_battery_value_scaling = 1.0
        self.metric_battery_value_export_scaling = 0.8
        self.calculate_pv90_plan = False
        self.pv_metric90_weight = 0.15
        self.load_scaling90 = 0.7
        self.metric_future_rate_offset_import = 0.0
        self.metric_future_rate_offset_export = 0.0
        self.metric_inday_adjust_damping = 1.0
        self.metric_standing_charge = 0.0
        self.metric_self_sufficiency = 0.0
        self.metric_dynamic_load_adjust = False
        self.metric_pv_calibration_enable = True
        self.dynamic_load_baseline = {}
        self.iboost_value_scaling = 1.0
        self.rate_import = {}
        self.rate_import_no_io = {}
        self.rate_import_base = {}
        self.rate_export = {}
        self.rate_export_base = {}
        self.rate_gas = {}
        self.rate_slots = []
        self.low_rates = []
        self.high_export_rates = []
        self.cost_today_sofar = 0
        self.carbon_today_sofar = 0
        self.octopus_free_slots = []
        self.octopus_saving_slots = []
        self.car_charging_slots = []
        self.reserve = 0
        self.reserve_percent = 0.0
        self.reserve_current = 0
        self.reserve_current_percent = 0.0
        self.battery_loss = 1.0
        self.battery_loss_discharge = 1.0
        self.inverter_loss = 1.0
        self.inverter_freeze_export_discharge_rate = 0.0
        self.inverter_hybrid = True
        self.pv_ac_limit = 0
        self.inverter_soc_reset = False
        self.inverter_set_charge_before = True
        self.best_soc_min = 0
        self.best_soc_max = 0
        self.best_soc_keep = 0
        self.best_soc_keep_weight = 0.5
        self.rate_min = 0
        self.rate_min_minute = 0
        self.rate_min_forward = {}
        self.rate_min_base = 0
        self.rate_max = 0
        self.rate_max_minute = 0
        self.rate_max_base = 0
        self.rate_export_cost_threshold = 99
        self.rate_import_cost_threshold = 99
        self.rate_best_cost_threshold_charge = None
        self.rate_best_cost_threshold_export = None
        self.rate_average = 0
        self.rate_export_min = 0
        self.rate_export_min_minute = 0
        self.rate_export_max = 0
        self.rate_export_max_minute = 0
        self.rate_export_max_forward = {}
        self.rate_export_average = 0
        self.rate_gas_min = 0
        self.rate_gas_max = 0
        self.rate_gas_average = 0
        self.rate_gas_min_minute = 0
        self.rate_gas_max_minute = 0
        self.set_soc_minutes = 5
        self.set_window_minutes = 5
        self.debug_enable = False
        self.import_today = {}
        self.import_today_now = 0
        self.export_today = {}
        self.export_today_now = 0
        self.pv_today = {}
        self.pv_today_now = 0
        self.pv_power = 0
        self.load_power = 0
        self.battery_power = 0
        self.grid_power = 0
        self.io_adjusted = {}
        self.current_charge_limit = 0.0
        self.current_charge_limit_kwh = 0.0
        self.inverter_limit = 0.0
        self.export_limit = 0.0
        self.charge_limit = []
        self.charge_limit_best = []
        self.charge_window = []
        self.charge_window_best = []
        self.car_charging_battery_size = [100]
        self.car_charging_limit = [100]
        self.car_charging_soc = [0]
        self.car_charging_soc_next = [None]
        self.car_charging_rate = [7.4]
        self.car_charging_loss = 1.0
        self.export_window = []
        self.export_limits = []
        self.export_limits_best = []
        self.export_window_best = []
        self.battery_rate_max_charge = 0.0333
        self.battery_rate_max_charge_dc = 0
        self.battery_rate_max_discharge = 0.0333
        self.battery_rate_max_export = 0.0333
        self.battery_rate_min = 0
        self.battery_rate_max_scaling = 1.0
        self.battery_rate_max_scaling_discharge = 1.0
        self.car_energy_reported_load = False
        self.charge_rate_now = 0
        self.discharge_rate_now = 0
        self.car_charging_hold = False
        self.car_charging_manual_soc = []
        self.car_charging_threshold = 99
        self.car_charging_energy = {}
        self.car_charging_energy_warned = False
        self.octopus_intelligent_charging = False
        self.octopus_intelligent_ignore_unplugged = False
        self.octopus_intelligent_consider_full = False
        self.notify_devices = ["notify"]
        self.octopus_url_cache = {}
        self.ge_url_cache = {}
        self.github_url_cache = {}
        self.load_minutes = {}
        self.load_minutes_now = 0
        self.load_minutes_age = 0
        self.load_last_period = 0
        self.load_last_status = "baseline"
        self.load_last_car_slot = False
        self.battery_capacity_nominal = False
        self.battery_scaling_auto = False
        self.releases = {}
        self.balance_inverters_enable = False
        self.balance_inverters_charge = True
        self.balance_inverters_discharge = True
        self.balance_inverters_crosscharge = True
        self.balance_inverters_threshold_charge = 1.0
        self.balance_inverters_threshold_discharge = 1.0
        self.load_inday_adjustment = 1.0
        self.set_read_only = True
        self.set_read_only_axle = False
        self.set_reserve_enable = False
        self.metric_cloud_coverage = 0.0
        self.future_energy_rates_import = {}
        self.future_energy_rates_export = {}
        self.load_scaling_dynamic = {}
        self.battery_charge_power_curve = {}
        self.battery_charge_power_curve_default = {}
        self.battery_charge_power_curve_auto = False
        self.battery_discharge_power_curve = {}
        self.battery_discharge_power_curve_default = {}
        self.battery_discharge_power_curve_auto = False
        self.computed_charge_curve = False
        self.computed_discharge_curve = False
        self.isCharging = False
        self.isCharging_Target = 0
        self.isExporting = False
        self.isExporting_Target = 0
        self.savings_today_predbat = 0.0
        self.savings_today_predbat_soc = 0.0
        self.savings_today_pvbat = 0.0
        self.savings_today_actual = 0.0
        self.savings_last_updated = None
        self.cost_yesterday_car = 0.0
        self.cost_total_car = 0.0
        self.rate_import = {}
        self.rate_import_replicated = {}
        self.rate_export = {}
        self.rate_export_replicated = {}
        self.rate_slots = []
        self.low_rates = []
        self.high_export_rates = []
        self.axle_sessions = []
        self.cost_today_sofar = 0
        self.carbon_today_sofar = 0
        self.import_today = {}
        self.export_today = {}
        self.pv_today = {}
        self.load_minutes = {}
        self.load_minutes_age = 0
        self.load_forecast = {}
        self.load_forecast_array = []
        self.pv_forecast_minute = {}
        self.pv_forecast_minute10 = {}
        self.pv_forecast_minute90 = {}
        # (p50, p90) content signatures from the previous plan run, used to spot a p90 that has been
        # left behind by a p50 reassigned underneath it - see Plan.refresh_pv_forecast_minute90()
        self.pv_forecast_minute90_signatures = None
        self.load_scaling_dynamic = {}
        self.carbon_intensity = {}
        self.carbon_history = {}
        self.carbon_enable = False
        self.iboost_enable = False
        self.iboost_gas = False
        self.iboost_solar = False
        self.iboost_solar_excess = False
        self.iboost_gas_export = False
        self.iboost_smart = False
        self.iboost_smart_min_length = 30
        self.iboost_on_export = False
        self.iboost_prevent_discharge = False
        self.iboost_smart_threshold = 0
        self.iboost_rate_threshold = 9999
        self.iboost_rate_threshold_export = 9999
        self.iboost_plan = []
        self.iboost_energy_subtract = True
        self.iboost_running = False
        self.iboost_running_full = False
        self.iboost_running_solar = False
        self.last_service_hash = {}
        self.count_inverter_writes = {}
        self.rate_slots = []
        self.low_rates = []
        self.high_export_rates = []
        self.octopus_slots = [[] for _ in range(8)]
        self.cost_today_sofar = 0
        self.carbon_today_sofar = 0
        self.import_today = {}
        self.export_today = {}
        self.pv_today = {}
        self.load_minutes = {}
        self.load_minutes_age = 0
        self.battery_temperature_charge_curve = {}
        self.battery_temperature_discharge_curve = {}
        self.battery_temperature_history = {}
        self.battery_temperature_prediction = {}
        self.alerts = []
        self.alert_active_keep = {}
        self.manual_soc_keep = {}
        self.all_active_keep = {}
        self.set_charge_low_power = False
        self.set_export_low_power = False
        self.config_root = "./"
        self.inverter_can_charge_during_export = True
        self.octopus_last_joined_try = None
        # None = not yet confirmed, True = the current Power Down join service is confirmed registered.
        # Deliberately never set to False - a failed probe still re-tries every join rather than being
        # cached, since the underlying result can be an ambiguous timeout, not just "not registered".
        # See the join logic in octopus.py and TODO(#4599).
        self.octopus_join_service_power_down = None
        self.calculate_savings_max_charge_slots = 1
        self.inverter_data_last_fetch = None
        self.octopus_url_cache_loaded = False
        self.github_url_cache_loaded = False
        self.load_forecast_history = False
        self.prediction_kernel_enable = False

        for root in CONFIG_ROOTS:
            if os.path.exists(root):
                self.config_root = root
                break
        self.config_root_p = self.config_root
        self.log("Config root is {}".format(self.config_root))

    def update_time(self, print=True):
        """
        Update the current time/date
        """
        self.local_tz = pytz.timezone(self.args.get("timezone", "Europe/London"))
        skew = self.args.get("clock_skew", 0)
        if skew:
            self.log("Warn: Clock skew is set to {} minutes".format(skew))
        self.now_utc_real = datetime.now(self.local_tz)
        now_utc = self.now_utc_real + timedelta(minutes=skew)
        now = datetime.now() + timedelta(minutes=skew)
        now = now.replace(second=0, microsecond=0, minute=(now.minute - (now.minute % PREDICT_STEP)))
        now_utc = now_utc.replace(second=0, microsecond=0, minute=(now_utc.minute - (now_utc.minute % PREDICT_STEP)))

        self.now_utc = now_utc
        self.now = now
        self.midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
        self.midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0)

        self.difference_minutes = minutes_since_yesterday(now)
        self.minutes_now = int((now - self.midnight).seconds / 60 / PREDICT_STEP) * PREDICT_STEP
        self.minutes_to_midnight = 24 * 60 - self.minutes_now
        self.log("--------------- PredBat - update at {} with clock skew {} minutes, minutes now {}".format(now_utc, skew, self.minutes_now))

    # @profile
    def _emit_snapshot_metrics(self):
        """Emit point-in-time metrics after each update cycle."""
        from predbat_metrics import metrics

        m = metrics()

        m.up.labels(version=THIS_VERSION).set(1)
        m.last_update_timestamp.set_to_current_time()

        # Plan age
        if self.plan_last_updated:
            plan_age = self.now_utc - self.plan_last_updated
            m.plan_age_minutes.set(plan_age.total_seconds() / 60.0)

        # Battery state
        m.battery_soc_kwh.set(self.soc_kw)
        m.battery_soc_percent.set(self.soc_percent)
        m.battery_max_kwh.set(self.soc_max)
        m.charge_rate_kw.set(self.charge_rate_now * MINUTE_WATT / 1000.0)
        m.discharge_rate_kw.set(self.discharge_rate_now * MINUTE_WATT / 1000.0)
        m.grid_power.set(self.grid_power / 1000.0)
        m.battery_power.set(self.battery_power / 1000.0)
        m.load_power.set(self.load_power / 1000.0)
        m.pv_power.set(self.pv_power / 1000.0)

        # Currency symbol
        m.currency_symbol = self.currency_symbols[0]

        # Cost and savings
        m.cost_today.set(self.cost_today_sofar)
        m.savings_today_pvbat.set(self.savings_today_pvbat)
        m.savings_today_actual.set(self.savings_today_actual)
        m.savings_today_predbat.set(self.savings_today_predbat)

        # Config validity
        m.config_valid.set(0 if self.arg_errors else 1)
        m.config_warnings.set(len(self.arg_errors) if self.arg_errors else 0)

        # Errors
        if self.had_errors:
            m.errors_total.labels(type="general").inc()

        # Control ownership ledger
        conflict_events = self.control_ledger.recent_events(time.time())
        m.control_conflicts_24h.set(len(conflict_events))
        sustained = self.control_ledger.sustained_controls(conflict_events)
        m.control_conflicts_sustained_total.set(len(sustained))
        m.control_conflicts_events = self.control_ledger.newest_events(20)
        m.control_conflicts_sustained_controls = sustained

    def save_plan(self):
        """Save the current best plan via the storage component so it can be restored on next startup."""
        storage = self.components.get_component("storage") if self.components else None
        if not storage:
            self.log("Warning: Storage component unavailable, cannot save plan")
            return
        plan_data = {
            "charge_window_best": self.charge_window_best,
            "charge_limit_best": self.charge_limit_best,
            "export_window_best": self.export_window_best,
            "export_limits_best": self.export_limits_best,
            "plan_preclip": self.plan_preclip,
            "plan_last_updated": self.plan_last_updated.isoformat() if self.plan_last_updated else None,
            "plan_last_updated_minutes": self.plan_last_updated_minutes,
        }
        try:
            expiry = self.now_utc + timedelta(hours=8)
            run_async(storage.save("predbat", "plan", plan_data, format="json", expiry=expiry))
            self.log("Saved plan to storage")
        except Exception as e:
            self.log("Warning: Failed to save plan: {}".format(e))

    def load_plan(self):
        """Restore a previously saved plan from storage if it is recent enough."""
        storage = self.components.get_component("storage") if self.components else None
        if not storage:
            self.log("Warning: Storage component unavailable, cannot load plan")
            return

        try:
            plan_data = run_async(storage.load("predbat", "plan"))
        except Exception as e:
            self.log("Warning: Failed to load saved plan: {}".format(e))
            return

        if not plan_data:
            self.log("No saved plan found in storage")
            return

        if not isinstance(plan_data, dict):
            self.log("Warning: Saved plan has unexpected type {}, ignoring".format(type(plan_data).__name__))
            return

        saved_updated = plan_data.get("plan_last_updated")
        if not saved_updated:
            self.log("Saved plan has no timestamp, ignoring")
            return

        try:
            saved_dt = datetime.fromisoformat(saved_updated)
        except (ValueError, TypeError):
            self.log("Warning: Saved plan timestamp is invalid, ignoring")
            return

        if saved_dt.tzinfo is None:
            saved_dt = saved_dt.replace(tzinfo=timezone.utc)

        age_minutes = (self.now_utc - saved_dt).total_seconds() / 60

        self.charge_window_best = plan_data.get("charge_window_best", [])
        self.charge_limit_best = plan_data.get("charge_limit_best", [])
        self.export_window_best = plan_data.get("export_window_best", [])
        self.export_limits_best = plan_data.get("export_limits_best", [])
        # The pre-clip snapshot plan selection scores against. Older saves predate it, and it is only ever a
        # four part plan, so anything else is discarded and the comparison falls back to the clipped plans.
        preclip = plan_data.get("plan_preclip")
        self.plan_preclip = tuple(preclip) if isinstance(preclip, (list, tuple)) and len(preclip) == 4 else None
        self.plan_last_updated = saved_dt
        self.plan_last_updated_minutes = plan_data.get("plan_last_updated_minutes", 0)
        self.plan_valid = True
        self.log("Restored saved plan from {:.0f} minutes ago: {} charge windows, {} export windows".format(age_minutes, len(self.charge_window_best), len(self.export_window_best)))

    def record_final_run_status(self, status, status_extra):
        """
        Publish the component health dashboard item and record the final run status for this cycle.
        Any component that is active but not alive fails the run, even if the plan itself computed successfully.
        """
        failed_components = []
        if self.components:
            all_components = self.components.get_all()
            active_components = self.components.get_active()
            error_count = 0
            component_status = {}
            component_error_count = {}
            all_healthy = True
            for component_name in all_components:
                is_active = self.components.is_active(component_name)
                is_alive = self.components.is_alive(component_name)
                component_error_count[component_name] = self.components.get_error_count(component_name)
                if is_active and not is_alive:
                    # Component is active but not alive - error state
                    component_status[component_name] = "error"
                    all_healthy = False
                    error_count += 1
                    component = self.components.get_component(component_name)
                    if not component.is_calculating():
                        failed_components.append(COMPONENT_LIST.get(component_name, {}).get("name", component_name))
                elif is_active:
                    component_status[component_name] = "running"
                else:
                    component_status[component_name] = "disabled"
            self.dashboard_item(
                "binary_sensor." + self.prefix + "_components_healthy",
                state="on" if all_healthy else "off",
                attributes={
                    "friendly_name": "Predbat components healthy",
                    "icon": "mdi:cog-outline" if all_healthy else "mdi:cog-off-outline",
                    "components": component_status,
                    "component_error_count": component_error_count,
                    "active_count": len(active_components),
                    "total_count": len(all_components),
                    "error_count": error_count,
                },
            )

        if self.had_errors:
            self.log("Error: Completed run status {} with Errors reported (check log)".format(status))
        elif failed_components:
            error_status = "Error: Complete run status {} with component errors: {}".format(status, ", ".join(failed_components))
            self.log(error_status)
            self.record_status(
                error_status,
                debug="best_charge_limit={} best_charge_window={} best_export_limit= {} best_export_window={}".format(self.charge_limit_best, self.charge_window_best, self.export_limits_best, self.export_window_best),
                notify=True,
                had_errors=True,
            )
        else:
            self.log("Info: Completed run status {}".format(status))
            self.record_status(
                status,
                debug="best_charge_limit={} best_charge_window={} best_export_limit= {} best_export_window={}".format(self.charge_limit_best, self.charge_window_best, self.export_limits_best, self.export_window_best),
                notify=True,
                extra=status_extra,
            )

    def update_pred(self, scheduled=True):
        """
        Update the prediction state, everything is called from here right now
        """
        recompute = False
        status_extra = ""
        self.had_errors = False

        self.update_time()
        self.save_current_config()

        # Check our version, don't check for cloud version which can't update directly
        if not self.get_arg("user_id", None):
            self.download_predbat_releases()

        # Check if we are still running the template configuration, if so don't run the plan
        if self.get_arg("template", False):
            self.log("Error: You have not completed editing the apps.yaml template, Predbat cannot run. Please comment out 'Template: True' line in apps.yaml to start Predbat running")
            self.record_status("Error: Template Configuration, remove 'Template: True' line in apps.yaml to start predbat running", had_errors=True)
            return

        self.expose_config("active", True)
        self.fetch_config_options()
        sensor_force_replan = self.fetch_sensor_data()

        # Check if any sensor changes require a replan
        if sensor_force_replan:
            self.log("Sensor changes require a replan, will recompute the plan")
            recompute = True

        # Open the control-ledger cycle BEFORE the first inverter read. fetch_inverter_data()
        # runs update_status(), which writes scheduled_charge_enable through write_and_poll_switch
        # and so CONFIRMS ownership - stamping those with the previous cycle number made the next
        # observe() of them hit the STALE rung whenever execute_plan() had written the entity in
        # the prior run. Every write in a run must share that run's cycle number.
        self.control_ledger.begin_cycle()

        # Fetch inverter data
        if not self.fetch_inverter_data():
            self.log("Error: Failed to fetch inverter data, not able to compute a plan")
            self.record_status("Error: Failed to fetch inverter data, not able to compute a plan", had_errors=True)
            return

        # Check if we have valid import rates
        if self.rate_min == self.rate_max == 0:
            self.log("Error: Import rates are all zero, not able to compute a plan")
            self.record_status("Error: Import rates are all zero, not able to compute a plan", had_errors=True)
            return

        if self.dynamic_load():
            self.log("Dynamic load adjustment changed, will recompute the plan")
            recompute = True

        if not scheduled or not self.plan_valid or recompute:
            self.log("Will recompute the plan as it is invalid")
            recompute = True
        else:
            plan_age = self.now_utc - self.plan_last_updated
            plan_age_minutes = plan_age.seconds / 60.0
            self.log("Plan was last updated on {} and is now {} minutes old".format(self.plan_last_updated, dp1(plan_age_minutes)))

        # Calculate the new plan (or re-use existing)
        recompute = self.calculate_plan(recompute=recompute)

        # Persist the plan so it can be restored immediately on next startup
        if recompute and self.plan_valid:
            self.save_plan()

        # Publish rate data
        self.publish_rate_and_threshold()

        # Execute the plan, re-read the inverter first if we had to calculate (as time passes during calculations)
        if recompute:
            if not self.fetch_inverter_data():
                self.log("Error: Failed to fetch inverter data, not able to execute the plan")
                self.record_status("Error: Failed to fetch inverter data, not able to execute the plan", had_errors=True)
                return
        status, status_extra = self.execute_plan()

        # If the plan was not updated, and the time has expired lets update it now
        if not recompute:
            plan_age = self.now_utc - self.plan_last_updated
            plan_age_minutes = plan_age.seconds / 60.0

            if (plan_age_minutes + RUN_EVERY) > self.calculate_plan_every:
                recompute = True
                self.log("Will recompute the plan as it is now {} minutes old and will exceed the max age of {} minutes before the next run".format(dp1(plan_age_minutes), self.calculate_plan_every))
                plan_random_delay = self.get_arg("plan_random_delay", 0)
                if plan_random_delay > 0:
                    delay = random.uniform(0, plan_random_delay)
                    self.log("Adding a random delay of {:.1f} seconds before recalculating the plan....".format(delay))
                    time.sleep(delay)
                # Calculate an updated plan, fetch the inverter data again and execute the plan
                self.calculate_plan(recompute=True)
                if not self.fetch_inverter_data():
                    self.log("Error: Failed to fetch inverter data, not able to execute the plan")
                    self.record_status("Error: Failed to fetch inverter data, not able to execute the plan", had_errors=True)
                    return
                status, status_extra = self.execute_plan()
            else:
                self.log("Will not recompute the plan, it is {} minutes old and max age is {} minutes".format(dp1(plan_age_minutes), self.calculate_plan_every))

        # Notify listeners that plan has been executed (consumed by gateway, plugins, etc.)
        if self.plugin_system:
            self.plugin_system.call_hooks(
                "on_plan_executed",
                charge_windows=self.charge_window_best,
                charge_limits=self.charge_limit_best,
                export_windows=self.export_window_best,
                export_limits=self.export_limits_best,
                charge_rate_w=int(self.battery_rate_max_charge * MINUTE_WATT),
                discharge_rate_w=int(self.battery_rate_max_discharge * MINUTE_WATT),
                soc_max=self.soc_max,
                reserve=self.reserve,
                timezone=str(self.local_tz),
            )

        # iBoost solar diverter model update state, only on 5 minute intervals
        if self.iboost_enable and scheduled:
            if self.iboost_energy_today:
                # If we have a realtime sensor just use that data
                self.iboost_next = self.iboost_today
            elif recompute and (self.minutes_now >= 0) and (self.minutes_now < self.calculate_plan_every):
                # Reset at midnight
                self.iboost_next = 0
            # Save next iBoost model value
            self.expose_config("iboost_today", self.iboost_next)
            self.log("iBoost model today updated to {}".format(self.iboost_next))

        # Update register writes counter
        previous_inverter_writes = self.load_previous_value_from_ha(self.prefix + ".inverter_register_writes")
        try:
            previous_inverter_writes = int(previous_inverter_writes)
        except (ValueError, TypeError):
            previous_inverter_writes = 0
        for id in self.count_inverter_writes.keys():
            previous_inverter_writes += self.count_inverter_writes[id]
            self.count_inverter_writes[id] = 0

        self.dashboard_item(
            self.prefix + ".inverter_register_writes",
            state=dp2(previous_inverter_writes),
            attributes={
                "friendly_name": "Total register writes (all inverters)",
                "state_class": "measurement",
                "unit_of_measurement": "writes",
                "icon": "mdi:counter",
            },
        )
        self.log("Total inverter register writes now {}".format(previous_inverter_writes))

        # Control interference detection. Restored once per process from the entity's own
        # attributes so the 24h window survives a pod restart - without this "3 in 24h"
        # silently means "3 since the last restart".
        now_ts = time.time()
        if not self.control_ledger_restored:
            self.control_ledger.restore(self.load_previous_value_from_ha(self.prefix + ".control_conflicts", attribute="events"))
            self.control_ledger_restored = True
        # prune() and recent_events() are NOT the same filter and neither can stand in for the
        # other. prune() maintains the durable store, dropping only what has genuinely aged out;
        # a future-dated event (a clock behind NTP) stays, because the clock corrects and the
        # history must survive until it does. recent_events() is what can be JUDGED right now, so
        # it is what the count and the sustained list are built from. The "events" attribute
        # publishes the stored list, because that attribute IS the store restore() reads back -
        # publishing only the events it can judge would delete the rest on the next run.
        self.control_ledger.prune(now_ts)
        conflict_events = self.control_ledger.recent_events(now_ts)
        sustained = self.control_ledger.sustained_controls(conflict_events)
        self.dashboard_item(
            self.prefix + ".control_conflicts",
            state=len(conflict_events),
            attributes={
                "friendly_name": "Settings changed outside Predbat (24h)",
                "state_class": "measurement",
                "unit_of_measurement": "changes",
                "icon": "mdi:account-alert",
                # Newest 20 BY TIME. The stored list is sorted by restore(), so on a pod whose clock
                # is behind, this run's real event carries a small "at", sorts to index 0, and a
                # plain [-20:] tail discarded the very event just detected - from the attribute that
                # IS the durable store.
                "events": self.control_ledger.newest_events(20),
                "sustained": sustained,
            },
        )
        if conflict_events:
            self.log("Control interference: {} change(s) in the last 24h, sustained on {}".format(len(conflict_events), sustained or "nothing"))

        if self.calculate_savings:
            # Get current totals
            savings_total_predbat = self.load_previous_value_from_ha(self.prefix + ".savings_total_predbat")
            try:
                savings_total_predbat = float(savings_total_predbat)
            except (ValueError, TypeError):
                savings_total_predbat = 0.0
            savings_total_last_updated = self.load_previous_value_from_ha(self.prefix + ".savings_total_predbat", attribute="last_updated")
            savings_total_start_date = self.load_previous_value_from_ha(self.prefix + ".savings_total_predbat", attribute="start_date")
            todays_date = self.now_utc_real.strftime("%Y-%m-%d")

            # If there is no date its a new install so we will save tomorrow as the first point
            if not savings_total_last_updated:
                savings_total_last_updated = todays_date
            if not savings_total_start_date:
                savings_total_start_date = todays_date

            savings_total_pvbat = self.load_previous_value_from_ha(self.prefix + ".savings_total_pvbat")
            try:
                savings_total_pvbat = float(savings_total_pvbat)
            except (ValueError, TypeError):
                savings_total_pvbat = 0.0

            savings_total_soc = self.load_previous_value_from_ha(self.prefix + ".savings_total_soc")
            try:
                savings_total_soc = float(savings_total_soc)
            except (ValueError, TypeError):
                savings_total_soc = 0.0

            savings_total_actual = self.load_previous_value_from_ha(self.prefix + ".savings_total_actual")
            try:
                savings_total_actual = float(savings_total_actual)
            except (ValueError, TypeError):
                savings_total_actual = 0.0

            if self.num_cars > 0:
                cost_total_car = self.load_previous_value_from_ha(self.prefix + ".cost_total_car")
                try:
                    cost_total_car = float(cost_total_car)
                except (ValueError, TypeError):
                    cost_total_car = 0.0
            else:
                cost_total_car = 0

            # Increment total at 1am once we have today's data stable (cloud data can lag)
            if self.minutes_now > 60 and savings_total_last_updated and savings_total_last_updated != todays_date and scheduled and not self.set_read_only:
                savings_total_predbat += self.savings_today_predbat
                savings_total_pvbat += self.savings_today_pvbat
                savings_total_soc = self.savings_today_predbat_soc
                savings_total_actual += self.savings_today_actual
                savings_total_last_updated = todays_date
                cost_total_car += self.cost_yesterday_car

            self.dashboard_item(
                self.prefix + ".savings_total_predbat",
                state=dp2(savings_total_predbat),
                attributes={
                    "friendly_name": "Total Predbat savings",
                    "state_class": "measurement",
                    "unit_of_measurement": self.currency_symbols[1],
                    "pounds": dp2(savings_total_predbat / 100.0),
                    "icon": "mdi:cash-multiple",
                    "start_date": savings_total_start_date,
                    "last_updated": savings_total_last_updated,
                },
            )
            self.dashboard_item(
                self.prefix + ".savings_total_soc",
                state=dp2(savings_total_soc),
                attributes={
                    "friendly_name": "Predbat savings, yesterday SoC",
                    "state_class": "measurement",
                    "unit_of_measurement": "kWh",
                    "icon": "mdi:battery-50",
                    "start_date": savings_total_start_date,
                    "last_updated": savings_total_last_updated,
                },
            )
            self.dashboard_item(
                self.prefix + ".savings_total_actual",
                state=dp2(savings_total_actual),
                attributes={
                    "friendly_name": "Actual total energy cost",
                    "state_class": "measurement",
                    "unit_of_measurement": self.currency_symbols[1],
                    "pounds": dp2(savings_total_actual / 100.0),
                    "icon": "mdi:cash-multiple",
                    "start_date": savings_total_start_date,
                    "last_updated": savings_total_last_updated,
                },
            )
            self.dashboard_item(
                self.prefix + ".savings_total_pvbat",
                state=dp2(savings_total_pvbat),
                attributes={
                    "friendly_name": "Total Savings vs no PV/Battery system",
                    "state_class": "measurement",
                    "unit_of_measurement": self.currency_symbols[1],
                    "pounds": dp2(savings_total_pvbat / 100.0),
                    "icon": "mdi:cash-multiple",
                    "start_date": savings_total_start_date,
                    "last_updated": savings_total_last_updated,
                },
            )
            if self.num_cars > 0:
                self.dashboard_item(
                    self.prefix + ".cost_total_car",
                    state=dp2(cost_total_car),
                    attributes={
                        "friendly_name": "Total car cost (approx)",
                        "state_class": "measurement",
                        "unit_of_measurement": self.currency_symbols[1],
                        "pounds": dp2(cost_total_car / 100.0),
                        "icon": "mdi:cash-multiple",
                        "start_date": savings_total_start_date,
                        "last_updated": savings_total_last_updated,
                    },
                )

        # Car SoC increment
        if scheduled:
            for car_n in range(self.num_cars):
                if car_n < len(self.car_charging_manual_soc) and self.car_charging_manual_soc[car_n]:
                    car_postfix = "" if car_n == 0 else "_" + str(car_n)
                    self.log("Car {} charging Manual SoC current is {} next is {}".format(car_n, self.car_charging_soc[car_n], self.car_charging_soc_next[car_n]))
                    if self.car_charging_soc_next[car_n] is not None:
                        self.expose_config("car_charging_manual_soc_kwh" + car_postfix, dp3(self.car_charging_soc_next[car_n]))

        # Holiday days left countdown, subtract a day at midnight every day
        if scheduled and self.holiday_days_left > 0 and self.minutes_now < RUN_EVERY:
            self.holiday_days_left -= 1
            self.expose_config("holiday_days_left", self.holiday_days_left)
            self.log("Holiday days left is now {}".format(self.holiday_days_left))

        if self.debug_enable:
            self.create_debug_yaml()

        self.record_final_run_status(status, status_extra)

        self.expose_config("active", False)
        self.save_current_config()

        # Call plugin update hooks
        if self.plugin_system:
            self.plugin_system.call_hooks("on_update")

        # Emit snapshot metrics (no-ops when prometheus_client absent)
        self._emit_snapshot_metrics()

        if self.comparison:
            if (scheduled and self.minutes_now < RUN_EVERY) or self.get_arg("compare_active", False):
                # Compare tariffs either when triggered or daily at midnight
                self.expose_config("compare_active", True)
                self.comparison.run_all()
                self.expose_config("compare_active", False)
            else:
                # Otherwise just update HA sensors to prevent then expiring
                self.comparison.publish_only()
        else:
            self.expose_config("compare_active", False)

        # Free memory
        if not self.debug_enable:
            self.load_minutes_step = {}
            self.load_minutes_step10 = {}
            self.pv_forecast_minute_step = {}
            self.pv_forecast_minute10_step = {}

        gc.collect()

        # Schedule inverter update for 30 seconds time to allow the inverter to process the changes we just made before we fetch the data again
        # This allows the power flow to update for the user more quickly.
        self.inverter_data_last_fetch = datetime.now() - timedelta(seconds=INVERTER_QUICK_UPDATE_SECONDS) + timedelta(seconds=30)

    async def async_download_predbat_version(self, version):
        """
        Sync wrapper for async download_predbat_version
        """
        return await self.run_in_executor(self.download_predbat_version, version)

    def download_predbat_version(self, version):
        """
        Download a version of Predbat from GitHub

        Args:
            version (str): The version of Predbat to download.

        Returns:
            bool: True if the download and update were successful, False otherwise.
        """
        if version == THIS_VERSION:
            self.log("Warn: Predbat update requested for the same version as we are running ({}), no update required".format(version))
            return

        selected_tag = version.split(" ")[0] if version else ""
        if selected_tag == "main":
            repository = self.get_predbat_repository()
        else:
            repository = DEFAULT_PREDBAT_REPOSITORY

        self.log("Update Predbat to version {} from repository {}".format(version, repository))
        self.expose_config("version", True, force=True, in_progress=True)

        files = predbat_update_download(version, repository=repository)
        if files:
            # Notify before killing threads so the WebSocket is still healthy
            if self.get_arg("set_system_notify"):
                self.call_notify("Predbat: update to: {}".format(version))

            # Kill the current threads
            self.log("Kill current threads before update")
            self.stop_thread = True

            # Perform the update
            self.log("Perform the update.....")
            if predbat_update_move(version, files):
                self.log("Update to version {} completed".format(version))
                return True
            else:
                self.log("Warn: Predbat update failed to move Predbat version {}".format(version))
                return False
        else:
            self.log("Warn: Predbat update failed to download Predbat version {}".format(version))
        return False

    def validate_is_int(self, value):
        """
        Validate that a value is an integer
        """
        try:
            value = int(value)
        except (ValueError, TypeError):
            return False
        return True

    def validate_is_float(self, value):
        """
        Validate that a value is a float
        """
        try:
            value = float(value)
        except (ValueError, TypeError):
            return False
        return True

    def validate_is_boolean(self, value):
        """
        Validate that a value is a boolean switch
        """
        if isinstance(value, str) and value.lower() in ["on", "off"]:
            return True

        try:
            value = bool(value)
        except (ValueError, TypeError):
            return False
        return True

    def validate_config(self):
        """
        Uses APPS_SCHEMA to validate the self.args configuration read from apps.yaml
        """
        errors = 0
        self.arg_errors = {}
        for name in APPS_SCHEMA:
            spec = APPS_SCHEMA[name]
            required = spec.get("required", False)
            expected_type = spec.get("type", "string")

            # Check required
            if required and name not in self.args:
                self.log("Warn: Validation of apps.yaml found missing configuration item '{}'".format(name))

            # Check type
            if name in self.args:
                value = self.get_arg(name, indirect=False)
                expected_types = expected_type.split("|")
                allowed = spec.get("allowed", None)
                entries = spec.get("entries", None)
                optional_entries = spec.get("optional_entries", False)
                required_entries = None
                matches = False
                if entries is not None:
                    if isinstance(entries, str):
                        required_entries = self.get_arg(entries, 0, indirect=False)
                    else:
                        required_entries = int(entries)

                    if isinstance(value, list):
                        if len(value) < required_entries:
                            if not optional_entries:
                                self.log("Warn: Validation of apps.yaml found configuration item '{}' has {} entries, expected {} based on {}".format(name, len(value), required_entries, entries))
                                self.arg_errors[name] = "Invalid number of entries, expected {}".format(required_entries)
                                errors += 1
                                continue
                    elif required_entries > 1:
                        if not optional_entries:
                            self.log("Warn: Validation of apps.yaml found configuration item '{}' is not a list, but requires {} entries based on {}".format(name, required_entries, entries))
                            self.arg_errors[name] = "Invalid type, expected list"
                            errors += 1
                            continue

                for expected_type in expected_types:
                    if expected_type == "none":
                        if self.get_arg(name, indirect=False) is None:
                            matches = True
                            break
                    elif expected_type == "integer" or expected_type == "integer_list":
                        if expected_type == "integer" and isinstance(value, int):
                            value = [value]
                        elif expected_type == "integer_list":
                            value = self.get_arg(name, [], indirect=False)

                        if isinstance(value, list):
                            matches = True
                            if required_entries is not None and len(value) > required_entries:
                                value = value[:required_entries]
                            for item in value:
                                if not self.validate_is_int(item):
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} is not an integer".format(name, item))
                                    self.arg_errors[name] = "Invalid type, expected integer item {}".format(item)
                                    errors += 1
                                    break
                                if not spec.get("zero", True) and int(item) == 0:
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' is zero".format(name))
                                    self.arg_errors[name] = "Invalid value, expected non-zero integer item {}".format(item)
                                    errors += 1
                                    break
                    elif expected_type == "float" or expected_type == "float_list":
                        if expected_type == "float" and (isinstance(value, float) or isinstance(value, int)):
                            value = [value]
                        elif expected_type == "float_list":
                            value = self.get_arg(name, [], indirect=False)

                        if isinstance(value, list):
                            matches = True
                            if required_entries is not None and len(value) > required_entries:
                                value = value[:required_entries]
                            for item in value:
                                if not self.validate_is_float(item):
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} is not a float".format(name, item))
                                    self.arg_errors[name] = "Invalid type, expected float item {}".format(item)
                                    errors += 1
                                    break
                    elif expected_type == "string" or expected_type == "string_list":
                        if expected_type == "string" and isinstance(value, str):
                            value = [value]
                        elif expected_type == "string_list":
                            value = self.get_arg(name, [], indirect=False)

                        if isinstance(value, list):
                            matches = True
                            if required_entries is not None and len(value) > required_entries:
                                value = value[:required_entries]
                            for item in value:
                                if not isinstance(item, str):
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} is not a string".format(name, item))
                                    self.arg_errors[name] = "Invalid type, expected string item {}".format(item)
                                    errors += 1
                                    break

                                if spec.get("empty", False) and not item:
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' is empty".format(name))
                                    self.arg_errors[name] = "Invalid value, expected non-empty string"
                                    errors += 1
                                    break

                                if allowed and item not in allowed:
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' value {} is not in allowed list {}".format(name, item, allowed))
                                    self.arg_errors[name] = "Invalid value {}, expected one of {}".format(item, allowed)
                                    errors += 1
                                    break
                    elif expected_type == "boolean" or expected_type == "boolean_list":
                        if expected_type == "boolean" and self.validate_is_boolean(value):
                            value = [value]
                        elif expected_type == "boolean_list":
                            value = self.get_arg(name, [], indirect=False)

                        if isinstance(value, list):
                            if required_entries is not None and len(value) > required_entries:
                                value = value[:required_entries]
                            matches = True
                            for item in value:
                                if not self.validate_is_boolean(item):
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} is not a boolean".format(name, item))
                                    self.arg_errors[name] = "Invalid type, expected boolean item {}".format(item)
                                    errors += 1
                                    break
                    elif expected_type == "integer" or expected_type == "integer_list":
                        if expected_type == "integer" and isinstance(value, int):
                            value = [value]
                        elif expected_type == "integer_list":
                            value = self.get_arg(name, [], indirect=False)

                        if isinstance(value, list):
                            if required_entries is not None and len(value) > required_entries:
                                value = value[:required_entries]

                            matches = True
                            for item in value:
                                if not self.validate_is_int(item):
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} is not an integer".format(name, item))
                                    self.arg_errors[name] = "Invalid type, expected integer item {}".format(item)
                                    errors += 1
                                    break

                                if allowed and item not in allowed:
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' value {} is not in allowed list {}".format(name, item, allowed))
                                    self.arg_errors[name] = "Invalid value {}, expected one of {}".format(item, allowed)
                                    errors += 1
                                    break

                    elif expected_type == "dict" or expected_type == "dict_list":
                        if expected_type == "dict" and isinstance(value, dict):
                            value = [value]
                        elif expected_type == "dict_list":
                            value = self.get_arg(name, [], indirect=False)
                        if isinstance(value, list):
                            if required_entries is not None and len(value) > required_entries:
                                value = value[:required_entries]

                            matches = True
                            for item in value:
                                if not isinstance(item, dict):
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} is not a dict. Must be in the format 'item: value'".format(name, item))
                                    self.arg_errors[name] = "Invalid type, element {} expected dict".format(item)
                                    errors += 1
                                    break
                    elif expected_type == "int_float_dict":
                        if spec.get("or_auto", False) and value == "auto":
                            matches = True
                        elif isinstance(value, dict):
                            matches = True
                            for key in value:
                                if not self.validate_is_int(key):
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {}, value {} is not a int => float value".format(name, key, value))
                                    self.arg_errors[name] = "Invalid element key {} expected int".format(key)
                                    errors += 1
                                    break
                                if not self.validate_is_float(value[key]):
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} value {} is not a int => float value".format(name, key, value))
                                    self.arg_errors[name] = "Invalid element key {} value {}, expected int => float".format(key, value[key])
                                    errors += 1
                                    break
                    elif expected_type == "sensor_list" or expected_type == "sensor":
                        sensor_type = spec.get("sensor_type", None)

                        if expected_type == "sensor" and isinstance(value, str):
                            value = [value]
                        elif expected_type == "sensor_list":
                            value = self.get_arg(name, [], indirect=False)

                        if isinstance(value, list):
                            # Auto trim to length
                            if required_entries is not None and len(value) > required_entries:
                                value = value[:required_entries]

                            matches = True
                            for sensor in value:
                                sensor_types = []
                                if sensor_type:
                                    sensor_types = sensor_type.split("|")

                                if "action" in sensor_types and isinstance(sensor, str) and "." in sensor:
                                    # Allow action sensors
                                    continue
                                if ("integer" in sensor_types or "float" in sensor_types) and self.validate_is_int(sensor) and not spec.get("modify", False):
                                    # Allow fixed integer values
                                    continue
                                if "float" in sensor_types and self.validate_is_float(sensor) and not spec.get("modify", False):
                                    # Allow fixed float values
                                    continue
                                if "string" in sensor_types and isinstance(sensor, str) and not spec.get("modify", False) and not "." in sensor:
                                    # Allow fixed string values
                                    continue

                                if not isinstance(sensor, str):
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} is not a valid entity_id (must be a string)".format(name, sensor))
                                    self.arg_errors[name] = "Invalid entity_id in element {}".format(sensor)
                                    errors += 1
                                    break
                                if "." not in sensor:
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} is not a valid entity_id (must contain a dot)".format(name, sensor))
                                    self.arg_errors[name] = "Invalid entity_id in element {}".format(sensor)
                                    errors += 1
                                    break

                                if spec.get("modify", False):
                                    prefix = sensor.split(".")[0]
                                    if prefix not in ["switch", "select", "input_number", "number", "time", "input_number", "input_datetime"]:
                                        if sensor.startswith("sensor.predbat_"):
                                            # We can ignore predbat generated sensors as they are control placeholders
                                            pass
                                        else:
                                            self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} which cannot be modified".format(name, sensor))
                                            self.arg_errors[name] = "Invalid entity_id in element {}, can not be modified".format(sensor)
                                            errors += 1
                                            break

                                state = self.get_state_wrapper(sensor)
                                if state is None:
                                    self.log("Expected types {} state {}".format(sensor_types, state))
                                    if "none" in sensor_types:
                                        # Allow None values for sensors
                                        continue
                                    else:
                                        self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} returned value 'None', not a valid value".format(name, sensor))
                                        self.arg_errors[name] = "Invalid value None in element {}".format(sensor)
                                        errors += 1
                                        break

                                validated = False
                                if "float" in sensor_types and self.validate_is_float(state):
                                    validated = True
                                if "integer" in sensor_types and self.validate_is_int(state):
                                    validated = True
                                if "boolean" in sensor_types and self.validate_is_boolean(state):
                                    validated = True
                                if "dict" in sensor_types and isinstance(state, dict):
                                    validated = True
                                if "list" in sensor_types and isinstance(state, list):
                                    validated = True
                                if "string" in sensor_types and isinstance(state, str):
                                    validated = True

                                if not validated:
                                    self.log("Warn: Validation of apps.yaml found configuration item '{}' element {} is not a valid type {}".format(name, sensor, sensor_type))
                                    self.arg_errors[name] = "Invalid type in element {}, expected {}".format(sensor, sensor_type)
                                    errors += 1
                                    break

                    if matches:
                        break
                if not matches:
                    self.log("Warn: Validation of apps.yaml found configuration item '{}' is not of type '{}' value was {}".format(name, expected_type, value))
                    self.arg_errors[name] = "Invalid type, expected {}".format(expected_type)
                    errors += 1
        if errors:
            self.log("Error: Validation of apps.yaml found {} configuration errors".format(errors))
        else:
            self.log("Validation of apps.yaml was successful")

        return errors

    def validate_config_schedule_retry(self, errors):
        """
        Called immediately after validate_config() with its error count. If validation failed,
        (re-)arms a retry sequence so a self-healed condition (e.g. a slow-starting integration's
        sensor not populated yet) clears its own error status without needing a manual restart -
        see #4379. A clean validation cancels any retry sequence already in progress.
        """
        if errors:
            retries = int(self.get_arg("validate_config_retries", 2))
            if retries > 0:
                self.validate_config_retries_remaining = retries
                retry_minutes = max(0, int(self.get_arg("validate_config_retry_minutes", 1)))
                self.validate_config_next_retry_time = self.now_utc + timedelta(minutes=retry_minutes)
            else:
                self.validate_config_retries_remaining = 0
                self.validate_config_next_retry_time = None
        else:
            self.validate_config_retries_remaining = 0
            self.validate_config_next_retry_time = None

    def validate_config_check_retry(self):
        """
        Called every 15 seconds from update_time_loop(). Re-runs validate_config() if a retry is
        due, only while a retry sequence is armed (i.e. only following an actual validation
        failure - see #4379) - a no-op the rest of the time.
        """
        if self.validate_config_retries_remaining <= 0:
            return
        if self.validate_config_next_retry_time is None or self.now_utc < self.validate_config_next_retry_time:
            return

        self.validate_config_retries_remaining -= 1
        errors = self.validate_config()
        if errors == 0:
            self.log("Info: Config validation retry succeeded, previous errors have now cleared")
            self.validate_config_retries_remaining = 0
            self.validate_config_next_retry_time = None
        elif self.validate_config_retries_remaining <= 0:
            self.log("Warn: Config validation still failing after all retries, giving up until the next restart or config change")
            self.validate_config_next_retry_time = None
        else:
            retry_minutes = self.get_arg("validate_config_retry_minutes", 1)
            self.validate_config_next_retry_time = self.now_utc + timedelta(minutes=retry_minutes)

    def is_running(self):
        """
        Check if the app is running
        """
        if self.stop_thread:
            return False
        if not self.dashboard_index:
            return False
        if self.fatal_error:
            return False

        if not self.ha_interface:
            return False

        if self.components:
            if not self.components.is_all_alive():
                return False

        # Read predbat.status
        status_entity = self.prefix + ".status"
        predbat_error = self.get_state_wrapper(status_entity, attribute="error", default=True)
        if predbat_error is None or predbat_error:
            return False
        predbat_last_updated = self.get_state_wrapper(status_entity, attribute="last_updated", default=None)
        if predbat_last_updated is None:
            return False

        try:
            predbat_last_updated = datetime.fromisoformat(predbat_last_updated)
        except ValueError:
            return False

        # Check if the last updated time is within the last 15 minutes
        if (datetime.now() - predbat_last_updated).total_seconds() > 15 * 60:
            return False
        return True

    def initialize(self):
        """
        Setup the app, called once each time the app starts
        """
        # Snapshot of apps.yaml exactly as the user wrote it, before Predbat's own defaulting
        # (auto_config/load_user_config) or any component's automatic_config() touches self.args -
        # lets ComponentBase.set_arg_auto() tell "user explicitly configured this" apart from
        # "Predbat defaulted it" or "another component already overwrote it" (issue #4494 follow-up).
        self.args_from_apps_yaml = copy.deepcopy(self.args)
        self.apps_yaml_override_warned = set()  # {arg} already warned about via set_arg_auto()
        self.log("Predbat: Startup {}".format(__name__))
        self.update_time(print=False)
        self.started_time = self.now_utc_real
        run_every = RUN_EVERY * 60
        now = self.now

        try:
            self.reset()
            self.update_time(print=False)

            # Start all sub-components
            self.components = Components(self)
            self.components.initialize(phase=0)

            # Initialise plugin system early so plugins can register web endpoints
            # before the web server starts
            try:
                self.log("Initialising plugin system")
                self.plugin_system = PluginSystem(self)
                self.plugin_system.discover_plugins()
            except Exception as e:
                self.log("Warning: Failed to initialise plugin system: {}".format(e))
                self.plugin_system = None

            # Now start all sub-components (including web server which will pick up registered endpoints)
            if not self.components.start(phase=0):
                self.log("Error: Some components failed to start (phase0)")
                self.record_status("Error: Some components failed to start (phase0)", had_errors=True)

            if not self.ha_interface:
                raise ValueError("HA interface not found")

            # Call plugin initialisation hooks after components are ready
            if self.plugin_system:
                self.plugin_system.call_hooks("on_init")

            # Printable config root
            self.config_root_p = self.config_root
            slug = self.ha_interface.get_slug()
            if slug:
                # and use slug name to determine printable config_root pathname when writing debug info to the log file
                self.config_root_p = "/addon_configs/" + slug

            self.log("Config root is {} and printable config_root_p is now {}".format(self.config_root, self.config_root_p))

            self.ha_interface.update_states()
            self.auto_config()
            self.load_user_config(quiet=False, register=False, load_config=True)
            self.comparison = Compare(self)

            self.components.initialize(phase=1)
            if not self.components.start(phase=1):
                self.log("Error: Some components failed to start (phase 1)")
                self.record_status("Error: Some components failed to start (phase 1)", had_errors=True)

            self.components.initialize(phase=2)
            if not self.components.start(phase=2):
                self.log("Error: Some components failed to start (phase 2)")
                self.record_status("Error: Some components failed to start (phase 2)", had_errors=True)

            self.load_user_config(quiet=False, register=True)
            self.auto_config(final=True)
            self.validate_config_schedule_retry(self.validate_config())

            # Restore the last saved plan so it is immediately active before the first calculation
            self.load_plan()

        except Exception as e:
            self.log("Error: Exception raised {}".format(e))
            self.log("Error: " + traceback.format_exc())
            self.record_status("Error: Exception raised {}".format(e), debug=traceback.format_exc(), had_errors=True)
            raise e

        # Started
        self.publish_last_started()

        # Run every N minutes aligned to the minute
        seconds_now = (now - self.midnight).seconds

        # Calculate next run time to exactly align with the run_every time
        seconds_offset = seconds_now % run_every
        seconds_next = seconds_now + (run_every - seconds_offset)
        next_time = self.midnight + timedelta(seconds=seconds_next)
        self.log("Predbat: Next run time will be {} and then every {} seconds".format(next_time, run_every))

        # First run is now
        self.update_pending = True

        # And then every N minutes
        if not INVERTER_TEST:
            self.run_every(self.run_time_loop, next_time, run_every, random_start=0, random_end=0)
            self.run_every(self.update_time_loop, datetime.now(), 15, random_start=0, random_end=0)
        else:
            self.update_time_loop(None)

        # Balance inverters
        run_every_balance = self.get_arg("balance_inverters_seconds", 60)
        if run_every_balance > 0:
            self.log("Balance inverters will run every {} seconds (if enabled)".format(run_every_balance))
            seconds_offset_balance = seconds_now % run_every_balance
            seconds_next_balance = seconds_now + (run_every_balance - seconds_offset_balance) + 15  # Offset to start after Predbat update task
            next_time_balance = self.midnight + timedelta(seconds=seconds_next_balance)
            self.run_every(self.run_time_loop_balance, next_time_balance, run_every_balance, random_start=0, random_end=0)

        # Predheat
        predheat = self.args.get("predheat", {})
        if predheat:
            self.predheat = PredHeat(self)
            self.predheat.initialize()

    async def terminate(self):
        """
        Called once each time the app terminates
        """
        self.log("Predbat terminating")
        self.stop_thread = True
        if self.components:
            await self.components.stop()
        self.log("Info: Components stopped")

        await asyncio.sleep(0)
        self.log("Predbat terminated")

    def update_time_loop(self, cb_args):
        """
        Called every 15 seconds
        """
        if not self.ha_interface or (not self.ha_interface.websocket_active and not self.ha_interface.db_primary):
            self.log("Error: HA interface not active and db_primary is {}".format(self.ha_interface.db_primary))
            self.fatal_error = True
            raise Exception("HA interface not active")

        self.check_entity_refresh()
        self.validate_config_check_retry()
        if self.update_pending and not self.prediction_started:
            # Full update required
            self.update_pending = False
            self.prediction_started = True
            try:
                self.load_user_config()
                self.validate_config_schedule_retry(self.validate_config())
                self.update_pred(scheduled=False)
                self.create_entity_list()
            except Exception as e:
                self.log("Error: Exception raised {}".format(e))
                self.log("Error: " + traceback.format_exc())
                self.record_status("Error: Exception raised {}".format(e), debug=traceback.format_exc(), had_errors=True)
                raise e
            finally:
                self.prediction_started = False
                # Always clear the active flag, even on early return or exception, so the
                # web spinner and predbat.active switch don't get stuck on
                self.expose_config("active", False)
        elif not self.prediction_started:
            time_now = datetime.now()
            inverter_data_last_fetch = self.inverter_data_last_fetch
            if inverter_data_last_fetch is None:
                # If we have never fetched inverter data, set the last fetch time to the past to trigger an update
                inverter_data_last_fetch = time_now - timedelta(hours=24)
            # Find the time since we last fetched inverter data and if it is greater than the quick update threshold, perform a quick update of the inverter data.
            tdiff = time_now - inverter_data_last_fetch
            if tdiff.total_seconds() >= INVERTER_QUICK_UPDATE_SECONDS:
                # Perform quick update of inverter data for the dashboard only
                self.prediction_started = True
                try:
                    self.quick_inverter_data_update()
                except Exception as e:
                    self.log("Error: Exception raised {}".format(e))
                    self.log("Error: " + traceback.format_exc())
                    self.record_status("Error: Exception raised {}".format(e), debug=traceback.format_exc(), had_errors=True)
                    raise e
                finally:
                    self.prediction_started = False

    def check_entity_refresh(self):
        """
        Check if we need to refresh the config entities with HA
        """
        # Check if we need to refresh the config entities with HA
        config_refresh = self.get_state_wrapper(entity_id=self.prefix + ".config_refresh")
        config_refresh_stamp = None
        if config_refresh:
            try:
                config_refresh_stamp = datetime.strptime(config_refresh, TIME_FORMAT)
            except (ValueError, TypeError):
                config_refresh_stamp = None

        age = CONFIG_REFRESH_PERIOD
        if config_refresh_stamp:
            tdiff = self.now_utc - config_refresh_stamp
            age = tdiff.seconds / 60 + tdiff.days * 60 * 24
            if age >= CONFIG_REFRESH_PERIOD:
                self.log("Info: Refresh config entities due to their age of {} minutes".format(age))
                self.update_pending = True
        else:
            self.log("Info: Refresh config entities as config_refresh state is unknown")
            self.update_pending = True

    def run_time_loop(self, cb_args):
        """
        Called every N minutes
        """
        if not self.ha_interface or (not self.ha_interface.websocket_active and not self.ha_interface.db_primary):
            self.log("Error: HA interface not active")
            self.fatal_error = True
            raise Exception("HA interface not active")

        self.check_entity_refresh()
        if not self.prediction_started:
            self.prediction_started = True
            try:
                config_changed = False
                if self.update_pending:
                    self.update_pending = False
                    self.ha_interface.update_states()
                    self.load_user_config()
                    self.validate_config_schedule_retry(self.validate_config())
                    config_changed = True

                # Run the prediction
                self.update_pred(scheduled=True)

                if config_changed:
                    self.create_entity_list()
            except Exception as e:
                self.log("Error: Exception raised {}".format(e))
                self.log("Error: " + traceback.format_exc())
                self.record_status("Error: Exception raised {}".format(e), debug=traceback.format_exc(), had_errors=True)
                raise e
            finally:
                self.prediction_started = False
                # Always clear the active flag, even on early return or exception, so the
                # web spinner and predbat.active switch don't get stuck on
                self.expose_config("active", False)

    def run_time_loop_balance(self, cb_args):
        """
        Called every N second for balance inverters
        """
        if self.get_arg("template", False):
            return

        if not self.prediction_started and self.balance_inverters_enable and not self.set_read_only:
            try:
                self.balance_inverters()
            except Exception as e:
                self.log("Error: Exception raised {}".format(e))
                self.log("Error: " + traceback.format_exc())
                self.record_status("Error: Exception raised {}".format(e), debug=traceback.format_exc(), had_errors=True)
                raise e

    def register_hook(self, hook_name, callback):
        """
        Register a hook callback (convenience method for plugins)
        """
        if self.plugin_system:
            self.plugin_system.register_hook(hook_name, callback)