Merge branch 'davidusb-geek:master' into weather

Author: paulhomes

src/emhass/command_line.py

@@ -41,6 +41,7 @@ def retrieve_home_assistant_data(
     if get_data_from_file:
         with open(emhass_conf["data_path"] / test_df_literal, "rb") as inp:
             rh.df_final, days_list, var_list, rh.ha_config = pickle.load(inp)
+            rh.var_list = var_list
         # Assign variables based on set_type
         retrieve_hass_conf["sensor_power_load_no_var_loads"] = str(var_list[0])
         if optim_conf.get("set_use_pv", True):

src/emhass/forecast.py

@@ -1279,6 +1279,7 @@ def get_load_forecast(
                     )
                     var_interp = [var_list[0]]
                     self.var_list = [var_list[0]]
+                    rh.var_list = self.var_list
                     self.var_load_new = self.var_load + "_positive"
             else:
                 days_list = get_days_list(days_min_load_forecast)

src/emhass/retrieve_hass.py

@@ -328,6 +328,12 @@ def prepare_data(
         :rtype: pandas.DataFrame
         
         """
+        self.logger.debug("prepare_data self.var_list=%s", self.var_list)
+        self.logger.debug("prepare_data var_load=%s", var_load)
+        self.logger.debug("prepare_data load_negative=%s", load_negative)
+        self.logger.debug("prepare_data set_zero_min=%s", set_zero_min)
+        self.logger.debug("prepare_data var_replace_zero=%s", var_replace_zero)
+        self.logger.debug("prepare_data var_interp=%s", var_interp)
         try:
             if load_negative:  # Apply the correct sign to load power
                 self.df_final[var_load + "_positive"] = -self.df_final[var_load]
@@ -347,16 +353,22 @@ def prepare_data(
             )
             return False
         # Confirm var_replace_zero & var_interp contain only sensors contained in var_list
-        if isinstance(var_replace_zero, list) and all(
-            item in var_replace_zero for item in self.var_list
-        ):
-            pass
+        if isinstance(var_replace_zero, list):
+            original_list = var_replace_zero[:]
+            var_replace_zero = [item for item in var_replace_zero if item in self.var_list]
+            removed = set(original_list) - set(var_replace_zero)
+            for item in removed:
+                self.logger.warning(
+                    f"Sensor '{item}' in var_replace_zero not found in self.var_list and has been removed.")
         else:
             var_replace_zero = []
-        if isinstance(var_interp, list) and all(
-            item in var_interp for item in self.var_list
-        ):
-            pass
+        if isinstance(var_interp, list):
+            original_list = var_interp[:]
+            var_interp = [item for item in var_interp if item in self.var_list]
+            removed = set(original_list) - set(var_interp)
+            for item in removed:
+                self.logger.warning(
+                    f"Sensor '{item}' in var_interp not found in self.var_list and has been removed.")
         else:
             var_interp = []
         # Apply minimum values

tests/test_forecast.py

@@ -79,6 +79,7 @@ def setUp(self):
                 self.rh.df_final, self.days_list, self.var_list, self.rh.ha_config = (
                     pickle.load(inp)
                 )
+                self.rh.var_list = self.var_list
             self.retrieve_hass_conf["sensor_power_load_no_var_loads"] = str(
                 self.var_list[0]
             )
@@ -561,6 +562,7 @@ def test_get_forecasts_with_lists(self):
         if self.get_data_from_file:
             with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp:
                 rh.df_final, days_list, var_list, rh.ha_config = pickle.load(inp)
+                rh.var_list = var_list
             retrieve_hass_conf["sensor_power_load_no_var_loads"] = str(self.var_list[0])
             retrieve_hass_conf["sensor_power_photovoltaics"] = str(self.var_list[1])
             retrieve_hass_conf["sensor_linear_interp"] = [
@@ -801,6 +803,7 @@ def test_get_forecasts_with_lists_special_case(self):
         if self.get_data_from_file:
             with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp:
                 rh.df_final, days_list, var_list, rh.ha_config = pickle.load(inp)
+                rh.var_list = var_list
             retrieve_hass_conf["sensor_power_load_no_var_loads"] = str(self.var_list[0])
             retrieve_hass_conf["sensor_power_photovoltaics"] = str(self.var_list[1])
             retrieve_hass_conf["sensor_linear_interp"] = [

tests/test_optimization.py

@@ -77,6 +77,7 @@ def setUp(self):
                 self.rh.df_final, self.days_list, self.var_list, self.rh.ha_config = (
                     pickle.load(inp)
                 )
+                self.rh.var_list = self.var_list
             self.retrieve_hass_conf["sensor_power_load_no_var_loads"] = str(
                 self.var_list[0]
             )

tests/test_retrieve_hass.py

@@ -101,6 +101,7 @@ def setUp(self):
                 self.rh.df_final, self.days_list, self.var_list, self.rh.ha_config = (
                     pickle.load(inp)
                 )
+                self.rh.var_list = self.var_list
         # Else obtain sensor values from HA
         else:
             if model_type == "long_train_data":
@@ -287,6 +288,39 @@ def test_prepare_data_negative_load(self):
             self.rh.df_final.index.tz, self.retrieve_hass_conf["time_zone"]
         )
 
+    # Tests that the prepare_data method does convert missing PV values to zero
+    # and also ignores any missing sensor columns.
+    def test_prepare_data_missing_pv(self):
+        load_sensor = self.retrieve_hass_conf["sensor_power_load_no_var_loads"]
+        actual_pv_sensor = self.retrieve_hass_conf["sensor_power_photovoltaics"]
+        forecast_pv_sensor = self.retrieve_hass_conf["sensor_power_photovoltaics_forecast"]
+        var_replace_zero = [actual_pv_sensor, forecast_pv_sensor, 'sensor.missing1']
+        var_interp = [actual_pv_sensor, load_sensor, 'sensor.missing2']
+        # Replace actual and forecast PV zero values with NaN's (to test they get replaced back)
+        self.rh.df_final[actual_pv_sensor] = self.rh.df_final[actual_pv_sensor].replace(0, np.nan)
+        self.rh.df_final[forecast_pv_sensor] = self.rh.df_final[forecast_pv_sensor].replace(0, np.nan)
+        # Verify a non-zero number of missing values in the actual and forecast PV columns before prepare_data
+        self.assertTrue(self.rh.df_final[actual_pv_sensor].isna().sum() > 0)
+        self.assertTrue(self.rh.df_final[forecast_pv_sensor].isna().sum() > 0)
+        self.rh.prepare_data(
+            load_sensor,
+            load_negative=False,
+            set_zero_min=True,
+            var_replace_zero=var_replace_zero,
+            var_interp=var_interp
+        )
+        self.assertIsInstance(self.rh.df_final, type(pd.DataFrame()))
+        self.assertEqual(
+            self.rh.df_final.index.isin(self.days_list).sum(),
+            self.df_raw.index.isin(self.days_list).sum(),
+        )
+        # Check the before and after actual and forecast PV columns have the same number of values
+        self.assertEqual(len(self.df_raw[actual_pv_sensor]), len(self.rh.df_final[actual_pv_sensor]))
+        self.assertEqual(len(self.df_raw[forecast_pv_sensor]), len(self.rh.df_final[forecast_pv_sensor]))
+        # Verify no missing values in the actual and forecast PV columns after prepare_data
+        self.assertTrue(self.rh.df_final[actual_pv_sensor].isna().sum() == 0)
+        self.assertTrue(self.rh.df_final[forecast_pv_sensor].isna().sum() == 0)
+
     # Test publish data
     def test_publish_data(self):
         response, data = self.rh.post_data(