Merge pull request #693 from davidusb-geek/davidusb-geek/fix/solve_secrets_bigm_hang

Solve secrets handling and MILP big-M solve hang issues

Author: davidusb-geek

docs/config.md

@@ -28,22 +28,27 @@ We will need to define these parameters to retrieve data from Home Assistant. Th
 - `use_influxdb`: Enable InfluxDB (version 1.x) as a data source instead of the Home Assistant API. This allows for longer historical data retention and better performance for machine learning models. InfluxDB v2 is not currently supported.
 - `influxdb_host`: The IP address or hostname of your InfluxDB instance. Defaults to `localhost`.
 - `influxdb_port`: The port number for your InfluxDB instance. Defaults to 8086.
-- `influxdb_username`: Username for authenticating with InfluxDB. Leave empty if no authentication is required.
-- `influxdb_password`: Password for authenticating with InfluxDB. Leave empty if no authentication is required.
 - `influxdb_database`: The name of the InfluxDB database containing your Home Assistant data. Defaults to `homeassistant`.
 - `influxdb_measurement`: The measurement name where your sensor data is stored. Defaults to `W` for the Home Assistant integration.
 - `influxdb_retention_policy`: The retention policy to use for InfluxDB queries. Defaults to `autogen`.
 
-A second part of this section is given by some privacy-sensitive parameters that should be included in a `secrets_emhass.yaml` file alongside the `config_emhass.yaml` file.
+A second part of this section is given by some privacy-sensitive parameters that should be included as:
+- The list of secret parameters filled in the form on the Add-on **Configuration** pane if using the Add-on installation method.
+- A `secrets_emhass.yaml` file alongside the `config_emhass.yaml` file if using the Docker standalone or legacy installation method.
 
-The parameters in the `secrets_emhass.yaml` file are:
+The **secrets** parameters are:
 
 - `hass_url`: The URL to your Home Assistant instance. For example: https://myhass.duckdns.org/
 - `long_lived_token`: A Long-Lived Access Token from the Lovelace profile page.
 - `time_zone`: The time zone of your system. For example: Europe/Paris.
 - `lat`: The latitude. For example: 45.0.
 - `lon`: The longitude. For example: 6.0
 - `alt`: The altitude in meters. For example: 100.0
+- `solcast_api_key`: The Solcast API key (weather_forecast_method=solcast)
+- `solcast_rooftop_id`: The Solcast rooftop ID (weather_forecast_method=solcast)
+- `solar_forecast_kwp`: The PV peak installed power in kW used for the 'solar.forecast' API call (weather_forecast_method=solar.forecast)
+- `influxdb_username`: Username for authenticating with InfluxDB. Leave empty if no authentication is required.
+- `influxdb_password`: Password for authenticating with InfluxDB. Leave empty if no authentication is required.
 
 ## Optimization configuration parameters
 

docs/passing_data.md

@@ -138,13 +138,14 @@ Then on the EMHASS configuration you need to set:
   "influxdb_port": 8086,
   "influxdb_measurement": "state",
   "influxdb_retention_policy": "autogen",
-  "influxdb_password": "influxdb_password",
-  "influxdb_username": "influxdb_user",
   "influxdb_use_ssl": false,
   "influxdb_verify_ssl": false,
 }
 ```
 
+Finally, if using the Add-on, you need to fill both "influxdb_password" and "influxdb_username" in the Add-on **Configuration** pane.
+If using the Docker standalone or legacy installation method, then you need to set these in the `secrets_emhass.yaml` file.
+
 
 ## Passing in secret parameters
 Secret parameters are passed differently, depending on which method you choose. Alternative options are also present for passing secrets, if you are running EMHASS separately from Home Assistant. _(I.e. not via EMHASS-Add-on)_ 

src/emhass/data/associations.csv

@@ -11,8 +11,6 @@ retrieve_hass_conf,var_interp,sensor_linear_interp,list_sensor_linear_interp
 retrieve_hass_conf,use_influxdb,use_influxdb
 retrieve_hass_conf,influxdb_host,influxdb_host
 retrieve_hass_conf,influxdb_port,influxdb_port
-retrieve_hass_conf,influxdb_username,influxdb_username
-retrieve_hass_conf,influxdb_password,influxdb_password
 retrieve_hass_conf,influxdb_database,influxdb_database
 retrieve_hass_conf,influxdb_measurement,influxdb_measurement
 retrieve_hass_conf,influxdb_retention_policy,influxdb_retention_policy
@@ -22,10 +20,17 @@ retrieve_hass_conf,method_ts_round,method_ts_round
 retrieve_hass_conf,continual_publish,continual_publish
 retrieve_hass_conf,use_websocket,use_websocket
 retrieve_hass_conf,ssl_no_verify,ssl_no_verify
+params_secrets,hass_url,hass_url
+params_secrets,long_lived_token,long_lived_token
 params_secrets,time_zone,time_zone
 params_secrets,lat,Latitude
 params_secrets,lon,Longitude
 params_secrets,alt,Altitude
+params_secrets,solcast_api_key,solcast_api_key
+params_secrets,solcast_rooftop_id,solcast_rooftop_id
+params_secrets,solar_forecast_kwp,solar_forecast_kwp
+params_secrets,influxdb_username,influxdb_username
+params_secrets,influxdb_password,influxdb_password
 optim_conf,costfun,costfun
 optim_conf,logging_level,logging_level
 optim_conf,set_use_pv,set_use_pv

src/emhass/data/config_defaults.json

@@ -29,8 +29,6 @@
   "use_influxdb": false,
   "influxdb_host": "localhost",
   "influxdb_port": 8086,
-  "influxdb_username": "",
-  "influxdb_password": "",
   "influxdb_database": "homeassistant",
   "influxdb_measurement": "W",
   "influxdb_retention_policy": "autogen",

src/emhass/forecast.py

@@ -1191,6 +1191,8 @@ def get_forecast_out_from_csv_or_list(
             forecast_out = pd.concat(forecast_parts, axis=0)
         else:
             forecast_out = pd.DataFrame()
+        if not forecast_out.empty and forecast_out.index.dtype != df_final.index.dtype:
+            forecast_out.index = forecast_out.index.astype(df_final.index.dtype)
         # Merge with final DataFrame to align indices
         merged = pd.merge_asof(
             df_final.sort_index(),
@@ -1581,6 +1583,8 @@ def get_load_cost_forecast(
         :rtype: pd.DataFrame
 
         """
+        if df_final.index.dtype != "datetime64[ns, " + str(self.time_zone) + "]":
+            df_final.index = df_final.index.astype("datetime64[ns, " + str(self.time_zone) + "]")
         csv_path = self.emhass_conf["data_path"] / csv_path
         if method == "hp_hc_periods":
             df_final[self.var_load_cost] = self.optim_conf["load_offpeak_hours_cost"]
@@ -1667,6 +1671,8 @@ def get_prod_price_forecast(
         :rtype: pd.DataFrame
 
         """
+        if df_final.index.dtype != "datetime64[ns, " + str(self.time_zone) + "]":
+            df_final.index = df_final.index.astype("datetime64[ns, " + str(self.time_zone) + "]")
         csv_path = self.emhass_conf["data_path"] / csv_path
         if method == "constant":
             df_final[self.var_prod_price] = self.optim_conf["photovoltaic_production_sell_price"]

src/emhass/machine_learning_forecaster.py

@@ -622,7 +622,7 @@ async def tune(
                 exog=self.data_test.drop(self.var_model, axis=1),
             )
 
-            freq_hours = self.data_exo.index.freq.delta.seconds / 3600
+            freq_hours = pd.to_timedelta(self.data_exo.index.freq).total_seconds() / 3600
             self.lags_opt = int(np.round(len(self.optimize_results.iloc[0]["lags"])))
             self.days_needed = int(np.round(self.lags_opt * freq_hours / 24))