Refactor sensitive data encryption and fixed updating Telegram service …

…#882

terrariumAPI.py

@@ -1235,6 +1235,7 @@ def notification_service_detail(self, service):
     def notification_service_update(self, service):
         try:
             service = NotificationService[service]
+            request.json['state'] = service.state or {}
             service.set(**request.json)
             orm.commit()
 
@@ -1687,7 +1688,7 @@ def setting_list(self):
         return {
             "data": [
                 self.setting_detail(setting.id)
-                for setting in Setting.select(lambda s: not s.id in ["password", "encryption_salt"])
+                for setting in Setting.select(lambda s: not s.id in ["password"])
             ]
         }
 
@@ -1698,9 +1699,7 @@ def setting_detail(self, setting):
 
         try:
             data = Setting[setting].to_dict()
-            if data["id"] in ["meross_cloud_username", "meross_cloud_password"]:
-                data["value"] = terrariumUtils.decrypt(data["value"])
-            elif "exclude_ids" == data["id"]:
+            if "exclude_ids" == data["id"]:
                 ids = data["value"].split(",")
                 data["value"] = []
 

terrariumCloud.py

@@ -11,7 +11,7 @@
 
 from time import sleep, time
 
-from terrariumUtils import classproperty, terrariumCache, terrariumSingleton, terrariumAsync, terrariumUtils
+from terrariumUtils import classproperty, terrariumCache, terrariumSingleton, terrariumAsync
 import os
 
 # pip install meross-iot
@@ -27,8 +27,8 @@
 class TerrariumMerossCloud(terrariumSingleton):
     @classproperty
     def is_enabled(__cls__):
-        EMAIL = terrariumUtils.decrypt(os.environ.get("MEROSS_EMAIL", ""))
-        PASSWORD = terrariumUtils.decrypt(os.environ.get("MEROSS_PASSWORD", ""))
+        EMAIL = os.environ.get("MEROSS_EMAIL", "")
+        PASSWORD = os.environ.get("MEROSS_PASSWORD", "")
 
         return EMAIL != "" and PASSWORD != ""
 

terrariumDatabase.py

@@ -84,8 +84,13 @@ def create_defaults(version):
             Setting(**setting)
             orm.commit()
         except orm.core.TransactionIntegrityError:
-            # Setting is already in the database. Ignore
-            pass
+            # Setting is already in the database. Ignore except version. Update that.
+            if "version" == setting["id"]:
+                Setting[setting["id"]].value = setting["value"]
+
+    # Clear old obsolete settings
+    settings = [setting['id'] for setting in setting_defaults]
+    Setting.select(lambda p: p.id not in settings).delete(bulk=True)
 
 
 def load_advanced_settings():
@@ -282,24 +287,29 @@ class NotificationService(db.Entity):
 
     ENCRYPTED_FIELDS = ["username", "password", "user_key", "api_token", "access_secret", "token", "allowed_users"]
 
-    def __encrypt_sensitive_fields(self):
+    def _encrypt_sensitive_fields(self):
         # Encrypt sensitive fields
         for field in self.ENCRYPTED_FIELDS:
-            if field in self.setup:
+            # Only update when the data was decrypted.
+            # If decrypted value is the same as original value, the value was NOT encrypted and needs to be encrypted
+            if field in self.setup and "" != self.setup[field] and self.setup[field] == terrariumUtils.decrypt(self.setup[field]):
                 self.setup[field] = terrariumUtils.encrypt(self.setup[field])
 
+    def _decrypt_sensitive_fields(self, data):
+        # Decrypt sensitive fields
+        for field in self.ENCRYPTED_FIELDS:
+            if field in data["setup"]:
+                data["setup"][field] = terrariumUtils.decrypt(data["setup"][field])
+
     def before_insert(self):
-        self.__encrypt_sensitive_fields()
+        self._encrypt_sensitive_fields()
 
     def before_update(self):
-        self.__encrypt_sensitive_fields()
+        self._encrypt_sensitive_fields()
 
     def to_dict(self, only=None, exclude=None, with_collections=False, with_lazy=False, related_objects=False):
         data = copy.deepcopy(super().to_dict(only, exclude, with_collections, with_lazy, related_objects))
-        # Encrypt sensitive fields
-        for field in self.ENCRYPTED_FIELDS:
-            if field in data["setup"]:
-                data["setup"][field] = terrariumUtils.decrypt(data["setup"][field])
+        self._decrypt_sensitive_fields(data)
 
         return data
 
@@ -578,20 +588,29 @@ class Setting(db.Entity):
     id = orm.PrimaryKey(str)
     value = orm.Optional(str)
 
-    ENCRYPTED_FIELDS = ["meross_cloud_username", "meross_cloud_password"]
+    ENCRYPTED_FIELDS = ["weather_source", "meross_cloud_username", "meross_cloud_password"]
 
-    def __encrypt_sensitive_fields(self):
-        if self.id in self.ENCRYPTED_FIELDS and "" != self.value:
+    def _encrypt_sensitive_fields(self):
+        # Encrypt sensitive fields
+        if "" != self.value and self.id in self.ENCRYPTED_FIELDS and self.value == terrariumUtils.decrypt(self.value):
             self.value = terrariumUtils.encrypt(self.value)
 
+    def _decrypt_sensitive_fields(self, data):
+        # Decrypt sensitive fields
+        if "" != data['value'] and data['id'] in self.ENCRYPTED_FIELDS:
+            data['value'] = terrariumUtils.decrypt(data['value'])
+
     def before_insert(self):
-        self.__encrypt_sensitive_fields()
+        self._encrypt_sensitive_fields()
 
     def before_update(self):
-        self.__encrypt_sensitive_fields()
+        self._encrypt_sensitive_fields()
 
     def to_dict(self, only=None, exclude=None, with_collections=False, with_lazy=False, related_objects=False):
-        return copy.deepcopy(super().to_dict(only, exclude, with_collections, with_lazy, related_objects))
+        data = copy.deepcopy(super().to_dict(only, exclude, with_collections, with_lazy, related_objects))
+        self._decrypt_sensitive_fields(data)
+
+        return data
 
 
 class Webcam(db.Entity):

terrariumEngine.py

@@ -87,6 +87,11 @@ def __init__(self, version):
             "asyncio": terrariumAsync(),
         }
 
+        # Create a salt for encryption. And set it as environment variable
+        salt = terrariumUtils.get_script_data("grep -i serial /proc/cpuinfo").decode().strip()
+        salt = salt.split(":")[1].strip()
+        os.environ["SALT"] = salt
+
         self.meross_cloud = None
 
         self.version = version
@@ -241,27 +246,23 @@ def load_settings(self):
         settings = {}
         with orm.db_session():
             for setting in Setting.select():
-                if terrariumUtils.is_float(setting.value):
-                    settings[setting.id] = float(setting.value)
-                elif setting.value.lower() in ["true", "false", "on", "off", "1", "0"]:
-                    settings[setting.id] = terrariumUtils.is_true(setting.value)
-                elif setting.id.lower() in ["power_price", "water_price"]:
-                    settings[setting.id] = float(setting.value) if terrariumUtils.is_float(setting.value) else 0.0
+                setting = setting.to_dict()
+
+                if terrariumUtils.is_float(setting['value']):
+                    settings[setting['id']] = float(setting['value'])
+                elif setting['value'].lower() in ["true", "false", "on", "off", "1", "0"]:
+                    settings[setting['id']] = terrariumUtils.is_true(setting['value'])
+                elif setting['id'].lower() in ["power_price", "water_price"]:
+                    settings[setting['id']] = float(setting['value']) if terrariumUtils.is_float(setting['value']) else 0.0
                 else:
-                    settings[setting.id] = setting.value
+                    settings[setting['id']] = setting['value']
 
-                logger.debug(f"Loaded setting: {setting.id} with value: {setting.value}")
+                logger.debug(f"Loaded setting: {setting['id']} with value: {setting['value']}")
 
         settings["exclude_ids"] = [] if "" == settings["exclude_ids"] else settings["exclude_ids"].split(",")
         # Force port number to an int.....
         settings["port"] = int(settings["port"])
 
-        # Create a salt for encryption. And set it as environment variable
-        cmd = "grep -i serial /proc/cpuinfo"
-        salt = terrariumUtils.get_script_data(cmd).decode().strip()
-        salt = salt.split(":")[1].strip()
-        os.environ["SALT"] = salt
-
         # Set meross login into the current bash environment
         # But first check if the credentials have changed.
         meross_login = (
@@ -372,8 +373,8 @@ def load_settings(self):
                 self.meross_cloud.stop()
 
             self.meross_cloud = TerrariumMerossCloud(
-                terrariumUtils.decrypt(settings["meross_cloud_username"]),
-                terrariumUtils.decrypt(settings["meross_cloud_password"]),
+                settings["meross_cloud_username"],
+                settings["meross_cloud_password"],
             )
 
     # -=NEW=-

terrariumNotification.py

@@ -576,8 +576,8 @@ def send_message(self, msg_type, subject, message, data=None, attachments=[]):
                     smtp_settings[smtp_security] = True
 
                 if "" != self.setup["username"]:
-                    smtp_settings["user"] = terrariumUtils.decrypt(self.setup["username"])
-                    smtp_settings["password"] = terrariumUtils.decrypt(self.setup["password"])
+                    smtp_settings["user"] = self.setup["username"]
+                    smtp_settings["password"] = self.setup["password"]
 
                 response = email_message.send(to=(receiver, receiver), smtp=smtp_settings)
 
@@ -2317,9 +2317,7 @@ def load_setup(self, setup_data):
                 self.connection.on_connect = self.on_connect
                 if self.setup["ssl"]:
                     self.connection.tls_set(tls_version=mqtt.ssl.PROTOCOL_TLS)
-                self.connection.username_pw_set(
-                    terrariumUtils.decrypt(self.setup["username"]), terrariumUtils.decrypt(self.setup["password"])
-                )
+                self.connection.username_pw_set(self.setup["username"], self.setup["password"])
                 self.connection.connect(self.setup["address"], self.setup["port"], 30)
                 self.connection.loop_start()
                 logger.info(f'Connecting to MQTT Broker at address: {self.setup["address"]}:{self.setup["port"]} ...')
@@ -2695,7 +2693,7 @@ def _run():
                 logger.exception(f"Error in telegram service: {ex}")
 
         old_chat_ids = []
-        if setup_data['state'] and setup_data['state']['chat_ids']:
+        if setup_data['state'] and 'chat_ids' in setup_data['state']:
             old_chat_ids = setup_data['state']['chat_ids']
 
         self.setup = {