Merge pull request #1956 from Andry925/feat/sqlite-storage-config-validation-and-init-guardrails

feat:sqlite storage config validation and init guardrails

Author: samson0v

tests/unit/service/test_storage.py

@@ -144,7 +144,7 @@ def test_sqlite_storage(self):
 class TestSQLiteEventStorageRotation(TestCase):
 
     def setUp(self):
-        self.directory = path.join("storage", "data")
+        self.directory = "./data/"
         self.db_path = path.join(self.directory, "data.db")
         if not path.exists(self.directory):
             try:
@@ -153,7 +153,7 @@ def setUp(self):
             except Exception as e:
                 LOG.warning(f"Failed to create directory {self.directory}: {e}")
         self.config = {
-            "data_file_path": self.db_path,
+            "data_file_path": self.directory,
             "messages_ttl_check_in_hours": 1,
             "messages_ttl_in_days": 7,
             "max_read_records_count": 1000,

thingsboard_gateway/storage/sqlite/database.py

@@ -105,7 +105,7 @@ def init_table(self):
             self.__log.exception("Failed to create table or migrate data! Error: %s", e)
 
     def run(self):
-        self.__log.info("Database thread started %r", id(self))
+        self.__log.debug("Database thread started %r", id(self))
         interval = self.settings.oversize_check_period * 60
         sleep_time = 0.2
 
@@ -131,7 +131,7 @@ def run(self):
 
             except Exception as e:
                 self.__log.exception("Error in database thread: %s", exc_info=e)
-        self.__log.info("Database thread stopped %r", id(self))
+        self.__log.debug("Database thread stopped %r", id(self))
         self.db.close()
 
     def process_file_limit(self):

thingsboard_gateway/storage/sqlite/sqlite_event_storage.py

@@ -36,10 +36,12 @@ def __init__(self, config, logger, main_stop_event):
         self.__log.info("Sqlite Storage initializing...")
         self.write_queue = Queue(-1)
         self.stopped = Event()
-        if isinstance(config, StorageSettings):
-            self.__settings = config
-        else:
-            self.__settings = StorageSettings(config)
+
+        self.__settings = config if isinstance(config, StorageSettings) else StorageSettings(config)
+        if self.__settings.warnings:
+            for warning in self.__settings.warnings:
+                self.__log.warning("%s", str(warning))
+
         self.__ensure_data_folder_exists()
         self.__pointer = Pointer(self.__settings.data_file_path, log=self.__log)
         self.__default_database_name = self.__settings.db_file_name

thingsboard_gateway/storage/sqlite/storage_settings.py

@@ -26,17 +26,25 @@ def __init__(self, config, enable_validation=True):
         self.max_read_records_count = config.get("max_read_records_count", 1000)
         self.batch_size = config.get("writing_batch_size", 1000)
         self.directory_path = path.dirname(self.data_file_path)
-        self.db_file_name = path.basename(self.data_file_path)
+        self.db_file_name = "data.db"
         self.size_limit = config.get("size_limit", 1024)
         self.max_db_amount = config.get("max_db_amount", 10)
         self.oversize_check_period = config.get("oversize_check_period", 1)
-        self.validate_settings()
+        self.warnings = self.validate_settings()
 
     def validate_settings(self):
-        if not self.db_file_name:
-            self.db_file_name = "data.db"
+        warnings = []
+        if path.basename(self.data_file_path):
+            warnings.append(
+                "You can not specify the file name, using default file name: data.db or make sure you set slash at the end of the path;")
+
         if self.size_limit < 1 and self.enable_validation:
             self.size_limit = 1
+            warnings.append("The size limit is too small - using the minimum value 1 MB;")
+
         if self.oversize_check_period < 1 and self.enable_validation:
             self.oversize_check_period = 1
+            warnings.append("The oversize check period is too small - using the minimum value 1 minute;")
+
+        return warnings