Merge pull request #1381 from okorach:trim-background-task-data

Trim-background-task-data

Author: okorach

.flake8

@@ -1,6 +1,6 @@
 [flake8]
 # E501 = Line too long
-ignore = E501,E401,W503,E128,C901,W504,E302,E265,E741,W291,W292,W293,W391,F401,E226,F841,F821,F541,D400,D401,E800,D102,D105,D107,I900,F632,E261
+ignore = E501,E401,W503,E128,C901,W504,E302,E265,E741,W291,W292,W293,W391,F401,E226,F841,F821,F541,D400,D401,E800,D102,D105,D107,I900,F632,E261,E731
 max-line-length = 150
 #exclude = tests/*
 #max-complexity = 10

migration/README.md

@@ -116,6 +116,7 @@ When sonar-migration complete successfully they return exit code 0. En case of f
 - Support of incremental dump of projects extracts
 - Display of HTTP requests duration in DEBUG logs
 - Fixes in documentation
+- Trimmed background task data to keep only what is need (to reduce memory and output JSON size)
 
 ## Version 0.2
 

migration/what-is-new.md

@@ -6,6 +6,7 @@
 - Support of incremental dump of projects extracts
 - Display of HTTP requests duration in DEBUG logs
 - Fixes in documentation
+- Trimmed background task data to keep only what is need (to reduce memory and output JSON size)
 
 # Version 0.2
 

pylintrc

@@ -86,7 +86,8 @@ disable=raw-checker-failed,
         W0611,
         W0238,
         R0911,
-        C0325
+        C0325,
+        W1309
 
 
 # Enable the message, report, category or checker with the given id(s). You can

sonar/projects.py

@@ -80,6 +80,46 @@
     "webhooks",
 )
 
+_UNNEEDED_CONTEXT_DATA = (
+    "sonar.announcement.message",
+    "sonar.auth.github.allowUsersToSignUp",
+    "sonar.auth.github.apiUrl",
+    "sonar.auth.github.appId",
+    "sonar.auth.github.enabled",
+    "sonar.auth.github.groupsSync",
+    "sonar.auth.github.organizations",
+    "sonar.auth.github.webUrl",
+    "sonar.builtInQualityProfiles.disableNotificationOnUpdate",
+    "sonar.core.id",
+    "sonar.core.serverBaseURL",
+    "sonar.core.startTime",
+    "sonar.dbcleaner.branchesToKeepWhenInactive",
+    "sonar.forceAuthentication",
+    "sonar.host.url",
+    "sonar.java.jdkHome",
+    "sonar.links.ci",
+    "sonar.links.homepage",
+    "sonar.links.issue",
+    "sonar.links.scm",
+    "sonar.links.scm_dev",
+    "sonar.plugins.risk.consent",
+)
+
+_UNNEEDED_TASK_DATA = (
+    "analysisId",
+    "componentId",
+    "hasScannerContext",
+    "id",
+    "warningCount",
+    "componentQualifier",
+    "nodeName",
+    "componentName",
+    "componentKey",
+    "submittedAt",
+    "executedAt",
+    "type",
+)
+
 
 class Project(components.Component):
     """
@@ -954,9 +994,9 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str,
         :rtype: dict
         """
         log.info("Exporting %s", str(self))
+        json_data = self._json.copy()
+        json_data.update({"key": self.key, "name": self.name})
         try:
-            json_data = self._json.copy()
-            json_data.update({"key": self.key, "name": self.name})
             json_data["binding"] = self.__export_get_binding()
             nc = self.new_code()
             if nc != "":
@@ -983,10 +1023,16 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str,
                 last_task = self.last_task()
                 json_data["backgroundTasks"] = {}
                 if last_task:
+                    ctxt = last_task.scanner_context()
+                    if ctxt:
+                        ctxt = {k: v for k, v in ctxt.items() if k not in _UNNEEDED_CONTEXT_DATA}
+                    t_hist = []
+                    for t in self.task_history():
+                        t_hist.append({k: v for k, v in t._json.items() if k not in _UNNEEDED_TASK_DATA})
                     json_data["backgroundTasks"] = {
-                        "lastTaskScannerContext": last_task.scanner_context(),
-                        "lastTaskWarnings": last_task.warnings(),
-                        "taskHistory": [t._json for t in self.task_history()],
+                        "lastTaskScannerContext": ctxt,
+                        # "lastTaskWarnings": last_task.warnings(),
+                        "taskHistory": t_hist,
                     }
 
             settings_dict = settings.get_bulk(endpoint=self.endpoint, component=self, settings_list=settings_list, include_not_set=False)
@@ -997,14 +1043,17 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str,
                 json_data.update(s.to_json())
         except HTTPError as e:
             if e.response.status_code == HTTPStatus.FORBIDDEN:
-                log.critical("Insufficient privileges to access %s, export of this project skipped", str(self))
-                json_data = {"error": "Insufficient permissions while extracting project"}
+                log.critical("Insufficient privileges to access %s, export of this project interrupted", str(self))
+                json_data["error"] = "Insufficient permissions while exporting project, export interrupted"
             else:
-                log.critical("HTTP error %s while exporting %s, export of this project skipped", str(e), str(self))
-                json_data = {"error": f"HTTP error {str(e)} while extracting project"}
+                log.critical("HTTP error %s while exporting %s, export of this project interrupted", str(e), str(self))
+                json_data["error"] = f"HTTP error {str(e)} while extracting project"
         except ConnectionError as e:
-            log.critical("Connecting error %s while extracting %s, extract of this project skipped", str(self), str(e))
-            json_data = {"error": f"Connection error {str(e)} while extracting prooject"}
+            log.critical("Connecting error %s while exporting %s, export of this project interrupted", str(self), str(e))
+            json_data["error"] = f"Connection error {str(e)} while extracting project, export interrupted"
+        except Exception as e:
+            log.critical("Connecting error %s while exporting %s, export of this project interrupted", str(self), str(e))
+            json_data["error"] = f"Exception {str(e)} while exporting project, export interrupted"
         log.info("Exporting %s done", str(self))
         return util.remove_nones(json_data)