diff --git a/.flake8 b/.flake8 index 2f0ff0d6d..d56588b0b 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ [flake8] # E501 = Line too long -ignore = E501,E401,W503,E128,C901,W504,E302,E265,E741,W291,W292,W293,W391,F401,E226,F841,F821,F541,D400,D401,E800,D102,D105,D107,I900,F632,E261 +ignore = E501,E401,W503,E128,C901,W504,E302,E265,E741,W291,W292,W293,W391,F401,E226,F841,F821,F541,D400,D401,E800,D102,D105,D107,I900,F632,E261,E731 max-line-length = 150 #exclude = tests/* #max-complexity = 10 diff --git a/migration/README.md b/migration/README.md index 85d8deae0..6906427d9 100644 --- a/migration/README.md +++ b/migration/README.md @@ -116,6 +116,7 @@ When sonar-migration complete successfully they return exit code 0. En case of f - Support of incremental dump of projects extracts - Display of HTTP requests duration in DEBUG logs - Fixes in documentation +- Trimmed background task data to keep only what is need (to reduce memory and output JSON size) ## Version 0.2 diff --git a/migration/what-is-new.md b/migration/what-is-new.md index dc5b47192..2fff1b12b 100644 --- a/migration/what-is-new.md +++ b/migration/what-is-new.md @@ -6,6 +6,7 @@ - Support of incremental dump of projects extracts - Display of HTTP requests duration in DEBUG logs - Fixes in documentation +- Trimmed background task data to keep only what is need (to reduce memory and output JSON size) # Version 0.2 diff --git a/pylintrc b/pylintrc index 3a99abbd2..856da153e 100644 --- a/pylintrc +++ b/pylintrc @@ -86,7 +86,8 @@ disable=raw-checker-failed, W0611, W0238, R0911, - C0325 + C0325, + W1309 # Enable the message, report, category or checker with the given id(s). You can diff --git a/sonar/projects.py b/sonar/projects.py index f11558d44..5f9ed8fc2 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -80,6 +80,46 @@ "webhooks", ) +_UNNEEDED_CONTEXT_DATA = ( + "sonar.announcement.message", + "sonar.auth.github.allowUsersToSignUp", + "sonar.auth.github.apiUrl", + "sonar.auth.github.appId", + "sonar.auth.github.enabled", + "sonar.auth.github.groupsSync", + "sonar.auth.github.organizations", + "sonar.auth.github.webUrl", + "sonar.builtInQualityProfiles.disableNotificationOnUpdate", + "sonar.core.id", + "sonar.core.serverBaseURL", + "sonar.core.startTime", + "sonar.dbcleaner.branchesToKeepWhenInactive", + "sonar.forceAuthentication", + "sonar.host.url", + "sonar.java.jdkHome", + "sonar.links.ci", + "sonar.links.homepage", + "sonar.links.issue", + "sonar.links.scm", + "sonar.links.scm_dev", + "sonar.plugins.risk.consent", +) + +_UNNEEDED_TASK_DATA = ( + "analysisId", + "componentId", + "hasScannerContext", + "id", + "warningCount", + "componentQualifier", + "nodeName", + "componentName", + "componentKey", + "submittedAt", + "executedAt", + "type", +) + class Project(components.Component): """ @@ -954,9 +994,9 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, :rtype: dict """ log.info("Exporting %s", str(self)) + json_data = self._json.copy() + json_data.update({"key": self.key, "name": self.name}) try: - json_data = self._json.copy() - json_data.update({"key": self.key, "name": self.name}) json_data["binding"] = self.__export_get_binding() nc = self.new_code() if nc != "": @@ -983,10 +1023,16 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, last_task = self.last_task() json_data["backgroundTasks"] = {} if last_task: + ctxt = last_task.scanner_context() + if ctxt: + ctxt = {k: v for k, v in ctxt.items() if k not in _UNNEEDED_CONTEXT_DATA} + t_hist = [] + for t in self.task_history(): + t_hist.append({k: v for k, v in t._json.items() if k not in _UNNEEDED_TASK_DATA}) json_data["backgroundTasks"] = { - "lastTaskScannerContext": last_task.scanner_context(), - "lastTaskWarnings": last_task.warnings(), - "taskHistory": [t._json for t in self.task_history()], + "lastTaskScannerContext": ctxt, + # "lastTaskWarnings": last_task.warnings(), + "taskHistory": t_hist, } settings_dict = settings.get_bulk(endpoint=self.endpoint, component=self, settings_list=settings_list, include_not_set=False) @@ -997,14 +1043,17 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, json_data.update(s.to_json()) except HTTPError as e: if e.response.status_code == HTTPStatus.FORBIDDEN: - log.critical("Insufficient privileges to access %s, export of this project skipped", str(self)) - json_data = {"error": "Insufficient permissions while extracting project"} + log.critical("Insufficient privileges to access %s, export of this project interrupted", str(self)) + json_data["error"] = "Insufficient permissions while exporting project, export interrupted" else: - log.critical("HTTP error %s while exporting %s, export of this project skipped", str(e), str(self)) - json_data = {"error": f"HTTP error {str(e)} while extracting project"} + log.critical("HTTP error %s while exporting %s, export of this project interrupted", str(e), str(self)) + json_data["error"] = f"HTTP error {str(e)} while extracting project" except ConnectionError as e: - log.critical("Connecting error %s while extracting %s, extract of this project skipped", str(self), str(e)) - json_data = {"error": f"Connection error {str(e)} while extracting prooject"} + log.critical("Connecting error %s while exporting %s, export of this project interrupted", str(self), str(e)) + json_data["error"] = f"Connection error {str(e)} while extracting project, export interrupted" + except Exception as e: + log.critical("Connecting error %s while exporting %s, export of this project interrupted", str(self), str(e)) + json_data["error"] = f"Exception {str(e)} while exporting project, export interrupted" log.info("Exporting %s done", str(self)) return util.remove_nones(json_data)