From f160d7b5178133873044ba826e5e7332e3813c12 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 10:32:22 +0200 Subject: [PATCH 1/6] Fixes #1380 --- sonar/projects.py | 55 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/sonar/projects.py b/sonar/projects.py index f11558d44..185fc2337 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -80,6 +80,46 @@ "webhooks", ) +_UNNEEDED_CONTEXT_DATA = ( + "sonar.announcement.message", + "sonar.auth.github.allowUsersToSignUp", + "sonar.auth.github.apiUrl", + "sonar.auth.github.appId", + "sonar.auth.github.enabled", + "sonar.auth.github.groupsSync", + "sonar.auth.github.organizations", + "sonar.auth.github.webUrl", + "sonar.builtInQualityProfiles.disableNotificationOnUpdate", + "sonar.core.id", + "sonar.core.serverBaseURL", + "sonar.core.startTime", + "sonar.dbcleaner.branchesToKeepWhenInactive", + "sonar.forceAuthentication", + "sonar.host.url", + "sonar.java.jdkHome", + "sonar.links.ci", + "sonar.links.homepage", + "sonar.links.issue", + "sonar.links.scm", + "sonar.links.scm_dev", + "sonar.plugins.risk.consent", +) + +_UNNEEDED_TASK_DATA = ( + "analysisId", + "componentId", + "hasScannerContext", + "id", + "warningCount", + "componentQualifier", + "nodeName", + "componentName", + "componentKey", + "submittedAt", + "executedAt", + "type", +) + class Project(components.Component): """ @@ -953,6 +993,9 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, :return: All project configuration settings :rtype: dict """ + + remove_useless = lambda d, useless: {k: v for k, v in d if k not in useless} + log.info("Exporting %s", str(self)) try: json_data = self._json.copy() @@ -983,10 +1026,16 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, last_task = self.last_task() json_data["backgroundTasks"] = {} if last_task: + ctxt = last_task.scanner_context() + if ctxt: + ctxt = {k: v for k, v in ctxt.items() if k not in _UNNEEDED_CONTEXT_DATA} + t_hist = [] + for t in self.task_history(): + t_hist.append({k: v for k, v in t._json.items() if k not in _UNNEEDED_TASK_DATA}) json_data["backgroundTasks"] = { - "lastTaskScannerContext": last_task.scanner_context(), - "lastTaskWarnings": last_task.warnings(), - "taskHistory": [t._json for t in self.task_history()], + "lastTaskScannerContext": ctxt, + # "lastTaskWarnings": last_task.warnings(), + "taskHistory": t_hist, } settings_dict = settings.get_bulk(endpoint=self.endpoint, component=self, settings_list=settings_list, include_not_set=False) From 597c81e4635d639e7a53dd5f65b67c6d3dbc9b99 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 10:33:47 +0200 Subject: [PATCH 2/6] Update what's new --- migration/README.md | 1 + migration/what-is-new.md | 1 + 2 files changed, 2 insertions(+) diff --git a/migration/README.md b/migration/README.md index 85d8deae0..6906427d9 100644 --- a/migration/README.md +++ b/migration/README.md @@ -116,6 +116,7 @@ When sonar-migration complete successfully they return exit code 0. En case of f - Support of incremental dump of projects extracts - Display of HTTP requests duration in DEBUG logs - Fixes in documentation +- Trimmed background task data to keep only what is need (to reduce memory and output JSON size) ## Version 0.2 diff --git a/migration/what-is-new.md b/migration/what-is-new.md index dc5b47192..2fff1b12b 100644 --- a/migration/what-is-new.md +++ b/migration/what-is-new.md @@ -6,6 +6,7 @@ - Support of incremental dump of projects extracts - Display of HTTP requests duration in DEBUG logs - Fixes in documentation +- Trimmed background task data to keep only what is need (to reduce memory and output JSON size) # Version 0.2 From d0c8a7bfec112f002024589d78e604b194be902c Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 10:37:38 +0200 Subject: [PATCH 3/6] Exclude rule that forbids lambdas --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 2f0ff0d6d..d56588b0b 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ [flake8] # E501 = Line too long -ignore = E501,E401,W503,E128,C901,W504,E302,E265,E741,W291,W292,W293,W391,F401,E226,F841,F821,F541,D400,D401,E800,D102,D105,D107,I900,F632,E261 +ignore = E501,E401,W503,E128,C901,W504,E302,E265,E741,W291,W292,W293,W391,F401,E226,F841,F821,F541,D400,D401,E800,D102,D105,D107,I900,F632,E261,E731 max-line-length = 150 #exclude = tests/* #max-complexity = 10 From 00a51fc46ec2e37e9372a09974e79e15a227fa6c Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 10:44:23 +0200 Subject: [PATCH 4/6] Catch all exceptions in project export --- sonar/projects.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sonar/projects.py b/sonar/projects.py index 185fc2337..bb52162dd 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -993,13 +993,10 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, :return: All project configuration settings :rtype: dict """ - - remove_useless = lambda d, useless: {k: v for k, v in d if k not in useless} - log.info("Exporting %s", str(self)) + json_data = self._json.copy() + json_data.update({"key": self.key, "name": self.name}) try: - json_data = self._json.copy() - json_data.update({"key": self.key, "name": self.name}) json_data["binding"] = self.__export_get_binding() nc = self.new_code() if nc != "": @@ -1046,14 +1043,17 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, json_data.update(s.to_json()) except HTTPError as e: if e.response.status_code == HTTPStatus.FORBIDDEN: - log.critical("Insufficient privileges to access %s, export of this project skipped", str(self)) - json_data = {"error": "Insufficient permissions while extracting project"} + log.critical("Insufficient privileges to access %s, export of this project interrupted", str(self)) + json_data["error"] = f"Insufficient permissions while exporting project, export interrupted" else: - log.critical("HTTP error %s while exporting %s, export of this project skipped", str(e), str(self)) - json_data = {"error": f"HTTP error {str(e)} while extracting project"} + log.critical("HTTP error %s while exporting %s, export of this project interrupted", str(e), str(self)) + json_data["error"] = f"HTTP error {str(e)} while extracting project" except ConnectionError as e: - log.critical("Connecting error %s while extracting %s, extract of this project skipped", str(self), str(e)) - json_data = {"error": f"Connection error {str(e)} while extracting prooject"} + log.critical("Connecting error %s while exporting %s, export of this project interrupted", str(self), str(e)) + json_data["error"] = f"Connection error {str(e)} while extracting project, export interrupted" + except Exception as e: + log.critical("Connecting error %s while exporting %s, export of this project interrupted", str(self), str(e)) + json_data["error"] = f"Exception {str(e)} while exporting project, export interrupted" log.info("Exporting %s done", str(self)) return util.remove_nones(json_data) From 54b314b29d4fd4e288b0b78b1226b09773d4b8a7 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 10:44:58 +0200 Subject: [PATCH 5/6] Cleanup --- sonar/projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar/projects.py b/sonar/projects.py index bb52162dd..5f9ed8fc2 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -1044,7 +1044,7 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, except HTTPError as e: if e.response.status_code == HTTPStatus.FORBIDDEN: log.critical("Insufficient privileges to access %s, export of this project interrupted", str(self)) - json_data["error"] = f"Insufficient permissions while exporting project, export interrupted" + json_data["error"] = "Insufficient permissions while exporting project, export interrupted" else: log.critical("HTTP error %s while exporting %s, export of this project interrupted", str(e), str(self)) json_data["error"] = f"HTTP error {str(e)} while extracting project" From 8b551068d76ea49baf47dd34bc09db117d46024f Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 10:47:22 +0200 Subject: [PATCH 6/6] Remove rule that's duplicate with SonarLint --- pylintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index 3a99abbd2..856da153e 100644 --- a/pylintrc +++ b/pylintrc @@ -86,7 +86,8 @@ disable=raw-checker-failed, W0611, W0238, R0911, - C0325 + C0325, + W1309 # Enable the message, report, category or checker with the given id(s). You can