From 6e79b18fc25c2be869926631c47a31db52ba7158 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 14:49:36 +0200 Subject: [PATCH 01/11] Fixes #1334 --- sonar/branches.py | 29 ++++++++++++++++++++--------- sonar/hotspots.py | 9 +++++++++ sonar/projects.py | 18 +++++++++++++----- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/sonar/branches.py b/sonar/branches.py index d093bd7bd..337818930 100644 --- a/sonar/branches.py +++ b/sonar/branches.py @@ -220,7 +220,8 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: :return: The branch new code period definition :rtype: str """ - from sonar import issues + from sonar.issues import count as issue_count + from sonar.hotspots import count as hotspot_count log.debug("Exporting %s", str(self)) data = {settings.NEW_CODE_PERIOD: self.new_code()} @@ -243,16 +244,26 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: if export_settings.get("MODE", "") == "MIGRATION": tpissues = self.count_third_party_issues() issue_data = {"thirdParty": tpissues if len(tpissues) > 0 else 0} + proj_key = self.concerned_object.key if self.endpoint.version() >= (10, 0, 0): - issue_data["falsePositives"] = issues.count( - self.endpoint, components=self.concerned_object.key, branch=self.name, issueStatuses="FALSE_POSITIVE" - ) - issue_data["accepted"] = issues.count(self.endpoint, components=self.concerned_object.key, branch=self.name, issueStatuses="ACCEPTED") + params = {"components": proj_key, "branch": self.name} + data["issues"] = { + "falsePositives": issue_count(self.endpoint, issueStatuses="FALSE_POSITIVE", **params), + "accepted": issue_count(self.endpoint, issueStatuses="ACCEPTED", **params), + } + params = {"project": proj_key, "branch": self.name} else: - issue_data["falsePositives"] = issues.count( - self.endpoint, componentKeys=self.concerned_object.key, branch=self.name, resolutions="FALSE-POSITIVE" - ) - issue_data["wontFix"] = issues.count(self.endpoint, componentKeys=self.concerned_object.key, branch=self.name, resolutions="WONTFIX") + params = {"componentKeys": proj_key, "branch": self.name} + data["issues"] = { + "falsePositives": issue_count(self.endpoint, resolutions="FALSE-POSITIVE", **params), + "wontFix": issue_count(self.endpoint, resolutions="WONTFIX", **params), + } + params = {"projectKey": proj_key, "branch": self.name} + data["hotspots"] = { + "acknowledged": hotspot_count(self.endpoint, resolution="ACKNOWLEDGED", **params), + "safe": hotspot_count(self.endpoint, resolution="SAFE", **params), + "fixed": hotspot_count(self.endpoint, resolution="FIXED", **params), + } data["issues"] = issue_data log.debug("%s has these notable issues %s", str(self), str(data["issues"])) data = util.remove_nones(data) diff --git a/sonar/hotspots.py b/sonar/hotspots.py index 9ed58768d..8f6f20607 100644 --- a/sonar/hotspots.py +++ b/sonar/hotspots.py @@ -491,3 +491,12 @@ def post_search_filter(hotspots_dict: dict[str, Hotspot], filters: types.ApiPara filtered_findings.pop(key, None) return filtered_findings + + +def count(endpoint: pf.Platform, **kwargs) -> int: + """Returns number of hotspots of a search""" + params = {} if not kwargs else kwargs.copy() + params["ps"] = 1 + nbr_hotspots = len(search(endpoint=endpoint, filters=params)) + log.debug("Hotspot counts with filters %s is %d hotspots", str(kwargs), nbr_hotspots) + return nbr_hotspots diff --git a/sonar/projects.py b/sonar/projects.py index 85f46ff77..fc16dcd22 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -953,7 +953,8 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, :return: All project configuration settings :rtype: dict """ - from sonar import issues + from sonar.issues import count as issue_count + from sonar.hotspots import count as hotspot_count log.info("Exporting %s", str(self)) try: @@ -999,12 +1000,19 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, tpissues = self.count_third_party_issues() issue_data = {"thirdParty": tpissues if len(tpissues) > 0 else 0} if self.endpoint.version() >= (10, 0, 0): - issue_data["falsePositives"] = issues.count(self.endpoint, components=self.key, issueStatuses="FALSE_POSITIVE") - issue_data["accepted"] = issues.count(self.endpoint, components=self.key, issueStatuses="ACCEPTED") + issue_data["falsePositives"] = issue_count(self.endpoint, components=self.key, issueStatuses="FALSE_POSITIVE") + issue_data["accepted"] = issue_count(self.endpoint, components=self.key, issueStatuses="ACCEPTED") + params = {"project": self.key} else: - issue_data["falsePositives"] = issues.count(self.endpoint, componentKeys=self.key, resolutions="FALSE-POSITIVE") - issue_data["wontFix"] = issues.count(self.endpoint, componentKeys=self.key, resolutions="WONTFIX") + issue_data["falsePositives"] = issue_count(self.endpoint, componentKeys=self.key, resolutions="FALSE-POSITIVE") + issue_data["wontFix"] = issue_count(self.endpoint, componentKeys=self.key, resolutions="WONTFIX") + params = {"projectKey": self.key} json_data["issues"] = issue_data + json_data["hotspots"] = { + "acknowledged": hotspot_count(self.endpoint, resolution="ACKNOWLEDGED", **params), + "safe": hotspot_count(self.endpoint, resolution="SAFE", **params), + "fixed": hotspot_count(self.endpoint, resolution="FIXED", **params), + } log.debug("%s has these notable issues %s", str(self), str(json_data["issues"])) settings_dict = settings.get_bulk(endpoint=self.endpoint, component=self, settings_list=settings_list, include_not_set=False) From f9b39e0ca144eda498f83202020a80e6be4e382b Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 15:00:50 +0200 Subject: [PATCH 02/11] Add tests on hotspots --- test/test_migration.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_migration.py b/test/test_migration.py index 05d4b01c8..dda8c02e6 100644 --- a/test/test_migration.py +++ b/test/test_migration.py @@ -80,9 +80,12 @@ def test_migration() -> None: iss = p["branches"]["master"]["issues"] assert iss["accepted"] > 0 - assert iss["fslePositives"] > 0 + assert iss["falsePositives"] > 0 assert iss["thirdParty"] == 0 + assert p["branches"]["master"]["hotspots"]["safe"] > 0 + assert p["branches"]["master"]["hotspots"]["acknowledged"] == 0 + p = json_config["projects"]["checkstyle-issues"] assert len(p["branches"]["issues"]["thirdParty"]) > 0 From 306b9bd2ea7c5920e0d5bdbd5d1d1a63f87f1a81 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 15:11:53 +0200 Subject: [PATCH 03/11] Fix bug --- sonar/branches.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sonar/branches.py b/sonar/branches.py index 337818930..1b8397a6f 100644 --- a/sonar/branches.py +++ b/sonar/branches.py @@ -247,17 +247,13 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: proj_key = self.concerned_object.key if self.endpoint.version() >= (10, 0, 0): params = {"components": proj_key, "branch": self.name} - data["issues"] = { - "falsePositives": issue_count(self.endpoint, issueStatuses="FALSE_POSITIVE", **params), - "accepted": issue_count(self.endpoint, issueStatuses="ACCEPTED", **params), - } + issue_data["falsePositives"] = issue_count(self.endpoint, issueStatuses="FALSE_POSITIVE", **params) + issue_data["accepted"] = issue_count(self.endpoint, issueStatuses="ACCEPTED", **params) params = {"project": proj_key, "branch": self.name} else: params = {"componentKeys": proj_key, "branch": self.name} - data["issues"] = { - "falsePositives": issue_count(self.endpoint, resolutions="FALSE-POSITIVE", **params), - "wontFix": issue_count(self.endpoint, resolutions="WONTFIX", **params), - } + issue_data["falsePositives"] = (issue_count(self.endpoint, resolutions="FALSE-POSITIVE", **params),) + issue_data["wontFix"] = (issue_count(self.endpoint, resolutions="WONTFIX", **params),) params = {"projectKey": proj_key, "branch": self.name} data["hotspots"] = { "acknowledged": hotspot_count(self.endpoint, resolution="ACKNOWLEDGED", **params), From 891a3f94fe041df0e116ccf3713b2c4975c27f4e Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 15:12:08 +0200 Subject: [PATCH 04/11] Adjust test for 9.9- --- test/test_migration.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_migration.py b/test/test_migration.py index dda8c02e6..36a93917d 100644 --- a/test/test_migration.py +++ b/test/test_migration.py @@ -79,7 +79,10 @@ def test_migration() -> None: assert p["ncloc"]["total"] > 0 iss = p["branches"]["master"]["issues"] - assert iss["accepted"] > 0 + if util.SQ.version() >= (10, 0, 0): + assert iss["accepted"] > 0 + else: + assert iss["wontFix"] > 0 assert iss["falsePositives"] > 0 assert iss["thirdParty"] == 0 From 5724f412c40a3faf6e327a33903508308fa05359 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 15:20:17 +0200 Subject: [PATCH 05/11] Remove annoying warning --- sonar/hotspots.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonar/hotspots.py b/sonar/hotspots.py index 8f6f20607..f2e95d8fd 100644 --- a/sonar/hotspots.py +++ b/sonar/hotspots.py @@ -441,7 +441,8 @@ def get_search_filters(endpoint: pf.Platform, params: types.ApiParams) -> types. criterias["status"] = util.allowed_values_string(criterias["status"], STATUSES) if "resolution" in criterias: criterias["resolution"] = util.allowed_values_string(criterias["resolution"], RESOLUTIONS) - log.warning("hotspot 'status' criteria incompatible with 'resolution' criteria, ignoring 'status'") + if "status" in criterias: + log.warning("hotspot 'status' criteria incompatible with 'resolution' criteria, ignoring 'status'") criterias["status"] = "REVIEWED" if endpoint.version() >= (10, 2, 0): criterias = util.dict_remap(original_dict=criterias, remapping={PROJECT_FILTER_OLD: PROJECT_FILTER}) From 1bf71b188cf2a97f940c85a2d9131b3547ef86fd Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 15:20:33 +0200 Subject: [PATCH 06/11] Remove annoying warning log --- sonar/hotspots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar/hotspots.py b/sonar/hotspots.py index f2e95d8fd..889bab005 100644 --- a/sonar/hotspots.py +++ b/sonar/hotspots.py @@ -443,7 +443,7 @@ def get_search_filters(endpoint: pf.Platform, params: types.ApiParams) -> types. criterias["resolution"] = util.allowed_values_string(criterias["resolution"], RESOLUTIONS) if "status" in criterias: log.warning("hotspot 'status' criteria incompatible with 'resolution' criteria, ignoring 'status'") - criterias["status"] = "REVIEWED" + criterias["status"] = "REVIEWED" if endpoint.version() >= (10, 2, 0): criterias = util.dict_remap(original_dict=criterias, remapping={PROJECT_FILTER_OLD: PROJECT_FILTER}) return util.dict_subset(criterias, SEARCH_CRITERIAS) From bd9c19fa027fd7aeff966fb4e1a7a095ea7ec599 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 18:46:20 +0200 Subject: [PATCH 07/11] Handle all types in list_to_csv --- sonar/utilities.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/sonar/utilities.py b/sonar/utilities.py index f8344eec2..3779a21c6 100644 --- a/sonar/utilities.py +++ b/sonar/utilities.py @@ -209,19 +209,21 @@ def csv_to_list(string: str, separator: str = ",") -> list[str]: return [s.strip() for s in string.split(separator)] -def list_to_csv(array: Union[None, str, list[str]], separator: str = ",", check_for_separator: bool = False) -> Optional[str]: +def list_to_csv(array: Union[None, str, int, float, list[str]], separator: str = ",", check_for_separator: bool = False) -> Optional[str]: """Converts a list of strings to CSV""" if isinstance(array, str): return csv_normalize(array, separator) if " " in array else array if array is None: return None - if check_for_separator: - # Don't convert to string if one array item contains the string separator - s = separator.strip() - for item in array: - if s in item: - return array - return separator.join([v.strip() for v in array]) + if isinstance(array, (list, set, tuple)): + if check_for_separator: + # Don't convert to string if one array item contains the string separator + s = separator.strip() + for item in array: + if s in item: + return array + return separator.join([v.strip() for v in array]) + return str(array) def csv_normalize(string: str, separator: str = ",") -> str: @@ -587,6 +589,15 @@ def dict_remap(original_dict: dict[str, str], remapping: dict[str, str]) -> dict return remapped_filters +def list_re_value(a_list: list[str], mapping: dict[str, str]) -> list[str]: + """Adjust findings search filters based on Sonar version""" + if not a_list or len(a_list) == 0: + return [] + for old, new in mapping.items(): + a_list = [new if v == old else v for v in a_list] + return a_list + + def dict_stringify(original_dict: dict[str, str]) -> dict[str, str]: """Covert dict list values into CSV string""" if not original_dict: From 9c00d21058e68a34450c73c286673ea92c15a2db Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 18:47:28 +0200 Subject: [PATCH 08/11] Fixes #1334 --- sonar/branches.py | 27 ++++++++++----------------- sonar/projects.py | 23 ++++++++++------------- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/sonar/branches.py b/sonar/branches.py index 1b8397a6f..61ae9299f 100644 --- a/sonar/branches.py +++ b/sonar/branches.py @@ -241,26 +241,19 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: loc_distrib = {m.split("=")[0]: int(m.split("=")[1]) for m in lang_distrib.split(";")} loc_distrib["total"] = self.loc() data["ncloc"] = loc_distrib - if export_settings.get("MODE", "") == "MIGRATION": tpissues = self.count_third_party_issues() - issue_data = {"thirdParty": tpissues if len(tpissues) > 0 else 0} - proj_key = self.concerned_object.key - if self.endpoint.version() >= (10, 0, 0): - params = {"components": proj_key, "branch": self.name} - issue_data["falsePositives"] = issue_count(self.endpoint, issueStatuses="FALSE_POSITIVE", **params) - issue_data["accepted"] = issue_count(self.endpoint, issueStatuses="ACCEPTED", **params) - params = {"project": proj_key, "branch": self.name} - else: - params = {"componentKeys": proj_key, "branch": self.name} - issue_data["falsePositives"] = (issue_count(self.endpoint, resolutions="FALSE-POSITIVE", **params),) - issue_data["wontFix"] = (issue_count(self.endpoint, resolutions="WONTFIX", **params),) - params = {"projectKey": proj_key, "branch": self.name} + params = self.search_params() + data["issues"] = { + "thirdParty": tpissues if len(tpissues) > 0 else 0, + "falsePositives": issue_count(self.endpoint, issueStatuses=["FALSE_POSITIVE"], **params), + } + status = "accepted" if self.endpoint.version() >= (10, 2, 0) else "wontFix" + data["issues"][status] = issue_count(self.endpoint, issueStatuses=[status.upper()], **params) data["hotspots"] = { - "acknowledged": hotspot_count(self.endpoint, resolution="ACKNOWLEDGED", **params), - "safe": hotspot_count(self.endpoint, resolution="SAFE", **params), - "fixed": hotspot_count(self.endpoint, resolution="FIXED", **params), + "acknowledged": hotspot_count(self.endpoint, resolution=["ACKNOWLEDGED"], **params), + "safe": hotspot_count(self.endpoint, resolution=["SAFE"], **params), + "fixed": hotspot_count(self.endpoint, resolution=["FIXED"], **params), } - data["issues"] = issue_data log.debug("%s has these notable issues %s", str(self), str(data["issues"])) data = util.remove_nones(data) return None if len(data) == 0 else data diff --git a/sonar/projects.py b/sonar/projects.py index fc16dcd22..b537eaa62 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -998,20 +998,17 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, "taskHistory": [t._json for t in self.task_history()], } tpissues = self.count_third_party_issues() - issue_data = {"thirdParty": tpissues if len(tpissues) > 0 else 0} - if self.endpoint.version() >= (10, 0, 0): - issue_data["falsePositives"] = issue_count(self.endpoint, components=self.key, issueStatuses="FALSE_POSITIVE") - issue_data["accepted"] = issue_count(self.endpoint, components=self.key, issueStatuses="ACCEPTED") - params = {"project": self.key} - else: - issue_data["falsePositives"] = issue_count(self.endpoint, componentKeys=self.key, resolutions="FALSE-POSITIVE") - issue_data["wontFix"] = issue_count(self.endpoint, componentKeys=self.key, resolutions="WONTFIX") - params = {"projectKey": self.key} - json_data["issues"] = issue_data + params = self.search_params() + json_data["issues"] = { + "thirdParty": tpissues if len(tpissues) > 0 else 0, + "falsePositives": issue_count(self.endpoint, issueStatuses=["FALSE_POSITIVE"], **params), + } + status = "accepted" if self.endpoint.version() >= (10, 2, 0) else "wontFix" + json_data["issues"][status] = issue_count(self.endpoint, issueStatuses=[status.upper()], **params) json_data["hotspots"] = { - "acknowledged": hotspot_count(self.endpoint, resolution="ACKNOWLEDGED", **params), - "safe": hotspot_count(self.endpoint, resolution="SAFE", **params), - "fixed": hotspot_count(self.endpoint, resolution="FIXED", **params), + "acknowledged": hotspot_count(self.endpoint, resolution=["ACKNOWLEDGED"], **params), + "safe": hotspot_count(self.endpoint, resolution=["SAFE"], **params), + "fixed": hotspot_count(self.endpoint, resolution=["FIXED"], **params), } log.debug("%s has these notable issues %s", str(self), str(json_data["issues"])) From 7d2208e7c4089b413f4fbbaa04dab06b513eeedf Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 18:48:03 +0200 Subject: [PATCH 09/11] Small ps if just counting hotspots --- sonar/hotspots.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/sonar/hotspots.py b/sonar/hotspots.py index 889bab005..00707efec 100644 --- a/sonar/hotspots.py +++ b/sonar/hotspots.py @@ -387,9 +387,10 @@ def search(endpoint: pf.Platform, filters: types.ApiParams = None) -> dict[str, new_params = get_search_filters(endpoint=endpoint, params=filters) new_params = util.dict_remap(original_dict=new_params, remapping=_FILTERS_HOTSPOTS_REMAPPING) filters_iterations = split_search_filters(new_params) + ps = 500 if "ps" not in new_params else new_params["ps"] for inline_filters in filters_iterations: p = 1 - inline_filters["ps"] = 500 + inline_filters["ps"] = ps log.info("Searching hotspots with sanitized filters %s", str(inline_filters)) while True: inline_filters["p"] = p @@ -403,7 +404,7 @@ def search(endpoint: pf.Platform, filters: types.ApiParams = None) -> dict[str, nbr_hotspots = 0 return {} raise e - nbr_pages = (nbr_hotspots + 499) // 500 + nbr_pages = (nbr_hotspots + ps - 1) // ps log.debug("Number of hotspots: %d - Page: %d/%d", nbr_hotspots, inline_filters["p"], nbr_pages) if nbr_hotspots > 10000: raise TooManyHotspotsError( @@ -433,6 +434,7 @@ def get_object(endpoint: pf.Platform, key: str, data: dict[str] = None, from_exp def get_search_filters(endpoint: pf.Platform, params: types.ApiParams) -> types.ApiParams: """Returns the filtered list of params that are allowed for api/hotspots/search""" + log.debug("Sanitizing hotspot search criteria %s", str(params)) if params is None: return {} criterias = util.remove_nones(params.copy()) @@ -441,12 +443,12 @@ def get_search_filters(endpoint: pf.Platform, params: types.ApiParams) -> types. criterias["status"] = util.allowed_values_string(criterias["status"], STATUSES) if "resolution" in criterias: criterias["resolution"] = util.allowed_values_string(criterias["resolution"], RESOLUTIONS) - if "status" in criterias: - log.warning("hotspot 'status' criteria incompatible with 'resolution' criteria, ignoring 'status'") - criterias["status"] = "REVIEWED" - if endpoint.version() >= (10, 2, 0): - criterias = util.dict_remap(original_dict=criterias, remapping={PROJECT_FILTER_OLD: PROJECT_FILTER}) - return util.dict_subset(criterias, SEARCH_CRITERIAS) + criterias["status"] = "REVIEWED" + if endpoint.version() <= (10, 2, 0): + criterias = util.dict_remap(original_dict=criterias, remapping={PROJECT_FILTER: PROJECT_FILTER_OLD}) + criterias = util.dict_subset(criterias, SEARCH_CRITERIAS) + log.debug("Sanitized hotspot search criteria %s", str(criterias)) + return criterias def split_filter(params: types.ApiParams, criteria: str) -> list[types.ApiParams]: @@ -476,7 +478,7 @@ def split_search_filters(params: types.ApiParams) -> list[types.ApiParams]: def post_search_filter(hotspots_dict: dict[str, Hotspot], filters: types.ApiParams) -> dict[str, Hotspot]: """Filters a dict of hotspots with provided filters""" filtered_findings = hotspots_dict.copy() - log.info("Post filtering findings with %s", str(filters)) + log.debug("Post filtering findings with %s", str(filters)) if "createdAfter" in filters: min_date = util.string_to_date(filters["createdAfter"]) if "createdBefore" in filters: @@ -499,5 +501,5 @@ def count(endpoint: pf.Platform, **kwargs) -> int: params = {} if not kwargs else kwargs.copy() params["ps"] = 1 nbr_hotspots = len(search(endpoint=endpoint, filters=params)) - log.debug("Hotspot counts with filters %s is %d hotspots", str(kwargs), nbr_hotspots) + log.debug("Hotspot counts with filters %s returned %d hotspots", str(kwargs), nbr_hotspots) return nbr_hotspots From 78f6686c40d03582cdfcffa492d71d07890d7efa Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 18:48:22 +0200 Subject: [PATCH 10/11] Refactoring issue filters per search --- sonar/issues.py | 96 ++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 50 deletions(-) diff --git a/sonar/issues.py b/sonar/issues.py index fe50e4fb1..e8410681e 100644 --- a/sonar/issues.py +++ b/sonar/issues.py @@ -43,6 +43,12 @@ COMPONENT_FILTER_OLD = "componentKeys" COMPONENT_FILTER = "components" +OLD_STATUS = "resolutions" +NEW_STATUS = "issueStatuses" + +OLD_FP = "FALSE-POSITIVE" +NEW_FP = "FALSE_POSITIVE" + _SEARCH_CRITERIAS = ( COMPONENT_FILTER_OLD, COMPONENT_FILTER, @@ -77,18 +83,17 @@ "author", "issues", "languages", - "resolutions", + OLD_STATUS, "resolved", "rules", "scopes", # 10.2 new filter "impactSeverities", # 10.4 new filter - "issueStatuses", + NEW_STATUS, ) _FILTERS_10_2_REMAPPING = {"severities": "impactSeverities"} -_FILTERS_10_4_REMAPPING = {"statuses": "issueStatuses"} TYPES = ("BUG", "VULNERABILITY", "CODE_SMELL") SEVERITIES = ("BLOCKER", "CRITICAL", "MAJOR", "MINOR", "INFO") @@ -102,7 +107,7 @@ "impactSoftwareQualities": IMPACT_SOFTWARE_QUALITIES, "impactSeverities": IMPACT_SEVERITIES, "statuses": STATUSES, - "resolutions": RESOLUTIONS, + OLD_STATUS: RESOLUTIONS, } _TOO_MANY_ISSUES_MSG = "Too many issues, recursing..." @@ -336,7 +341,7 @@ def is_wont_fix(self) -> bool: :return: Whether the issue is won't fix :rtype: bool """ - return self.resolution == "WONT-FIX" + return self.resolution == "WONTFIX" def is_accepted(self) -> bool: """ @@ -350,7 +355,7 @@ def is_false_positive(self) -> bool: :return: Whether the issue is a false positive :rtype: bool """ - return self.resolution == "FALSE-POSITIVE" + return self.resolution in ("FALSE-POSITIVE", "FALSE_POSITIVE") def strictly_identical_to(self, another_finding: Issue, ignore_component: bool = False) -> bool: """ @@ -447,7 +452,7 @@ def __apply_event(self, event: str, settings: ConfigSettings) -> bool: else: self.reopen() # self.add_comment(f"Issue re-open {origin}", settings[SYNC_ADD_COMMENTS]) - elif event_type == "FALSE-POSITIVE": + elif event_type in ("FALSE-POSITIVE", "FALSE_POSITIVE"): self.mark_as_false_positive() # self.add_comment(f"False positive {origin}", settings[SYNC_ADD_COMMENTS]) elif event_type == "WONT-FIX": @@ -747,8 +752,6 @@ def search(endpoint: pf.Platform, params: ApiParams = None, raise_error: bool = filters = pre_search_filters(endpoint=endpoint, params=params) # if endpoint.version() >= (10, 2, 0): # new_params = util.dict_remap_and_stringify(new_params, _FILTERS_10_2_REMAPPING) - if endpoint.version() >= (10, 4, 0): - filters = _change_filters_for_10_4(filters) log.debug("Search filters = %s", str(filters)) if not filters: @@ -822,11 +825,10 @@ def count(endpoint: pf.Platform, **kwargs) -> int: params = {} if not kwargs else kwargs.copy() params["ps"] = 1 try: - log.debug("Count params = %s", str(params)) nbr_issues = len(search(endpoint=endpoint, params=params)) except TooManyIssuesError as e: nbr_issues = e.nbr_issues - log.debug("Issue search %s would return %d issues", str(kwargs), nbr_issues) + log.debug("Count issues with filters %s returned %d issues", str(kwargs), nbr_issues) return nbr_issues @@ -852,7 +854,7 @@ def count_by_rule(endpoint: pf.Platform, **kwargs) -> dict[str, int]: if d["val"] not in rulecount: rulecount[d["val"]] = 0 rulecount[d["val"]] += d["count"] - log.debug("Rule counts = %s", util.json_dump(rulecount)) + # log.debug("Rule counts = %s", util.json_dump(rulecount)) return rulecount @@ -868,42 +870,36 @@ def pre_search_filters(endpoint: pf.Platform, params: ApiParams) -> ApiParams: """Returns the filtered list of params that are allowed for api/issue/search""" if not params: return {} - filters = util.dict_subset(util.remove_nones(params.copy()), _SEARCH_CRITERIAS) - if endpoint.version() >= (10, 2, 0): - if COMPONENT_FILTER_OLD in filters: - filters[COMPONENT_FILTER] = filters.pop(COMPONENT_FILTER_OLD) - if "types" in filters: - __MAP = {"BUG": "RELIABILITY", "CODE_SMELL": "MAINTAINABILITY", "VULNERABILITY": "SECURITY", "SECURITY_HOTSPOT": "SECURITY"} - filters["impactSoftwareQualities"] = [__MAP[t] for t in filters.pop("types")] - if len(filters["impactSoftwareQualities"]) == 0: - filters.pop("impactSoftwareQualities") - if "severities" in filters: - __MAP = {"BLOCKER": "HIGH", "CRITICAL": "HIGH", "MAJOR": "MEDIUM", "MINOR": "LOW", "INFO": "LOW"} - filters["impactSeverities"] = [__MAP[t] for t in filters.pop("severities")] - if len(filters["impactSeverities"]) == 0: - filters.pop("impactSeverities") - for k, v in FILTERS_MAP.items(): - if k in filters: - filters[k] = util.allowed_values_string(filters[k], v) - if filters.get("languages", None) is not None: - filters["languages"] = util.list_to_csv(filters["languages"]) - + log.debug("Sanitizing issue search filters %s", str(params)) + version = endpoint.version() + filters = util.dict_remap(original_dict=params.copy(), remapping={"project": COMPONENT_FILTER}) + filters = util.dict_subset(util.remove_nones(filters), _SEARCH_CRITERIAS) + if version < (10, 2, 0): + # Starting from 10.2 - "componentKeys" was renamed "components" + filters = util.dict_remap(original_dict=filters, remapping={COMPONENT_FILTER: COMPONENT_FILTER_OLD}) + else: + # Starting from 10.2 - Issue types were replaced by software qualities, and severities replaced by impacts + __MAP = {"BUG": "RELIABILITY", "CODE_SMELL": "MAINTAINABILITY", "VULNERABILITY": "SECURITY", "SECURITY_HOTSPOT": "SECURITY"} + filters["impactSoftwareQualities"] = util.list_re_value(filters.pop("types", None), __MAP) + if len(filters["impactSoftwareQualities"]) == 0: + filters.pop("impactSoftwareQualities") + __MAP = {"BLOCKER": "HIGH", "CRITICAL": "HIGH", "MAJOR": "MEDIUM", "MINOR": "LOW", "INFO": "LOW"} + filters["impactSeverities"] = util.list_re_value(filters.pop("severities", None), __MAP) + if len(filters["impactSeverities"]) == 0: + filters.pop("impactSeverities") + + if version < (10, 4, 0): + log.debug("Sanitizing issue search filters - fixing resolutions") + filters = util.dict_remap(original_dict=filters, remapping={NEW_STATUS: OLD_STATUS}) + if OLD_STATUS in filters: + filters[OLD_STATUS] = util.list_re_value(filters[OLD_STATUS], mapping={NEW_FP: OLD_FP}) + else: + # Starting from 10.4 - "resolutions" was renamed "issuesStatuses", "FALSE-POSITIVE" was renamed "FALSE_POSITIVE" + filters = util.dict_remap(original_dict=filters, remapping={OLD_STATUS: NEW_STATUS}) + if NEW_STATUS in filters: + filters[NEW_STATUS] = util.list_re_value(filters[NEW_STATUS], mapping={OLD_FP: NEW_FP}) + + filters = {k: util.allowed_values_string(v, FILTERS_MAP[k]) if k in FILTERS_MAP else v for k, v in filters.items()} + filters = {k: util.list_to_csv(v) for k, v in filters.items() if v} + log.debug("Sanitized issue search filters %s", str(filters)) return filters - - -def _change_filters_for_10_4(filters: ApiParams) -> ApiParams: - """Adjust filters for new 10.4 issues/search API parameters""" - if not filters: - return None - new_filters = util.dict_remap(filters.copy(), _FILTERS_10_4_REMAPPING) - statuses = [] - for f in "resolutions", "issueStatuses": - if f in new_filters: - statuses += util.csv_to_list(new_filters[f]) - new_filters.pop("resolutions", None) - if len(statuses) > 0: - if "FALSE-POSITIVE" in statuses: - statuses.remove("FALSE-POSITIVE") - statuses.append("FALSE_POSITIVE") - new_filters["issueStatuses"] = util.list_to_csv(statuses) - return new_filters From 82f29aed8ba15434df3cc9f7d335345c10f3a1aa Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 28 Sep 2024 18:57:56 +0200 Subject: [PATCH 11/11] Adjust test for 9.9 (No SLint last connection date) --- test/test_migration.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_migration.py b/test/test_migration.py index 36a93917d..e9ec5e3b2 100644 --- a/test/test_migration.py +++ b/test/test_migration.py @@ -60,8 +60,11 @@ def test_migration() -> None: u = json_config["users"]["admin"] assert "sonar-users" in u["groups"] assert u["local"] and u["active"] - assert "sonarQubeLastConnectionDate" in u - assert "sonarLintLastConnectionDate" in u + if util.SQ.version() >= (10, 0, 0): + assert "sonarQubeLastConnectionDate" in u + assert "sonarLintLastConnectionDate" in u + else: + assert "lastConnectionDate" in u assert json_config["users"]["olivier"]["externalProvider"] == "sonarqube" u = json_config["users"]["olivier-korach65532"]