diff --git a/cli/config.py b/cli/config.py index 56230a6f..026047f5 100644 --- a/cli/config.py +++ b/cli/config.py @@ -32,6 +32,12 @@ from cli import options from sonar import exceptions, errcodes, utilities, version from sonar.util import types, constants as c +from sonar.util import platform_helper as pfhelp +from sonar.util import project_helper as pjhelp +from sonar.util import portfolio_helper as foliohelp +from sonar.util import qualityprofile_helper as qphelp +from sonar.util import rule_helper as rhelp + import sonar.logging as log from sonar import platform, rules, qualityprofiles, qualitygates, users, groups from sonar import projects, portfolios, applications @@ -39,7 +45,6 @@ TOOL_NAME = "sonar-config" -DONT_INLINE_LISTS = "dontInlineLists" FULL_EXPORT = "fullExport" EXPORT_EMPTY = "exportEmpty" @@ -104,14 +109,6 @@ def __parse_args(desc: str) -> object: f"By default the export will show the value as '{utilities.DEFAULT}' " "and the setting will not be imported at import time", ) - parser.add_argument( - f"--{DONT_INLINE_LISTS}", - required=False, - default=False, - action="store_true", - help="By default, sonar-config exports multi-valued settings as comma separated strings instead of arrays (if there is not comma in values). " - "Set this flag if you want to force export multi valued settings as arrays", - ) parser.add_argument( f"--{EXPORT_EMPTY}", required=False, @@ -210,7 +207,6 @@ def export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> Non export_settings = kwargs.copy() export_settings.update( { - "INLINE_LISTS": not kwargs.get(DONT_INLINE_LISTS, False), "EXPORT_DEFAULTS": True, "FULL_EXPORT": kwargs.get(FULL_EXPORT, False), "MODE": mode, @@ -268,8 +264,6 @@ def __prep_json_for_write(json_data: types.ObjectJsonRepr, export_settings: type if not export_settings.get(EXPORT_EMPTY, False): log.debug("Removing empties") json_data = utilities.clean_data(json_data, remove_empty=True) - if export_settings.get("INLINE_LISTS", True): - json_data = utilities.inline_lists(json_data, exceptions=("conditions",)) return json_data @@ -309,22 +303,23 @@ def convert_json(**kwargs) -> dict[str, Any]: with open(kwargs["convertFrom"], encoding="utf-8") as fd: old_json = json.loads(fd.read()) mapping = { - "platform": platform.old_to_new_json, - "globalSettings": platform.global_settings_old_to_new_json, - "qualityProfiles": qualityprofiles.old_to_new_json, - "qualityGates": qualitygates.old_to_new_json, - "projects": projects.old_to_new_json, - "portfolios": portfolios.old_to_new_json, - "applications": applications.old_to_new_json, - "users": users.old_to_new_json, - "groups": groups.old_to_new_json, - "rules": rules.old_to_new_json, + "platform": pfhelp.convert_basics_json, + "globalSettings": pfhelp.convert_global_settings_json, + "qualityGates": qualitygates.convert_qgs_json, + "qualityProfiles": qphelp.convert_qps_json, + "projects": pjhelp.convert_projects_json, + "portfolios": foliohelp.convert_portfolios_json, + "applications": applications.convert_apps_json, + "users": users.convert_users_json, + "groups": groups.convert_groups_json, + "rules": rhelp.convert_rules_json, } new_json = {} for k, func in mapping.items(): if k in old_json: log.info("Converting %s", k) new_json[k] = func(old_json[k]) + new_json = __normalize_json(new_json, remove_empty=False, remove_none=True) with open(kwargs["convertTo"], mode="w", encoding="utf-8") as fd: print(utilities.json_dump(new_json), file=fd) return new_json diff --git a/sonar/applications.py b/sonar/applications.py index 42745e5c..55eb679f 100644 --- a/sonar/applications.py +++ b/sonar/applications.py @@ -42,6 +42,7 @@ import sonar.utilities as util from sonar.audit import rules, problem import sonar.util.constants as c +from sonar.util import common_json_helper _CLASS_LOCK = Lock() _IMPORTABLE_PROPERTIES = ("key", "name", "description", "visibility", "branches", "permissions", "tags") @@ -344,7 +345,7 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: "tags": self.get_tags(), } ) - json_data = old_to_new_json_one(json_data) + json_data = convert_app_json(json_data) return util.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False)) def set_permissions(self, data: types.JsonPermissions) -> application_permissions.ApplicationPermissions: @@ -518,7 +519,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, **kwarg """ check_supported(endpoint) write_q = kwargs.get("write_q", None) - key_regexp = kwargs.get("key_list", ".*") + key_regexp = kwargs.get("key_list", ".+") app_list = {k: v for k, v in get_list(endpoint).items() if not key_regexp or re.match(key_regexp, k)} apps_settings = [] @@ -547,7 +548,7 @@ def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings, **kwargs) return [] log.info("--- Auditing applications ---") problems = [] - key_regexp = kwargs.get("key_list", None) or ".*" + key_regexp = kwargs.get("key_list", ".+") for obj in [o for o in get_list(endpoint).values() if not key_regexp or re.match(key_regexp, o.key)]: problems += obj.audit(audit_settings, **kwargs) return problems @@ -600,19 +601,17 @@ def search_by_name(endpoint: pf.Platform, name: str) -> dict[str, Application]: return data -def old_to_new_json_one(old_app_json: dict[str, Any]) -> dict[str, Any]: +def convert_app_json(old_app_json: dict[str, Any]) -> dict[str, Any]: """Converts sonar-config old JSON report format to new format for a single application""" - new_json = old_app_json.copy() - if "permissions" in old_app_json: - new_json["permissions"] = util.perms_to_list(old_app_json["permissions"]) + new_json = common_json_helper.convert_common_fields(old_app_json.copy()) if "branches" in old_app_json: new_json["branches"] = util.dict_to_list(old_app_json["branches"], "name") return new_json -def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]: +def convert_apps_json(old_json: dict[str, Any]) -> dict[str, Any]: """Converts sonar-config old JSON report format to new format""" new_json = old_json.copy() for k, v in new_json.items(): - new_json[k] = old_to_new_json_one(v) + new_json[k] = convert_app_json(v) return util.dict_to_list(new_json, "key") diff --git a/sonar/groups.py b/sonar/groups.py index 096908a6..c57f89c3 100644 --- a/sonar/groups.py +++ b/sonar/groups.py @@ -434,6 +434,6 @@ def exists(endpoint: pf.Platform, name: str) -> bool: return Group.get_object(endpoint=endpoint, name=name) is not None -def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]: +def convert_groups_json(old_json: dict[str, Any]) -> dict[str, Any]: """Converts sonar-config old groups JSON report format to new format""" return util.dict_to_list(old_json, "name", "description") diff --git a/sonar/permissions/permission_templates.py b/sonar/permissions/permission_templates.py index c8259bf2..0bded1fc 100644 --- a/sonar/permissions/permission_templates.py +++ b/sonar/permissions/permission_templates.py @@ -21,13 +21,14 @@ """Abstraction of the SonarQube permission template concept""" from __future__ import annotations -from typing import Optional +from typing import Optional, Any import json import re import sonar.logging as log from sonar.util import types, cache +from sonar.util import platform_helper as phelp from sonar import sqobject, utilities, exceptions from sonar.permissions import template_permissions import sonar.platform as pf @@ -42,8 +43,6 @@ _CREATE_API = "permissions/create_template" _UPDATE_API = "permissions/update_template" -_IMPORTABLE_PROPERTIES = ("name", "description", "pattern", "defaultFor", "permissions") - class PermissionTemplate(sqobject.SqObject): """Abstraction of the Sonar permission template concept""" @@ -183,7 +182,7 @@ def to_json(self, export_settings: types.ConfigSettings = None) -> types.ObjectJ json_data.pop("pattern") json_data["creationDate"] = utilities.date_to_string(self.creation_date) json_data["lastUpdate"] = utilities.date_to_string(self.last_update) - return utilities.remove_nones(utilities.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False))) + return phelp.convert_template_json(json_data, export_settings.get("FULL_EXPORT", False)) def _audit_pattern(self, audit_settings: types.ConfigSettings) -> list[pb.Problem]: log.debug("Auditing %s projectKeyPattern ('%s')", str(self), str(self.project_key_pattern)) diff --git a/sonar/permissions/permissions.py b/sonar/permissions/permissions.py index e10bcf7e..0f5799eb 100644 --- a/sonar/permissions/permissions.py +++ b/sonar/permissions/permissions.py @@ -86,10 +86,14 @@ def to_json(self, perm_type: str | None = None, csv: bool = False) -> types.Json if not csv: return self.permissions.get(perm_type, {}) if is_valid(perm_type) else self.permissions perms = {} - for p in normalize(perm_type): - if p not in self.permissions or len(self.permissions[p]) == 0: + for ptype in normalize(perm_type): + for k, v in self.permissions.get(ptype, {}).copy().items(): + if len(v) == 0: + self.permissions[ptype].pop(k) + for ptype in normalize(perm_type): + if ptype not in self.permissions or len(self.permissions[ptype]) == 0: continue - perms[p] = {k: encode(v) for k, v in self.permissions.get(p, {}).items()} + perms[ptype] = {k: encode(v) for k, v in self.permissions.get(ptype, {}).items()} return perms if len(perms) > 0 else None def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: diff --git a/sonar/platform.py b/sonar/platform.py index fe0086a2..17c6f0b6 100644 --- a/sonar/platform.py +++ b/sonar/platform.py @@ -40,6 +40,7 @@ import sonar.utilities as util from sonar.util import types, update_center import sonar.util.constants as c +import sonar.util.platform_helper as pfhelp from sonar import errcodes, settings, devops, version, sif, exceptions, organizations from sonar.permissions import permissions, global_permissions, permission_templates @@ -262,7 +263,7 @@ def delete(self, api: str, params: types.ApiParams = None, **kwargs) -> requests def __run_request(self, request: callable, api: str, params: types.ApiParams = None, **kwargs) -> requests.Response: """Makes an HTTP request to SonarQube""" mute = kwargs.pop("mute", ()) - api = _normalize_api(api) + api = pfhelp.normalize_api(api) headers = {"user-agent": self._user_agent, "accept": _APP_JSON} | kwargs.get("headers", {}) params = params or {} with_org = kwargs.pop("with_organization", True) @@ -498,21 +499,7 @@ def export(self, export_settings: types.ConfigSettings, full: bool = False) -> t if not self.is_sonarcloud(): json_data[settings.DEVOPS_INTEGRATION] = devops.export(self, export_settings=export_settings) - # Convert dicts to lists - special_categories = (settings.LANGUAGES_SETTINGS, settings.DEVOPS_INTEGRATION, "permissions", "permissionTemplates") - for categ in [cat for cat in settings.CATEGORIES if cat not in special_categories]: - json_data[categ] = util.sort_list_by_key(util.dict_to_list(json_data[categ], "key"), "key") - for k, v in json_data[settings.LANGUAGES_SETTINGS].items(): - json_data[settings.LANGUAGES_SETTINGS][k] = util.sort_list_by_key(util.dict_to_list(v, "key"), "key") - json_data[settings.LANGUAGES_SETTINGS] = util.dict_to_list(json_data[settings.LANGUAGES_SETTINGS], "language", "settings") - json_data[settings.DEVOPS_INTEGRATION] = util.dict_to_list(json_data[settings.DEVOPS_INTEGRATION], "key") - json_data["permissions"] = util.perms_to_list(json_data["permissions"]) - for v in json_data["permissionTemplates"].values(): - if "permissions" in v: - v["permissions"] = util.perms_to_list(v["permissions"]) - json_data["permissionTemplates"] = util.dict_to_list(json_data["permissionTemplates"], "key") - - return util.order_dict(json_data, [*settings.CATEGORIES, "permissions", "permissionTemplates"]) + return pfhelp.convert_global_settings_json(json_data) def set_webhooks(self, webhooks_data: types.ObjectJsonRepr) -> bool: """Sets global webhooks with a list of webhooks represented as JSON @@ -791,19 +778,6 @@ def set_standard_experience(self) -> bool: this.context = Platform(os.getenv("SONAR_HOST_URL", "http://localhost:9000"), os.getenv("SONAR_TOKEN", "")) -def _normalize_api(api: str) -> str: - """Normalizes an API based on its multiple original forms""" - if api.startswith("/api/"): - pass - elif api.startswith("api/"): - api = "/" + api - elif api.startswith("/"): - api = "/api" + api - else: - api = "/api/" + api - return api - - def _audit_setting_value(key: str, platform_settings: dict[str, Any], audit_settings: types.ConfigSettings, url: str) -> list[Problem]: """Audits a particular platform setting is set to expected value""" if (v := _get_multiple_values(4, audit_settings[key], "MEDIUM", "CONFIGURATION")) is None: @@ -963,29 +937,3 @@ def audit(endpoint: Platform, audit_settings: types.ConfigSettings, **kwargs) -> pbs = endpoint.audit(audit_settings) "write_q" in kwargs and kwargs["write_q"].put(pbs) return pbs - - -def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]: - """Converts sonar-config "plaform" section old JSON report format to new format""" - if "plugins" in old_json: - old_json["plugins"] = util.dict_to_list(old_json["plugins"], "key") - return old_json - - -def global_settings_old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]: - """Converts sonar-config "globalSettings" section old JSON report format to new format""" - new_json = {} - special_categories = (settings.LANGUAGES_SETTINGS, settings.DEVOPS_INTEGRATION, "permissions", "permissionTemplates") - for categ in [cat for cat in settings.CATEGORIES if cat not in special_categories]: - new_json[categ] = util.sort_list_by_key(util.dict_to_list(old_json[categ], "key"), "key") - for k, v in old_json[settings.LANGUAGES_SETTINGS].items(): - new_json[settings.LANGUAGES_SETTINGS] = new_json.get(settings.LANGUAGES_SETTINGS, None) or {} - new_json[settings.LANGUAGES_SETTINGS][k] = util.sort_list_by_key(util.dict_to_list(v, "key"), "key") - new_json[settings.LANGUAGES_SETTINGS] = util.dict_to_list(new_json[settings.LANGUAGES_SETTINGS], "language", "settings") - new_json[settings.DEVOPS_INTEGRATION] = util.dict_to_list(old_json[settings.DEVOPS_INTEGRATION], "key") - new_json["permissions"] = util.perms_to_list(old_json["permissions"]) - for v in old_json["permissionTemplates"].values(): - if "permissions" in v: - v["permissions"] = util.perms_to_list(v["permissions"]) - new_json["permissionTemplates"] = util.dict_to_list(old_json["permissionTemplates"], "key") - return new_json diff --git a/sonar/portfolios.py b/sonar/portfolios.py index 1b8f4a9b..6ae53f71 100644 --- a/sonar/portfolios.py +++ b/sonar/portfolios.py @@ -45,6 +45,7 @@ import sonar.utilities as util from sonar.audit import rules, problem from sonar.portfolio_reference import PortfolioReference +from sonar.util import portfolio_helper as phelp if TYPE_CHECKING: from sonar.util import types @@ -368,7 +369,8 @@ def to_json(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr subportfolios = self.sub_portfolios() if not self.is_sub_portfolio(): json_data["visibility"] = self._visibility - json_data["permissions"] = util.perms_to_list(self.permissions().export(export_settings=export_settings)) + if pf_perms := self.permissions().export(export_settings=export_settings): + json_data["permissions"] = util.perms_to_list(pf_perms) json_data["tags"] = self._tags if subportfolios: json_data["portfolios"] = {} @@ -388,7 +390,10 @@ def to_json(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: """Exports a portfolio (for sonar-config)""" log.info("Exporting %s", str(self)) - return util.remove_nones(util.filter_export(self.to_json(export_settings), _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False))) + json_data = util.remove_nones( + util.filter_export(self.to_json(export_settings), _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False)) + ) + return phelp.convert_portfolio_json(json_data) def permissions(self) -> pperms.PortfolioPermissions: """Returns a portfolio permissions (if toplevel) or None if sub-portfolio""" @@ -455,26 +460,20 @@ def set_manual_mode(self) -> Portfolio: def set_tags_mode(self, tags: list[str], branch: Optional[str] = None) -> Portfolio: """Sets a portfolio to tags mode""" - if branch is None: - branch = c.DEFAULT_BRANCH - self.post("views/set_tags_mode", params={"portfolio": self.key, "tags": util.list_to_csv(tags), "branch": get_api_branch(branch)}) - self._selection_mode = {_SELECTION_MODE_TAGS: tags, "branch": branch} + self.post("views/set_tags_mode", params={"portfolio": self.key, "tags": util.list_to_csv(tags), "branch": branch}) + self._selection_mode = {_SELECTION_MODE_TAGS: tags, "branch": branch or c.DEFAULT_BRANCH} return self def set_regexp_mode(self, regexp: str, branch: Optional[str] = None) -> Portfolio: """Sets a portfolio to regexp mode""" - if branch is None: - branch = c.DEFAULT_BRANCH - self.post("views/set_regexp_mode", params={"portfolio": self.key, "regexp": regexp, "branch": get_api_branch(branch)}) - self._selection_mode = {_SELECTION_MODE_REGEXP: regexp, "branch": branch} + self.post("views/set_regexp_mode", params={"portfolio": self.key, "regexp": regexp, "branch": branch}) + self._selection_mode = {_SELECTION_MODE_REGEXP: regexp, "branch": branch or c.DEFAULT_BRANCH} return self def set_remaining_projects_mode(self, branch: Optional[str] = None) -> Portfolio: """Sets a portfolio to remaining projects mode""" - if branch is None: - branch = c.DEFAULT_BRANCH - self.post("views/set_remaining_projects_mode", params={"portfolio": self.key, "branch": get_api_branch(branch)}) - self._selection_mode = {"rest": True, "branch": branch} + self.post("views/set_remaining_projects_mode", params={"portfolio": self.key, "branch": branch}) + self._selection_mode = {"rest": True, "branch": branch or c.DEFAULT_BRANCH} return self def set_none_mode(self) -> Portfolio: @@ -808,17 +807,6 @@ def recompute(endpoint: pf.Platform) -> None: endpoint.post(Portfolio.API["REFRESH"]) -def _find_sub_portfolio(key: str, data: types.ApiPayload) -> types.ApiPayload: - """Finds a subportfolio in a JSON hierarchy""" - for subp in data.get("subViews", []): - if subp["key"] == key: - return subp - child = _find_sub_portfolio(key, subp) - if child is not None: - return child - return {} - - def __create_portfolio_hierarchy(endpoint: pf.Platform, data: types.ApiPayload, parent_key: str) -> int: """Creates the hierarchy of portfolios that are new defined by reference""" nbr_creations = 0 @@ -840,34 +828,3 @@ def __create_portfolio_hierarchy(endpoint: pf.Platform, data: types.ApiPayload, o.root_portfolio = o_parent.root_portfolio nbr_creations += __create_portfolio_hierarchy(endpoint, subp, parent_key=key) return nbr_creations - - -def get_api_branch(branch: str) -> str: - """Returns the value to pass to the API for the branch parameter""" - return branch if branch != c.DEFAULT_BRANCH else None - - -def clear_cache(endpoint: pf.Platform) -> None: - """Clears the cache of an endpoint""" - Portfolio.clear_cache(endpoint) - - -def old_to_new_json_one(old_json: dict[str, Any]) -> dict[str, Any]: - """Converts the sonar-config old JSON report format for a single portfolio to the new one""" - new_json = old_json.copy() - for key in "children", "portfolios": - if key in new_json: - new_json[key] = old_to_new_json(new_json[key]) - if "permissions" in old_json: - new_json["permissions"] = util.perms_to_list(old_json["permissions"]) - if "branches" in old_json: - new_json["branches"] = util.dict_to_list(old_json["branches"], "name") - return new_json - - -def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]: - """Converts the sonar-config portfolios old JSON report format to the new one""" - new_json = old_json.copy() - for k, v in new_json.items(): - new_json[k] = old_to_new_json_one(v) - return util.dict_to_list(new_json, "key") diff --git a/sonar/projects.py b/sonar/projects.py index 907bd99f..bea11419 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -52,6 +52,7 @@ from sonar.audit.rules import get_rule, RuleId from sonar.audit.problem import Problem import sonar.util.constants as c +import sonar.util.project_helper as phelp _CLASS_LOCK = Lock() @@ -82,50 +83,11 @@ "visibility", "qualityGate", "webhooks", - "aiCodeFix", + phelp.AI_CODE_FIX, ) _PROJECT_QUALIFIER = "qualifier=TRK" -_UNNEEDED_CONTEXT_DATA = ( - "sonar.announcement.message", - "sonar.auth.github.allowUsersToSignUp", - "sonar.auth.github.apiUrl", - "sonar.auth.github.appId", - "sonar.auth.github.enabled", - "sonar.auth.github.groupsSync", - "sonar.auth.github.organizations", - "sonar.auth.github.webUrl", - "sonar.builtInQualityProfiles.disableNotificationOnUpdate", - "sonar.core.id", - "sonar.core.serverBaseURL", - "sonar.core.startTime", - "sonar.dbcleaner.branchesToKeepWhenInactive", - "sonar.forceAuthentication", - "sonar.host.url", - "sonar.java.jdkHome", - "sonar.links.ci", - "sonar.links.homepage", - "sonar.links.issue", - "sonar.links.scm", - "sonar.links.scm_dev", - "sonar.plugins.risk.consent", -) - -_UNNEEDED_TASK_DATA = ( - "analysisId", - "componentId", - "hasScannerContext", - "id", - "warningCount", - "componentQualifier", - "nodeName", - "componentName", - "componentKey", - "submittedAt", - "executedAt", - "type", -) # Keys to exclude when applying settings in update() _SETTINGS_WITH_SPECIFIC_IMPORT = ( @@ -980,10 +942,10 @@ def migration_export(self, export_settings: types.ConfigSettings) -> types.Objec if last_task: ctxt = last_task.scanner_context() if ctxt: - ctxt = {k: v for k, v in ctxt.items() if k not in _UNNEEDED_CONTEXT_DATA} + ctxt = {k: v for k, v in ctxt.items() if k not in phelp.UNNEEDED_CONTEXT_DATA} t_hist = [] for t in self.task_history(): - t_hist.append({k: v for k, v in t.sq_json.items() if k not in _UNNEEDED_TASK_DATA}) + t_hist.append({k: v for k, v in t.sq_json.items() if k not in phelp.UNNEEDED_TASK_DATA}) json_data["backgroundTasks"] = { "lastTaskScannerContext": ctxt, # "lastTaskWarnings": last_task.warnings(), @@ -1003,15 +965,12 @@ def export(self, export_settings: types.ConfigSettings, settings_list: Optional[ json_data.update({"key": self.key, "name": self.name}) try: json_data["binding"] = self.__export_get_binding() - nc = self.new_code() - if nc != "": - json_data[settings.NEW_CODE_PERIOD] = nc - json_data["qualityProfiles"] = util.dict_to_list(self.__export_get_qp(), "language", "name") + json_data["qualityProfiles"] = self.__export_get_qp() json_data["links"] = self.links() - json_data["permissions"] = util.perms_to_list(self.permissions().to_json(csv=export_settings.get("INLINE_LISTS", True))) + json_data["permissions"] = self.permissions().to_json(csv=export_settings.get("INLINE_LISTS", True)) if self.endpoint.version() >= (10, 7, 0): - json_data["aiCodeFix"] = self.ai_code_fix() - json_data["branches"] = util.dict_to_list(self.__get_branch_export(export_settings), "name") + json_data[phelp.AI_CODE_FIX] = self.ai_code_fix() + json_data["branches"] = self.__get_branch_export(export_settings) json_data["tags"] = self.get_tags() json_data["visibility"] = self.visibility() (json_data["qualityGate"], qg_is_default) = self.quality_gate() @@ -1040,7 +999,11 @@ def export(self, export_settings: types.ConfigSettings, settings_list: Optional[ if contains_ai: json_data[_CONTAINS_AI_CODE] = contains_ai with_inherited = export_settings.get("INCLUDE_INHERITED", False) - json_data["settings"] = [s.to_json() for s in settings_dict.values() if with_inherited or not s.inherited and s.key != "visibility"] + json_data["settings"] = {} + settings_to_export = {k: s for k, s in settings_dict.items() if with_inherited or not s.inherited and s.key != "visibility"} + for k, s in settings_to_export.items(): + json_data["settings"] |= s.to_json() + return phelp.convert_project_json(json_data) except Exception as e: traceback.print_exc() @@ -1343,9 +1306,6 @@ def count(endpoint: pf.Platform, params: types.ApiParams = None) -> int: """Counts projects :param params: list of parameters to filter projects to search - :type params: dict - :return: Count of projects - :rtype: int """ new_params = {} if params is None else params.copy() new_params.update({"ps": 1, "p": 1}) @@ -1696,23 +1656,3 @@ def import_zips(endpoint: pf.Platform, project_list: list[str], threads: int = 2 log.info("%d/%d imports (%d%%) - Latest: %s - %s", i, nb_projects, int(i * 100 / nb_projects), proj_key, status) log.info("%s", ", ".join([f"{k}:{v}" for k, v in statuses_count.items()])) return statuses - - -def old_to_new_json_one(old_json: dict[str, Any]) -> dict[str, Any]: - """Converts the sonar-config projects old JSON report format for a single project to the new one""" - new_json = old_json.copy() - if "permissions" in old_json: - new_json["permissions"] = util.perms_to_list(old_json["permissions"]) - if "branches" in old_json: - new_json["branches"] = util.dict_to_list(old_json["branches"], "name") - if "settings" in old_json: - new_json["settings"] = util.dict_to_list(old_json["settings"], "key") - return new_json - - -def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]: - """Converts the sonar-config projects old JSON report format to the new one""" - new_json = old_json.copy() - for k, v in new_json.items(): - new_json[k] = old_to_new_json_one(v) - return util.dict_to_list(new_json, "key") diff --git a/sonar/qualitygates.py b/sonar/qualitygates.py index 89f0a921..cfe90fcc 100644 --- a/sonar/qualitygates.py +++ b/sonar/qualitygates.py @@ -38,6 +38,7 @@ from sonar.audit.rules import get_rule, RuleId from sonar.audit.problem import Problem +from sonar.util import common_json_helper __MAX_ISSUES_SHOULD_BE_ZERO = "Any numeric threshold on number of issues should be 0 or should be removed from QG conditions" @@ -563,6 +564,7 @@ def search_by_name(endpoint: pf.Platform, name: str) -> dict[str, QualityGate]: return util.search_by_name(endpoint, name, QualityGate.API[c.LIST], "qualitygates") -def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]: +def convert_qgs_json(old_json: dict[str, Any]) -> dict[str, Any]: """Converts the sonar-config quality gates old JSON report format to the new one""" + old_json = common_json_helper.convert_common_fields(old_json, with_permissions=False) return util.dict_to_list(old_json, "name") diff --git a/sonar/qualityprofiles.py b/sonar/qualityprofiles.py index 1ef59f60..26ea2641 100644 --- a/sonar/qualityprofiles.py +++ b/sonar/qualityprofiles.py @@ -32,6 +32,7 @@ import sonar.logging as log import sonar.platform as pf from sonar.util import types, cache, constants as c +from sonar.util import qualityprofile_helper as qphelp from sonar import exceptions from sonar import rules, languages import sonar.permissions.qualityprofile_permissions as permissions @@ -41,9 +42,6 @@ from sonar.audit.rules import get_rule, RuleId from sonar.audit.problem import Problem -_KEY_PARENT = "parent" -_CHILDREN_KEY = "children" - _IMPORTABLE_PROPERTIES = ("name", "language", "parentName", "isBuiltIn", "isDefault", "rules", "permissions", "prioritizedRules") _CLASS_LOCK = Lock() @@ -363,7 +361,7 @@ def update(self, data: types.ObjectJsonRepr) -> QualityProfile: QualityProfile.CACHE.pop(self) self.name = data["name"] QualityProfile.CACHE.put(self) - self.set_parent(data.pop(_KEY_PARENT, None)) + self.set_parent(data.pop(qphelp.KEY_PARENT, None)) self.set_rules(data.get("rules", []) + data.get("addedRules", [])) self.activate_rules(data.get("modifiedRules", [])) self.set_permissions(data.get("permissions", [])) @@ -371,12 +369,12 @@ def update(self, data: types.ObjectJsonRepr) -> QualityProfile: if data.get("isDefault", False): self.set_as_default() - for child_name, child_data in data.get(_CHILDREN_KEY, {}).items(): + for child_name, child_data in data.get(qphelp.KEY_CHILDREN, {}).items(): try: child_qp = get_object(self.endpoint, child_name, self.language) except exceptions.ObjectNotFound: child_qp = QualityProfile.create(self.endpoint, child_name, self.language) - child_qp.update(child_data | {_KEY_PARENT: self.name}) + child_qp.update(child_data | {qphelp.KEY_PARENT: self.name}) return self def to_json(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: @@ -394,7 +392,9 @@ def to_json(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr json_data["rules"] = [] for rule in self.rules().values(): data = { - k: v for k, v in rule.export(full).items() if k not in ("isTemplate", "templateKey", "language", "tags", "severities", "impacts") + k: v + for k, v in rule.export(full).items() + if k not in ("isTemplate", "templateKey", "language", "tags", "severity", "severities", "impacts") } if self.rule_is_prioritized(rule.key): data["prioritized"] = True @@ -430,7 +430,6 @@ def rule_impacts(self, rule_key: str, substitute_with_default: bool = True) -> d :param str rule_key: The rule key to get severities for :return: The impacts of the rule in the quality profile - :rtype: dict[str, str] """ return rules.Rule.get_object(self.endpoint, rule_key).impacts(self.key, substitute_with_default=substitute_with_default) @@ -439,7 +438,6 @@ def rule_severity(self, rule_key: str, substitute_with_default: bool = True) -> :param str rule_key: The rule key to get severities for :return: The severity - :rtype: str """ return rules.Rule.get_object(self.endpoint, rule_key).rule_severity(self.key, substitute_with_default=substitute_with_default) @@ -742,12 +740,12 @@ def hierarchize_language(qp_list: dict[str, str], endpoint: pf.Platform, languag continue parent_qp_name = qp_json_data.pop("parentName") parent_qp = hierarchy[parent_qp_name] - if _CHILDREN_KEY not in parent_qp: - parent_qp[_CHILDREN_KEY] = {} + if qphelp.KEY_CHILDREN not in parent_qp: + parent_qp[qphelp.KEY_CHILDREN] = {} this_qp = get_object(endpoint=endpoint, name=qp_name, language=language) qp_json_data |= this_qp.diff(get_object(endpoint=endpoint, name=parent_qp_name, language=language)) qp_json_data.pop("rules", None) - parent_qp[_CHILDREN_KEY][qp_name] = qp_json_data + parent_qp[qphelp.KEY_CHILDREN][qp_name] = qp_json_data to_remove.append(qp_name) for qp_name in to_remove: hierarchy.pop(qp_name) @@ -766,29 +764,6 @@ def hierarchize(qp_list: types.ObjectJsonRepr, endpoint: pf.Platform) -> types.O return {lang: hierarchize_language(lang_qp_list, endpoint=endpoint, language=lang) for lang, lang_qp_list in qp_list.items()} -def flatten_language(language: str, qp_list: types.ObjectJsonRepr) -> types.ObjectJsonRepr: - """Converts a hierarchical list of QP of a given language into a flat list""" - flat_list = {} - for qp_name, qp_data in qp_list.copy().items(): - if _CHILDREN_KEY in qp_data: - children = flatten_language(language, qp_data[_CHILDREN_KEY]) - for child in children.values(): - if "parent" not in child: - child["parent"] = f"{language}:{qp_name}" - qp_data.pop(_CHILDREN_KEY) - flat_list.update(children) - flat_list[f"{language}:{qp_name}"] = qp_data - return flat_list - - -def flatten(qp_list: types.ObjectJsonRepr) -> types.ObjectJsonRepr: - """Organize a hierarchical list of QP in a flat list""" - flat_list = {} - for lang, lang_qp_list in qp_list.items(): - flat_list.update(flatten_language(lang, lang_qp_list)) - return flat_list - - def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, **kwargs) -> types.ObjectJsonRepr: """Exports all or a list of quality profiles configuration as dict @@ -807,7 +782,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, **kwarg qp_list[lang] = {} qp_list[lang][name] = json_data qp_list = hierarchize(qp_list, endpoint=endpoint) - qp_list = __convert_profiles_to_list(qp_list) + qp_list = qphelp.convert_qps_json(qp_list) if write_q := kwargs.get("write_q", None): write_q.put(qp_list) write_q.put(util.WRITE_END) @@ -893,27 +868,3 @@ def exists(endpoint: pf.Platform, name: str, language: str) -> bool: return True except exceptions.ObjectNotFound: return False - - -def convert_one_qp_yaml(qp: types.ObjectJsonRepr) -> types.ObjectJsonRepr: - """Converts a QP in a modified version more suitable for YAML export""" - - if _CHILDREN_KEY in qp: - qp[_CHILDREN_KEY] = {k: convert_one_qp_yaml(q) for k, q in qp[_CHILDREN_KEY].items()} - qp[_CHILDREN_KEY] = util.dict_to_list(qp[_CHILDREN_KEY], "name") - return qp - - -def __convert_children_to_list(qp_json: dict[str, Any]) -> list[dict[str, Any]]: - """Converts a profile's children profiles to list""" - for v in qp_json.values(): - if "children" in v: - v["children"] = __convert_children_to_list(v["children"]) - return util.dict_to_list(qp_json, "name") - - -def old_to_new_json(qp_json: dict[str, Any]) -> list[dict[str, Any]]: - """Converts a language top level list of profiles to list""" - for k, v in qp_json.items(): - qp_json[k] = __convert_children_to_list(v) - return util.dict_to_list(qp_json, "language", "profiles") diff --git a/sonar/rules.py b/sonar/rules.py index f4e6e3c8..92183fac 100644 --- a/sonar/rules.py +++ b/sonar/rules.py @@ -34,6 +34,7 @@ import sonar.sqobject as sq from sonar.util import types, cache, constants as c, issue_defs as idefs from sonar import platform, utilities, exceptions, languages +from sonar.util import rule_helper as rhelp TYPE_TO_QUALITY = { idefs.TYPE_BUG: idefs.QUALITY_RELIABILITY, @@ -486,7 +487,7 @@ def export(endpoint: platform.Platform, export_settings: types.ConfigSettings, * for k in ("instantiated", "extended", "standard", "thirdParty"): if len(rule_list.get(k, {})) == 0: rule_list.pop(k, None) - rule_list = old_to_new_json(rule_list) + rule_list = rhelp.convert_rules_json(rule_list) if write_q := kwargs.get("write_q", None): write_q.put(rule_list) write_q.put(utilities.WRITE_END) @@ -579,12 +580,3 @@ def severities(endpoint: platform.Platform, json_data: dict[str, any]) -> Option return {impact["softwareQuality"]: impact["severity"] for impact in json_data.get("impacts", [])} else: return json_data.get("severity", None) - - -def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]: - """Converts the sonar-config rules old JSON report format to the new one""" - new_json = {} - for k in ("instantiated", "extended", "standard", "thirdParty"): - if k in old_json: - new_json[k] = utilities.dict_to_list(old_json[k], "key") - return new_json diff --git a/sonar/users.py b/sonar/users.py index cdd3bede..4412c0a7 100644 --- a/sonar/users.py +++ b/sonar/users.py @@ -42,7 +42,7 @@ _GROUPS_API_SC = "users/groups" -SETTABLE_PROPERTIES = ("login", "name", "scmAccounts", "email", "groups", "local") +SETTABLE_PROPERTIES = ("login", "name", "email", "groups", "scmAccounts", "local") USER_API = "v2/users-management/users" @@ -448,7 +448,8 @@ def to_json(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr json_data.pop("local") for key in "sonarQubeLastConnectionDate", "externalLogin", "externalProvider", "id", "managed": json_data.pop(key, None) - return util.filter_export(json_data, SETTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False)) + json_data = util.filter_export(json_data, SETTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False)) + return convert_user_json(json_data) def search(endpoint: pf.Platform, params: types.ApiParams = None) -> dict[str, User]: @@ -570,6 +571,16 @@ def exists(endpoint: pf.Platform, login: str) -> bool: return User.get_object(endpoint=endpoint, login=login) is not None -def old_to_new_json(old_json: dict[str, Any]) -> dict[str, Any]: +def convert_user_json(old_json: dict[str, Any]) -> dict[str, Any]: + """Converts a user JSON from old to new format""" + for k in "groups", "scmAccounts": + if k in old_json: + old_json[k] = util.csv_to_list(old_json[k]) + return util.order_dict(old_json, SETTABLE_PROPERTIES) + + +def convert_users_json(old_json: dict[str, Any]) -> dict[str, Any]: """Converts the sonar-config users old JSON report format to the new one""" + for k, u in old_json.items(): + old_json[k] = convert_user_json(u) return util.dict_to_list(old_json, "login") diff --git a/sonar/util/common_json_helper.py b/sonar/util/common_json_helper.py new file mode 100644 index 00000000..c8b3c7d7 --- /dev/null +++ b/sonar/util/common_json_helper.py @@ -0,0 +1,37 @@ +# +# sonar-tools +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +"""Helper tools for the Rule object""" + +from typing import Any +from sonar import utilities +from sonar import logging as log + + +def convert_common_fields(json_data: dict[str, Any], with_permissions: bool = True) -> dict[str, Any]: + """Converts Sonar objects common fields from old to new JSON format""" + if "permissions" in json_data and json_data["permissions"] is None: + json_data.pop("permissions") + if with_permissions and "permissions" in json_data: + json_data["permissions"] = utilities.perms_to_list(json_data["permissions"]) + for perm in json_data["permissions"]: + perm["permissions"] = utilities.csv_to_list(perm["permissions"]) + if "tags" in json_data: + json_data["tags"] = utilities.csv_to_list(json_data["tags"]) + return json_data diff --git a/sonar/util/platform_helper.py b/sonar/util/platform_helper.py new file mode 100644 index 00000000..6325e2c4 --- /dev/null +++ b/sonar/util/platform_helper.py @@ -0,0 +1,72 @@ +# +# sonar-tools +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +"""Helper tools for the Platform object""" + +from typing import Any +from sonar import settings +from sonar import utilities as util +from sonar.util import common_json_helper + +_PERM_TPL_IMPORTABLE_PROPERTIES = ("name", "description", "pattern", "defaultFor", "permissions") + + +def normalize_api(api: str) -> str: + """Normalizes an API based on its multiple original forms""" + if api.startswith("/api/"): + pass + elif api.startswith("api/"): + api = "/" + api + elif api.startswith("/"): + api = "/api" + api + else: + api = "/api/" + api + return api + + +def convert_basics_json(old_json: dict[str, Any]) -> dict[str, Any]: + """Converts sonar-config "plaform" section old JSON report format to new format""" + if "plugins" in old_json: + old_json["plugins"] = util.dict_to_list(old_json["plugins"], "key") + return old_json + + +def convert_template_json(json_data: dict[str, Any], full: bool = False) -> dict[str, Any]: + """Converts a Perm Template JSON from old to new format""" + json_data = common_json_helper.convert_common_fields(json_data) + return util.remove_nones(util.filter_export(json_data, _PERM_TPL_IMPORTABLE_PROPERTIES, full)) + + +def convert_global_settings_json(old_json: dict[str, Any], full: bool = False) -> dict[str, Any]: + """Converts sonar-config "globalSettings" section old JSON report format to new format""" + new_json = old_json.copy() + special_categories = (settings.LANGUAGES_SETTINGS, settings.DEVOPS_INTEGRATION, "permissions", "permissionTemplates") + for categ in [cat for cat in settings.CATEGORIES if cat not in special_categories]: + new_json[categ] = util.sort_list_by_key(util.dict_to_list(dict(sorted(old_json[categ].items())), "key"), "key") + for k, v in old_json[settings.LANGUAGES_SETTINGS].items(): + new_json[settings.LANGUAGES_SETTINGS] = new_json.get(settings.LANGUAGES_SETTINGS, None) or {} + new_json[settings.LANGUAGES_SETTINGS][k] = util.sort_list_by_key(util.dict_to_list(v, "key"), "key") + new_json[settings.LANGUAGES_SETTINGS] = util.dict_to_list(dict(sorted(new_json[settings.LANGUAGES_SETTINGS].items())), "language", "settings") + new_json[settings.DEVOPS_INTEGRATION] = util.dict_to_list(dict(sorted(old_json[settings.DEVOPS_INTEGRATION].items())), "key") + for k, v in new_json["permissionTemplates"].items(): + new_json["permissionTemplates"][k] = convert_template_json(new_json["permissionTemplates"][k], full) + new_json["permissionTemplates"] = util.dict_to_list(new_json["permissionTemplates"], "key") + new_json = common_json_helper.convert_common_fields(new_json) + + return util.order_dict(new_json, [*settings.CATEGORIES, "permissions", "permissionTemplates"]) diff --git a/sonar/util/portfolio_helper.py b/sonar/util/portfolio_helper.py new file mode 100644 index 00000000..68397d52 --- /dev/null +++ b/sonar/util/portfolio_helper.py @@ -0,0 +1,44 @@ +# +# sonar-tools +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +"""Helper tools for the Portfolio object""" + +from typing import Any +from sonar import utilities as util +from sonar.util import common_json_helper + + +def convert_portfolio_json(old_json: dict[str, Any]) -> dict[str, Any]: + """Converts the sonar-config old JSON report format for a single portfolio to the new one""" + new_json = common_json_helper.convert_common_fields(old_json.copy()) + if "projects" in new_json: + new_json["projects"] = common_json_helper.convert_common_fields(new_json["projects"]) + for key in "children", "portfolios": + if key in new_json: + new_json[key] = convert_portfolios_json(new_json[key]) + if "branches" in old_json: + new_json["branches"] = util.dict_to_list(old_json["branches"], "name") + return new_json + + +def convert_portfolios_json(old_json: dict[str, Any]) -> dict[str, Any]: + """Converts the sonar-config portfolios old JSON report format to the new one""" + for k, v in old_json.items(): + old_json[k] = convert_portfolio_json(v) + return util.dict_to_list(old_json, "key") diff --git a/sonar/util/project_helper.py b/sonar/util/project_helper.py new file mode 100644 index 00000000..0c929e07 --- /dev/null +++ b/sonar/util/project_helper.py @@ -0,0 +1,109 @@ +# +# sonar-tools +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +"""Helper tools for the Project object""" + +from typing import Any +from sonar import utilities as util +from sonar.util import common_json_helper + +UNNEEDED_TASK_DATA = ( + "analysisId", + "componentId", + "hasScannerContext", + "id", + "warningCount", + "componentQualifier", + "nodeName", + "componentName", + "componentKey", + "submittedAt", + "executedAt", + "type", +) + +UNNEEDED_CONTEXT_DATA = ( + "sonar.announcement.message", + "sonar.auth.github.allowUsersToSignUp", + "sonar.auth.github.apiUrl", + "sonar.auth.github.appId", + "sonar.auth.github.enabled", + "sonar.auth.github.groupsSync", + "sonar.auth.github.organizations", + "sonar.auth.github.webUrl", + "sonar.builtInQualityProfiles.disableNotificationOnUpdate", + "sonar.core.id", + "sonar.core.serverBaseURL", + "sonar.core.startTime", + "sonar.dbcleaner.branchesToKeepWhenInactive", + "sonar.forceAuthentication", + "sonar.host.url", + "sonar.java.jdkHome", + "sonar.links.ci", + "sonar.links.homepage", + "sonar.links.issue", + "sonar.links.scm", + "sonar.links.scm_dev", + "sonar.plugins.risk.consent", +) + +AI_CODE_FIX = "aiCodeFix" + +_JSON_KEY_ORDER = ( + "key", + "name", + "tags", + "visibility", + "settings", + "binding", + "branches", + "permissions", + "qualityGate", + "qualityProfiles", + "links", + "webhooks", +) + + +def convert_project_json(old_json: dict[str, Any]) -> dict[str, Any]: + """Converts the sonar-config projects old JSON report format for a single project to the new one""" + new_json = old_json.copy() + if "qualityProfiles" in old_json: + new_json["qualityProfiles"] = util.dict_to_list(old_json["qualityProfiles"], "language", "name") + if "branches" in old_json: + new_json["branches"] = util.dict_to_list(old_json["branches"], "name") + for k, v in old_json.items(): + if k not in _JSON_KEY_ORDER: + new_json.pop(k, None) + new_json["settings"] = new_json.get("settings", None) or {} + new_json["settings"][k] = v + if "settings" in new_json: + if AI_CODE_FIX in new_json["settings"] and not new_json["settings"][AI_CODE_FIX]: + new_json["settings"].pop(AI_CODE_FIX) + new_json["settings"] = util.dict_to_list(dict(sorted(new_json["settings"].items())), "key", "value") + new_json = common_json_helper.convert_common_fields(new_json) + return util.order_dict(new_json, _JSON_KEY_ORDER) + + +def convert_projects_json(old_json: dict[str, Any]) -> dict[str, Any]: + """Converts the sonar-config projects old JSON report format to the new one""" + new_json = old_json.copy() + for k, v in new_json.items(): + new_json[k] = convert_project_json(v) + return util.dict_to_list(new_json, "key") diff --git a/sonar/util/qualityprofile_helper.py b/sonar/util/qualityprofile_helper.py new file mode 100644 index 00000000..2ad72c5a --- /dev/null +++ b/sonar/util/qualityprofile_helper.py @@ -0,0 +1,74 @@ +# +# sonar-tools +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +"""Helper tools for the Project object""" + +from typing import Any +from sonar import utilities as util +from sonar.util import types +from sonar.util import common_json_helper + +KEY_PARENT = "parent" +KEY_CHILDREN = "children" +KEY_ORDER = ("name", "isBuiltIn", "isDefault", "children", "addedRules", "modifiedRules", "permissions") + + +def flatten_language(language: str, qp_list: types.ObjectJsonRepr) -> types.ObjectJsonRepr: + """Converts a hierarchical list of QP of a given language into a flat list""" + flat_list = {} + for qp_name, qp_data in qp_list.copy().items(): + if KEY_CHILDREN in qp_data: + children = flatten_language(language, qp_data[KEY_CHILDREN]) + for child in children.values(): + if "parent" not in child: + child["parent"] = f"{language}:{qp_name}" + qp_data.pop(KEY_CHILDREN) + flat_list.update(children) + flat_list[f"{language}:{qp_name}"] = qp_data + return flat_list + + +def flatten(qp_list: types.ObjectJsonRepr) -> types.ObjectJsonRepr: + """Organize a hierarchical list of QP in a flat list""" + flat_list = {} + for lang, lang_qp_list in qp_list.items(): + flat_list.update(flatten_language(lang, lang_qp_list)) + return flat_list + + +def __convert_qp_json(qp_json: dict[str, Any]) -> list[dict[str, Any]]: + """Converts a profile's children profiles to list""" + + for k, v in sorted(qp_json.items()): + for rtype in "addedRules", "modifiedRules": + for r in v.get(rtype, {}): + if "severities" in r: + r["impacts"] = r["severities"] + r.pop("severities") + if "children" in v: + v["children"] = __convert_qp_json(v["children"]) + qp_json[k] = util.order_keys(common_json_helper.convert_common_fields(v, with_permissions=False), *KEY_ORDER) + return util.dict_to_list(qp_json, "name") + + +def convert_qps_json(qp_json: dict[str, Any]) -> list[dict[str, Any]]: + """Converts a language top level list of profiles to list""" + for k, v in sorted(qp_json.items()): + qp_json[k] = __convert_qp_json(v) + return util.dict_to_list(qp_json, "language", "profiles") diff --git a/sonar/util/rule_helper.py b/sonar/util/rule_helper.py new file mode 100644 index 00000000..55b8c14f --- /dev/null +++ b/sonar/util/rule_helper.py @@ -0,0 +1,40 @@ +# +# sonar-tools +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +"""Helper tools for the Rule object""" + +from typing import Any +from sonar import utilities +from sonar.util import common_json_helper + + +def convert_rule_json(old_json: dict[str, Any]) -> dict[str, Any]: + """Converts a rule JSON from old to new export format""" + return common_json_helper.convert_common_fields(old_json) + + +def convert_rules_json(old_json: dict[str, Any]) -> dict[str, Any]: + """Converts the sonar-config rules old JSON report format to the new one""" + new_json = {} + for k in ("instantiated", "extended", "standard", "thirdParty"): + if k in old_json: + for r in old_json[k].values(): + r = convert_rule_json(r) + new_json[k] = utilities.dict_to_list(dict(sorted(old_json[k].items())), "key") + return new_json diff --git a/sonar/utilities.py b/sonar/utilities.py index f7befb7f..4c552030 100644 --- a/sonar/utilities.py +++ b/sonar/utilities.py @@ -200,6 +200,8 @@ def remove_nones(d: Any) -> Any: def clean_data(d: Any, remove_empty: bool = True, remove_none: bool = True) -> Any: """Recursively removes empty lists and dicts and none from a dict""" # log.debug("Cleaning up %s", json_dump(d)) + if isinstance(d, str): + return convert_string(d) if not isinstance(d, (list, dict)): return d @@ -857,6 +859,6 @@ def order_list(list_to_order: list[str], *key_order: str) -> list[str]: def perms_to_list(perms: dict[str, Any]) -> list[str, Any]: """Converts permissions in dict format to list format""" - if not perms: + if not perms or not isinstance(perms, dict): return perms return dict_to_list(perms.get("groups", {}), "group", "permissions") + dict_to_list(perms.get("users", {}), "user", "permissions")