Merge pull request #1289 from okorach:improve-yaml-2

Improve YAML output

Author: okorach

cli/config.py

@@ -108,11 +108,34 @@ def __write_export(config: dict[str, str], file: str, format: str) -> None:
     """Writes the configuration in file"""
     with utilities.open_file(file) as fd:
         if format == "yaml":
-            print(yaml.dump(config), file=fd)
+            print(yaml.dump(__convert_for_yaml(config), sort_keys=False), file=fd)
         else:
             print(utilities.json_dump(config), file=fd)
 
 
+def __convert_for_yaml(json_export: dict[str, any]) -> dict[str, any]:
+    """Converts the default JSON produced by export to a modified version more suitable for YAML"""
+    if "globalSettings" in json_export:
+        json_export["globalSettings"] = platform.convert_for_yaml(json_export["globalSettings"])
+    if "qualityGates" in json_export:
+        json_export["qualityGates"] = qualitygates.convert_for_yaml(json_export["qualityGates"])
+    if "qualityProfiles" in json_export:
+        json_export["qualityProfiles"] = qualityprofiles.convert_for_yaml(json_export["qualityProfiles"])
+    if "projects" in json_export:
+        json_export["projects"] = projects.convert_for_yaml(json_export["projects"])
+    if "portfolios" in json_export:
+        json_export["portfolios"] = portfolios.convert_for_yaml(json_export["portfolios"])
+    if "applications" in json_export:
+        json_export["applications"] = applications.convert_for_yaml(json_export["applications"])
+    if "users" in json_export:
+        json_export["users"] = users.convert_for_yaml(json_export["users"])
+    if "groups" in json_export:
+        json_export["groups"] = groups.convert_for_yaml(json_export["groups"])
+    if "rules" in json_export:
+        json_export["rules"] = rules.convert_for_yaml(json_export["rules"])
+    return json_export
+
+
 def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
     """Exports a platform configuration in a JSON file"""
     export_settings = {

doc/what-is-new.md

@@ -1,5 +1,8 @@
 # Next version yet unreleased
 
+- `sonar-tools` is now available as a docker image
+- `sonar-config` export can now export configuration as a YAML file (Only JSON was available previously). Import of YAML is not yet available
+
 # Version 3.3
 
 - `sonar-config`: Improved / Hardened several elements for both import and export

sonar/applications.py

@@ -512,7 +512,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
     for k in apps_settings:
         # remove key from JSON value, it's already the dict key
         apps_settings[k].pop("key")
-    return apps_settings
+    return dict(sorted(apps_settings.items()))
 
 
 def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings, key_list: types.KeyList = None) -> list[problem.Problem]:
@@ -576,3 +576,16 @@ def search_by_name(endpoint: pf.Platform, name: str) -> dict[str, Application]:
             data[app.key] = app
     # return {app.key: app for app in _OBJECTS.values() if app.name == name}
     return data
+
+
+def convert_for_yaml(original_json: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
+    """Convert the original JSON defined for JSON export into a JSON format more adapted for YAML export"""
+    new_json = util.dict_to_list(original_json, "key")
+    for app_json in new_json:
+        app_json["branches"] = util.dict_to_list(app_json["branches"], "name")
+        for b in app_json["branches"]:
+            if "projects" in b:
+                b["projects"] = [{"key": k, "branch": br} for k, br in b["projects"].items()]
+        if "permissions" in app_json:
+            app_json["permissions"] = permissions.convert_for_yaml(app_json["permissions"])
+    return new_json

sonar/devops.py

@@ -153,8 +153,8 @@ def to_json(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr
         :return: The configuration of the DevOps platform (except secrets)
         :rtype: dict
         """
-        json_data = self._json.copy()
-        json_data.update({"key": self.key, "type": self.type, "url": self.url})
+        json_data = {"key": self.key, "type": self.type, "url": self.url}
+        json_data.update(self._json.copy())
         return util.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False))
 
     def set_pat(self, pat, user_name=None):

sonar/groups.py

@@ -267,7 +267,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings) -> type
     """
     log.info("Exporting groups")
     g_list = {}
-    for g_name, g_obj in search(endpoint=endpoint).items():
+    for g_name, g_obj in sorted(search(endpoint=endpoint).items()):
         if not export_settings["FULL_EXPORT"] and g_obj.is_default():
             continue
         g_list[g_name] = "" if g_obj.description is None else g_obj.description
@@ -365,3 +365,8 @@ def exists(group_name: str, endpoint: pf.Platform) -> bool:
     :rtype: bool
     """
     return get_object(name=group_name, endpoint=endpoint) is not None
+
+
+def convert_for_yaml(original_json: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
+    """Convert the original JSON defined for JSON export into a JSON format more adapted for YAML export"""
+    return util.dict_to_list(original_json, "name", "description")

sonar/permissions/permissions.py

@@ -382,3 +382,12 @@ def black_list(perms: types.JsonPermissions, disallowed_perms: list[str]) -> typ
         for user_or_group, original_perms in sub_perms.items():
             resulting_perms[perm_type][user_or_group] = [p for p in original_perms if p not in disallowed_perms]
     return resulting_perms
+
+
+def convert_for_yaml(json_perms: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
+    """Converts permissions in a format that is more friendly for YAML"""
+    converted_perms = []
+    for ptype in "groups", "users":
+        if ptype in json_perms:
+            converted_perms += utilities.dict_to_list(json_perms[ptype], ptype[:-1], "permissions")
+    return converted_perms

sonar/platform.py

@@ -820,3 +820,19 @@ def _check_for_retry(response: requests.models.Response) -> tuple[bool, str]:
         log.debug("Moved permanently to URL %s", new_url)
         return True, new_url
     return False, None
+
+
+def convert_for_yaml(original_json: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
+    """Convert the original JSON defined for JSON export into a JSON format more adapted for YAML export"""
+    if "languages" in original_json:
+        original_json["languages"] = util.dict_to_list(original_json["languages"], "language")
+    if "permissions" in original_json:
+        original_json["permissions"] = permissions.convert_for_yaml(original_json["permissions"])
+    if "permissionTemplates" in original_json:
+        for tpl in original_json["permissionTemplates"].values():
+            if "permissions" in tpl:
+                tpl["permissions"] = permissions.convert_for_yaml(tpl["permissions"])
+        original_json["permissionTemplates"] = util.dict_to_list(original_json["permissionTemplates"], "name")
+    if "devopsIntegration" in original_json:
+        original_json["devopsIntegration"] = util.dict_to_list(original_json["devopsIntegration"], "name")
+    return original_json

sonar/portfolios.py

@@ -336,26 +336,24 @@ def audit(self, audit_settings: types.ConfigSettings) -> list[problem.Problem]:
     def to_json(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
         """Returns the portfolio representation as JSON"""
         self.refresh()
-        json_data = {}
+        json_data = {"key": self.key, "name": self.name}
+        if self._description:
+            json_data["description"] = self._description
         subportfolios = self.sub_portfolios()
+        if not self.is_sub_portfolio:
+            json_data["visibility"] = self._visibility
+            json_data["permissions"] = self.permissions().export(export_settings=export_settings)
+        json_data["tags"] = self._tags
         if subportfolios:
             json_data["portfolios"] = {}
             for s in subportfolios.values():
                 subp_json = s.to_json(export_settings)
                 subp_key = subp_json.pop("key")
                 json_data["portfolios"][subp_key] = subp_json
-        if not self.is_sub_portfolio:
-            json_data["permissions"] = self.permissions().export(export_settings=export_settings)
-            json_data["visibility"] = self._visibility
-        json_data["key"] = self.key
-        json_data["name"] = self.name
-        json_data["tags"] = self._tags
-        json_data["applications"] = self._applications
-        if self._description:
-            json_data["description"] = self._description
         mode = self.selection_mode().copy()
         if mode and "none" not in mode:
             json_data["projects"] = mode
+        json_data["applications"] = self._applications
         return json_data
 
     def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
@@ -744,7 +742,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
         nb_portfolios = count(endpoint=endpoint)
     i = 0
     exported_portfolios = {}
-    for k, p in get_list(endpoint=endpoint, key_list=key_list).items():
+    for k, p in sorted(get_list(endpoint=endpoint, key_list=key_list).items()):
         if not p.is_sub_portfolio:
             exported_portfolios[k] = p.export(export_settings)
             exported_portfolios[k].pop("key")
@@ -797,3 +795,20 @@ def __create_portfolio_hierarchy(endpoint: pf.Platform, data: types.ApiPayload,
 def get_api_branch(branch: str) -> str:
     """Returns the value to pass to the API for the branch parameter"""
     return branch if branch != settings.DEFAULT_BRANCH else None
+
+
+def convert_for_yaml(original_json: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
+    """Convert the original JSON defined for JSON export into a JSON format more adapted for YAML export"""
+    new_json = util.dict_to_list(original_json, "key")
+    for p_json in new_json:
+        try:
+            p_json["projects"] = [{"key": k, "branch": br} for k, br in p_json["projects"]["manual"].items()]
+        except KeyError:
+            pass
+        if "portfolios" in p_json:
+            p_json["portfolios"] = convert_for_yaml(p_json["portfolios"])
+        if "applications" in p_json:
+            p_json["applications"] = [{"key": k, "branches": br} for k, br in p_json["applications"].items()]
+        if "permissions" in p_json:
+            p_json["permissions"] = perms.convert_for_yaml(p_json["permissions"])
+    return new_json

sonar/projects.py

@@ -1385,7 +1385,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
         worker.setName(f"ProjectExport{i}")
         worker.start()
     q.join()
-    return project_settings
+    return dict(sorted(project_settings.items()))
 
 
 def exists(key: str, endpoint: pf.Platform) -> bool:
@@ -1489,3 +1489,22 @@ def export_zip(endpoint: pf.Platform, key_list: types.KeyList = None, threads: i
         },
         "project_exports": exports,
     }
+
+
+def convert_proj_for_yaml(proj_json: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
+    """Convert the original JSON defined for JSON export into a JSON format more adapted for YAML export"""
+    if "branches" in proj_json:
+        proj_json["branches"] = util.dict_to_list(proj_json["branches"], "name")
+    if "qualityProfiles" in proj_json:
+        proj_json["qualityProfiles"] = util.dict_to_list(proj_json["qualityProfiles"], "language", "name")
+    if "permissions" in proj_json:
+        proj_json["permissions"] = perms.convert_for_yaml(proj_json["permissions"])
+    return proj_json
+
+
+def convert_for_yaml(original_json: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
+    """Convert the original JSON defined for JSON export into a JSON format more adapted for YAML export"""
+    new_json = []
+    for proj in util.dict_to_list(original_json, "key"):
+        new_json.append(convert_proj_for_yaml(proj))
+    return new_json

sonar/qualitygates.py

@@ -380,7 +380,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings) -> type
     :rtype: dict
     """
     log.info("Exporting quality gates")
-    return {k: qg.to_json(export_settings) for k, qg in get_list(endpoint).items()}
+    return {k: qg.to_json(export_settings) for k, qg in sorted(get_list(endpoint).items())}
 
 
 def import_config(endpoint: pf.Platform, config_data: types.ObjectJsonRepr) -> bool:
@@ -465,3 +465,8 @@ def _decode_condition(c: str) -> tuple[str, str, str]:
 def search_by_name(endpoint: pf.Platform, name: str) -> dict[str, QualityGate]:
     """Searches quality gates matching name"""
     return util.search_by_name(endpoint, name, APIS["list"], "qualitygates")
+
+
+def convert_for_yaml(original_json: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
+    """Convert the original JSON defined for JSON export into a JSON format more adapted for YAML export"""
+    return util.dict_to_list(original_json, "name")