Change-plugins-export-format (#2053)

* New utility functions

* Fixes plugins export format

* Utility functions update

Author: okorach

cli/config.py

@@ -125,10 +125,7 @@ def __parse_args(desc: str) -> object:
 def __normalize_json(json_data: dict[str, any], remove_empty: bool = True, remove_none: bool = True) -> dict[str, any]:
     """Sorts a JSON file and optionally remove empty and none values"""
     log.info("Normalizing JSON - remove empty = %s, remove nones = %s", str(remove_empty), str(remove_none))
-    if remove_empty:
-        json_data = utilities.remove_empties(json_data)
-    if remove_none:
-        json_data = utilities.remove_nones(json_data)
+    json_data = utilities.clean_data(json_data, remove_none=remove_none, remove_empty=remove_empty)
     json_data = utilities.order_keys(json_data, *_SECTIONS_ORDER)
     for key in [k for k in _SECTIONS_TO_SORT if k in json_data]:
         json_data[key] = {k: json_data[key][k] for k in sorted(json_data[key])}
@@ -260,7 +257,7 @@ def __prep_json_for_write(json_data: types.ObjectJsonRepr, export_settings: type
         json_data = utilities.remove_nones(json_data)
         if not export_settings.get(EXPORT_EMPTY, False):
             log.debug("Removing empties")
-            json_data = utilities.remove_empties(json_data)
+            json_data = utilities.clean_data(json_data, remove_empty=True)
     if export_settings.get("INLINE_LISTS", True):
         json_data = utilities.inline_lists(json_data, exceptions=("conditions",))
     return json_data

sonar/platform.py

@@ -189,7 +189,12 @@ def basics(self) -> dict[str, str]:
         if self.is_sonarcloud():
             return {**data, "organization": self.organization}
 
-        return {**data, "version": util.version_to_string(self.version()[:3]), "serverId": self.server_id(), "plugins": self.plugins()}
+        return {
+            **data,
+            "version": util.version_to_string(self.version()[:3]),
+            "serverId": self.server_id(),
+            "plugins": util.dict_to_list(self.plugins(), "key"),
+        }
 
     def default_user_group(self) -> str:
         """

sonar/utilities.py

@@ -172,18 +172,6 @@ def convert_to_type(value: Any) -> Any:
     return value
 
 
-def remove_nones(d: dict[str, any]) -> dict[str, any]:
-    """Removes elements of the dict that are None values"""
-    new_d = d.copy()
-    for k, v in d.items():
-        if v is None:
-            new_d.pop(k)
-            continue
-        if isinstance(v, dict):
-            new_d[k] = remove_nones(v)
-    return new_d
-
-
 def none_to_zero(d: dict[str, any], key_match: str = "^.+$") -> dict[str, any]:
     """Replaces None values in a dict with 0"""
     new_d = d.copy()
@@ -197,21 +185,36 @@ def none_to_zero(d: dict[str, any], key_match: str = "^.+$") -> dict[str, any]:
     return new_d
 
 
-def remove_empties(d: dict[str, any]) -> dict[str, any]:
+def remove_nones(d: Any) -> Any:
+    """Removes elements of the data that are None values"""
+    return clean_data(d, remove_empty=False, remove_none=True)
+
+
+def clean_data(d: Any, remove_empty: bool = True, remove_none: bool = True) -> Any:
     """Recursively removes empty lists and dicts and none from a dict"""
     # log.debug("Cleaning up %s", json_dump(d))
-    new_d = d.copy()
-    for k, v in d.items():
-        if isinstance(v, str) and v == "":
-            new_d.pop(k)
-            continue
-        if not isinstance(v, (list, dict)):
-            continue
-        if len(v) == 0:
-            new_d.pop(k)
-        elif isinstance(v, dict):
-            new_d[k] = remove_empties(v)
-    return new_d
+    if not isinstance(d, (list, dict)):
+        return d
+
+    if isinstance(d, list):
+        # Remove empty strings and nones
+        if remove_empty:
+            d = [elem for elem in d if not (isinstance(elem, str) and elem == "")]
+        if remove_none:
+            d = [elem for elem in d if elem is not None]
+        return [clean_data(elem, remove_empty, remove_none) for elem in d]
+
+    # Remove empty dict string values
+    if remove_empty:
+        new_d = {k: v for k, v in d.items() if not isinstance(v, str) or v != ""}
+    if remove_none:
+        new_d = {k: v for k, v in d.items() if v is not None}
+
+    # Remove empty dict list or dict values
+    new_d = {k: v for k, v in new_d.items() if not isinstance(v, (list, dict)) or len(v) > 0}
+
+    # Recurse
+    return {k: clean_data(v, remove_empty, remove_none) for k, v in new_d.items()}
 
 
 def sort_lists(data: Any, redact_tokens: bool = True) -> Any:
@@ -278,23 +281,21 @@ def list_to_regexp(str_list: list[str]) -> str:
     return "(" + "|".join(str_list) + ")" if len(str_list) > 0 else ""
 
 
-def list_to_csv(
-    array: Union[None, str, int, float, list[str], set[str], tuple[str]], separator: str = ",", check_for_separator: bool = False
-) -> Optional[str]:
+def list_to_csv(array: Union[None, str, int, float, list[str], set[str], tuple[str]], separator: str = ",", check_for_separator: bool = False) -> Any:
     """Converts a list of strings to CSV"""
     if isinstance(array, str):
         return csv_normalize(array, separator) if " " in array else array
     if array is None:
         return None
-    if isinstance(array, (list, set, tuple)):
+    if isinstance(array, (list, set, tuple)) and all(isinstance(e, str) for e in array):
         if check_for_separator:
             # Don't convert to string if one array item contains the string separator
             s = separator.strip()
             for item in array:
                 if s in item:
                     return array
         return separator.join([v.strip() for v in array])
-    return str(array)
+    return array
 
 
 def csv_normalize(string: str, separator: str = ",") -> str:
@@ -314,10 +315,10 @@ def union(list1: list[any], list2: list[any]) -> list[any]:
     return list1 + [value for value in list2 if value not in list1]
 
 
-def difference(list1: list[any], list2: list[any]) -> list[any]:
+def difference(list1: list[Any], list2: list[Any]) -> list[Any]:
     """Computes difference of 2 lists"""
     # FIXME - This should be sets
-    return [value for value in list1 if value not in list2]
+    return list(set(list1) - set(list2))
 
 
 def quote(string: str, sep: str) -> str:
@@ -578,15 +579,11 @@ def __prefix(value: Any) -> Any:
         return value
 
 
-def filter_export(json_data: dict[str, any], key_properties: list[str], full: bool) -> dict[str, any]:
+def filter_export(json_data: dict[str, Any], key_properties: list[str], full: bool) -> dict[str, Any]:
     """Filters dict for export removing or prefixing non-key properties"""
-    new_json_data = json_data.copy()
-    for k in json_data:
-        if k not in key_properties:
-            if full and k != "actions":
-                new_json_data[f"_{k}"] = __prefix(new_json_data.pop(k))
-            else:
-                new_json_data.pop(k)
+    new_json_data = {k: json_data[k] for k in key_properties if k in json_data}
+    if full:
+        new_json_data |= {f"_{k}": __prefix(v) for k, v in json_data.items() if k not in key_properties}
     return new_json_data
 
 
@@ -726,15 +723,15 @@ def dict_remap_and_stringify(original_dict: dict[str, str], remapping: dict[str,
     return dict_stringify(dict_remap(original_dict, remapping))
 
 
-def list_to_dict(original_list: list[dict[str, any]], key_field: str) -> dict[str, any]:
+def list_to_dict(original_list: list[dict[str, Any]], key_field: str) -> dict[str, any]:
     """Converts a list to dict with list key_field as dict key"""
     converted_dict = {elem[key_field]: elem for elem in original_list}
     for e in converted_dict.values():
         e.pop(key_field)
     return converted_dict
 
 
-def dict_to_list(original_dict: dict[str, any], key_field: str, value_field: Optional[str] = "value") -> list[str, any]:
+def dict_to_list(original_dict: dict[str, Any], key_field: str, value_field: Optional[str] = "value") -> list[str, any]:
     """Converts a dict to list adding dict key in list key_field"""
     if isinstance(original_dict, list):
         return original_dict
@@ -794,20 +791,6 @@ def pretty_print_json(file: str) -> bool:
     return True
 
 
-def order_keys(original_dict: dict[str, any], *keys) -> dict[str, any]:
-    """Orders a dict keys in a chosen order, existings keys not in *keys are pushed to the end
-    :param dict[str, any] original_dict: Dict to order
-    :param str *keys: List of keys in desired order
-    :return: same dict with keys in desired order
-    """
-    ordered_dict = {}
-    for key in [k for k in keys if k in original_dict]:
-        ordered_dict[key] = original_dict[key]
-    for key in [k for k in original_dict if k not in keys]:
-        ordered_dict[key] = original_dict[key]
-    return ordered_dict
-
-
 def flatten(original_dict: dict[str, any]) -> dict[str, any]:
     """Flattens a recursive dict into a flat one"""
     flat_dict = {}
@@ -825,3 +808,39 @@ def similar_strings(key1: str, key2: str, max_distance: int = 5) -> bool:
         return False
     max_distance = min(len(key1) // 2, len(key2) // 2, max_distance)
     return (len(key2) >= 7 and (re.match(key2, key1))) or Levenshtein.distance(key1, key2, score_cutoff=6) <= max_distance
+
+
+def sort_list_by_key(list_to_sort: list[dict[str, Any]], key: str, priority_field: Optional[str] = None) -> list[dict[str, Any]]:
+    """Sorts a lits of dicts by a given key, exception for the priority field that would go first"""
+    f_elem = None
+    if priority_field:
+        f_elem = next((elem for elem in list_to_sort if priority_field in elem), None)
+    tmp_dict = {elem[key]: elem for elem in list_to_sort if elem != f_elem}
+    first_elem = [f_elem] if f_elem else []
+    return first_elem + list(dict(sorted(tmp_dict.items())).values())
+
+
+def order_keys(original_dict: dict[str, any], *keys) -> dict[str, any]:
+    """Orders a dict keys in a chosen order, existings keys not in *keys are pushed to the end
+    :param dict[str, any] original_dict: Dict to order
+    :param str *keys: List of keys in desired order
+    :return: same dict with keys in desired order
+    """
+    ordered_dict = {}
+    for key in [k for k in keys if k in original_dict]:
+        ordered_dict[key] = original_dict[key]
+    for key in [k for k in original_dict if k not in keys]:
+        ordered_dict[key] = original_dict[key]
+    return ordered_dict
+
+
+def order_dict(d: dict[str, Any], key_order: list[str]) -> dict[str, Any]:
+    """Orders keys of a dictionary in a given order"""
+    new_d = {k: d[k] for k in key_order if k in d}
+    return new_d | {k: v for k, v in d.items() if k not in new_d}
+
+
+def order_list(l: list[str], *key_order) -> list[str]:
+    """Orders elements of a list in a given order"""
+    new_l = [k for k in key_order if k in l]
+    return new_l + [k for k in l if k not in new_l]