Change-sonar-config-format

cli/config.py

@@ -22,7 +22,7 @@
 Exports SonarQube platform configuration as JSON
 """
 
-from typing import TextIO
+from typing import TextIO, Any
 from threading import Thread
 from queue import Queue
 
@@ -122,16 +122,20 @@ def __parse_args(desc: str) -> object:
     return options.parse_and_check(parser=parser, logger_name=TOOL_NAME)
 
 
-def __normalize_json(json_data: dict[str, any], remove_empty: bool = True, remove_none: bool = True) -> dict[str, any]:
+def __normalize_json(json_data: dict[str, Any], remove_empty: bool = True, remove_none: bool = True) -> dict[str, any]:
     """Sorts a JSON file and optionally remove empty and none values"""
     log.info("Normalizing JSON - remove empty = %s, remove nones = %s", str(remove_empty), str(remove_none))
-    if remove_empty:
-        json_data = utilities.remove_empties(json_data)
-    if remove_none:
-        json_data = utilities.remove_nones(json_data)
+    json_data = utilities.clean_data(json_data, remove_empty, remove_none)
     json_data = utilities.order_keys(json_data, *_SECTIONS_ORDER)
     for key in [k for k in _SECTIONS_TO_SORT if k in json_data]:
-        json_data[key] = {k: json_data[key][k] for k in sorted(json_data[key])}
+        if isinstance(json_data[key], (list, tuple, set)):
+            if len(json_data[key]) > 0:
+                sort_field = next((k for k in ("key", "name", "login") if k in json_data[key][0]), None)
+                if sort_field:
+                    tmp_d = {v[sort_field]: v for v in json_data[key]}
+                    json_data[key] = list(dict(sorted(tmp_d.items())).values())
+        else:
+            json_data[key] = {k: json_data[key][k] for k in sorted(json_data[key])}
     return json_data
 
 
@@ -171,28 +175,37 @@ def write_objects(queue: Queue[types.ObjectJsonRepr], fd: TextIO, object_type: s
     """
     done = False
     prefix = ""
+    objects_exported_as_lists = ("projects", "applications", "users", "portfolios")
+    objects_exported_as_whole = ("qualityGates", "groups")
     log.info("Waiting %s to write...", object_type)
-    print(f'"{object_type}": ' + "{", file=fd)
+    if object_type in objects_exported_as_lists:
+        start, stop = ("[", "]")
+    elif object_type in objects_exported_as_whole:
+        start, stop = ("", "")
+    else:
+        start, stop = ("{", "}")
+    print(f'"{object_type}": ' + start, file=fd)
     while not done:
         obj_json = queue.get()
         if not (done := obj_json is utilities.WRITE_END):
             if object_type == "groups":
                 obj_json = __prep_json_for_write(obj_json, {**export_settings, EXPORT_EMPTY: True})
             else:
                 obj_json = __prep_json_for_write(obj_json, export_settings)
-            if object_type in ("projects", "applications", "portfolios", "users"):
-                if object_type == "users":
-                    key = obj_json.pop("login", None)
-                else:
-                    key = obj_json.pop("key", None)
-                log.debug("Writing %s key '%s'", object_type[:-1], key)
+            key = "" if isinstance(obj_json, list) else obj_json.get("key", obj_json.get("login", obj_json.get("name", "unknown")))
+            log.debug("Writing %s key '%s'", object_type[:-1], key)
+            if object_type in objects_exported_as_lists:
+                print(f"{prefix}{utilities.json_dump(obj_json)}", end="", file=fd)
+            elif object_type in objects_exported_as_whole:
+                print(f"{prefix}{utilities.json_dump(obj_json)}", end="", file=fd)
+            elif object_type in ("applications", "portfolios", "users"):
                 print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd)
             else:
                 log.debug("Writing %s", object_type)
                 print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd)
             prefix = ",\n"
         queue.task_done()
-    print("\n}", file=fd, end="")
+    print("\n" + stop, file=fd, end="")
     log.info("Writing %s complete", object_type)
 
 
@@ -257,10 +270,7 @@ def __prep_json_for_write(json_data: types.ObjectJsonRepr, export_settings: type
     if export_settings.get("MODE", "CONFIG") == "MIGRATION":
         return json_data
     if not export_settings.get("FULL_EXPORT", False):
-        json_data = utilities.remove_nones(json_data)
-        if not export_settings.get(EXPORT_EMPTY, False):
-            log.debug("Removing empties")
-            json_data = utilities.remove_empties(json_data)
+        json_data = utilities.clean_data(json_data, remove_empty=not export_settings.get(EXPORT_EMPTY, False), remove_none=True)
     if export_settings.get("INLINE_LISTS", True):
         json_data = utilities.inline_lists(json_data, exceptions=("conditions",))
     return json_data

conf/build.sh

@@ -21,6 +21,7 @@ CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 build_docs=0
 build_docker=0
+offline=0
 
 . "${CONF_DIR}/env.sh"
 
@@ -32,6 +33,9 @@ while [[ $# -ne 0 ]]; do
         docker)
             build_docker=1
             ;;
+        offline)
+            offline=1
+            ;;
         *)
             ;;
     esac
@@ -41,9 +45,18 @@ done
 echo "======= FORMATTING CODE ========="
 ruff format
 echo "======= BUILDING PACKAGE ========="
-rm -rf "${ROOT_DIR}/build/lib/sonar" "${ROOT_DIR}/build/lib/cli" "${ROOT_DIR}"/build/scripts*/sonar-tools "${ROOT_DIR}"/dist/sonar_tools*
-# python -m build
-poetry build
+if [[ "${offline}" = "1" ]]; then
+    cp "${ROOT_DIR}/conf/offline/setup.py" "${ROOT_DIR}/"
+    cp "${ROOT_DIR}/conf/offline/sonar-tools" "${ROOT_DIR}/"
+    mv "${ROOT_DIR}/pyproject.toml" "${ROOT_DIR}/pyproject.toml.bak"
+    python setup.py bdist_wheel
+    mv "${ROOT_DIR}/pyproject.toml.bak" "${ROOT_DIR}/pyproject.toml"
+    rm "${ROOT_DIR}/setup.py" "${ROOT_DIR}/sonar-tools"
+    # python -m build
+else
+    rm -rf "${ROOT_DIR}/build/lib/sonar" "${ROOT_DIR}/build/lib/cli" "${ROOT_DIR}"/build/scripts*/sonar-tools "${ROOT_DIR}"/dist/sonar_tools*
+    poetry build
+fi
 
 if [[ "${build_docs}" = "1" ]]; then
     echo "======= BUILDING DOCS ========="

conf/offline/setup.py

@@ -0,0 +1,84 @@
+#
+# sonar-tools
+# Copyright (C) 2019-2025 Olivier Korach
+# mailto:olivier.korach AT gmail DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
+"""
+
+Package setup
+
+"""
+
+import setuptools
+from sonar import version
+
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+setuptools.setup(
+    name="sonar-tools",
+    version=version.PACKAGE_VERSION,
+    scripts=["sonar-tools"],
+    author="Olivier Korach",
+    author_email="[email protected]",
+    description="A collection of utility scripts for SonarQube Server or Cloud",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/okorach/sonar-tools",
+    project_urls={
+        "Bug Tracker": "https://github.com/okorach/sonar-tools/issues",
+        "Documentation": "https://github.com/okorach/sonar-tools/README.md",
+        "Source Code": "https://github.com/okorach/sonar-tools",
+    },
+    packages=setuptools.find_packages(),
+    package_data={"sonar": ["LICENSE", "audit/rules.json", "config.json", "audit/sonar-audit.properties"]},
+    install_requires=[
+        "argparse",
+        "datetime",
+        "python-dateutil",
+        "requests",
+        "jprops",
+        "levenshtein",
+        "PyYAML ",
+    ],
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)",
+        "Operating System :: OS Independent",
+    ],
+    entry_points={
+        "console_scripts": [
+            "sonar-audit = cli.audit:main",
+            "sonar-projects-export = cli.projects_export:main",
+            "sonar-projects-import = cli.projects_import:main",
+            "sonar-projects = cli.projects_cli:main",
+            "sonar-measures-export = cli.measures_export:main",
+            "sonar-housekeeper = cli.housekeeper:main",
+            "sonar-issues-sync = cli.findings_sync:main",
+            "sonar-findings-sync = cli.findings_sync:main",
+            "sonar-custom-measures = cli.cust_measures:main",
+            "sonar-issues-export = cli.findings_export:main",
+            "sonar-findings-export = cli.findings_export:main",
+            "sonar-loc = cli.loc:main",
+            "sonar-config = cli.config:main",
+            "support-audit = cli.support:main",
+            "sonar-rules = cli.rules_cli:main",
+        ]
+    },
+    python_requires=">=3.8",
+)

conf/offline/sonar-tools

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+#
+# sonar-tools
+# Copyright (C) 2019-2025 Olivier Korach
+# mailto:olivier.korach AT gmail DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
+"""Main entry point for sonar-tools"""
+
+from sonar import version
+
+print(f'''
+sonar-tools version {version.PACKAGE_VERSION}
+(c) Olivier Korach 2019-2025
+Collections of utilities for SonarQube Server and Cloud:
+- sonar-audit: Audits a SonarQube Server or Cloud platform for bad practices, performance, configuration problems
+- sonar-housekeeper: Deletes projects that have not been analyzed since a given number of days
+- sonar-loc: Produces a list of projects with their LoC count as computed by SonarQube Server or Cloud
+  commercial licenses (ie taking the largest branch or PR)
+- sonar-measures-export: Exports measures/metrics of one, several or all projects of the platform in CSV or JSON
+  (Can also export measures history)
+- sonar-findings-export: Exports findings (potentially filtered) from the platform in CSV or JSON
+  (also available as sonar-issues-export for backward compatibility, but deprecated)
+- sonar-findings-sync: Synchronizes issues between 2 branches of a same project, a whole project
+  branches of 2 different projects (potentially on different platforms).
+  (also available as sonar-issues-sync for backward compatibility, but deprecated)
+- sonar-projects: Exports / Imports projects to/from zip file (Import works for EE and higher)
+- sonar-config: Exports and imports an entire (or subsets of a) SonarQube Server or Cloud platform configuration as code (JSON)
+- sonar-rules: Exports Sonar rules
+See tools built-in -h help and https://github.com/okorach/sonar-tools for more documentation
+''')

sonar/app_branches.py

@@ -171,10 +171,13 @@ def export(self) -> types.ObjectJsonRepr:
         :param full: Whether to do a full export including settings that can't be set, defaults to False
         :type full: bool, optional
         """
-        log.info("Exporting %s from %s", self, self.sq_json)
-        jsondata = {"projects": {b["key"]: b["branch"] if b["selected"] else utilities.DEFAULT for b in self.sq_json["projects"]}}
+        log.info("Exporting %s from %s", self, utilities.json_dump(self.sq_json))
+        jsondata = {"name": self.name}
         if self.is_main():
             jsondata["isMain"] = True
+        br_projects = [b for b in self.sq_json["projects"] if b.get("selected", True)]
+        br_projects = [{"key": b["key"], "branch": None if b["isMain"] else b["branch"]} for b in br_projects]
+        jsondata["projects"] = utilities.remove_nones(br_projects)
         return jsondata
 
     def update(self, name: str, project_branches: list[Branch]) -> bool:

sonar/applications.py

@@ -339,12 +339,13 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
                 "description": None if self._description == "" else self._description,
                 "visibility": self.visibility(),
                 # 'projects': self.projects(),
-                "branches": {br.name: br.export() for br in self.branches().values()},
+                "branches": [br.export() for br in self.branches().values()],
                 "permissions": self.permissions().export(export_settings=export_settings),
                 "tags": self.get_tags(),
             }
         )
-        return util.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False))
+        json_data = util.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False))
+        return util.clean_data(json_data)
 
     def set_permissions(self, data: types.JsonPermissions) -> application_permissions.ApplicationPermissions:
         """Sets an application permissions
@@ -525,11 +526,9 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, **kwarg
         app_json = app.export(export_settings)
         if write_q:
             write_q.put(app_json)
-        else:
-            app_json.pop("key")
-            apps_settings[k] = app_json
+        apps_settings[k] = app_json
     write_q and write_q.put(util.WRITE_END)
-    return apps_settings
+    return dict(sorted(app_json.items())).values()
 
 
 def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings, **kwargs) -> list[problem.Problem]:
@@ -602,12 +601,4 @@ def search_by_name(endpoint: pf.Platform, name: str) -> dict[str, Application]:
 
 def convert_for_yaml(original_json: types.ObjectJsonRepr) -> types.ObjectJsonRepr:
     """Convert the original JSON defined for JSON export into a JSON format more adapted for YAML export"""
-    new_json = util.dict_to_list(util.remove_nones(original_json), "key")
-    for app_json in new_json:
-        app_json["branches"] = util.dict_to_list(app_json["branches"], "name")
-        for b in app_json["branches"]:
-            if "projects" in b:
-                b["projects"] = [{"key": k, "branch": br} for k, br in b["projects"].items()]
-        if "permissions" in app_json:
-            app_json["permissions"] = permissions.convert_for_yaml(app_json["permissions"])
-    return new_json
+    return original_json

sonar/devops.py

@@ -239,12 +239,8 @@ def export(endpoint: platform.Platform, export_settings: types.ConfigSettings) -
     :meta private:
     """
     log.info("Exporting DevOps integration settings")
-    json_data = {}
-    for s in get_list(endpoint).values():
-        export_data = s.to_json(export_settings)
-        key = export_data.pop("key")
-        json_data[key] = export_data
-    return json_data
+    devops_list = {s.key: s.to_json(export_settings) for s in get_list(endpoint).values()}
+    return list(dict(sorted(devops_list.items())).values())
 
 
 def import_config(endpoint: platform.Platform, config_data: types.ObjectJsonRepr, key_list: types.KeyList = None) -> int:

sonar/groups.py

@@ -345,11 +345,12 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, **kwarg
     """
 
     log.info("Exporting groups")
-    g_list = {}
+    g_list = []
     for g_name, g_obj in get_list(endpoint=endpoint).items():
         if not export_settings.get("FULL_EXPORT", False) and g_obj.is_default():
             continue
-        g_list[g_name] = "" if g_obj.description is None else g_obj.description
+        g_list.append({"name": g_name, "description": g_obj.description})
+    util.clean_data(g_list, remove_empty=False)
     log.info("%s groups to export", len(g_list))
     if write_q := kwargs.get("write_q", None):
         write_q.put(g_list)