Sonar migration tool

cli/migration.py

@@ -0,0 +1,170 @@
+#!/usr/local/bin/python3
+#
+# sonar-tools
+# Copyright (C) 2022-2024 Olivier Korach
+# mailto:olivier.korach AT gmail DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+"""
+    Exports SonarQube platform configuration as JSON
+"""
+import sys
+import json
+import yaml
+
+from cli import options
+from sonar import exceptions, errcodes, utilities
+import sonar.logging as log
+from sonar import platform, rules, qualityprofiles, qualitygates, users, groups
+from sonar import projects, portfolios, applications
+
+_EVERYTHING = [
+    options.WHAT_SETTINGS,
+    options.WHAT_USERS,
+    options.WHAT_GROUPS,
+    options.WHAT_GATES,
+    options.WHAT_RULES,
+    options.WHAT_PROFILES,
+    options.WHAT_PROJECTS,
+    options.WHAT_APPS,
+    options.WHAT_PORTFOLIOS,
+]
+
+__JSON_KEY_PLATFORM = "platform"
+
+__JSON_KEY_SETTINGS = "globalSettings"
+__JSON_KEY_USERS = "users"
+__JSON_KEY_GROUPS = "groups"
+__JSON_KEY_GATES = "qualityGates"
+__JSON_KEY_RULES = "rules"
+__JSON_KEY_PROFILES = "qualityProfiles"
+__JSON_KEY_PROJECTS = "projects"
+__JSON_KEY_APPS = "applications"
+__JSON_KEY_PORTFOLIOS = "portfolios"
+
+__MAP = {
+    options.WHAT_SETTINGS: __JSON_KEY_SETTINGS,
+    options.WHAT_USERS: __JSON_KEY_USERS,
+    options.WHAT_GROUPS: __JSON_KEY_GROUPS,
+    options.WHAT_GATES: __JSON_KEY_GATES,
+    options.WHAT_RULES: __JSON_KEY_RULES,
+    options.WHAT_PROFILES: __JSON_KEY_PROFILES,
+    options.WHAT_PROJECTS: __JSON_KEY_PROJECTS,
+    options.WHAT_APPS: __JSON_KEY_APPS,
+    options.WHAT_PORTFOLIOS: __JSON_KEY_PORTFOLIOS,
+}
+
+
+def __parse_args(desc):
+    parser = options.set_common_args(desc)
+    parser = options.set_key_arg(parser)
+    parser = options.set_output_file_args(parser, allowed_formats=("json",))
+    parser = options.add_thread_arg(parser, "migration export")
+    parser = options.set_what(parser, what_list=_EVERYTHING, operation="export")
+    parser = options.add_import_export_arg(parser, "migration")
+    parser.add_argument(
+        "--exportDefaults",
+        required=False,
+        default=False,
+        action="store_true",
+        help="Also exports settings values that are the platform defaults. "
+        f"By default the export will show the value as '{utilities.DEFAULT}' "
+        "and the setting will not be imported at import time",
+    )
+    args = options.parse_and_check(parser=parser, logger_name="sonar-migration")
+    return args
+
+
+def __write_export(config: dict[str, str], file: str) -> None:
+    """Writes the configuration in file"""
+    with utilities.open_file(file) as fd:
+        print(utilities.json_dump(config), file=fd)
+
+
+def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
+    """Exports a platform configuration in a JSON file"""
+    export_settings = {
+        "INLINE_LISTS": False,
+        "EXPORT_DEFAULTS": True,
+        # "FULL_EXPORT": kwargs["fullExport"],
+        "FULL_EXPORT": False,
+        "MODE": "MIGRATION",
+        "THREADS": kwargs[options.NBR_THREADS],
+    }
+    if "projects" in what and kwargs[options.KEYS]:
+        non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)]
+        if len(non_existing_projects) > 0:
+            utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY)
+
+    calls = {
+        options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export],
+        options.WHAT_RULES: [__JSON_KEY_RULES, rules.export],
+        options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export],
+        options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export],
+        options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export],
+        options.WHAT_APPS: [__JSON_KEY_APPS, applications.export],
+        options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export],
+        options.WHAT_USERS: [__JSON_KEY_USERS, users.export],
+        options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export],
+    }
+
+    log.info("Exporting configuration from %s", kwargs[options.URL])
+    key_list = kwargs[options.KEYS]
+    sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()}
+    for what_item, call_data in calls.items():
+        if what_item not in what:
+            continue
+        ndx, func = call_data
+        try:
+            sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list)
+        except exceptions.UnsupportedOperation as e:
+            log.warning(e.message)
+    sq_settings = utilities.remove_empties(sq_settings)
+    # if not kwargs.get("dontInlineLists", False):
+    #    sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",))
+    __write_export(sq_settings, kwargs[options.REPORT_FILE])
+
+
+def main() -> None:
+    """Main entry point for sonar-config"""
+    start_time = utilities.start_clock()
+    try:
+        kwargs = utilities.convert_args(__parse_args("Extract SonarQube platform configuration"))
+        endpoint = platform.Platform(**kwargs)
+        endpoint.verify_connection()
+    except (options.ArgumentsError, exceptions.ObjectNotFound) as e:
+        utilities.exit_fatal(e.message, e.errcode)
+
+    what = utilities.check_what(kwargs.pop(options.WHAT, None), _EVERYTHING, "exported")
+    if options.WHAT_PROFILES in what and options.WHAT_RULES not in what:
+        what.append(options.WHAT_RULES)
+    kwargs[options.FORMAT] = "json"
+    if kwargs[options.REPORT_FILE] is None:
+        kwargs[options.REPORT_FILE] = f"sonar-migration.{endpoint.server_id()}.json"
+    try:
+        __export_config(endpoint, what, **kwargs)
+    except exceptions.ObjectNotFound as e:
+        utilities.exit_fatal(e.message, errcodes.NO_SUCH_KEY)
+    except (PermissionError, FileNotFoundError) as e:
+        utilities.exit_fatal(f"OS error while exporting config: {e}", exit_code=errcodes.OS_ERROR)
+    log.info("Exporting SQ to SC migration data from %s completed", kwargs[options.URL])
+    log.info("Migration file '%s' created", kwargs[options.REPORT_FILE])
+    utilities.stop_clock(start_time)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

deploy-migration.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# sonar-tools
+# Copyright (C) 2019-2024 Olivier Korach
+# mailto:olivier.korach AT gmail DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
+build_docs=1
+build_image=1
+release=0
+
+while [ $# -ne 0 ]; do
+    case $1 in
+        nodoc)
+            build_docs=0
+            ;;
+        nodocker)
+            build_image=0
+            ;;
+        pypi)
+            release=1
+            ;;
+        *)
+            ;;
+    esac
+    shift
+done
+
+black --line-length=150 .
+rm -rf build dist
+python3 setup_migration.py bdist_wheel
+
+# Deploy locally for tests
+pip install --upgrade --force-reinstall dist/sonar_migration-*-py3-*.whl
+
+if [ "$build_image" == "1" ]; then
+    docker build -t sonar-migration:latest .
+fi
+
+if [ "$build_docs" == "1" ]; then
+    rm -rf api-doc/build
+    sphinx-build -b html api-doc/source api-doc/build
+fi
+
+# Deploy on pypi.org once released
+if [ "$release" = "1" ]; then
+    echo "Confirm release [y/n] ?"
+    read -r confirm
+    if [ "$confirm" = "y" ]; then
+        python3 -m twine upload dist/sonar_migration-*-py3-*.whl
+    fi
+fi

deploy.sh

@@ -45,7 +45,7 @@ rm -rf build dist
 python3 setup.py bdist_wheel
 
 # Deploy locally for tests
-pip install --upgrade --force-reinstall dist/*-py3-*.whl
+pip install --upgrade --force-reinstall dist/sonar-tools-*-py3-*.whl
 
 if [ "$build_image" == "1" ]; then
     docker build -t sonar-tools:latest .
@@ -61,6 +61,6 @@ if [ "$release" = "1" ]; then
     echo "Confirm release [y/n] ?"
     read -r confirm
     if [ "$confirm" = "y" ]; then
-        python3 -m twine upload dist/*
+        python3 -m twine upload dist/sonar-tools-*-py3-*.whl
     fi
 fi

setup_migration.py

@@ -0,0 +1,67 @@
+#
+# sonar-tools
+# Copyright (C) 2019-2024 Olivier Korach
+# mailto:olivier.korach AT gmail DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
+"""
+
+   Package setup
+
+"""
+import setuptools
+from sonar import version
+
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+setuptools.setup(
+    name="sonar-migration",
+    version=version.PACKAGE_VERSION,
+    scripts=["sonar_migration"],
+    author="Olivier Korach",
+    author_email="[email protected]",
+    description="A collection of utility scripts for SonarQube and SonarCloud",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/okorach/sonar-tools",
+    project_urls={
+        "Bug Tracker": "https://github.com/okorach/sonar-tools/issues",
+        "Documentation": "https://github.com/okorach/sonar-tools/README.md",
+        "Source Code": "https://github.com/okorach/sonar-tools",
+    },
+    packages=setuptools.find_packages(),
+    package_data={"sonar": ["LICENSE", "audit/rules.json", "audit/sonar-audit.properties"]},
+    install_requires=[
+        "argparse",
+        "datetime",
+        "python-dateutil",
+        "requests",
+        "jprops",
+    ],
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)",
+        "Operating System :: OS Independent",
+    ],
+    entry_points={
+        "console_scripts": [
+            "sonar-migration = cli.migration:main",
+        ]
+    },
+    python_requires=">=3.8",
+)

sonar-project.properties

@@ -14,5 +14,5 @@ sonar.python.pylint.reportPaths=build/pylint-report.out
 
 sonar.exclusions=api-doc/**/*, build/**/*, test/**/*
 sonar.coverage.exclusions=test/**/*, shellcheck2sonar.py, cli/cust_measures.py, sonar/custom_measures.py, cli/support.py, cli/projects_export.py, , cli/projects_import.py
-
+sonar.cpd.exclusions = cli/migration.py
 sonar.tests=test

sonar/branches.py

@@ -230,6 +230,14 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
             data[settings.NEW_CODE_PERIOD] = self.new_code()
         if export_settings.get("FULL_EXPORT", True):
             data.update({"name": self.name, "project": self.concerned_object.key})
+        if export_settings["MODE"] == "MIGRATION":
+            data["lastAnalysis"] = util.date_to_string(self.last_analysis())
+            lang_distrib = self.get_measure("ncloc_language_distribution")
+            loc_distrib = {}
+            if lang_distrib:
+                loc_distrib = {m.split("=")[0]: int(m.split("=")[1]) for m in lang_distrib.split(";")}
+            loc_distrib["total"] = self.loc()
+            data["ncloc"] = loc_distrib
         data = util.remove_nones(data)
         return None if len(data) == 0 else data
 

sonar/platform.py

@@ -167,7 +167,7 @@ def basics(self) -> dict[str, str]:
         if self.is_sonarcloud():
             return {**data, "organization": self.organization}
 
-        return {**data, "version": util.version_to_string(self.version()[:3]), "serverId": self.server_id()}
+        return {**data, "version": util.version_to_string(self.version()[:3]), "serverId": self.server_id(), "plugins": self.plugins()}
 
     def get(self, api: str, params: types.ApiParams = None, exit_on_error: bool = False, mute: tuple[HTTPStatus] = (), **kwargs) -> requests.Response:
         """Makes an HTTP GET request to SonarQube
@@ -310,9 +310,12 @@ def plugins(self) -> dict[str, str]:
         """
         if self.is_sonarcloud():
             return {}
+        sysinfo = self.sys_info()
+        if "Application Nodes" in sysinfo:
+            sysinfo = sysinfo["Application Nodes"][0]
         if self.version() < (9, 7, 0):
-            return self.sys_info()["Statistics"]["plugins"]
-        return self.sys_info()["Plugins"]
+            return sysinfo["Statistics"]["plugins"]
+        return sysinfo["Plugins"]
 
     def get_settings(self, settings_list: list[str] = None) -> dict[str, any]:
         """Returns a list of (or all) platform global settings value from their key

sonar/portfolios.py

@@ -753,7 +753,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
         else:
             log.debug("Skipping export of %s, it's a standard sub-portfolio", str(p))
         i += 1
-        if i % 50 == 0 or i == nb_portfolios:
+        if i % 10 == 0 or i == nb_portfolios:
             log.info("Exported %d/%d portfolios (%d%%)", i, nb_portfolios, (i * 100) // nb_portfolios)
     return exported_portfolios