diff --git a/cli/migration.py b/cli/migration.py new file mode 100644 index 000000000..bc9e2acea --- /dev/null +++ b/cli/migration.py @@ -0,0 +1,170 @@ +#!/usr/local/bin/python3 +# +# sonar-tools +# Copyright (C) 2022-2024 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +""" + Exports SonarQube platform configuration as JSON +""" +import sys +import json +import yaml + +from cli import options +from sonar import exceptions, errcodes, utilities +import sonar.logging as log +from sonar import platform, rules, qualityprofiles, qualitygates, users, groups +from sonar import projects, portfolios, applications + +_EVERYTHING = [ + options.WHAT_SETTINGS, + options.WHAT_USERS, + options.WHAT_GROUPS, + options.WHAT_GATES, + options.WHAT_RULES, + options.WHAT_PROFILES, + options.WHAT_PROJECTS, + options.WHAT_APPS, + options.WHAT_PORTFOLIOS, +] + +__JSON_KEY_PLATFORM = "platform" + +__JSON_KEY_SETTINGS = "globalSettings" +__JSON_KEY_USERS = "users" +__JSON_KEY_GROUPS = "groups" +__JSON_KEY_GATES = "qualityGates" +__JSON_KEY_RULES = "rules" +__JSON_KEY_PROFILES = "qualityProfiles" +__JSON_KEY_PROJECTS = "projects" +__JSON_KEY_APPS = "applications" +__JSON_KEY_PORTFOLIOS = "portfolios" + +__MAP = { + options.WHAT_SETTINGS: __JSON_KEY_SETTINGS, + options.WHAT_USERS: __JSON_KEY_USERS, + options.WHAT_GROUPS: __JSON_KEY_GROUPS, + options.WHAT_GATES: __JSON_KEY_GATES, + options.WHAT_RULES: __JSON_KEY_RULES, + options.WHAT_PROFILES: __JSON_KEY_PROFILES, + options.WHAT_PROJECTS: __JSON_KEY_PROJECTS, + options.WHAT_APPS: __JSON_KEY_APPS, + options.WHAT_PORTFOLIOS: __JSON_KEY_PORTFOLIOS, +} + + +def __parse_args(desc): + parser = options.set_common_args(desc) + parser = options.set_key_arg(parser) + parser = options.set_output_file_args(parser, allowed_formats=("json",)) + parser = options.add_thread_arg(parser, "migration export") + parser = options.set_what(parser, what_list=_EVERYTHING, operation="export") + parser = options.add_import_export_arg(parser, "migration") + parser.add_argument( + "--exportDefaults", + required=False, + default=False, + action="store_true", + help="Also exports settings values that are the platform defaults. " + f"By default the export will show the value as '{utilities.DEFAULT}' " + "and the setting will not be imported at import time", + ) + args = options.parse_and_check(parser=parser, logger_name="sonar-migration") + return args + + +def __write_export(config: dict[str, str], file: str) -> None: + """Writes the configuration in file""" + with utilities.open_file(file) as fd: + print(utilities.json_dump(config), file=fd) + + +def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: + """Exports a platform configuration in a JSON file""" + export_settings = { + "INLINE_LISTS": False, + "EXPORT_DEFAULTS": True, + # "FULL_EXPORT": kwargs["fullExport"], + "FULL_EXPORT": False, + "MODE": "MIGRATION", + "THREADS": kwargs[options.NBR_THREADS], + } + if "projects" in what and kwargs[options.KEYS]: + non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)] + if len(non_existing_projects) > 0: + utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY) + + calls = { + options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export], + options.WHAT_RULES: [__JSON_KEY_RULES, rules.export], + options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export], + options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export], + options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], + options.WHAT_APPS: [__JSON_KEY_APPS, applications.export], + options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export], + options.WHAT_USERS: [__JSON_KEY_USERS, users.export], + options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export], + } + + log.info("Exporting configuration from %s", kwargs[options.URL]) + key_list = kwargs[options.KEYS] + sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()} + for what_item, call_data in calls.items(): + if what_item not in what: + continue + ndx, func = call_data + try: + sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list) + except exceptions.UnsupportedOperation as e: + log.warning(e.message) + sq_settings = utilities.remove_empties(sq_settings) + # if not kwargs.get("dontInlineLists", False): + # sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) + __write_export(sq_settings, kwargs[options.REPORT_FILE]) + + +def main() -> None: + """Main entry point for sonar-config""" + start_time = utilities.start_clock() + try: + kwargs = utilities.convert_args(__parse_args("Extract SonarQube platform configuration")) + endpoint = platform.Platform(**kwargs) + endpoint.verify_connection() + except (options.ArgumentsError, exceptions.ObjectNotFound) as e: + utilities.exit_fatal(e.message, e.errcode) + + what = utilities.check_what(kwargs.pop(options.WHAT, None), _EVERYTHING, "exported") + if options.WHAT_PROFILES in what and options.WHAT_RULES not in what: + what.append(options.WHAT_RULES) + kwargs[options.FORMAT] = "json" + if kwargs[options.REPORT_FILE] is None: + kwargs[options.REPORT_FILE] = f"sonar-migration.{endpoint.server_id()}.json" + try: + __export_config(endpoint, what, **kwargs) + except exceptions.ObjectNotFound as e: + utilities.exit_fatal(e.message, errcodes.NO_SUCH_KEY) + except (PermissionError, FileNotFoundError) as e: + utilities.exit_fatal(f"OS error while exporting config: {e}", exit_code=errcodes.OS_ERROR) + log.info("Exporting SQ to SC migration data from %s completed", kwargs[options.URL]) + log.info("Migration file '%s' created", kwargs[options.REPORT_FILE]) + utilities.stop_clock(start_time) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/deploy-migration.sh b/deploy-migration.sh new file mode 100755 index 000000000..bc8fc299c --- /dev/null +++ b/deploy-migration.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# sonar-tools +# Copyright (C) 2019-2024 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +build_docs=1 +build_image=1 +release=0 + +while [ $# -ne 0 ]; do + case $1 in + nodoc) + build_docs=0 + ;; + nodocker) + build_image=0 + ;; + pypi) + release=1 + ;; + *) + ;; + esac + shift +done + +black --line-length=150 . +rm -rf build dist +python3 setup_migration.py bdist_wheel + +# Deploy locally for tests +pip install --upgrade --force-reinstall dist/sonar_migration-*-py3-*.whl + +if [ "$build_image" == "1" ]; then + docker build -t sonar-migration:latest . +fi + +if [ "$build_docs" == "1" ]; then + rm -rf api-doc/build + sphinx-build -b html api-doc/source api-doc/build +fi + +# Deploy on pypi.org once released +if [ "$release" = "1" ]; then + echo "Confirm release [y/n] ?" + read -r confirm + if [ "$confirm" = "y" ]; then + python3 -m twine upload dist/sonar_migration-*-py3-*.whl + fi +fi \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index fe05d2dd6..c74572fbb 100755 --- a/deploy.sh +++ b/deploy.sh @@ -45,7 +45,7 @@ rm -rf build dist python3 setup.py bdist_wheel # Deploy locally for tests -pip install --upgrade --force-reinstall dist/*-py3-*.whl +pip install --upgrade --force-reinstall dist/sonar-tools-*-py3-*.whl if [ "$build_image" == "1" ]; then docker build -t sonar-tools:latest . @@ -61,6 +61,6 @@ if [ "$release" = "1" ]; then echo "Confirm release [y/n] ?" read -r confirm if [ "$confirm" = "y" ]; then - python3 -m twine upload dist/* + python3 -m twine upload dist/sonar-tools-*-py3-*.whl fi fi \ No newline at end of file diff --git a/setup_migration.py b/setup_migration.py new file mode 100644 index 000000000..9a69dafef --- /dev/null +++ b/setup_migration.py @@ -0,0 +1,67 @@ +# +# sonar-tools +# Copyright (C) 2019-2024 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +""" + + Package setup + +""" +import setuptools +from sonar import version + + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() +setuptools.setup( + name="sonar-migration", + version=version.PACKAGE_VERSION, + scripts=["sonar_migration"], + author="Olivier Korach", + author_email="olivier.korach@gmail.com", + description="A collection of utility scripts for SonarQube and SonarCloud", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/okorach/sonar-tools", + project_urls={ + "Bug Tracker": "https://github.com/okorach/sonar-tools/issues", + "Documentation": "https://github.com/okorach/sonar-tools/README.md", + "Source Code": "https://github.com/okorach/sonar-tools", + }, + packages=setuptools.find_packages(), + package_data={"sonar": ["LICENSE", "audit/rules.json", "audit/sonar-audit.properties"]}, + install_requires=[ + "argparse", + "datetime", + "python-dateutil", + "requests", + "jprops", + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Operating System :: OS Independent", + ], + entry_points={ + "console_scripts": [ + "sonar-migration = cli.migration:main", + ] + }, + python_requires=">=3.8", +) diff --git a/sonar-project.properties b/sonar-project.properties index 2aa70fb90..54eea5317 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -14,5 +14,5 @@ sonar.python.pylint.reportPaths=build/pylint-report.out sonar.exclusions=api-doc/**/*, build/**/*, test/**/* sonar.coverage.exclusions=test/**/*, shellcheck2sonar.py, cli/cust_measures.py, sonar/custom_measures.py, cli/support.py, cli/projects_export.py, , cli/projects_import.py - +sonar.cpd.exclusions = cli/migration.py sonar.tests=test diff --git a/sonar/branches.py b/sonar/branches.py index 7e9b75815..c824e0d29 100644 --- a/sonar/branches.py +++ b/sonar/branches.py @@ -230,6 +230,14 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: data[settings.NEW_CODE_PERIOD] = self.new_code() if export_settings.get("FULL_EXPORT", True): data.update({"name": self.name, "project": self.concerned_object.key}) + if export_settings["MODE"] == "MIGRATION": + data["lastAnalysis"] = util.date_to_string(self.last_analysis()) + lang_distrib = self.get_measure("ncloc_language_distribution") + loc_distrib = {} + if lang_distrib: + loc_distrib = {m.split("=")[0]: int(m.split("=")[1]) for m in lang_distrib.split(";")} + loc_distrib["total"] = self.loc() + data["ncloc"] = loc_distrib data = util.remove_nones(data) return None if len(data) == 0 else data diff --git a/sonar/platform.py b/sonar/platform.py index e25279a54..d50e948a3 100644 --- a/sonar/platform.py +++ b/sonar/platform.py @@ -167,7 +167,7 @@ def basics(self) -> dict[str, str]: if self.is_sonarcloud(): return {**data, "organization": self.organization} - return {**data, "version": util.version_to_string(self.version()[:3]), "serverId": self.server_id()} + return {**data, "version": util.version_to_string(self.version()[:3]), "serverId": self.server_id(), "plugins": self.plugins()} def get(self, api: str, params: types.ApiParams = None, exit_on_error: bool = False, mute: tuple[HTTPStatus] = (), **kwargs) -> requests.Response: """Makes an HTTP GET request to SonarQube @@ -310,9 +310,12 @@ def plugins(self) -> dict[str, str]: """ if self.is_sonarcloud(): return {} + sysinfo = self.sys_info() + if "Application Nodes" in sysinfo: + sysinfo = sysinfo["Application Nodes"][0] if self.version() < (9, 7, 0): - return self.sys_info()["Statistics"]["plugins"] - return self.sys_info()["Plugins"] + return sysinfo["Statistics"]["plugins"] + return sysinfo["Plugins"] def get_settings(self, settings_list: list[str] = None) -> dict[str, any]: """Returns a list of (or all) platform global settings value from their key diff --git a/sonar/portfolios.py b/sonar/portfolios.py index 0f61b6834..93676a0ae 100644 --- a/sonar/portfolios.py +++ b/sonar/portfolios.py @@ -753,7 +753,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis else: log.debug("Skipping export of %s, it's a standard sub-portfolio", str(p)) i += 1 - if i % 50 == 0 or i == nb_portfolios: + if i % 10 == 0 or i == nb_portfolios: log.info("Exported %d/%d portfolios (%d%%)", i, nb_portfolios, (i * 100) // nb_portfolios) return exported_portfolios diff --git a/sonar/projects.py b/sonar/projects.py index 29911c43c..16b66f90b 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -104,6 +104,8 @@ def __init__(self, endpoint: pf.Platform, key: str) -> None: self._ncloc_with_branches = None self._binding = {"has_binding": True, "binding": None} self._new_code = None + self._ci = None + self._revision = None _OBJECTS[self.uuid()] = self log.debug("Created object %s", str(self)) @@ -212,7 +214,7 @@ def reload(self, data: types.ApiPayload) -> Project: self._last_analysis = util.string_to_date(data["analysisDate"]) else: self._last_analysis = None - self.revision = data.get("revision", None) + self._revision = data.get("revision", self._revision) return self def url(self) -> str: @@ -554,6 +556,10 @@ def last_task(self) -> Optional[tasks.Task]: """Returns the last analysis background task of a problem, or none if not found""" return tasks.search_last(component_key=self.key, endpoint=self.endpoint, type="REPORT") + def task_history(self) -> Optional[tasks.Task]: + """Returns the last analysis background task of a problem, or none if not found""" + return tasks.search_all(component_key=self.key, endpoint=self.endpoint, type="REPORT") + def scanner(self) -> str: """Returns the project type (MAVEN, GRADLE, DOTNET, OTHER, UNKNOWN)""" last_task = self.last_task() @@ -562,6 +568,28 @@ def scanner(self) -> str: last_task.concerned_object = self return last_task.scanner() + def ci(self) -> str: + """Returns the detected CI tool used, or undetected, or unknown if HTTP request fails""" + log.debug("Collecting detected CI") + if not self._ci or not self._revision: + self._ci, self._revision = "unknown", "unknown" + try: + data = json.loads(self.get("project_analyses/search", params={"project": self.key, "ps": 1}).text)["analyses"] + if len(data) > 0: + self._ci, self._revision = data[0].get("detectedCI", "unknown"), data[0].get("revision", "unknown") + except HTTPError: + log.warning("HTTP Error, can't retrieve CI tool and revision") + except KeyError: + log.warning("KeyError, can't retrieve CI tool and revision") + return self._ci + + def revision(self) -> str: + """Returns the last analysis commit, or unknown if HTTP request fails or no revision""" + log.debug("Collecting revision") + if not self._revision: + self.ci() + return self._revision + def __audit_scanner(self, audit_settings: types.ConfigSettings) -> list[Problem]: proj_type, scanner = self.get_type(), self.scanner() log.debug("%s is of type %s and uses scanner %s", str(self), proj_type, scanner) @@ -931,6 +959,26 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, if hooks is not None: json_data["webhooks"] = hooks json_data = util.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False)) + + if export_settings["MODE"] == "MIGRATION": + json_data["lastAnalysis"] = util.date_to_string(self.last_analysis()) + json_data["detectedCi"] = self.ci() + json_data["revision"] = self.revision() + lang_distrib = self.get_measure("ncloc_language_distribution") + loc_distrib = {} + if lang_distrib: + loc_distrib = {m.split("=")[0]: int(m.split("=")[1]) for m in lang_distrib.split(";")} + loc_distrib["total"] = self.loc() + json_data["ncloc"] = loc_distrib + last_task = self.last_task() + json_data["backgroundTasks"] = {} + if last_task: + json_data["backgroundTasks"] = { + "lastTaskScannerContext": last_task.scanner_context(), + "lastTaskWarnings": last_task.warnings(), + "taskHistory": [t._json for t in self.task_history()], + } + settings_dict = settings.get_bulk(endpoint=self.endpoint, component=self, settings_list=settings_list, include_not_set=False) # json_data.update({s.to_json() for s in settings_dict.values() if include_inherited or not s.inherited}) for s in settings_dict.values(): @@ -1363,6 +1411,11 @@ def __export_thread(queue: Queue[Project], results: dict[str, str], export_setti project = queue.get() results[project.key] = project.export(export_settings=export_settings) results[project.key].pop("key", None) + with _CLASS_LOCK: + export_settings["EXPORTED"] += 1 + nb, tot = export_settings["EXPORTED"], export_settings["NBR_PROJECTS"] + if nb % 10 == 0 or nb == tot: + log.info("%d/%d projects exported (%d%%)", nb, tot, (nb * 100) // tot) queue.task_done() @@ -1379,7 +1432,11 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis qp.projects() q = Queue(maxsize=0) - for p in get_list(endpoint=endpoint, key_list=key_list).values(): + proj_list = get_list(endpoint=endpoint, key_list=key_list) + export_settings["NBR_PROJECTS"] = len(proj_list) + export_settings["EXPORTED"] = 0 + log.info("Exporting %d projects", export_settings["NBR_PROJECTS"]) + for p in proj_list.values(): q.put(p) project_settings = {} for i in range(export_settings.get("THREADS", 8)): diff --git a/sonar/tasks.py b/sonar/tasks.py index cee0410bc..595e6131c 100644 --- a/sonar/tasks.py +++ b/sonar/tasks.py @@ -549,9 +549,9 @@ def search_last(endpoint: pf.Platform, component_key: str, **params) -> Optional return bg_tasks[0] -def search_all(endpoint: pf.Platform, component_key: str) -> list[Task]: +def search_all(endpoint: pf.Platform, component_key: str, **params) -> list[Task]: """Search all background tasks of a given component""" - return search(endpoint=endpoint, component_key=component_key) + return search(endpoint=endpoint, component_key=component_key, **params) def _get_suspicious_exclusions(patterns: str) -> list[str]: diff --git a/sonar_migration b/sonar_migration new file mode 100755 index 000000000..61a5390e8 --- /dev/null +++ b/sonar_migration @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# +# sonar-tools +# Copyright (C) 2019-2024 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +"""Main entry point for sonar-migration""" + +from sonar import version + +print(f''' +sonar-migration version {version.PACKAGE_VERSION} + +run: sonar-migration -u -t + +See tools built-in -h help and https://github.com/okorach/sonar-tools for more documentation +''')