Skip to content

Commit b0254b5

Browse files
okorachokorach
authored
Merge pull request #1313 from okorach:sonar-migration-tool
Sonar migration tool
2 parents 88330de + a9b74b6 commit b0254b5

File tree

11 files changed

+414
-11
lines changed

11 files changed

+414
-11
lines changed

‎cli/migration.py‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#!/usr/local/bin/python3
2+
#
3+
# sonar-tools
4+
# Copyright (C) 2022-2024 Olivier Korach
5+
# mailto:olivier.korach AT gmail DOT com
6+
#
7+
# This program is free software; you can redistribute it and/or
8+
# modify it under the terms of the GNU Lesser General Public
9+
# License as published by the Free Software Foundation; either
10+
# version 3 of the License, or (at your option) any later version.
11+
#
12+
# This program is distributed in the hope that it will be useful,
13+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15+
# Lesser General Public License for more details.
16+
#
17+
# You should have received a copy of the GNU Lesser General Public License
18+
# along with this program; if not, write to the Free Software Foundation,
19+
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20+
#
21+
"""
22+
Exports SonarQube platform configuration as JSON
23+
"""
24+
import sys
25+
import json
26+
import yaml
27+
28+
from cli import options
29+
from sonar import exceptions, errcodes, utilities
30+
import sonar.logging as log
31+
from sonar import platform, rules, qualityprofiles, qualitygates, users, groups
32+
from sonar import projects, portfolios, applications
33+
34+
_EVERYTHING = [
35+
options.WHAT_SETTINGS,
36+
options.WHAT_USERS,
37+
options.WHAT_GROUPS,
38+
options.WHAT_GATES,
39+
options.WHAT_RULES,
40+
options.WHAT_PROFILES,
41+
options.WHAT_PROJECTS,
42+
options.WHAT_APPS,
43+
options.WHAT_PORTFOLIOS,
44+
]
45+
46+
__JSON_KEY_PLATFORM = "platform"
47+
48+
__JSON_KEY_SETTINGS = "globalSettings"
49+
__JSON_KEY_USERS = "users"
50+
__JSON_KEY_GROUPS = "groups"
51+
__JSON_KEY_GATES = "qualityGates"
52+
__JSON_KEY_RULES = "rules"
53+
__JSON_KEY_PROFILES = "qualityProfiles"
54+
__JSON_KEY_PROJECTS = "projects"
55+
__JSON_KEY_APPS = "applications"
56+
__JSON_KEY_PORTFOLIOS = "portfolios"
57+
58+
__MAP = {
59+
options.WHAT_SETTINGS: __JSON_KEY_SETTINGS,
60+
options.WHAT_USERS: __JSON_KEY_USERS,
61+
options.WHAT_GROUPS: __JSON_KEY_GROUPS,
62+
options.WHAT_GATES: __JSON_KEY_GATES,
63+
options.WHAT_RULES: __JSON_KEY_RULES,
64+
options.WHAT_PROFILES: __JSON_KEY_PROFILES,
65+
options.WHAT_PROJECTS: __JSON_KEY_PROJECTS,
66+
options.WHAT_APPS: __JSON_KEY_APPS,
67+
options.WHAT_PORTFOLIOS: __JSON_KEY_PORTFOLIOS,
68+
}
69+
70+
71+
def __parse_args(desc):
72+
parser = options.set_common_args(desc)
73+
parser = options.set_key_arg(parser)
74+
parser = options.set_output_file_args(parser, allowed_formats=("json",))
75+
parser = options.add_thread_arg(parser, "migration export")
76+
parser = options.set_what(parser, what_list=_EVERYTHING, operation="export")
77+
parser = options.add_import_export_arg(parser, "migration")
78+
parser.add_argument(
79+
"--exportDefaults",
80+
required=False,
81+
default=False,
82+
action="store_true",
83+
help="Also exports settings values that are the platform defaults. "
84+
f"By default the export will show the value as '{utilities.DEFAULT}' "
85+
"and the setting will not be imported at import time",
86+
)
87+
args = options.parse_and_check(parser=parser, logger_name="sonar-migration")
88+
return args
89+
90+
91+
def __write_export(config: dict[str, str], file: str) -> None:
92+
"""Writes the configuration in file"""
93+
with utilities.open_file(file) as fd:
94+
print(utilities.json_dump(config), file=fd)
95+
96+
97+
def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
98+
"""Exports a platform configuration in a JSON file"""
99+
export_settings = {
100+
"INLINE_LISTS": False,
101+
"EXPORT_DEFAULTS": True,
102+
# "FULL_EXPORT": kwargs["fullExport"],
103+
"FULL_EXPORT": False,
104+
"MODE": "MIGRATION",
105+
"THREADS": kwargs[options.NBR_THREADS],
106+
}
107+
if "projects" in what and kwargs[options.KEYS]:
108+
non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)]
109+
if len(non_existing_projects) > 0:
110+
utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY)
111+
112+
calls = {
113+
options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export],
114+
options.WHAT_RULES: [__JSON_KEY_RULES, rules.export],
115+
options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export],
116+
options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export],
117+
options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export],
118+
options.WHAT_APPS: [__JSON_KEY_APPS, applications.export],
119+
options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export],
120+
options.WHAT_USERS: [__JSON_KEY_USERS, users.export],
121+
options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export],
122+
}
123+
124+
log.info("Exporting configuration from %s", kwargs[options.URL])
125+
key_list = kwargs[options.KEYS]
126+
sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()}
127+
for what_item, call_data in calls.items():
128+
if what_item not in what:
129+
continue
130+
ndx, func = call_data
131+
try:
132+
sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list)
133+
except exceptions.UnsupportedOperation as e:
134+
log.warning(e.message)
135+
sq_settings = utilities.remove_empties(sq_settings)
136+
# if not kwargs.get("dontInlineLists", False):
137+
# sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",))
138+
__write_export(sq_settings, kwargs[options.REPORT_FILE])
139+
140+
141+
def main() -> None:
142+
"""Main entry point for sonar-config"""
143+
start_time = utilities.start_clock()
144+
try:
145+
kwargs = utilities.convert_args(__parse_args("Extract SonarQube platform configuration"))
146+
endpoint = platform.Platform(**kwargs)
147+
endpoint.verify_connection()
148+
except (options.ArgumentsError, exceptions.ObjectNotFound) as e:
149+
utilities.exit_fatal(e.message, e.errcode)
150+
151+
what = utilities.check_what(kwargs.pop(options.WHAT, None), _EVERYTHING, "exported")
152+
if options.WHAT_PROFILES in what and options.WHAT_RULES not in what:
153+
what.append(options.WHAT_RULES)
154+
kwargs[options.FORMAT] = "json"
155+
if kwargs[options.REPORT_FILE] is None:
156+
kwargs[options.REPORT_FILE] = f"sonar-migration.{endpoint.server_id()}.json"
157+
try:
158+
__export_config(endpoint, what, **kwargs)
159+
except exceptions.ObjectNotFound as e:
160+
utilities.exit_fatal(e.message, errcodes.NO_SUCH_KEY)
161+
except (PermissionError, FileNotFoundError) as e:
162+
utilities.exit_fatal(f"OS error while exporting config: {e}", exit_code=errcodes.OS_ERROR)
163+
log.info("Exporting SQ to SC migration data from %s completed", kwargs[options.URL])
164+
log.info("Migration file '%s' created", kwargs[options.REPORT_FILE])
165+
utilities.stop_clock(start_time)
166+
sys.exit(0)
167+
168+
169+
if __name__ == "__main__":
170+
main()

‎deploy-migration.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
#
3+
# sonar-tools
4+
# Copyright (C) 2019-2024 Olivier Korach
5+
# mailto:olivier.korach AT gmail DOT com
6+
#
7+
# This program is free software; you can redistribute it and/or
8+
# modify it under the terms of the GNU Lesser General Public
9+
# License as published by the Free Software Foundation; either
10+
# version 3 of the License, or (at your option) any later version.
11+
#
12+
# This program is distributed in the hope that it will be useful,
13+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15+
# Lesser General Public License for more details.
16+
#
17+
# You should have received a copy of the GNU Lesser General Public License
18+
# along with this program; if not, write to the Free Software Foundation,
19+
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20+
#
21+
22+
build_docs=1
23+
build_image=1
24+
release=0
25+
26+
while [ $# -ne 0 ]; do
27+
case $1 in
28+
nodoc)
29+
build_docs=0
30+
;;
31+
nodocker)
32+
build_image=0
33+
;;
34+
pypi)
35+
release=1
36+
;;
37+
*)
38+
;;
39+
esac
40+
shift
41+
done
42+
43+
black --line-length=150 .
44+
rm -rf build dist
45+
python3 setup_migration.py bdist_wheel
46+
47+
# Deploy locally for tests
48+
pip install --upgrade --force-reinstall dist/sonar_migration-*-py3-*.whl
49+
50+
if [ "$build_image" == "1" ]; then
51+
docker build -t sonar-migration:latest .
52+
fi
53+
54+
if [ "$build_docs" == "1" ]; then
55+
rm -rf api-doc/build
56+
sphinx-build -b html api-doc/source api-doc/build
57+
fi
58+
59+
# Deploy on pypi.org once released
60+
if [ "$release" = "1" ]; then
61+
echo "Confirm release [y/n] ?"
62+
read -r confirm
63+
if [ "$confirm" = "y" ]; then
64+
python3 -m twine upload dist/sonar_migration-*-py3-*.whl
65+
fi
66+
fi

‎deploy.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ rm -rf build dist
4545
python3 setup.py bdist_wheel
4646

4747
# Deploy locally for tests
48-
pip install --upgrade --force-reinstall dist/*-py3-*.whl
48+
pip install --upgrade --force-reinstall dist/sonar-tools-*-py3-*.whl
4949

5050
if [ "$build_image" == "1" ]; then
5151
docker build -t sonar-tools:latest .
@@ -61,6 +61,6 @@ if [ "$release" = "1" ]; then
6161
echo "Confirm release [y/n] ?"
6262
read -r confirm
6363
if [ "$confirm" = "y" ]; then
64-
python3 -m twine upload dist/*
64+
python3 -m twine upload dist/sonar-tools-*-py3-*.whl
6565
fi
6666
fi

‎setup_migration.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#
2+
# sonar-tools
3+
# Copyright (C) 2019-2024 Olivier Korach
4+
# mailto:olivier.korach AT gmail DOT com
5+
#
6+
# This program is free software; you can redistribute it and/or
7+
# modify it under the terms of the GNU Lesser General Public
8+
# License as published by the Free Software Foundation; either
9+
# version 3 of the License, or (at your option) any later version.
10+
#
11+
# This program is distributed in the hope that it will be useful,
12+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
# Lesser General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU Lesser General Public License
17+
# along with this program; if not, write to the Free Software Foundation,
18+
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
#
20+
21+
"""
22+
23+
Package setup
24+
25+
"""
26+
import setuptools
27+
from sonar import version
28+
29+
30+
with open("README.md", "r", encoding="utf-8") as fh:
31+
long_description = fh.read()
32+
setuptools.setup(
33+
name="sonar-migration",
34+
version=version.PACKAGE_VERSION,
35+
scripts=["sonar_migration"],
36+
author="Olivier Korach",
37+
author_email="[email protected]",
38+
description="A collection of utility scripts for SonarQube and SonarCloud",
39+
long_description=long_description,
40+
long_description_content_type="text/markdown",
41+
url="https://github.com/okorach/sonar-tools",
42+
project_urls={
43+
"Bug Tracker": "https://github.com/okorach/sonar-tools/issues",
44+
"Documentation": "https://github.com/okorach/sonar-tools/README.md",
45+
"Source Code": "https://github.com/okorach/sonar-tools",
46+
},
47+
packages=setuptools.find_packages(),
48+
package_data={"sonar": ["LICENSE", "audit/rules.json", "audit/sonar-audit.properties"]},
49+
install_requires=[
50+
"argparse",
51+
"datetime",
52+
"python-dateutil",
53+
"requests",
54+
"jprops",
55+
],
56+
classifiers=[
57+
"Programming Language :: Python :: 3",
58+
"License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)",
59+
"Operating System :: OS Independent",
60+
],
61+
entry_points={
62+
"console_scripts": [
63+
"sonar-migration = cli.migration:main",
64+
]
65+
},
66+
python_requires=">=3.8",
67+
)

‎sonar-project.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,5 @@ sonar.python.pylint.reportPaths=build/pylint-report.out
1414

1515
sonar.exclusions=api-doc/**/*, build/**/*, test/**/*
1616
sonar.coverage.exclusions=test/**/*, shellcheck2sonar.py, cli/cust_measures.py, sonar/custom_measures.py, cli/support.py, cli/projects_export.py, , cli/projects_import.py
17-
17+
sonar.cpd.exclusions = cli/migration.py
1818
sonar.tests=test

‎sonar/branches.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,14 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
230230
data[settings.NEW_CODE_PERIOD] = self.new_code()
231231
if export_settings.get("FULL_EXPORT", True):
232232
data.update({"name": self.name, "project": self.concerned_object.key})
233+
if export_settings["MODE"] == "MIGRATION":
234+
data["lastAnalysis"] = util.date_to_string(self.last_analysis())
235+
lang_distrib = self.get_measure("ncloc_language_distribution")
236+
loc_distrib = {}
237+
if lang_distrib:
238+
loc_distrib = {m.split("=")[0]: int(m.split("=")[1]) for m in lang_distrib.split(";")}
239+
loc_distrib["total"] = self.loc()
240+
data["ncloc"] = loc_distrib
233241
data = util.remove_nones(data)
234242
return None if len(data) == 0 else data
235243

‎sonar/platform.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ def basics(self) -> dict[str, str]:
167167
if self.is_sonarcloud():
168168
return {**data, "organization": self.organization}
169169

170-
return {**data, "version": util.version_to_string(self.version()[:3]), "serverId": self.server_id()}
170+
return {**data, "version": util.version_to_string(self.version()[:3]), "serverId": self.server_id(), "plugins": self.plugins()}
171171

172172
def get(self, api: str, params: types.ApiParams = None, exit_on_error: bool = False, mute: tuple[HTTPStatus] = (), **kwargs) -> requests.Response:
173173
"""Makes an HTTP GET request to SonarQube
@@ -310,9 +310,12 @@ def plugins(self) -> dict[str, str]:
310310
"""
311311
if self.is_sonarcloud():
312312
return {}
313+
sysinfo = self.sys_info()
314+
if "Application Nodes" in sysinfo:
315+
sysinfo = sysinfo["Application Nodes"][0]
313316
if self.version() < (9, 7, 0):
314-
return self.sys_info()["Statistics"]["plugins"]
315-
return self.sys_info()["Plugins"]
317+
return sysinfo["Statistics"]["plugins"]
318+
return sysinfo["Plugins"]
316319

317320
def get_settings(self, settings_list: list[str] = None) -> dict[str, any]:
318321
"""Returns a list of (or all) platform global settings value from their key

‎sonar/portfolios.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -753,7 +753,7 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
753753
else:
754754
log.debug("Skipping export of %s, it's a standard sub-portfolio", str(p))
755755
i += 1
756-
if i % 50 == 0 or i == nb_portfolios:
756+
if i % 10 == 0 or i == nb_portfolios:
757757
log.info("Exported %d/%d portfolios (%d%%)", i, nb_portfolios, (i * 100) // nb_portfolios)
758758
return exported_portfolios
759759

0 commit comments

Comments
 (0)