From 2961e096827ac1ff6324f11e929ca55114e198b1 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Tue, 8 Oct 2024 10:12:16 +0200 Subject: [PATCH 1/5] Fix queue export --- cli/config.py | 107 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 80 insertions(+), 27 deletions(-) diff --git a/cli/config.py b/cli/config.py index 73c79d87e..238d41207 100644 --- a/cli/config.py +++ b/cli/config.py @@ -23,7 +23,8 @@ """ import sys import os -from threading import Lock +from threading import Thread, Lock +from queue import Queue import json import yaml @@ -219,39 +220,91 @@ def __export_config_sync(endpoint: platform.Platform, what: list[str], **kwargs) log.info("Synchronous export of configuration from %s completed", kwargs["url"]) +def write_objects(queue: Queue, fd, object_type: str) -> None: + """ + Thread to write projects in the JSON file + """ + done = False + prefix = "" + log.info("Waiting %s to write...", object_type) + print(f'"{object_type}": ' + "{", file=fd) + while not done: + obj_json = queue.get() + done = obj_json is None + if not done: + if object_type in ("projects", "applications", "portfolios", "users"): + if object_type == "users": + key = obj_json.pop("login", None) + else: + key = obj_json.pop("key", None) + log.debug("Writing %s key '%s'", object_type[:-1], key) + print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd) + else: + log.debug("Writing %s", object_type) + print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd) + prefix = ",\n" + queue.task_done() + print("\n}", file=fd, end="") + log.info("Writing %s complete", object_type) + + def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs) -> None: """Exports a platform configuration in a JSON file""" + file = kwargs[options.REPORT_FILE] export_settings = { - "INLINE_LISTS": not kwargs["dontInlineLists"], - "EXPORT_DEFAULTS": kwargs["exportDefaults"], - "FULL_EXPORT": kwargs["fullExport"], + "INLINE_LISTS": False, + "EXPORT_DEFAULTS": True, + # "FULL_EXPORT": kwargs["fullExport"], + "FULL_EXPORT": False, + "MODE": "MIGRATION", "THREADS": kwargs[options.NBR_THREADS], - options.REPORT_FILE: kwargs[options.REPORT_FILE], - "WRITE_CALLBACK": write_project, + "SKIP_ISSUES": True, } - log.info("Exporting configuration from %s (asynchronously)", kwargs[options.URL]) + if "projects" in what and kwargs[options.KEYS]: + non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)] + if len(non_existing_projects) > 0: + utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY) + + calls = { + options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export], + options.WHAT_RULES: [__JSON_KEY_RULES, rules.export], + options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export], + options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export], + options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], + options.WHAT_APPS: [__JSON_KEY_APPS, applications.export], + options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export], + options.WHAT_USERS: [__JSON_KEY_USERS, users.export], + options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export], + } + + log.info("Exporting configuration from %s", kwargs[options.URL]) key_list = kwargs[options.KEYS] sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()} - for what_item, call_data in _EXPORT_CALLS.items(): - if what_item not in what or what_item == options.WHAT_PROJECTS: - continue - ndx, func = call_data - try: - sq_settings[ndx] = utilities.remove_empties(func(endpoint, export_settings=export_settings, key_list=key_list)) - __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) - except exceptions.UnsupportedOperation as e: - log.warning(e.message) - except exceptions.ObjectNotFound as e: - log.error(e.message) - if not kwargs["dontInlineLists"]: - sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) - __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) - - __remove_chars_at_end(kwargs[options.REPORT_FILE], 3) - __add_project_header(kwargs[options.REPORT_FILE]) - projects.export(endpoint, export_settings=export_settings, key_list=None) - __add_project_footer(kwargs[options.REPORT_FILE]) - log.info("Asynchronous export of configuration from %s completed", kwargs["url"]) + is_first = True + q = Queue(maxsize=0) + with utilities.open_file(file, mode="w") as fd: + print("{", file=fd) + for what_item, call_data in calls.items(): + if what_item not in what: + continue + ndx, func = call_data + try: + if not is_first: + print(",", file=fd) + is_first = False + worker = Thread(target=write_objects, args=(q, fd, ndx)) + worker.daemon = True + worker.name = f"Write{ndx[:1].upper()}{ndx[1:10]}" + worker.start() + sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list, write_q=q) + q.join() + except exceptions.UnsupportedOperation as e: + log.warning(e.message) + sq_settings = utilities.remove_empties(sq_settings) + # if not kwargs.get("dontInlineLists", False): + # sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) + print("\n}", file=fd) + log.info("Exporting migration data from %s completed", kwargs["url"]) def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: From 219b90091bf99085c56527663518be4f79e29efe Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Tue, 8 Oct 2024 10:12:28 +0200 Subject: [PATCH 2/5] Remove useless import os --- migration/migration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/migration/migration.py b/migration/migration.py index 18a6cacaa..f0c3ac802 100644 --- a/migration/migration.py +++ b/migration/migration.py @@ -22,7 +22,6 @@ Exports SonarQube platform configuration as JSON """ import sys -import os from threading import Thread, Lock from queue import Queue From 3cfe0ec15607b696e7120fc4da83bee9de9eba79 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Tue, 8 Oct 2024 11:05:24 +0200 Subject: [PATCH 3/5] Fix inlining --- cli/config.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/config.py b/cli/config.py index 238d41207..b1f41634e 100644 --- a/cli/config.py +++ b/cli/config.py @@ -31,6 +31,7 @@ from cli import options from sonar import exceptions, errcodes, utilities +from sonar.util import types import sonar.logging as log from sonar import platform, rules, qualityprofiles, qualitygates, users, groups from sonar import projects, portfolios, applications @@ -220,7 +221,7 @@ def __export_config_sync(endpoint: platform.Platform, what: list[str], **kwargs) log.info("Synchronous export of configuration from %s completed", kwargs["url"]) -def write_objects(queue: Queue, fd, object_type: str) -> None: +def write_objects(queue: Queue, fd, object_type: str, export_settings: types.ConfigSettings) -> None: """ Thread to write projects in the JSON file """ @@ -232,6 +233,8 @@ def write_objects(queue: Queue, fd, object_type: str) -> None: obj_json = queue.get() done = obj_json is None if not done: + if export_settings.get("INLINE_LISTS", True): + obj_json = utilities.inline_lists(obj_json, exceptions=("conditions",)) if object_type in ("projects", "applications", "portfolios", "users"): if object_type == "users": key = obj_json.pop("login", None) @@ -252,7 +255,7 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs """Exports a platform configuration in a JSON file""" file = kwargs[options.REPORT_FILE] export_settings = { - "INLINE_LISTS": False, + "INLINE_LISTS": not kwargs["dontInlineLists"], "EXPORT_DEFAULTS": True, # "FULL_EXPORT": kwargs["fullExport"], "FULL_EXPORT": False, @@ -292,7 +295,7 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs if not is_first: print(",", file=fd) is_first = False - worker = Thread(target=write_objects, args=(q, fd, ndx)) + worker = Thread(target=write_objects, args=(q, fd, ndx, export_settings)) worker.daemon = True worker.name = f"Write{ndx[:1].upper()}{ndx[1:10]}" worker.start() @@ -301,8 +304,6 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs except exceptions.UnsupportedOperation as e: log.warning(e.message) sq_settings = utilities.remove_empties(sq_settings) - # if not kwargs.get("dontInlineLists", False): - # sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) print("\n}", file=fd) log.info("Exporting migration data from %s completed", kwargs["url"]) From f6b80a787356ed658a23733c50e93577472d072c Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Tue, 8 Oct 2024 14:29:41 +0200 Subject: [PATCH 4/5] Quality pass --- cli/config.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/cli/config.py b/cli/config.py index b1f41634e..38c0f3e8b 100644 --- a/cli/config.py +++ b/cli/config.py @@ -268,18 +268,6 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs if len(non_existing_projects) > 0: utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY) - calls = { - options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export], - options.WHAT_RULES: [__JSON_KEY_RULES, rules.export], - options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export], - options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export], - options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], - options.WHAT_APPS: [__JSON_KEY_APPS, applications.export], - options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export], - options.WHAT_USERS: [__JSON_KEY_USERS, users.export], - options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export], - } - log.info("Exporting configuration from %s", kwargs[options.URL]) key_list = kwargs[options.KEYS] sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()} @@ -287,7 +275,7 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs q = Queue(maxsize=0) with utilities.open_file(file, mode="w") as fd: print("{", file=fd) - for what_item, call_data in calls.items(): + for what_item, call_data in _EXPORT_CALLS.items(): if what_item not in what: continue ndx, func = call_data From 7a1997d3512beaecafdf7a075e3782d08a156a2a Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Tue, 8 Oct 2024 14:36:09 +0200 Subject: [PATCH 5/5] Allow output in stdout even in streaming mode --- cli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/config.py b/cli/config.py index 38c0f3e8b..3001560a6 100644 --- a/cli/config.py +++ b/cli/config.py @@ -298,7 +298,7 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: """Exports the configuration of the SonarQube platform""" - if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what or kwargs[options.FORMAT] != "json" or not kwargs[options.REPORT_FILE]: + if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what or kwargs[options.FORMAT] != "json": __export_config_sync(endpoint=endpoint, what=what, **kwargs) else: __export_config_async(endpoint=endpoint, what=what, **kwargs)