From 06f6b6c2cb27ac7a38c6eaebcb20d15fcaa4d34e Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 13:13:48 +0200 Subject: [PATCH 1/6] Fix crash when exporting with keys and async --- cli/config.py | 72 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/cli/config.py b/cli/config.py index c9ffd9d65..c22c32d30 100644 --- a/cli/config.py +++ b/cli/config.py @@ -75,6 +75,18 @@ _WRITE_LOCK = Lock() +_EXPORT_CALLS = { + options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export], + options.WHAT_RULES: [__JSON_KEY_RULES, rules.export], + options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export], + options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export], + options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], + options.WHAT_APPS: [__JSON_KEY_APPS, applications.export], + options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export], + options.WHAT_USERS: [__JSON_KEY_USERS, users.export], + options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export], +} + def __parse_args(desc): parser = options.set_common_args(desc) parser = options.set_key_arg(parser) @@ -173,8 +185,8 @@ def __convert_for_yaml(json_export: dict[str, any]) -> dict[str, any]: return json_export -def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: - """Exports a platform configuration in a JSON file""" +def __export_config_sync(endpoint: platform.Platform, what: list[str], **kwargs) -> None: + """Exports config in a synchronous way""" export_settings = { "INLINE_LISTS": not kwargs["dontInlineLists"], "EXPORT_DEFAULTS": kwargs["exportDefaults"], @@ -186,42 +198,62 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)] if len(non_existing_projects) > 0: utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY) + log.info("Exporting configuration synchronously from %s", kwargs[options.URL]) + key_list = kwargs[options.KEYS] + sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()} + for what_item, call_data in _EXPORT_CALLS.items(): + if what_item not in what: + continue + ndx, func = call_data + try: + sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list) + except exceptions.UnsupportedOperation as e: + log.warning(e.message) + sq_settings = utilities.remove_empties(sq_settings) + if not kwargs["dontInlineLists"]: + sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) + __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) + log.info("Synchronous export of configuration from %s completed", kwargs["url"]) - calls = { - options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export], - options.WHAT_RULES: [__JSON_KEY_RULES, rules.export], - options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export], - options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export], - # options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], - options.WHAT_APPS: [__JSON_KEY_APPS, applications.export], - options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export], - options.WHAT_USERS: [__JSON_KEY_USERS, users.export], - options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export], - } - log.info("Exporting configuration from %s", kwargs[options.URL]) +def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs) -> None: + """Exports a platform configuration in a JSON file""" + export_settings = { + "INLINE_LISTS": not kwargs["dontInlineLists"], + "EXPORT_DEFAULTS": kwargs["exportDefaults"], + "FULL_EXPORT": kwargs["fullExport"], + "THREADS": kwargs[options.NBR_THREADS], + options.REPORT_FILE: kwargs[options.REPORT_FILE], + "WRITE_CALLBACK": write_project, + } + log.info("Exporting configuration from %s (asynchronously)", kwargs[options.URL]) key_list = kwargs[options.KEYS] sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()} - for what_item, call_data in calls.items(): + for what_item, call_data in _EXPORT_CALLS.items(): if what_item not in what or what_item == options.WHAT_PROJECTS: continue ndx, func = call_data try: - sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list) + sq_settings[ndx] = utilities.remove_empties(func(endpoint, export_settings=export_settings, key_list=key_list)) __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) except exceptions.UnsupportedOperation as e: log.warning(e.message) - sq_settings = utilities.remove_empties(sq_settings) if not kwargs["dontInlineLists"]: sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) + __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) - export_settings["WRITE_CALLBACK"] = write_project __remove_chars_at_end(kwargs[options.REPORT_FILE], 3) __add_project_header(kwargs[options.REPORT_FILE]) - projects.export(endpoint, export_settings=export_settings, key_list=key_list) + projects.export(endpoint, export_settings=export_settings, key_list=None) __add_project_footer(kwargs[options.REPORT_FILE]) + log.info("Asynchronous export of configuration from %s completed", kwargs["url"]) + - log.info("Exporting configuration from %s completed", kwargs["url"]) +def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: + if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what: + __export_config_async(endpoint=endpoint, what=what, **kwargs) + else: + __export_config_sync(endpoint=endpoint, what=what, **kwargs) def __import_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: From 27ef9ac4c4c4fd84b06b136a88211effbd394451 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 13:39:14 +0200 Subject: [PATCH 2/6] Never exit on BAD_REQUEST, NOT_FOUND and UNAUTHORIZED --- sonar/platform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sonar/platform.py b/sonar/platform.py index 280a31b38..fbcc98d89 100644 --- a/sonar/platform.py +++ b/sonar/platform.py @@ -57,6 +57,8 @@ _SONAR_TOOLS_AGENT = f"sonar-tools {version.PACKAGE_VERSION}" _UPDATE_CENTER = "https://raw.githubusercontent.com/SonarSource/sonar-update-center-properties/master/update-center-source.properties" +_NORMAL_HTTP_ERRORS = (HTTPStatus.UNAUTHORIZED, HTTPStatus.NOT_FOUND, HTTPStatus.BAD_REQUEST) + LTA = None LATEST = None _HARDCODED_LTA = (9, 9, 6) @@ -246,7 +248,7 @@ def __run_request( self.url = new_url r.raise_for_status() except requests.exceptions.HTTPError as e: - if exit_on_error or (r.status_code not in mute and r.status_code == HTTPStatus.UNAUTHORIZED): + if exit_on_error: # or (r.status_code not in mute and r.status_code not in _NORMAL_HTTP_ERRORS): util.log_and_exit(r) else: _, msg = util.http_error(r) From 045b22af213cea1bc1b90eb8aa1d1c0980bbe4d8 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 13:39:46 +0200 Subject: [PATCH 3/6] Fix error when app key does not exists --- cli/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cli/config.py b/cli/config.py index c22c32d30..f9d89e263 100644 --- a/cli/config.py +++ b/cli/config.py @@ -87,6 +87,7 @@ options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export], } + def __parse_args(desc): parser = options.set_common_args(desc) parser = options.set_key_arg(parser) @@ -209,6 +210,8 @@ def __export_config_sync(endpoint: platform.Platform, what: list[str], **kwargs) sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list) except exceptions.UnsupportedOperation as e: log.warning(e.message) + except exceptions.ObjectNotFound as e: + log.error(e.message) sq_settings = utilities.remove_empties(sq_settings) if not kwargs["dontInlineLists"]: sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) @@ -238,6 +241,8 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) except exceptions.UnsupportedOperation as e: log.warning(e.message) + except exceptions.ObjectNotFound as e: + log.error(e.message) if not kwargs["dontInlineLists"]: sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) @@ -251,9 +256,9 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what: - __export_config_async(endpoint=endpoint, what=what, **kwargs) - else: __export_config_sync(endpoint=endpoint, what=what, **kwargs) + else: + __export_config_async(endpoint=endpoint, what=what, **kwargs) def __import_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: From 6e9cd219fe04efb124135b201e03e4efb4c633d7 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 13:40:09 +0200 Subject: [PATCH 4/6] Async write only if WRITE_CALLBACK is set --- sonar/projects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonar/projects.py b/sonar/projects.py index 5f9ed8fc2..8ee07da94 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -1473,7 +1473,8 @@ def __export_thread(queue: Queue[Project], results: dict[str, str], export_setti while not queue.empty(): project = queue.get() results[project.key] = project.export(export_settings=export_settings) - export_settings["WRITE_CALLBACK"](results[project.key], export_settings["file"]) + if export_settings.get("WRITE_CALLBACK", None): + export_settings["WRITE_CALLBACK"](results[project.key], export_settings["file"]) results[project.key].pop("key", None) with _CLASS_LOCK: export_settings["EXPORTED"] += 1 From c5164df95419ede811e1c34a983a269243e6acc9 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 13:43:52 +0200 Subject: [PATCH 5/6] Use sync export if format is YAML --- cli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/config.py b/cli/config.py index f9d89e263..5afef4099 100644 --- a/cli/config.py +++ b/cli/config.py @@ -255,7 +255,7 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: - if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what: + if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what or kwargs[options.FORMAT] != "json": __export_config_sync(endpoint=endpoint, what=what, **kwargs) else: __export_config_async(endpoint=endpoint, what=what, **kwargs) From 0f9f3955541b80b60088dd5fc32690304a5d1cdb Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 15:51:30 +0200 Subject: [PATCH 6/6] Don't keep project list JSON export if async --- sonar/projects.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sonar/projects.py b/sonar/projects.py index 8ee07da94..662df88d4 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -1472,10 +1472,12 @@ def __export_thread(queue: Queue[Project], results: dict[str, str], export_setti """Project export callback function for multitheaded export""" while not queue.empty(): project = queue.get() - results[project.key] = project.export(export_settings=export_settings) + exp_json = project.export(export_settings=export_settings) if export_settings.get("WRITE_CALLBACK", None): - export_settings["WRITE_CALLBACK"](results[project.key], export_settings["file"]) - results[project.key].pop("key", None) + export_settings["WRITE_CALLBACK"](exp_json, export_settings["file"]) + else: + results[project.key] = exp_json + results[project.key].pop("key", None) with _CLASS_LOCK: export_settings["EXPORTED"] += 1 nb, tot = export_settings["EXPORTED"], export_settings["NBR_PROJECTS"]