diff --git a/cli/config.py b/cli/config.py index c9ffd9d65..5afef4099 100644 --- a/cli/config.py +++ b/cli/config.py @@ -75,6 +75,19 @@ _WRITE_LOCK = Lock() +_EXPORT_CALLS = { + options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export], + options.WHAT_RULES: [__JSON_KEY_RULES, rules.export], + options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export], + options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export], + options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], + options.WHAT_APPS: [__JSON_KEY_APPS, applications.export], + options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export], + options.WHAT_USERS: [__JSON_KEY_USERS, users.export], + options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export], +} + + def __parse_args(desc): parser = options.set_common_args(desc) parser = options.set_key_arg(parser) @@ -173,8 +186,8 @@ def __convert_for_yaml(json_export: dict[str, any]) -> dict[str, any]: return json_export -def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: - """Exports a platform configuration in a JSON file""" +def __export_config_sync(endpoint: platform.Platform, what: list[str], **kwargs) -> None: + """Exports config in a synchronous way""" export_settings = { "INLINE_LISTS": not kwargs["dontInlineLists"], "EXPORT_DEFAULTS": kwargs["exportDefaults"], @@ -186,42 +199,66 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)] if len(non_existing_projects) > 0: utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY) + log.info("Exporting configuration synchronously from %s", kwargs[options.URL]) + key_list = kwargs[options.KEYS] + sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()} + for what_item, call_data in _EXPORT_CALLS.items(): + if what_item not in what: + continue + ndx, func = call_data + try: + sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list) + except exceptions.UnsupportedOperation as e: + log.warning(e.message) + except exceptions.ObjectNotFound as e: + log.error(e.message) + sq_settings = utilities.remove_empties(sq_settings) + if not kwargs["dontInlineLists"]: + sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) + __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) + log.info("Synchronous export of configuration from %s completed", kwargs["url"]) - calls = { - options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export], - options.WHAT_RULES: [__JSON_KEY_RULES, rules.export], - options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export], - options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export], - # options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], - options.WHAT_APPS: [__JSON_KEY_APPS, applications.export], - options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export], - options.WHAT_USERS: [__JSON_KEY_USERS, users.export], - options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export], - } - log.info("Exporting configuration from %s", kwargs[options.URL]) +def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs) -> None: + """Exports a platform configuration in a JSON file""" + export_settings = { + "INLINE_LISTS": not kwargs["dontInlineLists"], + "EXPORT_DEFAULTS": kwargs["exportDefaults"], + "FULL_EXPORT": kwargs["fullExport"], + "THREADS": kwargs[options.NBR_THREADS], + options.REPORT_FILE: kwargs[options.REPORT_FILE], + "WRITE_CALLBACK": write_project, + } + log.info("Exporting configuration from %s (asynchronously)", kwargs[options.URL]) key_list = kwargs[options.KEYS] sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()} - for what_item, call_data in calls.items(): + for what_item, call_data in _EXPORT_CALLS.items(): if what_item not in what or what_item == options.WHAT_PROJECTS: continue ndx, func = call_data try: - sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list) + sq_settings[ndx] = utilities.remove_empties(func(endpoint, export_settings=export_settings, key_list=key_list)) __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) except exceptions.UnsupportedOperation as e: log.warning(e.message) - sq_settings = utilities.remove_empties(sq_settings) + except exceptions.ObjectNotFound as e: + log.error(e.message) if not kwargs["dontInlineLists"]: sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) + __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) - export_settings["WRITE_CALLBACK"] = write_project __remove_chars_at_end(kwargs[options.REPORT_FILE], 3) __add_project_header(kwargs[options.REPORT_FILE]) - projects.export(endpoint, export_settings=export_settings, key_list=key_list) + projects.export(endpoint, export_settings=export_settings, key_list=None) __add_project_footer(kwargs[options.REPORT_FILE]) + log.info("Asynchronous export of configuration from %s completed", kwargs["url"]) - log.info("Exporting configuration from %s completed", kwargs["url"]) + +def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: + if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what or kwargs[options.FORMAT] != "json": + __export_config_sync(endpoint=endpoint, what=what, **kwargs) + else: + __export_config_async(endpoint=endpoint, what=what, **kwargs) def __import_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None: diff --git a/sonar/platform.py b/sonar/platform.py index 280a31b38..fbcc98d89 100644 --- a/sonar/platform.py +++ b/sonar/platform.py @@ -57,6 +57,8 @@ _SONAR_TOOLS_AGENT = f"sonar-tools {version.PACKAGE_VERSION}" _UPDATE_CENTER = "https://raw.githubusercontent.com/SonarSource/sonar-update-center-properties/master/update-center-source.properties" +_NORMAL_HTTP_ERRORS = (HTTPStatus.UNAUTHORIZED, HTTPStatus.NOT_FOUND, HTTPStatus.BAD_REQUEST) + LTA = None LATEST = None _HARDCODED_LTA = (9, 9, 6) @@ -246,7 +248,7 @@ def __run_request( self.url = new_url r.raise_for_status() except requests.exceptions.HTTPError as e: - if exit_on_error or (r.status_code not in mute and r.status_code == HTTPStatus.UNAUTHORIZED): + if exit_on_error: # or (r.status_code not in mute and r.status_code not in _NORMAL_HTTP_ERRORS): util.log_and_exit(r) else: _, msg = util.http_error(r) diff --git a/sonar/projects.py b/sonar/projects.py index 5f9ed8fc2..662df88d4 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -1472,9 +1472,12 @@ def __export_thread(queue: Queue[Project], results: dict[str, str], export_setti """Project export callback function for multitheaded export""" while not queue.empty(): project = queue.get() - results[project.key] = project.export(export_settings=export_settings) - export_settings["WRITE_CALLBACK"](results[project.key], export_settings["file"]) - results[project.key].pop("key", None) + exp_json = project.export(export_settings=export_settings) + if export_settings.get("WRITE_CALLBACK", None): + export_settings["WRITE_CALLBACK"](exp_json, export_settings["file"]) + else: + results[project.key] = exp_json + results[project.key].pop("key", None) with _CLASS_LOCK: export_settings["EXPORTED"] += 1 nb, tot = export_settings["EXPORTED"], export_settings["NBR_PROJECTS"]