diff --git a/cli/config.py b/cli/config.py index 80b626c9b..c9ffd9d65 100644 --- a/cli/config.py +++ b/cli/config.py @@ -22,6 +22,9 @@ Exports SonarQube platform configuration as JSON """ import sys +import os +from threading import Lock + import json import yaml @@ -31,6 +34,8 @@ from sonar import platform, rules, qualityprofiles, qualitygates, users, groups from sonar import projects, portfolios, applications +WRITE_FILE = None + _EVERYTHING = [ options.WHAT_SETTINGS, options.WHAT_USERS, @@ -67,6 +72,8 @@ options.WHAT_PORTFOLIOS: __JSON_KEY_PORTFOLIOS, } +_WRITE_LOCK = Lock() + def __parse_args(desc): parser = options.set_common_args(desc) @@ -113,6 +120,36 @@ def __write_export(config: dict[str, str], file: str, format: str) -> None: print(utilities.json_dump(config), file=fd) +def __remove_chars_at_end(file: str, nb_bytes: int) -> None: + """Writes the configuration in file""" + with open(file, mode="rb+") as fd: + fd.seek(-nb_bytes, os.SEEK_END) + fd.truncate() + + +def __add_project_header(file: str) -> None: + """Writes the configuration in file""" + with open(file, mode="a", encoding="utf-8") as fd: + print(',\n "projects": {\n', file=fd) + + +def __add_project_footer(file: str) -> None: + """Closes projects section""" + __remove_chars_at_end(file, 2) + with open(file, mode="a", encoding="utf-8") as fd: + print("\n }\n}", file=fd) + + +def write_project(project_json: dict[str, any], file: str) -> None: + """ + writes a project JSON in a file + """ + key = project_json.pop("key") + with _WRITE_LOCK: + with utilities.open_file(file, mode="a") as fd: + print(f'"{key}": {utilities.json_dump(project_json)},', file=fd) + + def __convert_for_yaml(json_export: dict[str, any]) -> dict[str, any]: """Converts the default JSON produced by export to a modified version more suitable for YAML""" if "globalSettings" in json_export: @@ -143,6 +180,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N "EXPORT_DEFAULTS": kwargs["exportDefaults"], "FULL_EXPORT": kwargs["fullExport"], "THREADS": kwargs[options.NBR_THREADS], + options.REPORT_FILE: kwargs[options.REPORT_FILE], } if "projects" in what and kwargs[options.KEYS]: non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)] @@ -154,7 +192,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N options.WHAT_RULES: [__JSON_KEY_RULES, rules.export], options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export], options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export], - options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], + # options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], options.WHAT_APPS: [__JSON_KEY_APPS, applications.export], options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export], options.WHAT_USERS: [__JSON_KEY_USERS, users.export], @@ -165,17 +203,24 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N key_list = kwargs[options.KEYS] sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()} for what_item, call_data in calls.items(): - if what_item not in what: + if what_item not in what or what_item == options.WHAT_PROJECTS: continue ndx, func = call_data try: sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list) + __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) except exceptions.UnsupportedOperation as e: log.warning(e.message) sq_settings = utilities.remove_empties(sq_settings) if not kwargs["dontInlineLists"]: sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) - __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT]) + + export_settings["WRITE_CALLBACK"] = write_project + __remove_chars_at_end(kwargs[options.REPORT_FILE], 3) + __add_project_header(kwargs[options.REPORT_FILE]) + projects.export(endpoint, export_settings=export_settings, key_list=key_list) + __add_project_footer(kwargs[options.REPORT_FILE]) + log.info("Exporting configuration from %s completed", kwargs["url"]) diff --git a/migration/migration.py b/migration/migration.py index 4f0609edb..350b1573c 100644 --- a/migration/migration.py +++ b/migration/migration.py @@ -22,6 +22,8 @@ Exports SonarQube platform configuration as JSON """ import sys +import os +from threading import Lock from cli import options from sonar import exceptions, errcodes, utilities, version @@ -65,6 +67,8 @@ options.WHAT_PORTFOLIOS: __JSON_KEY_PORTFOLIOS, } +_WRITE_LOCK = Lock() + def __parse_args(desc): parser = options.set_common_args(desc) @@ -93,6 +97,36 @@ def __parse_args(desc): return args +def __remove_chars_at_end(file: str, nb_bytes: int) -> None: + """Writes the configuration in file""" + with open(file, mode="rb+") as fd: + fd.seek(-nb_bytes, os.SEEK_END) + fd.truncate() + + +def __add_project_header(file: str) -> None: + """Writes the configuration in file""" + with open(file, mode="a", encoding="utf-8") as fd: + print(',\n "projects": {\n', file=fd) + + +def __add_project_footer(file: str) -> None: + """Closes projects section""" + __remove_chars_at_end(file, 2) + with open(file, mode="a", encoding="utf-8") as fd: + print("\n }\n}", file=fd) + + +def write_project(project_json: dict[str, any], file: str) -> None: + """ + writes a project JSON in a file + """ + key = project_json.pop("key") + with _WRITE_LOCK: + with utilities.open_file(file, mode="a") as fd: + print(f'"{key}": {utilities.json_dump(project_json)},', file=fd) + + def __write_export(config: dict[str, str], file: str) -> None: """Writes the configuration in file""" with utilities.open_file(file) as fd: @@ -108,6 +142,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N "FULL_EXPORT": False, "MODE": "MIGRATION", "THREADS": kwargs[options.NBR_THREADS], + options.REPORT_FILE: kwargs[options.REPORT_FILE], "SKIP_ISSUES": kwargs["skipIssues"], } if "projects" in what and kwargs[options.KEYS]: @@ -120,7 +155,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N options.WHAT_RULES: [__JSON_KEY_RULES, rules.export], options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export], options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export], - options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], + # options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export], options.WHAT_APPS: [__JSON_KEY_APPS, applications.export], options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export], options.WHAT_USERS: [__JSON_KEY_USERS, users.export], @@ -136,12 +171,20 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N ndx, func = call_data try: sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list) + __write_export(sq_settings, kwargs[options.REPORT_FILE]) except exceptions.UnsupportedOperation as e: log.warning(e.message) sq_settings = utilities.remove_empties(sq_settings) # if not kwargs.get("dontInlineLists", False): # sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",)) - __write_export(sq_settings, kwargs[options.REPORT_FILE]) + + log.info("Exporting project migration data streaming projects in '%s'", kwargs[options.REPORT_FILE]) + export_settings["WRITE_CALLBACK"] = write_project + __remove_chars_at_end(kwargs[options.REPORT_FILE], 3) + __add_project_header(kwargs[options.REPORT_FILE]) + projects.export(endpoint, export_settings=export_settings, key_list=key_list) + __add_project_footer(kwargs[options.REPORT_FILE]) + log.info("Exporting migration data from %s completed", kwargs["url"]) def main() -> None: diff --git a/sonar/projects.py b/sonar/projects.py index 1a0d02a62..f11558d44 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -1424,6 +1424,7 @@ def __export_thread(queue: Queue[Project], results: dict[str, str], export_setti while not queue.empty(): project = queue.get() results[project.key] = project.export(export_settings=export_settings) + export_settings["WRITE_CALLBACK"](results[project.key], export_settings["file"]) results[project.key].pop("key", None) with _CLASS_LOCK: export_settings["EXPORTED"] += 1