Stream project extract

cli/config.py

@@ -22,6 +22,9 @@
     Exports SonarQube platform configuration as JSON
 """
 import sys
+import os
+from threading import Lock
+
 import json
 import yaml
 
@@ -31,6 +34,8 @@
 from sonar import platform, rules, qualityprofiles, qualitygates, users, groups
 from sonar import projects, portfolios, applications
 
+WRITE_FILE = None
+
 _EVERYTHING = [
     options.WHAT_SETTINGS,
     options.WHAT_USERS,
@@ -67,6 +72,8 @@
     options.WHAT_PORTFOLIOS: __JSON_KEY_PORTFOLIOS,
 }
 
+_WRITE_LOCK = Lock()
+
 
 def __parse_args(desc):
     parser = options.set_common_args(desc)
@@ -113,6 +120,36 @@ def __write_export(config: dict[str, str], file: str, format: str) -> None:
             print(utilities.json_dump(config), file=fd)
 
 
+def __remove_chars_at_end(file: str, nb_bytes: int) -> None:
+    """Writes the configuration in file"""
+    with open(file, mode="rb+") as fd:
+        fd.seek(-nb_bytes, os.SEEK_END)
+        fd.truncate()
+
+
+def __add_project_header(file: str) -> None:
+    """Writes the configuration in file"""
+    with open(file, mode="a", encoding="utf-8") as fd:
+        print(',\n   "projects": {\n', file=fd)
+
+
+def __add_project_footer(file: str) -> None:
+    """Closes projects section"""
+    __remove_chars_at_end(file, 2)
+    with open(file, mode="a", encoding="utf-8") as fd:
+        print("\n   }\n}", file=fd)
+
+
+def write_project(project_json: dict[str, any], file: str) -> None:
+    """
+    writes a project JSON in a file
+    """
+    key = project_json.pop("key")
+    with _WRITE_LOCK:
+        with utilities.open_file(file, mode="a") as fd:
+            print(f'"{key}": {utilities.json_dump(project_json)},', file=fd)
+
+
 def __convert_for_yaml(json_export: dict[str, any]) -> dict[str, any]:
     """Converts the default JSON produced by export to a modified version more suitable for YAML"""
     if "globalSettings" in json_export:
@@ -143,6 +180,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N
         "EXPORT_DEFAULTS": kwargs["exportDefaults"],
         "FULL_EXPORT": kwargs["fullExport"],
         "THREADS": kwargs[options.NBR_THREADS],
+        options.REPORT_FILE: kwargs[options.REPORT_FILE],
     }
     if "projects" in what and kwargs[options.KEYS]:
         non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)]
@@ -154,7 +192,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N
         options.WHAT_RULES: [__JSON_KEY_RULES, rules.export],
         options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export],
         options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export],
-        options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export],
+        # options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export],
         options.WHAT_APPS: [__JSON_KEY_APPS, applications.export],
         options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export],
         options.WHAT_USERS: [__JSON_KEY_USERS, users.export],
@@ -165,17 +203,24 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N
     key_list = kwargs[options.KEYS]
     sq_settings = {__JSON_KEY_PLATFORM: endpoint.basics()}
     for what_item, call_data in calls.items():
-        if what_item not in what:
+        if what_item not in what or what_item == options.WHAT_PROJECTS:
             continue
         ndx, func = call_data
         try:
             sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list)
+            __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT])
         except exceptions.UnsupportedOperation as e:
             log.warning(e.message)
     sq_settings = utilities.remove_empties(sq_settings)
     if not kwargs["dontInlineLists"]:
         sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",))
-    __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT])
+
+    export_settings["WRITE_CALLBACK"] = write_project
+    __remove_chars_at_end(kwargs[options.REPORT_FILE], 3)
+    __add_project_header(kwargs[options.REPORT_FILE])
+    projects.export(endpoint, export_settings=export_settings, key_list=key_list)
+    __add_project_footer(kwargs[options.REPORT_FILE])
+
     log.info("Exporting configuration from %s completed", kwargs["url"])
 
 

migration/migration.py

@@ -22,6 +22,8 @@
     Exports SonarQube platform configuration as JSON
 """
 import sys
+import os
+from threading import Lock
 
 from cli import options
 from sonar import exceptions, errcodes, utilities, version
@@ -65,6 +67,8 @@
     options.WHAT_PORTFOLIOS: __JSON_KEY_PORTFOLIOS,
 }
 
+_WRITE_LOCK = Lock()
+
 
 def __parse_args(desc):
     parser = options.set_common_args(desc)
@@ -93,6 +97,36 @@ def __parse_args(desc):
     return args
 
 
+def __remove_chars_at_end(file: str, nb_bytes: int) -> None:
+    """Writes the configuration in file"""
+    with open(file, mode="rb+") as fd:
+        fd.seek(-nb_bytes, os.SEEK_END)
+        fd.truncate()
+
+
+def __add_project_header(file: str) -> None:
+    """Writes the configuration in file"""
+    with open(file, mode="a", encoding="utf-8") as fd:
+        print(',\n   "projects": {\n', file=fd)
+
+
+def __add_project_footer(file: str) -> None:
+    """Closes projects section"""
+    __remove_chars_at_end(file, 2)
+    with open(file, mode="a", encoding="utf-8") as fd:
+        print("\n   }\n}", file=fd)
+
+
+def write_project(project_json: dict[str, any], file: str) -> None:
+    """
+    writes a project JSON in a file
+    """
+    key = project_json.pop("key")
+    with _WRITE_LOCK:
+        with utilities.open_file(file, mode="a") as fd:
+            print(f'"{key}": {utilities.json_dump(project_json)},', file=fd)
+
+
 def __write_export(config: dict[str, str], file: str) -> None:
     """Writes the configuration in file"""
     with utilities.open_file(file) as fd:
@@ -108,6 +142,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N
         "FULL_EXPORT": False,
         "MODE": "MIGRATION",
         "THREADS": kwargs[options.NBR_THREADS],
+        options.REPORT_FILE: kwargs[options.REPORT_FILE],
         "SKIP_ISSUES": kwargs["skipIssues"],
     }
     if "projects" in what and kwargs[options.KEYS]:
@@ -120,7 +155,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N
         options.WHAT_RULES: [__JSON_KEY_RULES, rules.export],
         options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export],
         options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export],
-        options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export],
+        # options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export],
         options.WHAT_APPS: [__JSON_KEY_APPS, applications.export],
         options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export],
         options.WHAT_USERS: [__JSON_KEY_USERS, users.export],
@@ -136,12 +171,20 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N
         ndx, func = call_data
         try:
             sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list)
+            __write_export(sq_settings, kwargs[options.REPORT_FILE])
         except exceptions.UnsupportedOperation as e:
             log.warning(e.message)
     sq_settings = utilities.remove_empties(sq_settings)
     # if not kwargs.get("dontInlineLists", False):
     #    sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",))
-    __write_export(sq_settings, kwargs[options.REPORT_FILE])
+
+    log.info("Exporting project migration data streaming projects in '%s'", kwargs[options.REPORT_FILE])
+    export_settings["WRITE_CALLBACK"] = write_project
+    __remove_chars_at_end(kwargs[options.REPORT_FILE], 3)
+    __add_project_header(kwargs[options.REPORT_FILE])
+    projects.export(endpoint, export_settings=export_settings, key_list=key_list)
+    __add_project_footer(kwargs[options.REPORT_FILE])
+    log.info("Exporting migration data from %s completed", kwargs["url"])
 
 
 def main() -> None:

sonar/projects.py

@@ -1424,6 +1424,7 @@ def __export_thread(queue: Queue[Project], results: dict[str, str], export_setti
     while not queue.empty():
         project = queue.get()
         results[project.key] = project.export(export_settings=export_settings)
+        export_settings["WRITE_CALLBACK"](results[project.key], export_settings["file"])
         results[project.key].pop("key", None)
         with _CLASS_LOCK:
             export_settings["EXPORTED"] += 1