Mutualize migration and config (#1423)

* Mutualize config and migration

* Rename tests

* Add constants for cli options

* Fix type of parse_and_check

* Add type hints

Author: okorach

cli/config.py

@@ -36,22 +36,11 @@
 from sonar import platform, rules, qualityprofiles, qualitygates, users, groups
 from sonar import projects, portfolios, applications
 
-WRITE_FILE = None
-
-_EVERYTHING = [
-    options.WHAT_SETTINGS,
-    options.WHAT_USERS,
-    options.WHAT_GROUPS,
-    options.WHAT_GATES,
-    options.WHAT_RULES,
-    options.WHAT_PROFILES,
-    options.WHAT_PROJECTS,
-    options.WHAT_APPS,
-    options.WHAT_PORTFOLIOS,
-]
-
 TOOL_NAME = "sonar-config"
 
+DONT_INLINE_LISTS = "dontInlineLists"
+FULL_EXPORT = "fullExport"
+
 __JSON_KEY_PLATFORM = "platform"
 __JSON_KEY_SETTINGS = "globalSettings"
 __JSON_KEY_USERS = "users"
@@ -63,21 +52,8 @@
 __JSON_KEY_APPS = "applications"
 __JSON_KEY_PORTFOLIOS = "portfolios"
 
-__MAP = {
-    options.WHAT_SETTINGS: __JSON_KEY_SETTINGS,
-    options.WHAT_USERS: __JSON_KEY_USERS,
-    options.WHAT_GROUPS: __JSON_KEY_GROUPS,
-    options.WHAT_GATES: __JSON_KEY_GATES,
-    options.WHAT_RULES: __JSON_KEY_RULES,
-    options.WHAT_PROFILES: __JSON_KEY_PROFILES,
-    options.WHAT_PROJECTS: __JSON_KEY_PROJECTS,
-    options.WHAT_APPS: __JSON_KEY_APPS,
-    options.WHAT_PORTFOLIOS: __JSON_KEY_PORTFOLIOS,
-}
-
-
 _EXPORT_CALLS = {
-    "platform": [__JSON_KEY_PLATFORM, platform.basics],
+    __JSON_KEY_PLATFORM: [__JSON_KEY_PLATFORM, platform.basics],
     options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export],
     options.WHAT_RULES: [__JSON_KEY_RULES, rules.export],
     options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export],
@@ -89,16 +65,19 @@
     options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export],
 }
 
+_EVERYTHING = list(_EXPORT_CALLS.keys())[1:]
+
 
-def __parse_args(desc):
+def __parse_args(desc: str) -> object:
+    """Sets and parses all sonar-config options"""
     parser = options.set_common_args(desc)
     parser = options.set_key_arg(parser)
     parser = options.set_output_file_args(parser, allowed_formats=("json", "yaml"))
     parser = options.add_thread_arg(parser, "project export")
     parser = options.set_what(parser, what_list=_EVERYTHING, operation="export or import")
     parser = options.add_import_export_arg(parser, "configuration")
     parser.add_argument(
-        "--fullExport",
+        f"--{FULL_EXPORT}",
         required=False,
         default=False,
         action="store_true",
@@ -115,7 +94,7 @@ def __parse_args(desc):
         "and the setting will not be imported at import time",
     )
     parser.add_argument(
-        "--dontInlineLists",
+        f"--{DONT_INLINE_LISTS}",
         required=False,
         default=False,
         action="store_true",
@@ -157,96 +136,55 @@ def __convert_for_yaml(json_export: dict[str, any]) -> dict[str, any]:
     return json_export
 
 
-def __export_config_sync(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
-    """Exports config in a synchronous way"""
-    export_settings = {
-        "INLINE_LISTS": not kwargs["dontInlineLists"],
-        "EXPORT_DEFAULTS": kwargs["exportDefaults"],
-        "FULL_EXPORT": kwargs["fullExport"],
-        "THREADS": kwargs[options.NBR_THREADS],
-        options.REPORT_FILE: kwargs[options.REPORT_FILE],
-    }
-    if "projects" in what and kwargs[options.KEYS]:
-        non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)]
-        if len(non_existing_projects) > 0:
-            utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY)
-    log.info("Exporting configuration synchronously from %s", kwargs[options.URL])
-    key_list = kwargs[options.KEYS]
-    what.append("platform")
-    sq_settings = {}
-    for what_item, call_data in _EXPORT_CALLS.items():
-        if what_item not in what:
-            continue
-        ndx, func = call_data
-        try:
-            sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list)
-        except exceptions.UnsupportedOperation as e:
-            log.warning(e.message)
-        except exceptions.ObjectNotFound as e:
-            log.error(e.message)
-    sq_settings = __prep_json_for_write(sq_settings, export_settings)
-    __write_export(sq_settings, kwargs[options.REPORT_FILE], kwargs[options.FORMAT])
-    log.info("Synchronous export of configuration from %s completed", kwargs["url"])
-
-
-def __prep_json_for_write(json_data: types.ObjectJsonRepr, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
-    """Cleans up the JSON before writing"""
-    json_data = utilities.sort_lists(json_data)
-    if not export_settings.get("FULL_EXPORT", False):
-        json_data = utilities.remove_empties(utilities.remove_nones(json_data))
-    if export_settings.get("INLINE_LISTS", True):
-        json_data = utilities.inline_lists(json_data, exceptions=("conditions",))
-    return json_data
-
-
 def write_objects(queue: Queue[types.ObjectJsonRepr], fd: TextIO, object_type: str, export_settings: types.ConfigSettings) -> None:
     """
     Thread to write projects in the JSON file
     """
+    done = False
     prefix = ""
     log.info("Waiting %s to write...", object_type)
     print(f'"{object_type}": ' + "{", file=fd)
-    while True:
+    while not done:
         obj_json = queue.get()
-        if obj_json is None:
-            queue.task_done()
-            break
-        obj_json = __prep_json_for_write(obj_json, export_settings)
-        if object_type in ("projects", "applications", "portfolios", "users"):
-            if object_type == "users":
-                key = obj_json.pop("login", None)
+        done = obj_json is None
+        if not done:
+            obj_json = __prep_json_for_write(obj_json, export_settings)
+            if object_type in ("projects", "applications", "portfolios", "users"):
+                if object_type == "users":
+                    key = obj_json.pop("login", None)
+                else:
+                    key = obj_json.pop("key", None)
+                log.debug("Writing %s key '%s'", object_type[:-1], key)
+                print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd)
             else:
-                key = obj_json.pop("key", None)
-            log.debug("Writing %s key '%s'", object_type[:-1], key)
-            print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd)
-        else:
-            log.debug("Writing %s", object_type)
-            print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd)
-        prefix = ",\n"
+                log.debug("Writing %s", object_type)
+                print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd)
+            prefix = ",\n"
         queue.task_done()
     print("\n}", file=fd, end="")
     log.info("Writing %s complete", object_type)
 
 
-def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
+def export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
     """Exports a platform configuration in a JSON file"""
     file = kwargs[options.REPORT_FILE]
+    mode = kwargs.get("mode", "CONFIG")
     export_settings = {
-        "INLINE_LISTS": not kwargs["dontInlineLists"],
-        "EXPORT_DEFAULTS": kwargs["exportDefaults"],
-        "FULL_EXPORT": kwargs["fullExport"],
-        "MODE": "CONFIG",
+        "INLINE_LISTS": False if mode == "MIGRATION" else not kwargs.get(DONT_INLINE_LISTS, False),
+        "EXPORT_DEFAULTS": True,
+        "FULL_EXPORT": False if mode == "MIGRATION" else kwargs.get(FULL_EXPORT, False),
+        "MODE": mode,
         "THREADS": kwargs[options.NBR_THREADS],
-        "SKIP_ISSUES": True,
+        "SKIP_ISSUES": kwargs.get("skipIssues", False),
     }
     if "projects" in what and kwargs[options.KEYS]:
         non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)]
         if len(non_existing_projects) > 0:
             utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY)
 
+    what.append(__JSON_KEY_PLATFORM)
     log.info("Exporting configuration from %s", kwargs[options.URL])
     key_list = kwargs[options.KEYS]
-    what.append("platform")
     is_first = True
     q = Queue(maxsize=0)
     with utilities.open_file(file, mode="w") as fd:
@@ -268,16 +206,20 @@ def __export_config_async(endpoint: platform.Platform, what: list[str], **kwargs
             except exceptions.UnsupportedOperation as e:
                 log.warning(e.message)
         print("\n}", file=fd)
-    utilities.normalize_json_file(file, remove_empty=True, remove_none=True)
+    utilities.normalize_json_file(file, remove_empty=False, remove_none=True)
     log.info("Exporting migration data from %s completed", kwargs["url"])
 
 
-def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
-    """Exports the configuration of the SonarQube platform"""
-    if kwargs[options.KEYS] or options.WHAT_PROJECTS not in what or kwargs[options.FORMAT] != "json":
-        __export_config_sync(endpoint=endpoint, what=what, **kwargs)
-    else:
-        __export_config_async(endpoint=endpoint, what=what, **kwargs)
+def __prep_json_for_write(json_data: types.ObjectJsonRepr, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
+    """Cleans up the JSON before writing"""
+    json_data = utilities.sort_lists(json_data)
+    if export_settings.get("MODE", "CONFIG") == "MIGRATION":
+        return json_data
+    if not export_settings.get("FULL_EXPORT", False):
+        json_data = utilities.remove_empties(utilities.remove_nones(json_data))
+    if export_settings.get("INLINE_LISTS", True):
+        json_data = utilities.inline_lists(json_data, exceptions=("conditions",))
+    return json_data
 
 
 def __import_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
@@ -330,7 +272,7 @@ def main() -> None:
     kwargs[options.FORMAT] = utilities.deduct_format(kwargs[options.FORMAT], kwargs[options.REPORT_FILE], allowed_formats=("json", "yaml"))
     if kwargs[options.EXPORT]:
         try:
-            __export_config(endpoint, what, **kwargs)
+            export_config(endpoint, what, **kwargs)
         except exceptions.ObjectNotFound as e:
             utilities.exit_fatal(e.message, errcodes.NO_SUCH_KEY)
         except (PermissionError, FileNotFoundError) as e:

cli/options.py

@@ -176,7 +176,7 @@ def __check_file_writeable(file: str) -> None:
         os.remove(file)
 
 
-def parse_and_check(parser: ArgumentParser, logger_name: str = None, verify_token: bool = True, is_migration: bool = False) -> ArgumentParser:
+def parse_and_check(parser: ArgumentParser, logger_name: str = None, verify_token: bool = True, is_migration: bool = False) -> object:
     """Parses arguments, applies default settings and perform common environment checks"""
     try:
         args = parser.parse_args()

migration/migration.py

@@ -22,14 +22,11 @@
     Exports SonarQube platform configuration as JSON
 """
 import sys
-from threading import Thread
-from queue import Queue
 
-from cli import options
+from cli import options, config
 from sonar import exceptions, errcodes, utilities, version
 import sonar.logging as log
-from sonar import platform, rules, qualityprofiles, qualitygates, users, groups
-from sonar import projects, portfolios, applications
+from sonar import platform
 
 TOOL_NAME = "sonar-migration"
 
@@ -97,94 +94,6 @@ def __parse_args(desc):
     return args
 
 
-def write_objects(queue: Queue, fd, object_type: str) -> None:
-    """
-    Thread to write projects in the JSON file
-    """
-    done = False
-    prefix = ""
-    log.info("Waiting %s to write...", object_type)
-    print(f'"{object_type}": ' + "{", file=fd)
-    while not done:
-        obj_json = queue.get()
-        done = obj_json is None
-        if not done:
-            obj_json = utilities.remove_nones(obj_json)
-            if object_type in ("projects", "applications", "portfolios", "users"):
-                if object_type == "users":
-                    key = obj_json.pop("login", None)
-                else:
-                    key = obj_json.pop("key", None)
-                log.debug("Writing %s key '%s'", object_type[:-1], key)
-                print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd)
-            else:
-                log.debug("Writing %s", object_type)
-                print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd)
-            prefix = ",\n"
-        queue.task_done()
-    print("\n}", file=fd, end="")
-    log.info("Writing %s complete", object_type)
-
-
-def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> None:
-    """Exports a platform configuration in a JSON file"""
-    file = kwargs[options.REPORT_FILE]
-    export_settings = {
-        "INLINE_LISTS": False,
-        "EXPORT_DEFAULTS": True,
-        # "FULL_EXPORT": kwargs["fullExport"],
-        "FULL_EXPORT": False,
-        "MODE": "MIGRATION",
-        "THREADS": kwargs[options.NBR_THREADS],
-        "SKIP_ISSUES": kwargs["skipIssues"],
-    }
-    if "projects" in what and kwargs[options.KEYS]:
-        non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)]
-        if len(non_existing_projects) > 0:
-            utilities.exit_fatal(f"Project key(s) '{','.join(non_existing_projects)}' do(es) not exist", errcodes.NO_SUCH_KEY)
-
-    calls = {
-        "platform": [__JSON_KEY_PLATFORM, platform.basics],
-        options.WHAT_SETTINGS: [__JSON_KEY_SETTINGS, platform.export],
-        options.WHAT_RULES: [__JSON_KEY_RULES, rules.export],
-        options.WHAT_PROFILES: [__JSON_KEY_PROFILES, qualityprofiles.export],
-        options.WHAT_GATES: [__JSON_KEY_GATES, qualitygates.export],
-        options.WHAT_PROJECTS: [__JSON_KEY_PROJECTS, projects.export],
-        options.WHAT_APPS: [__JSON_KEY_APPS, applications.export],
-        options.WHAT_PORTFOLIOS: [__JSON_KEY_PORTFOLIOS, portfolios.export],
-        options.WHAT_USERS: [__JSON_KEY_USERS, users.export],
-        options.WHAT_GROUPS: [__JSON_KEY_GROUPS, groups.export],
-    }
-    what.append("platform")
-    log.info("Exporting configuration from %s", kwargs[options.URL])
-    key_list = kwargs[options.KEYS]
-    is_first = True
-    q = Queue(maxsize=0)
-    with utilities.open_file(file, mode="w") as fd:
-        print("{", file=fd)
-        for what_item, call_data in calls.items():
-            if what_item not in what:
-                continue
-            ndx, func = call_data
-            try:
-                if not is_first:
-                    print(",", file=fd)
-                is_first = False
-                worker = Thread(target=write_objects, args=(q, fd, ndx))
-                worker.daemon = True
-                worker.name = f"Write{ndx[:1].upper()}{ndx[1:10]}"
-                worker.start()
-                func(endpoint, export_settings=export_settings, key_list=key_list, write_q=q)
-                q.join()
-            except exceptions.UnsupportedOperation as e:
-                log.warning(e.message)
-        # if not kwargs.get("dontInlineLists", False):
-        #    sq_settings = utilities.inline_lists(sq_settings, exceptions=("conditions",))
-        print("\n}", file=fd)
-    utilities.normalize_json_file(file, remove_empty=False, remove_none=True)
-    log.info("Exporting migration data from %s completed", kwargs["url"])
-
-
 def main() -> None:
     """Main entry point for sonar-config"""
     start_time = utilities.start_clock()
@@ -203,7 +112,7 @@ def main() -> None:
     if kwargs[options.REPORT_FILE] is None:
         kwargs[options.REPORT_FILE] = f"sonar-migration.{endpoint.server_id()}.json"
     try:
-        __export_config(endpoint, what, **kwargs)
+        config.export_config(endpoint, what, mode="MIGRATION", **kwargs)
     except exceptions.ObjectNotFound as e:
         utilities.exit_fatal(e.message, errcodes.NO_SUCH_KEY)
     except (PermissionError, FileNotFoundError) as e:

test/test_config.py

@@ -102,7 +102,7 @@ def test_config_non_existing_project() -> None:
     util.clean(util.JSON_FILE)
 
 
-def test_config_inline_commas() -> None:
+def test_config_inline_lists() -> None:
     """test_config_inline_commas"""
     util.clean(util.JSON_FILE)
     with pytest.raises(SystemExit):
@@ -124,7 +124,7 @@ def test_config_inline_commas() -> None:
     util.clean(util.JSON_FILE)
 
 
-def test_config_no_inline_commas() -> None:
+def test_config_dont_inline_lists() -> None:
     """test_config_no_inline_commas"""
     util.clean(util.JSON_FILE)
     with pytest.raises(SystemExit):