Pass write Q as parameter of export

Author: okorach

migration/migration.py

@@ -118,8 +118,8 @@ def write_objects(queue: Queue, fd, object_type: str) -> None:
                 log.info("Writing %s key '%s'", object_type, key)
                 print(f'{prefix}"{key}": {utilities.json_dump(obj_json)}', end="", file=fd)
             else:
-                log.info("Writing %s global", object_type)
-                print(f"{prefix}{utilities.json_dump(obj_json)[1:-1]}", end="", file=fd)
+                log.info("Writing %s", object_type)
+                print(f"{prefix}{utilities.json_dump(obj_json)[2:-1]}", end="", file=fd)
             prefix = ",\n"
         queue.task_done()
     print("\n}", file=fd, end="")
@@ -172,10 +172,9 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N
                 is_first = False
                 worker = Thread(target=write_objects, args=(q, fd, ndx))
                 worker.daemon = True
-                worker.name = f"Write{ndx}"
+                worker.name = f"Write{ndx[:1].upper()}{ndx[1:10]}"
                 worker.start()
-                export_settings["WRITE_QUEUE"] = q
-                sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list)
+                sq_settings[ndx] = func(endpoint, export_settings=export_settings, key_list=key_list, write_q=q)
                 q.join()
             except exceptions.UnsupportedOperation as e:
                 log.warning(e.message)

sonar/applications.py

@@ -19,6 +19,7 @@
 #
 
 from __future__ import annotations
+from queue import Queue
 from typing import Union
 
 import json
@@ -498,7 +499,9 @@ def exists(endpoint: pf.Platform, key: str) -> bool:
         return False
 
 
-def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None) -> types.ObjectJsonRepr:
+def export(
+    endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None, write_q: Queue = None
+) -> types.ObjectJsonRepr:
     """Exports applications as JSON
 
     :param Platform endpoint: Reference to the Sonar platform
@@ -514,12 +517,13 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
     apps_settings = {}
     for k, app in sorted(get_list(endpoint, key_list).items()):
         app_json = app.export(export_settings)
-        if export_settings.get("WRITE_QUEUE", None):
-            export_settings["WRITE_QUEUE"].put(app_json)
+        if write_q:
+            write_q.put(app_json)
         else:
             app_json.pop("key")
             apps_settings[k] = app_json
-    export_settings["WRITE_QUEUE"].put(None)
+    if write_q:
+        write_q.put(None)
     return apps_settings
 
 

sonar/groups.py

@@ -19,6 +19,7 @@
 #
 
 from __future__ import annotations
+from queue import Queue
 from typing import Optional
 import sonar.logging as log
 import sonar.platform as pf
@@ -256,7 +257,9 @@ def get_list(endpoint: pf.Platform) -> dict[str, Group]:
     return search(endpoint)
 
 
-def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None) -> types.ObjectJsonRepr:
+def export(
+    endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: Optional[types.KeyList] = None, write_q: Optional[Queue] = None
+) -> types.ObjectJsonRepr:
     """Exports groups representation in JSON
 
     :param Platform endpoint: reference to the SonarQube platform
@@ -272,9 +275,9 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
         if not export_settings["FULL_EXPORT"] and g_obj.is_default():
             continue
         g_list[g_name] = "" if g_obj.description is None else g_obj.description
-    if export_settings.get("WRITE_QUEUE", None):
-        export_settings["WRITE_QUEUE"].put(g_list)
-    export_settings["WRITE_QUEUE"].put(None)
+    if write_q:
+        write_q.put(g_list)
+        write_q.put(None)
     return g_list
 
 

sonar/platform.py

@@ -27,6 +27,7 @@
 from http import HTTPStatus
 import sys
 import os
+from queue import Queue
 from typing import Optional
 import time
 import datetime
@@ -869,7 +870,9 @@ def convert_for_yaml(original_json: types.ObjectJsonRepr) -> types.ObjectJsonRep
     return original_json
 
 
-def export(endpoint: Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None) -> types.ObjectJsonRepr:
+def export(
+    endpoint: Platform, export_settings: types.ConfigSettings, key_list: Optional[types.KeyList] = None, write_q: Optional[Queue] = None
+) -> types.ObjectJsonRepr:
     """Exports all or a list of projects configuration as dict
 
     :param Platform endpoint: reference to the SonarQube platform
@@ -879,7 +882,7 @@ def export(endpoint: Platform, export_settings: types.ConfigSettings, key_list:
     :rtype: ObjectJsonRepr
     """
     exp = endpoint.export(export_settings)
-    if export_settings.get("WRITE_QUEUE", None):
-        export_settings["WRITE_QUEUE"].put(exp)
-        export_settings["WRITE_QUEUE"].put(None)
+    if write_q:
+        write_q.put(exp)
+        write_q.put(None)
     return exp

sonar/portfolios.py

@@ -24,6 +24,7 @@
 """
 
 from __future__ import annotations
+from queue import Queue
 from typing import Union, Optional
 import json
 import datetime
@@ -727,7 +728,9 @@ def search_by_key(endpoint: pf.Platform, key: str) -> types.ApiPayload:
     return util.search_by_key(endpoint, key, Portfolio.SEARCH_API, "components")
 
 
-def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None) -> types.ObjectJsonRepr:
+def export(
+    endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: Optional[types.KeyList] = None, write_q: Optional[Queue] = None
+) -> types.ObjectJsonRepr:
     """Exports portfolios as JSON
 
     :param Platform endpoint: Reference to the SonarQube platform
@@ -749,8 +752,8 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
         try:
             if not p.is_sub_portfolio:
                 exp = p.export(export_settings)
-                if export_settings.get("WRITE_QUEUE", None):
-                    export_settings["WRITE_QUEUE"].put(exp)
+                if write_q:
+                    write_q.put(exp)
                 else:
                     exp.pop("key")
                     exported_portfolios[k] = exp
@@ -763,7 +766,8 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
         i += 1
         if i % 10 == 0 or i == nb_portfolios:
             log.info("Exported %d/%d portfolios (%d%%)", i, nb_portfolios, (i * 100) // nb_portfolios)
-    export_settings["WRITE_QUEUE"].put(None)
+    if write_q:
+        write_q.put(None)
     return exported_portfolios
 
 

sonar/projects.py

@@ -26,6 +26,7 @@
 from __future__ import annotations
 
 import os
+from queue import Queue
 import re
 import json
 from datetime import datetime
@@ -1473,14 +1474,13 @@ def audit(endpoint: pf.Platform, audit_settings: types.ConfigSettings, key_list:
     return problems
 
 
-def __export_thread(queue: Queue[Project], results: dict[str, str], export_settings: types.ConfigSettings) -> None:
+def __export_thread(queue: Queue[Project], results: dict[str, str], export_settings: types.ConfigSettings, write_q: Optional[Queue] = None) -> None:
     """Project export callback function for multitheaded export"""
     while not queue.empty():
         project = queue.get()
         exp_json = project.export(export_settings=export_settings)
-        if export_settings.get("WRITE_QUEUE", None):
-            log.info("Putting %s in queue %s", project.key, str(export_settings["WRITE_QUEUE"]))
-            export_settings["WRITE_QUEUE"].put(exp_json)
+        if write_q:
+            write_q.put(exp_json)
         else:
             results[project.key] = exp_json
             results[project.key].pop("key", None)
@@ -1490,10 +1490,11 @@ def __export_thread(queue: Queue[Project], results: dict[str, str], export_setti
         if nb % 10 == 0 or nb == tot:
             log.info("%d/%d projects exported (%d%%)", nb, tot, (nb * 100) // tot)
         queue.task_done()
-    log.info("Putting DONE in queue %s", str(export_settings["WRITE_QUEUE"]))
 
 
-def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None) -> types.ObjectJsonRepr:
+def export(
+    endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: Optional[types.KeyList] = None, write_q: Optional[Queue] = None
+) -> types.ObjectJsonRepr:
     """Exports all or a list of projects configuration as dict
 
     :param Platform endpoint: reference to the SonarQube platform
@@ -1515,12 +1516,13 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
     project_settings = {}
     for i in range(export_settings.get("THREADS", 8)):
         log.debug("Starting project export thread %d", i)
-        worker = Thread(target=__export_thread, args=(q, project_settings, export_settings))
-        worker.setDaemon(True)
-        worker.setName(f"ProjectExport{i}")
+        worker = Thread(target=__export_thread, args=(q, project_settings, export_settings, write_q))
+        worker.daemon = True
+        worker.name = f"ProjectExport{i}"
         worker.start()
     q.join()
-    export_settings["WRITE_QUEUE"].put(None)
+    if write_q:
+        write_q.put(None)
     return dict(sorted(project_settings.items()))
 
 

sonar/qualitygates.py

@@ -24,8 +24,8 @@
 """
 
 from __future__ import annotations
-
-from typing import Union
+from queue import Queue
+from typing import Union, Optional
 
 from http import HTTPStatus
 import json
@@ -380,7 +380,9 @@ def get_list(endpoint: pf.Platform) -> dict[str, QualityGate]:
     return qg_list
 
 
-def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None) -> types.ObjectJsonRepr:
+def export(
+    endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: Optional[types.KeyList] = None, write_q: Optional[Queue] = None
+) -> types.ObjectJsonRepr:
     """Exports quality gates as JSON
 
     :param Platform endpoint: Reference to the Sonar platform
@@ -391,9 +393,9 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
     """
     log.info("Exporting quality gates")
     qg_list = {k: qg.to_json(export_settings) for k, qg in sorted(get_list(endpoint).items())}
-    if export_settings.get("WRITE_QUEUE", None):
-        export_settings["WRITE_QUEUE"].put(qg_list)
-        export_settings["WRITE_QUEUE"].put(None)
+    if write_q:
+        write_q.put(qg_list)
+        write_q.put(None)
     return qg_list
 
 

sonar/qualityprofiles.py

@@ -19,7 +19,8 @@
 #
 
 from __future__ import annotations
-from typing import Union
+from queue import Queue
+from typing import Union, Optional
 import json
 from datetime import datetime
 
@@ -578,7 +579,9 @@ def hierarchize(qp_list: dict[str, str], endpoint: pf.Platform) -> types.ObjectJ
     return qp_list
 
 
-def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None) -> types.ObjectJsonRepr:
+def export(
+    endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: Optional[types.KeyList] = None, write_q: Optional[Queue] = None
+) -> types.ObjectJsonRepr:
     """Exports all or a list of quality profiles configuration as dict
 
     :param Platform endpoint: reference to the SonarQube platform
@@ -598,9 +601,9 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
             qp_list[lang] = {}
         qp_list[lang][name] = json_data
     qp_list = hierarchize(qp_list, endpoint)
-    if export_settings.get("WRITE_QUEUE", None):
-        export_settings["WRITE_QUEUE"].put(qp_list)
-        export_settings["WRITE_QUEUE"].put(None)
+    if write_q:
+        write_q.put(qp_list)
+        write_q.put(None)
     return dict(sorted(qp_list.items()))
 
 

sonar/rules.py

@@ -23,6 +23,7 @@
 
 """
 from __future__ import annotations
+from queue import Queue
 import json
 from typing import Optional
 from http import HTTPStatus
@@ -271,7 +272,9 @@ def get_object(endpoint: platform.Platform, key: str) -> Optional[Rule]:
         return None
 
 
-def export(endpoint: platform.Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None) -> types.ObjectJsonRepr:
+def export(
+    endpoint: platform.Platform, export_settings: types.ConfigSettings, key_list: Optional[types.KeyList] = None, write_q: Optional[Queue] = None
+) -> types.ObjectJsonRepr:
     """Returns a JSON export of all rules"""
     log.info("Exporting rules")
     full = export_settings.get("FULL_EXPORT", False)
@@ -299,9 +302,9 @@ def export(endpoint: platform.Platform, export_settings: types.ConfigSettings, k
         rule_list["standard"] = other_rules
     if export_settings.get("MODE", "") == "MIGRATION":
         rule_list["thirdParty"] = {r.key: r.export() for r in third_party(endpoint=endpoint)}
-    if export_settings.get("WRITE_QUEUE", None):
-        export_settings["WRITE_QUEUE"].put(rule_list)
-        export_settings["WRITE_QUEUE"].put(None)
+    if write_q:
+        write_q.put(rule_list)
+        write_q.put(None)
     return rule_list
 
 

sonar/users.py

@@ -19,7 +19,7 @@
 #
 
 from __future__ import annotations
-
+from queue import Queue
 from typing import Union, Optional
 import datetime as dt
 import json
@@ -404,7 +404,9 @@ def search(endpoint: pf.Platform, params: types.ApiParams = None) -> dict[str, U
     return sqobject.search_objects(endpoint=endpoint, object_class=User, params=params)
 
 
-def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: types.KeyList = None) -> types.ObjectJsonRepr:
+def export(
+    endpoint: pf.Platform, export_settings: types.ConfigSettings, key_list: Optional[types.KeyList] = None, write_q: Optional[Queue] = None
+) -> types.ObjectJsonRepr:
     """Exports all users in JSON representation
 
     :param Platform endpoint: reference to the SonarQube platform
@@ -417,11 +419,12 @@ def export(endpoint: pf.Platform, export_settings: types.ConfigSettings, key_lis
     u_list = {}
     for u_login, u_obj in sorted(search(endpoint=endpoint).items()):
         u_list[u_login] = u_obj.to_json(export_settings)
-        if export_settings.get("WRITE_QUEUE", None):
-            export_settings["WRITE_QUEUE"].put(u_list[u_login])
+        if write_q:
+            write_q.put(u_list[u_login])
         else:
             u_list[u_login].pop("login", None)
-    export_settings["WRITE_QUEUE"].put(None)
+    if write_q:
+        write_q.put(None)
     return u_list