Merge pull request #1375 from okorach:optional-expensive-extract

Make expensive issue count extract optional

Author: okorach

migration/README.md

@@ -42,6 +42,7 @@ The user corresponding to the token must have sufficiently elevated permissions
 `ERROR` and above is always active.
 - `-c` or `--clientCert` : Allows to specify an optional client certificate file (as .pem file)
 - `--httpTimeout` : Sets the timeout for HTTP(S) requests to the SonarQube platform
+- `--skipIssues` : Skips the "expensive" issue count extract from the migration. This reduces by a factor of 2 to 3 the extract duration and the number of API calls
 - `--skipVersionCheck` : `sonar-migration` occasionnally checks on pypi.org if there is a new version of **sonar-migration** available, and output a warning log if that is the case. You can skip this check with this option.
 - `-l <logFile>` : Send logs to **<logFile>**, stdout by default
 
@@ -107,6 +108,11 @@ When sonar-migration complete successfully they return exit code 0. En case of f
 
 # What's New - Release notes
 
+## Version 0.3
+
+- Robustness: Handle `connectionError` errors in project extract threads
+- Added option `--skipIssues` to skip expensive issue count extraction task from the extract which may be necessary on large platforms extracts
+
 ## Version 0.2
 
 - Added export of:

migration/migration.py

@@ -73,6 +73,13 @@ def __parse_args(desc):
     parser = options.add_thread_arg(parser, "migration export")
     parser = options.set_what(parser, what_list=_EVERYTHING, operation="export")
     parser = options.add_import_export_arg(parser, "migration")
+    parser.add_argument(
+        "--skipIssues",
+        required=False,
+        default=False,
+        action="store_true",
+        help="Skips the export of issues count which might be costly iin terms of API calls",
+    )
     parser.add_argument(
         "--exportDefaults",
         required=False,
@@ -101,6 +108,7 @@ def __export_config(endpoint: platform.Platform, what: list[str], **kwargs) -> N
         "FULL_EXPORT": False,
         "MODE": "MIGRATION",
         "THREADS": kwargs[options.NBR_THREADS],
+        "SKIP_ISSUES": kwargs["skipIssues"],
     }
     if "projects" in what and kwargs[options.KEYS]:
         non_existing_projects = [key for key in kwargs[options.KEYS] if not projects.exists(key, endpoint)]
@@ -140,7 +148,7 @@ def main() -> None:
     """Main entry point for sonar-config"""
     start_time = utilities.start_clock()
     try:
-        kwargs = utilities.convert_args(__parse_args("Extract SonarQube platform configuration"))
+        kwargs = utilities.convert_args(__parse_args("Extract SonarQube to SonarCloud migration data"))
         endpoint = platform.Platform(**kwargs)
         endpoint.verify_connection()
         endpoint.set_user_agent(f"sonar-migration {version.MIGRATION_TOOL_VERSION}")

migration/what-is-new.md

@@ -1,4 +1,8 @@
-# Next version yet unreleased
+# Version 0.3
+
+- Robustness: Handle `connectionError` errors in project extract threads
+- Added option `--skipIssues` to skip expensive issue count extraction task from the extract which may be necessary on large platforms extracts
+
 
 # Version 0.2
 

sonar/branches.py

@@ -231,7 +231,7 @@ def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr:
         if export_settings.get("FULL_EXPORT", True):
             data.update({"name": self.name, "project": self.concerned_object.key})
         if export_settings.get("MODE", "") == "MIGRATION":
-            data.update(self.migration_export())
+            data.update(self.migration_export(export_settings))
         data = util.remove_nones(data)
         return None if len(data) == 0 else data
 

sonar/components.py

@@ -167,7 +167,7 @@ def get_hotspots(self, filters: types.ApiParams = None) -> dict[str, object]:
             params.update(filters)
         return search(endpoint=self.endpoint, filters=params)
 
-    def migration_export(self) -> dict[str, any]:
+    def migration_export(self, export_settings: types.ConfigSettings) -> dict[str, any]:
         from sonar.issues import count as issue_count
         from sonar.hotspots import count as hotspot_count
 
@@ -178,6 +178,10 @@ def migration_export(self) -> dict[str, any]:
             loc_distrib = {m.split("=")[0]: int(m.split("=")[1]) for m in lang_distrib.split(";")}
         loc_distrib["total"] = self.loc()
         json_data["ncloc"] = loc_distrib
+        if export_settings["SKIP_ISSUES"]:
+            log.debug("Issues count extract skipped for %s`", str(self))
+            return json_data
+
         tpissues = self.count_third_party_issues()
         inst_issues = self.count_instantiated_rules_issues()
         params = self.search_params()

sonar/projects.py

@@ -977,7 +977,7 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str,
             json_data = util.filter_export(json_data, _IMPORTABLE_PROPERTIES, export_settings.get("FULL_EXPORT", False))
 
             if export_settings.get("MODE", "") == "MIGRATION":
-                json_data.update(self.migration_export())
+                json_data.update(self.migration_export(export_settings))
                 json_data["detectedCi"] = self.ci()
                 json_data["revision"] = self.revision()
                 last_task = self.last_task()

sonar/version.py

@@ -25,4 +25,4 @@
 """
 
 PACKAGE_VERSION = "3.5"
-MIGRATION_TOOL_VERSION = "0.2"
+MIGRATION_TOOL_VERSION = "0.3"