Fixes #2039

Author: okorach-sonar

sonar/errcodes.py

@@ -69,3 +69,6 @@
 OBJECT_NOT_FOUND = 16
 
 SONAR_INTERNAL_ERROR = 17
+
+
+TOO_MANY_RESULTS = 18

sonar/exceptions.py

@@ -74,3 +74,11 @@ class ConnectionError(SonarException):
 
     def __init__(self, message: str) -> None:
         super().__init__(message, errcodes.CONNECTION_ERROR)
+
+
+class TooManyResults(SonarException):
+    """When a call to APIs returns too many results."""
+
+    def __init__(self, nbr_results: int, message: str) -> None:
+        super().__init__(message, errcodes.TOO_MANY_RESULTS)
+        self.nbr_results = nbr_results

sonar/projects.py

@@ -41,6 +41,8 @@
 import sonar.platform as pf
 
 from sonar.util import types, cache
+from sonar.util import project_utils as putils
+
 from sonar import exceptions, errcodes
 from sonar import sqobject, components, qualitygates, qualityprofiles, tasks, settings, webhooks, devops
 import sonar.permissions.permissions as perms
@@ -1443,14 +1445,22 @@ def count(endpoint: pf.Platform, params: types.ApiParams = None) -> int:
 def search(endpoint: pf.Platform, params: types.ApiParams = None, threads: int = 8) -> dict[str, Project]:
     """Searches projects in SonarQube
 
-    :param endpoint: Reference to the SonarQube platform
-    :param params: list of parameters to narrow down the search
-    :returns: list of projects
+    :param Platform endpoint: Reference to the SonarQube platform
+    :param ApiParams params: list of filter parameters to narrow down the search
+    :param int threads: Number of threads to use for the search
     """
     new_params = {} if params is None else params.copy()
-    if not endpoint.is_sonarcloud():
+    if not endpoint.is_sonarcloud() and not new_params.get("filter", None):
         new_params["filter"] = _PROJECT_QUALIFIER
-    return sqobject.search_objects(endpoint=endpoint, object_class=Project, params=new_params, threads=threads)
+    try:
+        log.info("Searching projects with parameters: %s", str(new_params))
+        return sqobject.search_objects(endpoint=endpoint, object_class=Project, params=new_params, threads=threads)
+    except exceptions.TooManyResults as e:
+        log.warning(e.message)
+        filter_1, filter_2 = putils.split_loc_filter(new_params["filter"])
+        return search(endpoint, params={**new_params, "filter": filter_1}, threads=threads) | search(
+            endpoint, params={**new_params, "filter": filter_2}, threads=threads
+        )
 
 
 def get_list(endpoint: pf.Platform, key_list: types.KeyList = None, threads: int = 8, use_cache: bool = True) -> dict[str, Project]:
@@ -1461,8 +1471,8 @@ def get_list(endpoint: pf.Platform, key_list: types.KeyList = None, threads: int
     :return: the list of all projects
     :rtype: dict{key: Project}
     """
-    with _CLASS_LOCK:
-        if key_list is None or len(key_list) == 0 or not use_cache:
+    if key_list is None or len(key_list) == 0 or not use_cache:
+        with _CLASS_LOCK:
             log.info("Listing projects")
             p_list = dict(sorted(search(endpoint=endpoint, threads=threads).items()))
             return p_list

sonar/sqobject.py

@@ -234,6 +234,10 @@ def search_objects(endpoint: object, object_class: any, params: types.ApiParams,
     data = __get(endpoint, api, {**new_params, p_field: 1})
     nb_pages = utilities.nbr_pages(data, api_version)
     nb_objects = max(len(data[returned_field]), utilities.nbr_total_elements(data, api_version))
+
+    if nb_objects > c.ELASTIC_MAX_RESULTS:
+        raise exceptions.TooManyResults(nb_objects, f"Too many {cname}s ({nb_objects}) returned by search")
+
     log.info(
         "Searching %d %ss, %d pages of %d elements, %d pages in parallel...",
         nb_objects,

sonar/util/constants.py

@@ -74,3 +74,5 @@
 
 SQS_USERS = "sonar-users"  # SonarQube Server users default group name
 SQC_USERS = "Members"  # SonarQube Cloud users default group name
+
+ELASTIC_MAX_RESULTS = 10000  # ElasticSearch max results limit

sonar/util/project_utils.py

@@ -0,0 +1,55 @@
+#
+# sonar-tools
+# Copyright (C) 2025 Olivier Korach
+# mailto:olivier.korach AT gmail DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
+import math
+
+
+def split_loc_filter(loc_filter: str) -> tuple[str, str]:
+    """Parses a ncloc filter and returns new filters to split the search"""
+    __FILTER_AND = " and "
+    loc_min, loc_max = 0, 100000000
+    new_filters = []
+    for f in loc_filter.split(__FILTER_AND):
+        if f.startswith("ncloc>="):
+            try:
+                loc_min = int(f[len("ncloc>=") :])
+            except ValueError:
+                pass
+        elif f.startswith("ncloc>"):
+            try:
+                loc_min = int(f[len("ncloc>") :]) + 1
+            except ValueError:
+                pass
+        elif f.startswith("ncloc<="):
+            try:
+                loc_max = int(f[len("ncloc<=") :])
+            except ValueError:
+                pass
+        elif f.startswith("ncloc<"):
+            try:
+                loc_max = int(f[len("ncloc<") :]) - 1
+            except ValueError:
+                pass
+        else:
+            new_filters.append(f)
+    loc_middle = int(math.exp((math.log2(loc_max) - math.log2(max(loc_min, 1))) / 2))
+    slice1 = __FILTER_AND.join(new_filters + [f"ncloc>={loc_min}", f"ncloc<={loc_middle}"])
+    slice2 = __FILTER_AND.join(new_filters + [f"ncloc>{loc_middle}", f"ncloc<={loc_max}"])
+    return slice1, slice2