diff --git a/conf/release.Dockerfile b/conf/release.Dockerfile index bec6e8d6..b9f75c73 100644 --- a/conf/release.Dockerfile +++ b/conf/release.Dockerfile @@ -28,11 +28,11 @@ COPY ./LICENSE . COPY ./sonar/audit sonar/audit RUN pip install --upgrade pip \ -&& pip install sonar-tools==3.17 +&& pip install sonar-tools==3.16.1 USER ${USERNAME} WORKDIR /home/${USERNAME} -HEALTHCHECK --interval=180s --timeout=5s CMD [ "sonar-tools-help" ] +HEALTHCHECK --interval=180s --timeout=5s CMD [ "sonar-tools" ] -CMD [ "sonar-tools-help" ] +CMD [ "sonar-tools" ] diff --git a/doc/what-is-new.md b/doc/what-is-new.md index d779b6db..fd68dc39 100644 --- a/doc/what-is-new.md +++ b/doc/what-is-new.md @@ -1,5 +1,9 @@ # Next version +# Version 3.16.1 + +- Patch for [Issue #2039](https://github.com/okorach/sonar-tools/issues/2039) that may affect merely all the Sonar Tools + # Version 3.16 * `sonar-config`: diff --git a/pyproject.toml b/pyproject.toml index 839b14d8..727b16d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sonar-tools" -version = "3.17" +version = "3.16.1" description = "A collection of utility tools for the SonarQube ecosystem" authors = [ {name = "Olivier Korach", email = "olivier.korach@gmail.com"}, diff --git a/sonar/errcodes.py b/sonar/errcodes.py index f1102dfc..06ee807e 100644 --- a/sonar/errcodes.py +++ b/sonar/errcodes.py @@ -69,3 +69,6 @@ OBJECT_NOT_FOUND = 16 SONAR_INTERNAL_ERROR = 17 + + +TOO_MANY_RESULTS = 18 diff --git a/sonar/exceptions.py b/sonar/exceptions.py index 9f6f9297..0611c0b9 100644 --- a/sonar/exceptions.py +++ b/sonar/exceptions.py @@ -75,3 +75,11 @@ class ConnectionError(SonarException): def __init__(self, message: str) -> None: super().__init__(message, errcodes.CONNECTION_ERROR) + + +class TooManyResults(SonarException): + """When a call to APIs returns too many results.""" + + def __init__(self, nbr_results: int, message: str) -> None: + super().__init__(message, errcodes.TOO_MANY_RESULTS) + self.nbr_results = nbr_results diff --git a/sonar/projects.py b/sonar/projects.py index 33d1a178..76e767ae 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -40,6 +40,8 @@ import sonar.platform as pf from sonar.util import types, cache +from sonar.util import project_utils as putils + from sonar import exceptions, errcodes from sonar import sqobject, components, qualitygates, qualityprofiles, tasks, settings, webhooks, devops import sonar.permissions.permissions as perms @@ -1357,14 +1359,22 @@ def count(endpoint: pf.Platform, params: types.ApiParams = None) -> int: def search(endpoint: pf.Platform, params: types.ApiParams = None, threads: int = 8) -> dict[str, Project]: """Searches projects in SonarQube - :param endpoint: Reference to the SonarQube platform - :param params: list of parameters to narrow down the search - :returns: list of projects + :param Platform endpoint: Reference to the SonarQube platform + :param ApiParams params: list of filter parameters to narrow down the search + :param int threads: Number of threads to use for the search """ new_params = {} if params is None else params.copy() - if not endpoint.is_sonarcloud(): + if not endpoint.is_sonarcloud() and not new_params.get("filter", None): new_params["filter"] = _PROJECT_QUALIFIER - return sqobject.search_objects(endpoint=endpoint, object_class=Project, params=new_params, threads=threads) + try: + log.info("Searching projects with parameters: %s", str(new_params)) + return sqobject.search_objects(endpoint=endpoint, object_class=Project, params=new_params, threads=threads) + except exceptions.TooManyResults as e: + log.warning(e.message) + filter_1, filter_2 = putils.split_loc_filter(new_params["filter"]) + return search(endpoint, params={**new_params, "filter": filter_1}, threads=threads) | search( + endpoint, params={**new_params, "filter": filter_2}, threads=threads + ) def get_list(endpoint: pf.Platform, key_list: types.KeyList = None, threads: int = 8, use_cache: bool = True) -> dict[str, Project]: @@ -1375,8 +1385,8 @@ def get_list(endpoint: pf.Platform, key_list: types.KeyList = None, threads: int :return: the list of all projects :rtype: dict{key: Project} """ - with _CLASS_LOCK: - if key_list is None or len(key_list) == 0 or not use_cache: + if key_list is None or len(key_list) == 0 or not use_cache: + with _CLASS_LOCK: log.info("Listing projects") p_list = dict(sorted(search(endpoint=endpoint, threads=threads).items())) return p_list diff --git a/sonar/sqobject.py b/sonar/sqobject.py index b25bc619..3f5a4885 100644 --- a/sonar/sqobject.py +++ b/sonar/sqobject.py @@ -242,6 +242,10 @@ def search_objects(endpoint: object, object_class: Any, params: types.ApiParams, data = __get(endpoint, api, {**new_params, p_field: 1}) nb_pages = utilities.nbr_pages(data, api_version) nb_objects = max(len(data[returned_field]), utilities.nbr_total_elements(data, api_version)) + + if nb_objects > c.ELASTIC_MAX_RESULTS: + raise exceptions.TooManyResults(nb_objects, f"Too many {cname}s ({nb_objects}) returned by search") + log.info( "Searching %d %ss, %d pages of %d elements, %d pages in parallel...", nb_objects, diff --git a/sonar/util/constants.py b/sonar/util/constants.py index 6a87b6a9..c067ad5e 100644 --- a/sonar/util/constants.py +++ b/sonar/util/constants.py @@ -74,3 +74,5 @@ SQS_USERS = "sonar-users" # SonarQube Server users default group name SQC_USERS = "Members" # SonarQube Cloud users default group name + +ELASTIC_MAX_RESULTS = 10000 # ElasticSearch max results limit diff --git a/sonar/util/project_utils.py b/sonar/util/project_utils.py new file mode 100644 index 00000000..deec9284 --- /dev/null +++ b/sonar/util/project_utils.py @@ -0,0 +1,55 @@ +# +# sonar-tools +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +import math + + +def split_loc_filter(loc_filter: str) -> tuple[str, str]: + """Parses a ncloc filter and returns new filters to split the search""" + __FILTER_AND = " and " + loc_min, loc_max = 0, 100000000 + new_filters = [] + for f in loc_filter.split(__FILTER_AND): + if f.startswith("ncloc>="): + try: + loc_min = int(f[len("ncloc>=") :]) + except ValueError: + pass + elif f.startswith("ncloc>"): + try: + loc_min = int(f[len("ncloc>") :]) + 1 + except ValueError: + pass + elif f.startswith("ncloc<="): + try: + loc_max = int(f[len("ncloc<=") :]) + except ValueError: + pass + elif f.startswith("ncloc<"): + try: + loc_max = int(f[len("ncloc<") :]) - 1 + except ValueError: + pass + else: + new_filters.append(f) + loc_middle = int(math.exp((math.log2(loc_max) - math.log2(max(loc_min, 1))) / 2)) + slice1 = __FILTER_AND.join(new_filters + [f"ncloc>={loc_min}", f"ncloc<={loc_middle}"]) + slice2 = __FILTER_AND.join(new_filters + [f"ncloc>{loc_middle}", f"ncloc<={loc_max}"]) + return slice1, slice2 diff --git a/sonar/version.py b/sonar/version.py index 799a0302..15ac22f5 100644 --- a/sonar/version.py +++ b/sonar/version.py @@ -24,5 +24,5 @@ """ -PACKAGE_VERSION = "3.17" -MIGRATION_TOOL_VERSION = "0.7" +PACKAGE_VERSION = "3.16.1" +MIGRATION_TOOL_VERSION = "0.6-snapshot"