From 9138424713634ca915befdea8be76072dcc9eb5f Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Wed, 29 Oct 2025 13:19:13 -0500 Subject: [PATCH 1/4] Bump version --- sonar/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar/version.py b/sonar/version.py index 77021d236..e3cd52c62 100644 --- a/sonar/version.py +++ b/sonar/version.py @@ -24,5 +24,5 @@ """ -PACKAGE_VERSION = "3.16" +PACKAGE_VERSION = "3.16.1" MIGRATION_TOOL_VERSION = "0.6-snapshot" From 1f34e991f1c56ec6b8f194b63c03f20071bb5227 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Wed, 29 Oct 2025 13:19:30 -0500 Subject: [PATCH 2/4] Fixes #2039 --- sonar/errcodes.py | 3 ++ sonar/exceptions.py | 8 ++++++ sonar/projects.py | 24 +++++++++++----- sonar/sqobject.py | 4 +++ sonar/util/constants.py | 2 ++ sonar/util/project_utils.py | 55 +++++++++++++++++++++++++++++++++++++ 6 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 sonar/util/project_utils.py diff --git a/sonar/errcodes.py b/sonar/errcodes.py index f1102dfc5..06ee807ef 100644 --- a/sonar/errcodes.py +++ b/sonar/errcodes.py @@ -69,3 +69,6 @@ OBJECT_NOT_FOUND = 16 SONAR_INTERNAL_ERROR = 17 + + +TOO_MANY_RESULTS = 18 diff --git a/sonar/exceptions.py b/sonar/exceptions.py index e97efe3b0..d7fd506bf 100644 --- a/sonar/exceptions.py +++ b/sonar/exceptions.py @@ -74,3 +74,11 @@ class ConnectionError(SonarException): def __init__(self, message: str) -> None: super().__init__(message, errcodes.CONNECTION_ERROR) + + +class TooManyResults(SonarException): + """When a call to APIs returns too many results.""" + + def __init__(self, nbr_results: int, message: str) -> None: + super().__init__(message, errcodes.TOO_MANY_RESULTS) + self.nbr_results = nbr_results diff --git a/sonar/projects.py b/sonar/projects.py index 59af14392..843d72e95 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -41,6 +41,8 @@ import sonar.platform as pf from sonar.util import types, cache +from sonar.util import project_utils as putils + from sonar import exceptions, errcodes from sonar import sqobject, components, qualitygates, qualityprofiles, tasks, settings, webhooks, devops import sonar.permissions.permissions as perms @@ -1443,14 +1445,22 @@ def count(endpoint: pf.Platform, params: types.ApiParams = None) -> int: def search(endpoint: pf.Platform, params: types.ApiParams = None, threads: int = 8) -> dict[str, Project]: """Searches projects in SonarQube - :param endpoint: Reference to the SonarQube platform - :param params: list of parameters to narrow down the search - :returns: list of projects + :param Platform endpoint: Reference to the SonarQube platform + :param ApiParams params: list of filter parameters to narrow down the search + :param int threads: Number of threads to use for the search """ new_params = {} if params is None else params.copy() - if not endpoint.is_sonarcloud(): + if not endpoint.is_sonarcloud() and not new_params.get("filter", None): new_params["filter"] = _PROJECT_QUALIFIER - return sqobject.search_objects(endpoint=endpoint, object_class=Project, params=new_params, threads=threads) + try: + log.info("Searching projects with parameters: %s", str(new_params)) + return sqobject.search_objects(endpoint=endpoint, object_class=Project, params=new_params, threads=threads) + except exceptions.TooManyResults as e: + log.warning(e.message) + filter_1, filter_2 = putils.split_loc_filter(new_params["filter"]) + return search(endpoint, params={**new_params, "filter": filter_1}, threads=threads) | search( + endpoint, params={**new_params, "filter": filter_2}, threads=threads + ) def get_list(endpoint: pf.Platform, key_list: types.KeyList = None, threads: int = 8, use_cache: bool = True) -> dict[str, Project]: @@ -1461,8 +1471,8 @@ def get_list(endpoint: pf.Platform, key_list: types.KeyList = None, threads: int :return: the list of all projects :rtype: dict{key: Project} """ - with _CLASS_LOCK: - if key_list is None or len(key_list) == 0 or not use_cache: + if key_list is None or len(key_list) == 0 or not use_cache: + with _CLASS_LOCK: log.info("Listing projects") p_list = dict(sorted(search(endpoint=endpoint, threads=threads).items())) return p_list diff --git a/sonar/sqobject.py b/sonar/sqobject.py index 8978d17b9..4f1baf03c 100644 --- a/sonar/sqobject.py +++ b/sonar/sqobject.py @@ -234,6 +234,10 @@ def search_objects(endpoint: object, object_class: any, params: types.ApiParams, data = __get(endpoint, api, {**new_params, p_field: 1}) nb_pages = utilities.nbr_pages(data, api_version) nb_objects = max(len(data[returned_field]), utilities.nbr_total_elements(data, api_version)) + + if nb_objects > c.ELASTIC_MAX_RESULTS: + raise exceptions.TooManyResults(nb_objects, f"Too many {cname}s ({nb_objects}) returned by search") + log.info( "Searching %d %ss, %d pages of %d elements, %d pages in parallel...", nb_objects, diff --git a/sonar/util/constants.py b/sonar/util/constants.py index 6ee08ec2e..ffe9a2fcc 100644 --- a/sonar/util/constants.py +++ b/sonar/util/constants.py @@ -74,3 +74,5 @@ SQS_USERS = "sonar-users" # SonarQube Server users default group name SQC_USERS = "Members" # SonarQube Cloud users default group name + +ELASTIC_MAX_RESULTS = 10000 # ElasticSearch max results limit diff --git a/sonar/util/project_utils.py b/sonar/util/project_utils.py new file mode 100644 index 000000000..deec9284e --- /dev/null +++ b/sonar/util/project_utils.py @@ -0,0 +1,55 @@ +# +# sonar-tools +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +import math + + +def split_loc_filter(loc_filter: str) -> tuple[str, str]: + """Parses a ncloc filter and returns new filters to split the search""" + __FILTER_AND = " and " + loc_min, loc_max = 0, 100000000 + new_filters = [] + for f in loc_filter.split(__FILTER_AND): + if f.startswith("ncloc>="): + try: + loc_min = int(f[len("ncloc>=") :]) + except ValueError: + pass + elif f.startswith("ncloc>"): + try: + loc_min = int(f[len("ncloc>") :]) + 1 + except ValueError: + pass + elif f.startswith("ncloc<="): + try: + loc_max = int(f[len("ncloc<=") :]) + except ValueError: + pass + elif f.startswith("ncloc<"): + try: + loc_max = int(f[len("ncloc<") :]) - 1 + except ValueError: + pass + else: + new_filters.append(f) + loc_middle = int(math.exp((math.log2(loc_max) - math.log2(max(loc_min, 1))) / 2)) + slice1 = __FILTER_AND.join(new_filters + [f"ncloc>={loc_min}", f"ncloc<={loc_middle}"]) + slice2 = __FILTER_AND.join(new_filters + [f"ncloc>{loc_middle}", f"ncloc<={loc_max}"]) + return slice1, slice2 From d41c0adad270984d6f227f13531e14ffb167f7f3 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Wed, 29 Oct 2025 13:37:11 -0500 Subject: [PATCH 3/4] Update doc about release 3.16.1 --- doc/what-is-new.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/what-is-new.md b/doc/what-is-new.md index d779b6dbc..fd68dc39f 100644 --- a/doc/what-is-new.md +++ b/doc/what-is-new.md @@ -1,5 +1,9 @@ # Next version +# Version 3.16.1 + +- Patch for [Issue #2039](https://github.com/okorach/sonar-tools/issues/2039) that may affect merely all the Sonar Tools + # Version 3.16 * `sonar-config`: From 6db0e650d515e62d149092f25a87981cea96f5d8 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Wed, 29 Oct 2025 13:37:23 -0500 Subject: [PATCH 4/4] Adjust to 3.16.1 --- conf/release.Dockerfile | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/release.Dockerfile b/conf/release.Dockerfile index bec6e8d61..b9f75c735 100644 --- a/conf/release.Dockerfile +++ b/conf/release.Dockerfile @@ -28,11 +28,11 @@ COPY ./LICENSE . COPY ./sonar/audit sonar/audit RUN pip install --upgrade pip \ -&& pip install sonar-tools==3.17 +&& pip install sonar-tools==3.16.1 USER ${USERNAME} WORKDIR /home/${USERNAME} -HEALTHCHECK --interval=180s --timeout=5s CMD [ "sonar-tools-help" ] +HEALTHCHECK --interval=180s --timeout=5s CMD [ "sonar-tools" ] -CMD [ "sonar-tools-help" ] +CMD [ "sonar-tools" ] diff --git a/pyproject.toml b/pyproject.toml index 839b14d8d..727b16d83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sonar-tools" -version = "3.17" +version = "3.16.1" description = "A collection of utility tools for the SonarQube ecosystem" authors = [ {name = "Olivier Korach", email = "olivier.korach@gmail.com"},