Handle-exceptions-at-the-root-request (#2003)

* Fix bug

* Convert exceptions at the root

* Adjust to exceptions in platform

* Make gitlab groups a list setting

* Remove au lowercasing for API errors

* Adjust to exceptions in platform

* Allow new param as multi-valued

* Adjust to QG not found message

* Separate exception for HTTP and Connection Error

* Simplify import_zip excepton handling

* Disallowed to remove admin permission to the admin user

* Adjust exception handling

* Add illegal issue transition

* Add illegal issue transition

* Fix

* Raise IllegalIssueTransition

* Remove Illegal Issue Transition Exception

* Remove illegal issue err code

* Remove nolint option from the run_scanner call

* Formatting

* Fix key callection for object not found

* Handle ObjectNotFound for import_zip

* Remove exception handling in delete()

* Fixes on WebHook corner cases

* Add auto gen of unsupported operation for bad parameter value

* Remove ConnectionError, it's a type of RequestException

* Remove useless CLI parameters process

* Raise UnsupportedOperationw hen passing a wrong severity

* Verify exception when passing incorrect parameters

* Verify exception when import non existing project key

* Verify exception when webhook with too small secret string

* Handle update when reference portfolio already exists

* Remove exception handling (handled in Platform)

* Add exception handling for incorretc API param value

* Quality pass

* Add type hints

* Quality pass

* Quality pass

* Fix key retrieval in case of exception

* Add message to detect not found

* Refactoring

* Adjust to 9.9

* Adjust to 9.9

* Use poetry to install dependencies

* Fix run_linters

* Find ruff from python

* Remove python to ruff ruff

* Install deps before running linters

* Remove install before runninng linters

* Try to run linters in same env

Author: okorach

.github/workflows/build.yml

@@ -28,8 +28,12 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ -f requirements-to-build.txt ]; then pip install -r requirements-to-build.txt; fi
+          python -m pip install poetry
+          poetry install
+      - name: Build package
+        run: |
+          poetry build
+        continue-on-error: false
         # Linting is done in the run_linters.sh script
 
       - name: Prep tests
@@ -49,9 +53,10 @@ jobs:
       - name: Run linters
         working-directory: .
         run: |
+          python -m pip install ruff pylint flake8
           chmod +x conf/run_linters.sh
           conf/run_linters.sh
-      #- name: Cache SonarQube packages
+      # - name: Cache SonarQube packages
       #  uses: actions/cache@v4
       #  with:
       #    path: ./.sonar

conf/prep_all_tests.sh

@@ -43,7 +43,7 @@ function create_fresh_project {
 
 conf/run_linters.sh
 
-create_fresh_project "${SYNC_PROJECT_KEY}" "${SONAR_HOST_URL_TEST:?}" "${SONAR_TOKEN_TEST_ADMIN_USER}" "${SONAR_TOKEN_TEST_ADMIN_ANALYSIS}" -nolint
+create_fresh_project "${SYNC_PROJECT_KEY}" "${SONAR_HOST_URL_TEST:?}" "${SONAR_TOKEN_TEST_ADMIN_USER}" "${SONAR_TOKEN_TEST_ADMIN_ANALYSIS}"
 create_fresh_project "${SYNC_PROJECT_KEY}" "${SONAR_HOST_URL_LATEST:?}" "${SONAR_TOKEN_LATEST_ADMIN_USER}" "${SONAR_TOKEN_LATEST_ADMIN_ANALYSIS}"
 create_fresh_project "${SYNC_PROJECT_KEY}" "${SONAR_HOST_URL_CB:?}" "${SONAR_TOKEN_CB_ADMIN_USER}" "${SONAR_TOKEN_CB_ADMIN_ANALYSIS}"
 create_fresh_project "${SYNC_PROJECT_KEY}" "${SONAR_HOST_URL_9:?}" "${SONAR_TOKEN_9_ADMIN_USER}" "${SONAR_TOKEN_9_ADMIN_ANALYSIS}"

conf/run_linters.sh

@@ -72,7 +72,7 @@ fi
 if [[ "${localbuild}" = "true" ]]; then
     if [[ "${linters_to_run}" == *"shellcheck"* ]]; then
         echo "===> Running shellcheck"
-        shellcheck $(find "${ROOT_DIR}" . -name '*.sh') \
+        shellcheck "$(find "${ROOT_DIR}" . -name '*.sh')" \
             -s bash -f json | jq | tee "${BUILD_DIR}/shellcheck-report.json" | "${CONF_DIR}"/shellcheck2sonar.py "${external_format}" > "${SHELLCHECK_REPORT}"
         [[ ! -s "${SHELLCHECK_REPORT}" ]] && rm -f "${SHELLCHECK_REPORT}"
         cat "${BUILD_DIR}/shellcheck-report.json"

migration/build.sh

@@ -29,12 +29,6 @@ while [[ $# -ne 0 ]]; do
         nodocker)
             build_image=0
             ;;
-        pypi)
-            release=1
-            ;;
-        dockerhub)
-            release_docker=1
-            ;;
         *)
             ;;
     esac

sonar/components.py

@@ -118,7 +118,7 @@ def get_issues(self, filters: types.ApiParams = None) -> dict[str, object]:
         """Returns list of issues for a component, optionally on branches or/and PRs"""
         from sonar.issues import search_all
 
-        filters = {k: list(set(v) if isinstance(v, (list, set, tuple)) else v) for k, v in (filters or {}).items() if v is not None}
+        filters = {k: list(set(v)) if isinstance(v, (list, set, tuple)) else v for k, v in (filters or {}).items() if v is not None}
         log.info("Searching issues for %s with filters %s", str(self), str(filters))
         issue_list = search_all(endpoint=self.endpoint, params=self.api_params() | {"additionalFields": "comments"} | filters)
         self.nbr_issues = len(issue_list)

sonar/findings.py

@@ -23,6 +23,7 @@
 import concurrent.futures
 from datetime import datetime
 from typing import Optional
+import re
 from http import HTTPStatus
 from requests import RequestException
 import Levenshtein
@@ -32,6 +33,7 @@
 import sonar.platform as pf
 from sonar.util import types
 from sonar.util import constants as c, issue_defs as idefs
+from sonar import exceptions
 
 import sonar.utilities as util
 from sonar import projects, rules
@@ -460,6 +462,10 @@ def do_transition(self, transition: str) -> bool:
             return self.post("issues/do_transition", {"issue": self.key, "transition": transition}).ok
         except (ConnectionError, RequestException) as e:
             util.handle_error(e, f"applying transition {transition}", catch_http_statuses=(HTTPStatus.BAD_REQUEST, HTTPStatus.NOT_FOUND))
+        except exceptions.SonarException as e:
+            if re.match(r"Transition from state [A-Za-z]+ does not exist", e.message):
+                raise exceptions.UnsupportedOperation(e.message) from e
+            raise
         return False
 
     def get_branch_and_pr(self, data: types.ApiPayload) -> tuple[Optional[str], Optional[str]]:

sonar/issues.py

@@ -294,18 +294,14 @@ def add_comment(self, comment: str) -> bool:
         log.debug("Adding comment '%s' to %s", comment, str(self))
         try:
             r = self.post("issues/add_comment", {"issue": self.key, "text": comment})
-        except (ConnectionError, requests.RequestException) as e:
+        except requests.RequestException as e:
             util.handle_error(e, "adding comment", catch_all=True)
             return False
         return r.ok
 
     def __set_severity(self, **params) -> bool:
-        try:
-            log.debug("Changing severity of %s from '%s' to '%s'", str(self), self.severity, str(params))
-            r = self.post("issues/set_severity", {"issue": self.key, **params})
-        except (ConnectionError, requests.RequestException) as e:
-            util.handle_error(e, "changing issue severity", catch_all=True)
-            return False
+        log.debug("Changing severity of %s from '%s' to '%s'", str(self), self.severity, str(params))
+        r = self.post("issues/set_severity", {"issue": self.key, **params})
         return r.ok
 
     def set_severity(self, severity: str) -> bool:

sonar/permissions/project_permissions.py

@@ -72,6 +72,9 @@ def _set_perms(
             self.read()
         for p in permissions.PERMISSION_TYPES:
             to_remove = diff_func(self.permissions.get(p, {}), new_perms.get(p, {}))
+            if p == "users" and "admin" in to_remove:
+                # Don't remove admin permission to the admin user, this is not possible anyway
+                to_remove["admin"] = [v for v in to_remove["admin"] if v != "admin"]
             self._post_api(apis["remove"][p], field[p], to_remove, **kwargs)
             to_add = diff_func(new_perms.get(p, {}), self.permissions.get(p, {}))
             self._post_api(apis["add"][p], field[p], to_add, **kwargs)

sonar/platform.py

@@ -28,6 +28,7 @@
 from http import HTTPStatus
 import sys
 import os
+import re
 from typing import Optional
 import time
 import datetime
@@ -264,9 +265,8 @@ def __run_request(self, request: callable, api: str, params: types.ApiParams = N
             headers["Authorization"] = f"Bearer {self.__token}"
             if with_org:
                 params["organization"] = self.organization
-        req_type, url = "", ""
+        req_type, url = getattr(request, "__name__", repr(request)).upper(), ""
         if log.get_level() <= log.DEBUG:
-            req_type = getattr(request, "__name__", repr(request)).upper()
             url = self.__urlstring(api, params, kwargs.get("data", {}))
             log.debug("%s: %s", req_type, url)
         kwargs["headers"] = headers
@@ -288,10 +288,24 @@ def __run_request(self, request: callable, api: str, params: types.ApiParams = N
                     self.local_url = new_url
             r.raise_for_status()
         except HTTPError as e:
-            lvl = log.DEBUG if r.status_code in mute else log.ERROR
+            code = r.status_code
+            lvl = log.DEBUG if code in mute else log.ERROR
             log.log(lvl, "%s (%s request)", util.error_msg(e), req_type)
-            raise e
-        except (ConnectionError, RequestException) as e:
+            err_msg = util.sonar_error(e.response)
+            err_msg_lower = err_msg.lower()
+            key = next((params[k] for k in ("key", "project", "component", "componentKey") if k in params), "Unknown")
+            if any(
+                msg in err_msg_lower for msg in ("not found", "no quality gate has been found", "does not exist", "could not find")
+            ):  # code == HTTPStatus.NOT_FOUND:
+                raise exceptions.ObjectNotFound(key, err_msg) from e
+            if any(msg in err_msg_lower for msg in ("already exists", "already been taken")):
+                raise exceptions.ObjectAlreadyExists(key, err_msg) from e
+            if re.match(r"(Value of parameter .+ must be one of|No enum constant)", err_msg):
+                raise exceptions.UnsupportedOperation(err_msg) from e
+            if any(msg in err_msg_lower for msg in ("insufficient privileges", "insufficient permissions")):
+                raise exceptions.SonarException(err_msg, errcodes.SONAR_API_AUTHORIZATION) from e
+            raise exceptions.SonarException(err_msg, errcodes.SONAR_API) from e
+        except ConnectionError as e:
             util.handle_error(e, "")
         return r
 

sonar/portfolios.py

@@ -545,7 +545,7 @@ def add_application_branch(self, app_key: str, branch: str = c.DEFAULT_BRANCH) -
         self._applications[app_key].append(branch)
         return True
 
-    def add_subportfolio(self, key: str, name: str = None, by_ref: bool = False) -> object:
+    def add_subportfolio(self, key: str, name: str = None, by_ref: bool = False) -> Portfolio:
         """Adds a subportfolio to a portfolio, defined by key, name and by reference option"""
 
         log.info("Adding sub-portfolios to %s", str(self))
@@ -644,7 +644,12 @@ def update(self, data: dict[str, str], recurse: bool) -> None:
             if subp_data.get("byReference", False):
                 o_subp = Portfolio.get_object(self.endpoint, key)
                 if o_subp.key not in key_list:
-                    self.add_subportfolio(o_subp.key, name=o_subp.name, by_ref=True)
+                    try:
+                        self.add_subportfolio(o_subp.key, name=o_subp.name, by_ref=True)
+                    except exceptions.SonarException as e:
+                        # If the exception is that the portfolio already references, just pass
+                        if "already references" not in e.message:
+                            raise
             else:
                 try:
                     o_subp = Portfolio.get_object(self.endpoint, key)