diff --git a/cli/findings_export.py b/cli/findings_export.py index 13afcf133..da32f1b95 100755 --- a/cli/findings_export.py +++ b/cli/findings_export.py @@ -32,7 +32,7 @@ from queue import Queue import threading from threading import Thread -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException from cli import options from sonar.util.types import ConfigSettings @@ -294,8 +294,8 @@ def __get_component_findings(queue: Queue[tuple[object, ConfigSettings]], write_ findings_list = findings.export_findings( component.endpoint, component.key, branch=params.get("branch", None), pull_request=params.get("pullRequest", None) ) - except HTTPError as e: - log.critical("Error %s while exporting findings of %s, skipped", str(e), str(component)) + except (ConnectionError, RequestException) as e: + log.critical("%s while exporting findings of %s, skipped", util.error_msg(e), str(component)) findings_list = {} write_queue.put([findings_list, False]) else: @@ -323,8 +323,8 @@ def __get_component_findings(queue: Queue[tuple[object, ConfigSettings]], write_ if (i_statuses or not status_list) and (i_resols or not resol_list) and (i_types or not type_list) and (i_sevs or not sev_list): try: findings_list = component.get_issues(filters=new_params) - except HTTPError as e: - log.critical("Error %s while exporting issues of %s, skipped", str(e), str(component)) + except (ConnectionError, RequestException) as e: + log.critical("%s while exporting issues of %s, skipped", util.error_msg(e), str(component)) findings_list = {} else: log.debug("Status = %s, Types = %s, Resol = %s, Sev = %s", str(i_statuses), str(i_types), str(i_resols), str(i_sevs)) @@ -333,8 +333,8 @@ def __get_component_findings(queue: Queue[tuple[object, ConfigSettings]], write_ if (h_statuses or not status_list) and (h_resols or not resol_list) and (h_types or not type_list) and (h_sevs or not sev_list): try: findings_list.update(component.get_hotspots(filters=new_params)) - except HTTPError as e: - log.critical("Error %s while exporting hotspots of object key %s, skipped", str(e), str(component)) + except (ConnectionError, RequestException) as e: + log.error("%s while exporting hotspots of object key %s, skipped", util.error_msg(e), str(component)) else: log.debug("Status = %s, Types = %s, Resol = %s, Sev = %s", str(h_statuses), str(h_types), str(h_resols), str(h_sevs)) log.info("Selected types, severities, resolutions or statuses disables issue search") @@ -351,8 +351,8 @@ def store_findings(components_list: dict[str, object], params: ConfigSettings) - try: log.debug("Queue %s task %s put", str(my_queue), str(comp)) my_queue.put((comp, params.copy())) - except HTTPError as e: - log.critical("Error %s while exporting findings of %s, skipped", str(e), str(comp)) + except (ConnectionError, RequestException) as e: + log.critical("%s while exporting findings of %s, skipped", util.error_msg(e), str(comp)) threads = params.get(options.NBR_THREADS, 4) for i in range(min(threads, len(components_list))): diff --git a/cli/loc.py b/cli/loc.py index 71ee67724..50e25f287 100644 --- a/cli/loc.py +++ b/cli/loc.py @@ -24,7 +24,7 @@ import sys import csv import datetime -from requests.exceptions import HTTPError +from requests import RequestException from cli import options import sonar.logging as log @@ -51,8 +51,8 @@ def __get_csv_row(o: object, **kwargs) -> tuple[list[str], str]: """Returns CSV row of object""" try: loc = o.loc() - except HTTPError as e: - log.warning("HTTP Error %s, LoC export of %s skipped", str(e), str(o)) + except (ConnectionError, RequestException) as e: + log.warning("%s, LoC export of %s skipped", util.error_msg(e), str(o)) loc = "" arr = [o.key, loc] obj_type = type(o).__name__.lower() @@ -113,8 +113,8 @@ def __get_object_json_data(o: object, **kwargs) -> dict[str, str]: d = {parent_type: o.concerned_object.key, "branch": o.name, "ncloc": ""} try: d["ncloc"] = o.loc() - except HTTPError as e: - log.warning("HTTP Error %s, LoC export of %s skipped", str(e), str(o)) + except (ConnectionError, RequestException) as e: + log.warning("%s, LoC export of %s skipped", util.error_msg(e), str(o)) if kwargs[options.WITH_NAME]: d[f"{parent_type}Name"] = o.name if obj_type in ("branch", "applicationbranch"): diff --git a/cli/measures_export.py b/cli/measures_export.py index 99321b5b8..dfca37249 100755 --- a/cli/measures_export.py +++ b/cli/measures_export.py @@ -29,8 +29,7 @@ from typing import Union -from http import HTTPStatus -from requests.exceptions import HTTPError +from requests import RequestException from sonar.util import types from cli import options import sonar.logging as log @@ -56,12 +55,7 @@ def __last_analysis(component: object) -> str: def __get_json_measures_history(obj: object, wanted_metrics: types.KeyList) -> dict[str, str]: """Returns the measure history of an object (project, branch, application, portfolio)""" - data = {} - try: - data["history"] = obj.get_measures_history(wanted_metrics) - except HTTPError as e: - log.error("HTTP Error %s, measures history export of %s skipped", str(e), str(obj)) - return data + return {"history": obj.get_measures_history(wanted_metrics)} def __get_object_measures(obj: object, wanted_metrics: types.KeyList) -> dict[str, str]: @@ -288,11 +282,8 @@ def __get_measures(obj: object, wanted_metrics: types.KeyList, hist: bool) -> Un data.update(__get_json_measures_history(obj, wanted_metrics)) else: data.update(__get_object_measures(obj, wanted_metrics)) - except HTTPError as e: - if e.response.status_code == HTTPStatus.FORBIDDEN: - log.error("Insufficient permission to retrieve measures of %s, export skipped for this object", str(obj)) - else: - log.error("HTTP Error %s while retrieving measures of %s, export skipped for this object", str(e), str(obj)) + except (ConnectionError, RequestException) as e: + log.error("%s, measures export skipped for %s", util.error_msg(e), str(obj)) return None return data diff --git a/sonar/app_branches.py b/sonar/app_branches.py index ddf55b49b..a45400de9 100644 --- a/sonar/app_branches.py +++ b/sonar/app_branches.py @@ -24,7 +24,7 @@ import json from http import HTTPStatus -from requests.exceptions import HTTPError +from requests import RequestException from requests.utils import quote import sonar.logging as log @@ -34,7 +34,7 @@ from sonar.applications import Application as App from sonar.branches import Branch -from sonar import exceptions, projects +from sonar import exceptions, projects, utilities import sonar.sqobject as sq _OBJECTS = {} @@ -109,9 +109,11 @@ def create(cls, app: App, name: str, project_branches: list[Branch]) -> Applicat params["projectBranch"].append(br_name) try: app.endpoint.post(APIS["create"], params=params) - except HTTPError as e: + except (ConnectionError, RequestException) as e: if e.response.status_code == HTTPStatus.BAD_REQUEST: raise exceptions.ObjectAlreadyExists(f"app.App {app.key} branch '{name}", e.response.text) + log.critical("%s while creating branch '%s' of '%s'", utilities.error_msg(e), name, str(app)) + raise return ApplicationBranch(app=app, name=name, project_branches=project_branches) @classmethod @@ -182,9 +184,12 @@ def update(self, name: str, project_branches: list[Branch]) -> bool: params["projectBranch"].append(br_name) try: ok = self.endpoint.post(APIS["update"], params=params).ok - except HTTPError as e: + except (ConnectionError, RequestException) as e: if e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(str(self), e.response.text) + log.error("%s while updating '%s'", utilities.error_msg(e), str(self)) + raise + self.name = name self._project_branches = project_branches return ok diff --git a/sonar/applications.py b/sonar/applications.py index b09e7402c..cc11c576c 100644 --- a/sonar/applications.py +++ b/sonar/applications.py @@ -26,7 +26,7 @@ from datetime import datetime from http import HTTPStatus from threading import Lock -from requests.exceptions import HTTPError +from requests import RequestException import sonar.logging as log import sonar.platform as pf @@ -90,9 +90,11 @@ def get_object(cls, endpoint: pf.Platform, key: str) -> Application: return _OBJECTS[uu] try: data = json.loads(endpoint.get(APIS["get"], params={"application": key}).text)["application"] - except HTTPError as e: + except (ConnectionError, RequestException) as e: if e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(key, f"Application key '{key}' not found") + log.error("%s while getting app key '%s'", util.error_msg(e), key) + raise return cls.load(endpoint, data) @classmethod @@ -131,9 +133,11 @@ def create(cls, endpoint: pf.Platform, key: str, name: str) -> Application: check_supported(endpoint) try: endpoint.post(APIS["create"], params={"key": key, "name": name}) - except HTTPError as e: + except (ConnectionError, RequestException) as e: if e.response.status_code == HTTPStatus.BAD_REQUEST: raise exceptions.ObjectAlreadyExists(key, e.response.text) + log.critical("%s while creating app key '%s'", util.error_msg(e), key) + raise return Application(endpoint, key, name) def refresh(self) -> None: @@ -146,10 +150,11 @@ def refresh(self) -> None: try: self.reload(json.loads(self.get("navigation/component", params={"component": self.key}).text)) self.reload(json.loads(self.get(APIS["get"], params=self.search_params()).text)["application"]) - except HTTPError as e: + except (ConnectionError, RequestException) as e: if e.response.status_code == HTTPStatus.NOT_FOUND: _OBJECTS.pop(self.uuid(), None) raise exceptions.ObjectNotFound(self.key, f"{str(self)} not found") + log.error("%s while refreshing %s", util.error_msg(e), str(self)) raise def __str__(self) -> str: @@ -383,11 +388,12 @@ def add_projects(self, project_list: list[str]) -> bool: try: r = self.post("applications/add_project", params={"application": self.key, "project": proj}) ok = ok and r.ok - except HTTPError as e: + except (ConnectionError, RequestException) as e: if e.response.status_code == HTTPStatus.NOT_FOUND: log.warning("Project '%s' not found, can't be added to %s", proj, self) ok = False else: + log.error("%s while adding projects to %s", util.error_msg(e), str(self)) raise return ok diff --git a/sonar/branches.py b/sonar/branches.py index f043adb44..48769ea47 100644 --- a/sonar/branches.py +++ b/sonar/branches.py @@ -22,7 +22,7 @@ from http import HTTPStatus import json from urllib.parse import unquote -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException import requests.utils from sonar import platform @@ -89,9 +89,11 @@ def get_object(cls, concerned_object: projects.Project, branch_name: str) -> Bra return _OBJECTS[uu] try: data = json.loads(concerned_object.endpoint.get(APIS["list"], params={"project": concerned_object.key}).text) - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(HTTPError, e) and e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(concerned_object.key, f"Project '{concerned_object.key}' not found") + log.critical("%s while getting branch '%s' of %s", util.error_msg(e), branch_name, str(concerned_object)) + raise for br in data.get("branches", []): if br["name"] == branch_name: return cls.load(concerned_object, branch_name, br) @@ -127,9 +129,10 @@ def refresh(self) -> Branch: """ try: data = json.loads(self.get(APIS["list"], params={"project": self.concerned_object.key}).text) - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(HTTPError, e) and e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(self.key, f"{str(self)} not found in SonarQube") + log.error("%s while refreshing %s", util.error_msg(e), str(self)) for br in data.get("branches", []): if br["name"] == self.name: self._load(br) @@ -173,7 +176,7 @@ def is_main(self): self.refresh() return self._is_main - def delete(self): + def delete(self) -> bool: """Deletes a branch :raises ObjectNotFound: Branch not found for deletion @@ -182,9 +185,11 @@ def delete(self): """ try: return sq.delete_object(self, APIS["delete"], {"branch": self.name, "project": self.concerned_object.key}, _OBJECTS) - except HTTPError as e: - if e.response.status_code == HTTPStatus.BAD_REQUEST: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.BAD_REQUEST: log.warning("Can't delete %s, it's the main branch", str(self)) + else: + log.error("%s while deleting %s", util.error_msg(e), str(self)) return False def new_code(self) -> str: @@ -197,11 +202,10 @@ def new_code(self) -> str: elif self._new_code is None: try: data = json.loads(self.get(api=APIS["get_new_code"], params={"project": self.concerned_object.key}).text) - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(self.concerned_object.key, f"{str(self.concerned_object)} not found") - if e.response.status_code == HTTPStatus.FORBIDDEN: - log.error("Error 403 when getting new code period of %s", {str(self)}) + log.error("%s while getting new code period of %s", util.error_msg(e), str(self)) raise e for b in data["newCodePeriods"]: new_code = settings.new_code_to_string(b) @@ -261,9 +265,11 @@ def rename(self, new_name): log.info("Renaming main branch of %s from '%s' to '%s'", str(self.concerned_object), self.name, new_name) try: self.post(APIS["rename"], params={"project": self.concerned_object.key, "name": new_name}) - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(HTTPError, e) and e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(self.concerned_object.key, f"str{self.concerned_object} not found") + log.error("%s while renaming %s", util.error_msg(e), str(self)) + raise _OBJECTS.pop(self.uuid(), None) self.name = new_name _OBJECTS[self.uuid()] = self @@ -358,11 +364,8 @@ def audit(self, audit_settings: types.ConfigSettings) -> list[Problem]: log.debug("Auditing %s", str(self)) try: return self.__audit_last_analysis(audit_settings) + self.__audit_zero_loc() + self.__audit_never_analyzed() - except HTTPError as e: - if e.response.status_code == HTTPStatus.FORBIDDEN: - log.error("Not enough permission to fully audit %s", str(self)) - else: - log.error("HTTP error %s while auditing %s", str(e), str(self)) + except Exception as e: + log.error("%s while auditing %s, audit skipped", util.error_msg(e), str(self)) else: log.debug("Branch audit disabled, skipping audit of %s", str(self)) return [] diff --git a/sonar/devops.py b/sonar/devops.py index 61cb20711..e527d7f68 100644 --- a/sonar/devops.py +++ b/sonar/devops.py @@ -23,7 +23,7 @@ from http import HTTPStatus import json -from requests.exceptions import HTTPError +from requests import RequestException import sonar.logging as log from sonar.util import types @@ -109,10 +109,11 @@ def create(cls, endpoint: platform.Platform, key: str, plt_type: str, url_or_wor elif plt_type == "bitbucketcloud": params.update({"clientSecret": _TO_BE_SET, "clientId": _TO_BE_SET, "workspace": url_or_workspace}) endpoint.post(_CREATE_API_BBCLOUD, params=params) - except HTTPError as e: + except (ConnectionError, RequestException) as e: if e.response.status_code == HTTPStatus.BAD_REQUEST and endpoint.edition() in ("community", "developer"): log.warning("Can't set DevOps platform '%s', don't you have more that 1 of that type?", key) raise exceptions.UnsupportedOperation(f"Can't set DevOps platform '{key}', don't you have more that 1 of that type?") + log.error("%s while creating devops platform %s/%s/%s", util.error_msg(e), key, plt_type, url_or_workspace) raise o = DevopsPlatform(endpoint=endpoint, key=key, platform_type=plt_type) o.refresh() diff --git a/sonar/hotspots.py b/sonar/hotspots.py index f35479b83..996c3bc21 100644 --- a/sonar/hotspots.py +++ b/sonar/hotspots.py @@ -24,7 +24,7 @@ import json import re from http import HTTPStatus -from requests.exceptions import HTTPError +from requests import RequestException import requests.utils import sonar.logging as log @@ -402,12 +402,13 @@ def search(endpoint: pf.Platform, filters: types.ApiParams = None) -> dict[str, try: data = json.loads(endpoint.get(Hotspot.SEARCH_API, params=inline_filters, mute=(HTTPStatus.NOT_FOUND,)).text) nbr_hotspots = util.nbr_total_elements(data) - except HTTPError as e: + except (ConnectionError, RequestException) as e: if e.response.status_code == HTTPStatus.NOT_FOUND: log.warning("No hotspots found with search params %s", str(inline_filters)) nbr_hotspots = 0 return {} - raise e + log.error("%s while searching hotspots", util.error_msg(e)) + break nbr_pages = util.nbr_pages(data) log.debug("Number of hotspots: %d - Page: %d/%d", nbr_hotspots, inline_filters["p"], nbr_pages) if nbr_hotspots > Hotspot.MAX_SEARCH: @@ -494,8 +495,10 @@ def post_search_filter(hotspots_dict: dict[str, Hotspot], filters: types.ApiPara lang = rules.get_object(endpoint=finding.endpoint, key=finding.rule).language if lang not in filters["languages"]: filtered_findings.pop(key, None) + # pylint: disable-next=E0606 if "createdAfter" in filters and finding.creation_date < min_date: filtered_findings.pop(key, None) + # pylint: disable-next=E0606 if "createdBefore" in filters and finding.creation_date > max_date: filtered_findings.pop(key, None) diff --git a/sonar/issues.py b/sonar/issues.py index 020c06019..38202380c 100644 --- a/sonar/issues.py +++ b/sonar/issues.py @@ -29,7 +29,6 @@ from queue import Queue from threading import Thread import requests.utils -from requests.exceptions import HTTPError import sonar.logging as log import sonar.platform as pf @@ -732,8 +731,8 @@ def __search_thread(queue: Queue) -> None: i["pullRequest"] = page_params.get("pullRequest", None) issue_list[i["key"]] = get_object(endpoint=endpoint, key=i["key"], data=i) log.debug("Added %d issues in threaded search page %d", len(data["issues"]), page) - except HTTPError as e: - log.critical("HTTP Error while searching issues, search may be incomplete: %s", str(e)) + except Exception as e: + log.error("%s while searching issues, search may be incomplete", util.error_msg(e)) queue.task_done() @@ -794,6 +793,7 @@ def search(endpoint: pf.Platform, params: ApiParams = None, raise_error: bool = worker.setDaemon(True) worker.start() q.join() + log.debug("Issue search for %s completed with %d issues", str(params), len(issue_list)) return issue_list @@ -849,13 +849,16 @@ def count_by_rule(endpoint: pf.Platform, **kwargs) -> dict[str, int]: rulecount = {} for i in range(nbr_slices): params["rules"] = ",".join(ruleset[i * SLICE_SIZE : min((i + 1) * SLICE_SIZE - 1, len(ruleset))]) - data = json.loads(endpoint.get(Issue.SEARCH_API, params=params).text)["facets"][0]["values"] - for d in data: - if d["val"] not in ruleset: - continue - if d["val"] not in rulecount: - rulecount[d["val"]] = 0 - rulecount[d["val"]] += d["count"] + try: + data = json.loads(endpoint.get(Issue.SEARCH_API, params=params).text)["facets"][0]["values"] + for d in data: + if d["val"] not in ruleset: + continue + if d["val"] not in rulecount: + rulecount[d["val"]] = 0 + rulecount[d["val"]] += d["count"] + except Exception as e: + log.error("%s while counting issues per rule, count may be incomplete", util.error_msg(e)) return rulecount diff --git a/sonar/measures.py b/sonar/measures.py index 397e1356d..64f7a09fa 100644 --- a/sonar/measures.py +++ b/sonar/measures.py @@ -25,7 +25,7 @@ import json import re from http import HTTPStatus -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException from sonar import metrics, exceptions from sonar.util.types import ApiPayload, ApiParams, KeyList @@ -124,9 +124,10 @@ def get(concerned_object: object, metrics_list: KeyList, **kwargs) -> dict[str, try: data = json.loads(concerned_object.endpoint.get(Measure.API_READ, params={**kwargs, **params}).text) - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(concerned_object.key, f"{str(concerned_object)} not found") + log.error("%s while getting measures %s of %s", util.error_msg(e), str(metrics_list), str(concerned_object)) raise e m_dict = {m: None for m in metrics_list} for m in data["component"]["measures"]: @@ -154,9 +155,10 @@ def get_history(concerned_object: object, metrics_list: KeyList, **kwargs) -> li try: data = json.loads(concerned_object.endpoint.get(Measure.API_HISTORY, params={**kwargs, **params}).text) - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(concerned_object.key, f"{str(concerned_object)} not found") + log.error("%s while getting measures %s history of %s", util.error_msg(e), str(metrics_list), str(concerned_object)) raise e res_list = [] # last_metric, last_date = "", "" diff --git a/sonar/organizations.py b/sonar/organizations.py index b323e03f9..8a7470039 100644 --- a/sonar/organizations.py +++ b/sonar/organizations.py @@ -24,11 +24,10 @@ """ from __future__ import annotations -from typing import Optional import json from http import HTTPStatus from threading import Lock -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException import sonar.logging as log import sonar.platform as pf @@ -79,9 +78,10 @@ def get_object(cls, endpoint: pf.Platform, key: str) -> Organization: return _OBJECTS[uu] try: data = json.loads(endpoint.get(Organization.SEARCH_API, params={"organizations": key}).text) - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(key, f"Organization '{key}' not found") + log.error("%s getting organization %s", util.error_msg(e), key) raise e if len(data["organizations"]) == 0: raise exceptions.ObjectNotFound(key, f"Organization '{key}' not found") diff --git a/sonar/permissions/permission_templates.py b/sonar/permissions/permission_templates.py index 548ad343a..60492f048 100644 --- a/sonar/permissions/permission_templates.py +++ b/sonar/permissions/permission_templates.py @@ -22,7 +22,7 @@ import json import re -from requests.exceptions import HTTPError +from requests import RequestException import sonar.logging as log from sonar.util import types @@ -141,8 +141,9 @@ def set_as_default(self, what_list: list[str]) -> None: continue try: self.post("permissions/set_default_template", params={"templateId": self.key, "qualifier": qual}) - except HTTPError as e: - log.error("HTTP Error: %s", utilities.sonar_error(e.response)) + except (ConnectionError, RequestException) as e: + log.error("%s while setting %s as default", utilities.error_msg(e), str(self)) + raise def set_pattern(self, pattern: str) -> PermissionTemplate: """Sets a permission template pattern""" diff --git a/sonar/permissions/permissions.py b/sonar/permissions/permissions.py index 7f2c46cea..d3d0b2c76 100644 --- a/sonar/permissions/permissions.py +++ b/sonar/permissions/permissions.py @@ -25,11 +25,10 @@ import json from abc import ABC, abstractmethod -from http import HTTPStatus -from requests.exceptions import HTTPError +from requests import RequestException import sonar.logging as log -from sonar import utilities, errcodes +from sonar import utilities from sonar.util import types from sonar.audit.rules import get_rule, RuleId from sonar.audit.problem import Problem @@ -227,10 +226,10 @@ def _get_api(self, api: str, perm_type: str, ret_field: str, **extra_params) -> params = extra_params.copy() page, nbr_pages = 1, 1 counter = 0 - while page <= nbr_pages: + while page <= nbr_pages and counter <= 5: params["p"] = page - resp = self.endpoint.get(api, params=params) - if resp.ok: + try: + resp = self.endpoint.get(api, params=params) data = json.loads(resp.text) # perms.update({p[ret_field]: p["permissions"] for p in data[perm_type]}) for p in data[perm_type]: @@ -239,12 +238,10 @@ def _get_api(self, api: str, perm_type: str, ret_field: str, **extra_params) -> counter = 0 else: counter += 1 - elif resp.status_code not in (HTTPStatus.BAD_REQUEST, HTTPStatus.NOT_FOUND): - # Hack: Different versions of SonarQube return different codes (400 or 404) - utilities.exit_fatal(f"HTTP error {resp.status_code} - Exiting", errcodes.SONAR_API) - page, nbr_pages = page + 1, utilities.nbr_pages(data) - if counter > 5 or not resp.ok: - break + page, nbr_pages = page + 1, utilities.nbr_pages(data) + except (ConnectionError, RequestException) as e: + log.error("%s while retrieving %s permissions", utilities.error_msg(e), str(self)) + page += 1 return perms def _post_api(self, api: str, set_field: str, perms_dict: types.JsonPermissions, **extra_params) -> bool: @@ -259,8 +256,8 @@ def _post_api(self, api: str, set_field: str, perms_dict: types.JsonPermissions, params["permission"] = p try: r = self.endpoint.post(api, params=params) - except HTTPError as e: - log.error("HTTP Error: %s", utilities.sonar_error(e.response)) + except (ConnectionError, RequestException) as e: + log.error("%s while setting permissions %s", utilities.error_msg(e), str(self)) result = result and r.ok return result diff --git a/sonar/permissions/quality_permissions.py b/sonar/permissions/quality_permissions.py index 7d1b22b37..a7a7d7e1d 100644 --- a/sonar/permissions/quality_permissions.py +++ b/sonar/permissions/quality_permissions.py @@ -24,11 +24,11 @@ from typing import Optional import json -from http import HTTPStatus +from requests import RequestException from sonar.util import types import sonar.logging as log -from sonar import utilities, errcodes +from sonar import utilities from sonar.permissions import permissions MAX_PERMS = 25 @@ -71,16 +71,14 @@ def _get_api(self, api: str, perm_type: tuple[str, ...], ret_field: str, **extra page, nbr_pages = 1, 1 while page <= nbr_pages: params["p"] = page - resp = self.endpoint.get(api, params=params) - if resp.ok: + try: + resp = self.endpoint.get(api, params=params) data = json.loads(resp.text) perms += [p[ret_field] for p in data[perm_type]] - elif resp.status_code not in (HTTPStatus.BAD_REQUEST, HTTPStatus.NOT_FOUND): - # Hack: Different versions of SonarQube return different codes (400 or 404) - utilities.exit_fatal(f"HTTP error {resp.status_code} - Exiting", errcodes.SONAR_API) - else: - break - page, nbr_pages = page + 1, utilities.nbr_pages(data) + page, nbr_pages = page + 1, utilities.nbr_pages(data) + except (ConnectionError, RequestException) as e: + log.error("%s while retrieving %s permissions", utilities.error_msg(e), str(self)) + page += 1 return perms def _set_perms(self, new_perms: types.ObjectJsonRepr, apis: dict[str, dict[str, str]], field: str, diff_func: callable, **kwargs) -> bool: diff --git a/sonar/platform.py b/sonar/platform.py index a06ca4d2b..fc2676e6f 100644 --- a/sonar/platform.py +++ b/sonar/platform.py @@ -35,7 +35,7 @@ import tempfile import requests import jprops -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException, Timeout import sonar.logging as log import sonar.utilities as util @@ -52,7 +52,6 @@ WRONG_CONFIG_MSG = "Audit config property %s has wrong value %s, skipping audit" _NON_EXISTING_SETTING_SKIPPED = "Setting %s does not exist, skipping..." -_HTTP_ERROR = "%s Error: %s HTTP status code %d - %s" _SONAR_TOOLS_AGENT = f"sonar-tools {version.PACKAGE_VERSION}" _UPDATE_CENTER = "https://raw.githubusercontent.com/SonarSource/sonar-update-center-properties/master/update-center-source.properties" @@ -105,7 +104,8 @@ def __credentials(self) -> tuple[str, str]: def verify_connection(self) -> None: try: self.get("server/version") - except HTTPError as e: + except (ConnectionError, RequestException) as e: + log.critical("%s while verifying connection", util.error_msg(e)) raise exceptions.ConnectionError(util.sonar_error(e.response)) def version(self) -> tuple[int, int, int]: @@ -247,20 +247,20 @@ def __run_request( if retry: self.url = new_url r.raise_for_status() - except requests.exceptions.HTTPError as e: + except HTTPError as e: if exit_on_error: # or (r.status_code not in mute and r.status_code not in _NORMAL_HTTP_ERRORS): - util.log_and_exit(r) + log_and_exit(e) else: - _, msg = util.http_error(r) - lvl = log.ERROR - if r.status_code in mute: - lvl = log.DEBUG - log.log(lvl, _HTTP_ERROR, req_type, self.__urlstring(api, params), r.status_code, msg) + lvl = log.DEBUG if r.status_code in mute else log.ERROR + log.log(lvl, "%s (%s request)", util.error_msg(e), req_type) raise e - except requests.exceptions.Timeout as e: + except Timeout as e: util.exit_fatal(str(e), errcodes.HTTP_TIMEOUT) - except requests.RequestException as e: - util.exit_fatal(str(e), errcodes.SONAR_API) + except (ConnectionError, RequestException) as e: + if exit_on_error: # or (r.status_code not in mute and r.status_code not in _NORMAL_HTTP_ERRORS): + util.exit_fatal(str(e), errcodes.SONAR_API) + log.error(str(e)) + raise return r def global_permissions(self): @@ -285,13 +285,14 @@ def sys_info(self) -> dict[str, any]: try: resp = self.get("system/info", mute=(HTTPStatus.INTERNAL_SERVER_ERROR,)) success = True - except HTTPError as e: + except (ConnectionError, RequestException) as e: # Hack: SonarQube randomly returns Error 500 on this API, retry up to 10 times - if e.response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR and counter < 10: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR and counter < 10: log.error("HTTP Error 500 for api/system/info, retrying...") time.sleep(0.5) counter += 1 else: + log.error("%s while getting system info", util.error_msg(e)) raise e self.__sys_info = json.loads(resp.text) success = True @@ -656,6 +657,7 @@ def _audit_lta_latest(self) -> list[Problem]: v = latest() if not v: return [] + # pylint: disable-next=E0606 return [Problem(rule, self.url, ".".join(sq_vers), ".".join(v))] @@ -810,11 +812,11 @@ def __lta_and_latest() -> tuple[tuple[int, int, int], tuple[int, int, int]]: if len(v) == 2: v.append("0") LATEST = tuple(int(n) for n in v) - log.debug("Sonar update center says LTA (ex-LTS) = %s, LATEST = %s", str(LTA), str(LATEST)) - except (EnvironmentError, requests.exceptions.HTTPError): + log.info("Sonar update center says LTA (ex-LTS) = %s, LATEST = %s", str(LTA), str(LATEST)) + except (EnvironmentError, HTTPError): LTA = _HARDCODED_LTA LATEST = _HARDCODED_LATEST - log.debug("Sonar update center read failed, hardcoding LTA (ex-LTS) = %s, LATEST = %s", str(LTA), str(LATEST)) + log.info("Sonar update center read failed, hardcoding LTA (ex-LTS) = %s, LATEST = %s", str(LTA), str(LATEST)) try: os.remove(tmpfile) except EnvironmentError: @@ -899,3 +901,11 @@ def basics( write_q.put(exp) write_q.put(None) return exp + + +def log_and_exit(exception: Exception) -> None: + """If HTTP response is not OK, display an error log and exit""" + err_code, msg = util.http_error_and_code(exception) + if err_code is None: + return + util.exit_fatal(msg, err_code) diff --git a/sonar/portfolio_reference.py b/sonar/portfolio_reference.py index 7fba42b91..fceca290d 100644 --- a/sonar/portfolio_reference.py +++ b/sonar/portfolio_reference.py @@ -25,13 +25,13 @@ from __future__ import annotations from http import HTTPStatus -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException import sonar.logging as log import sonar.platform as pf from sonar.util import types -from sonar import exceptions +from sonar import exceptions, utilities import sonar.sqobject as sq _OBJECTS = {} @@ -73,9 +73,11 @@ def create(cls, reference: object, parent: object, params: types.ApiParams = Non try: parent.endpoint.post("views/add_portfolio", params={"portfolio": parent.key, "reference": reference.key}) - except HTTPError as e: - if e.response.status_code == HTTPStatus.BAD_REQUEST: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.BAD_REQUEST: raise exceptions.ObjectAlreadyExists + log.critical("%s while creating portfolio reference to %s in %s", utilities.error_msg(e), str(reference), str(parent)) + raise e return PortfolioReference(reference=reference, parent=parent) def __str__(self) -> str: diff --git a/sonar/portfolios.py b/sonar/portfolios.py index c2ae5f9a1..ee176932e 100644 --- a/sonar/portfolios.py +++ b/sonar/portfolios.py @@ -30,7 +30,7 @@ import datetime from http import HTTPStatus from threading import Lock -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException import sonar.logging as log import sonar.platform as pf @@ -262,8 +262,9 @@ def add_reference_subportfolio(self, reference: Portfolio) -> object: self.post("views/add_portfolio", params={"portfolio": self.key, "reference": reference.key}, mute=(HTTPStatus.BAD_REQUEST,)) else: self.post("views/add_local_view", params={"key": self.key, "ref_key": reference.key}, mute=(HTTPStatus.BAD_REQUEST,)) - except HTTPError as e: - if e.response.status_code != HTTPStatus.BAD_REQUEST: + except (ConnectionError, RequestException) as e: + if not isinstance(e, HTTPError) or e.response.status_code != HTTPStatus.BAD_REQUEST: + log.error("%s while adding reference subportfolio to %s", util.error_msg(e), str(self)) raise self._sub_portfolios.update({reference.key: ref}) return ref @@ -274,8 +275,9 @@ def add_standard_subportfolio(self, key: str, name: str, **kwargs) -> Portfolio: try: if self.endpoint.version() < (9, 3, 0): self.post("views/add_sub_view", params={"key": self.key, "name": name, "subKey": key}, mute=(HTTPStatus.BAD_REQUEST,)) - except HTTPError as e: - if e.response.status_code != HTTPStatus.BAD_REQUEST: + except (ConnectionError, RequestException) as e: + if not isinstance(e, HTTPError) or e.response.status_code != HTTPStatus.BAD_REQUEST: + log.error("%s while adding standard subportfolio to %s", util.error_msg(e), str(self)) raise self._sub_portfolios.update({subp.key: subp}) return subp @@ -428,10 +430,11 @@ def add_project_branches(self, branch_dict: dict[str, Union[str, object]]) -> Po if e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(self.key, f"Project '{key}' or branch '{branch}' not found, can't be added to {str(self)}") if e.response.status_code == HTTPStatus.BAD_REQUEST: - # Project or branch already in portfolio - pass - else: + log.error("%s while adding project branches to %s", util.error_msg(e), str(self)) raise + except (ConnectionError, RequestException) as e: + log.error("%s while adding project branches to %s", util.error_msg(e), str(self)) + raise return self def set_manual_mode(self) -> Portfolio: @@ -520,10 +523,11 @@ def add_application_branch(self, app_key: str, branch: str = settings.DEFAULT_BR log.info("%s: Adding %s", str(self), str(app_branch)) params = {"key": self.key, "application": app_key, "branch": branch} self.post("views/add_application_branch", params=params, mute=(HTTPStatus.BAD_REQUEST,)) - except HTTPError as e: - if e.response.status_code != HTTPStatus.BAD_REQUEST: + except (ConnectionError, RequestException) as e: + if not isinstance(e, HTTPError) or e.response.status_code != HTTPStatus.BAD_REQUEST: + log.error("%s while adding application branch to %s", util.error_msg(e), str(self)) raise - log.warning(util.sonar_error(e.response)) + log.warning(util.error_msg(e)) if app_key not in self._applications: self._applications[app_key] = [] self._applications[app_key].append(branch) @@ -579,11 +583,8 @@ def get_project_list(self) -> list[str]: data = json.loads(self.get("api/measures/component_tree", params=params).text) nbr_projects = util.nbr_total_elements(data) proj_key_list += [c["refKey"] for c in data["components"]] - except HTTPError as e: - if e.response.status_code in (HTTPStatus.BAD_REQUEST, HTTPStatus.NOT_FOUND): - log.warning("HTTP Error %s while collecting projects from %s, stopping collection", str(e), str(self)) - else: - log.critical("HTTP Error %s while collecting projects from %s, proceeding anyway", str(e), str(self)) + except (ConnectionError, RequestException) as e: + log.error("%s while collecting projects from %s, stopping collection", util.error_msg(e), str(self)) break nbr_pages = util.nbr_pages(data) log.debug("Number of projects: %d - Page: %d/%d", nbr_projects, page, nbr_pages) @@ -790,9 +791,8 @@ def export( exported_portfolios[k] = exp else: log.debug("Skipping export of %s, it's a standard sub-portfolio", str(p)) - except HTTPError as e: - _, msg = util.http_error(e.response) - log.error("%s while exporting %s, export will be empty for this portfolio", msg, str(p)) + except (ConnectionError, RequestException) as e: + log.error("%s while exporting %s, export will be empty for this portfolio", util.error_msg(e), str(p)) exported_portfolios[k] = {} i += 1 if i % 10 == 0 or i == nb_portfolios: diff --git a/sonar/projects.py b/sonar/projects.py index 0c3fa3c47..5f2005f71 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -34,7 +34,7 @@ from http import HTTPStatus from threading import Thread, Lock from queue import Queue -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException import sonar.logging as log import sonar.platform as pf @@ -168,8 +168,9 @@ def get_object(cls, endpoint: pf.Platform, key: str) -> Project: log.error("Project key '%s' not found", key) raise exceptions.ObjectNotFound(key, f"Project key '{key}' not found") return cls.load(endpoint, data["components"][0]) - except HTTPError as e: - if e.response.status_code != HTTPStatus.FORBIDDEN: + except (ConnectionError, RequestException) as e: + if not isinstance(e, HTTPError) or e.response.status_code != HTTPStatus.FORBIDDEN: + log.error("%s while getting project '%s'", util.error_msg(e), key) raise data = json.loads(endpoint.get(_NAV_API, params={"component": key}).text) if "errors" in data: @@ -207,9 +208,11 @@ def create(cls, endpoint: pf.Platform, key: str, name: str) -> Project: """ try: endpoint.post(_CREATE_API, params={"project": key, "name": name}) - except HTTPError as e: - if e.response.status_code == HTTPStatus.BAD_REQUEST: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.BAD_REQUEST: raise exceptions.ObjectAlreadyExists(key, e.response.text) + log.error("%s while creating project '%s'", util.error_msg(e), key) + raise o = cls(endpoint, key) o.name = name return o @@ -377,12 +380,12 @@ def binding(self): resp = self.get("alm_settings/get_binding", params={"project": self.key}, mute=(HTTPStatus.NOT_FOUND,)) self._binding["has_binding"] = True self._binding["binding"] = json.loads(resp.text) - except HTTPError as e: - if e.response.status_code in (HTTPStatus.NOT_FOUND, HTTPStatus.BAD_REQUEST): + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code in (HTTPStatus.NOT_FOUND, HTTPStatus.BAD_REQUEST): # Hack: 8.9 returns 404, 9.x returns 400 self._binding["has_binding"] = False else: - log.error("alm_settings/get_binding returning status code %d", e.response.status_code) + log.error("%s while getting '%s' bindinfs", util.error_msg(e), str(self)) raise e log.debug("Binding = %s", util.json_dump(self._binding["binding"])) return self._binding["binding"] @@ -556,12 +559,11 @@ def __audit_binding_valid(self, audit_settings: types.ConfigSettings) -> list[Pr try: _ = self.get("alm_settings/validate_binding", params={"project": self.key}) log.debug("%s binding is valid", str(self)) - except HTTPError as e: + except (ConnectionError, RequestException) as e: # Hack: 8.9 returns 404, 9.x returns 400 - if e.response.status_code in (HTTPStatus.BAD_REQUEST, HTTPStatus.NOT_FOUND): + if isinstance(e, HTTPError) and e.response.status_code in (HTTPStatus.BAD_REQUEST, HTTPStatus.NOT_FOUND): return [Problem(get_rule(RuleId.PROJ_INVALID_BINDING), self, str(self))] - else: - util.exit_fatal(f"alm_settings/validate_binding returning status code {e.response.status_code}, exiting", errcodes.SONAR_API) + log.error("%s while auditing %s binding, skipped", util.error_msg(e), str(self)) return [] def get_type(self) -> str: @@ -617,8 +619,8 @@ def ci(self) -> str: data = json.loads(self.get("project_analyses/search", params={"project": self.key, "ps": 1}).text)["analyses"] if len(data) > 0: self._ci, self._revision = data[0].get("detectedCI", "unknown"), data[0].get("revision", "unknown") - except HTTPError: - log.warning("HTTP Error, can't retrieve CI tool and revision") + except (ConnectionError, RequestException) as e: + log.warning("%s while getting %s CI tool", util.error_msg(e), str(self)) except KeyError: log.warning("KeyError, can't retrieve CI tool and revision") return self._ci @@ -664,11 +666,11 @@ def audit(self, audit_settings: types.ConfigSettings) -> list[Problem]: problems += self._audit_bg_task(audit_settings) problems += self.__audit_binding_valid(audit_settings) problems += self.__audit_scanner(audit_settings) - except HTTPError as e: - if e.response.status_code == HTTPStatus.FORBIDDEN: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.FORBIDDEN: log.error("Not enough permission to fully audit %s", str(self)) else: - log.error("HTTP error %s while auditing %s", str(e), str(self)) + log.error("%s while auditing %s", util.error_msg(e), str(self)) return problems def export_zip(self, timeout: int = 180) -> dict[str, str]: @@ -688,6 +690,8 @@ def export_zip(self, timeout: int = 180) -> dict[str, str]: resp = self.post("project_dump/export", params={"key": self.key}) except HTTPError as e: return {"status": f"HTTP_ERROR {e.response.status_code}"} + except (ConnectionError, RequestException) as e: + return {"status": str(e)} data = json.loads(resp.text) status = tasks.Task(endpoint=self.endpoint, task_id=data["taskId"], concerned_object=self, data=data).wait_for_completion(timeout=timeout) if status != tasks.SUCCESS: @@ -706,8 +710,9 @@ def export_async(self) -> Union[str, None]: log.info("Exporting %s (asynchronously)", str(self)) try: return json.loads(self.post("project_dump/export", params={"key": self.key}).text)["taskId"] - except HTTPError: - return None + except (ConnectionError, RequestException) as e: + log.error("%s while exporting zip of %s CI", util.error_msg(e), str(self)) + return None def import_zip(self) -> bool: """Imports a project zip file in SonarQube @@ -1046,19 +1051,13 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, if not export_settings.get("INCLUDE_INHERITED", False) and s.inherited: continue json_data.update(s.to_json()) - except HTTPError as e: - if e.response.status_code == HTTPStatus.FORBIDDEN: - log.critical("Insufficient privileges to access %s, export of this project interrupted", str(self)) - json_data["error"] = "Insufficient permissions while exporting project, export interrupted" - else: - log.critical("HTTP error %s while exporting %s, export of this project interrupted", str(e), str(self)) - json_data["error"] = f"HTTP error {str(e)} while extracting project" - except ConnectionError as e: - log.critical("Connecting error %s while exporting %s, export of this project interrupted", str(self), str(e)) - json_data["error"] = f"Connection error {str(e)} while extracting project, export interrupted" + except (ConnectionError, RequestException) as e: + errmsg = util.error_msg(e) + log.error("Exception: %s while exporting %s, export of this project interrupted", errmsg, str(self)) + json_data["error"] = f"{errmsg} while extracting project" except Exception as e: - log.critical("Connecting error %s while exporting %s, export of this project interrupted", str(self), str(e)) - json_data["error"] = f"Exception {str(e)} while exporting project, export interrupted" + log.critical("Exception: %s while exporting %s, export of this project interrupted", errmsg, str(self)) + json_data["error"] = f"{errmsg} while extracting project" log.debug("Exporting %s done", str(self)) return util.remove_nones(json_data) @@ -1091,10 +1090,10 @@ def set_permissions(self, desired_permissions: types.ObjectJsonRepr) -> bool: try: self.permissions().set(desired_permissions) return True - except HTTPError as e: - if e.response.status_code != HTTPStatus.BAD_REQUEST: + except (ConnectionError, RequestException) as e: + log.error("%s while setting permissions of %s", util.error_msg(e), str(self)) + if not isinstance(e, HTTPError) or e.response.status_code != HTTPStatus.BAD_REQUEST: raise e - log.error(util.sonar_error(e.response)) return False def set_links(self, desired_links: types.ObjectJsonRepr) -> bool: @@ -1429,11 +1428,8 @@ def __audit_thread(queue: Queue[Project], results: list[Problem], audit_settings results.append(Problem(get_rule(RuleId.PROJ_DUPLICATE_BINDING), project, str(project), str(bindings[bindkey]))) else: bindings[bindkey] = project - except HTTPError as e: - if e.response.status_code == HTTPStatus.FORBIDDEN: - log.error("Not enough permission to fully audit %s", str(project)) - else: - log.error("HTTP error %s while auditing %s", str(e), str(project)) + except (ConnectionError, RequestException) as e: + log.error("%s while auditing %s", util.error_msg(e), str(project)) queue.task_done() log.debug("%s audit complete", str(project)) log.debug("Queue empty, exiting thread") @@ -1486,10 +1482,10 @@ def __export_thread(queue: Queue[Project], results: dict[str, str], export_setti with _CLASS_LOCK: export_settings["EXPORTED"] += 1 nb, tot = export_settings["EXPORTED"], export_settings["NBR_PROJECTS"] - log.debug("%d/%d projects exported (%d%%)", nb, tot, (nb * 100) // tot) - if nb % 10 == 0 or tot - nb < 10: - log.info("%d/%d projects exported (%d%%)", nb, tot, (nb * 100) // tot) + lvl = log.INFO if nb % 10 == 0 or tot - nb < 10 else log.DEBUG + log.log(lvl, "%d/%d projects exported (%d%%)", nb, tot, (nb * 100) // tot) queue.task_done() + log.info("Project export queue empty, export complete") def export( diff --git a/sonar/qualitygates.py b/sonar/qualitygates.py index 265bba8f0..d08251148 100644 --- a/sonar/qualitygates.py +++ b/sonar/qualitygates.py @@ -29,7 +29,7 @@ from http import HTTPStatus import json -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException import sonar.logging as log import sonar.sqobject as sq @@ -173,9 +173,11 @@ def projects(self) -> dict[str, projects.Project]: params["p"] = page try: resp = self.get(APIS["get_projects"], params=params) - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(self.name, f"{str(self)} not found") + log.error("%s while getting %s projects", util.error_msg(e), str(self)) + raise data = json.loads(resp.text) for prj in data["results"]: log.info("Proj = %s", str(prj)) diff --git a/sonar/qualityprofiles.py b/sonar/qualityprofiles.py index 39dcc4301..d9745f7b1 100644 --- a/sonar/qualityprofiles.py +++ b/sonar/qualityprofiles.py @@ -23,10 +23,9 @@ import json from datetime import datetime -from http import HTTPStatus from queue import Queue from threading import Thread, Lock -from requests import HTTPError +from requests import RequestException import requests.utils import sonar.logging as log @@ -260,11 +259,11 @@ def activate_rule(self, rule_key: str, severity: str = None, **params) -> bool: api_params = {"key": self.key, "rule": rule_key, "severity": severity} if len(params) > 0: api_params["params"] = ";".join([f"{k}={v}" for k, v in params.items()]) - r = self.post("qualityprofiles/activate_rule", params=api_params) - if r.status_code == HTTPStatus.NOT_FOUND: - log.error("Rule %s not found, can't activate it in %s", rule_key, str(self)) - elif r.status_code == HTTPStatus.BAD_REQUEST: - log.error("HTTP error %d while trying to activate rule %s in %s", r.status_code, rule_key, str(self)) + try: + r = self.post("qualityprofiles/activate_rule", params=api_params) + except (ConnectionError, RequestException) as e: + log.error("%s while trying to activate rule %s in %s", util.error_msg(e), rule_key, str(self)) + return False return r.ok def activate_rules(self, ruleset: dict[str, str]) -> bool: @@ -283,9 +282,9 @@ def activate_rules(self, ruleset: dict[str, str]) -> bool: ok = ok and self.activate_rule(rule_key=r_key, severity=sev, **r_data["params"]) else: ok = ok and self.activate_rule(rule_key=r_key, severity=sev) - except HTTPError as e: + except (ConnectionError, RequestException) as e: ok = False - log.warning("Activation of rule '%s' in %s failed: HTTP Error %d", r_key, str(self), e.response.status_code) + log.error("%s while activating rules in '%s'", util.error_msg(e), r_key) return ok def update(self, data: types.ObjectJsonRepr, queue: Queue) -> QualityProfile: diff --git a/sonar/rules.py b/sonar/rules.py index 37ffe7ec7..7c4886e6e 100644 --- a/sonar/rules.py +++ b/sonar/rules.py @@ -27,7 +27,7 @@ import json from typing import Optional from http import HTTPStatus -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException import sonar.logging as log import sonar.sqobject as sq @@ -131,9 +131,10 @@ def get_object(cls, endpoint: platform.Platform, key: str) -> Rule: log.debug("Reading rule key '%s'", key) try: r = endpoint.get(_DETAILS_API, params={"key": key}) - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.NOT_FOUND: raise exceptions.ObjectNotFound(key=key, message=f"Rule key '{key}' does not exist") + log.error("%s while getting rule'%s'", utilities.error_msg(e), key) return Rule(endpoint=endpoint, key=key, data=json.loads(r.text)["rule"]) @classmethod diff --git a/sonar/settings.py b/sonar/settings.py index d0ac23142..673b48bcd 100644 --- a/sonar/settings.py +++ b/sonar/settings.py @@ -26,7 +26,7 @@ import json from typing import Union from http import HTTPStatus -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException import sonar.logging as log import sonar.platform as pf @@ -454,10 +454,11 @@ def set_new_code_period(endpoint: pf.Platform, nc_type: str, nc_value: str, proj ok = ok and endpoint.post(API_SET, params={"key": "sonar.leak.period", "value": nc_value, "project": project_key}).ok else: ok = endpoint.post(API_NEW_CODE_SET, params={"type": nc_type, "value": nc_value, "project": project_key, "branch": branch}).ok - except HTTPError as e: - if e.response.status_code == HTTPStatus.BAD_REQUEST: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.BAD_REQUEST: raise exceptions.UnsupportedOperation(f"Can't set project new code period: {e.response.text}") - raise + log.error("%s setting new code period of '%s'", util.error_msg(e), str(project_key)) + return False return ok @@ -486,10 +487,11 @@ def set_visibility(endpoint: pf.Platform, visibility: str, component: object = N else: log.debug("Setting setting '%s' to value '%s'", PROJECT_DEFAULT_VISIBILITY, str(visibility)) return endpoint.post("projects/update_default_visibility", params={"projectVisibility": visibility}).ok - except HTTPError as e: - if e.response.status_code == HTTPStatus.BAD_REQUEST: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.BAD_REQUEST: raise exceptions.UnsupportedOperation(f"Can't set project default visibility: {e.response.text}") - raise + log.error("%s setting new code period of '%s'", util.error_msg(e), str(component)) + return False def set_setting(endpoint: pf.Platform, key: str, value: any, component: object = None) -> bool: @@ -501,8 +503,8 @@ def set_setting(endpoint: pf.Platform, key: str, value: any, component: object = else: try: s.set(value) - except HTTPError as e: - log.error("Setting '%s' cannot be set: %s", key, util.sonar_error(e.response)) + except (ConnectionError, RequestException) as e: + log.error("%s while setting setting '%s'", util.error_msg(e), key, str(component)) return False except exceptions.UnsupportedOperation as e: log.error("Setting '%s' cannot be set: %s", key, e.message) diff --git a/sonar/sqobject.py b/sonar/sqobject.py index f8605bf72..ed5bb5930 100644 --- a/sonar/sqobject.py +++ b/sonar/sqobject.py @@ -29,7 +29,7 @@ from queue import Queue from threading import Thread import requests -from requests.exceptions import HTTPError +from requests import HTTPError, RequestException import sonar.logging as log from sonar.util import types @@ -116,8 +116,8 @@ def __search_thread(queue: Queue) -> None: objects[obj[key_field]] = object_class.load(endpoint=endpoint, data=obj) else: objects[obj[key_field]] = object_class(endpoint, obj[key_field], data=obj) - except HTTPError as e: - log.critical("HTTP error while searching %s, search skipped: %s", object_class.__name__, str(e)) + except (ConnectionError, RequestException) as e: + log.critical("%s while searching %s, search skipped", utilities.error_msg(e), object_class.__name__) queue.task_done() @@ -163,10 +163,11 @@ def delete_object(object: SqObject, api: str, params: types.ApiParams, map: dict map.pop(object.uuid(), None) log.info("Successfully deleted %s", str(object)) return r.ok - except HTTPError as e: - if e.response.status_code == HTTPStatus.NOT_FOUND: + except (ConnectionError, RequestException) as e: + if isinstance(e, HTTPError) and e.response.status_code == HTTPStatus.NOT_FOUND: map.pop(object.uuid(), None) raise exceptions.ObjectNotFound(object.key, f"{str(object)} not found for delete") + log.error("%s while deleting object '%s'", utilities.error_msg(e), str(object)) raise diff --git a/sonar/utilities.py b/sonar/utilities.py index 9ae910f81..ce9fb766a 100644 --- a/sonar/utilities.py +++ b/sonar/utilities.py @@ -52,7 +52,7 @@ def check_last_version(package_url: str) -> None: try: r = requests.get(url=package_url, headers={"Accept": "application/vnd.pypi.simple.v1+json"}, timeout=10) r.raise_for_status() - except (requests.RequestException, requests.exceptions.HTTPError, requests.exceptions.Timeout) as e: + except Exception as e: log.info("Can't access pypi.org, error %s", str(e)) return txt_version = json.loads(r.text)["versions"][-1] @@ -423,10 +423,11 @@ def sonar_error(response: requests.models.Response) -> str: return "" -def http_error(response: requests.models.Response) -> tuple[str, int]: +def http_error_and_code(exception: requests.HTTPError) -> tuple[int, str]: """Returns the Sonar error code of an API HTTP response, or None if no error""" + response = exception.response if response.ok: - return None, None + return None, "No error" tool_msg = f"For request URL {response.request.url}\n" code = response.status_code if code == HTTPStatus.UNAUTHORIZED: @@ -441,12 +442,13 @@ def http_error(response: requests.models.Response) -> tuple[str, int]: return err_code, f"{tool_msg}: {sonar_error(response)}" -def log_and_exit(response: requests.models.Response) -> None: - """If HTTP response is not OK, display an error log and exit""" - err_code, msg = http_error(response) - if err_code is None: - return - exit_fatal(msg, err_code) +def error_msg(exception: Exception) -> str: + """Returns the error of an Sonar API HTTP response, or None if no error""" + if isinstance(exception, requests.HTTPError): + _, errmsg = http_error_and_code(exception) + return errmsg + else: + return str(exception) def object_key(key_or_obj):