Merge pull request #451 from okorach:handle-non-existing-project-keys

Handle non existing project keys gracefully

Author: okorach

README.md

@@ -53,7 +53,8 @@ Using login/password is not possible.
 The user corresponding to the token must have enough permissions to achieve the tool tasks
 - `-v` : Logging verbosity level (`WARN`, `ÌNFO` or `DEBUG`). The default is `INFO`.
 `ERROR` and above is always active.
-- 
+
+See common [error exit codes](#exit-codes) at the bottom of this page
 
 # <a name="sonar-audit"></a>sonar-audit
 
@@ -243,16 +244,19 @@ export SONAR_TOKEN=15ee09df11fb9b8234b7a1f1ac5fce2e4e93d75d
 sonar-projects-import -f exported_projects.csv
 ```
 
-# Tools coming soon
-
-## sonar-issues-recover
+# <a name="exit-codes"></a>Exit codes
 
-Tries to recover issues that were mistakenly closed following a scan with incorrect parameters. This tool is only useful for SonarQube instances in version 7.9.x and lower since this feature is built-in with SonarQube 8.x and higher
+When tools complete successfully they return exit code 0. En case of fatal error the following exit codes may be returned:
+- Code 1: Authentication error (Incorrect token provided)
+- Code 2: Authorization error (provided token has insufficient permissions)
+- Code 3: Other general Sonar API HTTP error
+- Code 4: No token provided
+- Code 5: Non existing project key provided
+- Code 6: Incorrect finding search criteria provided
+- Code 7: Unsupported operation requested (because of SonarQube edition or configuration)
+- Code 8: Audit rule loading failed (at startup)
+- Code 9: SIF audit error (file not found, can't open file, not a legit JSON file, ...)
 
-Issue recovery means:
-- Reapplying all transitions to the issue to reach its final state before close (Usually *False positive* or *Won't Fix*)
-- Reapplying all manual comments
-- Reapplying all severity or issue type change
 
 ### :information_source: Limitations
 - The script has to be run before the closed issue purge period (SonarQube parameter `sonar.dbcleaner.daysBeforeDeletingClosedIssues` whose default value is **30 days**)

sonar/audit/__init__.py

@@ -19,9 +19,9 @@
 #
 import sys
 from sonar.audit import rules
+from sonar import utilities, options
 
 try:
     rules.load()
 except rules.RuleConfigError as e:
-    print(e.message)
-    sys.exit(3)
+    utilities.exit_fatal(e.message, options.ERR_RULES_LOADING_FAILED)

sonar/audit/audit.py

@@ -25,9 +25,10 @@
 '''
 import sys
 import json
+
 import sonar.portfolios as pf
 import sonar.applications as apps
-from sonar import users, groups, version, env, qualityprofiles, qualitygates, projects, sif
+from sonar import users, groups, version, env, qualityprofiles, qualitygates, projects, sif, options
 import sonar.utilities as util
 from sonar.audit import problem, config
 
@@ -94,8 +95,7 @@ def __parser_args(desc):
                         'or outputs to stdout if it already exist')
     args = parser.parse_args()
     if args.sif is None and args.config is None and args.token is None:
-        util.logger.critical("Token is missing (Argument -t/--token) when not analyzing local SIF")
-        sys.exit(4)
+        util.exit_fatal("Token is missing (Argument -t/--token) when not analyzing local SIF", options.ERR_TOKEN_MISSING)
     return args
 
 def main():
@@ -114,17 +114,13 @@ def main():
         try:
             problems = _audit_sif(kwargs['sif'])
         except json.decoder.JSONDecodeError:
-            print(f"File {kwargs['sif']} does not seem to be a legit JSON file, aborting...")
-            sys.exit(3)
+            util.exit_fatal(f"File {kwargs['sif']} does not seem to be a legit JSON file, aborting...", options.ERR_SIF_AUDIT_ERROR)
         except FileNotFoundError:
-            print(f"File {kwargs['sif']} does not exist, aborting...")
-            sys.exit(4)
+            util.exit_fatal(f"File {kwargs['sif']} does not exist, aborting...", options.ERR_SIF_AUDIT_ERROR)
         except PermissionError:
-            print(f"No permissiont to open file {kwargs['sif']}, aborting...")
-            sys.exit(5)
+            util.exit_fatal(f"No permissiont to open file {kwargs['sif']}, aborting...", options.ERR_SIF_AUDIT_ERROR)
         except sif.NotSystemInfo:
-            print(f"File {kwargs['sif']} does not seem to be a system info or support info file, aborting...")
-            sys.exit(6)
+            util.exit_fatal(f"File {kwargs['sif']} does not seem to be a system info or support info file, aborting...", options.ERR_SIF_AUDIT_ERROR)
     else:
         problems = _audit_sq(sq, settings, args.what)
 

sonar/env.py

@@ -29,17 +29,14 @@
 import requests
 
 import sonar.utilities as util
-
+from sonar import options
 from sonar.audit import rules, config
 import sonar.audit.severities as sev
 import sonar.audit.types as typ
 import sonar.audit.problem as pb
 
 from sonar import sif
 
-AUTHENTICATION_ERROR_MSG = "Authentication error. Is token valid ?"
-AUTORIZATION_ERROR_MSG = "Insufficient permissions to perform operation"
-HTTP_FATAL_ERROR_MSG = "HTTP fatal error %d - %s"
 WRONG_CONFIG_MSG = "Audit config property %s has wrong value %s, skipping audit"
 
 _NON_EXISTING_SETTING_SKIPPED = "Setting %s does not exist, skipping..."
@@ -129,12 +126,11 @@ def get(self, api, params=None, exit_on_error=True):
             else:
                 r = requests.get(url=self.url + api, auth=self.credentials(), params=params)
             r.raise_for_status()
-        except requests.exceptions.HTTPError as errh:
+        except requests.exceptions.HTTPError:
             if exit_on_error:
-                _log_and_exit(r.status_code, errh)
+                _log_and_exit(r.status_code)
         except requests.RequestException as e:
-            util.logger.error(str(e))
-            raise SystemExit(e) from e
+            util.exit_fatal(str(e), options.ERR_SONAR_API)
         return r
 
     def post(self, api, params=None):
@@ -146,11 +142,10 @@ def post(self, api, params=None):
             else:
                 r = requests.post(url=self.url + api, auth=self.credentials(), params=params)
             r.raise_for_status()
-        except requests.exceptions.HTTPError as errh:
-            _log_and_exit(r.status_code, errh)
+        except requests.exceptions.HTTPError:
+            _log_and_exit(r.status_code)
         except requests.RequestException as e:
-            util.logger.error(str(e))
-            raise SystemExit(e) from e
+            util.exit_fatal(str(e), options.ERR_SONAR_API)
         return r
 
     def delete(self, api, params=None):
@@ -162,11 +157,10 @@ def delete(self, api, params=None):
             else:
                 r = requests.delete(url=self.url + api, auth=self.credentials(), params=params)
             r.raise_for_status()
-        except requests.exceptions.HTTPError as errh:
-            _log_and_exit(r.status_code, errh)
+        except requests.exceptions.HTTPError:
+            _log_and_exit(r.status_code)
         except requests.RequestException as e:
-            util.logger.error(str(e))
-            raise SystemExit(e) from e
+            util.exit_fatal(str(e), options.ERR_SONAR_API)
 
     def urlstring(self, api, params):
         first = True
@@ -247,9 +241,7 @@ def _audit_admin_password(self):
             else:
                 util.logger.info("User 'admin' default password has been changed")
         except requests.RequestException as e:
-            util.logger.error("HTTP request exception for %s/%s: %s", self.url,
-                              'api/authentication/validate', str(e))
-            raise
+            util.exit_fatal(str(e), options.ERR_SONAR_API)
         return problems
 
     def __get_permissions(self, perm_type):
@@ -371,16 +363,13 @@ def _normalize_api(api):
     return api
 
 
-def _log_and_exit(code, err):
+def _log_and_exit(code):
     if code == 401:
-        util.logger.fatal(AUTHENTICATION_ERROR_MSG)
-        raise SystemExit(err)
+        util.exit_fatal(f"HTTP error {code} - Authentication error. Is token valid ?", options.ERR_SONAR_API_AUTHENTICATION)
     if code == 403:
-        util.logger.fatal(AUTORIZATION_ERROR_MSG)
-        raise SystemExit(err)
+        util.exit_fatal(f"HTTP error {code} - Insufficient permissions to perform operation", options.ERR_SONAR_API_AUTHORIZATION)
     if (code // 100) != 2:
-        util.logger.fatal(HTTP_FATAL_ERROR_MSG, code, err)
-        raise SystemExit(err)
+        util.exit_fatal(f"HTTP error {code} - Exiting", options.ERR_SONAR_API)
 
 
 def get(api, params=None, ctxt=None, exit_on_error=True):

sonar/findings/export.py

@@ -160,20 +160,20 @@ def __get_list(project, list_str, list_type):
 def __verify_inputs(params):
     diff = util.difference(util.csv_to_list(params.get('resolutions', None)), issues.RESOLUTIONS + hotspots.RESOLUTIONS)
     if diff:
-        util.logger.fatal("Resolutions %s are not legit status", str(diff))
-        return False
+        util.exit_fatal(f"Resolutions {str(diff)} are not legit resolutions", options.ERR_WRONG_SEARCH_CRITERIA)
+
     diff = util.difference(util.csv_to_list(params.get('statuses', None)), issues.STATUSES + hotspots.STATUSES)
     if diff:
-        util.logger.fatal("Status %s are not legit status", str(diff))
-        return False
+        util.exit_fatal(f"Statuses {str(diff)} are not legit statuses", options.ERR_WRONG_SEARCH_CRITERIA)
+
     diff = util.difference(util.csv_to_list(params.get('severities', None)), issues.SEVERITIES + hotspots.SEVERITIES)
     if diff:
-        util.logger.fatal("Severities %s are not legit severities", str(diff))
-        return False
+        util.exit_fatal(f"Severities {str(diff)} are not legit severities", options.ERR_WRONG_SEARCH_CRITERIA)
+
     diff = util.difference(util.csv_to_list(params.get('types', None)), issues.TYPES + hotspots.TYPES)
     if diff:
-        util.logger.fatal("Types %s are not legit types", str(diff))
-        return False
+        util.exit_fatal(f"Types {str(diff)} are not legit types", options.ERR_WRONG_SEARCH_CRITERIA)
+
     return True
 
 
@@ -230,20 +230,15 @@ def main():
 
     project_key = kwargs.get('projectKeys', None)
     params = util.remove_nones(kwargs.copy())
-    if not __verify_inputs(params):
-        sys.exit(2)
+    __verify_inputs(params)
+
     for p in ('statuses', 'createdAfter', 'createdBefore', 'resolutions', 'severities', 'types', 'tags'):
         if params.get(p, None) is not None:
             if params['useFindings']:
                 util.logger.warning("Selected search criteria %s will disable --useFindings", params[p])
             params['useFindings'] = False
             break
-    if project_key is None:
-        project_list = projects.get_key_list(endpoint=sqenv)
-    else:
-        project_list = []
-        for key in util.csv_to_list(project_key):
-            project_list.append(projects.get_object(key, endpoint=sqenv).key)
+    project_list = projects.get_projects_list(project_key, sqenv)
 
     fmt = kwargs['format']
     file = kwargs.pop('file', None)

sonar/findings/sync.py

@@ -28,9 +28,7 @@
     Only issues with a 100% match are synchronized. When there's a doubt, nothing is done
 '''
 
-
-import sys
-from sonar import env, projects, branches, version, syncer
+from sonar import env, projects, branches, version, syncer, options
 import sonar.utilities as util
 
 _WITH_COMMENTS = {'additionalFields': 'comments'}
@@ -144,8 +142,7 @@ def main():
                          counters.get('nb_tgt_has_changelog', 0))
 
     except env.NonExistingObjectError as e:
-        util.logger.critical(e.message)
-        sys.exit(1)
+        util.exit_fatal(e.message, options.ERR_NO_SUCH_PROJECT_KEY)
 
 
 if __name__ == '__main__':

sonar/measures_export.py

@@ -187,11 +187,8 @@ def main():
     wanted_metrics = __get_wanted_metrics(args, endpoint)
     (fmt, file) = __get_fmt_and_file(args)
 
-    filters = None
-    if args.projectKeys is not None:
-        filters = {'projects': args.projectKeys.replace(' ', '')}
-    util.logger.info("Getting project list")
-    project_list = projects.search(endpoint=endpoint, params=filters).values()
+    project_list = projects.get_projects_list(args.projectKeys, endpoint)
+
     is_first = True
     obj_list = []
     if with_branches:

sonar/options.py

@@ -30,3 +30,14 @@
 
 CSV_SEPARATOR = 'csvSeparator'
 FORMAT = 'format'
+
+ERR_SONAR_API_AUTHENTICATION = 1
+ERR_SONAR_API_AUTHORIZATION = 2
+ERR_SONAR_API = 3
+ERR_TOKEN_MISSING = 4
+
+ERR_NO_SUCH_PROJECT_KEY = 5
+ERR_WRONG_SEARCH_CRITERIA = 6
+ERR_UNSUPPORTED_OPERATION = 7
+ERR_RULES_LOADING_FAILED = 8
+ERR_SIF_AUDIT_ERROR = 9

sonar/projects.py

@@ -70,6 +70,8 @@ def __load__(self, data=None):
         if data is None:
             resp = env.get(PROJECT_SEARCH_API, ctxt=self.endpoint, params={'projects': self.key})
             data = json.loads(resp.text)
+            if not data['components']:
+                raise env.NonExistingObjectError(self.key, "Project key does not exist")
             data = data['components'][0]
         self.name = data['name']
         self.visibility = data['visibility']
@@ -219,8 +221,7 @@ def binding(self):
                 self._binding['has_binding'] = True
                 self._binding['binding'] = json.loads(resp.text)
             else:
-                util.logger.fatal("alm_settings/get_binding returning status code %d, exiting", resp.status_code)
-                raise SystemExit(1)
+                util.exit_fatal(f"alm_settings/get_binding returning status code {resp.status_code}, exiting", options.ERR_SONAR_API)
         return self._binding['binding']
 
     def is_part_of_monorepo(self):
@@ -442,8 +443,7 @@ def __audit_binding_valid(self, audit_settings):
             rule = rules.get_rule(rules.RuleId.PROJ_INVALID_BINDING)
             return [pb.Problem(rule.type, rule.severity, rule.msg.format(str(self)), concerned_object=self)]
         else:
-            util.logger.fatal("alm_settings/validate_binding returning status code %d, exiting", resp.status_code)
-            raise SystemExit(1)
+            util.exit_fatal(f"alm_settings/get_binding returning status code {resp.status_code}, exiting", options.ERR_SONAR_API)
 
     def audit(self, audit_settings):
         util.logger.debug("Auditing %s", str(self))
@@ -629,6 +629,20 @@ def get_object_list(endpoint=None, params=None):
     return search(endpoint, params).values()
 
 
+def get_projects_list(str_key_list, endpoint):
+    if str_key_list is None:
+        util.logger.info("Getting project list")
+        project_list = search(endpoint=endpoint).values()
+    else:
+        project_list = []
+        try:
+            for key in util.csv_to_list(str_key_list):
+                project_list.append(get_object(key, endpoint=endpoint).key)
+        except env.NonExistingObjectError as e:
+            util.exit_fatal(f"Project key '{e.key}' does not exist, aborting...", options.ERR_NO_SUCH_PROJECT_KEY)
+    return project_list
+
+
 def key_obj(key_or_obj):
     if isinstance(key_or_obj, str):
         return (key_or_obj, get_object(key_or_obj))

sonar/projects_export.py

@@ -25,7 +25,7 @@
 '''
 import sys
 import os
-from sonar import env, projects
+from sonar import env, projects, options
 import sonar.utilities as util
 
 
@@ -37,9 +37,7 @@ def main():
     sq = env.Environment(some_url=args.url, some_token=args.token)
 
     if (sq.edition() in ('community', 'developer') and sq.version(digits=2) < (9, 2)):
-        util.logger.critical("Can't export projects on Community and Developer Edition before 9.2, aborting...")
-        print("Can't export project on Community and Developer Edition before 9.2, aborting...")
-        sys.exit(1)
+        util.exit_fatal("Can't export projects on Community and Developer Edition before 9.2, aborting...", options.ERR_UNSUPPORTED_OPERATION)
 
     project_list = projects.search(endpoint=sq)
     nb_projects = len(project_list)
@@ -51,9 +49,8 @@ def main():
         try:
             dump = p.export(timeout=args.exportTimeout)
         except env.UnsupportedOperation as e:
-            util.logger.critical("%s", e.message)
-            print(f"{e.message}")
-            sys.exit(1)
+            util.exit_fatal(e.message, options.ERR_UNSUPPORTED_OPERATION)
+
         status = dump['status']
         if status in statuses:
             statuses[status] += 1