Maintainability

cli/loc.py

@@ -198,7 +198,21 @@ def __parse_args(desc):
     return args
 
 
-def main():
+def __check_options(edition: str, kwargs: dict[str, str]) -> dict[str, str]:
+    """Verifies a certain number of options for compatibility with edition"""
+    kwargs[options.FORMAT] = util.deduct_format(kwargs[options.FORMAT], kwargs[options.REPORT_FILE])
+    if kwargs[options.WITH_BRANCHES] and edition == "community":
+        util.exit_fatal(f"No branches in {edition} edition, aborting...", errcodes.UNSUPPORTED_OPERATION)
+    if kwargs[options.COMPONENT_TYPE] == "portfolios" and edition in ("community", "developer"):
+        util.exit_fatal(f"No portfolios in {edition} edition, aborting...", errcodes.UNSUPPORTED_OPERATION)
+    if kwargs[options.COMPONENT_TYPE] == "portfolios" and kwargs[options.WITH_BRANCHES]:
+        log.warning("Portfolio LoC export selected, branch option is ignored")
+        kwargs[options.WITH_BRANCHES] = False
+    return kwargs
+
+
+def main() -> None:
+    """sonar-loc entry point"""
     start_time = util.start_clock()
     try:
         kwargs = util.convert_args(
@@ -209,16 +223,7 @@ def main():
     except (options.ArgumentsError, exceptions.ConnectionError) as e:
         util.exit_fatal(e.message, e.errcode)
 
-    kwargs[options.FORMAT] = util.deduct_format(kwargs[options.FORMAT], kwargs[options.REPORT_FILE])
-
-    edition = endpoint.edition()
-    if kwargs[options.WITH_BRANCHES] and edition == "community":
-        util.exit_fatal(f"No branches in {edition} edition, aborting...", errcodes.UNSUPPORTED_OPERATION)
-    if kwargs[options.COMPONENT_TYPE] == "portfolios" and edition in ("community", "developer"):
-        util.exit_fatal(f"No portfolios in {edition} edition, aborting...", errcodes.UNSUPPORTED_OPERATION)
-    if kwargs[options.COMPONENT_TYPE] == "portfolios" and kwargs[options.WITH_BRANCHES]:
-        log.warning("Portfolio LoC export selected, branch option is ignored")
-        kwargs[options.WITH_BRANCHES] = False
+    kwargs = __check_options(endpoint.edition(), kwargs)
 
     try:
         if kwargs[options.COMPONENT_TYPE] == "portfolios":

cli/measures_export.py

@@ -26,6 +26,9 @@
 """
 import sys
 import csv
+
+from typing import Union
+
 from http import HTTPStatus
 from requests.exceptions import HTTPError
 from sonar.util import types
@@ -277,6 +280,23 @@ def __check_options_vs_edition(edition: str, params: dict[str, str]) -> dict[str
     return params
 
 
+def __get_measures(obj: object, wanted_metrics: types.KeyList, hist: bool) -> Union[dict[str, any], None]:
+    """Returns object measures (last measures or history of measures)"""
+    data = obj.component_data()
+    try:
+        if hist:
+            data.update(__get_json_measures_history(obj, wanted_metrics))
+        else:
+            data.update(__get_object_measures(obj, wanted_metrics))
+    except HTTPError as e:
+        if e.response.status_code == HTTPStatus.FORBIDDEN:
+            log.error("Insufficient permission to retrieve measures of %s, export skipped for this object", str(obj))
+        else:
+            log.error("HTTP Error %s while retrieving measures of %s, export skipped for this object", str(e), str(obj))
+        return None
+    return data
+
+
 def main() -> None:
     """Entry point for sonar-measures-export"""
     start_time = util.start_clock()
@@ -299,19 +319,9 @@ def main() -> None:
 
         measure_list = []
         for obj in obj_list:
-            data = obj.component_data()
-            try:
-                if kwargs["history"]:
-                    data.update(__get_json_measures_history(obj, wanted_metrics))
-                else:
-                    data.update(__get_object_measures(obj, wanted_metrics))
-            except HTTPError as e:
-                if e.response.status_code == HTTPStatus.FORBIDDEN:
-                    log.error("Insufficient permission to retrieve measures of %s, export skipped for this object", str(obj))
-                else:
-                    log.error("HTTP Error %s while retrieving measures of %s, export skipped for this object", str(e), str(obj))
-                continue
-            measure_list += [data]
+            data = __get_measures(obj, wanted_metrics, kwargs["history"])
+            if data is not None:
+                measure_list += [data]
 
         if fmt == "json":
             with util.open_file(file) as fd:

sonar/platform.py

@@ -100,7 +100,7 @@ def __credentials(self) -> tuple[str, str]:
 
     def verify_connection(self) -> None:
         try:
-            self.get("projects/search", params={"qualifiers": "TRK", "ps": 1})
+            self.get("server/version")
         except HTTPError as e:
             raise exceptions.ConnectionError(util.sonar_error(e.response))
 

sonar/qualityprofiles.py

@@ -482,7 +482,7 @@ def audit(self, audit_settings: types.ConfigSettings = None) -> list[Problem]:
         problems = []
         age = util.age(self.last_update(), rounded=True)
         if age > audit_settings.get("audit.qualityProfiles.maxLastChangeAge", 180):
-            problems.append(Problem(get_rule(RuleId.QP_LAST_CHANGE_DATE), self, str(self, age)))
+            problems.append(Problem(get_rule(RuleId.QP_LAST_CHANGE_DATE), self, str(self), age))
 
         total_rules = rules.count(endpoint=self.endpoint, languages=self.language)
         if self.nbr_rules < int(total_rules * audit_settings.get("audit.qualityProfiles.minNumberOfRules", 0.5)):

sonar/rules.py

@@ -247,7 +247,7 @@ def export(endpoint: platform.Platform, export_settings: types.ConfigSettings, k
         rule_list["instantiated"] = instantiated_rules
     if len(extended_rules) > 0:
         rule_list["extended"] = extended_rules
-    if len(other_rules) > 0:
+    if len(other_rules) > 0 and full:
         rule_list["standard"] = other_rules
     return rule_list
 

test/it.sh

@@ -20,43 +20,10 @@
 #
 set -euo pipefail
 
+DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+source "$DIR/test-tools.sh"
 DB_BACKUPS_DIR=~/backup
 
-function logmsg {
-    echo "$@" | tee -a "$IT_LOG_FILE"
-}
-
-function run_test {
-    file=$1; shift
-    logmsg "$@"
-    if [ "$SONAR_HOST_URL" == "$SONAR_HOST_URL_SONARCLOUD" ]; then
-        "$@" -o okorach
-    else
-        "$@"
-    fi
-    check "$file"
-}
-
-function run_test_stdout {
-    file=$1; shift
-    logmsg "$@" ">$file"
-    if [ "$SONAR_HOST_URL" == "$SONAR_HOST_URL_SONARCLOUD" ]; then
-        "$@" -o okorach >"$file"
-    else
-        "$@" >"$file"
-    fi
-    check "$file"
-}
-
-check() {
-    if [ -s "$1" ]; then
-        logmsg "Output file $1 is OK"
-    else
-        logmsg "Output file $1 is missing or empty"
-        # exit 1
-    fi
-}
-
 function backup_for {
     case $1 in
         lts|lta|lts-ce|lta-ce|lts-de|lta-de)
@@ -122,140 +89,128 @@ for env in "$@"
 do
 
     logmsg "Install sonar-tools current local version: root = $TMP"
-    cd "$REPO_ROOT"; ./deploy.sh nodoc; cd -
+    cd "$REPO_ROOT"; ./deploy.sh nodoc nodocker 1>$IT_LOG_FILE 2>&1; cd -
 
     if [ "$env" = "sonarcloud" ]; then
         logmsg "Running with environment $env"
         export SONAR_TOKEN=$SONAR_TOKEN_SONARCLOUD
         export SONAR_HOST_URL=$SONAR_HOST_URL_SONARCLOUD        
     else
         id="it$$"
-        logmsg "Running with environment $env - sonarId $id"
+        logmsg "Creating IT test environment $env - sonarId $id"
         sqport=10020
         # echo sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p 6020 -f "$(backup_for "$env")"
-        sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p 6020 -f "$(backup_for "$env")"
+        sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p 6020 -f "$(backup_for "$env")" 1>$IT_LOG_FILE 2>&1
         export SONAR_TOKEN=$SONAR_TOKEN_ADMIN_USER
         export SONAR_HOST_URL="http://localhost:$sqport"
     fi
 
-    logmsg "IT $env sonar-measures-export"
-    f="$TMP/measures-$env-unrel.csv"; run_test "$f" sonar-measures-export -b -f "$f" -m _main --withURL
-    f="$TMP/measures-$env-2.csv";     run_test_stdout "$f" sonar-measures-export -b -m _main --withURL
-    f="$TMP/measures-$env-3.csv";     run_test_stdout "$f" sonar-measures-export -b -p -r -d -m _all
+    f="measures-$env-unrel.csv"; run_test "$f" sonar-measures-export -b -m _main --withURL
+    f="measures-$env-2.csv";     run_test_stdout "$f" sonar-measures-export -b -m _main --withURL
+    f="measures-$env-3.csv";     run_test_stdout "$f" sonar-measures-export -b -p -r -d -m _all
 
-    f="$TMP/measures-$env-1.json";    run_test "$f" sonar-measures-export -b -f "$f" -m _all
-    f="$TMP/measures-$env-2.json";    run_test_stdout "$f" sonar-measures-export -b -p -r -d -m _all --format json
-    f="$TMP/measures-$env-3.csv";     run_test "$f" sonar-measures-export -b -f "$f" --csvSeparator '+' -m _main
+    f="measures-$env-1.json";    run_test "$f" sonar-measures-export -b -m _all
+    f="measures-$env-2.json";    run_test_stdout "$f" sonar-measures-export -b -p -r -d -m _all --format json
+    f="measures-$env-3.csv";     run_test "$f" sonar-measures-export -b --csvSeparator '+' -m _main
 
-    f="$TMP/measures-history-$env-1.csv";     run_test "$f" sonar-measures-export -b -f "$f" --history
-    f="$TMP/measures-history-$env-2.csv";     run_test "$f" sonar-measures-export -b -f "$f" -k okorach_sonar-tools --history --asTable
-    f="$TMP/measures-history-$env-3.json";    run_test "$f" sonar-measures-export -b -f "$f" --history
+    f="measures-history-$env-1.csv";     run_test "$f" sonar-measures-export -b --history
+    f="measures-history-$env-2.csv";     run_test "$f" sonar-measures-export -b -k okorach_sonar-tools --history --asTable
+    f="measures-history-$env-3.json";    run_test "$f" sonar-measures-export -b --history
 
     logmsg "IT $env sonar-findings-export"
 
-    f="$TMP/findings-$env-unrel.csv";  run_test "$f" sonar-findings-export -v DEBUG -f "$f"
-    f="$TMP/findings-$env-1.json";     run_test "$f" sonar-findings-export -f "$f"
-    f="$TMP/findings-$env-2.json";     run_test_stdout "$f" sonar-findings-export -v DEBUG --format json -k okorach_audio-video-tools,okorach_sonar-tools
-    f="$TMP/findings-$env-3.json";     run_test_stdout "$f" sonar-findings-export -v DEBUG --format json -k okorach_audio-video-tools,okorach_sonar-tools --useFindings
-    f="$TMP/findings-$env-4.csv";      run_test_stdout "$f" sonar-findings-export --format csv -k okorach_audio-video-tools,okorach_sonar-tools --csvSeparator '+'
+    f="findings-$env-unrel.csv";  run_test "$f" sonar-findings-export -v DEBUG
+    f="findings-$env-1.json";     run_test "$f" sonar-findings-export
+    f="findings-$env-2.json";     run_test_stdout "$f" sonar-findings-export -v DEBUG --format json -k okorach_audio-video-tools,okorach_sonar-tools
+    f="findings-$env-3.json";     run_test_stdout "$f" sonar-findings-export -v DEBUG --format json -k okorach_audio-video-tools,okorach_sonar-tools --useFindings
+    f="findings-$env-4.csv";      run_test_stdout "$f" sonar-findings-export --format csv -k okorach_audio-video-tools,okorach_sonar-tools --csvSeparator '+'
 
     if [ "$env" = "sonarcloud" ]; then
         logmsg "IT $env sonar-audit SKIPPED"
         logmsg "IT $env sonar-housekeeper SKIPPED"
     else
         logmsg "IT $env sonar-audit"
-        f="$TMP/audit-$env-unrel.csv";     run_test_stdout "$f" sonar-audit
-        f="$TMP/audit-$env-1.json";        run_test "$f" sonar-audit -f "$f"
-        f="$TMP/audit-$env-2.json";        run_test_stdout "$f" sonar-audit --format json --what qualitygates,qualityprofiles,settings
-        f="$TMP/audit-$env-3.csv";         run_test_stdout "$f" sonar-audit  --csvSeparator '+' --format csv
+        f="audit-$env-unrel.csv";     run_test_stdout "$f" sonar-audit
+        f="audit-$env-1.json";        run_test "$f" sonar-audit
+        f="audit-$env-2.json";        run_test_stdout "$f" sonar-audit --format json --what qualitygates,qualityprofiles,settings
+        f="audit-$env-3.csv";         run_test_stdout "$f" sonar-audit  --csvSeparator '+' --format csv
 
         logmsg "IT $env sonar-housekeeper"
-        f="$TMP/housekeeper-$env-1.csv";   run_test_stdout "$f" sonar-housekeeper -P 365 -B 90 -T 180 -R 30
+        f="housekeeper-$env-1.csv";   run_test_stdout "$f" sonar-housekeeper -P 365 -B 90 -T 180 -R 30
     fi
 
     logmsg "IT $env sonar-loc"
-    f="$TMP/loc-$env-1.csv";           run_test_stdout "$f" sonar-loc
-    f="$TMP/loc-$env-unrel.csv";       run_test_stdout "$f" sonar-loc -n -a
-    f="$TMP/loc-$env-2.csv";           run_test "$f" sonar-loc -n -a -f "$f" --csvSeparator ';'
+    f="loc-$env-1.csv";           run_test_stdout "$f" sonar-loc
+    f="loc-$env-unrel.csv";       run_test_stdout "$f" sonar-loc -n -a
+    f="loc-$env-2.csv";           run_test "$f" sonar-loc -n -a --csvSeparator ';'
 
     logmsg "sonar-rules $env"
-    f="$TMP/rules-$env-1.csv";         run_test_stdout "$f" sonar-rules -e
-    f="$TMP/rules-$env-2.csv";         run_test "$f" sonar-rules -e -f "$f"
-    f="$TMP/rules-$env-3.json";        run_test_stdout "$f" sonar-rules -e --format json
-    f="$TMP/rules-$env-4.json";        run_test "$f" sonar-rules -e -f "$f"
+    f="rules-$env-1.csv";         run_test_stdout "$f" sonar-rules -e
+    f="rules-$env-2.csv";         run_test "$f" sonar-rules -e
+    f="rules-$env-3.json";        run_test_stdout "$f" sonar-rules -e --format json
+    f="rules-$env-4.json";        run_test "$f" sonar-rules -e
 
     logmsg "sonar-config $env"
-    f="$TMP/config-$env-1.json";       run_test_stdout "$f" sonar-config -e -w "qualitygates, qualityprofiles, projects" -k okorach_audio-video-tools,okorach_sonar-tools
-    f="$TMP/config-$env-2.json";       run_test_stdout "$f" sonar-config --export
-    f="$TMP/config-$env-unrel.json";   run_test "$f" sonar-config --export -f "$f"
+    f="config-$env-1.json";       run_test_stdout "$f" sonar-config -e -w "qualitygates, qualityprofiles, projects" -k okorach_audio-video-tools,okorach_sonar-tools
+    f="config-$env-2.json";       run_test_stdout "$f" sonar-config --export
+    f="config-$env-unrel.json";   run_test "$f" sonar-config --export
 
     if [ $noExport -eq 1 ]; then
         logmsg "sonar-projects-export $env test skipped"
     elif [ "$env" = "sonarcloud" ]; then
         logmsg "sonar-projects-export $env SKIPPED"
     else
         logmsg "sonar-projects-export $env"
-        sonar-projects-export
+        f="proj-export-$env-2.json"; run_test "$f" sonar-projects-export
     fi
 
     logmsg "sonar-findings-export $env ADMIN export"
-    f1="$TMP/findings-$env-admin.csv";   run_test "$f1" sonar-findings-export -v DEBUG -f "$f1" -k okorach_audio-video-tools,okorach_sonar-tools
+    f1="findings-$env-admin.csv";   run_test "$f1" sonar-findings-export -v DEBUG -k okorach_audio-video-tools,okorach_sonar-tools
 
     if [ "$env" = "sonarcloud" ]; then
         logmsg "sonar-projects-export $env SKIPPED"
     else
         logmsg "sonar-findings-export $env USER export"
         export SONAR_TOKEN=$SONAR_TOKEN_USER_USER
-        f2="$TMP/findings-$env-user.csv";    run_test "$f2" sonar-findings-export -v DEBUG -f "$f2" -k okorach_audio-video-tools,okorach_sonar-tools
+        f2="findings-$env-user.csv";    run_test "$f2" sonar-findings-export -v DEBUG -k okorach_audio-video-tools,okorach_sonar-tools
     fi
 
     # Restore admin token as long as previous version is 2.9 or less
     logmsg "Restore sonar-tools last released version"
-    echo "Y" | pip uninstall sonar-tools
-    pip install sonar-tools
+    pip install --force-reinstall sonar-tools 1>$IT_LOG_FILE 2>&1; 
 
     export SONAR_TOKEN="$SONAR_TOKEN_ADMIN_USER"
     logmsg "IT released tools $env"
-    sonar-measures-export -b -f "$TMP/measures-$env-rel.csv" -m _main --withURL
-    sonar-findings-export -f "$TMP/findings-$env-rel.csv"
-    if [ "$env" != "sonarcloud" ]; then
-        sonar-audit >"$TMP/audit-$env-rel.csv" || echo "OK"
-    fi
-    sonar-loc -n -a >"$TMP/loc-$env-rel.csv"
-    sonar-config -e >"$TMP/config-$env-rel.json"
-
-    echo "IT compare released and unreleased $env" | tee -a "$IT_LOG_FILE"
-    for f in measures findings audit loc
+    f="measures-$env-rel.csv"; run_test "$f" sonar-measures-export -b -m _main --withURL
+    # sonar-measures-export -b -f "$f"
+    f="findings-$env-rel.csv"; run_test "$f" sonar-findings-export
+    # f="audit-$env-rel.csv"; run_test "$f" sonar-audit -f "audit-$env-rel.csv"
+    f="loc-$env-rel.csv"; run_test "$f" sonar-loc -n -a
+    f="config-$env-rel.json"; run_test "$f" sonar-config -e
+
+    logmsg "IT compare released and unreleased $env"
+    for f in measures findings loc
     do
         root="$TMP/$f-$env"
-        logmsg "=========================="
-        logmsg "$f-$env diff"
-        logmsg "=========================="
-        sort "$root-rel.csv" >"$root-rel.sorted.csv"
-        sort "$root-unrel.csv" >"$root-unrel.sorted.csv"
-        diff "$root-rel.sorted.csv" "$root-unrel.sorted.csv" | tee -a "$IT_LOG_FILE" || echo "" 
+        announce_test "$f-$env diff"
+        sort -o "$root-rel.csv" "$root-rel.csv"
+        sort -o "$root-unrel.csv" "$root-unrel.csv"
+        test_passed_if_identical "$root-rel.csv" "$root-unrel.csv"
     done
     for f in config
     do
         root="$TMP/$f-$env"
-        logmsg "=========================="
-        logmsg "$f-$env diff"
-        logmsg "=========================="
-        diff "$root-rel.json" "$root-unrel.json" | tee -a "$IT_LOG_FILE" || echo "" 
+        announce_test "$f-$env diff"
+        test_passed_if_identical "$root-rel.json" "$root-unrel.json"
     done
-    logmsg "=========================="
-    logmsg "findings-$env admin vs user diff"
-    logmsg "=========================="
-    f1="$TMP/findings-$env-admin.csv"
-    f2="$TMP/findings-$env-user.csv"
-    diff "$f1" "$f2" | tee -a "$IT_LOG_FILE" || echo ""
-
+    announce_test "findings-$env admin vs user diff"
+    test_passed_if_identical "$TMP/findings-$env-admin.csv" "$TMP/findings-$env-user.csv"
     if [ "$env" != "sonarcloud" ]; then
         logmsg "Deleting environment sonarId $id"
         sonar delete -i "$id"
     fi
 done
 
 logmsg "====================================="
-logmsg "          IT tests success"
+logmsg "          IT tests end"
 logmsg "====================================="