diff --git a/cli/loc.py b/cli/loc.py index eda1f5fba..71ee67724 100644 --- a/cli/loc.py +++ b/cli/loc.py @@ -198,7 +198,21 @@ def __parse_args(desc): return args -def main(): +def __check_options(edition: str, kwargs: dict[str, str]) -> dict[str, str]: + """Verifies a certain number of options for compatibility with edition""" + kwargs[options.FORMAT] = util.deduct_format(kwargs[options.FORMAT], kwargs[options.REPORT_FILE]) + if kwargs[options.WITH_BRANCHES] and edition == "community": + util.exit_fatal(f"No branches in {edition} edition, aborting...", errcodes.UNSUPPORTED_OPERATION) + if kwargs[options.COMPONENT_TYPE] == "portfolios" and edition in ("community", "developer"): + util.exit_fatal(f"No portfolios in {edition} edition, aborting...", errcodes.UNSUPPORTED_OPERATION) + if kwargs[options.COMPONENT_TYPE] == "portfolios" and kwargs[options.WITH_BRANCHES]: + log.warning("Portfolio LoC export selected, branch option is ignored") + kwargs[options.WITH_BRANCHES] = False + return kwargs + + +def main() -> None: + """sonar-loc entry point""" start_time = util.start_clock() try: kwargs = util.convert_args( @@ -209,16 +223,7 @@ def main(): except (options.ArgumentsError, exceptions.ConnectionError) as e: util.exit_fatal(e.message, e.errcode) - kwargs[options.FORMAT] = util.deduct_format(kwargs[options.FORMAT], kwargs[options.REPORT_FILE]) - - edition = endpoint.edition() - if kwargs[options.WITH_BRANCHES] and edition == "community": - util.exit_fatal(f"No branches in {edition} edition, aborting...", errcodes.UNSUPPORTED_OPERATION) - if kwargs[options.COMPONENT_TYPE] == "portfolios" and edition in ("community", "developer"): - util.exit_fatal(f"No portfolios in {edition} edition, aborting...", errcodes.UNSUPPORTED_OPERATION) - if kwargs[options.COMPONENT_TYPE] == "portfolios" and kwargs[options.WITH_BRANCHES]: - log.warning("Portfolio LoC export selected, branch option is ignored") - kwargs[options.WITH_BRANCHES] = False + kwargs = __check_options(endpoint.edition(), kwargs) try: if kwargs[options.COMPONENT_TYPE] == "portfolios": diff --git a/cli/measures_export.py b/cli/measures_export.py index e39bc609c..99321b5b8 100755 --- a/cli/measures_export.py +++ b/cli/measures_export.py @@ -26,6 +26,9 @@ """ import sys import csv + +from typing import Union + from http import HTTPStatus from requests.exceptions import HTTPError from sonar.util import types @@ -277,6 +280,23 @@ def __check_options_vs_edition(edition: str, params: dict[str, str]) -> dict[str return params +def __get_measures(obj: object, wanted_metrics: types.KeyList, hist: bool) -> Union[dict[str, any], None]: + """Returns object measures (last measures or history of measures)""" + data = obj.component_data() + try: + if hist: + data.update(__get_json_measures_history(obj, wanted_metrics)) + else: + data.update(__get_object_measures(obj, wanted_metrics)) + except HTTPError as e: + if e.response.status_code == HTTPStatus.FORBIDDEN: + log.error("Insufficient permission to retrieve measures of %s, export skipped for this object", str(obj)) + else: + log.error("HTTP Error %s while retrieving measures of %s, export skipped for this object", str(e), str(obj)) + return None + return data + + def main() -> None: """Entry point for sonar-measures-export""" start_time = util.start_clock() @@ -299,19 +319,9 @@ def main() -> None: measure_list = [] for obj in obj_list: - data = obj.component_data() - try: - if kwargs["history"]: - data.update(__get_json_measures_history(obj, wanted_metrics)) - else: - data.update(__get_object_measures(obj, wanted_metrics)) - except HTTPError as e: - if e.response.status_code == HTTPStatus.FORBIDDEN: - log.error("Insufficient permission to retrieve measures of %s, export skipped for this object", str(obj)) - else: - log.error("HTTP Error %s while retrieving measures of %s, export skipped for this object", str(e), str(obj)) - continue - measure_list += [data] + data = __get_measures(obj, wanted_metrics, kwargs["history"]) + if data is not None: + measure_list += [data] if fmt == "json": with util.open_file(file) as fd: diff --git a/sonar/platform.py b/sonar/platform.py index 34fba5f9b..e25279a54 100644 --- a/sonar/platform.py +++ b/sonar/platform.py @@ -100,7 +100,7 @@ def __credentials(self) -> tuple[str, str]: def verify_connection(self) -> None: try: - self.get("projects/search", params={"qualifiers": "TRK", "ps": 1}) + self.get("server/version") except HTTPError as e: raise exceptions.ConnectionError(util.sonar_error(e.response)) diff --git a/sonar/qualityprofiles.py b/sonar/qualityprofiles.py index e6085fc01..d00beeec0 100644 --- a/sonar/qualityprofiles.py +++ b/sonar/qualityprofiles.py @@ -482,7 +482,7 @@ def audit(self, audit_settings: types.ConfigSettings = None) -> list[Problem]: problems = [] age = util.age(self.last_update(), rounded=True) if age > audit_settings.get("audit.qualityProfiles.maxLastChangeAge", 180): - problems.append(Problem(get_rule(RuleId.QP_LAST_CHANGE_DATE), self, str(self, age))) + problems.append(Problem(get_rule(RuleId.QP_LAST_CHANGE_DATE), self, str(self), age)) total_rules = rules.count(endpoint=self.endpoint, languages=self.language) if self.nbr_rules < int(total_rules * audit_settings.get("audit.qualityProfiles.minNumberOfRules", 0.5)): diff --git a/sonar/rules.py b/sonar/rules.py index 0670732a9..c11139fee 100644 --- a/sonar/rules.py +++ b/sonar/rules.py @@ -247,7 +247,7 @@ def export(endpoint: platform.Platform, export_settings: types.ConfigSettings, k rule_list["instantiated"] = instantiated_rules if len(extended_rules) > 0: rule_list["extended"] = extended_rules - if len(other_rules) > 0: + if len(other_rules) > 0 and full: rule_list["standard"] = other_rules return rule_list diff --git a/test/it.sh b/test/it.sh index 1305ce5b1..1cf03fb50 100755 --- a/test/it.sh +++ b/test/it.sh @@ -20,43 +20,10 @@ # set -euo pipefail +DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source "$DIR/test-tools.sh" DB_BACKUPS_DIR=~/backup -function logmsg { - echo "$@" | tee -a "$IT_LOG_FILE" -} - -function run_test { - file=$1; shift - logmsg "$@" - if [ "$SONAR_HOST_URL" == "$SONAR_HOST_URL_SONARCLOUD" ]; then - "$@" -o okorach - else - "$@" - fi - check "$file" -} - -function run_test_stdout { - file=$1; shift - logmsg "$@" ">$file" - if [ "$SONAR_HOST_URL" == "$SONAR_HOST_URL_SONARCLOUD" ]; then - "$@" -o okorach >"$file" - else - "$@" >"$file" - fi - check "$file" -} - -check() { - if [ -s "$1" ]; then - logmsg "Output file $1 is OK" - else - logmsg "Output file $1 is missing or empty" - # exit 1 - fi -} - function backup_for { case $1 in lts|lta|lts-ce|lta-ce|lts-de|lta-de) @@ -122,7 +89,7 @@ for env in "$@" do logmsg "Install sonar-tools current local version: root = $TMP" - cd "$REPO_ROOT"; ./deploy.sh nodoc; cd - + cd "$REPO_ROOT"; ./deploy.sh nodoc nodocker 1>$IT_LOG_FILE 2>&1; cd - if [ "$env" = "sonarcloud" ]; then logmsg "Running with environment $env" @@ -130,64 +97,63 @@ do export SONAR_HOST_URL=$SONAR_HOST_URL_SONARCLOUD else id="it$$" - logmsg "Running with environment $env - sonarId $id" + logmsg "Creating IT test environment $env - sonarId $id" sqport=10020 # echo sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p 6020 -f "$(backup_for "$env")" - sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p 6020 -f "$(backup_for "$env")" + sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p 6020 -f "$(backup_for "$env")" 1>$IT_LOG_FILE 2>&1 export SONAR_TOKEN=$SONAR_TOKEN_ADMIN_USER export SONAR_HOST_URL="http://localhost:$sqport" fi - logmsg "IT $env sonar-measures-export" - f="$TMP/measures-$env-unrel.csv"; run_test "$f" sonar-measures-export -b -f "$f" -m _main --withURL - f="$TMP/measures-$env-2.csv"; run_test_stdout "$f" sonar-measures-export -b -m _main --withURL - f="$TMP/measures-$env-3.csv"; run_test_stdout "$f" sonar-measures-export -b -p -r -d -m _all + f="measures-$env-unrel.csv"; run_test "$f" sonar-measures-export -b -m _main --withURL + f="measures-$env-2.csv"; run_test_stdout "$f" sonar-measures-export -b -m _main --withURL + f="measures-$env-3.csv"; run_test_stdout "$f" sonar-measures-export -b -p -r -d -m _all - f="$TMP/measures-$env-1.json"; run_test "$f" sonar-measures-export -b -f "$f" -m _all - f="$TMP/measures-$env-2.json"; run_test_stdout "$f" sonar-measures-export -b -p -r -d -m _all --format json - f="$TMP/measures-$env-3.csv"; run_test "$f" sonar-measures-export -b -f "$f" --csvSeparator '+' -m _main + f="measures-$env-1.json"; run_test "$f" sonar-measures-export -b -m _all + f="measures-$env-2.json"; run_test_stdout "$f" sonar-measures-export -b -p -r -d -m _all --format json + f="measures-$env-3.csv"; run_test "$f" sonar-measures-export -b --csvSeparator '+' -m _main - f="$TMP/measures-history-$env-1.csv"; run_test "$f" sonar-measures-export -b -f "$f" --history - f="$TMP/measures-history-$env-2.csv"; run_test "$f" sonar-measures-export -b -f "$f" -k okorach_sonar-tools --history --asTable - f="$TMP/measures-history-$env-3.json"; run_test "$f" sonar-measures-export -b -f "$f" --history + f="measures-history-$env-1.csv"; run_test "$f" sonar-measures-export -b --history + f="measures-history-$env-2.csv"; run_test "$f" sonar-measures-export -b -k okorach_sonar-tools --history --asTable + f="measures-history-$env-3.json"; run_test "$f" sonar-measures-export -b --history logmsg "IT $env sonar-findings-export" - f="$TMP/findings-$env-unrel.csv"; run_test "$f" sonar-findings-export -v DEBUG -f "$f" - f="$TMP/findings-$env-1.json"; run_test "$f" sonar-findings-export -f "$f" - f="$TMP/findings-$env-2.json"; run_test_stdout "$f" sonar-findings-export -v DEBUG --format json -k okorach_audio-video-tools,okorach_sonar-tools - f="$TMP/findings-$env-3.json"; run_test_stdout "$f" sonar-findings-export -v DEBUG --format json -k okorach_audio-video-tools,okorach_sonar-tools --useFindings - f="$TMP/findings-$env-4.csv"; run_test_stdout "$f" sonar-findings-export --format csv -k okorach_audio-video-tools,okorach_sonar-tools --csvSeparator '+' + f="findings-$env-unrel.csv"; run_test "$f" sonar-findings-export -v DEBUG + f="findings-$env-1.json"; run_test "$f" sonar-findings-export + f="findings-$env-2.json"; run_test_stdout "$f" sonar-findings-export -v DEBUG --format json -k okorach_audio-video-tools,okorach_sonar-tools + f="findings-$env-3.json"; run_test_stdout "$f" sonar-findings-export -v DEBUG --format json -k okorach_audio-video-tools,okorach_sonar-tools --useFindings + f="findings-$env-4.csv"; run_test_stdout "$f" sonar-findings-export --format csv -k okorach_audio-video-tools,okorach_sonar-tools --csvSeparator '+' if [ "$env" = "sonarcloud" ]; then logmsg "IT $env sonar-audit SKIPPED" logmsg "IT $env sonar-housekeeper SKIPPED" else logmsg "IT $env sonar-audit" - f="$TMP/audit-$env-unrel.csv"; run_test_stdout "$f" sonar-audit - f="$TMP/audit-$env-1.json"; run_test "$f" sonar-audit -f "$f" - f="$TMP/audit-$env-2.json"; run_test_stdout "$f" sonar-audit --format json --what qualitygates,qualityprofiles,settings - f="$TMP/audit-$env-3.csv"; run_test_stdout "$f" sonar-audit --csvSeparator '+' --format csv + f="audit-$env-unrel.csv"; run_test_stdout "$f" sonar-audit + f="audit-$env-1.json"; run_test "$f" sonar-audit + f="audit-$env-2.json"; run_test_stdout "$f" sonar-audit --format json --what qualitygates,qualityprofiles,settings + f="audit-$env-3.csv"; run_test_stdout "$f" sonar-audit --csvSeparator '+' --format csv logmsg "IT $env sonar-housekeeper" - f="$TMP/housekeeper-$env-1.csv"; run_test_stdout "$f" sonar-housekeeper -P 365 -B 90 -T 180 -R 30 + f="housekeeper-$env-1.csv"; run_test_stdout "$f" sonar-housekeeper -P 365 -B 90 -T 180 -R 30 fi logmsg "IT $env sonar-loc" - f="$TMP/loc-$env-1.csv"; run_test_stdout "$f" sonar-loc - f="$TMP/loc-$env-unrel.csv"; run_test_stdout "$f" sonar-loc -n -a - f="$TMP/loc-$env-2.csv"; run_test "$f" sonar-loc -n -a -f "$f" --csvSeparator ';' + f="loc-$env-1.csv"; run_test_stdout "$f" sonar-loc + f="loc-$env-unrel.csv"; run_test_stdout "$f" sonar-loc -n -a + f="loc-$env-2.csv"; run_test "$f" sonar-loc -n -a --csvSeparator ';' logmsg "sonar-rules $env" - f="$TMP/rules-$env-1.csv"; run_test_stdout "$f" sonar-rules -e - f="$TMP/rules-$env-2.csv"; run_test "$f" sonar-rules -e -f "$f" - f="$TMP/rules-$env-3.json"; run_test_stdout "$f" sonar-rules -e --format json - f="$TMP/rules-$env-4.json"; run_test "$f" sonar-rules -e -f "$f" + f="rules-$env-1.csv"; run_test_stdout "$f" sonar-rules -e + f="rules-$env-2.csv"; run_test "$f" sonar-rules -e + f="rules-$env-3.json"; run_test_stdout "$f" sonar-rules -e --format json + f="rules-$env-4.json"; run_test "$f" sonar-rules -e logmsg "sonar-config $env" - f="$TMP/config-$env-1.json"; run_test_stdout "$f" sonar-config -e -w "qualitygates, qualityprofiles, projects" -k okorach_audio-video-tools,okorach_sonar-tools - f="$TMP/config-$env-2.json"; run_test_stdout "$f" sonar-config --export - f="$TMP/config-$env-unrel.json"; run_test "$f" sonar-config --export -f "$f" + f="config-$env-1.json"; run_test_stdout "$f" sonar-config -e -w "qualitygates, qualityprofiles, projects" -k okorach_audio-video-tools,okorach_sonar-tools + f="config-$env-2.json"; run_test_stdout "$f" sonar-config --export + f="config-$env-unrel.json"; run_test "$f" sonar-config --export if [ $noExport -eq 1 ]; then logmsg "sonar-projects-export $env test skipped" @@ -195,61 +161,50 @@ do logmsg "sonar-projects-export $env SKIPPED" else logmsg "sonar-projects-export $env" - sonar-projects-export + f="proj-export-$env-2.json"; run_test "$f" sonar-projects-export fi logmsg "sonar-findings-export $env ADMIN export" - f1="$TMP/findings-$env-admin.csv"; run_test "$f1" sonar-findings-export -v DEBUG -f "$f1" -k okorach_audio-video-tools,okorach_sonar-tools + f1="findings-$env-admin.csv"; run_test "$f1" sonar-findings-export -v DEBUG -k okorach_audio-video-tools,okorach_sonar-tools if [ "$env" = "sonarcloud" ]; then logmsg "sonar-projects-export $env SKIPPED" else logmsg "sonar-findings-export $env USER export" export SONAR_TOKEN=$SONAR_TOKEN_USER_USER - f2="$TMP/findings-$env-user.csv"; run_test "$f2" sonar-findings-export -v DEBUG -f "$f2" -k okorach_audio-video-tools,okorach_sonar-tools + f2="findings-$env-user.csv"; run_test "$f2" sonar-findings-export -v DEBUG -k okorach_audio-video-tools,okorach_sonar-tools fi # Restore admin token as long as previous version is 2.9 or less logmsg "Restore sonar-tools last released version" - echo "Y" | pip uninstall sonar-tools - pip install sonar-tools + pip install --force-reinstall sonar-tools 1>$IT_LOG_FILE 2>&1; export SONAR_TOKEN="$SONAR_TOKEN_ADMIN_USER" logmsg "IT released tools $env" - sonar-measures-export -b -f "$TMP/measures-$env-rel.csv" -m _main --withURL - sonar-findings-export -f "$TMP/findings-$env-rel.csv" - if [ "$env" != "sonarcloud" ]; then - sonar-audit >"$TMP/audit-$env-rel.csv" || echo "OK" - fi - sonar-loc -n -a >"$TMP/loc-$env-rel.csv" - sonar-config -e >"$TMP/config-$env-rel.json" - - echo "IT compare released and unreleased $env" | tee -a "$IT_LOG_FILE" - for f in measures findings audit loc + f="measures-$env-rel.csv"; run_test "$f" sonar-measures-export -b -m _main --withURL + # sonar-measures-export -b -f "$f" + f="findings-$env-rel.csv"; run_test "$f" sonar-findings-export + # f="audit-$env-rel.csv"; run_test "$f" sonar-audit -f "audit-$env-rel.csv" + f="loc-$env-rel.csv"; run_test "$f" sonar-loc -n -a + f="config-$env-rel.json"; run_test "$f" sonar-config -e + + logmsg "IT compare released and unreleased $env" + for f in measures findings loc do root="$TMP/$f-$env" - logmsg "==========================" - logmsg "$f-$env diff" - logmsg "==========================" - sort "$root-rel.csv" >"$root-rel.sorted.csv" - sort "$root-unrel.csv" >"$root-unrel.sorted.csv" - diff "$root-rel.sorted.csv" "$root-unrel.sorted.csv" | tee -a "$IT_LOG_FILE" || echo "" + announce_test "$f-$env diff" + sort -o "$root-rel.csv" "$root-rel.csv" + sort -o "$root-unrel.csv" "$root-unrel.csv" + test_passed_if_identical "$root-rel.csv" "$root-unrel.csv" done for f in config do root="$TMP/$f-$env" - logmsg "==========================" - logmsg "$f-$env diff" - logmsg "==========================" - diff "$root-rel.json" "$root-unrel.json" | tee -a "$IT_LOG_FILE" || echo "" + announce_test "$f-$env diff" + test_passed_if_identical "$root-rel.json" "$root-unrel.json" done - logmsg "==========================" - logmsg "findings-$env admin vs user diff" - logmsg "==========================" - f1="$TMP/findings-$env-admin.csv" - f2="$TMP/findings-$env-user.csv" - diff "$f1" "$f2" | tee -a "$IT_LOG_FILE" || echo "" - + announce_test "findings-$env admin vs user diff" + test_passed_if_identical "$TMP/findings-$env-admin.csv" "$TMP/findings-$env-user.csv" if [ "$env" != "sonarcloud" ]; then logmsg "Deleting environment sonarId $id" sonar delete -i "$id" @@ -257,5 +212,5 @@ do done logmsg "=====================================" -logmsg " IT tests success" +logmsg " IT tests end" logmsg "=====================================" diff --git a/test/test-tools.sh b/test/test-tools.sh index 76b2ead99..7a40fde4d 100644 --- a/test/test-tools.sh +++ b/test/test-tools.sh @@ -36,24 +36,35 @@ function logmsg { function run_test { file=$1; shift - logmsg "$@" + announce_test "$@ -f $file" + file="$REPO_ROOT/tmp/$file" + # logmsg "=========================================" + # logmsg "$@" + # logmsg "=========================================" if [ "$SONAR_HOST_URL" == "$SONAR_HOST_URL_SONARCLOUD" ]; then - "$@" -o okorach + "$@" -o okorach -l $IT_LOG_FILE -f "$file" >/dev/null else - "$@" + # echo "$@" -o okorach -l $IT_LOG_FILE -f "$file" + "$@" -l $IT_LOG_FILE -f "$file" 2>/dev/null fi - check "$file" + test_passed_if_file_not_empty "$file" } function run_test_stdout { file=$1; shift - logmsg "$@" ">$file" + announce_test "$@ >$file" + file="$REPO_ROOT/tmp/$file" + + # logmsg "=========================================" + # logmsg "$@ >$file" + # logmsg "=========================================" if [ "$SONAR_HOST_URL" == "$SONAR_HOST_URL_SONARCLOUD" ]; then - "$@" -o okorach >"$file" + "$@" -o okorach -l $IT_LOG_FILE >"$file" 2>/dev/null else - "$@" >"$file" + # echo "$@" -o okorach -l $IT_LOG_FILE "> $file" + "$@" -l $IT_LOG_FILE >"$file" 2>/dev/null fi - check "$file" + test_passed_if_file_not_empty "$file" } check_file_not_empty() { @@ -66,7 +77,14 @@ check_file_not_empty() { } test_passed_if_identical() { - diff $* + diff $* >> $IT_LOG_FILE + code=$? + test_result $code + return $code +} + +test_passed_if_file_not_empty() { + [ -s "$1" ] code=$? test_result $code return $code