diff --git a/cli/housekeeper.py b/cli/housekeeper.py index d9566097..ef6bc1c3 100644 --- a/cli/housekeeper.py +++ b/cli/housekeeper.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # sonar-tools # Copyright (C) 2019-2025 Olivier Korach @@ -124,6 +123,12 @@ def _parse_arguments() -> object: default=_DEFAULT_BRANCH_OBSOLESCENCE, help=f"Deletes branches not to be kept and not analyzed since a given number of days, by default {_DEFAULT_BRANCH_OBSOLESCENCE} days", ) + parser.add_argument( + "--keepWhenInactive", + required=False, + type=str, + help="Regexp of branches to keep when inactive, overrides the SonarQube default sonar.dbcleaner.branchesToKeepWhenInactive value", + ) parser.add_argument( "-R", "--pullrequestsMaxAge", @@ -195,12 +200,13 @@ def main() -> None: sq.verify_connection() sq.set_user_agent(f"{TOOL_NAME} {version.PACKAGE_VERSION}") - mode, proj_age, branch_age, pr_age, token_age = ( + mode, proj_age, branch_age, pr_age, token_age, keep_regexp = ( kwargs["mode"], kwargs["projectsMaxAge"], kwargs["branchesMaxAge"], kwargs["pullrequestsMaxAge"], kwargs["tokensMaxAge"], + kwargs.get("keepWhenInactive", None), ) settings = { "audit.tokens.maxAge": token_age, @@ -209,6 +215,7 @@ def main() -> None: PROJ_MAX_AGE: proj_age, "audit.projects.branches.maxLastAnalysisAge": branch_age, "audit.projects.pullRequests.maxLastAnalysisAge": pr_age, + "audit.projects.branches.keepWhenInactive": keep_regexp, c.AUDIT_MODE_PARAM: "housekeeper", options.NBR_THREADS: kwargs[options.NBR_THREADS], } diff --git a/doc/sonar-housekeeper.md b/doc/sonar-housekeeper.md index 4d82bf0d..29a7beb4 100644 --- a/doc/sonar-housekeeper.md +++ b/doc/sonar-housekeeper.md @@ -1,12 +1,13 @@ -# +# sonar-housekeeper Deletes obsolete/outdated data from SonarQube: - Projects whose last analysis date (on any branch) is older than a given number of days. - User tokens older than a given number of days - Inactive branches (Branches not analyzed for a given number of days), excepted branches marked as "keep when inactive" + There is a possibility to override the "keep when inactive" regexp if the original SonarQube one was wrong and caused many branches to be kepf - Inactive pull requests (PRs not analyzed for a given number of days) -Usage: `sonar-housekeeper [-P ] [-B ] [-R ] [-T ] [--mode delete]` +Usage: `sonar-housekeeper [-P ] [-B ] [-R ] [-T ] [--keepWhenInactive `: Will search for projects not analyzed since more than `` days. To avoid deleting too recent projects it is denied to specify less than 90 days @@ -14,13 +15,30 @@ To avoid deleting too recent projects it is denied to specify less than 90 days Branches marked as "keep when inactive" are excluded from housekeeping - `-R `: Will search for pull requests not analyzed since more than `` days - `-T `: Will search for tokens created since more than `` days +- `--keepWhenInactive `: Overrides the SonarQube `sonar.dbcleaner.branchesToKeepWhenInactive` to with another regexp to consider branches to delete - `--mode delete`: If not specified, `sonar-housekeeper` will only perform a dry run and list projects branches, pull requests and tokens that would be deleted. If `--mode delete` is specified objects are actually deleted - `-h`, `-u`, `-t`, `-o`, `-v`, `-l`, `--httpTimeout`, `--threads`, `--clientCert`: See **sonar-tools** [common parameters](https://github.com/okorach/sonar-tools/blob/master/README.md) +## Examples -:warning: **sonar-tools** 2.7 or higher is required for `sonar-housekeeper` compatibility with SonarQube Server 10 +``` +export SONAR_HOST_URL=https://sonar.acme-corp.com +export SONAR_TOKEN=squ_XXXXXXYYYYYZZZAAAABBBBCCCDDDEEEFFFGGGGGG + +# Deletes all branches that have not been analyzed for 90 days and where the branch name does not match '(main|master|release|develop).*' +sonar-housekeeper -B 90 --keepWhenInactive '(main|master|release|develop).*' --mode delete + +# Lists projects that have not been analyzed on any branch or PR in the last 365 days, on SonarQube Cloud +sonar-housekeeper -P 365 -u https://sonarcloud.io -t -o + +# Deletes projects that have not been analyzed on any branch or PR in the last 365 days +sonar-housekeeper -P 365 --mode delete + +# Deletes tokens created more than 365 days ago +sonar-housekeeper -T 365 --mode delete +``` ## Required Permissions @@ -41,8 +59,3 @@ To avoid bad mistakes (mistakenly deleting too many projects), the tools will re ### :warning: Database backup **A database backup should always be taken before executing this script. There is no recovery.** - -### Example -``` -sonar-housekeeper -u https://sonar.acme-corp.com -t 15ee09df11fb9b8234b7a1f1ac5fce2e4e93d75d -``` diff --git a/sonar/branches.py b/sonar/branches.py index 115e9645..c8a30c7a 100644 --- a/sonar/branches.py +++ b/sonar/branches.py @@ -361,15 +361,20 @@ def __audit_never_analyzed(self) -> list[Problem]: def __audit_last_analysis(self, audit_settings: types.ConfigSettings) -> list[Problem]: if self.is_main(): - log.debug("%s is main (not purgeable)", str(self)) + log.info("%s is main (not purgeable)", str(self)) return [] if (age := util.age(self.last_analysis())) is None: log.debug("%s last analysis audit is disabled, skipped...", str(self)) return [] max_age = audit_settings.get("audit.projects.branches.maxLastAnalysisAge", 30) + preserved = audit_settings.get("audit.projects.branches.keepWhenInactive", None) problems = [] - if self.is_kept_when_inactive(): - log.debug("%s is kept when inactive (not purgeable)", str(self)) + log.info("EVALUATING %s age %d greater than %d days and not matches '%s'", str(self), age, max_age, preserved) + if preserved is not None and age > max_age and not re.match(rf"^{preserved}$", self.name): + log.info("%s age %d greater than %d days and not matches '%s'", str(self), age, max_age, preserved) + problems.append(Problem(get_rule(RuleId.BRANCH_LAST_ANALYSIS), self, str(self), age)) + elif self.is_kept_when_inactive(): + log.info("%s is kept when inactive (not purgeable)", str(self)) elif age > max_age: problems.append(Problem(get_rule(RuleId.BRANCH_LAST_ANALYSIS), self, str(self), age)) else: diff --git a/test/unit/test_housekeeper.py b/test/unit/test_housekeeper.py index cbebd245..288d7f9c 100644 --- a/test/unit/test_housekeeper.py +++ b/test/unit/test_housekeeper.py @@ -24,6 +24,9 @@ sonar-housekeeper tests """ +from collections.abc import Generator +import pytest + import utilities as tutil from sonar import errcodes from cli import housekeeper, options @@ -36,3 +39,14 @@ def test_housekeeper() -> None: """test_housekeeper""" for opts in __GOOD_OPTS: assert tutil.run_cmd(housekeeper.main, f"{CMD} {tutil.SQS_OPTS} {opts}") == errcodes.OK + +def test_keep_branches_override(csv_file: Generator[str]) -> None: + """test_keep_branches_override""" + if tutil.SQ.version() == "community": + pytest.skip("No branches in Community") + assert tutil.run_cmd(housekeeper.main, f"{CMD} {tutil.SQS_OPTS} -P 730 -T 730 -R 730 -B 90 -f {csv_file}") == errcodes.OK + nbr_br = tutil.csv_col_count_values(csv_file, 1, "BRANCH_LAST_ANALYSIS") + assert tutil.run_cmd(housekeeper.main, f"{CMD} {tutil.SQS_OPTS} -P 730 -T 730 -R 730 -B 90 -f {csv_file} --keepWhenInactive 'dontkeepanything'") == errcodes.OK + # With 'dontkeepanything' as branch regexp, more branches to delete should be found + assert tutil.csv_col_count_values(csv_file, 1, "BRANCH_LAST_ANALYSIS") > nbr_br + diff --git a/test/unit/utilities.py b/test/unit/utilities.py index 98404233..dffa10fd 100644 --- a/test/unit/utilities.py +++ b/test/unit/utilities.py @@ -296,10 +296,15 @@ def csv_col_has_values(csv_file: str, col_name: str, *values) -> bool: return False -def csv_col_count_values(csv_file: str, col_name: str, *values) -> int: +def csv_col_count_values(csv_file: str, col_name_or_nbr: Union[str, int], *values) -> int: + """Counts the number of times a given column has one of the given values""" values_to_search = list(values).copy() with open(csv_file, encoding="utf-8") as fd: - (col,) = get_cols(next(reader := csv.reader(fd)), col_name) + reader = csv.reader(fd) + if isinstance(col_name_or_nbr, int): + col = col_name_or_nbr - 1 + else: + (col,) = get_cols(next(reader, col_name_or_nbr)) counter = sum(1 if line[col] in values_to_search else 0 for line in reader) return counter