Allow-housekeeper-to-override-keep-when-inactive-pattern (#2013)

* Fixes #2005

* Add doc for #2005

* Quality pass

* Allow to select col to search by col number on top of col name

* Add tests for #2005

Author: okorach

cli/housekeeper.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 # sonar-tools
 # Copyright (C) 2019-2025 Olivier Korach
@@ -124,6 +123,12 @@ def _parse_arguments() -> object:
         default=_DEFAULT_BRANCH_OBSOLESCENCE,
         help=f"Deletes branches not to be kept and not analyzed since a given number of days, by default {_DEFAULT_BRANCH_OBSOLESCENCE} days",
     )
+    parser.add_argument(
+        "--keepWhenInactive",
+        required=False,
+        type=str,
+        help="Regexp of branches to keep when inactive, overrides the SonarQube default sonar.dbcleaner.branchesToKeepWhenInactive value",
+    )
     parser.add_argument(
         "-R",
         "--pullrequestsMaxAge",
@@ -195,12 +200,13 @@ def main() -> None:
         sq.verify_connection()
         sq.set_user_agent(f"{TOOL_NAME} {version.PACKAGE_VERSION}")
 
-        mode, proj_age, branch_age, pr_age, token_age = (
+        mode, proj_age, branch_age, pr_age, token_age, keep_regexp = (
             kwargs["mode"],
             kwargs["projectsMaxAge"],
             kwargs["branchesMaxAge"],
             kwargs["pullrequestsMaxAge"],
             kwargs["tokensMaxAge"],
+            kwargs.get("keepWhenInactive", None),
         )
         settings = {
             "audit.tokens.maxAge": token_age,
@@ -209,6 +215,7 @@ def main() -> None:
             PROJ_MAX_AGE: proj_age,
             "audit.projects.branches.maxLastAnalysisAge": branch_age,
             "audit.projects.pullRequests.maxLastAnalysisAge": pr_age,
+            "audit.projects.branches.keepWhenInactive": keep_regexp,
             c.AUDIT_MODE_PARAM: "housekeeper",
             options.NBR_THREADS: kwargs[options.NBR_THREADS],
         }

doc/sonar-housekeeper.md

@@ -1,26 +1,44 @@
-# <a name="sonar-housekeeper">
+# sonar-housekeeper
 
 Deletes obsolete/outdated data from SonarQube:
 - Projects whose last analysis date (on any branch) is older than a given number of days.
 - User tokens older than a given number of days
 - Inactive branches (Branches not analyzed for a given number of days), excepted branches marked as "keep when inactive"
+  There is a possibility to override the "keep when inactive" regexp if the original SonarQube one was wrong and caused many branches to be kepf
 - Inactive pull requests (PRs not analyzed for a given number of days)
 
-Usage: `sonar-housekeeper [-P <days>] [-B <days>] [-R <days>] [-T <days>] [--mode delete]`
+Usage: `sonar-housekeeper [-P <days>] [-B <days>] [-R <days>] [-T <days>] [--keepWhenInactive <regexp] [--mode delete]`
 
 - `-P <days>`: Will search for projects not analyzed since more than `<days>` days.
 To avoid deleting too recent projects it is denied to specify less than 90 days
 - `-B <days>`: Will search for projects branches not analyzed since more than `<days>` days.
 Branches marked as "keep when inactive" are excluded from housekeeping
 - `-R <days>`: Will search for pull requests not analyzed since more than `<days>` days
 - `-T <days>`: Will search for tokens created since more than `<days>` days
+- `--keepWhenInactive <regexp>`: Overrides the SonarQube `sonar.dbcleaner.branchesToKeepWhenInactive` to with another regexp to consider branches to delete
 - `--mode delete`: If not specified, `sonar-housekeeper` will only perform a dry run and list projects
 branches, pull requests and tokens that would be deleted.
 If `--mode delete` is specified objects are actually deleted
 - `-h`, `-u`, `-t`, `-o`, `-v`, `-l`, `--httpTimeout`, `--threads`, `--clientCert`: See **sonar-tools** [common parameters](https://github.com/okorach/sonar-tools/blob/master/README.md)
 
+## Examples
 
-:warning: **sonar-tools** 2.7 or higher is required for `sonar-housekeeper` compatibility with SonarQube Server 10
+```
+export SONAR_HOST_URL=https://sonar.acme-corp.com
+export SONAR_TOKEN=squ_XXXXXXYYYYYZZZAAAABBBBCCCDDDEEEFFFGGGGGG
+
+# Deletes all branches that have not been analyzed for 90 days and where the branch name does not match '(main|master|release|develop).*'
+sonar-housekeeper -B 90 --keepWhenInactive '(main|master|release|develop).*' --mode delete
+
+# Lists projects that have not been analyzed on any branch or PR in the last 365 days, on SonarQube Cloud
+sonar-housekeeper -P 365 -u https://sonarcloud.io -t <token> -o <organization>
+
+# Deletes projects that have not been analyzed on any branch or PR in the last 365 days
+sonar-housekeeper -P 365 --mode delete
+
+# Deletes tokens created more than 365 days ago
+sonar-housekeeper -T 365 --mode delete
+```
 
 ## Required Permissions
 
@@ -41,8 +59,3 @@ To avoid bad mistakes (mistakenly deleting too many projects), the tools will re
 
 ### :warning: Database backup
 **A database backup should always be taken before executing this script. There is no recovery.**
-
-### Example
-```
-sonar-housekeeper -u https://sonar.acme-corp.com -t 15ee09df11fb9b8234b7a1f1ac5fce2e4e93d75d
-```

sonar/branches.py

@@ -361,15 +361,20 @@ def __audit_never_analyzed(self) -> list[Problem]:
 
     def __audit_last_analysis(self, audit_settings: types.ConfigSettings) -> list[Problem]:
         if self.is_main():
-            log.debug("%s is main (not purgeable)", str(self))
+            log.info("%s is main (not purgeable)", str(self))
             return []
         if (age := util.age(self.last_analysis())) is None:
             log.debug("%s last analysis audit is disabled, skipped...", str(self))
             return []
         max_age = audit_settings.get("audit.projects.branches.maxLastAnalysisAge", 30)
+        preserved = audit_settings.get("audit.projects.branches.keepWhenInactive", None)
         problems = []
-        if self.is_kept_when_inactive():
-            log.debug("%s is kept when inactive (not purgeable)", str(self))
+        log.info("EVALUATING %s age %d greater than %d days and not matches '%s'", str(self), age, max_age, preserved)
+        if preserved is not None and age > max_age and not re.match(rf"^{preserved}$", self.name):
+            log.info("%s age %d greater than %d days and not matches '%s'", str(self), age, max_age, preserved)
+            problems.append(Problem(get_rule(RuleId.BRANCH_LAST_ANALYSIS), self, str(self), age))
+        elif self.is_kept_when_inactive():
+            log.info("%s is kept when inactive (not purgeable)", str(self))
         elif age > max_age:
             problems.append(Problem(get_rule(RuleId.BRANCH_LAST_ANALYSIS), self, str(self), age))
         else:

test/unit/test_housekeeper.py

@@ -24,6 +24,9 @@
 sonar-housekeeper tests
 """
 
+from collections.abc import Generator
+import pytest
+
 import utilities as tutil
 from sonar import errcodes
 from cli import housekeeper, options
@@ -36,3 +39,14 @@ def test_housekeeper() -> None:
     """test_housekeeper"""
     for opts in __GOOD_OPTS:
         assert tutil.run_cmd(housekeeper.main, f"{CMD} {tutil.SQS_OPTS} {opts}") == errcodes.OK
+
+def test_keep_branches_override(csv_file: Generator[str]) -> None:
+    """test_keep_branches_override"""
+    if tutil.SQ.version() == "community":
+        pytest.skip("No branches in Community")
+    assert tutil.run_cmd(housekeeper.main, f"{CMD} {tutil.SQS_OPTS} -P 730 -T 730 -R 730 -B 90 -f {csv_file}") == errcodes.OK
+    nbr_br = tutil.csv_col_count_values(csv_file, 1, "BRANCH_LAST_ANALYSIS")
+    assert tutil.run_cmd(housekeeper.main, f"{CMD} {tutil.SQS_OPTS} -P 730 -T 730 -R 730 -B 90 -f {csv_file} --keepWhenInactive 'dontkeepanything'") == errcodes.OK
+    # With 'dontkeepanything' as branch regexp, more branches to delete should be found
+    assert tutil.csv_col_count_values(csv_file, 1, "BRANCH_LAST_ANALYSIS") > nbr_br
+

test/unit/utilities.py

@@ -296,10 +296,15 @@ def csv_col_has_values(csv_file: str, col_name: str, *values) -> bool:
         return False
 
 
-def csv_col_count_values(csv_file: str, col_name: str, *values) -> int:
+def csv_col_count_values(csv_file: str, col_name_or_nbr: Union[str, int], *values) -> int:
+    """Counts the number of times a given column has one of the given values"""
     values_to_search = list(values).copy()
     with open(csv_file, encoding="utf-8") as fd:
-        (col,) = get_cols(next(reader := csv.reader(fd)), col_name)
+        reader = csv.reader(fd)
+        if isinstance(col_name_or_nbr, int):
+            col = col_name_or_nbr - 1
+        else:
+            (col,) = get_cols(next(reader, col_name_or_nbr)) 
         counter = sum(1 if line[col] in values_to_search else 0 for line in reader)
     return counter