Skip to content

Commit 76a5e6a

Browse files
authored
Allow-sonar-audit-filters (#2021)
* Fixes #2017
1 parent 19ebdae commit 76a5e6a

File tree

10 files changed

+198
-89
lines changed

10 files changed

+198
-89
lines changed

cli/audit.py

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
import json
2626
import csv
27+
import re
2728
from typing import TextIO, Optional
2829
from threading import Thread
2930
from queue import Queue
@@ -50,6 +51,8 @@
5051
options.WHAT_PORTFOLIOS: portfolios.audit,
5152
}
5253

54+
PROBLEM_KEYS = "problems"
55+
5356

5457
def _audit_sif(sysinfo: str, audit_settings: types.ConfigSettings) -> tuple[str, list[problem.Problem]]:
5558
"""Audits a SIF and return found problems"""
@@ -70,20 +73,35 @@ def _audit_sif(sysinfo: str, audit_settings: types.ConfigSettings) -> tuple[str,
7073
return sif_obj.server_id(), sif_obj.audit(audit_settings)
7174

7275

76+
def __filter_problems(problems: list[problem.Problem], settings: types.ConfigSettings) -> list[problem.Problem]:
77+
"""Filters audit problems by severity and/or type and/or problem key"""
78+
if settings.get(options.SEVERITIES, None):
79+
log.debug("Filtering audit problems with severities: %s", settings[options.SEVERITIES])
80+
problems = [p for p in problems if str(p.severity) in settings[options.SEVERITIES]]
81+
if settings.get(options.TYPES, None):
82+
log.debug("Filtering audit problems with types: %s", settings[options.TYPES])
83+
problems = [p for p in problems if str(p.type) in settings[options.TYPES]]
84+
if settings.get(PROBLEM_KEYS, None):
85+
log.debug("Filtering audit problems with keys: %s", settings[PROBLEM_KEYS])
86+
problems = [p for p in problems if re.match(rf"^{settings[PROBLEM_KEYS]}$", str(p.rule_id))]
87+
return problems
88+
89+
7390
def write_csv(queue: Queue[list[problem.Problem]], fd: TextIO, settings: types.ConfigSettings) -> None:
7491
"""Thread callback to write audit problems in a CSV file"""
7592
server_id = settings.get("SERVER_ID", None)
7693
with_url = settings.get("WITH_URL", False)
7794
csvwriter = csv.writer(fd, delimiter=settings.get("CSV_DELIMITER", ","))
7895
header = ["Server Id"] if server_id else []
79-
header += ["Audit Check", "Category", "Severity", "Message"]
96+
header += ["Problem", "Type", "Severity", "Message"]
8097
header += ["URL"] if with_url else []
8198
csvwriter.writerow(header)
8299
while (problems := queue.get()) is not util.WRITE_END:
100+
problems = __filter_problems(problems, settings)
83101
for p in problems:
84102
json_data = p.to_json(with_url)
85103
data = [] if not server_id else [server_id]
86-
data += list(json_data.values())
104+
data += [json_data[k] for k in ("problem", "type", "severity", "message") if k in json_data]
87105
csvwriter.writerow(data)
88106
queue.task_done()
89107
queue.task_done()
@@ -96,6 +114,7 @@ def write_json(queue: Queue[list[problem.Problem]], fd: TextIO, settings: types.
96114
comma = ""
97115
print("[", file=fd)
98116
while (problems := queue.get()) is not util.WRITE_END:
117+
problems = __filter_problems(problems, settings)
99118
for p in problems:
100119
json_data = p.to_json(with_url)
101120
if server_id:
@@ -166,6 +185,24 @@ def __parser_args(desc: str) -> object:
166185
nargs="*",
167186
help="Pass audit configuration settings on command line (-D<setting>=<value>)",
168187
)
188+
parser.add_argument(
189+
f"--{options.SEVERITIES}",
190+
required=False,
191+
default=None,
192+
help="Report only audit problems with the given severities (comma separate values LOW, MEDIUM, HIGH, CRITICAL)",
193+
)
194+
parser.add_argument(
195+
f"--{options.TYPES}",
196+
required=False,
197+
default=None,
198+
help="Report only audit problems of the given comma separated problem types",
199+
)
200+
parser.add_argument(
201+
f"--{PROBLEM_KEYS}",
202+
required=False,
203+
default=None,
204+
help="Report only audit problems whose type key matches the given regexp",
205+
)
169206
args = options.parse_and_check(parser=parser, logger_name=TOOL_NAME, verify_token=False)
170207
if args.sif is None and args.config is None:
171208
util.check_token(args.token)
@@ -189,14 +226,14 @@ def main() -> None:
189226
key, value = val[0].split("=", maxsplit=1)
190227
cli_settings[key] = value
191228
settings = audit_conf.load(TOOL_NAME, cli_settings)
229+
settings |= kwargs
192230
file = ofile = kwargs.pop(options.REPORT_FILE)
193231
fmt = util.deduct_format(kwargs[options.FORMAT], ofile)
194232
settings.update(
195233
{
196234
"FILE": file,
197235
"CSV_DELIMITER": kwargs[options.CSV_SEPARATOR],
198236
"WITH_URL": kwargs[options.WITH_URL],
199-
"threads": kwargs[options.NBR_THREADS],
200237
"format": fmt,
201238
}
202239
)
@@ -208,8 +245,8 @@ def main() -> None:
208245
file = kwargs["sif"]
209246
errcode = errcodes.SIF_AUDIT_ERROR
210247
(settings["SERVER_ID"], problems) = _audit_sif(file, settings)
248+
problems = __filter_problems(problems, settings)
211249
problem.dump_report(problems, file=ofile, server_id=settings["SERVER_ID"], format=fmt)
212-
213250
else:
214251
sq = platform.Platform(**kwargs)
215252
sq.verify_connection()

cli/findings_export.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ def parse_args(desc: str) -> Namespace:
125125
parser.add_argument(
126126
f"--{options.SEVERITIES}",
127127
required=False,
128-
help="Comma separated severities among" + util.list_to_csv(idefs.STD_SEVERITIES + hotspots.SEVERITIES),
128+
help="Comma separated severities among " + util.list_to_csv(idefs.STD_SEVERITIES + hotspots.SEVERITIES),
129129
)
130130
parser.add_argument(
131131
f"--{options.TYPES}",

doc/sonar-audit.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ When `--what` is not specified, everything is audited
3232
- `--what apps`: Audits applications
3333
- `-f <file>`: Sends audit output to `<file>`, `stdout` is the default. The output format is deducted from
3434
the file extension (JSON or CSV), except if `--format` is specified
35+
- `--severities`: The audit output will only reports problems of the given severities to pass as comma separated (LOW, MEDIUM, HIGH, CRITICAL)
36+
- `--types`: The audit output will only reports problems of the given types to pass as comma separated (BAD_PRACTICE, GOVERNANCE, HOUSEKEEPING, OPERATIONS, PERFORMANCE, SECURITY as of today, more may be added in the future)
37+
- `--problems`: The audit output will only report problems whose key match the given regexp. The key is the 2nd column of the CSV or the "problem" field of the JSON
3538
- `--sif <SystemInfoFile>`: Will audit the input SIF file, instead of connecting to a SonarQube Server or Cloud platform.
3639
In that case:
3740
- URL and token are not needed

sonar/audit/problem.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ def to_json(self, with_url=False):
5151
d = vars(self).copy()
5252
d.pop("concerned_object")
5353

54-
for k in ("severity", "type", "rule_id"):
54+
d["problem"] = str(d.pop("rule_id"))
55+
for k in ("severity", "type"):
5556
d[k] = str(d[k])
5657
if with_url:
5758
try:

sonar/portfolios.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ def __str__(self) -> str:
171171
"""Returns string representation of object"""
172172
return (
173173
f"subportfolio '{self.key}'"
174-
if self.sq_json.get("qualifier", _PORTFOLIO_QUALIFIER) == _SUBPORTFOLIO_QUALIFIER
174+
if self.sq_json and self.sq_json.get("qualifier", _PORTFOLIO_QUALIFIER) == _SUBPORTFOLIO_QUALIFIER
175175
else f"portfolio '{self.key}'"
176176
)
177177

test/unit/conftest.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -239,40 +239,44 @@ def csv_file() -> Generator[str]:
239239
"""setup of tests"""
240240
file = get_temp_filename("csv")
241241
yield file
242-
rm(file)
242+
if os.path.exists(file):
243+
rm(file)
243244

244245

245246
@pytest.fixture
246247
def txt_file() -> Generator[str]:
247248
"""setup of tests"""
248249
file = get_temp_filename("txt")
249250
yield file
250-
rm(file)
251+
if os.path.exists(file):
252+
rm(file)
251253

252254

253255
@pytest.fixture
254256
def json_file() -> Generator[str]:
255257
"""setup of tests"""
256258
file = get_temp_filename("json")
257259
yield file
258-
rm(file)
260+
if os.path.exists(file):
261+
rm(file)
259262

260263

261264
@pytest.fixture
262265
def yaml_file() -> Generator[str]:
263266
"""setup of tests"""
264267
file = get_temp_filename("yaml")
265-
rm(file)
266268
yield file
267-
rm(file)
269+
if os.path.exists(file):
270+
rm(file)
268271

269272

270273
@pytest.fixture
271274
def sarif_file() -> Generator[str]:
272275
"""setup of tests"""
273276
file = get_temp_filename("sarif")
274277
yield file
275-
rm(file)
278+
if os.path.exists(file):
279+
rm(file)
276280

277281

278282
@pytest.fixture

test/unit/test_audit.py

Lines changed: 2 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -19,75 +19,14 @@
1919
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
2020
#
2121

22-
"""sonar-audit tests"""
23-
24-
import os
25-
from collections.abc import Generator
22+
"""Project audit tests"""
2623

2724
import utilities as tutil
28-
from sonar import errcodes as e
29-
import cli.options as opt
30-
from cli import audit
3125
from sonar import projects
3226
from sonar.audit import rules
3327

34-
CMD = f"sonar-audit.py {tutil.SQS_OPTS}"
35-
36-
AUDIT_DISABLED = """
37-
audit.globalSettings = no
38-
audit.projects = false
39-
audit.qualityGates = no
40-
audit.qualityProfiles = no
41-
audit.users = no
42-
audit.groups = no
43-
audit.portfolios = no
44-
audit.applications = no
45-
audit.logs = no
46-
audit.plugins = no"""
47-
48-
49-
def test_audit_disabled(csv_file: Generator[str]) -> None:
50-
"""test_audit_disabled"""
51-
with open(".sonar-audit.properties", mode="w", encoding="utf-8") as fd:
52-
print(AUDIT_DISABLED, file=fd)
53-
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {csv_file}") == e.OK
54-
os.remove(".sonar-audit.properties")
55-
assert tutil.csv_nbr_lines(csv_file) == 0
56-
57-
58-
def test_audit_stdout() -> None:
59-
"""test_audit_stdout"""
60-
assert tutil.run_cmd(audit.main, CMD) == e.OK
61-
62-
63-
def test_audit_json(json_file: Generator[str]) -> None:
64-
"""test_audit_json"""
65-
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {json_file}") == e.OK
66-
67-
68-
def test_audit_proj_key(csv_file: Generator[str]) -> None:
69-
"""test_audit_proj_key"""
70-
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {csv_file} --{opt.WHAT} projects --{opt.KEY_REGEXP} {tutil.LIVE_PROJECT}") == e.OK
71-
72-
73-
def test_audit_proj_non_existing_key() -> None:
74-
"""test_audit_proj_non_existing_key"""
75-
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.WHAT} projects --{opt.KEY_REGEXP} {tutil.LIVE_PROJECT},bad_key") == e.ARGS_ERROR
76-
77-
78-
def test_audit_cmd_line_settings(csv_file: Generator[str]) -> None:
79-
"""test_audit_cmd_line_settings"""
80-
what_to_audit = ["logs", "projects", "portfolios", "applications", "qualityProfiles", "qualityGates", "users", "groups"]
81-
cli_opt = " ".join([f"-Daudit.{what}=true" for what in what_to_audit])
82-
assert tutil.run_cmd(audit.main, f"{CMD} {cli_opt} --{opt.REPORT_FILE} {csv_file}") == e.OK
83-
assert tutil.csv_nbr_lines(csv_file) > 0
84-
85-
cli_opt = " ".join([f"-Daudit.{what}=false" for what in what_to_audit + ["globalSettings"]])
86-
assert tutil.run_cmd(audit.main, f"{CMD} {cli_opt} --{opt.REPORT_FILE} {csv_file}") == e.OK
87-
assert tutil.csv_nbr_lines(csv_file) == 0
88-
8928

90-
def test_audit_proj_key_pattern(csv_file: Generator[str]) -> None:
29+
def test_audit_proj_key_pattern() -> None:
9130
"""test_audit_cmd_line_settings"""
9231
settings = {"audit.projects": True, "audit.projects.keyPattern": None}
9332
pbs = projects.audit(tutil.SQ, settings, key_list="BANKING.*")

test/unit/test_cli_audit.py

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/usr/bin/env python3
2+
#
3+
# sonar-tools tests
4+
# Copyright (C) 2024-2025 Olivier Korach
5+
# mailto:olivier.korach AT gmail DOT com
6+
#
7+
# This program is free software; you can redistribute it and/or
8+
# modify it under the terms of the GNU Lesser General Public
9+
# License as published by the Free Software Foundation; either
10+
# version 3 of the License, or (at your option) any later version.
11+
#
12+
# This program is distributed in the hope that it will be useful,
13+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15+
# Lesser General Public License for more details.
16+
#
17+
# You should have received a copy of the GNU Lesser General Public License
18+
# along with this program; if not, write to the Free Software Foundation,
19+
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20+
#
21+
22+
"""sonar-audit CLI tests"""
23+
24+
import os
25+
from collections.abc import Generator
26+
27+
import utilities as tutil
28+
from sonar import errcodes as e
29+
import cli.options as opt
30+
from cli import audit
31+
32+
CMD = f"sonar-audit.py {tutil.SQS_OPTS}"
33+
34+
AUDIT_DISABLED = """
35+
audit.globalSettings = no
36+
audit.projects = false
37+
audit.qualityGates = no
38+
audit.qualityProfiles = no
39+
audit.users = no
40+
audit.groups = no
41+
audit.portfolios = no
42+
audit.applications = no
43+
audit.logs = no
44+
audit.plugins = no"""
45+
46+
47+
def test_audit_disabled(csv_file: Generator[str]) -> None:
48+
"""Tests that nothing is output when all audits are disabled"""
49+
with open(".sonar-audit.properties", mode="w", encoding="utf-8") as fd:
50+
print(AUDIT_DISABLED, file=fd)
51+
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {csv_file}") == e.OK
52+
os.remove(".sonar-audit.properties")
53+
assert tutil.csv_nbr_lines(csv_file) == 0
54+
55+
56+
def test_audit_stdout() -> None:
57+
"""Tests audit to stdout"""
58+
assert tutil.run_cmd(audit.main, CMD) == e.OK
59+
60+
61+
def test_audit_json(json_file: Generator[str]) -> None:
62+
"""Test audit to json file"""
63+
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {json_file}") == e.OK
64+
65+
66+
def test_audit_proj_key(csv_file: Generator[str]) -> None:
67+
"""Tests that audit can select only specific project keys"""
68+
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {csv_file} --{opt.WHAT} projects --{opt.KEY_REGEXP} {tutil.LIVE_PROJECT}") == e.OK
69+
70+
71+
def test_audit_proj_non_existing_key() -> None:
72+
"""Tests that error is raised when the project key regexp does not select any project"""
73+
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.WHAT} projects --{opt.KEY_REGEXP} {tutil.LIVE_PROJECT},bad_key") == e.ARGS_ERROR
74+
75+
76+
def test_audit_cmd_line_settings(csv_file: Generator[str]) -> None:
77+
"""Verifies that passing audit settings from command line with -D<key>=<value> works"""
78+
what_to_audit = ["logs", "projects", "portfolios", "applications", "qualityProfiles", "qualityGates", "users", "groups"]
79+
cli_opt = " ".join([f"-Daudit.{what}=true" for what in what_to_audit])
80+
assert tutil.run_cmd(audit.main, f"{CMD} {cli_opt} --{opt.REPORT_FILE} {csv_file}") == e.OK
81+
assert tutil.csv_nbr_lines(csv_file) > 0
82+
83+
cli_opt = " ".join([f"-Daudit.{what}=false" for what in what_to_audit + ["globalSettings"]])
84+
assert tutil.run_cmd(audit.main, f"{CMD} {cli_opt} --{opt.REPORT_FILE} {csv_file}") == e.OK
85+
assert tutil.csv_nbr_lines(csv_file) == 0
86+
87+
88+
def test_filter_severity(csv_file: Generator[str]) -> None:
89+
"""Verify that filtering by severities works"""
90+
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {csv_file} --{opt.SEVERITIES} MEDIUM,HIGH") == e.OK
91+
assert tutil.csv_nbr_lines(csv_file) > 0
92+
assert tutil.csv_col_is_value(csv_file, "Severity", "MEDIUM", "HIGH")
93+
94+
95+
def test_filter_type(json_file: Generator[str]) -> None:
96+
"""Verify that filtering by severities works"""
97+
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {json_file} --{opt.TYPES} HOUSEKEEPING,SECURITY") == e.OK
98+
assert tutil.json_field_in_values(json_file, "type", "HOUSEKEEPING", "SECURITY")
99+
100+
101+
def test_filter_problem(csv_file: Generator[str]) -> None:
102+
"""Verify that filtering by problem id works"""
103+
regexp = "(OBJECT.+|QG.+)"
104+
assert tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {csv_file} --problems {regexp}") == e.OK
105+
assert tutil.csv_col_match(csv_file, "Problem", regexp)
106+
107+
108+
def test_filter_multiple(csv_file: Generator[str]) -> None:
109+
"""Verify that filtering by problem id works"""
110+
regexp = "(OBJECT.+|QG.+)"
111+
assert (
112+
tutil.run_cmd(audit.main, f"{CMD} --{opt.REPORT_FILE} {csv_file} --{opt.TYPES} HOUSEKEEPING --{opt.SEVERITIES} MEDIUM --problems {regexp}")
113+
== e.OK
114+
)
115+
assert tutil.csv_col_is_value(csv_file, "Severity", "MEDIUM")
116+
assert tutil.csv_col_is_value(csv_file, "Type", "HOUSEKEEPING")
117+
assert tutil.csv_col_match(csv_file, "Problem", regexp)

0 commit comments

Comments
 (0)