feat: return non 0 exit code when there are failures (#28)

* feat: return non 0 exit code when there are failures

When failures are reported in the destination report, you have two options:
- We should raise an exit code, this allows you to act on the error.
- We ignore it and return a 0 exit code, this allows us to ignore the results and continue as usual.

Author: Joris Conijn

Makefile

@@ -22,8 +22,7 @@ clean:
 	[[ -f .coverage ]] && rm .coverage || true
 
 run:
-	report2junit --source-type cfn-guard ./sample-reports/cfn-guard.json
-	report2junit --source-type cfn-nag ./sample-reports/cfn-nag.json
+	report2junit --ignore-failures ./sample-reports/cfn-guard.json ./sample-reports/cfn-nag.json
 
 build:
 	python setup.py sdist

README.md

@@ -57,3 +57,17 @@ report2junit ./sample-reports/cfn-nag.json ./sample-reports/cfn-guard.json
 # Or if you want to specify the destination:
 report2junit ./sample-reports/cfn-nag.json ./sample-reports/cfn-guard.json --destination-file ./sample-reports/junit-other.xml
 ```
+
+In some cases it is useful to explicitly stop when there are failures. For example when you want to enforce that there
+are no failures. Or on the other hand we could continue when there are failures. This behaviour can be influenced using
+the `--ignore-failures` and `--fail-on-failures` options. Where `--fail-on-failures` is the default.
+
+```bash
+# Convert the given report and when there are failures exit code 1 is returned.
+report2junit ./sample-reports/cfn-guard.json --fail-on-failures
+echo $?
+
+# Convert the given report and when there are failures exit code 0 is returned.
+report2junit ./sample-reports/cfn-guard.json --ignore-failures
+echo $?
+```

report2junit/__init__.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, List
 import os.path
 import click
 
@@ -10,27 +10,40 @@
 @click.command()
 @click.argument("source-files", nargs=-1)
 @click.option("--destination-file", required=False)
+@click.option("--fail-on-failures/--ignore-failures", default=True)
 def main(
-    source_files: str,
-    destination_file: Optional[str],
-):
+    source_files: List[str], destination_file: Optional[str], fail_on_failures: bool
+) -> None:
     """
     Convert2JUnit
 
     This tool allows you to convert various reports into the JUnit format.
     """
-    if not destination_file:
-        destination_file = os.path.join(os.path.dirname(source_files[0]), "junit.xml")
+    report = initialize_report(destination_file, source_files)
+    apply_source_files(report, source_files)
+    report.write()
 
-    report = JUnitOutput(destination_file)
+    if fail_on_failures and report.has_failures():
+        raise click.ClickException(
+            "report2junit detected that the report contains failures"
+        )
 
+
+def apply_source_files(report: JUnitOutput, source_files: List[str]) -> None:
     for source_file in source_files:
         source_file = os.path.abspath(source_file)
 
         if not report.apply(source_file):
             raise click.ClickException("Could not convert the report")
 
-    report.write()
+
+def initialize_report(
+    destination_file: Optional[str], source_files: List[str]
+) -> JUnitOutput:
+    if not destination_file:
+        destination_file = os.path.join(os.path.dirname(source_files[0]), "junit.xml")
+
+    return JUnitOutput(destination_file)
 
 
 if __name__ == "__main__":

report2junit/junit.py

@@ -2,12 +2,15 @@
 
 
 from typing import List
-from junit_xml import to_xml_report_string, TestSuite
+from junit_xml import to_xml_report_string, TestSuite, TestCase
 
 from report2junit.reports import Report
 
 
 class JUnitOutput:
+    __destination: str
+    __reports: List[TestSuite]
+
     def __init__(self, destination: str):
         self.__destination = destination
         self.__reports = self.__load_existing()
@@ -24,6 +27,15 @@ def apply(self, source: str) -> bool:
     def add_test_suite(self, report: TestSuite) -> None:
         self.__reports.append(report)
 
+    def has_failures(self) -> bool:
+        def report_has_failures(report: TestSuite) -> bool:
+            return any(map(case_has_failures, report.test_cases))
+
+        def case_has_failures(case: TestCase) -> bool:
+            return len(case.failures) > 0
+
+        return any(map(report_has_failures, self.__reports))
+
     def write(self) -> None:
         if len(self.__reports) == 0:
             raise Exception("No reports found to write")

tests/test_cli.py

@@ -7,22 +7,32 @@
 
 
 @pytest.mark.parametrize(
-    "input_file",
+    "fail_on_failures, input_file, expected_exit_code",
     [
-        "cfn-guard.json",
-        "cfn-guard-fully-compliant.json",
-        "cfn-nag.json",
-        "cfn-guard-skipped-only.json",
+        (False, "cfn-guard.json", 0),
+        (True, "cfn-guard.json", 1),
+        (False, "cfn-guard-fully-compliant.json", 0),
+        (True, "cfn-guard-fully-compliant.json", 0),
+        (False, "cfn-nag.json", 0),
+        (True, "cfn-nag.json", 1),
+        (False, "cfn-guard-skipped-only.json", 0),
+        (True, "cfn-guard-skipped-only.json", 0),
     ],
 )
-def test_single_report_conversion(input_file: str, sample_reports_path: str) -> None:
+def test_single_report_conversion(
+    fail_on_failures: bool,
+    input_file: str,
+    expected_exit_code: int,
+    sample_reports_path: str,
+) -> None:
     runner = CliRunner()
     m = mock_open()
 
     with patch("report2junit.junit.open", m):
         result = runner.invoke(
             main,
             [
+                "--fail-on-failures" if fail_on_failures else "--ignore-failures",
                 f"{sample_reports_path}/{input_file}",
             ],
         )
@@ -32,7 +42,7 @@ def test_single_report_conversion(input_file: str, sample_reports_path: str) ->
     handle.write.assert_called_once_with(
         expected_payload(input_file.replace(".json", ".xml"))
     )
-    assert result.exit_code == 0
+    assert result.exit_code == expected_exit_code
 
 
 def test_cfn_guard_conversion_explicit_destination(sample_reports_path: str) -> None:
@@ -43,6 +53,7 @@ def test_cfn_guard_conversion_explicit_destination(sample_reports_path: str) ->
         result = runner.invoke(
             main,
             [
+                "--ignore-failures",
                 f"{sample_reports_path}/cfn-guard.json",
                 "--destination-file",
                 f"{sample_reports_path}/cfn-guard-specific.xml",