feat(general): Include UNKNOWN check results in Checkov reports

README.md

@@ -51,7 +51,7 @@ Checkov also powers [**Prisma Cloud Application Security**](https://www.prismacl
  * Evaluates [Terraform Provider](https://registry.terraform.io/browse/providers) settings to regulate the creation, management, and updates of IaaS, PaaS or SaaS managed through Terraform.
  * Policies support evaluation of [variables](https://github.com/bridgecrewio/checkov/blob/main/docs/2.Basics/Handling%20Variables.md) to their optional default value.
  * Supports in-line [suppression](https://github.com/bridgecrewio/checkov/blob/main/docs/2.Basics/Suppressing%20and%20Skipping%20Policies.md) of accepted risks or false-positives to reduce recurring scan failures. Also supports global skip from using CLI.
- * [Output](https://github.com/bridgecrewio/checkov/blob/main/docs/2.Basics/Reviewing%20Scan%20Results.md) currently available as CLI, [CycloneDX](https://cyclonedx.org), JSON, JUnit XML, CSV, SARIF and github markdown and link to remediation [guides](https://docs.prismacloud.io/en/enterprise-edition/policy-reference/).
+ * [Output](https://github.com/bridgecrewio/checkov/blob/main/docs/2.Basics/Reviewing%20Scan%20Results.md) currently available as CLI, [CycloneDX](https://cyclonedx.org), JSON, JUnit XML, CSV, SARIF and github markdown and link to remediation [guides](https://docs.prismacloud.io/en/enterprise-edition/policy-reference/). Policies that cannot determine pass or fail return [UNKNOWN](https://github.com/bridgecrewio/checkov/blob/main/docs/2.Basics/Unknown%20Check%20Results.md) and are included in the report and summary.
 
 ## Screenshots
 

checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py

@@ -118,6 +118,7 @@ def post_runner(self, scan_report: Report) -> None:
             scan_report.failed_checks = self.extend_records_with_cloned_policies(scan_report.failed_checks)
             scan_report.passed_checks = self.extend_records_with_cloned_policies(scan_report.passed_checks)
             scan_report.skipped_checks = self.extend_records_with_cloned_policies(scan_report.skipped_checks)
+            scan_report.unknown_checks = self.extend_records_with_cloned_policies(scan_report.unknown_checks)
 
     def extend_records_with_cloned_policies(self, records: list[Record]) -> list[Record]:
         bc_check_ids = [record.bc_check_id for record in records]

checkov/common/bridgecrew/wrapper.py

@@ -57,7 +57,7 @@ def _put_json_object(s3_client: S3Client, json_obj: Any, bucket: str, object_pat
 
 def _extract_checks_metadata(report: Report, full_repo_object_key: str, on_prem: bool) -> dict[str, dict[str, Any]]:
     metadata: dict[str, dict[str, Any]] = defaultdict(dict)
-    for check in itertools.chain(report.passed_checks, report.failed_checks, report.skipped_checks):
+    for check in itertools.chain(report.passed_checks, report.failed_checks, report.skipped_checks, report.unknown_checks):
         metadata_key = f'{check.file_path}:{check.resource}'
         check_meta = {k: getattr(check, k, "") for k in check_metadata_keys}
         check_meta['file_object_path'] = full_repo_object_key + check.file_path
@@ -93,7 +93,10 @@ def reduce_scan_reports(scan_reports: list[Report], on_prem: Optional[bool] = Fa
                         for check in report.failed_checks],
                     "skipped_checks": [
                         {k: getattr(check, k) for k in reduced_keys}
-                        for check in report.skipped_checks]
+                        for check in report.skipped_checks],
+                    "unknown_checks": [
+                        {k: getattr(check, k) for k in reduced_keys}
+                        for check in report.unknown_checks]
                 },
                 "image_cached_results": report.image_cached_results
         }

checkov/common/output/csv.py

@@ -56,13 +56,13 @@ def __init__(self) -> None:
 
     def add_report(self, report: Report, git_org: str, git_repository: str) -> None:
         if report.check_type in (CheckType.SCA_PACKAGE, CheckType.SCA_IMAGE):
-            for record in itertools.chain(report.failed_checks, report.passed_checks, report.skipped_checks):
+            for record in itertools.chain(report.failed_checks, report.passed_checks, report.skipped_checks, report.unknown_checks):
                 if record.check_name == SCA_PACKAGE_SCAN_CHECK_NAME:
                     self.add_sca_package_resources(resource=record, git_org=git_org, git_repository=git_repository, check_type=report.check_type)
             for resource in sorted(report.extra_resources):
                 self.add_sca_package_resources(resource=resource, git_org=git_org, git_repository=git_repository, check_type=report.check_type)
         else:
-            for record in itertools.chain(report.failed_checks, report.passed_checks, report.skipped_checks):
+            for record in itertools.chain(report.failed_checks, report.passed_checks, report.skipped_checks, report.unknown_checks):
                 self.add_iac_resources(resource=record, git_org=git_org, git_repository=git_repository)
             for resource in sorted(report.extra_resources):
                 self.add_iac_resources(resource=resource, git_org=git_org, git_repository=git_repository)

checkov/common/output/record.py

@@ -200,6 +200,9 @@ def to_string(self, compact: bool = False, use_bc_ids: bool = False) -> str:
             status = CheckResult.SKIPPED.name
             status_color = "blue"
             suppress_comment = "\tSuppress comment: {}\n".format(self.check_result.get("suppress_comment", ""))
+        elif self.check_result["result"] == CheckResult.UNKNOWN:
+            status = CheckResult.UNKNOWN.name
+            status_color = "yellow"
 
         check_message = colored('Check: {}: "{}"\n'.format(self.get_output_id(use_bc_ids), self.check_name), "white")
         guideline_message = self.get_guideline_string(self.guideline)

checkov/common/output/report.py

@@ -50,6 +50,7 @@ def __init__(self, check_type: str):
         self.passed_checks: list[Record] = []
         self.failed_checks: list[Record] = []
         self.skipped_checks: list[Record] = []
+        self.unknown_checks: list[Record] = []
         self.parsing_errors: list[str] = []
         self.resources: set[str] = set()
         self.extra_resources: set[ExtraResource] = set()
@@ -81,12 +82,15 @@ def add_record(self, record: Record) -> None:
             self.failed_checks.append(record)
         if record.check_result["result"] == CheckResult.SKIPPED:
             self.skipped_checks.append(record)
+        if record.check_result["result"] == CheckResult.UNKNOWN:
+            self.unknown_checks.append(record)
 
     def get_summary(self) -> Dict[str, Union[int, str]]:
         return {
             "passed": len(self.passed_checks),
             "failed": len(self.failed_checks),
             "skipped": len(self.skipped_checks),
+            "unknown": len(self.unknown_checks),
             "parsing_errors": len(self.parsing_errors),
             "resource_count": len(self.resources),
             "checkov_version": version,
@@ -96,7 +100,7 @@ def get_json(self) -> str:
         return json.dumps(self.get_dict(), indent=4, cls=CustomJSONEncoder)
 
     def get_all_records(self) -> List[Record]:
-        return self.failed_checks + self.passed_checks + self.skipped_checks
+        return self.failed_checks + self.passed_checks + self.skipped_checks + self.unknown_checks
 
     def get_dict(self, is_quiet: bool = False, url: str | None = None, full_report: bool = False, s3_setup_failed: bool = False, support_path: str | None = None) -> dict[str, Any]:
         if not url and not s3_setup_failed:
@@ -118,7 +122,8 @@ def get_dict(self, is_quiet: bool = False, url: str | None = None, full_report:
                 "checks": {
                     "passed_checks": [check.__dict__ for check in self.passed_checks],
                     "failed_checks": [check.__dict__ for check in self.failed_checks],
-                    "skipped_checks": [check.__dict__ for check in self.skipped_checks]
+                    "skipped_checks": [check.__dict__ for check in self.skipped_checks],
+                    "unknown_checks": [check.__dict__ for check in self.unknown_checks],
                 },
                 "image_cached_results": [res.__dict__ for res in self.image_cached_results]
             }
@@ -129,6 +134,7 @@ def get_dict(self, is_quiet: bool = False, url: str | None = None, full_report:
                     "passed_checks": [check.__dict__ for check in self.passed_checks],
                     "failed_checks": [check.__dict__ for check in self.failed_checks],
                     "skipped_checks": [check.__dict__ for check in self.skipped_checks],
+                    "unknown_checks": [check.__dict__ for check in self.unknown_checks],
                     "parsing_errors": list(self.parsing_errors),
                 },
                 "summary": self.get_summary(),
@@ -265,6 +271,7 @@ def is_empty(self, full: bool = False) -> bool:
             len(self.passed_checks)
             + len(self.failed_checks)
             + len(self.skipped_checks)
+            + len(self.unknown_checks)
             + len(self.parsing_errors)
         )
 
@@ -291,17 +298,18 @@ def print_console(
         summary = self.get_summary()
         output_data = colored(f"{self.check_type} scan results:\n", "blue")
         if self.parsing_errors:
-            message = "\nPassed checks: {}, Failed checks: {}, Skipped checks: {}, Parsing errors: {}\n\n".format(
+            message = "\nPassed checks: {}, Failed checks: {}, Skipped checks: {}, Unknown checks: {}, Parsing errors: {}\n\n".format(
                 summary["passed"],
                 summary["failed"],
                 summary["skipped"],
+                summary["unknown"],
                 summary["parsing_errors"],
             )
         else:
             if self.check_type == CheckType.SCA_PACKAGE or self.check_type.lower().startswith(CheckType.SAST):
                 message = f"\nFailed checks: {summary['failed']}, Skipped checks: {summary['skipped']}\n\n"
             else:
-                message = f"\nPassed checks: {summary['passed']}, Failed checks: {summary['failed']}, Skipped checks: {summary['skipped']}\n\n"
+                message = f"\nPassed checks: {summary['passed']}, Failed checks: {summary['failed']}, Skipped checks: {summary['skipped']}, Unknown checks: {summary['unknown']}\n\n"
         if summary_position == 'top':
             output_data += colored(message, "cyan")
         # output for vulnerabilities is different
@@ -328,6 +336,8 @@ def print_console(
             for record in self.failed_checks:
                 output_data += record.to_string(compact=is_compact, use_bc_ids=use_bc_ids)
             if not is_quiet:
+                for record in self.unknown_checks:
+                    output_data += record.to_string(compact=is_compact, use_bc_ids=use_bc_ids)
                 for record in self.skipped_checks:
                     output_data += record.to_string(compact=is_compact, use_bc_ids=use_bc_ids)
 
@@ -372,14 +382,15 @@ def print_failed_github_md(self, use_bc_ids: bool = False) -> str:
         if result:
             summary = self.get_summary()
             if self.parsing_errors:
-                message = "Passed Checks: {}, Failed Checks: {}, Skipped Checks: {}, Parsing Errors: {}\n\n".format(
+                message = "Passed Checks: {}, Failed Checks: {}, Skipped Checks: {}, Unknown Checks: {}, Parsing Errors: {}\n\n".format(
                     summary["passed"],
                     summary["failed"],
                     summary["skipped"],
+                    summary["unknown"],
                     summary["parsing_errors"],
                 )
             else:
-                message = f"```\nPassed Checks: {summary['passed']}, Failed Checks: {summary['failed']}, Skipped Checks: {summary['skipped']}\n```\n\n"
+                message = f"```\nPassed Checks: {summary['passed']}, Failed Checks: {summary['failed']}, Skipped Checks: {summary['skipped']}, Unknown Checks: {summary['unknown']}\n```\n\n"
 
             table = tabulate(
                 result,
@@ -396,7 +407,7 @@ def get_test_suite(self, properties: Optional[Dict[str, Any]] = None, use_bc_ids
 
         test_cases = []
 
-        records = self.passed_checks + self.failed_checks + self.skipped_checks
+        records = self.passed_checks + self.failed_checks + self.skipped_checks + self.unknown_checks
         for record in records:
             severity = BcSeverities.NONE
             if record.severity:
@@ -431,6 +442,8 @@ def get_test_suite(self, properties: Optional[Dict[str, Any]] = None, use_bc_ids
                     test_case.add_skipped_info(f"{check_id} skipped for {test_name_detail}")
                 else:
                     test_case.add_skipped_info(record.check_result.get("suppress_comment", ""))
+            if record.check_result["result"] == CheckResult.UNKNOWN:
+                test_case.add_skipped_info(f"Check result unknown: {record.check_name}")
 
             test_cases.append(test_case)
 
@@ -615,9 +628,13 @@ def from_reduced_json(cls, json_report: dict[str, Any], check_type: str) -> Repo
         report = Report(check_type)
         report.image_cached_results = json_report['image_cached_results']
 
-        all_json_records = json_report["checks"]["passed_checks"] + \
-            json_report["checks"]["failed_checks"] + \
-            json_report["checks"]["skipped_checks"]
+        checks = json_report["checks"]
+        if "unknown_checks" not in checks:
+            checks = {**checks, "unknown_checks": []}
+        all_json_records = checks["passed_checks"] + \
+            checks["failed_checks"] + \
+            checks["skipped_checks"] + \
+            checks["unknown_checks"]
 
         for json_record in all_json_records:
             report.add_record(
@@ -631,6 +648,7 @@ def merge_reports(base_report: Report, report_to_merge: Report) -> None:
     base_report.passed_checks.extend(report_to_merge.passed_checks)
     base_report.failed_checks.extend(report_to_merge.failed_checks)
     base_report.skipped_checks.extend(report_to_merge.skipped_checks)
+    base_report.unknown_checks.extend(report_to_merge.unknown_checks)
     base_report.parsing_errors.extend(report_to_merge.parsing_errors)
     base_report.image_cached_results.extend(report_to_merge.image_cached_results)
     base_report.resources.update(report_to_merge.resources)
@@ -648,4 +666,5 @@ def dedupe_records(origin_records: list[Record]) -> list[Record]:
 
     report.passed_checks = dedupe_records(report.passed_checks)
     report.failed_checks = dedupe_records(report.failed_checks)
+    report.unknown_checks = dedupe_records(report.unknown_checks)
     return report

checkov/common/output/sarif.py

@@ -73,7 +73,7 @@ def _create_rules(self) -> list[dict[str, Any]]:
 
         for report in self.reports:
             if report.check_type in SCA_CHECKTYPES:
-                for record in itertools.chain(report.failed_checks, report.skipped_checks):
+                for record in itertools.chain(report.failed_checks, report.skipped_checks, report.unknown_checks):
                     rule = None
                     if record.check_id.startswith("BC_LIC"):
                         rule = self._create_license_rule(check_type=report.check_type, record=record)
@@ -85,7 +85,7 @@ def _create_rules(self) -> list[dict[str, Any]]:
                         rules.append(rule)
                         rule_idx += 1
             else:
-                for record in itertools.chain(report.failed_checks, report.skipped_checks):
+                for record in itertools.chain(report.failed_checks, report.skipped_checks, report.unknown_checks):
                     if record.check_id not in self.rule_index_map:
                         rule = self._create_iac_rule(check_type=report.check_type, record=record)
                         self.rule_index_map[rule["id"]] = rule_idx
@@ -199,12 +199,14 @@ def _create_results(self) -> list[dict[str, Any]]:
         results: "list[dict[str, Any]]" = []
 
         for report in self.reports:
-            for record in itertools.chain(report.failed_checks, report.skipped_checks):
+            for record in itertools.chain(report.failed_checks, report.skipped_checks, report.unknown_checks):
                 level = "warning"
                 if record.severity:
                     level = SEVERITY_TO_SARIF_LEVEL.get(record.severity.name.lower(), "none")
                 elif record.check_result.get("result") == CheckResult.FAILED:
                     level = "error"
+                elif record.check_result.get("result") == CheckResult.UNKNOWN:
+                    level = "note"
 
                 rule_id = self._create_rule_id(check_type=report.check_type, record=record)
                 if not rule_id or rule_id not in self.rule_index_map:

checkov/common/runners/runner_registry.py

@@ -723,7 +723,7 @@ def remove_runner(self, runner: _BaseRunner) -> None:
 
     @staticmethod
     def enrich_report_with_guidelines(scan_report: Report) -> None:
-        for record in itertools.chain(scan_report.failed_checks, scan_report.passed_checks, scan_report.skipped_checks):
+        for record in itertools.chain(scan_report.failed_checks, scan_report.passed_checks, scan_report.skipped_checks, scan_report.unknown_checks):
             guideline = metadata_integration.get_guideline(record.check_id)
             if guideline:
                 record.set_guideline(guideline)

checkov/common/typing.py

@@ -95,6 +95,7 @@ class _ReducedScanReportCheck(TypedDict):
     failed_checks: list[dict[str, Any]]
     passed_checks: list[dict[str, Any]]
     skipped_checks: list[dict[str, Any]]
+    unknown_checks: list[dict[str, Any]]
 
 
 class _CicdDetails(TypedDict, total=False):

docs/2.Basics/Reviewing Scan Results.md

@@ -9,6 +9,8 @@ nav_order: 6
 
 The results of Checkov scans can be viewed in CLI, JSON, JUnit XML, SARIF, or Markdown
 
+Some checks return **UNKNOWN** when a value cannot be determined (e.g. variable-dependent or plan `after_unknown`). These are included in the report and summary; see [Unknown Check Results](Unknown%20Check%20Results.md) for details and examples.
+
 > Note: For Markdown output, you need to use `github_failed_only` as the `--output` type
 
 ## Scan Result Sample (CLI)