Add success criteria summary to results

This commit adds a summary of each success criteria for each experiment,
along with the result of the success criteria. This can be useful to
help debug what causes the experiment result to be whatever it was.

Author: douglasjacobsen

lib/ramble/ramble/experiment_result.py

@@ -36,6 +36,7 @@ class ExperimentStatus(str, Enum):
     "raw_variables": "RAMBLE_RAW_VARIABLES",
     namespace.tags: "TAGS",
     "experiment_chain": "EXPERIMENT_CHAIN",
+    "success_criteria": "SUCCESS_CRITERIA",
 }
 
 
@@ -56,6 +57,7 @@ def __init__(self, app_inst):
         self.experiment_chain = app_inst.chain_order.copy()
         self.tags = list(app_inst.experiment_tags)
         self.contexts = []
+        self.success_criteria = {}
         self.software = {}
 
         self.keys = {}

lib/ramble/ramble/test/success_criteria_functionality/success_summary.py

@@ -0,0 +1,73 @@
+# Copyright 2022-2025 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+
+import pytest
+
+import ramble.workspace
+from ramble.main import RambleCommand
+
+# everything here uses the mock_workspace_path
+pytestmark = pytest.mark.usefixtures("mutable_config", "mutable_mock_workspace_path")
+
+workspace = RambleCommand("workspace")
+ramble_on = RambleCommand("on")
+
+
+def test_success_criteria_summary(mock_applications, workspace_name):
+    test_config = """
+ramble:
+  variables:
+    mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}'
+    batch_submit: '{execute_experiment}'
+    processes_per_node: '1'
+    n_threads: '1'
+  applications:
+    basic:
+      workloads:
+        working_wl:
+          experiments:
+            pass-experiment:
+              variables:
+                n_nodes: 1
+              success_criteria:
+              - name: always-pass
+                mode: string
+                match: '.*seconds.*'
+            fail-experiment:
+              variables:
+                n_nodes: 1
+              success_criteria:
+              - name: always-fail
+                mode: string
+                match: 'Blarg'
+  software:
+    packages: {}
+    environments: {}
+"""
+    with ramble.workspace.create(workspace_name) as ws:
+        ws.write()
+
+        config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name)
+
+        with open(config_path, "w+") as f:
+            f.write(test_config)
+        ws._re_read()
+
+        workspace("setup", global_args=["-w", workspace_name])
+        ramble_on(global_args=["-w", workspace_name])
+        workspace("analyze", "-f", "text", "json", "yaml", global_args=["-w", workspace_name])
+
+        for extension in ["txt", "json", "yaml"]:
+            with open(os.path.join(ws.root, "results.latest.txt")) as f:
+                data = f.read()
+                assert "Success criteria summary:" in data
+                assert "_application_function = PASSED" in data
+                assert "always-pass = PASSED" in data
+                assert "always-fail = FAILED" in data

lib/ramble/ramble/workspace/workspace.py

@@ -1662,6 +1662,10 @@ def dump_results(self, output_formats=None, print_results=False, summary_only=Fa
                             if software_key in exp and exp[software_key]:
                                 self.write_software_info(f, exp)
 
+                        if exp["SUCCESS_CRITERIA"]:
+                            f.write("  Success criteria summary:\n")
+                            for name, result in exp["SUCCESS_CRITERIA"].items():
+                                f.write(f"    {name} = {result}\n")
                 else:
                     logger.msg("No results to write")
 

var/ramble/repos/builtin/base_classes/application-base/base_class.py

@@ -2390,6 +2390,12 @@ def format_context(context_match, context_format):
 
         self._init_result()
 
+        for criteria_obj in criteria_list.all_criteria():
+            if criteria_obj.ok():
+                self.result.success_criteria[criteria_obj.name] = "PASSED"
+            else:
+                self.result.success_criteria[criteria_obj.name] = "FAILED"
+
         for context, fom_map in fom_values.items():
             context_map = {
                 "name": context,