Merge pull request #1086 from douglasjacobsen/success-summary

Add success criteria summary to results

Author: rfbgo

lib/ramble/ramble/experiment_result.py

@@ -36,6 +36,7 @@ class ExperimentStatus(str, Enum):
     "raw_variables": "RAMBLE_RAW_VARIABLES",
     namespace.tags: "TAGS",
     "experiment_chain": "EXPERIMENT_CHAIN",
+    "success_criteria": "SUCCESS_CRITERIA",
 }
 
 
@@ -56,6 +57,7 @@ def __init__(self, app_inst):
         self.experiment_chain = app_inst.chain_order.copy()
         self.tags = list(app_inst.experiment_tags)
         self.contexts = []
+        self.success_criteria = {}
         self.software = {}
 
         self.keys = {}

lib/ramble/ramble/test/success_criteria_functionality/success_summary.py

@@ -0,0 +1,73 @@
+# Copyright 2022-2025 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+
+import pytest
+
+import ramble.workspace
+from ramble.main import RambleCommand
+
+# everything here uses the mock_workspace_path
+pytestmark = pytest.mark.usefixtures("mutable_config", "mutable_mock_workspace_path")
+
+workspace = RambleCommand("workspace")
+ramble_on = RambleCommand("on")
+
+
+def test_success_criteria_summary(mock_applications, workspace_name):
+    test_config = """
+ramble:
+  variables:
+    mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}'
+    batch_submit: '{execute_experiment}'
+    processes_per_node: '1'
+    n_threads: '1'
+  applications:
+    basic:
+      workloads:
+        working_wl:
+          experiments:
+            pass-experiment:
+              variables:
+                n_nodes: 1
+              success_criteria:
+              - name: always-pass
+                mode: string
+                match: '.*seconds.*'
+            fail-experiment:
+              variables:
+                n_nodes: 1
+              success_criteria:
+              - name: always-fail
+                mode: string
+                match: 'Blarg'
+  software:
+    packages: {}
+    environments: {}
+"""
+    with ramble.workspace.create(workspace_name) as ws:
+        ws.write()
+
+        config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name)
+
+        with open(config_path, "w+") as f:
+            f.write(test_config)
+        ws._re_read()
+
+        workspace("setup", global_args=["-w", workspace_name])
+        ramble_on(global_args=["-w", workspace_name])
+        workspace("analyze", "-f", "text", "json", "yaml", global_args=["-w", workspace_name])
+
+        for extension in ["txt", "json", "yaml"]:
+            with open(os.path.join(ws.root, "results.latest.txt")) as f:
+                data = f.read()
+                assert "Success criteria summary:" in data
+                assert "_application_function = PASSED" in data
+                assert "always-pass = PASSED" in data
+                assert "always-fail = FAILED" in data

lib/ramble/ramble/workspace/workspace.py

@@ -1662,6 +1662,10 @@ def dump_results(self, output_formats=None, print_results=False, summary_only=Fa
                             if software_key in exp and exp[software_key]:
                                 self.write_software_info(f, exp)
 
+                        if exp["SUCCESS_CRITERIA"]:
+                            f.write("  Success criteria summary:\n")
+                            for name, result in exp["SUCCESS_CRITERIA"].items():
+                                f.write(f"    {name} = {result}\n")
                 else:
                     logger.msg("No results to write")
 

var/ramble/repos/builtin/base_classes/application-base/base_class.py

@@ -2390,6 +2390,12 @@ def format_context(context_match, context_format):
 
         self._init_result()
 
+        for criteria_obj in criteria_list.all_criteria():
+            if criteria_obj.ok():
+                self.result.success_criteria[criteria_obj.name] = "PASSED"
+            else:
+                self.result.success_criteria[criteria_obj.name] = "FAILED"
+
         for context, fom_map in fom_values.items():
             context_map = {
                 "name": context,