refactor test scripts (#152)

Author: rafaelmds

.gitignore

@@ -8,6 +8,7 @@ docs/_build
 
 # Ignore output when run the test
 tests/**/output
+tests/test_summary.txt
 
 # Ignore output when run the examples
 examples/*/density_*

tests/run_tests.sh

@@ -1,8 +1,5 @@
 #!/usr/bin/env bash
 
-# exit when any command fails
-set -e
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 # scenarios to test
@@ -15,10 +12,13 @@ declare -a scenarios=(
     "two_layers_with_variable_conductivity"
 )
 
+# Delete old test summary
+rm -rf "${SCRIPT_DIR}/test_summary.txt"
+
 # Run scenarios
 for scenario in "${scenarios[@]}"
 do
-    echo -e "\n==> Running scenario: ${scenario} <===\n"
+    echo -e "\n==> Running simulation: ${scenario} <==\n"
 
     # Remove old output
     rm -rf "${SCRIPT_DIR}/data/${scenario}/output"
@@ -31,6 +31,7 @@ do
         MANDYOC=${MANDYOC} bash run.sh
 done
 
-# Run tests
+echo -e "\n==> Verifying results <==\n"
+
 cd "${SCRIPT_DIR}" && \
-    pytest -v testing_results.py
+    pytest -v testing_results.py --tb=line -rf | tee test_summary.txt

tests/testing_results.py

@@ -1,41 +1,57 @@
 """
 Compare the Mandyoc output data with the expected data
 """
-import os
 import pytest
 import numpy as np
 import numpy.testing as npt
 from pathlib import Path
 
-# Test path
-base_path = Path(os.path.realpath(os.path.abspath(__file__))).parent
 
-scenarios = [
-    "vanKeken1997_case1a",
-    "Crameri2012_case2",
-    "continental_rift",
-    "punch",
-    "Schmalholz_2011",
-    "two_layers_with_variable_conductivity",
-]
+BASE_PATH = Path(__file__).parent.absolute()
 
-# Name of the files to compare
-fields = [
-    "time",
+MAIN_FIELDS = [
     "density",
     "heat",
     "pressure",
+    "step",
     "strain",
     "strain_rate",
-    "surface",
     "temperature",
+    "time",
     "velocity",
     "viscosity",
-    "step",
 ]
 
-# steps to compare
-steps = [0, 1, 10000]
+
+# Definition of each scenario
+SCENARIOS = {
+    "vanKeken1997_case1a": {
+        "steps": [0, 1],
+        "fields": MAIN_FIELDS,
+    },
+    "Crameri2012_case2": {
+        "steps": [0],
+        "fields": MAIN_FIELDS,
+        "num_procs": 1,
+    },
+    "continental_rift": {
+        "steps": [0, 1],
+        "fields": MAIN_FIELDS,
+    },
+    "punch": {
+        "steps": [0],
+        "fields": MAIN_FIELDS,
+    },
+    "Schmalholz_2011": {
+        "steps": [0, 1],
+        "fields": MAIN_FIELDS,
+    },
+    "two_layers_with_variable_conductivity": {
+        "steps": [0, 10000],
+        "fields": MAIN_FIELDS,
+    },
+}
+
 
 def read(filename):
     """
@@ -49,37 +65,40 @@ def read(filename):
     return data
 
 
-@pytest.mark.parametrize("step", steps)
-@pytest.mark.parametrize("field", fields)
-@pytest.mark.parametrize("scenario", scenarios)
-def test_result(scenario, field, step):
-    """Run tests"""
+def get_test_cases():
+    """Generates valid test combinations."""
+    cases = []
+    for name, config in SCENARIOS.items():
+        for step in config["steps"]:
+            for field in config["fields"]:
+                cases.append((name, field, step))
+    return cases
+
 
-    if scenario != 'Crameri2012_case2' and field == 'surface':
-        pytest.skip('No surface for this scenario')
+@pytest.mark.parametrize("scenario, field, step", get_test_cases())
+def test_result(scenario, field, step):
+    config = SCENARIOS[scenario]
 
-    if scenario == 'Crameri2012_case2' and step == 1:
-        pytest.skip('Tested with only one processor')
+    test_path = BASE_PATH / "data" / scenario / "output"
+    expected_path = BASE_PATH / "data" / scenario / "expected"
 
-    if (scenario == 'punch' and step == 1) or (scenario == 'punch' and field == 'step_1'):
-        pytest.skip('Test with only one step')
+    if not test_path.exists():
+        pytest.fail(f"Simulation directory missing. Scenario {scenario} likely failed to run.")
 
-    if (scenario != 'two_layers_with_variable_conductivity' and step == 10000) or (scenario == 'two_layers_with_variable_conductivity' and step == 1):
-        pytest.skip('Test step/file not available for this scenario')
+    # Handle the 'step' field
+    if field == 'step':
+        num_procs = config.get("num_procs", 2)
+        filenames = [f"step_{step}_{i}.txt" for i in range(num_procs)]
+    else:
+        filenames = [f"{field}_{step}.txt"]
 
-    test_path = base_path / "data" / scenario/ "output"
-    expected_path = base_path / "data" / scenario/ "expected"
+    for fname in filenames:
+        out_file = test_path / fname
+        exp_file = expected_path / fname
 
-    filenames = [f"{field}_{step}.txt"]
+        assert out_file.exists(), f"Missing output file: {out_file}"
 
-    if field == 'step':
-        if scenario in ['Crameri2012_case2']:
-            filenames = [f"step_{step}_0.txt"]
-        else:
-            filenames = [f"step_{step}_0.txt", f"step_{step}_1.txt"]
-
-    for filename in filenames:
-        output = read(test_path / filename)
-        expected = read(expected_path / filename)
+        output = read(out_file)
+        expected = read(exp_file)
 
         npt.assert_allclose(output, expected, rtol=2e-4, atol=1.0e-18)