Separate tests for examples & advanced_examples

Author: kostis-init

tests/test_examples.py

@@ -17,37 +17,39 @@
 from cpmpy.exceptions import NotSupportedError, TransformationNotImplementedError
 import itertools
 
-cwd = getcwd()
-if 'y' in cwd[-2:]:
-    EXAMPLES =  glob(join(".", "examples", "*.py")) + \
-                glob(join(".", "examples", "advanced", "*.py")) + \
-                glob(join(".", "examples", "csplib", "*.py"))
-else:
-    EXAMPLES = glob(join("..", "examples", "*.py")) + \
-               glob(join("..", "examples", "advanced", "*.py")) + \
-               glob(join("..", "examples", "csplib", "*.py"))
+prefix = '.' if 'y' in getcwd()[-2:] else '..'
+
+TO_SKIP = [
+    "prob001_convert_data.py"
+]
+
+EXAMPLES = glob(join(prefix, "examples", "*.py")) + \
+           glob(join(prefix, "examples", "csplib", "*.py"))
+EXAMPLES = [e for e in EXAMPLES if not any(x in e for x in TO_SKIP)]
+
+ADVANCED_EXAMPLES = glob(join(prefix, "examples", "advanced", "*.py"))
 
 # SOLVERS = SolverLookup.supported()
 SOLVERS = [
-        "ortools",
-        "gurobi",
-        "minizinc",
-        ]
+    "ortools",
+    "gurobi",
+    "minizinc",
+]
 
-@pytest.mark.parametrize(("solver", "example"), itertools.product(SOLVERS, EXAMPLES))
-@pytest.mark.timeout(60)  # some examples take long, use 10 second timeout
-def test_examples(solver, example):
-    """Loads example files and executes with default solver
+@pytest.mark.parametrize(("solver", "example"), itertools.product(SOLVERS, EXAMPLES))  # run the test for each combination of solver and example
+@pytest.mark.timeout(60)  # 60-second timeout for each test
+def test_example(solver, example):
+    """Loads the example file and executes its __main__ block with the given solver being set as default.
 
     Args:
         solver ([string]): Loaded with parametrized solver name
         example ([string]): Loaded with parametrized example filename
     """
-    if solver in ('gurobi', 'minizinc') and any(x in example for x in ["npuzzle", "tst_likevrp", "ortools_presolve_propagate", 'sudoku_ratrun1.py']):
-        return pytest.skip(reason=f"exclude {example} for gurobi, too slow or solver specific")
+    if solver in ('gurobi', 'minizinc') and any(x in example for x in ["npuzzle.py", "tst_likevrp.py", 'sudoku_', 'pareto_optimal.py', 'prob009_perfect_squares.py', 'blocks_world.py', 'flexible_jobshop.py']):
+        return pytest.skip(reason=f"exclude {example} for {solver}, too slow or solver-specific")
 
+    base_solvers = SolverLookup.base_solvers
     try:
-        base_solvers = SolverLookup.base_solvers
         solver_class = SolverLookup.lookup(solver)
         if not solver_class.supported():
             # check this here, as unsupported solvers can fail the example for various reasons
@@ -59,8 +61,10 @@ def test_examples(solver, example):
         runpy.run_path(example, run_name="__main__")  # many examples won't do anything `__name__ != "__main__"`
     except (NotSupportedError, TransformationNotImplementedError) as e:
         if solver == 'ortools':  # `from` augments exception trace
-            raise Exception("Example not supported by ortools, which is currently able to run all models, but raised") from e
-        pytest.skip(reason=f"Skipped, solver or its transformation does not support model, raised {type(e).__name__}: {e}")
+            raise Exception(
+                "Example not supported by ortools, which is currently able to run all models, but raised") from e
+        pytest.skip(
+            reason=f"Skipped, solver or its transformation does not support model, raised {type(e).__name__}: {e}")
     except ValueError as e:
         if "Unknown solver" in str(e):
             pytest.skip(reason=f"Skipped, example uses specific solver, raised: {e}")
@@ -71,3 +75,10 @@ def test_examples(solver, example):
     finally:
         SolverLookup.base_solvers = base_solvers
 
+
+@pytest.mark.parametrize("example", ADVANCED_EXAMPLES)
+@pytest.mark.timeout(10)
+def test_advanced_example(example):
+    """Loads the advanced example file and executes its __main__ block with no default solver set."""
+    sys.argv = [example]
+    runpy.run_path(example, run_name="__main__")