Merge pull request #86 from sandialabs/dev-weh

Updates to benchmarking logic

Author: whart222

Diff for: test/aml_comparisons/coek/solve0.cpp

@@ -61,7 +61,7 @@ int main(int argc, char* argv[])
         solver.set_option("TimeLimit", 0.0);
 
         auto res = solver.solve(model);
-        if (not check_optimal_termination(res)) {
+        if (res->termination_condition != coek::TerminationCondition::time_limit) {
             std::cout << "ERROR - failed to execute solver '" << solver_name << "'" << std::endl;
             std::cout << "MESSAGE - " << res->error_message << std::endl;
             return 2;
@@ -76,7 +76,7 @@ int main(int argc, char* argv[])
         solver.set_option("print_level", 1);
 
         auto res = solver.solve(Model);
-        if (not check_optimal_termination(res)) {
+        if (res->termination_condition != coek::TerminationCondition::time_limit) {
             std::cout << "ERROR - failed to execute solver '" << solver_name << "'" << std::endl;
             std::cout << "MESSAGE - " << res->error_message << std::endl;
             return 2;

Diff for: test/aml_comparisons/scripts/bench_gurobi.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Benchmark comparison
+#
+# This script should be run from the top-level _build directory
+#
+
+if [[ -d /coek_performance_results ]]; then
+    echo $CI_COMMIT_SHA
+    echo $CI_COMMIT_TITLE
+fi
+
+\rm -Rf results
+../test/aml_comparisons/scripts/run bench_coek 3
+../test/aml_comparisons/scripts/collect
+../test/aml_comparisons/scripts/dog solve0
+../test/aml_comparisons/scripts/dog writer
+
+if [[ -d /coek_performance_results ]]; then
+    CI_COMMIT_TITLE_MOD="${CI_COMMIT_TITLE// /_}"
+    python ../test/aml_comparisons/scripts/to_csv.py --dirname /coek_performance_results/${BRANCH_NAME} --branch_name ${BRANCH_NAME} --ci_commit_sha    ${CI_COMMIT_SHA} --ci_commit_title ${CI_COMMIT_TITLE_MOD}
+    python ../test/aml_comparisons/scripts/compare.py --artifact_dir=/coek_performance_results --branch_name=${BRANCH_NAME}
+fi
+

Diff for: test/aml_comparisons/scripts/dog

@@ -398,6 +398,9 @@ def collect_data(directory):
         collect_solve0_summary(directory)
 
 if __name__ == "__main__":
+    print("")
+    print("DOG", " ".join(sys.argv))
+    print("")
     directory = "results"
 
     parser = argparse.ArgumentParser()

Diff for: test/aml_comparisons/scripts/run

@@ -124,6 +124,7 @@ def test_writer(
     if not os.path.exists("results/writer/" + subdir):
         os.mkdir("results/writer/" + subdir)
 
+    print("")
     print("")
     print("TESTING ARGUMENTS")
     print("  test_type:  " + test_type)
@@ -133,7 +134,6 @@ def test_writer(
     print("  subdir:     " + subdir)
     print("  timeout:    " + str(timeout))
     print("  ntrials:    " + str(ntrials))
-    print("")
 
     for model in models:
         config_name = None
@@ -208,8 +208,8 @@ def test_writer(
                             )
                         else:
                             print(
-                                ". ERROR {}: rc={} output='{}'".format(
-                                    timefile, p.returncode, results
+                                ". ERROR (test_writer) {}:\n\tcmd={} rc={} output='{}'".format(
+                                    timefile, " ".join(run), p.returncode, results
                                 )
                             )
                             if os.path.exists(timefile):
@@ -218,7 +218,7 @@ def test_writer(
                     except subprocess.TimeoutExpired:
                         os.killpg(os.getpgid(p.pid), signal.SIGTERM)
                         errors.append(timefile)
-                        print(". TIMEOUT {} cmd={}".format(timefile, " ".join(run)))
+                        print(". TIMEOUT (test_writer) {} cmd={}".format(timefile, " ".join(run)))
                         with open(timefile, "w") as OUTPUT:
                             OUTPUT.write("timeout {}".format(timeout))
 
@@ -241,6 +241,7 @@ def test_solve0(
     if not os.path.exists("results/solve0/" + subdir):
         os.mkdir("results/solve0/" + subdir)
 
+    print("")
     print("")
     print("TESTING ARGUMENTS")
     print("  test_type:  " + test_type)
@@ -250,7 +251,6 @@ def test_solve0(
     print("  subdir:     " + subdir)
     print("  timeout:    " + str(timeout))
     print("  ntrials:    " + str(ntrials))
-    print("")
 
     for model in models:
         config_name = None
@@ -313,14 +313,14 @@ def test_solve0(
                         if p.returncode == 0:
                             print(". OK {}: rc={}".format(timefile, p.returncode))
                         else:
-                            print(". ERROR {}: rc={}".format(timefile, p.returncode))
+                            print(". ERROR (test_solve0) {}:\n\tcmd={} rc={}".format(timefile, " ".join(run), p.returncode))
                             if os.path.exists(timefile):
                                 os.remove(timefile)
                             errors.append(timefile)
                     except subprocess.TimeoutExpired:
                         os.killpg(os.getpgid(p.pid), signal.SIGTERM)
                         errors.append(timefile)
-                        print(". TIMEOUT {} cmd={}".format(timefile, " ".join(run)))
+                        print(". TIMEOUT (test_solve0) {} cmd={}".format(timefile, " ".join(run)))
                         with open(timefile, "w") as OUTPUT:
                             OUTPUT.write("timeout {}".format(timeout))
 
@@ -337,22 +337,10 @@ if __name__ == "__main__":
     else:
         ntrials = 1
 
-    build_dir = None
-    if os.path.exists("coek/coek_writer"):
-        build_dir = "."
-    elif os.path.exists(
-        os.path.join(currdir, "../../../build/test/aml_comparisons/coek/coek_writer")
-    ):
-        build_dir = os.path.join(currdir, "../../../build")
-    if build_dir is None:
-        print("Cannot find Coek build directory")
-        print(
-            os.path.join(
-                currdir, "../../../build/test/aml_comparisons/coek/coek_writer"
-            )
-        )
+    if not os.path.exists("test/aml_comparisons/coek/coek_writer"):
+        print("Must execute 'run' in the coek build directory")
         sys.exit(1)
-    build_dir = os.path.abspath(build_dir)
+    build_dir = os.path.abspath(".")
 
     errors = []
 
@@ -366,6 +354,15 @@ if __name__ == "__main__":
             ntrials=ntrials,
             errors=errors,
         )
+        test_solve0(
+            test_type="small",
+            models=all_gurobi_models,
+            solvers=["gurobi"],
+            executable=os.path.join(build_dir, "test/aml_comparisons/gurobi/gurobi_solve0"),
+            subdir="gurobi",
+            ntrials=ntrials,
+            errors=errors,
+        )
 
     elif sys.argv[1] == "bench_coek":
         test_solve0(