new step for tests using chaining API in github actions

Author: madhur-ob

.github/workflows/test.yml

@@ -43,6 +43,9 @@ jobs:
     - name: Execute Python tests
       run: tox
 
+    - name: Execute Python tests with low level chaining API
+      run: tox -e chaining
+
   R:
     name: core / R ${{ matrix.ver }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}

test/core/run_tests.py

@@ -230,7 +230,15 @@ def construct_cmd_from_click_api(mode):
             shutil.rmtree(tempdir)
 
 
-def run_all(ok_tests, ok_contexts, ok_graphs, debug, num_parallel, inherit_env):
+def run_all(
+    ok_tests,
+    ok_contexts,
+    ok_graphs,
+    debug,
+    num_parallel,
+    inherit_env,
+    use_chaining_api=False,
+):
 
     tests = [
         test
@@ -247,17 +255,22 @@ def run_all(ok_tests, ok_contexts, ok_graphs, debug, num_parallel, inherit_env):
     if debug or num_parallel is None:
         for test in tests:
             failed.extend(
-                run_test_cases((test, ok_contexts, ok_graphs, debug, base_env))
+                run_test_cases(
+                    (test, ok_contexts, ok_graphs, debug, base_env, use_chaining_api)
+                )
             )
     else:
-        args = [(test, ok_contexts, ok_graphs, debug, base_env) for test in tests]
+        args = [
+            (test, ok_contexts, ok_graphs, debug, base_env, use_chaining_api)
+            for test in tests
+        ]
         for fail in Pool(num_parallel).imap_unordered(run_test_cases, args):
             failed.extend(fail)
     return failed
 
 
 def run_test_cases(args):
-    test, ok_contexts, ok_graphs, debug, base_env = args
+    test, ok_contexts, ok_graphs, debug, base_env, use_chaining_api = args
     contexts = json.load(open("contexts.json"))
     graphs = list(iter_graphs())
     test_name = test.__class__.__name__
@@ -288,46 +301,40 @@ def run_test_cases(args):
                 if enabled_tests and (test_name not in map(str, enabled_tests)):
                     continue
 
-                log("running", formatter, context)
-                ret, path = run_test(
+                log(
+                    "running [using chaining api]" if use_chaining_api else "running",
                     formatter,
                     context,
-                    debug,
-                    contexts["checks"],
-                    base_env,
                 )
-
-                if ret:
-                    tstid = "%s in context %s" % (formatter, context["name"])
-                    failed.append((tstid, path))
-                    log("failed", formatter, context, real_bad=True)
-                    if debug:
-                        return failed
-                else:
-                    log("success", formatter, context, real_good=True)
-
-                log("running [with chaining api]", formatter, context)
                 ret, path = run_test(
                     formatter,
                     context,
                     debug,
                     contexts["checks"],
                     base_env,
-                    use_chaining_api=True,
+                    use_chaining_api,
                 )
 
                 if ret:
-                    tstid = "%s in context %s [with chaining api]" % (
-                        formatter,
-                        context["name"],
+                    tstid = (
+                        "%s in context %s [using chaining api]"
+                        if use_chaining_api
+                        else "%s in context %s" % (formatter, context["name"])
                     )
                     failed.append((tstid, path))
-                    log("failed [with chaining api]", formatter, context, real_bad=True)
+                    log(
+                        "failed [using chaining api]" if use_chaining_api else "failed",
+                        formatter,
+                        context,
+                        real_bad=True,
+                    )
                     if debug:
                         return failed
                 else:
                     log(
-                        "success [with chaining api]",
+                        "success [using chaining api]"
+                        if use_chaining_api
+                        else "success",
                         formatter,
                         context,
                         real_good=True,
@@ -365,6 +372,12 @@ def run_test_cases(args):
 @click.option(
     "--inherit-env", is_flag=True, default=False, help="Inherit env variables"
 )
+@click.option(
+    "--use-chaining-api",
+    is_flag=True,
+    default=False,
+    help="Use low-level chaining API to generate commands.",
+)
 @click.option(
     "--num-parallel",
     show_default=True,
@@ -379,6 +392,7 @@ def cli(
     num_parallel=None,
     debug=False,
     inherit_env=False,
+    use_chaining_api=False,
 ):
 
     parse = lambda x: {t.lower() for t in x.split(",") if t}
@@ -390,6 +404,7 @@ def cli(
         debug,
         num_parallel,
         inherit_env,
+        use_chaining_api,
     )
 
     if failed:

test_runner

@@ -16,15 +16,20 @@ install_extensions() {
 }
 
 run_tests() {
-  cd test/core && PYTHONPATH=`pwd`/../../ python3 run_tests.py --num-parallel 8 && cd ../../
+  cd test/core && PYTHONPATH=`pwd`/../../ python3 run_tests.py --num-parallel 8 "$@" && cd ../../
 }
 
 # We run realtime cards tests separately because there these tests validate the asynchronous updates to the
 # information stored in the datastore. So if there are other processes starving resources then these tests will
 # surely fail since a lot of checks have timeouts. 
 run_runtime_card_tests() {
   CARD_GRAPHS="small-foreach,small-parallel,nested-branches,single-linear-step,simple-foreach"
-  cd test/core && PYTHONPATH=`pwd`/../../ python3 run_tests.py --num-parallel 8 --contexts python3-all-local-cards-realtime --graphs $CARD_GRAPHS && cd ../../
+  cd test/core && PYTHONPATH=`pwd`/../../ python3 run_tests.py --num-parallel 8 --contexts python3-all-local-cards-realtime --graphs $CARD_GRAPHS "$@" && cd ../../
 }
 
-install_deps && install_extensions && run_tests && run_runtime_card_tests
+if [[ "$1" == "chain" ]]; then
+  shift
+  USE_CHAINING_API="--use-chaining-api"
+fi
+
+install_deps && install_extensions && run_tests "$USE_CHAINING_API" && run_runtime_card_tests "$USE_CHAINING_API"

tox.ini

@@ -3,3 +3,7 @@
 [testenv]
 allowlist_externals = ./test_runner
 commands = ./test_runner
+
+[testenv:chaining]
+allowlist_externals = ./test_runner
+commands = ./test_runner chain