Skip to content

Commit 49983ff

Browse files
madhur-obmadhur-ob
committed
tests using lower level chaining API
1 parent 2f0ab07 commit 49983ff

File tree

2 files changed

+118
-28
lines changed

2 files changed

+118
-28
lines changed

‎test/core/contexts.json

+24-24
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@
1919
"--quiet"
2020
],
2121
"run_options": [
22-
"--max-workers", "50",
23-
"--max-num-splits", "10000",
24-
"--tag", "\u523a\u8eab means sashimi",
25-
"--tag", "multiple tags should be ok"
22+
"--max-workers=50",
23+
"--max-num-splits=10000",
24+
"--tag=\u523a\u8eab means sashimi",
25+
"--tag=multiple tags should be ok"
2626
],
2727
"checks": [ "python3-cli", "python3-metadata"],
2828
"disabled_tests": [
@@ -51,10 +51,10 @@
5151
"--quiet"
5252
],
5353
"run_options": [
54-
"--max-workers", "50",
55-
"--max-num-splits", "10000",
56-
"--tag", "\u523a\u8eab means sashimi",
57-
"--tag", "multiple tags should be ok"
54+
"--max-workers=50",
55+
"--max-num-splits=10000",
56+
"--tag=\u523a\u8eab means sashimi",
57+
"--tag=multiple tags should be ok"
5858
],
5959
"checks": [ "python3-cli", "python3-metadata"],
6060
"enabled_tests": [
@@ -81,10 +81,10 @@
8181
"--quiet"
8282
],
8383
"run_options": [
84-
"--max-workers", "50",
85-
"--max-num-splits", "10000",
86-
"--tag", "\u523a\u8eab means sashimi",
87-
"--tag", "multiple tags should be ok"
84+
"--max-workers=50",
85+
"--max-num-splits=10000",
86+
"--tag=\u523a\u8eab means sashimi",
87+
"--tag=multiple tags should be ok"
8888
],
8989
"checks": [ "python3-cli", "python3-metadata"],
9090
"disabled_tests": [
@@ -111,10 +111,10 @@
111111
"--quiet"
112112
],
113113
"run_options": [
114-
"--max-workers", "50",
115-
"--max-num-splits", "10000",
116-
"--tag", "\u523a\u8eab means sashimi",
117-
"--tag", "multiple tags should be ok"
114+
"--max-workers=50",
115+
"--max-num-splits=10000",
116+
"--tag=\u523a\u8eab means sashimi",
117+
"--tag=multiple tags should be ok"
118118
],
119119
"checks": ["python3-cli", "python3-metadata"],
120120
"disabled_tests": [
@@ -140,10 +140,10 @@
140140
"METAFLOW_DEFAULT_METADATA": "service"
141141
},
142142
"run_options": [
143-
"--max-workers", "50",
144-
"--max-num-splits", "10000",
145-
"--tag", "\u523a\u8eab means sashimi",
146-
"--tag", "multiple tags should be ok"
143+
"--max-workers=50",
144+
"--max-num-splits=10000",
145+
"--tag=\u523a\u8eab means sashimi",
146+
"--tag=multiple tags should be ok"
147147
],
148148
"checks": ["python3-cli", "python3-metadata"],
149149
"disabled_tests": [
@@ -177,10 +177,10 @@
177177
"METAFLOW_DEFAULT_METADATA": "service"
178178
},
179179
"run_options": [
180-
"--max-workers", "50",
181-
"--max-num-splits", "10000",
182-
"--tag", "\u523a\u8eab means sashimi",
183-
"--tag", "multiple tags should be ok"
180+
"--max-workers=50",
181+
"--max-num-splits=10000",
182+
"--tag=\u523a\u8eab means sashimi",
183+
"--tag=multiple tags should be ok"
184184
],
185185
"checks": ["python3-cli", "python3-metadata"],
186186
"disabled_tests": [

‎test/core/run_tests.py

+94-4
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
import subprocess
1010
from multiprocessing import Pool
1111

12+
from metaflow.cli import start, run
1213
from metaflow._vendor import click
14+
from metaflow.click_api import MetaflowAPI, extract_all_params, click_to_python_types
1315

1416
from metaflow_test import MetaflowTest
1517
from metaflow_test.formatter import FlowFormatter
@@ -61,14 +63,72 @@ def log(msg, formatter=None, context=None, real_bad=False, real_good=False):
6163
click.echo("[pid %s] %s" % (pid, line))
6264

6365

64-
def run_test(formatter, context, debug, checks, env_base):
66+
def run_test(formatter, context, debug, checks, env_base, use_chaining_api=False):
6567
def run_cmd(mode):
6668
cmd = [context["python"], "-B", "test_flow.py"]
6769
cmd.extend(context["top_options"])
6870
cmd.extend((mode, "--run-id-file", "run-id"))
6971
cmd.extend(context["run_options"])
7072
return cmd
7173

74+
def construct_arg_dict(params_opts, cli_options):
75+
result_dict = {}
76+
has_value = False
77+
secondary_supplied = False
78+
79+
for arg in cli_options:
80+
if "=" in arg:
81+
given_opt, val = arg.split("=")
82+
has_value = True
83+
else:
84+
given_opt = arg
85+
86+
for key, each_param in params_opts.items():
87+
py_type = click_to_python_types[type(each_param.type)]
88+
if given_opt in each_param.opts:
89+
secondary_supplied = False
90+
elif given_opt in each_param.secondary_opts:
91+
secondary_supplied = True
92+
else:
93+
continue
94+
95+
if has_value:
96+
value = val
97+
else:
98+
if secondary_supplied:
99+
value = not each_param.default
100+
else:
101+
value = each_param.default
102+
103+
if each_param.multiple:
104+
if key not in result_dict:
105+
result_dict[key] = [py_type(value)]
106+
else:
107+
result_dict[key].append(py_type(value))
108+
else:
109+
result_dict[key] = py_type(value)
110+
111+
has_value = False
112+
secondary_supplied = False
113+
114+
return result_dict
115+
116+
def construct_cmd_from_click_api(mode):
117+
api = MetaflowAPI.from_cli("test_flow.py", start)
118+
_, _, param_opts, _, _ = extract_all_params(start)
119+
top_level_options = context["top_options"]
120+
top_level_dict = construct_arg_dict(param_opts, top_level_options)
121+
122+
_, _, param_opts, _, _ = extract_all_params(run)
123+
run_level_options = context["run_options"]
124+
run_level_dict = construct_arg_dict(param_opts, run_level_options)
125+
run_level_dict["run_id_file"] = "run-id"
126+
127+
cmd = getattr(api(**top_level_dict), mode)(**run_level_dict)
128+
command = [context["python"], "-B"]
129+
command.extend(cmd)
130+
return command
131+
72132
cwd = os.getcwd()
73133
tempdir = tempfile.mkdtemp("_metaflow_test")
74134
package = os.path.dirname(os.path.abspath(__file__))
@@ -123,7 +183,10 @@ def run_cmd(mode):
123183
return pre_ret, path
124184

125185
# run flow
126-
flow_ret = subprocess.call(run_cmd("run"), env=env)
186+
if use_chaining_api:
187+
flow_ret = subprocess.call(construct_cmd_from_click_api("run"), env=env)
188+
else:
189+
flow_ret = subprocess.call(run_cmd("run"), env=env)
127190
if flow_ret:
128191
if formatter.should_fail:
129192
log("Flow failed as expected.")
@@ -242,6 +305,33 @@ def run_test_cases(args):
242305
return failed
243306
else:
244307
log("success", formatter, context, real_good=True)
308+
309+
log("running [with chaining api]", formatter, context)
310+
ret, path = run_test(
311+
formatter,
312+
context,
313+
debug,
314+
contexts["checks"],
315+
base_env,
316+
use_chaining_api=True,
317+
)
318+
319+
if ret:
320+
tstid = "%s in context %s [with chaining api]" % (
321+
formatter,
322+
context["name"],
323+
)
324+
failed.append((tstid, path))
325+
log("failed [with chaining api]", formatter, context, real_bad=True)
326+
if debug:
327+
return failed
328+
else:
329+
log(
330+
"success [with chaining api]",
331+
formatter,
332+
context,
333+
real_good=True,
334+
)
245335
else:
246336
log("not a valid combination. Skipped.", formatter)
247337
return failed
@@ -258,13 +348,13 @@ def run_test_cases(args):
258348
"--tests",
259349
default="",
260350
type=str,
261-
help="A comma-separate list of graphs to include (default: all).",
351+
help="A comma-separated list of tests to include (default: all).",
262352
)
263353
@click.option(
264354
"--graphs",
265355
default="",
266356
type=str,
267-
help="A comma-separate list of graphs to include (default: all).",
357+
help="A comma-separated list of graphs to include (default: all).",
268358
)
269359
@click.option(
270360
"--debug",

0 commit comments

Comments
 (0)