Set phase earlier; minor other fixes

Author: romain-intel

metaflow/cli.py

@@ -475,6 +475,11 @@ def start(
         # be raised. For resume, since we ignore those options, we ignore the error.
         raise ctx.obj.delayed_config_exception
 
+    # Initialize the phase early so it can be used in the mutators
+    # The phase is determined by which CLI subcommand is being invoked (e.g. "run" → LAUNCH,
+    # "step" → TASK, "batch" → TRAMPOLINE).
+    system_context._update(phase=_phase_from_cli_args(getattr(ctx, "saved_args", None)))
+
     # Init all values in the flow mutators and then process them
     for decorator in ctx.obj.flow._flow_mutators:
         decorator.external_init()
@@ -564,13 +569,7 @@ def start(
 
     decorators._init(ctx.obj.flow)
 
-    # Populate the system context singleton for this process. The phase is
-    # determined by which CLI subcommand is being invoked (e.g. "run" → LAUNCH,
-    # "step" → TASK, "batch" → TRAMPOLINE).
-    saved_args = getattr(ctx, "saved_args", None)
-    phase = _phase_from_cli_args(saved_args)
     system_context._update(
-        phase=phase,
         flow=ctx.obj.flow,
         graph=ctx.obj.graph,
         environment=ctx.obj.environment,

metaflow/debug.py

@@ -22,16 +22,33 @@ class Debug(object):
     def __init__(self):
         import metaflow.metaflow_config as config
 
+        have_debug_options = False
         for typ in config.DEBUG_OPTIONS:
             if getattr(config, "DEBUG_%s" % typ.upper()):
                 op = partial(self.log, typ)
+                have_debug_options = True
             else:
                 op = self.noop
             # use debug.$type_exec(args) to log command line for subprocesses
             # of type $type
             setattr(self, "%s_exec" % typ, op)
             # use the debug.$type flag to check if logging is enabled for $type
             setattr(self, typ, op != self.noop)
+        # In some environments (I'm looking at you Bazel), the path to the filename is
+        # super long and not very useful. We will print it once and truncate the rest.
+        # This is not 100% accurate as each package is in a separate directory so the
+        # prefix length may be a bit different but it's good enough and removes a lot
+        # of noise while also keeping the cost low (instead of having to figure out
+        # the prefix for each package)
+        self.prefix_len = 0
+        if have_debug_options:
+            # Figure out the name of the current file and strip out everything before
+            #
+            self.prefix_len = len(inspect.stack()[0][1][: -len("metaflow/debug.py")])
+            self.log(
+                "options",
+                "File prefix is: %s" % inspect.stack()[0][1][: self.prefix_len],
+            )
 
     def log(self, typ, args):
         if is_stringish(args):
@@ -40,7 +57,10 @@ def log(self, typ, args):
             s = " ".join(args)
         lineno = inspect.currentframe().f_back.f_lineno
         filename = inspect.stack()[1][1]
-        print("debug[%s %s:%s]: %s" % (typ, filename, lineno, s), file=sys.stderr)
+        print(
+            "debug[%s %s:%s]: %s" % (typ, filename[self.prefix_len :], lineno, s),
+            file=sys.stderr,
+        )
 
     def __getattr__(self, name):
         # Small piece of code to get pyright and other linters to recognize that there