add ephemeral_storage to resources

Author: nick-macro

metaflow/plugins/aws/batch/batch_decorator.py

@@ -86,7 +86,8 @@ class BatchDecorator(StepDecorator):
         Number of elastic fabric adapter network devices to attach to container
     ephemeral_storage : int, default None
         The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
-        This is only relevant for Fargate compute environments
+        This is only relevant for Fargate compute environments.
+        If `@resources` is also present, the maximum value from all decorators is used.
     log_driver: str, optional, default None
         The log driver to use for the Amazon ECS container.
     log_options: List[str], optional, default None

metaflow/plugins/resources_decorator.py

@@ -32,6 +32,9 @@ class ResourcesDecorator(StepDecorator):
     shared_memory : int, optional, default None
         The value for the size (in MiB) of the /dev/shm volume for this step.
         This parameter maps to the `--shm-size` option in Docker.
+    ephemeral_storage : int, default None
+        The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
+        This is only relevant for Fargate compute environments.
     """
 
     name = "resources"
@@ -41,4 +44,5 @@ class ResourcesDecorator(StepDecorator):
         "disk": None,
         "memory": "4096",
         "shared_memory": None,
+        "ephemeral_storage": None,
     }