Allow Modal secret name to be specified in config (#793)

Author: tomwhite

cubed/runtime/executors/modal.py

@@ -80,12 +80,15 @@
     else:
         raise ValueError(f"Unrecognized cloud: {cloud}")
 
-if cloud == "aws":
-    secrets = [modal.Secret.from_name("my-aws-secret")]
-elif cloud == "gcp":
-    secrets = [modal.Secret.from_name("my-googlecloud-secret")]
-else:
-    raise ValueError(f"Unrecognized cloud: {cloud}")
+secret_name = executor_options.get("secret", None)
+if secret_name is None:
+    if cloud == "aws":
+        secret_name = "my-aws-secret"
+    elif secret_name == "gcp":
+        secrets_name = "my-googlecloud-secret"
+    else:
+        raise ValueError(f"Unrecognized cloud: {cloud}")
+secrets = [modal.Secret.from_name(secret_name)]
 
 
 def check_runtime_memory(spec):
@@ -155,7 +158,15 @@ def execute_dag(
     ) -> None:
         merged_kwargs = {**self.kwargs, **kwargs}
         # remove executor options as they should already have been used in defining the remote functions
-        for executor_option in ("memory", "retries", "timeout", "cloud", "region"):
+        for executor_option in (
+            "memory",
+            "retries",
+            "timeout",
+            "cloud",
+            "region",
+            "requirements_file",
+            "secret",
+        ):
             merged_kwargs.pop(executor_option, None)
         asyncio_run(
             self._async_execute_dag(

docs/configuration.md

@@ -212,6 +212,7 @@ Note that `batch_size` is not currently supported for Lithops.
 |------------------------------|---------|-------------------------------------------------------------------------------------|
 | `cloud`                      | `"aws"` | The cloud to run on. One of `"aws"` or `"gcp"`.                                     |
 | `region`                     | N/A     | The cloud region to run in. This must be set to match the region of your cloud store to avoid data transfer fees. See Modal's [Region selection](https://modal.com/docs/guide/region-selection) page for possible values. |
+| `secret`                     | `"my-aws-secret"` for AWS, `"my-googlecloud-secret"` for Google Cloud | The name of the [Modal secret](https://modal.com/docs/guide/secrets) to use.        |
 | `retries`                    | 2       | The number of times to retry a task if it fails.                                    |
 | `timeout`                    | 180     | Tasks that take longer than the timeout will be automatically killed and retried.   |
 | `enable_output`              | False   | Print Modal output to stdout and stderr things for debugging.                      |