Merge pull request #37 from MITLibraries/TIMX-356-limit-parallel-containers

Limit parallel containers of Transmogrifier

Author: ghukill

README.md

@@ -6,6 +6,15 @@ Compare transformed TIMDEX records from two versions (A,B) of Transmogrifier.
 
 `abdiff` is the name of the CLI application in this repository that performs an A/B test of Transmogrifier.
 
+## Development
+
+- To preview a list of available Makefile commands: `make help`
+- To install with dev dependencies: `make install`
+- To update dependencies: `make update`
+- To run unit tests: `make test`
+- To lint the repo: `make lint`
+- To run the app: `pipenv run abdiff --help`
+
 ## Concepts
 
 A **Job** in `abdiff` represents the A/B test for comparing the results from two versions of Transmogrifier.  When a job is first created, a working directory and a JSON file `job.json` with an initial set of configurations is created.
@@ -82,7 +91,9 @@ AWS_SESSION_TOKEN=# passed to Transmogrifier containers for use
 
 ```text
 WEBAPP_HOST=# host for flask webapp
-WEBAPP_PORT# port for flask webapp
+WEBAPP_PORT=# port for flask webapp
+TRANSMOGRIFIER_MAX_WORKERS=# max number of Transmogrifier containers to run in parallel; default is 6
+TRANSMOGRIFIER_TIMEOUT=# timeout for a single Transmogrifier container; default is 5 hours
 ```
 
 ## CLI commands
@@ -147,28 +158,3 @@ Options:
   -h, --help                Show this message and exit.
 ```
 
-## Development
-
-- To preview a list of available Makefile commands: `make help`
-- To install with dev dependencies: `make install`
-- To update dependencies: `make update`
-- To run unit tests: `make test`
-- To lint the repo: `make lint`
-- To run the app: `pipenv run abdiff --help`
-
-## Environment Variables
-
-### Required
-
-```shell
-WORKSPACE=### Set to `dev` for local development, this will be set to `stage` and `prod` in those environments by Terraform.
-```
-
-### Optional
-
-```shell
-```
-
-
-
-

abdiff/config.py

@@ -13,6 +13,8 @@ class Config:
     OPTIONAL_ENV_VARS = (
         "WEBAPP_HOST",
         "WEBAPP_PORT",
+        "TRANSMOGRIFIER_MAX_WORKERS",
+        "TRANSMOGRIFIER_TIMEOUT",
     )
 
     def __getattr__(self, name: str) -> Any:  # noqa: ANN401
@@ -31,6 +33,18 @@ def webapp_port(self) -> int:
         port = self.WEBAPP_PORT or "5000"
         return int(port)
 
+    @property
+    def transmogrifier_max_workers(self) -> int:
+        """Maximum number of Transmogrifier containers to run in parallel."""
+        max_workers = self.TRANSMOGRIFIER_MAX_WORKERS or 6
+        return int(max_workers)
+
+    @property
+    def transmogrifier_timeout(self) -> int:
+        """Timeout for a single Transmogrifier container."""
+        timeout = self.TRANSMOGRIFIER_TIMEOUT or 60 * 60 * 5  # 5 hours default
+        return int(timeout)
+
 
 def configure_logger(logger: logging.Logger, *, verbose: bool) -> str:
     if verbose:

abdiff/core/exceptions.py

@@ -3,30 +3,25 @@ def __init__(self, run_id: str) -> None:
         super().__init__(f"No Docker containers were found with label run='{run_id}'.")
 
 
-class DockerContainerRuntimeExceededTimeoutError(Exception):
-    def __init__(self, containers: list, timeout: int) -> None:
-        self.containers = containers
+class DockerContainerTimeoutError(Exception):
+    def __init__(self, container_id: str | None, timeout: int) -> None:
+        self.container_id = container_id
         self.timeout = timeout
         super().__init__(self.get_formatted_message())
 
     def get_formatted_message(self) -> str:
-        container_ids = [container.id for container in self.containers]
-        return (
-            f"Timeout of {self.timeout} seconds exceeded."
-            f"{len(container_ids)} container(s) is/are still running:"
-            f"{container_ids}."
-        )
+        return f"Container {self.container_id} exceed timeout of {self.timeout} seconds."
 
 
 # core function errors
-class DockerContainerRunFailedError(Exception):
-    def __init__(self, containers: list) -> None:
-        self.containers = containers
+class DockerContainerRuntimeError(Exception):
+    def __init__(self, container_id: str) -> None:
+        self.container_id = container_id
         super().__init__(self.get_formatted_message())
 
     def get_formatted_message(self) -> str:
         return (
-            f"The following Docker containers exited with an error: {self.containers}. "
+            f"Container {self.container_id} did not exit cleanly. "
             "Check the logs in transformed/logs.txt to identify the error."
         )