Release 0.5.0 (#205)

This is the PR for the release `0.5.0`

Author: sabinem

.env.dist.compose

@@ -17,6 +17,9 @@ ODTP_SECRET_KEY=
 # your github token
 GITHUB_TOKEN=
 
+# odtp password
+ODTP_PASSWORD=
+
 # mongodb user and password
 MONGO_DB_USER=
 MONGO_DB_PASSWORD=
@@ -40,8 +43,5 @@ ODTP_LOG_LEVEL=ERROR
 # Log Level when running executions
 RUN_LOG_LEVEL=INFO
 
-# App Path
-APP_PATH=
-
-# Install the package in editable mode.
-PIP_INSTALL_ARGS="--editable"
+# set to False if your docker installation does not allow the flag --gpus all
+ALLOW_DOCKER_GPUS=False

.env.dist.compose.dev

@@ -18,6 +18,9 @@ ODTP_SECRET_KEY=
 # your github token
 GITHUB_TOKEN=
 
+# odtp password
+ODTP_PASSWORD=
+
 # mongodb user and password
 MONGO_DB_USER=
 MONGO_DB_PASSWORD=
@@ -46,3 +49,6 @@ APP_PATH=
 
 # Install the package in editable mode.
 PIP_INSTALL_ARGS="--editable"
+
+# set to False if your docker installation does not allow the flag --gpus all
+ALLOW_DOCKER_GPUS=False

.env.dist.local

@@ -22,6 +22,9 @@ ODTP_SECRET_KEY=
 # your github token
 GITHUB_TOKEN=
 
+# odtp password
+ODTP_PASSWORD=
+
 # Dashboard parameters
 ODTP_DASHBOARD_PORT=
 ODTP_DASHBOARD_JSON_EDITOR=
@@ -34,3 +37,6 @@ ODTP_LOG_LEVEL=ERROR
 
 # Log Level when running executions
 RUN_LOG_LEVEL=INFO
+
+# set to False if your docker installation does not allow the flag --gpus all
+ALLOW_DOCKER_GPUS=False

CHANGELOG.md

@@ -1,20 +1,34 @@
 ## Changelog
 
+- v0.5.0
+    - gui and cli: making partial executions possible
+    - gui: allow to upload and store secret files per user. These files are then stored encrypted in the
+    users working directory and the user can link them to execution runs
+    - gui and cli: odtp.yml is now parsed when a component version is added and the default parameters are
+    stored. Therefore when an execution is created it is preset with the default parameter and port values.
+    These can be overwritten during execution runs.
+    - gui and cli: workflows are added as step between component versions and execution: a workflow is the
+    sequence of component versions. Workflows are not user owned and can be shared between users in the same
+    way as component versions are shared
+    - gui: gui allows now to deprecate workflows, executions and component versions. Deprecated items are
+    no longer offered in selects. Items can be deprecated and also reactivated
+    - env: an additional env variable is added ALLOW_DOCKER_GPUS that must be set to false if the used Docker
+    version does not allow GPU settings such as 
 - v0.4.0
     - gui: the general workflow for the user was improved (new Items are selected automatically for example)
     - gui: validation has been implemented in the forms to make sure data that is entered will be valid
     - gui: the run page for executions has been improved in the gui: logs are now also available from the gui when a workflow runs
     - gui: on the dashboard homepage you can now directly see whether all services including github are connected
     - gui: the workarea in the dashboard was removed and replace by an info section on top of the pages
-    - logging: improve logging at GUI/CLI: run commands are now logged to a file, while everything else is still logged to the command line: this is so run commands can be easier debugged by knowing what was run in docker and how that run was triggered 
+    - logging: improve logging at GUI/CLI: run commands are now logged to a file, while everything else is still logged to the command line: this is so run commands can be easier debugged by knowing what was run in docker and how that run was triggered
     - logging: the logs of the execution runs are now exposed as volumes, so that will be directly available when the component starts and will remain in the project path even after a component exited prematurely
     - execution: execution and step timestamps
     - cli: printing table from mongodb colleciton
-    - cli: delete methods for execution and related items. 
+    - cli: delete methods for execution and related items.
     - components: avoid rebuilding image if component is available
     - compose: a dev version for docker compose has been added to facilitate development
 
-- v0.3.1 
+- v0.3.1
     - hotfixes for bugs
     - removal of unused dependencies and adition of pandas
 
@@ -24,7 +38,7 @@
     - the execution run page is now only accessible via the execution
     - the workflow in the dashboard has been improved
     - the docker compose setup has now a dev and a prod file
-    - log levels for dashboard and docker runs can now be set separately   
+    - log levels for dashboard and docker runs can now be set separately
 
 - v0.2.8: bug fixes and settings updates
 

compose.dev.yml

@@ -23,12 +23,13 @@ services:
       ODTP_ACCESS_KEY: ${ODTP_ACCESS_KEY}
       ODTP_SECRET_KEY: ${ODTP_SECRET_KEY}
       GITHUB_TOKEN: ${GITHUB_TOKEN}
+      ODTP_PASSWORD: ${ODTP_PASSWORD}
       ODTP_MONGO_DB: ${ODTP_MONGO_DB}
       ODTP_DASHBOARD_PORT: ${ODTP_DASHBOARD_PORT}
       ODTP_DASHBOARD_RELOAD: ${ODTP_DASHBOARD_RELOAD}
-      ODTP_DASHBOARD_JSON_EDITOR: ${ODTP_DASHBOARD_JSON_EDITOR}
       ODTP_LOG_LEVEL: ${ODTP_LOG_LEVEL}
       RUN_LOG_LEVEL: ${RUN_LOG_LEVEL}
+      ALLOW_DOCKER_GPUS: ${ALLOW_DOCKER_GPUS:-True}
 
     stdin_open: true
     tty: true

compose.yml

@@ -19,11 +19,12 @@ services:
       ODTP_ACCESS_KEY: ${ODTP_ACCESS_KEY}
       ODTP_SECRET_KEY: ${ODTP_SECRET_KEY}
       GITHUB_TOKEN: ${GITHUB_TOKEN}
+      ODTP_PASSWORD: ${ODTP_PASSWORD}
       ODTP_MONGO_DB: ${ODTP_MONGO_DB}
       ODTP_DASHBOARD_PORT: ${ODTP_DASHBOARD_PORT}
-      ODTP_DASHBOARD_JSON_EDITOR: ${ODTP_DASHBOARD_JSON_EDITOR}
       ODTP_LOG_LEVEL: ${ODTP_LOG_LEVEL}
       RUN_LOG_LEVEL: ${RUN_LOG_LEVEL}
+      ALLOW_DOCKER_GPUS: ${ALLOW_DOCKER_GPUS:-True}
 
     stdin_open: true
     tty: true

odtp/cli/component.py

@@ -26,14 +26,18 @@ def prepare(
     image_name: str = typer.Option(
         ..., "--image_name", help="Specify the name of the component image"
     ),
+    image_link: str = typer.Option(
+        None, "--image_link", help="Specify the name of the component image link"
+    ),
     repository: str = typer.Option(
         ..., "--repository", help="Specify the git repository url"
     ),
 ):  
     try:
         componentManager = DockerManager(
             repo_url=repository, 
-            image_name=image_name, 
+            image_name=image_name,
+            image_link=image_link,
             project_folder=folder
         )
         componentManager.prepare_component()
@@ -51,9 +55,9 @@ def run(
     image_name: str = typer.Option(
         ..., "--image_name", help="Specify the name of the component image"
     ),
-    instance_name: str = typer.Option(
-        ..., "--instance_name", help="Specify the name of the instance"
-    ),    
+    container_name: str = typer.Option(
+        ..., "--container_name", help="Specify the name of the container"
+    ),   
     repository: str = typer.Option(
         ..., "--repository", help="Specify the git repository url"
     ),
@@ -62,6 +66,9 @@ def run(
     )] = None,
     parameter_file: Annotated[str, typer.Option(
         help="Specify the path to the environment file"
+    )] = None,
+    secrets_file: Annotated[str, typer.Option(
+        help="Specify the path to the secrets file"
     )] = None,     
     ports: Annotated[str, typer.Option(
         help="Specify port mappings separated by a plus sign i.e. 8501:8501+8201:8201"
@@ -76,10 +83,12 @@ def run(
         )
         ports = odtp_parse.parse_port_mappings_for_one_component(ports) 
         parameters = odtp_parse.parse_parameters_for_one_file(parameter_file)
+        secrets = odtp_parse.parse_parameters_for_one_file(secrets_file)
         componentManager.run_component(
-            parameters=parameters, 
+            parameters=parameters,
+            secrets=secrets, 
             ports=ports, 
-            instance_name=instance_name
+            container_name=container_name
         )
     except Exception as e:
         log.error(f"ERROR: Run of component failed: {e}")