refactor: simplify CLI, drop docker/buildah support, make az login optional

Readme.md

@@ -1,41 +1,57 @@
-### Introduction
+
+# Introduction
+
 This is an experimental CLI. It will allow you to manage user code deployments for a dagster instance that's deployed to kubernetes. It can package your code branch into a docker container, upload it to your ACR and update your existing Dagster instance (on kubernetes) to have your user code deployment.
-### Pre-requisites
-* You must have a valid kubectl config
-### Installation
+
+# Pre-requisites
+
+* Kubectl + valid kubectl config
+* Helm3
+* Podman
+* Python3.10+
+* AZ CLI (if you are on azure)
+
+# Installation
+
 * `pip install dagster-uc`
-* Create a configuration file in the root of your repository or in your home directory named `.config_user_code_deployments.yaml`, similar to this example:
+* Create a configuration file in the root of your repository or in your home directory named `.config_user_code_deployments.yaml`, similar to this example. You can also create one by doing `dagster-uc init-config -f '.config_user_code_deployments.yaml'`
+
 ```yaml
 dev:
-  repository_root: ./
-  dagster_gui_url: "http://dagster.company.com/"
-  docker_root: "."
-  dockerfile: "./k8s_deployments/Dockerfile"
-  acr: myacr.azurecr.io
+  cicd: false
   code_path: dagster_pipelines/repo.py
+  container_registry: myacr.azurecr.io
+  dagster_gui_url: null
   dagster_version: 1.8.4
+  docker_root: .
+  dockerfile: ./Dockerfile
   environment: dev
-  cicd: false
+  image_prefix: 'team-alpha'
+  kubernetes_context: "my-kubernetes-context"
   limits:
-    cpu: 4000m
-    memory: 2000Mi
-  namespace: my-k8s-namespace
-  node: my-k8s-nodepool
+    cpu: '2'
+    memory: 2Gi
+  namespace: .
+  node: small
+  repository_root: .
   requests:
-    cpu: 150m
-    memory: 750Mi
-  subscription: my-azure-subscription
-  acr_subscription: my-azure-subscription
+    cpu: '1'
+    memory: 1Gi
+  use_az_login: false
   user_code_deployment_env:
     - name: ON_K8S
       value: 'True'
     - name: ENVIRONMENT
       value: dev
   user_code_deployment_env_secrets:
     - name: my-env-secret
-  python_version: "python:3.10-slim"
-  kubernetes_context: "my-kubernetes-context"
+  user_code_deployments_configmap_name: dagster-user-deployments-values-yaml
+  dagster_workspace_yaml_configmap_name: dagster-workspace-yaml
+  uc_deployment_semaphore_name: dagster-uc-semaphore
+  verbose: false
 ```
-### Usage
+
+# Usage
+
 * In order to deploy the currently checked out git branch, run `dagster-uc deployment deploy`
-* In order to see all possible commands, run `dagster-uc --help`
+* In order to see all possible commands, run `dagster-uc --help`

dagster_uc/config.py

@@ -3,7 +3,41 @@
 
 import yaml
 
-from dagster_uc.initialize_logger import logger
+from dagster_uc.log import logger
+
+
+@dataclass
+class UserCodeDeploymentsConfig:
+    """This is a data class that holds the configuration parameters necessary
+    for running the user code deployment script.
+    """
+
+    environment: str
+    container_registry: str
+    dockerfile: str
+    image_prefix: str
+    namespace: str
+    node: str
+    code_path: str
+    docker_root: str
+    repository_root: str
+    dagster_version: str
+    user_code_deployment_env_secrets: list[
+        dict[str, str]
+    ]  # Must be list of dicts with key 'name' like so: [{"name": "sp-credentials"}, {"name": "lakefs-credentials"}]
+    user_code_deployment_env: list[
+        dict[str, str]
+    ]  # Must be list of dicts with keys 'name' and 'value' like so: [{"name": "MY_ENV_VAR", "value": "True"}, ...]
+    cicd: bool
+    limits: dict[str, str]
+    requests: dict[str, str]
+    kubernetes_context: str
+    dagster_gui_url: str | None = None
+    verbose: bool = False
+    use_az_login: bool = True
+    user_code_deployments_configmap_name: str = "dagster-user-deployments-values-yaml"
+    dagster_workspace_yaml_configmap_name: str = "dagster-workspace-yaml"
+    uc_deployment_semaphore_name: str = "dagster-uc-semaphore"
 
 
 def load_config(environment: str, path: str | None) -> "UserCodeDeploymentsConfig":
@@ -21,51 +55,18 @@ def load_config(environment: str, path: str | None) -> "UserCodeDeploymentsConfi
             raise Exception(
                 f"Could not load config file. Tried the following locations: {paths_to_try}\nCurrent folder: {os.getcwd()}\nContents of current folder: {os.listdir(os.getcwd())}\n\n Tip: Place the config file in the same folder you're calling this script from, or your home directory, or specify the path manually using --config-file <path>",
             )
+
     with open(path) as stream:
         raw_yaml = yaml.safe_load(stream)
         if environment not in raw_yaml:
             raise Exception(
                 f"Environment '{environment}' not specified in configuration file at '{path}'",
             )
         data = raw_yaml[environment]
+
     for field in fields(UserCodeDeploymentsConfig):
         if os.environ.get(field.name.upper(), None) is not None:
             data[field.name] = os.environ[field.name.upper()]
+
     logger.debug(f"Using configuration:\n {data}")
     return UserCodeDeploymentsConfig(**data)
-
-
-@dataclass
-class UserCodeDeploymentsConfig:
-    """This is a data class that holds the configuration parameters necessary
-    for running the user code deployment script.
-    """
-
-    environment: str
-    acr: str
-    dockerfile: str
-    namespace: str
-    subscription: str
-    acr_subscription: str
-    node: str
-    code_path: str
-    docker_root: str
-    repository_root: str
-    dagster_version: str
-    user_code_deployment_env_secrets: list[
-        dict[str, str]
-    ]  # Must be list of dicts with key 'name' like so: [{"name": "sp-credentials"}, {"name": "lakefs-credentials"}]
-    user_code_deployment_env: list[
-        dict[str, str]
-    ]  # Must be list of dicts with keys 'name' and 'value' like so: [{"name": "MY_ENV_VAR", "value": "True"}, ...]
-    cicd: bool
-    limits: dict[str, str]
-    requests: dict[str, str]
-    python_version: str
-    kubernetes_context: str
-    dagster_gui_url: str = ""
-    user_deployments_values_yaml: str = "dagster-user-deployments-values.yaml"
-    user_code_deployments_configmap_name: str = "dagster-user-deployments-values-yaml"
-    dagster_workspace_yaml_configmap_name = "dagster-workspace-yaml"
-    verbose: bool = False
-    uc_deployment_semaphore_name: str = "dagster-uc-semaphore"

dagster_uc/k8s_configmaps.py → dagster_uc/configmaps.py

@@ -1,7 +1,12 @@
 # This is a k8s yaml for defining a custom configmap with yaml data in it, loaded into a dictionary.
 # We will put the values.yaml for dagster's user-code deployment chart inside, and thereby use it to keep track of
 # currently deployed user code deployments and their configuration *on the k8s cluster*.
-BASE_CONFIGMAP = {"apiVersion": "v1", "kind": "ConfigMap", "data": {"yaml": ""}}
+BASE_CONFIGMAP = {
+    "apiVersion": "v1",
+    "kind": "ConfigMap",
+    "data": {"yaml": ""},
+    "metadata": {},
+}
 
 # This is the data we want to insert into our custom configmap. It's the 'values.yaml' that is to be fed into the
 # dagster user-code deployment chart later. This values.yaml starts out with an empty deployment list and is later
@@ -17,7 +22,7 @@
     "postgresqlSecretName": "dagster-postgresql-secret",
     "celeryConfigSecretName": "dagster-celery-config-secret",
     "deployments": [],
-    "imagePullSecrets": [{"name": "idp-acr"}],
+    "imagePullSecrets": [],
     "serviceAccount": {"create": True, "name": "", "annotations": {}},
     "rbacEnabled": True,
     "extraManifests": [],

dagster_uc/initialize_logger.py → dagster_uc/log.py

@@ -4,6 +4,7 @@
 logging.basicConfig(
     level=logging.WARNING,
     handlers=[logging.StreamHandler(sys.stdout)],
-    format="%(filename)s: " "%(levelname)s: " "%(funcName)s(): " "%(lineno)d:\t" "%(message)s",
+    format="%(asctime)s - %(name)s.%(funcName)s():%(lineno)d - %(levelname)s - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
 )
 logger = logging.getLogger(__name__)