Beter labels and multi tenancy support (#348)

* metrics and logging multi-tenancy

improves labeling and adds support for multi-tenancy as well as migration

* create an infra panel to observe cabotage's internal infra metrics

* rough-draft of container-level resource limit/req

* show metrics for pods that have terminated

* fix handling of migration to tenantified queries

Author: ewdurbin

cabotage/celery/tasks/branch_deploy.py

@@ -233,6 +233,9 @@ def _precreate_ingresses(environment):
                 "organization": org.slug,
                 "project": environment.project.slug,
                 "application": app.slug,
+                "cabotage.io/organization": org.k8s_identifier,
+                "cabotage.io/project": environment.project.k8s_identifier,
+                "cabotage.io/application": app.k8s_identifier,
             },
             ingresses=app_env.ingresses,
             org_k8s_identifier=org.k8s_identifier,

cabotage/celery/tasks/build.py

@@ -27,7 +27,11 @@
 from github.GithubException import GithubException, UnknownObjectException
 from github.GithubIntegration import GithubIntegration
 
-from cabotage.celery.tasks.deploy import run_deploy, run_job
+from cabotage.celery.tasks.deploy import (
+    _safe_labels_from_application,
+    run_deploy,
+    run_job,
+)
 from cabotage.celery.tasks.notify import (
     dispatch_autodeploy_notification,
     dispatch_pipeline_notification,
@@ -450,6 +454,7 @@ def build_release_buildkit(release):
                 },
             )
             context_configmap_object = release.release_build_context_configmap
+            safe_labels = _safe_labels_from_application(release.application)
             job_object = kubernetes.client.V1Job(
                 metadata=kubernetes.client.V1ObjectMeta(
                     name=f"releasebuild-{release.build_job_id}",
@@ -460,6 +465,7 @@ def build_release_buildkit(release):
                         "process": "build",
                         "build_id": release.build_job_id,
                         "build-job.cabotage.io": "true",
+                        **safe_labels,
                     },
                 ),
                 spec=kubernetes.client.V1JobSpec(
@@ -477,6 +483,7 @@ def build_release_buildkit(release):
                                 "build_id": release.build_job_id,
                                 "ca-admission.cabotage.io": "true",
                                 "resident-pod.cabotage.io": "true",
+                                **safe_labels,
                             },
                             annotations={
                                 "container.apparmor.security.beta.kubernetes.io/build": "unconfined",  # noqa: E501
@@ -960,6 +967,7 @@ def build_image_buildkit(image: Image):
                     "buildkitd.toml": buildkitd_toml,
                 },
             )
+            safe_labels = _safe_labels_from_application(image.application)
             job_object = kubernetes.client.V1Job(
                 metadata=kubernetes.client.V1ObjectMeta(
                     name=f"imagebuild-{image.build_job_id}",
@@ -970,6 +978,7 @@ def build_image_buildkit(image: Image):
                         "process": "build",
                         "build_id": image.build_job_id,
                         "build-job.cabotage.io": "true",
+                        **safe_labels,
                     },
                 ),
                 spec=kubernetes.client.V1JobSpec(
@@ -987,6 +996,7 @@ def build_image_buildkit(image: Image):
                                 "build_id": image.build_job_id,
                                 "ca-admission.cabotage.io": "true",
                                 "resident-pod.cabotage.io": "true",
+                                **safe_labels,
                             },
                             annotations={
                                 "container.apparmor.security.beta.kubernetes.io/build": "unconfined",  # noqa: E501
@@ -1496,6 +1506,7 @@ def build_omnibus_buildkit(image, release):
             ],
         )
 
+        safe_labels = _safe_labels_from_application(image.application)
         job_object = kubernetes.client.V1Job(
             metadata=kubernetes.client.V1ObjectMeta(
                 name=f"omnibusbuild-{image.build_job_id}",
@@ -1506,6 +1517,7 @@ def build_omnibus_buildkit(image, release):
                     "process": "build",
                     "build_id": image.build_job_id,
                     "build-job.cabotage.io": "true",
+                    **safe_labels,
                 },
             ),
             spec=kubernetes.client.V1JobSpec(
@@ -1523,6 +1535,7 @@ def build_omnibus_buildkit(image, release):
                             "build_id": image.build_job_id,
                             "ca-admission.cabotage.io": "true",
                             "resident-pod.cabotage.io": "true",
+                            **safe_labels,
                         },
                         annotations={
                             "container.apparmor.security.beta.kubernetes.io/image-build": "unconfined",  # noqa: E501

cabotage/celery/tasks/deploy.py

@@ -271,6 +271,37 @@ def k8s_label_value(release):
     return compact_k8s_name(*pairs)
 
 
+def _safe_labels_from_release(release):
+    """Build cabotage.io/-prefixed labels using k8s_identifiers.
+
+    These are collision-safe labels that sit alongside the legacy
+    slug-based labels.
+    """
+    org = release.application.project.organization
+    project = release.application.project
+    app = release.application
+    app_env = release.application_environment
+    labels = {
+        "cabotage.io/organization": org.k8s_identifier,
+        "cabotage.io/project": project.k8s_identifier,
+        "cabotage.io/application": app.k8s_identifier,
+    }
+    if app_env.k8s_identifier is not None:
+        labels["cabotage.io/environment"] = app_env.environment.k8s_identifier
+    return labels
+
+
+def _safe_labels_from_application(application):
+    """Build cabotage.io/-prefixed labels from an Application (for builds)."""
+    org = application.project.organization
+    project = application.project
+    return {
+        "cabotage.io/organization": org.k8s_identifier,
+        "cabotage.io/project": project.k8s_identifier,
+        "cabotage.io/application": application.k8s_identifier,
+    }
+
+
 def render_namespace(release):
     namespace_object = kubernetes.client.V1Namespace(
         metadata=kubernetes.client.V1ObjectMeta(
@@ -808,13 +839,15 @@ def render_service(release, process_name):
     resource_prefix = k8s_resource_prefix(release)
     service_name = f"{resource_prefix}-{process_name}"
     label_value = k8s_label_value(release)
+    safe_labels = _safe_labels_from_release(release)
     service_object = kubernetes.client.V1Service(
         metadata=kubernetes.client.V1ObjectMeta(
             name=service_name,
             labels={
                 "resident-service.cabotage.io": "true",
                 "app": resource_prefix,
                 "process": process_name,
+                **safe_labels,
             },
         ),
         spec=kubernetes.client.V1ServiceSpec(
@@ -874,6 +907,7 @@ def render_ingress(release, ingress):
 
     Convenience wrapper that extracts naming context from a Release.
     """
+    safe_labels = _safe_labels_from_release(release)
     return render_ingress_object(
         ingress=ingress,
         resource_prefix=k8s_resource_prefix(release),
@@ -882,6 +916,7 @@ def render_ingress(release, ingress):
             "project": release.application.project.slug,
             "application": release.application.slug,
             "app": k8s_label_value(release),
+            **safe_labels,
         },
         org_k8s_identifier=release.application.project.organization.k8s_identifier,
         process_names=list(release.processes) if release.processes else [],
@@ -1841,6 +1876,7 @@ def render_deployment(
     app_env = release.application_environment
     env_slug = app_env.environment.slug if app_env.environment else ""
     process_counts = app_env.process_counts or {}
+    safe_labels = _safe_labels_from_release(release)
     pod_labels = {
         "organization": release.application.project.organization.slug,
         "project": release.application.project.slug,
@@ -1852,6 +1888,7 @@ def render_deployment(
         "deployment": str(deployment_id),
         "ca-admission.cabotage.io": "true",
         "resident-pod.cabotage.io": "true",
+        **safe_labels,
     }
     deployment_object = kubernetes.client.V1Deployment(
         metadata=kubernetes.client.V1ObjectMeta(
@@ -1863,6 +1900,7 @@ def render_deployment(
                 "process": process_name,
                 "app": label_value,
                 "resident-deployment.cabotage.io": "true",
+                **safe_labels,
             },
         ),
         spec=kubernetes.client.V1DeploymentSpec(
@@ -2054,6 +2092,7 @@ def resize_deployment(namespace, release, process_name, pod_class_name):
 
 def render_job(namespace, release, service_account_name, process_name, job_id):
     label_value = k8s_label_value(release)
+    safe_labels = _safe_labels_from_release(release)
     job_object = kubernetes.client.V1Job(
         metadata=kubernetes.client.V1ObjectMeta(
             name=f"deployment-{job_id}",
@@ -2066,6 +2105,7 @@ def render_job(namespace, release, service_account_name, process_name, job_id):
                 "release": str(release.version),
                 "deployment": job_id,
                 "resident-job.cabotage.io": "true",
+                **safe_labels,
             },
         ),
         spec=kubernetes.client.V1JobSpec(
@@ -2084,6 +2124,7 @@ def render_job(namespace, release, service_account_name, process_name, job_id):
                         "deployment": job_id,
                         "ca-admission.cabotage.io": "true",
                         "resident-pod.cabotage.io": "true",
+                        **safe_labels,
                     }
                 ),
                 spec=render_podspec(release, process_name, service_account_name),
@@ -2130,6 +2171,7 @@ def render_cronjob(
         )
     process_counts = app_env.process_counts or {}
     suspended = process_counts.get(process_name, 0) == 0
+    safe_labels = _safe_labels_from_release(release)
     common_labels = {
         "organization": release.application.project.organization.slug,
         "project": release.application.project.slug,
@@ -2139,6 +2181,7 @@ def render_cronjob(
         "environment": env_slug,
         "release": str(release.version),
         "deployment": str(deployment_id),
+        **safe_labels,
     }
     job_labels = {
         **common_labels,
@@ -2160,6 +2203,7 @@ def render_cronjob(
                 "process": process_name,
                 "app": label_value,
                 "resident-cronjob.cabotage.io": "true",
+                **safe_labels,
             },
         ),
         spec=kubernetes.client.V1CronJobSpec(

cabotage/client/templates/_base.html

@@ -259,6 +259,10 @@
                    class="btn btn-ghost btn-sm text-base-content/60 hover:text-base-content text-xs font-medium">Projects</a>
                 <a href="{{ url_for('user.guide') }}"
                    class="btn btn-ghost btn-sm text-base-content/60 hover:text-base-content text-xs font-medium">Guide</a>
+                {% if current_user.admin %}
+                <a href="{{ url_for('user.infra_observe') }}"
+                   class="btn btn-ghost btn-sm text-base-content/60 hover:text-base-content text-xs font-medium">Infra</a>
+                {% endif %}
               </nav>
               <div class="dropdown dropdown-hover dropdown-end theme-toggle-wrap">
                 <div tabindex="0" role="button" id="theme-toggle"
@@ -611,6 +615,10 @@
                class="btn btn-ghost btn-sm justify-start text-sm">Projects</a>
             <a href="{{ url_for('user.guide') }}"
                class="btn btn-ghost btn-sm justify-start text-sm">Guide</a>
+            {% if current_user.admin %}
+            <a href="{{ url_for('user.infra_observe') }}"
+               class="btn btn-ghost btn-sm justify-start text-sm">Infrastructure</a>
+            {% endif %}
           </nav>
         {% endif %}