clone backing services when branching

Author: ewdurbin

cabotage/celery/tasks/branch_deploy.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import re
+from copy import deepcopy
 
 from flask import current_app
 from kubernetes.client.rest import ApiException
@@ -23,6 +24,7 @@
     IngressHost,
     IngressPath,
 )
+from cabotage.server.models.resources import PostgresResource, RedisResource
 from cabotage.server.models.utils import readable_k8s_hostname, safe_k8s_name
 from cabotage.utils.github import (
     find_or_create_pr_comment,
@@ -35,6 +37,45 @@
 Activity = activity_plugin.activity_cls
 
 
+def _clone_backing_services_for_branch_deploy(base_environment, environment):
+    """Create fresh backing-service rows in a PR environment.
+
+    The cloned resources inherit sizing/topology/version settings from the
+    base environment, but they are provisioned independently in the branch
+    deploy namespace with a clean lifecycle state.
+    """
+    for resource in base_environment.active_resources:
+        if isinstance(resource, PostgresResource):
+            cloned = PostgresResource(
+                environment_id=environment.id,
+                name=resource.name,
+                slug=resource.slug,
+                service_version=resource.service_version,
+                size_class=resource.size_class,
+                storage_size=resource.storage_size,
+                ha_enabled=resource.ha_enabled,
+                backup_strategy=resource.backup_strategy,
+                postgres_parameters=deepcopy(resource.postgres_parameters or {}),
+            )
+        elif isinstance(resource, RedisResource):
+            cloned = RedisResource(
+                environment_id=environment.id,
+                name=resource.name,
+                slug=resource.slug,
+                service_version=resource.service_version,
+                size_class=resource.size_class,
+                storage_size=resource.storage_size,
+                ha_enabled=resource.ha_enabled,
+                leader_replicas=resource.leader_replicas,
+                follower_replicas=resource.follower_replicas,
+            )
+        else:
+            continue
+        db.session.add(cloned)
+
+    db.session.flush()
+
+
 def _create_app_env_for_branch_deploy(
     application,
     environment,
@@ -651,6 +692,8 @@ def maybe_update_pr_comment_for_app_env(app_env):
 
 def create_branch_deploy(project, pr_number, head_sha, installation_id, head_ref=None):
     """Create an ephemeral environment for a PR and build images for all enrolled apps."""
+    from cabotage.celery.tasks.resources import reconcile_backing_services
+
     base_env = project.branch_deploy_base_environment
     env_slug = f"pr-{pr_number}"
 
@@ -695,6 +738,8 @@ def create_branch_deploy(project, pr_number, head_sha, installation_id, head_ref
         env_config_map[env_config.id] = new_env_config.id
     db.session.flush()
 
+    _clone_backing_services_for_branch_deploy(base_env, environment)
+
     activity = Activity(
         verb="create",
         object=environment,
@@ -721,6 +766,9 @@ def create_branch_deploy(project, pr_number, head_sha, installation_id, head_ref
 
     db.session.commit()
 
+    if environment.active_resources:
+        reconcile_backing_services.delay()
+
     # Create K8s namespace + ingresses early so cert-manager can start
     # issuing TLS certificates while image builds run in parallel.
     try:

cabotage/celery/tasks/build.py

@@ -47,6 +47,7 @@
 
 from cabotage.server.models.projects import (
     activity_plugin,
+    Environment,
     Image,
     Release,
     Deployment,
@@ -75,6 +76,136 @@
 log = logging.getLogger(__name__)
 
 
+def _branch_deploy_environment_backing_services_ready(environment):
+    resources = environment.active_resources
+    if not resources:
+        return True
+    return all(resource.provisioning_status == "ready" for resource in resources)
+
+
+def _branch_deploy_backing_services_ready(app_env):
+    environment = app_env.environment if app_env else None
+    if not environment or environment.forked_from_environment_id is None:
+        return True
+    return _branch_deploy_environment_backing_services_ready(environment)
+
+
+def _mark_image_waiting_for_backing_services(image):
+    image.image_metadata = {
+        **(image.image_metadata or {}),
+        "waiting_for_backing_services": True,
+    }
+    db.session.add(image)
+    db.session.commit()
+
+
+def _clear_image_waiting_for_backing_services(image):
+    metadata = dict(image.image_metadata or {})
+    if "waiting_for_backing_services" in metadata:
+        metadata.pop("waiting_for_backing_services", None)
+        image.image_metadata = metadata
+        db.session.add(image)
+        db.session.commit()
+
+
+def _latest_release_for_image(image):
+    app_env = image.application_environment
+    if not app_env:
+        return None
+    release = app_env.latest_release
+    if not release or not release.release_metadata:
+        return None
+    if release.release_metadata.get("source_image_id") == str(image.id):
+        return release
+    return None
+
+
+def _queue_autodeploy_release_for_image(image):
+    app_env = image.application_environment
+    if not app_env or not image.built:
+        return None
+
+    existing_release = _latest_release_for_image(image)
+    if existing_release is not None:
+        return existing_release
+
+    _clear_image_waiting_for_backing_services(image)
+
+    application = image.application
+    release = image.application.create_release(app_env=app_env)
+    release.release_metadata = {
+        **(image.image_metadata or {}),
+        "source_image_id": str(image.id),
+    }
+    db.session.add(release)
+    db.session.flush()
+    activity = Activity(
+        verb="create",
+        object=release,
+        data={
+            "user_id": "automation",
+            "deployment_id": image.image_metadata.get("id", None),
+            "description": image.image_metadata.get("description", None),
+            "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        },
+    )
+    db.session.add(activity)
+    db.session.commit()
+
+    CheckRun.from_metadata(image.image_metadata, app_env).progress(
+        "Building release...",
+        detail="Image built, release build starting.",
+        details_url=cabotage_url(application, f"releases/{release.id}"),
+        Image=f"images/{image.id}",
+        Release=f"releases/{release.id}",
+    )
+    run_release_build.delay(release_id=release.id)
+    try:
+        dispatch_autodeploy_notification(
+            "release_building",
+            image.id,
+            application,
+            app_env,
+            image_url=cabotage_url(application, f"images/{image.id}"),
+            release_url=cabotage_url(application, f"releases/{release.id}"),
+            image_metadata=image.image_metadata,
+        )
+    except Exception:
+        log.warning(
+            "Failed to dispatch autodeploy release_building notification",
+            exc_info=True,
+        )
+    return release
+
+
+def resume_branch_deploy_releases_for_environment(environment_id):
+    environment = Environment.query.filter_by(id=environment_id).first()
+    if (
+        environment is None
+        or environment.forked_from_environment_id is None
+        or not _branch_deploy_environment_backing_services_ready(environment)
+    ):
+        return
+
+    for app_env in environment.application_environments:
+        image = app_env.latest_image
+        if not image or not image.image_metadata:
+            continue
+        if not image.image_metadata.get("auto_deploy"):
+            continue
+        if not image.image_metadata.get("branch_deploy"):
+            continue
+
+        if current_app.config.get("CABOTAGE_OMNIBUS_BUILDS"):
+            if not image.built and image.image_metadata.get(
+                "waiting_for_backing_services"
+            ):
+                _clear_image_waiting_for_backing_services(image)
+                run_omnibus_build.delay(image_id=image.id)
+        elif image.built:
+            _queue_autodeploy_release_for_image(image)
+
+
 def _dispatch_image_failure(image, error_detail):
     try:
         app = image.application
@@ -1873,48 +2004,21 @@ def run_image_build(image_id: str, buildkit: bool = False):
         and image.image_metadata.get("auto_deploy", False)
     ):
         app_env = image.application_environment
-        release = image.application.create_release(app_env=app_env)
-        release.release_metadata = {
-            **(image.image_metadata or {}),
-            "source_image_id": str(image.id),
-        }
-        db.session.add(release)
-        db.session.flush()
-        activity = Activity(
-            verb="create",
-            object=release,
-            data={
-                "user_id": "automation",
-                "deployment_id": image.image_metadata.get("id", None),
-                "description": image.image_metadata.get("description", None),
-                "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
-            },
-        )
-        db.session.add(activity)
-        db.session.commit()
-        check.progress(
-            "Building release...",
-            detail="Image built, release build starting.",
-            details_url=cabotage_url(application, f"releases/{release.id}"),
-            Image=f"images/{image.id}",
-            Release=f"releases/{release.id}",
-        )
-        run_release_build.delay(release_id=release.id)
-        try:
-            dispatch_autodeploy_notification(
-                "release_building",
-                image.id,
-                application,
-                app_env,
-                image_url=cabotage_url(application, f"images/{image.id}"),
-                release_url=cabotage_url(application, f"releases/{release.id}"),
-                image_metadata=image.image_metadata,
-            )
-        except Exception:
-            log.warning(
-                "Failed to dispatch autodeploy release_building notification",
-                exc_info=True,
+        if image.image_metadata.get(
+            "branch_deploy"
+        ) and not _branch_deploy_backing_services_ready(app_env):
+            _mark_image_waiting_for_backing_services(image)
+            check.progress(
+                "Waiting for backing services...",
+                detail=(
+                    "Image built successfully. Waiting for backing services "
+                    "to become ready before release build."
+                ),
+                details_url=cabotage_url(application, f"images/{image.id}"),
+                Image=f"images/{image.id}",
             )
+        else:
+            _queue_autodeploy_release_for_image(image)
 
 
 @shared_task()
@@ -2245,6 +2349,24 @@ def run_omnibus_build(image_id: str):
 
     release = None
     app_env = image.application_environment
+    if (
+        image.image_metadata
+        and image.image_metadata.get("auto_deploy", False)
+        and image.image_metadata.get("branch_deploy")
+        and not _branch_deploy_backing_services_ready(app_env)
+    ):
+        _mark_image_waiting_for_backing_services(image)
+        check.progress(
+            "Waiting for backing services...",
+            detail=(
+                "Waiting for backing services to become ready before starting "
+                "the branch deploy build."
+            ),
+            details_url=cabotage_url(application, f"images/{image.id}"),
+            Image=f"images/{image.id}",
+        )
+        return
+
     try:
         try:
             # Create the release record upfront so build_omnibus_buildkit

cabotage/celery/tasks/resources.py

@@ -1495,6 +1495,9 @@ def _delete_redis(resource, core_api, custom_api, apps_api=None, rbac_api=None):
 def reconcile_backing_services():
     """Periodic task: converge all backing service resources to desired state."""
     from cabotage.server.models.resources import Resource
+    from cabotage.celery.tasks.build import (
+        resume_branch_deploy_releases_for_environment,
+    )
 
     resources = Resource.query.filter(
         Resource.provisioning_status != "deleting",
@@ -1535,6 +1538,10 @@ def reconcile_backing_services():
                     ensure_network_policies(networking_api, namespace)
                 reconcile_fn(resource, core_api, custom_api, apps_api, rbac_api)
                 db.session.commit()
+                if resource.environment.forked_from_environment_id is not None:
+                    resume_branch_deploy_releases_for_environment(
+                        resource.environment_id
+                    )
         except Exception:
             log.exception(
                 "Failed to reconcile %s resource %s (%s)",

cabotage/server/__init__.py

@@ -165,7 +165,7 @@ def __call__(self, *args: object, **kwargs: object) -> object:
         },
         "backing-service-reconciler": {
             "task": "cabotage.celery.tasks.resources.reconcile_backing_services",
-            "schedule": 30.0,
+            "schedule": 10.0,
             "args": None,
         },
     }