feat(campaign): generate additional namespaced names server-side

Author: tcjennings

src/lsst/cmservice/client/loaders.py

@@ -79,7 +79,7 @@ def _upsert_spec_block(
         if spec_block and not allow_update:
             return spec_block
 
-        logger.info(f"""Loading spec_block {spec_name} as {key}""")
+        logger.info(f"Loading spec_block {spec_name} as {key}")
 
         # A spec that "includes" another spec is effectively declaring a clone
         # of the referenced spec that should have already been "loaded" such
@@ -143,47 +143,6 @@ def _upsert_spec_block(
                 },
             )
 
-        # scripts and steps are lists, not mappings
-        if "scripts" in block_data:
-            # the 'name' of the script should be namespaced; the 'spec_block'
-            # of the script is a key to its specification alias within the
-            # campaign, which itself references a namespaced spec_block.
-            block_data["scripts"] = [
-                deep_update(
-                    s,
-                    {
-                        "Script": {
-                            "name": str(uuid5(namespace, s["Script"]["name"])),
-                            "spec_block": s["Script"]["spec_block"],
-                            "prerequisites": [
-                                str(uuid5(namespace, dep)) for dep in s["Script"].get("prerequisites", [])
-                            ],
-                        }
-                    },
-                )
-                for s in block_data["scripts"]
-            ]
-
-        if "steps" in block_data:
-            # steps are found in a campaign spec_block; the 'spec_block' and
-            # 'step' should be namespaced. Any names used in a prerequisites
-            # should be namespaced.
-            block_data["steps"] = [
-                deep_update(
-                    s,
-                    {
-                        "Step": {
-                            "name": str(uuid5(namespace, s["Step"]["name"])),
-                            "spec_block": str(uuid5(namespace, s["Step"]["spec_block"])),
-                            "prerequisites": [
-                                str(uuid5(namespace, dep)) for dep in s["Step"].get("prerequisites", [])
-                            ],
-                        }
-                    },
-                )
-                for s in block_data["steps"]
-            ]
-
         handler = block_data.pop("handler", None)
         if spec_block is None:
             return self._parent.spec_block.create(

src/lsst/cmservice/db/script.py

@@ -168,6 +168,7 @@ async def get_create_kwargs(
             parent_name = kwargs["parent_name"]
             name = kwargs["name"]
             spec_block_name = kwargs["spec_block_name"]
+            original_name = kwargs.get("original_name", name)
         except KeyError as e:
             raise CMMissingRowCreateInputError(f"Missing input to create Script: {e}") from e
         attempt = kwargs.get("attempt", 0)
@@ -177,11 +178,13 @@ async def get_create_kwargs(
         if isinstance(parent_level, int):
             parent_level = LevelEnum(parent_level)
 
+        # The fullname should reflect the element's original shortname not its
+        # namespaced name
         ret_dict = {
             "parent_level": parent_level,
             "name": name,
             "attempt": attempt,
-            "fullname": f"{parent_name}/{name}_{attempt:03}",
+            "fullname": f"{parent_name}/{original_name}_{attempt:03}",
             "method": ScriptMethodEnum[kwargs.get("method", "default")],
             "handler": kwargs.get("handler"),
             "data": kwargs.get("data", {}),

src/lsst/cmservice/db/step.py

@@ -161,6 +161,7 @@ async def get_create_kwargs(
             parent_name = kwargs["parent_name"]
             name = kwargs["name"]
             spec_block_name = kwargs["spec_block_name"]
+            original_name = kwargs.get("original_name", name)
         except KeyError as e:
             raise CMMissingRowCreateInputError(f"Missing input to create Step: {e}") from e
 
@@ -173,7 +174,7 @@ async def get_create_kwargs(
             "spec_block_id": spec_block.id,
             "parent_id": campaign.id,
             "name": name,
-            "fullname": f"{campaign.fullname}/{name}",
+            "fullname": f"{campaign.fullname}/{original_name}",
             "handler": kwargs.get("handler"),
             "data": kwargs.get("data", {}),
             "child_config": kwargs.get("child_config", {}),

src/lsst/cmservice/handlers/element_handler.py

@@ -157,6 +157,12 @@ async def prepare(
             assert isinstance(spec_block.scripts, Iterable)
             assert isinstance(element_campaign.data, dict)
 
+        if element_campaign.data.get("namespace"):
+            campaign_namespace = UUID(element_campaign.data.get("namespace"))
+        else:
+            campaign_namespace = None
+
+        # Campaigns, Steps, Groups, and Jobs may have Scripts
         for script_item in spec_block.scripts:
             try:
                 script_vals = script_item["Script"].copy()
@@ -165,6 +171,9 @@ async def prepare(
             test_type_and_raise(script_vals, dict, "ElementHandler Script yaml tag")
             try:
                 script_name = script_vals.pop("name")
+                namespaced_script_name = (
+                    str(uuid5(campaign_namespace, script_name)) if campaign_namespace else script_name
+                )
             except KeyError as msg:
                 raise CMYamlParseError(f"Unnnamed Script block {script_vals}") from msg
 
@@ -174,29 +183,34 @@ async def prepare(
 
             # If the spec_aliases does not have a key for the current script
             # name, then it is not an alias.
-            if (script_spec_block_name in spec_aliases) or (not element_campaign.data.get("namespace")):
+            if (script_spec_block_name in spec_aliases) or (not campaign_namespace):
                 script_spec_block_name = spec_aliases.get(script_spec_block_name, script_spec_block_name)
             else:
                 # generate a namespaced name from the current campaign
-                campaign_namespace = UUID(element_campaign.data.get("namespace"))
                 script_spec_block_name = str(uuid5(campaign_namespace, script_spec_block_name))
+
             new_script = await Script.create_row(
                 session,
                 parent_level=element.level,
                 spec_block_name=script_spec_block_name,
                 parent_name=element.fullname,
-                name=script_name,
+                name=namespaced_script_name,
+                original_name=script_name,
                 **script_vals,
             )
             await session.refresh(new_script, attribute_names=["id"])
-            script_ids_dict[script_name] = new_script.id
-            prereq_pairs += [(script_name, prereq_) for prereq_ in script_vals.get("prerequisites", [])]
+            script_ids_dict[namespaced_script_name] = new_script.id
+
+            prereq_list = [
+                str(uuid5(campaign_namespace, prereq)) if campaign_namespace else prereq
+                for prereq in script_vals.get("prerequisites", [])
+            ]
+            prereq_pairs += [(namespaced_script_name, prereq) for prereq in prereq_list]
 
         for depend_name, prereq_name in prereq_pairs:
             prereq_id = script_ids_dict[prereq_name]
             depend_id = script_ids_dict[depend_name]
-            _new_depend = await self._add_prerequisite(session, depend_id, prereq_id)
-            # await session.refresh(new_depend)
+            _ = await self._add_prerequisite(session, depend_id, prereq_id)
 
         await element.update_values(session, status=StatusEnum.prepared)
         return (True, StatusEnum.prepared)

src/lsst/cmservice/handlers/functions.py

@@ -1,6 +1,7 @@
 import os
 from collections.abc import Mapping
-from typing import Any
+from typing import TYPE_CHECKING, Any
+from uuid import UUID, uuid5
 
 import yaml
 from anyio import Path
@@ -296,6 +297,14 @@ async def add_steps(
 
     step_ids_dict = {step_.name: step_.id for step_ in current_steps}
 
+    if TYPE_CHECKING:
+        assert isinstance(campaign.data, dict)
+
+    if campaign.data.get("namespace"):
+        campaign_namespace = UUID(campaign.data.get("namespace"))
+    else:
+        campaign_namespace = None
+
     prereq_pairs = []
     for step_ in step_config_list:
         try:
@@ -304,24 +313,38 @@ async def add_steps(
             raise CMYamlParseError(f"Expecting Step not: {step_.keys()}") from msg
         child_name_ = step_config_.pop("name")
         spec_block_name = step_config_.pop("spec_block")
+
         if spec_block_name is None:  # pragma: no cover
             raise CMYamlParseError(
                 f"Step {child_name_} of {campaign.fullname} does contain 'spec_block'",
             )
+
+        namespaced_step_name = (
+            str(uuid5(campaign_namespace, child_name_)) if campaign_namespace else child_name_
+        )
+        namespaced_spec_block_name = (
+            str(uuid5(campaign_namespace, spec_block_name)) if campaign_namespace else spec_block_name
+        )
+
         spec_block_name = spec_aliases.get(spec_block_name, spec_block_name)
         step_config_ = deep_update(step_config_, child_config.get(child_name_, {}))
 
         new_step = await Step.create_row(
             session,
-            name=child_name_,
-            spec_block_name=spec_block_name,
+            name=namespaced_step_name,
+            spec_block_name=namespaced_spec_block_name,
+            original_name=child_name_,
             parent_name=campaign.fullname,
             **step_config_,
         )
         await session.refresh(new_step)
-        step_ids_dict[child_name_] = new_step.id
-        prereqs_names = step_config_.pop("prerequisites", [])
-        prereq_pairs += [(child_name_, prereq_) for prereq_ in prereqs_names]
+        step_ids_dict[namespaced_step_name] = new_step.id
+
+        prereq_list = [
+            str(uuid5(campaign_namespace, prereq)) if campaign_namespace else prereq
+            for prereq in step_config_.pop("prerequisites", [])
+        ]
+        prereq_pairs += [(namespaced_step_name, prereq) for prereq in prereq_list]
 
     for depend_name, prereq_name in prereq_pairs:
         prereq_id = step_ids_dict[prereq_name]

src/lsst/cmservice/routers/queues.py

@@ -6,8 +6,11 @@
 
 from .. import db, models
 from ..common.errors import CMMissingIDError
+from ..common.logging import LOGGER
 from . import wrappers
 
+logger = LOGGER.bind(module=__name__)
+
 # Template specialization
 # Specify the pydantic model for the table
 ResponseModelClass = models.Queue
@@ -72,6 +75,7 @@ async def process_element(
     except CMMissingIDError as msg:
         raise HTTPException(status_code=404, detail=f"{str(msg)}") from msg
     except Exception as msg:
+        logger.exception()
         raise HTTPException(status_code=500, detail=f"{str(msg)}") from msg
     return can_continue