feat(model):

- Improve Manifest and Step models
- Apply extra=forbid configuration to all models
- Implement submanifest support for Step fields.
- Add "render_jsonschema" helper script.

Author: tcjennings

script/render_jsonschema.py

@@ -0,0 +1,24 @@
+"""Loads the CM Service Manifest Models and generates JSONSchema files"""
+
+import json
+from pathlib import Path
+
+from lsst.cmservice.models.manifests import bps, butler, facility, lsst, steps, wms
+
+
+def main():
+    for manifest in [
+        bps.BpsSpec,
+        butler.ButlerSpec,
+        facility.FacilitySpec,
+        lsst.LsstSpec,
+        steps.StepSpec,
+        wms.WmsSpec,
+    ]:
+        manifest_schema = manifest.model_json_schema()
+        manifest_title = manifest_schema["title"]
+        Path(f"{manifest_title}.jsonschema").write_text(json.dumps(manifest_schema, indent=2))
+
+
+if __name__ == "__main__":
+    main()

src/lsst/cmservice/models/manifest.py

@@ -3,5 +3,5 @@
 from .manifests.butler import ButlerManifest as ButlerManifest
 from .manifests.facility import FacilityManifest as FacilityManifest
 from .manifests.lsst import LsstManifest as LsstManifest
-from .manifests.steps import GroupedStepManifest as GroupedStepManifest
+from .manifests.steps import StepManifest as StepManifest
 from .manifests.wms import WmsManifest as WmsManifest

src/lsst/cmservice/models/manifests/__init__.py

@@ -1,4 +1,7 @@
-from pydantic import AliasChoices, BaseModel, Field
+from typing import Any, cast
+
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, create_model
+from pydantic.alias_generators import to_snake
 
 from ...common.enums import ManifestKind
 from ...common.types import KindField
@@ -20,3 +23,52 @@ class LibraryManifest[SpecT](BaseModel):
         validation_alias=AliasChoices("spec", "configuration", "data"),
         serialization_alias="spec",
     )
+
+
+def model_title_generator(cls: type[BaseModel]) -> str:
+    """Given a pydantic Model, returns the name of the model in snake_case."""
+    return to_snake(cls.__name__)
+
+
+class SubfieldManifest[T: BaseModel]:
+    """Create a variant of a Pydantic Model with all fields optional.
+
+    This class allows the use of a more restrictive model as a subfield in
+    another model such that all fields in the submodel are "optional" in that
+    they do not need to be supplied, but if they are they remain subject to
+    any type restrictions.
+
+    Parameters
+    ----------
+    T : type[BaseModel]
+        The pydantic BaseModel type to enable for Subfield use.
+
+    Returns
+    -------
+    type[T]
+        A new model class with the same config and fields as the input model,
+        but all fields are made optional (i.e., may be missing but not None).
+
+    Notes
+    -----
+    - The original model's configuration is preserved.
+    - Nested submodels are handled correctly.
+    - Fields remain strongly typed - if present, they must match the original
+      type.
+    """
+
+    def __class_getitem__(cls, model: type[T]) -> type[T]:
+        fields: dict[str, Any] = {
+            field_name: (field_info.annotation, None) for field_name, field_info in model.model_fields.items()
+        }
+        subfield_model = create_model(f"{model.__name__}SubField", __config__=model.model_config, **fields)
+        subfield_model.model_rebuild()
+        return cast(type[T], subfield_model)
+
+
+SPEC_CONFIG = ConfigDict(
+    model_title_generator=model_title_generator,
+    extra="forbid",
+    str_strip_whitespace=True,
+)
+"""A common pydantic model configuration for use with multiple models."""

src/lsst/cmservice/models/manifests/bps.py

@@ -8,7 +8,7 @@
 
 from pydantic import Field
 
-from . import LibraryManifest, ManifestSpec
+from . import SPEC_CONFIG, LibraryManifest, ManifestSpec
 
 
 class BpsSpec(ManifestSpec):
@@ -17,37 +17,35 @@ class BpsSpec(ManifestSpec):
     a campaign template.
     """
 
-    pipeline_yaml: str | None = Field(default=None)
+    model_config = SPEC_CONFIG
+    pipeline_yaml: str | None = Field(
+        default=None,
+        description="The absolute path to a Pipeline YAML specification file with optional anchor. "
+        "The path must begin with a `/` or a `${...}` environment variable.",
+        pattern="^(/|\\$\\{.*\\})(.*)(\\.yaml)(#.*)?$",
+    )
     variables: dict[str, str] | None = Field(
         default=None,
-        description=(
-            "A mapping of name-value string pairs used to define addtional "
-            "top-level BPS substitution variables. Note that the values are quoted in the "
-            "output."
-        ),
+        description="A mapping of name-value string pairs used to define addtional "
+        "top-level BPS substitution variables. Note that the values are quoted in the "
+        "output.",
     )
     include_files: list[str] | None = Field(default=None)
     literals: dict[str, Any] | None = Field(
         default=None,
-        description=(
-            "A mapping of arbitrary top-level mapping sections to be added as additional literal YAML, "
-            "e.g., `finalJob`."
-        ),
+        description="A mapping of arbitrary top-level mapping sections to be added as additional "
+        "literal YAML, e.g., `finalJob`.",
     )
     environment: dict[str, str] | None = Field(
         default=None,
-        description=(
-            "A mapping of name-value string pairs used to defined additional "
-            "values under the `environment` heading."
-        ),
+        description="A mapping of name-value string pairs used to defined additional "
+        "values under the `environment` heading.",
     )
     payload: dict[str, str] | None = Field(
         default=None,
-        description=(
-            "A mapping of name-value string pairs used to define BPS payload "
-            "options. Note that these values are generated from other configuration "
-            "sources at runtime."
-        ),
+        description="A mapping of name-value string pairs used to define BPS payload "
+        "options. Note that these values are generated from other configuration "
+        "sources at runtime.",
     )
     extra_init_options: str | None = Field(
         default=None, description="Options added to the end of pipetaskinit"
@@ -61,9 +59,8 @@ class BpsSpec(ManifestSpec):
     extra_update_qgraph_options: str | None = Field(default=None)
     clustering: dict[str, Any] | None = Field(
         default=None,
-        description=(
-            "A mapping of clustering directives, added as literal YAML under the `clustering` heading."
-        ),
+        description="A mapping of clustering directives, added as literal YAML "
+        "under the `clustering` heading.",
     )
 
 

src/lsst/cmservice/models/manifests/butler.py

@@ -10,23 +10,15 @@
 
 from pydantic import AliasChoices, BaseModel, Field
 
-from . import LibraryManifest, ManifestSpec
-
-
-class ButlerSpec(ManifestSpec):
-    """Spec model for a Butler Manifest."""
-
-    collections: ButlerCollectionsSpec
-    predicates: Sequence[str] = Field(default_factory=list)
-    repo: str = Field(description="Name of a Butler known to the application's Butler Factory.")
-    include_files: list[str] | None = Field(default=None)
+from . import SPEC_CONFIG, LibraryManifest, ManifestSpec
 
 
 class ButlerCollectionsSpec(BaseModel):
     """Specification for the definition of Butler collections used throughout
     a campaign. This model is part of the "spec" of a Butler Library Manifest.
     """
 
+    model_config = SPEC_CONFIG
     campaign_input: list[str] = Field(
         description="The campaign source collection",
         default_factory=list,
@@ -52,4 +44,14 @@ class ButlerCollectionsSpec(BaseModel):
     )
 
 
+class ButlerSpec(ManifestSpec):
+    """Spec model for a Butler Manifest."""
+
+    model_config = SPEC_CONFIG
+    collections: ButlerCollectionsSpec
+    predicates: Sequence[str] = Field(default_factory=list)
+    repo: str = Field(description="Name of a Butler known to the application's Butler Factory.")
+    include_files: list[str] | None = Field(default=None)
+
+
 class ButlerManifest(LibraryManifest[ButlerSpec]): ...

src/lsst/cmservice/models/manifests/facility.py

@@ -4,10 +4,19 @@
 
 from __future__ import annotations
 
-from . import LibraryManifest, ManifestSpec
+from typing import Literal
 
+from pydantic import Field
 
-class FacilitySpec(ManifestSpec): ...
+from . import SPEC_CONFIG, LibraryManifest, ManifestSpec
+
+
+class FacilitySpec(ManifestSpec):
+    model_config = SPEC_CONFIG
+    facility: Literal["USDF", "IN2P3", "LANC", "RAL"] = Field(
+        default="USDF",
+        description="The name of the processing facility.",
+    )
 
 
 class FacilityManifest(LibraryManifest[FacilitySpec]): ...

src/lsst/cmservice/models/manifests/lsst.py

@@ -9,25 +9,44 @@
 
 from pydantic import Field
 
-from . import LibraryManifest, ManifestSpec
+from . import SPEC_CONFIG, LibraryManifest, ManifestSpec
 
 
 class LsstSpec(ManifestSpec):
     """Spec model for an LSST Manifest."""
 
+    model_config = SPEC_CONFIG
     lsst_version: str = Field(default="w_latest", description="LSST Stack version")
     lsst_distrib_dir: str = Field(description="Absolute path to a stack distribution location")
     prepend: str | None = Field(
         default=None,
         description="A newline-delimited string of shell actions to execute prior to stack setup.",
     )
+    custom_lsst_setup: str | None = Field(
+        default=None,
+        description="A `\n`-delimited string of optional commands to be added to any Bash script that sets "
+        "up the LSST Stack, to be executed verbatim *after* EUPS setup but *before* the payload command. "
+        "Can be used to customize the Stack with EUPS, for example.",
+    )
     append: str | None = Field(
         default=None,
-        description="A newline-delimited string of shell actions to execute after to stack setup.",
+        description="A newline-delimited string of shell actions to execute after the payload command.",
     )
     environment: Mapping = Field(
         default_factory=dict, description="A mapping of environment variables to set prior to stack setup"
     )
+    ticket: str | None = Field(
+        default=None,
+        description="An optional JIRA ticket number associated with the campaign.",
+    )
+    description: str | None = Field(
+        default=None,
+        description="An optional description of the campaign.",
+    )
+    campaign: str | None = Field(default=None, description="An optional name for the campaign.")
+    project: str | None = Field(
+        default=None, description="An optional project name or identifier associated with the campaign."
+    )
 
 
 class LsstManifest(LibraryManifest[LsstSpec]): ...