feat(api): Include order in paginated queries

- add crtime to element metadata
- add manifests route for campaigns

Author: tcjennings

src/lsst/cmservice/routers/v2/campaigns.py

@@ -15,7 +15,8 @@
 
 from ...common.graph import graph_from_edge_list_v2, graph_to_dict
 from ...common.logging import LOGGER
-from ...db.campaigns_v2 import Campaign, CampaignUpdate, Edge, Node
+from ...common.timestamp import element_time
+from ...db.campaigns_v2 import Campaign, CampaignUpdate, Edge, Manifest, Node
 from ...db.manifests_v2 import CampaignManifest
 from ...db.session import db_session_dependency
 
@@ -42,10 +43,16 @@ async def read_campaign_collection(
     offset: Annotated[int, Query()] = 0,
 ) -> Sequence[Campaign]:
     """A paginated API returning a list of all Campaigns known to the
-    application.
+    application, from newest to oldest.
     """
     try:
-        campaigns = await session.exec(select(Campaign).offset(offset).limit(limit))
+        statement = (
+            select(Campaign)
+            .order_by(Campaign.metadata_["crtime"].desc().nulls_last())
+            .offset(offset)
+            .limit(limit)
+        )
+        campaigns = await session.exec(statement)
 
         response.headers["Next"] = str(
             request.url_for("read_campaign_collection").include_query_params(
@@ -95,6 +102,9 @@ async def read_campaign_resource(
         response.headers["Edges"] = str(
             request.url_for("read_campaign_edge_collection", campaign_name=campaign.id)
         )
+        response.headers["Manifests"] = str(
+            request.url_for("read_campaign_manifest_collection", campaign_name=campaign.id)
+        )
         return campaign
     else:
         raise HTTPException(status_code=404)
@@ -196,22 +206,60 @@ async def read_campaign_node_collection(
 
     # The input could be a campaign UUID or it could be a literal name.
     # TODO this could just as well be a campaign query with a join to nodes
-    s = select(Node)
+    statement = select(Node).order_by(Node.metadata_["crtime"].asc().nulls_last())
+
     try:
         if campaign_id := UUID(campaign_name):
-            s = s.where(Node.namespace == campaign_id)
+            statement = statement.where(Node.namespace == campaign_id)
     except ValueError:
         # FIXME get an id from a name
         raise HTTPException(status_code=422, detail="campaign_name must be a uuid")
-    s = s.offset(offset).limit(limit)
-    nodes = await session.exec(s)
+    statement = statement.offset(offset).limit(limit)
+    nodes = await session.exec(statement)
     response.headers["Next"] = str(
         request.url_for(
             "read_campaign_node_collection",
-            campaign_name=campaign_name,
+            campaign_name=campaign_id,
         ).include_query_params(offset=(offset + limit), limit=limit),
     )
-    # TODO Previous
+    response.headers["Self"] = str(request.url_for("read_campaign_resource", campaign_name=campaign_id))
+    return nodes.all()
+
+
+@router.get(
+    "/{campaign_name}/manifests",
+    summary="Get campaign Manifests",
+)
+async def read_campaign_manifest_collection(
+    request: Request,
+    response: Response,
+    session: Annotated[AsyncSession, Depends(db_session_dependency)],
+    campaign_name: str,
+    limit: Annotated[int, Query(le=100)] = 10,
+    offset: Annotated[int, Query()] = 0,
+) -> Sequence[Manifest]:
+    """A paginated API returning a list of all Manifests in the namespace of a
+    single Campaign.
+    """
+
+    # The input could be a campaign UUID or it could be a literal name.
+    statement = select(Manifest).order_by(Manifest.metadata_["crtime"].asc().nulls_last())
+
+    try:
+        if campaign_id := UUID(campaign_name):
+            statement = statement.where(Manifest.namespace == campaign_id)
+    except ValueError:
+        # FIXME get an id from a name
+        raise HTTPException(status_code=422, detail="campaign_name must be a uuid")
+    statement = statement.offset(offset).limit(limit)
+    nodes = await session.exec(statement)
+    response.headers["Next"] = str(
+        request.url_for(
+            "read_campaign_manifest_collection",
+            campaign_name=campaign_id,
+        ).include_query_params(offset=(offset + limit), limit=limit),
+    )
+    response.headers["Self"] = str(request.url_for("read_campaign_resource", campaign_name=campaign_id))
     return nodes.all()
 
 
@@ -252,14 +300,16 @@ async def read_campaign_edge_collection(
             .join_from(Edge, target_nodes, Edge.target == target_nodes.id)
         )
     else:
-        s = select(Edge)
+        s = select(Edge).order_by(col(Edge.name).asc().nulls_last())
     try:
         if campaign_id := UUID(campaign_name):
             s = s.where(Edge.namespace == campaign_id)
     except ValueError:
         # FIXME get an id from a name
         raise HTTPException(status_code=422, detail="campaign_name must be a uuid")
     edges = await session.exec(s)
+
+    response.headers["Self"] = str(request.url_for("read_campaign_resource", campaign_name=campaign_id))
     return edges.all()
 
 
@@ -313,10 +363,12 @@ async def create_campaign_resource(
     # Create a campaign spec from the manifest, delegating the creation of new
     # dynamic fields to the model validation method, -OR- create new dynamic
     # fields here.
+    campaign_metadata = manifest.metadata_.model_dump()
+    campaign_metadata |= {"crtime": element_time()}
     campaign = Campaign.model_validate(
         dict(
-            name=manifest.metadata_.name,
-            metadata_=manifest.metadata_.model_dump(),
+            name=campaign_metadata.pop("name"),
+            metadata_=campaign_metadata,
             # owner = ...  # TODO Get username from gafaelfawr # noqa: ERA001
         )
     )
@@ -376,12 +428,8 @@ async def read_campaign_graph(
     edges = (await session.exec(statement)).all()
 
     # Organize the edges into a graph. The graph nodes are annotated with their
-    # current database attributes.
-    # TODO it makes sense for the graph to include expunged Nodes in the meta-
-    # data for campaign processing, but for the purposes of this api route,
-    # only the most relevant information should be associated with each node,
-    # e.g., its name, status, id, and its URL
+    # current database attributes according to the "simple" node view.
     graph = await graph_from_edge_list_v2(edges=edges, node_type=Node, session=session, node_view="simple")
 
-    response.headers["Self"] = ""
+    response.headers["Self"] = str(request.url_for("read_campaign_resource", campaign_name=campaign_id))
     return graph_to_dict(graph)

src/lsst/cmservice/routers/v2/edges.py

@@ -9,7 +9,7 @@
 from uuid import UUID, uuid5
 
 from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
-from sqlmodel import select
+from sqlmodel import col, select
 from sqlmodel.ext.asyncio.session import AsyncSession
 
 from ...common.logging import LOGGER
@@ -46,7 +46,8 @@ async def read_edges_collection(
     For campaign-scoped edges, one should use the /campaigns/{}/edges route.
     """
     try:
-        edges = await session.exec(select(Edge).offset(offset).limit(limit))
+        statement = select(Edge).order_by(col(Edge.name).desc()).offset(offset).limit(limit)
+        edges = await session.exec(statement)
         response.headers["Next"] = (
             request.url_for("read_edges_collection")
             .include_query_params(offset=(offset + limit), limit=limit)

src/lsst/cmservice/routers/v2/manifests.py

@@ -14,6 +14,7 @@
 
 from ...common.jsonpatch import JSONPatch, JSONPatchError, apply_json_patch
 from ...common.logging import LOGGER
+from ...common.timestamp import element_time
 from ...db.campaigns_v2 import Campaign, Manifest, _default_campaign_namespace
 from ...db.manifests_v2 import ManifestModel
 from ...db.session import db_session_dependency
@@ -51,7 +52,13 @@ async def read_manifest_collection(
             )
         )
     try:
-        nodes = await session.exec(select(Manifest).offset(offset).limit(limit))
+        statement = (
+            select(Manifest)
+            .order_by(Manifest.metadata_["crtime"].desc().nulls_last())
+            .offset(offset)
+            .limit(limit)
+        )
+        nodes = await session.exec(statement)
         return nodes.all()
     except Exception as msg:
         logger.exception()
@@ -150,13 +157,16 @@ async def create_one_or_more_manifests(
         _previous = (await session.exec(s)).one_or_none()
         _version = _previous.version if _previous else manifest.metadata_.version
         _version += 1
+
+        _manifest_metadata = manifest.metadata_.model_dump()
+        _manifest_metadata |= {"crtime": element_time()}
         _manifest = Manifest(
             id=uuid5(_namespace_uuid, f"{_name}.{_version}"),
-            name=_name,
+            name=_manifest_metadata.pop("name"),
             namespace=_namespace_uuid,
             kind=manifest.kind,
             version=_version,
-            metadata_=manifest.metadata_.model_dump(),
+            metadata_=_manifest_metadata,
             spec=manifest.spec.model_dump(),
         )
 
@@ -240,6 +250,7 @@ async def update_manifest_resource(
             )
 
     # create Manifest from new_manifest, add to session, and commit
+    new_manifest["metadata"] |= {"crtime": element_time()}
     new_manifest_db = Manifest.model_validate(new_manifest)
     session.add(new_manifest_db)
     await session.commit()

src/lsst/cmservice/routers/v2/nodes.py

@@ -14,6 +14,7 @@
 
 from ...common.jsonpatch import JSONPatch, JSONPatchError, apply_json_patch
 from ...common.logging import LOGGER
+from ...common.timestamp import element_time
 from ...db.campaigns_v2 import Campaign, Node
 from ...db.manifests_v2 import NodeManifest
 from ...db.session import db_session_dependency
@@ -47,7 +48,10 @@ async def read_nodes_collection(
     For campaign-scoped nodes, one should use the /campaigns/{}/nodes route.
     """
     try:
-        nodes = await session.exec(select(Node).offset(offset).limit(limit))
+        statement = (
+            select(Node).order_by(Node.metadata_["crtime"].desc().nulls_last()).offset(offset).limit(limit)
+        )
+        nodes = await session.exec(statement)
         response.headers["Next"] = (
             request.url_for("read_nodes_collection")
             .include_query_params(offset=(offset + limit), limit=limit)
@@ -136,12 +140,15 @@ async def create_node_resource(
 
     node_version = previous_node.version if previous_node else node_version
     node_version += 1
+    node_metadata = manifest.metadata_.model_dump()
+    node_metadata |= {"crtime": element_time()}
     node = Node(
         id=uuid5(node_namespace_uuid, f"{node_name}.{node_version}"),
-        name=node_name,
+        name=node_metadata.pop("name"),
         namespace=node_namespace_uuid,
         version=node_version,
         configuration=manifest.spec.model_dump(),
+        metadata_=node_metadata,
     )
 
     # Put the node in the database
@@ -228,6 +235,7 @@ async def update_node_resource(
             )
 
     # create Manifest from new_manifest, add to session, and commit
+    new_manifest["metadata"] |= {"crtime": element_time()}
     new_manifest_db = Node.model_validate(new_manifest)
     session.add(new_manifest_db)
     await session.commit()

tests/v2/test_manifest_routes.py

@@ -131,6 +131,13 @@ async def test_load_manifests(aclient: AsyncClient) -> None:
     assert len(manifests) == 3
     assert manifests[-1]["spec"]["one"] == 1
 
+    # Get all the loaded manifests from the campaign route
+    x = await aclient.get(f"/cm-service/v2/campaigns/{campaign_id}/manifests")
+    assert x.is_success
+    manifests = x.json()
+    assert len(manifests) == 2
+    assert manifests[-1]["spec"]["one"] == 1
+
 
 async def test_patch_manifest(aclient: AsyncClient) -> None:
     """Tests partial update of manifests and single resource retrieval."""