feat(api): Include order in paginated queries

- add crtime to element metadata

Author: tcjennings

src/lsst/cmservice/routers/v2/campaigns.py

@@ -15,6 +15,7 @@
 
 from ...common.graph import graph_from_edge_list_v2, graph_to_dict
 from ...common.logging import LOGGER
+from ...common.timestamp import element_time
 from ...db.campaigns_v2 import Campaign, CampaignUpdate, Edge, Node
 from ...db.manifests_v2 import CampaignManifest
 from ...db.session import db_session_dependency
@@ -42,10 +43,16 @@ async def read_campaign_collection(
     offset: Annotated[int, Query()] = 0,
 ) -> Sequence[Campaign]:
     """A paginated API returning a list of all Campaigns known to the
-    application.
+    application, from newest to oldest.
     """
     try:
-        campaigns = await session.exec(select(Campaign).offset(offset).limit(limit))
+        statement = (
+            select(Campaign)
+            .order_by(Campaign.metadata_["crtime"].desc().nulls_last())
+            .offset(offset)
+            .limit(limit)
+        )
+        campaigns = await session.exec(statement)
 
         response.headers["Next"] = str(
             request.url_for("read_campaign_collection").include_query_params(
@@ -196,15 +203,16 @@ async def read_campaign_node_collection(
 
     # The input could be a campaign UUID or it could be a literal name.
     # TODO this could just as well be a campaign query with a join to nodes
-    s = select(Node)
+    statement = select(Node).order_by(Node.metadata_["crtime"].asc().nulls_last())
+
     try:
         if campaign_id := UUID(campaign_name):
-            s = s.where(Node.namespace == campaign_id)
+            statement = statement.where(Node.namespace == campaign_id)
     except ValueError:
         # FIXME get an id from a name
         raise HTTPException(status_code=422, detail="campaign_name must be a uuid")
-    s = s.offset(offset).limit(limit)
-    nodes = await session.exec(s)
+    statement = statement.offset(offset).limit(limit)
+    nodes = await session.exec(statement)
     response.headers["Next"] = str(
         request.url_for(
             "read_campaign_node_collection",
@@ -252,7 +260,7 @@ async def read_campaign_edge_collection(
             .join_from(Edge, target_nodes, Edge.target == target_nodes.id)
         )
     else:
-        s = select(Edge)
+        s = select(Edge).order_by(col(Edge.name).asc().nulls_last())
     try:
         if campaign_id := UUID(campaign_name):
             s = s.where(Edge.namespace == campaign_id)
@@ -313,10 +321,12 @@ async def create_campaign_resource(
     # Create a campaign spec from the manifest, delegating the creation of new
     # dynamic fields to the model validation method, -OR- create new dynamic
     # fields here.
+    campaign_metadata = manifest.metadata_.model_dump()
+    campaign_metadata |= {"crtime": element_time()}
     campaign = Campaign.model_validate(
         dict(
-            name=manifest.metadata_.name,
-            metadata_=manifest.metadata_.model_dump(),
+            name=campaign_metadata.pop("name"),
+            metadata_=campaign_metadata,
             # owner = ...  # TODO Get username from gafaelfawr # noqa: ERA001
         )
     )
@@ -376,12 +386,8 @@ async def read_campaign_graph(
     edges = (await session.exec(statement)).all()
 
     # Organize the edges into a graph. The graph nodes are annotated with their
-    # current database attributes.
-    # TODO it makes sense for the graph to include expunged Nodes in the meta-
-    # data for campaign processing, but for the purposes of this api route,
-    # only the most relevant information should be associated with each node,
-    # e.g., its name, status, id, and its URL
+    # current database attributes according to the "simple" node view.
     graph = await graph_from_edge_list_v2(edges=edges, node_type=Node, session=session, node_view="simple")
 
-    response.headers["Self"] = ""
+    response.headers["Self"] = str(request.url_for("read_campaign_resource", campaign_name=campaign_id))
     return graph_to_dict(graph)

src/lsst/cmservice/routers/v2/edges.py

@@ -9,7 +9,7 @@
 from uuid import UUID, uuid5
 
 from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
-from sqlmodel import select
+from sqlmodel import col, select
 from sqlmodel.ext.asyncio.session import AsyncSession
 
 from ...common.logging import LOGGER
@@ -46,7 +46,8 @@ async def read_edges_collection(
     For campaign-scoped edges, one should use the /campaigns/{}/edges route.
     """
     try:
-        edges = await session.exec(select(Edge).offset(offset).limit(limit))
+        statement = select(Edge).order_by(col(Edge.name).desc()).offset(offset).limit(limit)
+        edges = await session.exec(statement)
         response.headers["Next"] = (
             request.url_for("read_edges_collection")
             .include_query_params(offset=(offset + limit), limit=limit)

src/lsst/cmservice/routers/v2/manifests.py

@@ -14,6 +14,7 @@
 
 from ...common.jsonpatch import JSONPatch, JSONPatchError, apply_json_patch
 from ...common.logging import LOGGER
+from ...common.timestamp import element_time
 from ...db.campaigns_v2 import Campaign, Manifest, _default_campaign_namespace
 from ...db.manifests_v2 import ManifestModel
 from ...db.session import db_session_dependency
@@ -51,7 +52,13 @@ async def read_manifest_collection(
             )
         )
     try:
-        nodes = await session.exec(select(Manifest).offset(offset).limit(limit))
+        statement = (
+            select(Manifest)
+            .order_by(Manifest.metadata_["crtime"].desc().nulls_last())
+            .offset(offset)
+            .limit(limit)
+        )
+        nodes = await session.exec(statement)
         return nodes.all()
     except Exception as msg:
         logger.exception()
@@ -150,13 +157,16 @@ async def create_one_or_more_manifests(
         _previous = (await session.exec(s)).one_or_none()
         _version = _previous.version if _previous else manifest.metadata_.version
         _version += 1
+
+        _manifest_metadata = manifest.metadata_.model_dump()
+        _manifest_metadata |= {"crtime": element_time()}
         _manifest = Manifest(
             id=uuid5(_namespace_uuid, f"{_name}.{_version}"),
-            name=_name,
+            name=_manifest_metadata.pop("name"),
             namespace=_namespace_uuid,
             kind=manifest.kind,
             version=_version,
-            metadata_=manifest.metadata_.model_dump(),
+            metadata_=_manifest_metadata,
             spec=manifest.spec.model_dump(),
         )
 
@@ -240,6 +250,7 @@ async def update_manifest_resource(
             )
 
     # create Manifest from new_manifest, add to session, and commit
+    new_manifest["metadata"] |= {"crtime": element_time()}
     new_manifest_db = Manifest.model_validate(new_manifest)
     session.add(new_manifest_db)
     await session.commit()

src/lsst/cmservice/routers/v2/nodes.py

@@ -14,6 +14,7 @@
 
 from ...common.jsonpatch import JSONPatch, JSONPatchError, apply_json_patch
 from ...common.logging import LOGGER
+from ...common.timestamp import element_time
 from ...db.campaigns_v2 import Campaign, Node
 from ...db.manifests_v2 import NodeManifest
 from ...db.session import db_session_dependency
@@ -47,7 +48,10 @@ async def read_nodes_collection(
     For campaign-scoped nodes, one should use the /campaigns/{}/nodes route.
     """
     try:
-        nodes = await session.exec(select(Node).offset(offset).limit(limit))
+        statement = (
+            select(Node).order_by(Node.metadata_["crtime"].desc().nulls_last()).offset(offset).limit(limit)
+        )
+        nodes = await session.exec(statement)
         response.headers["Next"] = (
             request.url_for("read_nodes_collection")
             .include_query_params(offset=(offset + limit), limit=limit)
@@ -136,12 +140,15 @@ async def create_node_resource(
 
     node_version = previous_node.version if previous_node else node_version
     node_version += 1
+    node_metadata = manifest.metadata_.model_dump()
+    node_metadata |= {"crtime": element_time()}
     node = Node(
         id=uuid5(node_namespace_uuid, f"{node_name}.{node_version}"),
-        name=node_name,
+        name=node_metadata.pop("name"),
         namespace=node_namespace_uuid,
         version=node_version,
         configuration=manifest.spec.model_dump(),
+        metadata_=node_metadata,
     )
 
     # Put the node in the database
@@ -228,6 +235,7 @@ async def update_node_resource(
             )
 
     # create Manifest from new_manifest, add to session, and commit
+    new_manifest["metadata"] |= {"crtime": element_time()}
     new_manifest_db = Node.model_validate(new_manifest)
     session.add(new_manifest_db)
     await session.commit()