From 80bf00d380a8cd26782fbfcbb268671302e69886 Mon Sep 17 00:00:00 2001
From: Toby Jennings <tobyj@slac.stanford.edu>
Date: Thu, 26 Jun 2025 16:34:07 -0500
Subject: [PATCH 1/3] bugfix(parse): Allow '-' in element names (fixes #3)

---
 src/lsst/cmservice/parsing/string.py | 10 +++++-----
 tests/test_parsing.py                |  1 +
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/lsst/cmservice/parsing/string.py b/src/lsst/cmservice/parsing/string.py
index 24a052ede..dc485a187 100644
--- a/src/lsst/cmservice/parsing/string.py
+++ b/src/lsst/cmservice/parsing/string.py
@@ -48,11 +48,11 @@ def parse_element_fullname(fullname: str) -> Fullname:
     fullname_r = re.compile(
         (
             r"^"
-            r"(?P<campaign>[\w]+){1}(?:\/)*"
-            r"(?P<step>[\w]+){0,1}(?:\/)*"
-            r"(?P<group>[\w]+){0,1}(?:\/)*"
-            r"(?P<job>[\w]+){0,1}(?:\/)*"
-            r"(?P<script>[\w]+){0,1}"
+            r"(?P<campaign>[\w-]+){1}(?:\/)*"
+            r"(?P<step>[\w-]+){0,1}(?:\/)*"
+            r"(?P<group>[\w-]+){0,1}(?:\/)*"
+            r"(?P<job>[\w-]+){0,1}(?:\/)*"
+            r"(?P<script>[\w-]+){0,1}"
             r"$"
         ),
         re.MULTILINE,
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 63fffb30d..101096725 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -11,6 +11,7 @@
         ("campaign_name/step_name/group0", "group"),
         ("campaign_name/step_name", "step"),
         ("campaign_name", "campaign"),
+        ("campaign-name", "campaign"),
     ],
 )
 def test_fullname_parsing(fullname: str, expected: str) -> None:

From ed69840a621b78389b7814345f4f7ad8bf372836 Mon Sep 17 00:00:00 2001
From: Toby Jennings <tobyj@slac.stanford.edu>
Date: Thu, 26 Jun 2025 16:51:16 -0500
Subject: [PATCH 2/3] bugfix(config): add config.bps.artifact_path - removes
 "prod_area" from scope of campaign configuration, set via application config
 instead.

---
 Makefile                                 |  1 +
 src/lsst/cmservice/config.py             |  5 +++++
 src/lsst/cmservice/handlers/jobs.py      |  7 +++----
 src/lsst/cmservice/handlers/scripts.py   | 22 +++++++++++-----------
 tests/fixtures/seeds/empty_config.yaml   |  1 -
 tests/fixtures/seeds/start_trivial.yaml  |  1 -
 tests/fixtures/seeds/test_hsc_micro.yaml |  2 --
 tests/fixtures/seeds/test_trivial.yaml   |  1 -
 8 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/Makefile b/Makefile
index d6d5066b8..3b6a39451 100644
--- a/Makefile
+++ b/Makefile
@@ -107,6 +107,7 @@ test: PGPORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
 test: export DB__URL=postgresql://cm-service@localhost:${PGPORT}/cm-service
 test: export DB__PASSWORD=INSECURE-PASSWORD
 test: export DB__TABLE_SCHEMA=cm_service_test
+test: export BPS__ARTIFACT_PATH=$(PWD)/output
 test: run-compose
 	alembic upgrade head
 	pytest -vvv --asyncio-mode=auto --cov=lsst.cmservice --cov-branch --cov-report=term --cov-report=html ${PYTEST_ARGS}
diff --git a/src/lsst/cmservice/config.py b/src/lsst/cmservice/config.py
index bfb045c15..2bd83bd68 100644
--- a/src/lsst/cmservice/config.py
+++ b/src/lsst/cmservice/config.py
@@ -60,6 +60,11 @@ class BpsConfiguration(BaseModel):
         default=16,
     )
 
+    artifact_path: str = Field(
+        description="Filesystem path location for writing artifacts (`prod_area`)",
+        default="/prod_area",
+    )
+
 
 class ButlerConfiguration(BaseModel):
     """Configuration settings for butler client operations.
diff --git a/src/lsst/cmservice/handlers/jobs.py b/src/lsst/cmservice/handlers/jobs.py
index 0f735b704..9eb5de9f0 100644
--- a/src/lsst/cmservice/handlers/jobs.py
+++ b/src/lsst/cmservice/handlers/jobs.py
@@ -75,11 +75,11 @@ async def _write_script(
         await session.refresh(parent, attribute_names=["c_"])
         data_dict = await script.data_dict(session)
         resolved_cols = await script.resolve_collections(session)
+        prod_area = os.path.expandvars(config.bps.artifact_path)
 
         # Resolve mandatory data element inputs. All of these values must be
         # provided somewhere along the SpecBlock chain.
         try:
-            prod_area = os.path.expandvars(data_dict["prod_area"])
             butler_repo = os.path.expandvars(data_dict["butler_repo"])
             lsst_version = os.path.expandvars(data_dict.get("lsst_version", "w_latest"))
             lsst_distrib_dir = os.path.expandvars(data_dict["lsst_distrib_dir"])
@@ -572,7 +572,7 @@ async def _write_script(
         if TYPE_CHECKING:
             assert isinstance(parent, Job)
         data_dict = await script.data_dict(session)
-        prod_area = await Path(os.path.expandvars(data_dict["prod_area"])).resolve()
+        prod_area = await Path(os.path.expandvars(config.bps.artifact_path)).resolve()
         resolved_cols = await script.resolve_collections(session)
         script_url = await self._set_script_files(session, script, prod_area)
         butler_repo = data_dict["butler_repo"]
@@ -623,8 +623,7 @@ async def _do_prepare(
     ) -> StatusEnum:
         if TYPE_CHECKING:
             assert isinstance(parent, Job)
-        data_dict = await script.data_dict(session)
-        prod_area = await Path(os.path.expandvars(data_dict["prod_area"])).resolve()
+        prod_area = await Path(os.path.expandvars(config.bps.artifact_path)).resolve()
 
         report_url = parent.metadata_.get("report_url") or (
             prod_area / parent.fullname / "manifest_report.yaml"
diff --git a/src/lsst/cmservice/handlers/scripts.py b/src/lsst/cmservice/handlers/scripts.py
index 3b5284270..d9de8549f 100644
--- a/src/lsst/cmservice/handlers/scripts.py
+++ b/src/lsst/cmservice/handlers/scripts.py
@@ -40,7 +40,7 @@ async def _write_script(
         data_dict = await script.data_dict(session)
         try:
             output_coll = resolved_cols["output"]
-            script_url = await self._set_script_files(session, script, data_dict["prod_area"])
+            script_url = await self._set_script_files(session, script, config.bps.artifact_path)
             butler_repo = data_dict["butler_repo"]
         except KeyError as e:
             raise CMMissingScriptInputError(f"{script.fullname} missing an input: {e}") from e
@@ -92,7 +92,7 @@ async def _write_script(
         try:
             output_coll = resolved_cols["output"]
             input_colls = resolved_cols["inputs"]
-            script_url = await self._set_script_files(session, script, data_dict["prod_area"])
+            script_url = await self._set_script_files(session, script, config.bps.artifact_path)
             butler_repo = data_dict["butler_repo"]
         except KeyError as msg:
             logger.exception()
@@ -158,7 +158,7 @@ async def _write_script(
         try:
             output_coll = resolved_cols["output"]
             input_coll = resolved_cols["input"]
-            script_url = await self._set_script_files(session, script, data_dict["prod_area"])
+            script_url = await self._set_script_files(session, script, config.bps.artifact_path)
             butler_repo = data_dict["butler_repo"]
         except KeyError as msg:
             raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg
@@ -238,7 +238,7 @@ async def _write_script(
             raise CMMissingScriptInputError(
                 "Must specify what to collect in ChainCollectScriptHandler, jobs or steps",
             )
-        script_url = await self._set_script_files(session, script, data_dict["prod_area"])
+        script_url = await self._set_script_files(session, script, config.bps.artifact_path)
         butler_repo = data_dict["butler_repo"]
         command = f"{config.butler.butler_bin} collection-chain {butler_repo} {output_coll}"
         for collect_coll_ in collect_colls:
@@ -298,7 +298,7 @@ async def _write_script(
         try:
             output_coll = resolved_cols["output"]
             input_coll = resolved_cols["input"]
-            script_url = await self._set_script_files(session, script, data_dict["prod_area"])
+            script_url = await self._set_script_files(session, script, config.bps.artifact_path)
             butler_repo = data_dict["butler_repo"]
             data_query = data_dict.get("data_query")
         except KeyError as msg:
@@ -354,7 +354,7 @@ async def _write_script(
         data_dict = await script.data_dict(session)
         try:
             output_coll = resolved_cols["output"]
-            script_url = await self._set_script_files(session, script, data_dict["prod_area"])
+            script_url = await self._set_script_files(session, script, config.bps.artifact_path)
             butler_repo = data_dict["butler_repo"]
         except KeyError as msg:
             raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg
@@ -410,7 +410,7 @@ async def _write_script(
         try:
             input_coll = resolved_cols["input"]
             output_coll = resolved_cols["output"]
-            script_url = await self._set_script_files(session, script, data_dict["prod_area"])
+            script_url = await self._set_script_files(session, script, config.bps.artifact_path)
             butler_repo = data_dict["butler_repo"]
         except KeyError as msg:
             raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg
@@ -474,7 +474,7 @@ async def _write_script(
         resolved_cols = await script.resolve_collections(session)
         data_dict = await script.data_dict(session)
         try:
-            script_url = await self._set_script_files(session, script, data_dict["prod_area"])
+            script_url = await self._set_script_files(session, script, config.bps.artifact_path)
             butler_repo = data_dict["butler_repo"]
             output_coll = resolved_cols["output"]
         except KeyError as msg:
@@ -534,7 +534,7 @@ async def _write_script(
     ) -> StatusEnum:
         resolved_cols = await script.resolve_collections(session)
         data_dict = await script.data_dict(session)
-        prod_area = os.path.expandvars(data_dict["prod_area"])
+        prod_area = os.path.expandvars(config.bps.artifact_path)
         script_url = await self._set_script_files(session, script, prod_area)
         butler_repo = data_dict["butler_repo"]
         usage_graph_url = os.path.expandvars(f"{prod_area}/{parent.fullname}/resource_usage.qgraph")
@@ -591,7 +591,7 @@ async def _write_script(
     ) -> StatusEnum:
         resolved_cols = await script.resolve_collections(session)
         data_dict = await script.data_dict(session)
-        prod_area = os.path.expandvars(data_dict["prod_area"])
+        prod_area = os.path.expandvars(config.bps.artifact_path)
         script_url = await self._set_script_files(session, script, prod_area)
         butler_repo = data_dict["butler_repo"]
         hips_maps_graph_url = os.path.expandvars(f"{prod_area}/{parent.fullname}/hips_maps.qgraph")
@@ -693,7 +693,7 @@ async def _write_script(
         try:
             input_coll = resolved_cols["input"]
             output_coll = resolved_cols["output"]
-            script_url = await self._set_script_files(session, script, data_dict["prod_area"])
+            script_url = await self._set_script_files(session, script, config.bps.artifact_path)
             butler_repo = data_dict["butler_repo"]
         except KeyError as msg:
             raise CMMissingScriptInputError(f"{script.fullname} missing an input: {msg}") from msg
diff --git a/tests/fixtures/seeds/empty_config.yaml b/tests/fixtures/seeds/empty_config.yaml
index 1b8f1d40b..d30242b50 100644
--- a/tests/fixtures/seeds/empty_config.yaml
+++ b/tests/fixtures/seeds/empty_config.yaml
@@ -29,7 +29,6 @@ Campaign:
       campaign_validation: "{out}/validate"
     data:
       butler_repo: "/repo/main"
-      prod_area: "output/archive"
       data_query: "instrument = 'HSC'"
       lsst_version: "${WEEKLY}"
 - Specification:
diff --git a/tests/fixtures/seeds/start_trivial.yaml b/tests/fixtures/seeds/start_trivial.yaml
index 4cf245675..ce8bccea1 100644
--- a/tests/fixtures/seeds/start_trivial.yaml
+++ b/tests/fixtures/seeds/start_trivial.yaml
@@ -6,6 +6,5 @@ Campaign:
     campaign_source: HSC/raw/RC2
   data:
     lsst_version: w_2024_38
-    prod_area: /tmp/trivial
     butler_repo: "/repo/main"
     data_query: "instrument = 'HSC' AND exposure in (30504, 30502) AND detector in (45, 46, 47, 48)"
diff --git a/tests/fixtures/seeds/test_hsc_micro.yaml b/tests/fixtures/seeds/test_hsc_micro.yaml
index f32ae2457..bca8fe430 100644
--- a/tests/fixtures/seeds/test_hsc_micro.yaml
+++ b/tests/fixtures/seeds/test_hsc_micro.yaml
@@ -23,7 +23,6 @@
             split_min_groups: 2
     data:
       butler_repo: "/repo/main"
-      prod_area: "output/archive"
       data_query: "instrument = 'HSC' AND exposure in (30504, 30502) AND detector in (45, 46, 47, 48)"
       lsst_version: w_2025_01
       bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml"
@@ -65,7 +64,6 @@
       campaign_validation: "{out}/validate"
     data:
       butler_repo: "/repo/main"
-      prod_area: "output/archive"
       data_query: "instrument = 'HSC'"
       lsst_version: "${WEEKLY}"
 - Specification:
diff --git a/tests/fixtures/seeds/test_trivial.yaml b/tests/fixtures/seeds/test_trivial.yaml
index a51b58c30..c9ee849a6 100644
--- a/tests/fixtures/seeds/test_trivial.yaml
+++ b/tests/fixtures/seeds/test_trivial.yaml
@@ -16,7 +16,6 @@
             split_method: no_split
     data:
       butler_repo: "/repo/main"
-      prod_area: "output/archive"
       data_query: "instrument = 'HSC' AND exposure in (30504, 30502) AND detector in (45, 46, 47, 48)"
       lsst_version: w_2023_46
 - Specification:

From cc17430dd74c9977fc888ba1542c7b229f2b737d Mon Sep 17 00:00:00 2001
From: Toby Jennings <tobyj@slac.stanford.edu>
Date: Thu, 26 Jun 2025 17:02:19 -0500
Subject: [PATCH 3/3] bugfix(daemon): Correct types in due date checks.

---
 src/lsst/cmservice/common/daemon.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lsst/cmservice/common/daemon.py b/src/lsst/cmservice/common/daemon.py
index 0a8cb5855..92f5e8034 100644
--- a/src/lsst/cmservice/common/daemon.py
+++ b/src/lsst/cmservice/common/daemon.py
@@ -18,11 +18,11 @@
 async def check_due_date(session: async_scoped_session, node: NodeMixin, time_next_check: datetime) -> None:
     """For a provided due date, check if the queue entry is overdue"""
 
-    due_date = node.metadata_.get("due_date", None)
+    due_date: int = node.metadata_.get("due_date", None)
     if due_date is None:
         return None
 
-    if time_next_check > due_date:
+    if time_next_check.timestamp() > due_date:
         campaign = await node.get_campaign(session)
         await notification.send_notification(for_status=StatusEnum.overdue, for_campaign=campaign)