Agamemnon now specifies separate locations for snapshot images and xrc data (#1783)

* Pass in xrc and snapshot directory from agamemnon

Author: rtuck99

src/mx_bluesky/common/parameters/components.py

@@ -182,6 +182,7 @@ def validate_directories(cls, values):
         # Plans using numtracker currently won't work with snapshot directories:
         # see https://github.com/DiamondLightSource/mx-bluesky/issues/1527
         if values["storage_directory"] != USE_NUMTRACKER:
+            # TODO don't write to storage here https://github.com/DiamondLightSource/mx-bluesky/issues/1780
             os.makedirs(values["storage_directory"], exist_ok=True)
 
             values["snapshot_directory"] = values.get(

src/mx_bluesky/hyperion/blueapi/parameters.py

@@ -38,6 +38,7 @@ class MultiSamplePinTypeParam(BaseModel):
 
 class RobotLoadThenCentreParams(HyperionParam):
     storage_directory: str
+    snapshot_directory: str
     file_name: str
     transmission_frac: float
     exposure_time_s: float
@@ -61,6 +62,7 @@ class MultiRotationScanParams(HyperionParam):
     comment: str
     file_name: str
     storage_directory: str
+    snapshot_directory: str
     exposure_time_s: float
     rotation_increment_deg: float
     snapshot_omegas_deg: list[float]

src/mx_bluesky/hyperion/external_interaction/agamemnon.py

@@ -199,7 +199,11 @@ def _populate_parameters_from_agamemnon(
     with_energy_params = _get_withenergy_parameters_from_agamemnon(agamemnon_params)
     pin_type = _get_pin_type_from_agamemnon_collect_parameters(agamemnon_params)
     collections = agamemnon_params["collection"]
-    visit_directory, file_name = path.split(agamemnon_params["prefix"])
+    rotation_storage_directory, rotation_file_name = path.split(
+        agamemnon_params["prefix"]
+    )
+    xrc_storage_directory, xrc_file_name = path.split(agamemnon_params["xrc_prefix"])
+    snapshot_directory = agamemnon_params["jpegs_dir"]
     use_roi_mode = get_hyperion_feature_settings().XRC_USE_ROI_MODE
     return [
         LoadCentreCollectParams.model_validate(
@@ -215,8 +219,9 @@ def _populate_parameters_from_agamemnon(
                 },
                 **with_energy_params,
                 "robot_load_then_centre": {
-                    "storage_directory": str(visit_directory) + "/xraycentring",
-                    "file_name": file_name,
+                    "snapshot_directory": str(snapshot_directory),
+                    "storage_directory": str(xrc_storage_directory),
+                    "file_name": xrc_file_name,
                     "pin_type": pin_type,
                     "omega_starts_deg": [0.0, 90.0],
                     "chi_start_deg": collection["chi"],
@@ -226,9 +231,10 @@ def _populate_parameters_from_agamemnon(
                 },
                 "multi_rotation_scan": {
                     "comment": collection["comment"],
-                    "storage_directory": str(visit_directory),
+                    "snapshot_directory": str(snapshot_directory),
+                    "storage_directory": str(rotation_storage_directory),
                     "exposure_time_s": collection["exposure_time"],
-                    "file_name": file_name,
+                    "file_name": rotation_file_name,
                     "transmission_frac": collection["transmission"],
                     "rotation_increment_deg": collection["omega_increment"],
                     "ispyb_experiment_type": collection["experiment_type"],

tests/test_data/agamemnon/example_collect_multipin.json

@@ -78,6 +78,8 @@
                 "min": 0.68876
             }
         },
-        "prefix": "/dls/tmp/data/year/cm00000-0/auto/test/test_xtal"
+        "prefix": "/dls/tmp/data/year/cm00000-0/auto/test/test_xtal",
+        "jpegs_dir": "/dls/tmp/data/year/cm00000-0/jpegs/auto/test/test_xtal",
+        "xrc_prefix": "/dls/tmp/data/year/cm00000-0/xraycentring/auto/test/test_xtal"
     }
 }

tests/test_data/agamemnon/example_native.json

@@ -47,6 +47,8 @@
       }
     ],
     "prefix": "/dls/i03/data/2025/mx34598-77/auto/CBLBA/CBLBA-x00242/CBLBA-x00242",
+    "jpegs_dir": "/dls/i03/data/2025/mx34598-77/jpegs/auto/CBLBA/CBLBA-x00242",
+    "xrc_prefix": "/dls/i03/data/2025/mx34598-77/xraycentring/auto/CBLBA/CBLBA-x00242/CBLBA-x00242",
     "recipe_limits": {
       "distance": {
         "max": 800.0,

tests/test_data/parameter_json_files/external_load_centre_collect_params.json

@@ -7,6 +7,7 @@
   "demand_energy_ev": 11100,
   "robot_load_then_centre": {
     "storage_directory": "{tmp_data}/123457/xraycentring",
+    "snapshot_directory": "{tmp_data}/snapshots/123457",
     "file_name": "robot_load_centring_file",
     "transmission_frac": 1.0,
     "exposure_time_s": 0.002,
@@ -21,6 +22,7 @@
     "comment": "Hyperion Rotation Scan - ",
     "file_name": "protk",
     "storage_directory": "{tmp_data}/123457/",
+    "snapshot_directory": "{tmp_data}/snapshots/123457",
     "transmission_frac": 0.05,
     "exposure_time_s": 0.004,
     "rotation_increment_deg": 0.1,

tests/test_data/parameter_json_files/external_load_centre_collect_params_multipin.json

@@ -7,6 +7,7 @@
   "demand_energy_ev": 11100,
   "robot_load_then_centre": {
     "storage_directory": "{tmp_data}/123457/xraycentring",
+    "snapshot_directory": "{tmp_data}/snapshots/123457",
     "file_name": "robot_load_centring_file",
     "transmission_frac": 1.0,
     "exposure_time_s": 0.002,
@@ -24,6 +25,7 @@
     "comment": "Hyperion Rotation Scan - ",
     "file_name": "protk",
     "storage_directory": "{tmp_data}/123457/",
+    "snapshot_directory": "{tmp_data}/snapshots/123457",
     "transmission_frac": 0.05,
     "exposure_time_s": 0.004,
     "rotation_increment_deg": 0.1,

tests/unit_tests/hyperion/experiment_plans/test_load_centre_collect_full_plan.py

@@ -1,5 +1,6 @@
 import dataclasses
 from collections.abc import Sequence
+from pathlib import Path
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock, call, patch
 
@@ -320,6 +321,23 @@ def test_params_with_unexpected_info_in_multi_rotation_scan_rejected(
         LoadCentreCollect(**params)
 
 
+def test_params_with_snapshot_directory_overrides_defaults(tmp_path: Path):
+    params = raw_params_from_file(
+        GOOD_TEST_LOAD_CENTRE_COLLECT_MULTI_ROTATION, tmp_path
+    )
+    params["robot_load_then_centre"]["snapshot_directory"] = (
+        "/some/other/path/to/snapshots"
+    )
+    params["multi_rotation_scan"]["snapshot_directory"] = "/another/path/to/snapshots"
+    load_centre_collect = LoadCentreCollect(**params)
+    assert load_centre_collect.robot_load_then_centre.snapshot_directory == Path(
+        "/some/other/path/to/snapshots"
+    )
+    assert load_centre_collect.multi_rotation_scan.snapshot_directory == Path(
+        "/another/path/to/snapshots"
+    )
+
+
 def test_can_serialize_load_centre_collect_robot_load_params(
     load_centre_collect_params,
 ):

tests/unit_tests/hyperion/external_interaction/test_agamemnon.py

@@ -288,11 +288,11 @@ def test_create_parameters_from_agamemnon_contains_expected_robot_load_then_cent
         assert str(robot_load_params.parameter_model_version) == "6.0.0"
         assert (
             robot_load_params.storage_directory
-            == "/dls/i03/data/2025/mx34598-77/auto/CBLBA/CBLBA-x00242/xraycentring"
+            == "/dls/i03/data/2025/mx34598-77/xraycentring/auto/CBLBA/CBLBA-x00242"
         )
         assert robot_load_params.file_name == "CBLBA-x00242"
         assert robot_load_params.snapshot_directory == PosixPath(
-            "/dls/i03/data/2025/mx34598-77/auto/CBLBA/CBLBA-x00242/xraycentring/snapshots"
+            "/dls/i03/data/2025/mx34598-77/jpegs/auto/CBLBA/CBLBA-x00242"
         )
         assert robot_load_params.use_roi_mode == expected_roi_mode
 
@@ -332,7 +332,7 @@ def test_create_parameters_from_agamemnon_contains_expected_rotation_data(
         )
         assert rotation_params.file_name == "CBLBA-x00242"
         assert rotation_params.snapshot_directory == PosixPath(
-            "/dls/i03/data/2025/mx34598-77/auto/CBLBA/CBLBA-x00242/snapshots"
+            "/dls/i03/data/2025/mx34598-77/jpegs/auto/CBLBA/CBLBA-x00242"
         )
 
     individual_scans = list(