Merged recent changes from 'main' branch

tieneupin · tieneupin · commit 797dba536aba · 2025-07-04T18:16:54.000+01:00
diff --git a/.bumpversion.toml b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "1.2.3"
+current_version = "1.2.4"
 commit = true
 tag = true
 
diff --git a/Helm/Chart.yaml b/Helm/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: cryoem-services
 description: An umbrella Helm chart for EM processing services
-version: 1.2.3
+version: 1.2.4
 dependencies:
   - name: bfactor_setup
   - name: clem_align_and_merge
diff --git a/Helm/charts/bfactor_setup/Chart.yaml b/Helm/charts/bfactor_setup/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: bfactor_setup
 description: A Helm chart for the service selecting particles to use for bfactors
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/cluster_submission/Chart.yaml b/Helm/charts/cluster_submission/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: cluster_submission
 description: A Helm chart for the cluster submission service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/cryolo/Chart.yaml b/Helm/charts/cryolo/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: cryolo
 description: A Helm chart for CrYOLO service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/cryolo_gpu/Chart.yaml b/Helm/charts/cryolo_gpu/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: cryolo_gpu
 description: A Helm chart for CrYOLO service running on a GPU
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/ctffind/Chart.yaml b/Helm/charts/ctffind/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: ctffind
 description: A Helm chart for CTFFind service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/denoise/Chart.yaml b/Helm/charts/denoise/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: denoise
 description: A Helm chart for EM Denoise service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/denoise_slurm/Chart.yaml b/Helm/charts/denoise_slurm/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: denoise_slurm
 description: A Helm chart for EM Denoise service which submits jobs to the IRIS cluster
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/extract/Chart.yaml b/Helm/charts/extract/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: extract
 description: A Helm chart for Extraction service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/extract_class/Chart.yaml b/Helm/charts/extract_class/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: extract_class
 description: A Helm chart for the extraction of particles for refinement
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/icebreaker/Chart.yaml b/Helm/charts/icebreaker/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: icebreaker
 description: A Helm chart for IceBreaker service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/images/Chart.yaml b/Helm/charts/images/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: images
 description: A Helm chart for the Images service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/ispyb_connector/Chart.yaml b/Helm/charts/ispyb_connector/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: ispyb_connector
 description: A Helm chart for the EMISPyB service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/membrain_seg/Chart.yaml b/Helm/charts/membrain_seg/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: membrain_seg
 description: A Helm chart for membrain-seg service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/motioncorr/Chart.yaml b/Helm/charts/motioncorr/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: motioncorr
 description: A Helm chart for MotionCorr service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/node_creator/Chart.yaml b/Helm/charts/node_creator/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: node_creator
 description: A Helm chart for node creation
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/postprocess/Chart.yaml b/Helm/charts/postprocess/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: postprocess
 description: A Helm chart for the Relion post-processing service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/process_recipe/Chart.yaml b/Helm/charts/process_recipe/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: process_recipe
 description: A Helm chart for the recipe processing service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/select_classes/Chart.yaml b/Helm/charts/select_classes/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: select_classes
 description: A Helm chart for class selection service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/select_particles/Chart.yaml b/Helm/charts/select_particles/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: select_particles
 description: A Helm chart for particle selection service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/tomo_align/Chart.yaml b/Helm/charts/tomo_align/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: tomo_align
 description: A Helm chart for TomoAlign tomography service
-version: 1.2.3
+version: 1.2.4
diff --git a/Helm/charts/tomo_align_slurm/Chart.yaml b/Helm/charts/tomo_align_slurm/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 name: tomo_align_slurm
 description: A Helm chart for TomoAlign service which submits jobs to the IRIS cluster
-version: 1.2.3
+version: 1.2.4
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "cryoemservices"
-version = "1.2.3"
+version = "1.2.4"
 description = "Services for CryoEM processing"
 readme = "README.md"
 keywords = [
diff --git a/recipes/em-tomo-align.json b/recipes/em-tomo-align.json
@@ -87,7 +87,8 @@
     "output": {
       "images": 6,
       "ispyb_connector": 10,
-      "movie": 6
+      "movie": 6,
+      "node_creator": 11
     },
     "queue": "segmentation",
     "service": "MembrainSeg"
diff --git a/src/cryoemservices/__init__.py b/src/cryoemservices/__init__.py
@@ -1,3 +1,3 @@
 from __future__ import annotations
 
-__version__ = "1.2.3"
+__version__ = "1.2.4"
diff --git a/src/cryoemservices/pipeliner_plugins/combine_star_files.py b/src/cryoemservices/pipeliner_plugins/combine_star_files.py
@@ -38,9 +38,10 @@ def combine_star_files(files_to_process: List[Path], output_dir: Path):
 
     # Make a temporary star file to get the table headings from
     reference_optics = None
-    with open(files_to_process[0], "r") as full_starfile, open(
-        output_dir / ".particles_tmp.star", "w"
-    ) as tmp_starfile:
+    with (
+        open(files_to_process[0], "r") as full_starfile,
+        open(output_dir / ".particles_tmp.star", "w") as tmp_starfile,
+    ):
         for line_counter in range(50):
             line = full_starfile.readline()
             if line.startswith("opticsGroup"):
@@ -95,9 +96,10 @@ def combine_star_files(files_to_process: List[Path], output_dir: Path):
 
         # Add the particles lines to the final star file
         file_particles_count = 0
-        with open(split_file, "r") as added_starfile, open(
-            output_dir / ".particles_all_tmp.star", "a"
-        ) as particles_file:
+        with (
+            open(split_file, "r") as added_starfile,
+            open(output_dir / ".particles_all_tmp.star", "a") as particles_file,
+        ):
             while True:
                 particle_line = added_starfile.readline()
                 if not particle_line:
@@ -131,20 +133,22 @@ def split_star_file(
     """
 
     # Make a temporary star file to get the table headings from
-    with open(file_to_process, "r") as full_starfile, open(
-        output_dir / ".particles_tmp.star", "w"
-    ) as tmp_starfile:
+    with (
+        open(file_to_process, "r") as full_starfile,
+        open(output_dir / ".particles_tmp.star", "w") as tmp_starfile,
+    ):
         for line_counter in range(50):
             line = full_starfile.readline()
             if not line:
+                line_counter -= 1
                 break
             tmp_starfile.write(line)
 
     star_dictionary = starfile.read(output_dir / ".particles_tmp.star")
     (output_dir / ".particles_tmp.star").unlink()
 
     # Find the number of lines in the full file
-    starfile_starter_lines = line_counter - star_dictionary["particles"].shape[0]
+    starfile_starter_lines = line_counter + 1 - star_dictionary["particles"].shape[0]
     count = 0
     with open(file_to_process, "r") as full_starfile:
         for count, line in enumerate(full_starfile):
diff --git a/src/cryoemservices/services/cryolo.py b/src/cryoemservices/services/cryolo.py
@@ -212,6 +212,7 @@ def cryolo(self, rw, header: dict, message: dict):
         )
 
         # Create the config file
+        cryolo_params.relion_options.cryolo_box_size = cryolo_params.cryolo_box_size
         cryolo_params.relion_options.cryolo_config_file = str(
             job_dir / "cryolo_config.json"
         )
@@ -246,7 +247,8 @@ def cryolo(self, rw, header: dict, message: dict):
         # Register the cryolo job with the node creator
         self.log.info(f"Sending {self.job_type} to node creator")
         node_creator_parameters: dict[str, Any] = {
-            "job_type": self.job_type,
+            "job_type": self.job_type
+            + (".tomo" if cryolo_params.experiment_type == "tomography" else ""),
             "input_file": cryolo_params.input_path,
             "output_file": cryolo_params.output_path,
             "relion_options": dict(cryolo_params.relion_options),
diff --git a/src/cryoemservices/services/denoise.py b/src/cryoemservices/services/denoise.py
@@ -260,6 +260,8 @@ def denoise(self, rw, header: dict, message: dict):
         segmentation_parameters = {
             "tomogram": str(denoised_full_path),
             "output_dir": str(segmentation_dir),
+            "pixel_size": str(denoise_params.relion_options.pixel_size_downscaled),
+            "relion_options": dict(denoise_params.relion_options),
         }
         cryolo_parameters = {
             "input_path": str(denoised_full_path),
diff --git a/src/cryoemservices/services/membrain_seg.py b/src/cryoemservices/services/membrain_seg.py
@@ -9,6 +9,7 @@
 
 from cryoemservices.services.common_service import CommonService
 from cryoemservices.util.models import MockRW
+from cryoemservices.util.relion_service_options import RelionServiceOptions
 from cryoemservices.util.slurm_submission import slurm_submission_for_services
 
 
@@ -28,6 +29,7 @@ class MembrainSegParameters(BaseModel):
     segmentation_threshold: Optional[float] = None
     cleanup_output: bool = True
     submit_to_slurm: bool = False
+    relion_options: RelionServiceOptions
 
 
 class MembrainSeg(CommonService):
@@ -38,6 +40,9 @@ class MembrainSeg(CommonService):
     # Logger name
     _logger_name = "cryoemservices.services.membrain_seg"
 
+    # Job name
+    job_type = "membrain.segment"
+
     def initializing(self):
         """Subscribe to a queue. Received messages must be acknowledged."""
         self.log.info("membrain-seg service starting")
@@ -153,6 +158,23 @@ def membrain_seg(self, rw, header: dict, message: dict):
         else:
             result = subprocess.run(command, capture_output=True)
 
+        # Send to node creator
+        self.log.info("Sending segmentation to node creator")
+        node_creator_parameters = {
+            "experiment_type": "tomography",
+            "job_type": self.job_type,
+            "input_file": membrain_seg_params.tomogram,
+            "output_file": str(segmented_path),
+            "relion_options": dict(membrain_seg_params.relion_options),
+            "command": " ".join(command),
+            "stdout": result.stdout.decode("utf8", "replace"),
+            "stderr": result.stderr.decode("utf8", "replace"),
+            "success": True,
+        }
+        if result.returncode:
+            node_creator_parameters["success"] = False
+        rw.send_to("node_creator", node_creator_parameters)
+
         # Stop here if the job failed
         if result.returncode:
             self.log.error(
diff --git a/src/cryoemservices/services/node_creator.py b/src/cryoemservices/services/node_creator.py
@@ -81,7 +81,7 @@ def adjust_job_counter(pipeline_name: str, job_number: int):
     "relion.import.movies": {"folder": "Import", "spa_input": {"fn_in_raw": "*.tiff"}},
     "relion.importtomo": {
         "folder": "Import",
-        "tomography_input": {"movie_files": "*.tiff", "mdoc_files": "*.mdoc"},
+        "tomography_input": {"fn_in_raw": "*.tiff", "fn_mdoc": "*.mdoc"},
     },
     "relion.motioncorr.own": {
         "folder": "MotionCorr",
@@ -113,7 +113,6 @@ def adjust_job_counter(pipeline_name: str, job_number: int):
     "cryolo.autopick": {
         "folder": "AutoPick",
         "spa_input": {"input_file": "corrected_micrographs.star"},
-        "tomography_input": {"input_file": "tomograms.star"},  # should be in_tomoset
     },
     "relion.extract": {
         "folder": "Extract",
@@ -186,7 +185,7 @@ def adjust_job_counter(pipeline_name: str, job_number: int):
         "folder": "ExcludeTiltImages",
         "tomography_input": {"in_tiltseries": "tilt_series_ctf.star"},
     },
-    "relion.aligntiltseries": {
+    "relion.aligntiltseries.aretomo": {
         "folder": "AlignTiltSeries",
         "tomography_input": {"in_tiltseries": "selected_tilt_series.star"},
     },
@@ -198,6 +197,14 @@ def adjust_job_counter(pipeline_name: str, job_number: int):
         "folder": "Denoise",
         "tomography_input": {"in_tomoset": "tomograms.star"},
     },
+    "membrain.segment": {
+        "folder": "Segmentation",
+        "tomography_input": {"in_tomoset": "tomograms.star"},
+    },
+    "cryolo.autopick.tomo": {
+        "folder": "AutoPick",
+        "tomography_input": {"input_file": "tomograms.star"},
+    },
 }
 
 
@@ -353,8 +360,8 @@ def node_creator(self, rw, header: dict, message: dict):
         elif job_info.job_type == "relion.import.movies":
             pipeline_options["fn_in_raw"] = job_info.input_file
         elif job_info.job_type == "relion.importtomo":
-            pipeline_options["movie_files"] = job_info.input_file.split(":")[0]
-            pipeline_options["mdoc_files"] = job_info.input_file.split(":")[1]
+            pipeline_options["fn_in_raw"] = job_info.input_file.split(":")[0]
+            pipeline_options["fn_mdoc"] = job_info.input_file.split(":")[1]
 
         # Mark the job completion status
         job_is_continue = False
@@ -424,7 +431,7 @@ def node_creator(self, rw, header: dict, message: dict):
         # Load this job as a pipeliner job to create the nodes
         pipeliner_job = read_job(f"{job_dir}/job.star")
         pipeliner_job.output_dir = str(relative_job_dir) + "/"
-        relion_commands = pipeliner_job.get_final_commands()
+        relion_commands = pipeliner_job.get_commands()
 
         # These parts would normally happen in pipeliner_job.prepare_to_run
         pipeliner_job.create_input_nodes()
@@ -548,7 +555,9 @@ def node_creator(self, rw, header: dict, message: dict):
             if not (job_dir / ".CCPEM_pipeliner_jobinfo").exists():
                 for command in relion_commands:
                     update_jobinfo_file(
-                        process.name, action="Run", command_list=command
+                        process.name,
+                        action="Run",
+                        command_list=command.get_final_command(output_dir=str(job_dir)),
                     )
             # Generate the default_pipeline.star file
             project.check_process_completion()
diff --git a/src/cryoemservices/services/select_classes.py b/src/cryoemservices/services/select_classes.py
diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py
diff --git a/src/cryoemservices/util/relion_service_options.py b/src/cryoemservices/util/relion_service_options.py
diff --git a/src/cryoemservices/util/tomo_output_files.py b/src/cryoemservices/util/tomo_output_files.py
diff --git a/tests/services/test_cryolo_service.py b/tests/services/test_cryolo_service.py
diff --git a/tests/services/test_denoise.py b/tests/services/test_denoise.py
diff --git a/tests/services/test_membrain_seg.py b/tests/services/test_membrain_seg.py
diff --git a/tests/services/test_node_creator.py b/tests/services/test_node_creator.py
diff --git a/tests/services/test_select_classes_service.py b/tests/services/test_select_classes_service.py
diff --git a/tests/services/test_tomo_align.py b/tests/services/test_tomo_align.py
