Removal of redundant tomography context code (#381)

Author: stephen-riggs

src/murfey/client/contexts/tomo.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 from pathlib import Path
 from threading import RLock
-from typing import Callable, Dict, List, Optional, OrderedDict
+from typing import Callable, Dict, List, OrderedDict
 
 import requests
 import xmltodict
@@ -38,7 +38,7 @@ def _get_tilt_angle_v5_11(p: Path) -> str:
 
 def _find_angle_index(split_name: List[str]) -> int:
     for i, part in enumerate(split_name):
-        if "." in part and part[0].isnumeric():
+        if "." in part and part[-1].isnumeric():
             return i
     return -1
 
@@ -110,8 +110,6 @@ def __init__(self, acquisition_software: str, basepath: Path):
         self._tilt_series_sizes: Dict[str, int] = {}
         self._completed_tilt_series: List[str] = []
         self._aligned_tilt_series: List[str] = []
-        self._motion_corrected_tilt_series: Dict[str, List[Path]] = {}
-        self._last_transferred_file: Path | None = None
         self._data_collection_stash: list = []
         self._processing_job_stash: dict = {}
         self._preprocessing_triggers: dict = {}
@@ -156,57 +154,6 @@ def _flush_preprocess(self, tag: str, app_id: int):
                     capture_post(tr[0], json=process_file)
             self._preprocessing_triggers.pop(tag)
 
-    def _check_for_alignment(
-        self,
-        movie_path: Path,
-        motion_corrected_path: Path,
-        url: str,
-        dcid: int,
-        pjid: int,
-        appid: int,
-        mvid: int,
-        tilt_angles: List,
-        manual_tilt_offset: Optional[float],
-        pixel_size: Optional[float],
-    ):
-        tilt_series = _construct_tilt_series_name(movie_path)
-
-        if self._motion_corrected_tilt_series.get(
-            tilt_series
-        ) and motion_corrected_path not in self._motion_corrected_tilt_series.get(
-            tilt_series, {}
-        ):
-            self._motion_corrected_tilt_series[tilt_series].append(
-                motion_corrected_path
-            )
-        else:
-            self._motion_corrected_tilt_series[tilt_series] = [motion_corrected_path]
-        if tilt_series in self._completed_tilt_series:
-            if (
-                len(self._motion_corrected_tilt_series[tilt_series])
-                == len(self._tilt_series[tilt_series])
-                and len(self._motion_corrected_tilt_series[tilt_series]) > 1
-                and tilt_series not in self._aligned_tilt_series
-            ):
-                try:
-
-                    series_data: dict = {
-                        "name": tilt_series,
-                        "file_tilt_list": str(tilt_angles),
-                        "dcid": dcid,
-                        "processing_job": pjid,
-                        "autoproc_program_id": appid,
-                        "motion_corrected_path": str(motion_corrected_path),
-                        "movie_id": mvid,
-                        "manual_tilt_offset": manual_tilt_offset,
-                        "pixel_size": pixel_size,
-                    }
-                    capture_post(url, json=series_data)
-                    with self._lock:
-                        self._aligned_tilt_series.append(tilt_series)
-                except Exception as e:
-                    logger.warning(f"Data error {e}")
-
     def _complete_process_file(
         self,
         incomplete_process_file: ProcessFileIncomplete,
@@ -302,7 +249,6 @@ def _add_tilt(
         file_path: Path,
         extract_tilt_angle: Callable[[Path], str],
         environment: MurfeyInstanceEnvironment | None = None,
-        required_position_files: List[Path] | None = None,
         required_strings: List[str] | None = None,
     ) -> List[str]:
         if not environment:
@@ -312,7 +258,6 @@ def _add_tilt(
         if not source:
             logger.warning(f"No source found for file {file_path}")
             return []
-        # required_position_files = required_position_files or []
         required_strings = (
             ["fractions"] if required_strings is None else required_strings
         )
@@ -472,24 +417,6 @@ def _add_tilt(
                 else:
                     self._tilt_series[tilt_series].append(file_path)
 
-        res = []
-        if self._last_transferred_file:
-            last_tilt_series = _construct_tilt_series_name(self._last_transferred_file)
-
-            last_tilt_angle = extract_tilt_angle(self._last_transferred_file)
-            self._last_transferred_file = file_path
-            if (
-                last_tilt_series != tilt_series
-                and last_tilt_angle != tilt_angle
-                or self._tilt_series_sizes.get(tilt_series)
-            ) or self._completed_tilt_series:
-                res = self._check_tilt_series(
-                    tilt_series,
-                    required_position_files or [],
-                    file_transferred_to,
-                    environment=environment,
-                )
-
         if environment:
             tilt_url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/tilt"
             tilt_data = {
@@ -543,99 +470,28 @@ def _add_tilt(
             }
             capture_post(preproc_url, json=preproc_data)
 
-        self._last_transferred_file = file_path
-        return res
+        return self._check_tilt_series(tilt_series)
 
     def _check_tilt_series(
         self,
         tilt_series: str,
-        required_position_files: List[Path],
-        file_transferred_to: Path | None,
-        environment: MurfeyInstanceEnvironment | None = None,
     ) -> List[str]:
         newly_completed_series: List[str] = []
-        if not self._tilt_series:
+        mdoc_tilt_series_size = self._tilt_series_sizes.get(tilt_series, 0)
+        if not self._tilt_series or not mdoc_tilt_series_size:
             return newly_completed_series
-        this_tilt_series_size = len(self._tilt_series.get(tilt_series, []))
-        tilt_series_size_check = (
-            (this_tilt_series_size == self._tilt_series_sizes.get(tilt_series))
-            if self._tilt_series_sizes.get(tilt_series)
-            else False
-        )
-        if tilt_series_size_check and not required_position_files:
-            if tilt_series not in self._completed_tilt_series:
-                self._completed_tilt_series.append(tilt_series)
-                newly_completed_series.append(tilt_series)
-        for ts, ta in self._tilt_series.items():
-            required_position_files_check = (
-                all(_f.is_file() for _f in required_position_files)
-                if required_position_files
-                else True
-            )
-            if self._tilt_series_sizes.get(ts):
-                completion_test = len(ta) >= self._tilt_series_sizes[ts]
-                if completion_test:
-                    completion_test = required_position_files_check
-            else:
-                completion_test = False
-            if ts not in self._completed_tilt_series and completion_test:
-                newly_completed_series.append(ts)
-                self._completed_tilt_series.append(ts)
-                if environment and file_transferred_to:
-                    file_tilt_list = []
-                    movie: str
-                    angle: str
-                    for movie, angle in environment.tilt_angles[ts]:
-                        if environment.motion_corrected_movies.get(Path(movie)):
-                            file_tilt_list.append(
-                                [
-                                    str(
-                                        environment.motion_corrected_movies[
-                                            Path(movie)
-                                        ][0]
-                                    ),
-                                    angle,
-                                    str(
-                                        environment.motion_corrected_movies[
-                                            Path(movie)
-                                        ][1]
-                                    ),
-                                ]
-                            )
-                        if environment.motion_corrected_movies.get(file_transferred_to):
-                            self._check_for_alignment(
-                                file_transferred_to,
-                                Path(
-                                    environment.motion_corrected_movies[  # key error PosixPath
-                                        file_transferred_to
-                                    ][
-                                        0
-                                    ]
-                                ),
-                                environment.url.geturl(),
-                                environment.data_collection_ids[ts],
-                                environment.processing_job_ids[ts]["em-tomo-align"],
-                                environment.autoproc_program_ids[ts]["em-tomo-align"],
-                                int(
-                                    environment.motion_corrected_movies[
-                                        file_transferred_to
-                                    ][1]
-                                ),
-                                file_tilt_list,
-                                environment.data_collection_parameters.get(
-                                    "manual_tilt_offset"
-                                ),
-                                environment.data_collection_parameters.get(
-                                    "pixel_size_on_image"
-                                ),
-                            )
+
+        counted_tilts = len(self._tilt_series.get(tilt_series, []))
+        tilt_series_size_check = counted_tilts >= mdoc_tilt_series_size
+        if tilt_series_size_check and tilt_series not in self._completed_tilt_series:
+            self._completed_tilt_series.append(tilt_series)
+            newly_completed_series.append(tilt_series)
         return newly_completed_series
 
     def _add_tomo_tilt(
         self,
         file_path: Path,
         environment: MurfeyInstanceEnvironment | None = None,
-        required_position_files: List[Path] | None = None,
         required_strings: List[str] | None = None,
     ) -> List[str]:
         required_strings = (
@@ -654,16 +510,10 @@ def _add_tomo_tilt(
                 tilt_info_extraction = tomo_tilt_info["5.7"]
         else:
             tilt_info_extraction = tomo_tilt_info["5.7"]
-        tilt_series = _construct_tilt_series_name(file_path)
         return self._add_tilt(
             file_path,
             tilt_info_extraction,
             environment=environment,
-            required_position_files=(
-                required_position_files
-                if required_position_files is not None
-                else [file_path.parent / (tilt_series + ".mdoc")]
-            ),
             required_strings=required_strings,
         )
 
@@ -729,7 +579,6 @@ def post_transfer(
                     completed_tilts = self._add_tomo_tilt(
                         transferred_file,
                         environment=environment,
-                        required_position_files=kwargs.get("required_position_files"),
                         required_strings=kwargs.get("required_strings")
                         or required_strings,
                     )
@@ -744,14 +593,7 @@ def post_transfer(
                 if environment:
                     source = self._get_source(transferred_file, environment)
                     if source:
-                        completed_tilts = self._check_tilt_series(
-                            tilt_series,
-                            kwargs.get("required_position_files") or [],
-                            self._file_transferred_to(
-                                environment, source, transferred_file
-                            ),
-                            environment=environment,
-                        )
+                        completed_tilts = self._check_tilt_series(tilt_series)
 
                     # Always update the tilt series length in the database after an mdoc
                     if environment.murfey_session is not None:

src/murfey/client/instance_environment.py

@@ -53,7 +53,6 @@ class MurfeyInstanceEnvironment(BaseModel):
     }
     data_collection_parameters: dict = {}
     movies: Dict[Path, MovieTracker] = {}
-    motion_corrected_movies: Dict[Path, List[str]] = {}
     listeners: Dict[str, Set[Callable]] = {}
     movie_tilt_pair: Dict[Path, str] = {}
     tilt_angles: Dict[str, List[List[str]]] = {}
@@ -118,7 +117,6 @@ def clear(self):
         self.autoproc_program_ids = {}
         self.data_collection_parameters = {}
         self.movies = {}
-        self.motion_corrected_movies = {}
         self.listeners = {}
         self.movie_tilt_pair = {}
         self.tilt_angles = {}

tests/client/test_context.py

@@ -10,7 +10,7 @@
 
 def test_tomography_context_initialisation_for_tomo(tmp_path):
     context = TomographyContext("tomo", tmp_path)
-    assert not context._last_transferred_file
+    assert not context._completed_tilt_series
     assert context._acquisition_software == "tomo"
 
 
@@ -38,10 +38,6 @@ def test_tomography_context_add_tomo_tilt(mock_post, mock_get, tmp_path):
     assert context._tilt_series == {
         "Position_1": [tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"]
     }
-    assert (
-        context._last_transferred_file
-        == tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"
-    )
     (tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff").touch()
     context.post_transfer(
         tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff",
@@ -101,10 +97,6 @@ def test_tomography_context_add_tomo_tilt_out_of_order(mock_post, mock_get, tmp_
     assert context._tilt_series == {
         "Position_1": [tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"]
     }
-    assert (
-        context._last_transferred_file
-        == tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"
-    )
     (tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff").touch()
     context.post_transfer(
         tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff",
@@ -195,10 +187,6 @@ def test_tomography_context_add_tomo_tilt_delayed_tilt(mock_post, mock_get, tmp_
     assert context._tilt_series == {
         "Position_1": [tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"]
     }
-    assert (
-        context._last_transferred_file
-        == tmp_path / "Position_1_001_[30.0]_date_time_fractions.tiff"
-    )
     (tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff").touch()
     context.post_transfer(
         tmp_path / "Position_1_002_[-30.0]_date_time_fractions.tiff",
@@ -236,7 +224,7 @@ def test_tomography_context_add_tomo_tilt_delayed_tilt(mock_post, mock_get, tmp_
 
 def test_tomography_context_initialisation_for_serialem(tmp_path):
     context = TomographyContext("serialem", tmp_path)
-    assert not context._last_transferred_file
+    assert not context._completed_tilt_series
     assert context._acquisition_software == "serialem"