Debug VideoMaker - fix failing test (#1418)

* Debug VideoMaker

* Update changelog

* Address comments

Author: CBroz1

.gitignore

@@ -4,6 +4,7 @@ mountainsort4_output/
 .idea/
 mysql_config
 memray*
+root_*.y*ml
 
 # Notebooks
 *.ipynb

CHANGELOG.md

@@ -43,6 +43,7 @@ import all foreign key references.
     - Ensure video files are properly added to `DLCProject` # 1367
     - DLC parameter handling improvements and default value corrections #1379
     - Fix ingestion nwb files with position objects but no spatial series #1405
+    - Ignore `percent_frames` when using `limit` in `DLCPosVideo` #1418
 - Spikesorting
     - Implement short-transaction `SpikeSortingRecording.make` for v0 #1338
 

src/spyglass/position/v1/dlc_utils_makevid.py

@@ -74,6 +74,8 @@ def __init__(
         # key_hash supports resume from previous run
         self.temp_dir = Path(temp_dir) / f"dlc_vid_{key_hash}"
         self.temp_dir.mkdir(parents=True, exist_ok=True)
+        if not self.temp_dir.exists():  # pragma: no cover
+            raise FileNotFoundError(f"Could not create {self.temp_dir}")
         logger.debug(f"Temporary directory: {self.temp_dir}")
 
         if not Path(video_filename).exists():
@@ -109,6 +111,7 @@ def __init__(
         self.timeout = 30 if test_mode else 300
 
         self.ffmpeg_log_args = ["-hide_banner", "-loglevel", "error"]
+
         self.ffmpeg_fmt_args = ["-c:v", "libx264", "-pix_fmt", "yuv420p"]
 
         prev_backend = matplotlib.get_backend()
@@ -123,7 +126,12 @@ def __init__(
         )
         self.process_frames()
         plt.close(self.fig)
-        logger.info(f"Finished video: {self.output_video_filename}")
+
+        if Path(self.output_video_filename).exists():
+            logger.info(f"Finished video: {self.output_video_filename}")
+        else:
+            logger.error(f"Failed to create: {self.output_video_filename}")
+
         logger.debug(f"Dropped frames: {self.dropped_frames}")
 
         if not debug:
@@ -187,6 +195,9 @@ def _set_frame_info(self):
         if self.debug:  # If debugging, limit frames to available data
             self.n_frames = min(len(self.position_mean), self.n_frames)
 
+        if self.n_frames == 0:  # pragma: no cover
+            raise ValueError("No frames to process!")  # pragma: no cover
+
         self.pad_len = len(str(self.n_frames))
 
     def _set_plot_bases(self):
@@ -490,11 +501,18 @@ def ffmpeg_extract(self, start_frame, end_frame):
         ]
         ret = subprocess.run(ffmpeg_cmd, stderr=subprocess.PIPE)
 
+        if ret.returncode != 0:  # pragma: no cover
+            logger.error(f"Error extracting frames: {ret.stderr}")
+        if not self.temp_dir.glob("orig_*.png"):  # pragma: no cover
+            raise FileNotFoundError("No frames were extracted!")
+
         extracted = len(list(self.temp_dir.glob("orig_*.png")))
         logger.debug(f"Extracted  frames: {start_frame}, len: {extracted}")
-        if extracted < self.batch_size - 1:
+
+        frame_diff = end_frame - start_frame + 1  # may be less than batch size
+        if extracted < frame_diff:
             logger.warning(
-                f"Could not extract frames: {extracted} / {self.batch_size-1}"
+                f"Could not extract frames: {extracted} / {frame_diff}"
             )
             one_err = "\n".join(str(ret.stderr).split("\\")[-3:-1])
             logger.debug(f"\nExtract Error: {one_err}")
@@ -536,9 +554,16 @@ def ffmpeg_stitch_partial(self, start_frame, output_partial_video):
         except subprocess.CalledProcessError as e:  # pragma: no cover
             logger.error(f"Err stitching video: {e.stderr}")  # pragma: no cover
 
+        if not Path(output_partial_video).exists():  # pragma: no cover
+            logger.error(f"Partial video not created: {output_partial_video}")
+
     def concat_partial_videos(self):
         """Concatenate all the partial videos into one final video."""
         partial_vids = sorted(self.temp_dir.glob("partial_*.mp4"))
+
+        if not partial_vids:  # pragma: no cover
+            raise FileNotFoundError("No partial videos to concatenate!")
+
         logger.debug(f"Concat part vids: {len(partial_vids)}")
         concat_list_path = self.temp_dir / "concat_list.txt"
         with open(concat_list_path, "w") as f:
@@ -568,6 +593,7 @@ def concat_partial_videos(self):
             )
         except subprocess.CalledProcessError as e:
             logger.error(f"Error stitching partial video: {e.stderr}")
+            raise  # pragma: no cover
 
 
 def make_video(**kwargs):

src/spyglass/position/v1/position_dlc_selection.py

@@ -487,10 +487,12 @@ def make(self, key):
             else None
         )
         frames = params.get("frames", None)
+        percent_frames = params.get("percent_frames", None)
 
         if limit := params.get("limit", None):  # new int param for debugging
             output_video_filename = Path(".") / f"TEST_VID_{limit}.mp4"
             video_frame_inds = video_frame_inds[:limit]
+            percent_frames = 1
             pos_info_df = pos_info_df.head(limit)
 
         video_maker = make_video(
@@ -505,7 +507,7 @@ def make(self, key):
             position_time=np.asarray(pos_info_df.index),
             processor=params.get("processor", "matplotlib"),
             frames=np.arange(frames[0], frames[1]) if frames else None,
-            percent_frames=params.get("percent_frames", None),
+            percent_frames=percent_frames,
             output_video_filename=output_video_filename,
             cm_to_pixels=meters_per_pixel * M_TO_CM,
             crop=pose_estimation_params.get("cropping"),

tests/position/v1/test_dlc_position.py

@@ -99,9 +99,8 @@ def test_si_interpolate(sgp, si_params_tbl, si_key, pose_estimation_key):
 
 @pytest.fixture(scope="session")
 def si_df(sgp, si_key, populate_si, bodyparts):
-    yield (
-        sgp.v1.DLCSmoothInterp() & {**si_key, "bodypart": bodyparts[0]}
-    ).fetch1_dataframe()
+    _ = si_key, populate_si, bodyparts
+    yield (sgp.v1.DLCSmoothInterp() & dj.Top()).fetch1_dataframe()
 
 
 def test_cohort_fetch1_dataframe(si_df):