Change participant ID to sample ID. Also sneak in small changes to remove saving heatmaps to disk and adding try/catch around summary report

Author: i-jey

ulc_mm_package/QtGUI/form_gui.py

@@ -85,7 +85,7 @@ def _load_ui(self):
 
         # Labels
         self.operator_lbl = QLabel("Operator ID")
-        self.participant_lbl = QLabel("Non-identifying participant ID")
+        self.sample_id_lbl = QLabel("Non-identifying sample ID")
         self.study_select_lbl = QLabel("Select study")
         self.flowcell_lbl = QLabel("Flowcell ID")
         self.notes_lbl = QLabel("Other notes")
@@ -94,7 +94,7 @@ def _load_ui(self):
 
         # Text boxes
         self.operator_val = QLineEdit()
-        self.participant_val = QLineEdit()
+        self.sample_id_val = QLineEdit()
 
         self.flowcell_val = QLineEdit()
         self.notes_val = QPlainTextEdit()
@@ -124,7 +124,7 @@ def _load_ui(self):
 
         # Place widgets
         self.main_layout.addWidget(self.operator_lbl, 0, 0)
-        self.main_layout.addWidget(self.participant_lbl, 1, 0)
+        self.main_layout.addWidget(self.sample_id_lbl, 1, 0)
         self.main_layout.addWidget(self.study_select_lbl, 2, 0)
         self.main_layout.addWidget(self.flowcell_lbl, 5, 0)
         self.main_layout.addWidget(self.site_lbl, 6, 0)
@@ -133,7 +133,7 @@ def _load_ui(self):
         self.main_layout.addWidget(self.exit_btn, 9, 0)
 
         self.main_layout.addWidget(self.operator_val, 0, 1)
-        self.main_layout.addWidget(self.participant_val, 1, 1)
+        self.main_layout.addWidget(self.sample_id_val, 1, 1)
         self.main_layout.addWidget(self.study_select_val, 2, 1)
         self.main_layout.addWidget(self.flowcell_val, 5, 1)
         self.main_layout.addWidget(self.site_val, 6, 1)
@@ -154,7 +154,7 @@ def get_form_input(self) -> dict:
 
         form_metadata = {
             "operator_id": self.operator_val.text(),
-            "participant_id": self.participant_val.text(),
+            "sample_id": self.sample_id_val.text(),
             "flowcell_id": self.flowcell_val.text(),
             "target_flowrate": (
                 TARGET_FLOWRATE.name.capitalize(),
@@ -173,7 +173,7 @@ def get_form_input(self) -> dict:
 
     def reset_parameters(self) -> None:
         """Clear specific inputs which are expected to be unique for the next run."""
-        self.participant_val.setText("")
+        self.sample_id_val.setText("")
         self.flowcell_val.setText("")
         self.notes_val.setPlainText("")
 

ulc_mm_package/QtGUI/liveview_gui.py

@@ -91,7 +91,7 @@ def closeEvent(self, event):
     def update_experiment(self, metadata: dict):
         # TODO standardize dict input
         self.operator_val.setText(f"{metadata['operator_id']}")
-        self.participant_val.setText(f"{metadata['participant_id']}")
+        self.sample_val.setText(f"{metadata['sample_id']}")
         self.flowcell_val.setText(f"{metadata['flowcell_id']}")
         self.target_flowrate_val.setText(f"{metadata['target_flowrate'][0]}")
         self.site_val.setText(f"{metadata['site']}")
@@ -560,35 +560,35 @@ def _load_metadata_ui(self):
 
         # Populate metadata tab
         self.operator_lbl = QLabel("Operator ID")
-        self.participant_lbl = QLabel("Participant ID")
+        self.sample_lbl = QLabel("Sample ID")
         self.flowcell_lbl = QLabel("Flowcell ID")
         self.target_flowrate_lbl = QLabel("Flowrate")
         self.site_lbl = QLabel("Site")
         self.notes_lbl = QLabel("Other notes")
 
         self.operator_val = QLineEdit()
-        self.participant_val = QLineEdit()
+        self.sample_val = QLineEdit()
         self.flowcell_val = QLineEdit()
         self.target_flowrate_val = QLineEdit()
         self.site_val = QLineEdit()
         self.notes_val = QPlainTextEdit()
 
         self.operator_val.setReadOnly(True)
-        self.participant_val.setReadOnly(True)
+        self.sample_val.setReadOnly(True)
         self.flowcell_val.setReadOnly(True)
         self.target_flowrate_val.setReadOnly(True)
         self.site_val.setReadOnly(True)
         self.notes_val.setReadOnly(True)
 
         self.metadata_layout.addWidget(self.operator_lbl, 1, 1)
-        self.metadata_layout.addWidget(self.participant_lbl, 2, 1)
+        self.metadata_layout.addWidget(self.sample_lbl, 2, 1)
         self.metadata_layout.addWidget(self.flowcell_lbl, 3, 1)
         self.metadata_layout.addWidget(self.target_flowrate_lbl, 4, 1)
         self.metadata_layout.addWidget(self.site_lbl, 5, 1)
         self.metadata_layout.addWidget(self.notes_lbl, 6, 1)
 
         self.metadata_layout.addWidget(self.operator_val, 1, 2)
-        self.metadata_layout.addWidget(self.participant_val, 2, 2)
+        self.metadata_layout.addWidget(self.sample_val, 2, 2)
         self.metadata_layout.addWidget(self.flowcell_val, 3, 2)
         self.metadata_layout.addWidget(self.target_flowrate_val, 4, 2)
         self.metadata_layout.addWidget(self.site_val, 5, 2)
@@ -601,7 +601,7 @@ def _load_metadata_ui(self):
 
     experiment_metadata = {
         "operator_id": "1234",
-        "participant_id": "567",
+        "sample_id": "567",
         "flowcell_id": "A2",
         "target_flowrate": ("Fast", 15),
         "site": "Uganda",

ulc_mm_package/QtGUI/oracle.py

@@ -430,7 +430,7 @@ def general_pause_handler(
             (
                 "The CAP module can now be removed."
                 "\n\nPlease empty both reservoirs and reload 12 uL of fresh "
-                "diluted blood (from the same participant) into the sample reservoir. Make sure to close the lid after."
+                "diluted blood (from the same sample) into the sample reservoir. Make sure to close the lid after."
                 '\n\nAfter reloading the reservoir and closing the lid, click "OK" to resume this run.'
             ),
             buttons=Buttons.OK,

ulc_mm_package/hardware/scope.py

@@ -140,7 +140,6 @@ def reset_for_end_experiment(self) -> None:
         # Close data storage
         closing_file_future = self.data_storage.close(
             self.predictions_handler.get_prediction_tensors(),
-            self.predictions_handler.heatmaps,
         )
         if closing_file_future is not None:
             while not closing_file_future.done():

ulc_mm_package/image_processing/data_storage.py

@@ -297,112 +297,122 @@ def close(
 
             ### Create summary report
             self.logger.info("> Creating summary report...")
-            summary_report_dir = self.get_experiment_path() / "summary_report"
-            Path.mkdir(summary_report_dir, exist_ok=True)
-
-            ### NOTE: xhtml2pdf fails if you provide relative image file paths, e.g ("../thumbnails/ring/1.png")
-            ### so provide absolute filepaths only. Note this means that the html file will be broken if viewed from anywhere other than the Pi.
-
-            class_to_all_thumbnails_abs_path: Dict[str, List[str]] = {
-                x: [
-                    str(y.resolve())
-                    for y in list(class_to_thumbnails_path[x].rglob("*.png"))
-                ]
-                for x in class_to_thumbnails_path.keys()
-            }
-
-            html_abs_path_temp_loc = (
-                summary_report_dir / f"{self.time_str}_temp_summary.html"
-            )
-            pdf_save_loc = summary_report_dir / f"{self.time_str}_summary.pdf"
-
-            # Create per-image metadata plot
-            per_image_metadata_plot_save_loc = str(
-                summary_report_dir / f"{self.time_str}_per_image_metadata_plot.jpg"
-            )
-
-            counts_plot_loc = str(summary_report_dir / "counts.jpg")
-            conf_plot_loc = str(summary_report_dir / "confs.jpg")
-            objectness_plot_loc = str(summary_report_dir / "objectness.jpg")
+            try:
+                summary_report_dir = self.get_experiment_path() / "summary_report"
+                Path.mkdir(summary_report_dir, exist_ok=True)
+
+                ### NOTE: xhtml2pdf fails if you provide relative image file paths, e.g ("../thumbnails/ring/1.png")
+                ### so provide absolute filepaths only. Note this means that the html file will be broken if viewed from anywhere other than the Pi.
+
+                class_to_all_thumbnails_abs_path: Dict[str, List[str]] = {
+                    x: [
+                        str(y.resolve())
+                        for y in list(class_to_thumbnails_path[x].rglob("*.png"))
+                    ]
+                    for x in class_to_thumbnails_path.keys()
+                }
+
+                html_abs_path_temp_loc = (
+                    summary_report_dir / f"{self.time_str}_temp_summary.html"
+                )
+                pdf_save_loc = summary_report_dir / f"{self.time_str}_summary.pdf"
 
-            # Only generate additional plots if DEBUG_REPORT environment variable is set to True
-            if DEBUG_REPORT:
-                with open(self.per_img_metadata_filename, "r") as per_img_metadata_file:
-                    make_per_image_metadata_plots(
-                        per_img_metadata_file, per_image_metadata_plot_save_loc
-                    )
+                # Create per-image metadata plot
+                per_image_metadata_plot_save_loc = str(
+                    summary_report_dir / f"{self.time_str}_per_image_metadata_plot.jpg"
+                )
 
-                try:
-                    make_cell_count_plot(pred_tensors, counts_plot_loc)
-                except Exception as e:
-                    self.logger.error(f"Failed to make cell count plot - {e}")
-                try:
-                    make_yogo_conf_plots(pred_tensors, conf_plot_loc)
-                except Exception as e:
-                    self.logger.error(f"Failed to make yogo confidence plots - {e}")
-                try:
-                    make_yogo_objectness_plots(pred_tensors, objectness_plot_loc)
-                except Exception as e:
-                    self.logger.error(f"Failed to make yogo objectness plots - {e}")
-
-            # Get cell counts
-            raw_cell_counts = np.asarray(get_class_counts(pred_tensors))
-            (
-                comp_parasitemia,
-                comp_parasitemia_err,
-            ) = self.compensator.get_res_from_counts(raw_cell_counts, units_ul_out=True)
-            # Associate class with counts
-            class_name_to_cell_count = {
-                x.capitalize(): y for (x, y) in zip(YOGO_CLASS_LIST, raw_cell_counts)
-            }
-            # 'parasites per ul' is # of rings / total rbcs * scaling factor (RBCS_PER_UL)
-
-            # Create parasitemia plot
-            parasitemia_plot_loc = str(self.get_parasitemia_vis_filename())
-            try:
-                make_parasitemia_plot(
+                counts_plot_loc = str(summary_report_dir / "counts.jpg")
+                conf_plot_loc = str(summary_report_dir / "confs.jpg")
+                objectness_plot_loc = str(summary_report_dir / "objectness.jpg")
+
+                # Only generate additional plots if DEBUG_REPORT environment variable is set to True
+                if DEBUG_REPORT:
+                    with open(
+                        self.per_img_metadata_filename, "r"
+                    ) as per_img_metadata_file:
+                        make_per_image_metadata_plots(
+                            per_img_metadata_file, per_image_metadata_plot_save_loc
+                        )
+
+                    try:
+                        make_cell_count_plot(pred_tensors, counts_plot_loc)
+                    except Exception as e:
+                        self.logger.error(f"Failed to make cell count plot - {e}")
+                    try:
+                        make_yogo_conf_plots(pred_tensors, conf_plot_loc)
+                    except Exception as e:
+                        self.logger.error(f"Failed to make yogo confidence plots - {e}")
+                    try:
+                        make_yogo_objectness_plots(pred_tensors, objectness_plot_loc)
+                    except Exception as e:
+                        self.logger.error(f"Failed to make yogo objectness plots - {e}")
+
+                # Get cell counts
+                raw_cell_counts = np.asarray(get_class_counts(pred_tensors))
+                (
                     comp_parasitemia,
                     comp_parasitemia_err,
+                ) = self.compensator.get_res_from_counts(
+                    raw_cell_counts, units_ul_out=True
+                )
+                # Associate class with counts
+                class_name_to_cell_count = {
+                    x.capitalize(): y
+                    for (x, y) in zip(YOGO_CLASS_LIST, raw_cell_counts)
+                }
+                # 'parasites per ul' is # of rings / total rbcs * scaling factor (RBCS_PER_UL)
+
+                # Create parasitemia plot
+                parasitemia_plot_loc = str(self.get_parasitemia_vis_filename())
+                try:
+                    make_parasitemia_plot(
+                        comp_parasitemia,
+                        comp_parasitemia_err,
+                        parasitemia_plot_loc,
+                    )
+                except Exception as e:
+                    self.logger.error(f"Failed to make parasitemia plot - {e}")
+
+                # HTML w/ absolute path
+                abs_css_file_path = str((summary_report_dir / CSS_FILE_NAME).resolve())
+                html_report_with_abs_path = make_html_report(
+                    self.compensator,
+                    self.time_str,
+                    self.experiment_level_metadata,
+                    per_image_metadata_plot_save_loc,
+                    raw_cell_counts,
+                    class_to_all_thumbnails_abs_path,
                     parasitemia_plot_loc,
+                    counts_plot_loc,
+                    conf_plot_loc,
+                    objectness_plot_loc,
+                    css_path=abs_css_file_path,
                 )
-            except Exception as e:
-                self.logger.error(f"Failed to make parasitemia plot - {e}")
-
-            # HTML w/ absolute path
-            abs_css_file_path = str((summary_report_dir / CSS_FILE_NAME).resolve())
-            html_report_with_abs_path = make_html_report(
-                self.compensator,
-                self.time_str,
-                self.experiment_level_metadata,
-                per_image_metadata_plot_save_loc,
-                raw_cell_counts,
-                class_to_all_thumbnails_abs_path,
-                parasitemia_plot_loc,
-                counts_plot_loc,
-                conf_plot_loc,
-                objectness_plot_loc,
-                css_path=abs_css_file_path,
-            )
 
-            # Copy the CSS file to the summary directory
-            shutil.copy(SUMMARY_REPORT_CSS_FILE, summary_report_dir)
+                # Copy the CSS file to the summary directory
+                shutil.copy(SUMMARY_REPORT_CSS_FILE, summary_report_dir)
 
-            # Save the temporary HTML file w/ absolute path so we can properly generate the PDF
-            save_html_report(html_report_with_abs_path, html_abs_path_temp_loc)
-            create_pdf_from_html(html_abs_path_temp_loc, pdf_save_loc)
+                # Save the temporary HTML file w/ absolute path so we can properly generate the PDF
+                save_html_report(html_report_with_abs_path, html_abs_path_temp_loc)
+                create_pdf_from_html(html_abs_path_temp_loc, pdf_save_loc)
 
-            # Make a copy of the summary PDF to the Desktop
-            shutil.copy(pdf_save_loc, DESKTOP_SUMMARY_DIR)
+                # Make a copy of the summary PDF to the Desktop
+                shutil.copy(pdf_save_loc, DESKTOP_SUMMARY_DIR)
 
-            # Remove intermediate files
-            remove(html_abs_path_temp_loc)
-            remove(summary_report_dir / CSS_FILE_NAME)
+                # Remove intermediate files
+                remove(html_abs_path_temp_loc)
+                remove(summary_report_dir / CSS_FILE_NAME)
 
-            if DEBUG_REPORT:
-                remove(counts_plot_loc)
-                remove(per_image_metadata_plot_save_loc)
-                remove(conf_plot_loc)
-                remove(objectness_plot_loc)
+                if DEBUG_REPORT:
+                    remove(counts_plot_loc)
+                    remove(per_image_metadata_plot_save_loc)
+                    remove(conf_plot_loc)
+                    remove(objectness_plot_loc)
+            except Exception as e:
+                self.logger.error(
+                    f"Unexpected exception when saving summary report: {e}. Skipping report and continuing..."
+                )
 
             # Write to a separate csv with just cell counts for each class
             self.logger.info("Writing cell counts to csv...")

ulc_mm_package/scope_constants.py

@@ -144,7 +144,7 @@ def IMG_HEIGHT(self) -> int:
 # ================ Data storage metadata ================ #
 EXPERIMENT_METADATA_KEYS = [
     "operator_id",
-    "participant_id",
+    "sample_id",
     "sample_collection_date",
     "sample_collection_time",
     "sample_age_hours",

ulc_mm_package/summary_report/make_summary_report.py

@@ -385,10 +385,8 @@ def make_html_report(
         if experiment_metadata["operator_id"]
         else "-"
     )
-    participant = (
-        experiment_metadata["participant_id"]
-        if experiment_metadata["participant_id"]
-        else "-"
+    sample = (
+        experiment_metadata["sample_id"] if experiment_metadata["sample_id"] else "-"
     )
     notes = experiment_metadata["notes"] if experiment_metadata["notes"] else "-"
     fc_id = (
@@ -401,7 +399,7 @@ def make_html_report(
         "css_file": css_path,
         "dataset_name": dataset_name,
         "operator_id": operator,
-        "participant_id": participant,
+        "sample_id": sample,
         "notes": notes,
         "flowcell_id": fc_id,
         "class_name_to_cell_count": format_cell_counts(compensator, cell_counts),
@@ -465,7 +463,7 @@ def create_pdf_from_html(path_to_html: Path, save_path: Path) -> None:
     # Dummy data
     exp_metadata = {
         "operator_id": "MK",
-        "participant_id": "1034",
+        "sample_id": "1034",
         "notes": "sample only",
         "flowcell_id": "A5",
     }

ulc_mm_package/summary_report/summary_template.html

@@ -56,8 +56,8 @@ <h1 style="text-align: center; color: red">RESEARCH USE ONLY - <strong>NOT FOR C
                 <td>{{ operator_id }}</td>
             </tr>
             <tr>
-                <td>Non-identifying participant ID</td>
-                <td>{{ participant_id }}</td>
+                <td>Non-identifying sample ID</td>
+                <td>{{ sample_id }}</td>
             </tr>
             <tr>
                 <td>Notes</td>