[feat] Add in a group of meta metrics to allow submetric running

configs/ot_cfm/perturbation_meta.yaml

@@ -0,0 +1,9 @@
+datasets:
+  - tag: defaultGarciaAlonsoPseudotimeEvenCells
+
+method:
+  name: ot_cfm
+  train_and_test_script: ./methods/ot_cfm/train_and_test.sh
+
+metrics:
+  - name: MetaPerturbation

configs/ot_cfm/perturbation_set.yaml

@@ -1,30 +1,42 @@
 datasets:
-  - tag: GarciaAlonsoGC
+  # - tag: GarciaAlonsoGC
+  - tag: defaultGarciaAlonsoPseudotimeEvenCells
 
 method:
   name: ot_cfm
   train_and_test_script: ./methods/ot_cfm/train_and_test.sh
 
 metrics:
   - name: PerturbationCellTypeProportion
+    # this specifies the cell type we want to perturb
+    # and we'll be filtering out the cells
     perturbation_set_config:
-      - gene_col_name: gene_symbols
-        knockin_genes: ['SOX4', 'EGR4', 'KLF6', 'KLF7']
-        knockout_genes: ['STRA8', 'ZGLP1', 'ZIC1']
-        timepoint_idx: 0
+      filter_cell_type: 'GC'
+      filter_tp_idx: 5
+      perturbations:
+        - gene_col_name: gene_symbols
+          knockin_genes: ['SOX4', 'EGR4', 'KLF6', 'KLF7']
+          knockout_genes: ['STRA8', 'ZGLP1', 'ZIC1']
+          timepoint_idx: 0 # this should be building off of filter_tp_idx
     trajectory_infer_model:
       name: CellTypist
       renormalize: True
   - name: PerturbationCellTypeProportion
     perturbation_set_config:
-      - gene_col_name: gene_symbols
-        knockin_genes: ['STRA8', 'ZGLP1', 'ZIC1']
-        knockout_genes: ['SOX4', 'EGR4', 'KLF6', 'KLF7']
-        timepoint_idx: 0
+      filter_cell_type: 'GC'
+      filter_tp_idx: 5
+      perturbations:
+        - gene_col_name: gene_symbols
+          knockin_genes: ['STRA8', 'ZGLP1', 'ZIC1']
+          knockout_genes: ['SOX4', 'EGR4', 'KLF6', 'KLF7']
+          timepoint_idx: 0
     trajectory_infer_model:
       name: CellTypist
       renormalize: True
   - name: PerturbationCellTypeProportion
+    perturbation_set_config:
+      filter_cell_type: 'GC'
+      filter_tp_idx: 5
     trajectory_infer_model:
       name: CellTypist
       renormalize: True

pyproject.toml

@@ -40,6 +40,7 @@ where = ["src"]  # Tells Python: "The actual code lives inside the src folder"
 scTimeBench = [
     "shared/dataset/*.yaml",
     "shared/dataset/cell_lineages/**/*.txt",
+    "shared/dataset/perturbation_lineages/**/*.yaml",
 ]
 
 [project.optional-dependencies]

src/scTimeBench/config.py

@@ -8,6 +8,8 @@
 import argparse
 import logging
 import os
+import random
+import numpy as np
 import yaml
 
 from enum import Enum
@@ -110,6 +112,13 @@ def __init__(self):
             help="Directory to save generated plots (default: plots)",
         )
 
+        parser.add_argument(
+            "--random_seed",
+            type=int,
+            default=42,
+            help="Random seed for reproducibility (default: 42)",
+        )
+
         parser.add_argument(
             "--clear_tables",
             action="store_true",
@@ -368,5 +377,9 @@ def __init__(self):
             for metric in self.metrics_skiplist
         ]
 
+        # finally set the random seed
+        random.seed(self.random_seed)
+        np.random.seed(self.random_seed)
+
         logging.info("Configuration successfully loaded")
         logging.debug("Configuration details: %s", self.__dict__)

src/scTimeBench/main.py

@@ -16,7 +16,11 @@
     scTimeBench.metrics  # to avoid unused import warning
     scTimeBench.shared.dataset  # to avoid unused import warning
 
-from scTimeBench.metrics.base import METRIC_REGISTRY, BaseMetric
+from scTimeBench.metrics.base import (
+    METRIC_REGISTRY,
+    BaseMetric,
+    create_submetric_instance,
+)
 from scTimeBench.metrics.method_manager import MethodManager
 from scTimeBench.shared.dataset.base import DATASET_REGISTRY
 
@@ -53,13 +57,7 @@ def run_metrics(config: Config):
     db_manager = database.DatabaseManager(config)
 
     for metric in config.metrics:
-        metric_name = metric["name"]
-        if metric_name not in METRIC_REGISTRY:
-            raise ValueError(f"Metric {metric_name} not found in registry.")
-        metric_class = METRIC_REGISTRY[metric_name]
-        metric_instance = metric_class(
-            config=config, db_manager=db_manager, metric_config=metric
-        )
+        metric_instance = create_submetric_instance(config, db_manager, metric)
         metric_instance.eval()
 
     db_manager.close()

src/scTimeBench/method_utils/method_runner.py

@@ -11,9 +11,15 @@
 import numpy as np
 import pandas as pd
 import scanpy as sc
+import scipy as sp
 
 from scTimeBench.shared.constants import RequiredOutputFiles
 from scTimeBench.shared.constants import ObservationColumns
+from scTimeBench.shared.utils import (
+    is_raw,
+    undo_log_normalization,
+    log_normalize_to_counts,
+)
 
 
 def get_parser():
@@ -131,12 +137,14 @@ def generate(self, test_ann_data):
                 result.write_h5ad(output_file)
 
             elif required_output == RequiredOutputFiles.PERTURBED_TEST_ANN_DATA:
-                # here we generate the perturbation set
-                first_tp_cells, all_tps = self._prep_from_first_tp(test_ann_data)
-                result = self._generate_perturbation(first_tp_cells, all_tps)
-                self._check_from_first_tp(first_tp_cells, all_tps, result)
+                result = self._generate_perturbation(test_ann_data)
                 result.write_h5ad(output_file)
 
+            elif required_output == RequiredOutputFiles.META_FLAG:
+                # this is just a placeholder file to indicate that the meta metric has been run
+                with open(output_file, "w") as f:
+                    f.write("This file indicates that the meta metric has been run.")
+
             else:
                 raise ValueError(f"Unknown required output: {required_output}")
 
@@ -238,12 +246,12 @@ def generate_gex_from_t_to_t1(self, test_ann_data, t, t1) -> sc.AnnData:
         raise NotImplementedError("Subclasses should implement this method.")
 
     # ** NOTE: DO NOT OVERWRITE THIS FUNCTION **
-    def _generate_perturbation(self, test_ann_data, all_tps) -> sc.AnnData:
+    def _generate_perturbation(self, test_ann_data) -> sc.AnnData:
         """
         Generate predicted gene expression across all timepoints for a perturbation.
         Returns: AnnData object with predicted gene expression across all timepoints.
         """
-        # now we need to read from the perturbation config to create the perturbation set
+        # here we generate the perturbation set
         from scTimeBench.shared.perturbation_set import PerturbationSet
         from scTimeBench.shared.constants import PERTURBATION_SET_CONFIG_FILENAME
 
@@ -255,8 +263,9 @@ def _generate_perturbation(self, test_ann_data, all_tps) -> sc.AnnData:
             perturbation_set_config = yaml.safe_load(f)
         perturbation_set = PerturbationSet(perturbation_set_config)
 
-        # this is the gex at timepoint tp
-        gex_tp = test_ann_data.copy()
+        # then do the initial filtering
+        first_tp_cells, all_tps = perturbation_set.apply_intial_filter(test_ann_data)
+        gex_tp = first_tp_cells.copy()
         all_gex = None
 
         for t_idx in range(len(all_tps)):
@@ -268,6 +277,14 @@ def _generate_perturbation(self, test_ann_data, all_tps) -> sc.AnnData:
             # let's print out the gene variable columns names:
             gex_tp = perturbation_set.apply_perturbation(gex_tp, t_idx)
 
+            # data handling to ensure that the output gex is properly normalized
+            if not is_raw(gex_tp):
+                if sp.sparse.issparse(gex_tp.X):
+                    gex_tp.X.data = np.clip(gex_tp.X.data, a_min=0, a_max=20)
+                else:
+                    gex_tp.X = np.clip(gex_tp.X, a_min=0, a_max=20)
+                gex_tp = log_normalize_to_counts(undo_log_normalization(gex_tp))
+
             # then we need to save this out to the final output file
             if all_gex is None:
                 all_gex = gex_tp.copy()
@@ -284,6 +301,7 @@ def _generate_perturbation(self, test_ann_data, all_tps) -> sc.AnnData:
             gex_tp = self.generate_gex_from_t_to_t1(gex_tp, t, t1)
 
         print("Finished generating perturbation across all timepoints.")
+        self._check_from_first_tp(first_tp_cells, all_tps, all_gex)
         return all_gex
 
 

src/scTimeBench/metrics/base.py

@@ -290,7 +290,7 @@ def _eval(self):
 
                 # finally, we evaluate on the test data (ground truth)
                 # and the predicted data from the method
-                self._submetric_eval(
+                return self._submetric_eval(
                     **self._prep_kwargs_for_submetric_eval(output_path, dataset, method)
                 )
 
@@ -464,7 +464,7 @@ def eval(self):
                 )
                 submetric_instance.eval()
         else:
-            self._eval()
+            return self._eval()
 
     # ** PREPROCESSING DATASET SECTION **
     def _resolve_tag(self, to_match, dataset_tag):
@@ -735,3 +735,18 @@ def _init_datasets(self):
         for dataset in self.datasets:
             logging.debug("-" * 100)
             logging.debug(dataset)
+
+
+def create_submetric_instance(
+    config: Config, db_manager: DatabaseManager, metric_config: dict
+) -> BaseMetric:
+    """
+    Factory that creates an instance of a submetric.
+    """
+    metric_name = metric_config["name"]
+    if metric_name not in METRIC_REGISTRY:
+        raise ValueError(f"Metric {metric_name} not found in registry.")
+
+    return METRIC_REGISTRY[metric_name](
+        config=config, db_manager=db_manager, metric_config=metric_config
+    )