Add --metrics CLI flag to filter which metrics run

src/lightspeed_evaluation/core/system/validator.py

@@ -155,6 +155,7 @@ def load_evaluation_data(
         data_path: str,
         tags: Optional[list[str]] = None,
         conv_ids: Optional[list[str]] = None,
+        metrics: Optional[list[str]] = None,
     ) -> list[EvaluationData]:
         """Load, filter, and validate evaluation data from YAML file.
 
@@ -168,6 +169,7 @@ def load_evaluation_data(
             data_path: Path to the evaluation data YAML file
             tags: Optional list of tags to filter by
             conv_ids: Optional list of conversation group IDs to filter by
+            metrics: Optional list of metrics to run (filters each turn's turn_metrics)
 
         Returns:
             Filtered and validated list of Evaluation Data
@@ -214,6 +216,16 @@ def load_evaluation_data(
         # Filter by scope before validation
         evaluation_data = self._filter_by_scope(evaluation_data, tags, conv_ids)
 
+        # Filter turn_metrics if --metrics was specified
+        if metrics:
+            metrics_set = set(metrics)
+            for eval_data in evaluation_data:
+                for turn in eval_data.turns:
+                    if turn.turn_metrics:
+                        turn.turn_metrics = [
+                            m for m in turn.turn_metrics if m in metrics_set
+                        ]
+
         # Semantic validation (metrics availability and requirements)
         if not self._validate_evaluation_data(evaluation_data):
             raise DataValidationError("Evaluation data validation failed")

src/lightspeed_evaluation/runner/evaluation.py

@@ -127,6 +127,7 @@ def run_evaluation(  # pylint: disable=too-many-locals
             eval_args.eval_data,
             tags=eval_args.tags,
             conv_ids=eval_args.conv_ids,
+            metrics=eval_args.metrics,
         )
 
         print(
@@ -217,6 +218,12 @@ def main() -> int:
         default=None,
         help="Filter by conversation group IDs (run only specified conversations)",
     )
+    parser.add_argument(
+        "--metrics",
+        nargs="+",
+        default=None,
+        help="Filter to only run specified metrics (e.g. custom:answer_correctness)",
+    )
     parser.add_argument(
         "--cache-warmup",
         action="store_true",