feat: integrate panel of judges

Author: asamal4

src/lightspeed_evaluation/core/constants.py

@@ -60,6 +60,9 @@
 
 DEFAULT_API_NUM_RETRIES = 3
 
+# Frameworks that don't require judge LLM (NLP, script-based evaluations)
+NON_LLM_FRAMEWORKS = frozenset({"nlp", "script"})
+
 DEFAULT_LLM_PROVIDER = "openai"
 DEFAULT_LLM_MODEL = "gpt-4o-mini"
 DEFAULT_SSL_VERIFY = True
@@ -96,6 +99,8 @@
     "api_output_tokens",
     "judge_llm_input_tokens",
     "judge_llm_output_tokens",
+    # Per-judge scores (JSON array with one entry for single judge)
+    "judge_scores",
     # Streaming performance metrics
     "time_to_first_token",
     "streaming_duration",

src/lightspeed_evaluation/core/llm/manager.py

@@ -163,6 +163,22 @@ def get_primary_judge(self) -> "LLMManager":
             return self.judge_managers[0]
         return self
 
+    def get_judges_for_metric(self, metric_identifier: str) -> list["LLMManager"]:
+        """Get list of judges to use for a specific metric.
+
+        Returns all judges if metric should use panel, otherwise returns
+        list with single primary judge. Always returns a list.
+
+        Args:
+            metric_identifier: Metric identifier (e.g., "ragas:faithfulness")
+
+        Returns:
+            List of LLMManager instances to use for this metric
+        """
+        if self.should_use_panel_for_metric(metric_identifier):
+            return self.get_judge_managers()
+        return [self.get_primary_judge()]
+
     def should_use_panel_for_metric(self, metric_identifier: str) -> bool:
         """Determine if a metric should use judge panel based on enabled_metrics.
 

src/lightspeed_evaluation/core/models/__init__.py

@@ -8,9 +8,10 @@
 from lightspeed_evaluation.core.models.data import (
     EvaluationData,
     EvaluationRequest,
-    MetricResult,
     EvaluationResult,
     EvaluationScope,
+    JudgeScore,
+    MetricResult,
     TurnData,
 )
 from lightspeed_evaluation.core.models.mixins import StreamingMetricsMixin
@@ -34,6 +35,7 @@
     "TurnData",
     "EvaluationData",
     "EvaluationRequest",
+    "JudgeScore",
     "MetricResult",
     "EvaluationResult",
     "EvaluationScope",

src/lightspeed_evaluation/core/models/data.py

@@ -419,6 +419,24 @@ def validate_conversation_metrics(
         return v
 
 
+class JudgeScore(BaseModel):
+    """Model for individual judge evaluation score in a judge panel.
+
+    Used when multiple judges evaluate the same metric, storing per-judge
+    details for transparency and analysis.
+    """
+
+    judge_id: str = Field(
+        ..., min_length=1, description="Judge identifier (model ID from llm_pool)"
+    )
+    score: Optional[float] = Field(
+        default=None, ge=0.0, le=1.0, description="Score between 0 and 1"
+    )
+    reason: str = Field(default="", description="Explanation from this judge")
+    input_tokens: int = Field(default=0, ge=0, description="Input tokens used")
+    output_tokens: int = Field(default=0, ge=0, description="Output tokens used")
+
+
 class MetricResult(BaseModel):
     """Model for framework metric result."""
 
@@ -439,6 +457,10 @@ class MetricResult(BaseModel):
     judge_llm_output_tokens: int = Field(
         default=0, ge=0, description="Judge LLM output tokens used"
     )
+    judge_scores: Optional[list[JudgeScore]] = Field(
+        default=None,
+        description="Per-judge scores when using judge panel (for transparency)",
+    )
 
     @field_validator("result")
     @classmethod

src/lightspeed_evaluation/core/output/generator.py

@@ -196,6 +196,13 @@ def _generate_csv_report(
                         # Special formatting for execution_time
                         if column == "execution_time" and value is not None:
                             row_data.append(f"{value:.3f}")
+                        # Convert judge_scores to JSON string
+                        elif column == "judge_scores" and value is not None:
+                            row_data.append(
+                                json.dumps(
+                                    [js.model_dump() for js in value], default=str
+                                )
+                            )
                         else:
                             row_data.append(value)
                     else:
@@ -254,6 +261,12 @@ def _generate_json_summary(  # pylint: disable=too-many-arguments,too-many-posit
                     "execution_time": round(r.execution_time, 3),
                     "judge_llm_input_tokens": r.judge_llm_input_tokens,
                     "judge_llm_output_tokens": r.judge_llm_output_tokens,
+                    # Judge panel scores (when using multiple judges)
+                    "judge_scores": (
+                        [js.model_dump() for js in r.judge_scores]
+                        if r.judge_scores
+                        else None
+                    ),
                     # Streaming performance metrics
                     "time_to_first_token": r.time_to_first_token,
                     "streaming_duration": r.streaming_duration,

src/lightspeed_evaluation/core/system/loader.py

@@ -16,6 +16,10 @@
     SystemConfig,
     VisualizationConfig,
 )
+from lightspeed_evaluation.core.models.system import (
+    JudgePanelConfig,
+    LLMPoolConfig,
+)
 from lightspeed_evaluation.core.system.setup import (
     setup_environment_variables,
     setup_logging,
@@ -156,6 +160,17 @@ def load_system_config(self, config_path: str) -> SystemConfig:
     def _create_system_config(self, config_data: dict[str, Any]) -> SystemConfig:
         """Create SystemConfig object from validated configuration data."""
         metrics_metadata = config_data.get("metrics_metadata", {})
+
+        # Parse llm_pool if present
+        llm_pool = None
+        if "llm_pool" in config_data:
+            llm_pool = LLMPoolConfig(**config_data["llm_pool"])
+
+        # Parse judge_panel if present
+        judge_panel = None
+        if "judge_panel" in config_data:
+            judge_panel = JudgePanelConfig(**config_data["judge_panel"])
+
         return SystemConfig(
             core=CoreConfig(**config_data.get("core", {})),
             llm=LLMConfig(**config_data.get("llm", {})),
@@ -164,6 +179,8 @@ def _create_system_config(self, config_data: dict[str, Any]) -> SystemConfig:
             output=OutputConfig(**config_data.get("output", {})),
             logging=LoggingConfig(**config_data.get("logging", {})),
             visualization=VisualizationConfig(**config_data.get("visualization", {})),
+            llm_pool=llm_pool,
+            judge_panel=judge_panel,
             default_turn_metrics_metadata=metrics_metadata.get("turn_level", {}),
             default_conversation_metrics_metadata=metrics_metadata.get(
                 "conversation_level", {}