[LEADS-389] Support user-defined metadata in evaluation data format for GDS quality grading and traceability

Author: bsatapat-jpg

README.md

@@ -320,6 +320,7 @@ For field tables, full YAML examples (file-only, file + SQLite, file + Postgres)
 | Field                           | Type           | Required | Description                                                          |
 |---------------------------------|----------------|----------|----------------------------------------------------------------------|
 | `conversation_group_id`         | string         | ✅       | Unique identifier for conversation                                   |
+| `metadata`                      | ConversationMetadata | ❌ | User-defined metadata for traceability and quality grading           |
 | `description`                   | string         | ❌       | Optional description                                                 |
 | `tag`                           | string         | ❌       | Tag for grouping eval conversations (default: "eval")             |
 | `setup_script`                  | string         | ❌       | Path to setup script (Optional, used when API is enabled)            |
@@ -333,6 +334,7 @@ For field tables, full YAML examples (file-only, file + SQLite, file + Postgres)
 | Field                 | Type             | Required | Description                          | API Populated         |
 |-----------------------|------------------|----------|--------------------------------------|-----------------------|
 | `turn_id`             | string           | ✅       | Unique identifier for the turn       | ❌                    |
+| `metadata`            | TurnMetadata     | ❌       | User-defined metadata for traceability and quality grading | ❌ |
 | `query`               | string           | ✅       | The question/prompt to evaluate      | ❌                    |
 | `response`            | string           | 📋       | Actual response from system          | ✅ (if API enabled)   |
 | `contexts`            | list[string]     | 📋       | Context information for evaluation   | ✅ (if API enabled)   |

src/lightspeed_evaluation/__init__.py

@@ -24,11 +24,14 @@
     from lightspeed_evaluation.core.llm import LLMManager
     from lightspeed_evaluation.core.models import (
         APIConfig,
+        ConversationMetadata,
+        DatasetMetadata,
         EvaluationData,
         EvaluationResult,
         LLMConfig,
         LoggingConfig,
         TurnData,
+        TurnMetadata,
         VisualizationConfig,
     )
     from lightspeed_evaluation.core.models.summary import EvaluationSummary
@@ -79,6 +82,12 @@
     "VisualizationConfig": ("lightspeed_evaluation.core.models", "VisualizationConfig"),
     "EvaluationData": ("lightspeed_evaluation.core.models", "EvaluationData"),
     "TurnData": ("lightspeed_evaluation.core.models", "TurnData"),
+    "TurnMetadata": ("lightspeed_evaluation.core.models", "TurnMetadata"),
+    "ConversationMetadata": (
+        "lightspeed_evaluation.core.models",
+        "ConversationMetadata",
+    ),
+    "DatasetMetadata": ("lightspeed_evaluation.core.models", "DatasetMetadata"),
     "EvaluationResult": ("lightspeed_evaluation.core.models", "EvaluationResult"),
     "EvaluationSummary": (
         "lightspeed_evaluation.core.models.summary",

src/lightspeed_evaluation/api.py

@@ -23,7 +23,7 @@
     print(summary.by_metric)
 """
 
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from lightspeed_evaluation.core.models import (
     EvaluationData,
@@ -35,22 +35,31 @@
 from lightspeed_evaluation.core.system import ConfigLoader
 from lightspeed_evaluation.pipeline.evaluation import EvaluationPipeline
 
+if TYPE_CHECKING:
+    from lightspeed_evaluation.core.models.data import DatasetMetadata
+
 
 def evaluate(
     config: SystemConfig,
     data: list[EvaluationData],
     output_dir: Optional[str] = None,
+    original_data_path: Optional[str] = None,
+    dataset_metadata: Optional["DatasetMetadata"] = None,
 ) -> list[EvaluationResult]:
     """Run evaluation on the provided data using the given configuration.
 
     Creates a fully-initialized pipeline from the ``SystemConfig``, runs
-    evaluation on every conversation in *data*, and returns the raw results.
+    evaluation on every conversations in *data*, and returns the raw results.
     No reports are generated -- file I/O is the caller's responsibility.
 
     Args:
         config: A pre-built SystemConfig instance.
         data: List of EvaluationData conversations to evaluate.
         output_dir: Optional override for the output directory.
+        original_data_path: Path to the original evaluation data file.
+            Required for saving amended data when agents are enabled.
+        dataset_metadata: Optional dataset-level metadata to preserve in
+            amended output files.
 
     Returns:
         List of EvaluationResult objects (one per metric per turn/conversation).
@@ -61,7 +70,11 @@ def evaluate(
     loader = ConfigLoader.from_config(config)
     pipeline = EvaluationPipeline(loader, output_dir)
     try:
-        return pipeline.run_evaluation(data)
+        return pipeline.run_evaluation(
+            data,
+            original_data_path=original_data_path,
+            dataset_metadata=dataset_metadata,
+        )
     finally:
         pipeline.close()
 

src/lightspeed_evaluation/core/models/__init__.py

@@ -14,13 +14,16 @@
     AttachmentData,
 )
 from lightspeed_evaluation.core.models.data import (
+    ConversationMetadata,
+    DatasetMetadata,
     EvaluationData,
     EvaluationRequest,
     EvaluationResult,
     EvaluationScope,
     JudgeScore,
     MetricResult,
     TurnData,
+    TurnMetadata,
 )
 from lightspeed_evaluation.core.models.llm import (
     EmbeddingConfig,
@@ -61,7 +64,10 @@
     "ProposalAgentConfig",
     # Data models
     "TurnData",
+    "TurnMetadata",
     "EvaluationData",
+    "ConversationMetadata",
+    "DatasetMetadata",
     "EvaluationRequest",
     "JudgeScore",
     "MetricResult",

src/lightspeed_evaluation/core/models/data.py

@@ -12,6 +12,39 @@
 logger = logging.getLogger(__name__)
 
 
+class TurnMetadata(BaseModel):
+    """Optional user-defined metadata for a single turn.
+
+    Schema-free: any key-value pairs are accepted.
+    See the Evaluation Data Collection Guide for recommended fields
+    (e.g. complexity, data_source, human_verified, persona).
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+
+class ConversationMetadata(BaseModel):
+    """Optional user-defined metadata for a conversation group.
+
+    Schema-free: any key-value pairs are accepted.
+    See the Evaluation Data Collection Guide for recommended fields
+    (e.g. scenario_category, use_case, interaction_type, topic).
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+
+class DatasetMetadata(BaseModel):
+    """Optional user-defined metadata for the entire evaluation dataset.
+
+    Schema-free: any key-value pairs are accepted.
+    See the Evaluation Data Collection Guide for recommended fields
+    (e.g. team_product, dataset_version, pii_confirmed_removed).
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+
 def _validate_and_deduplicate_metrics(
     metrics: list[str], metric_type: str = "metric"
 ) -> list[str]:
@@ -39,6 +72,10 @@ class TurnData(StreamingMetricsMixin):
     model_config = ConfigDict(extra="forbid")
 
     turn_id: str = Field(..., min_length=1, description="Turn ID (alphanumeric)")
+    metadata: Optional[TurnMetadata] = Field(
+        default=None,
+        description="User-defined metadata for traceability and quality grading",
+    )
     query: Optional[str] = Field(
         default=None,
         min_length=1,
@@ -428,6 +465,10 @@ class EvaluationData(BaseModel):
     conversation_group_id: str = Field(
         ..., min_length=1, description="Unique conversation group identifier"
     )
+    metadata: Optional[ConversationMetadata] = Field(
+        default=None,
+        description="User-defined metadata for traceability and quality grading",
+    )
     description: Optional[str] = Field(
         default=None,
         min_length=1,

src/lightspeed_evaluation/core/output/data_persistence.py

@@ -2,21 +2,28 @@
 
 from datetime import UTC, datetime
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
 import yaml
 
 from lightspeed_evaluation.core.constants import DEFAULT_OUTPUT_DIR
 from lightspeed_evaluation.core.models import EvaluationData
+from lightspeed_evaluation.core.models.data import DatasetMetadata
 
 
-# Use caching
 def save_evaluation_data(
     evaluation_data: list[EvaluationData],
     original_data_path: str,
     output_dir: str = DEFAULT_OUTPUT_DIR,
+    dataset_metadata: Optional[DatasetMetadata] = None,
 ) -> Optional[str]:
-    """Save amended evaluation data to output directory with timestamp."""
+    """Save amended evaluation data to output directory with timestamp.
+
+    When *dataset_metadata* is provided the file is written in the dict
+    format (``metadata`` + ``conversations`` keys) so that dataset-level
+    metadata is preserved across amend cycles.  Without metadata the
+    original list format is used for backward compatibility.
+    """
     original_path = Path(original_data_path)
     amended_data_path = None
 
@@ -33,10 +40,21 @@ def save_evaluation_data(
             / f"{original_path.stem}_amended_{timestamp}{original_path.suffix}"
         )
 
+        conversations = [
+            conv_data.model_dump(mode="json") for conv_data in evaluation_data
+        ]
+
+        output_data: Any = conversations
+        if dataset_metadata is not None:
+            output_data = {
+                "metadata": dataset_metadata.model_dump(mode="json", exclude_none=True),
+                "conversations": conversations,
+            }
+
         # Save amended data to output directory
         with open(amended_data_path, "w", encoding="utf-8") as f:
             yaml.dump(
-                [conv_data.model_dump(mode="json") for conv_data in evaluation_data],
+                output_data,
                 f,
                 default_flow_style=False,
                 sort_keys=False,

src/lightspeed_evaluation/core/system/validator.py

@@ -8,6 +8,7 @@
 from pydantic import ValidationError
 
 from lightspeed_evaluation.core.models import EvaluationData, TurnData
+from lightspeed_evaluation.core.models.data import DatasetMetadata
 from lightspeed_evaluation.core.system.exceptions import DataValidationError
 
 if TYPE_CHECKING:
@@ -167,6 +168,7 @@ def __init__(
         """
         self.validation_errors: list[str] = []
         self.evaluation_data: Optional[list[EvaluationData]] = None
+        self.dataset_metadata: Optional[DatasetMetadata] = None
         self.api_enabled = api_enabled
         self.original_data_path: Optional[str] = None
         self.fail_on_invalid_data = fail_on_invalid_data
@@ -189,6 +191,15 @@ def _conversation_level_metrics(self) -> set[str]:
     def _load_and_parse_yaml(self, data_path: str) -> list[EvaluationData]:
         """Load a YAML file and convert each entry to an EvaluationData model.
 
+        Supports two root formats for backward compatibility:
+
+        1. **List format** (original): YAML root is a list of conversations.
+        2. **Dict format** (new): YAML root is a dict with optional ``metadata``
+           and required ``conversations`` keys.
+
+        When the dict format is used, dataset-level metadata is parsed and
+        stored on ``self.dataset_metadata``.
+
         Args:
             data_path: Path to the evaluation data YAML file.
 
@@ -211,13 +222,12 @@ def _load_and_parse_yaml(self, data_path: str) -> list[EvaluationData]:
 
         if raw_data is None:
             raise DataValidationError("Empty or invalid YAML file")
-        if not isinstance(raw_data, list):
-            raise DataValidationError(
-                f"YAML root must be a list, got {type(raw_data).__name__}"
-            )
+
+        self.dataset_metadata = None
+        raw_conversations = self._extract_conversations_and_metadata(raw_data)
 
         evaluation_data = []
-        for i, data_dict in enumerate(raw_data):
+        for i, data_dict in enumerate(raw_conversations):
             try:
                 eval_data = EvaluationData(**data_dict)
                 evaluation_data.append(eval_data)
@@ -235,6 +245,57 @@ def _load_and_parse_yaml(self, data_path: str) -> list[EvaluationData]:
                 ) from e
         return evaluation_data
 
+    def _extract_conversations_and_metadata(self, raw_data: object) -> list[dict]:
+        """Extract conversation list and optional dataset metadata from raw YAML.
+
+        Args:
+            raw_data: Parsed YAML data (list or dict).
+
+        Returns:
+            List of raw conversation dicts.
+
+        Raises:
+            DataValidationError: If the structure is invalid.
+        """
+        if isinstance(raw_data, list):
+            return raw_data
+
+        if isinstance(raw_data, dict):
+            if "conversations" not in raw_data:
+                raise DataValidationError(
+                    "YAML root is a dict but missing required 'conversations' key. "
+                    "Expected either a list of conversations or a dict with "
+                    "'conversations' (and optional 'metadata') keys."
+                )
+
+            metadata_raw = raw_data.get("metadata")
+            if metadata_raw is not None:
+                if not isinstance(metadata_raw, dict):
+                    raise DataValidationError(
+                        f"'metadata' must be a mapping, "
+                        f"got {type(metadata_raw).__name__}"
+                    )
+                try:
+                    self.dataset_metadata = DatasetMetadata(**metadata_raw)
+                except ValidationError as e:
+                    error_details = format_pydantic_error(e)
+                    raise DataValidationError(
+                        f"Invalid dataset metadata: {error_details}"
+                    ) from e
+
+            raw_conversations = raw_data["conversations"]
+            if not isinstance(raw_conversations, list):
+                raise DataValidationError(
+                    "'conversations' must be a list, "
+                    f"got {type(raw_conversations).__name__}"
+                )
+            return raw_conversations
+
+        raise DataValidationError(
+            f"YAML root must be a list or a dict with 'conversations' key, "
+            f"got {type(raw_data).__name__}"
+        )
+
     def _apply_metrics_filter(
         self, evaluation_data: list[EvaluationData], metrics: list[str]
     ) -> None: