[LEADS-389] Support user-defined metadata in evaluation data format for GDS quality grading and traceability

Author: bsatapat-jpg

README.md

@@ -320,6 +320,7 @@ For field tables, full YAML examples (file-only, file + SQLite, file + Postgres)
 | Field                           | Type           | Required | Description                                                          |
 |---------------------------------|----------------|----------|----------------------------------------------------------------------|
 | `conversation_group_id`         | string         | ✅       | Unique identifier for conversation                                   |
+| `metadata`                      | ConversationMetadata | ❌ | User-defined metadata for traceability and quality grading           |
 | `description`                   | string         | ❌       | Optional description                                                 |
 | `tag`                           | string         | ❌       | Tag for grouping eval conversations (default: "eval")             |
 | `setup_script`                  | string         | ❌       | Path to setup script (Optional, used when API is enabled)            |
@@ -333,6 +334,7 @@ For field tables, full YAML examples (file-only, file + SQLite, file + Postgres)
 | Field                 | Type             | Required | Description                          | API Populated         |
 |-----------------------|------------------|----------|--------------------------------------|-----------------------|
 | `turn_id`             | string           | ✅       | Unique identifier for the turn       | ❌                    |
+| `metadata`            | TurnMetadata     | ❌       | User-defined metadata for traceability and quality grading | ❌ |
 | `query`               | string           | ✅       | The question/prompt to evaluate      | ❌                    |
 | `response`            | string           | 📋       | Actual response from system          | ✅ (if API enabled)   |
 | `contexts`            | list[string]     | 📋       | Context information for evaluation   | ✅ (if API enabled)   |

src/lightspeed_evaluation/__init__.py

@@ -24,11 +24,14 @@
     from lightspeed_evaluation.core.llm import LLMManager
     from lightspeed_evaluation.core.models import (
         APIConfig,
+        ConversationMetadata,
+        DatasetMetadata,
         EvaluationData,
         EvaluationResult,
         LLMConfig,
         LoggingConfig,
         TurnData,
+        TurnMetadata,
         VisualizationConfig,
     )
     from lightspeed_evaluation.core.models.summary import EvaluationSummary
@@ -79,6 +82,12 @@
     "VisualizationConfig": ("lightspeed_evaluation.core.models", "VisualizationConfig"),
     "EvaluationData": ("lightspeed_evaluation.core.models", "EvaluationData"),
     "TurnData": ("lightspeed_evaluation.core.models", "TurnData"),
+    "TurnMetadata": ("lightspeed_evaluation.core.models", "TurnMetadata"),
+    "ConversationMetadata": (
+        "lightspeed_evaluation.core.models",
+        "ConversationMetadata",
+    ),
+    "DatasetMetadata": ("lightspeed_evaluation.core.models", "DatasetMetadata"),
     "EvaluationResult": ("lightspeed_evaluation.core.models", "EvaluationResult"),
     "EvaluationSummary": (
         "lightspeed_evaluation.core.models.summary",

src/lightspeed_evaluation/api.py

@@ -23,7 +23,7 @@
     print(summary.by_metric)
 """
 
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from lightspeed_evaluation.core.models import (
     EvaluationData,
@@ -35,22 +35,31 @@
 from lightspeed_evaluation.core.system import ConfigLoader
 from lightspeed_evaluation.pipeline.evaluation import EvaluationPipeline
 
+if TYPE_CHECKING:
+    from lightspeed_evaluation.core.models.data import DatasetMetadata
+
 
 def evaluate(
     config: SystemConfig,
     data: list[EvaluationData],
     output_dir: Optional[str] = None,
+    original_data_path: Optional[str] = None,
+    dataset_metadata: Optional["DatasetMetadata"] = None,
 ) -> list[EvaluationResult]:
     """Run evaluation on the provided data using the given configuration.
 
     Creates a fully-initialized pipeline from the ``SystemConfig``, runs
-    evaluation on every conversation in *data*, and returns the raw results.
+    evaluation on every conversations in *data*, and returns the raw results.
     No reports are generated -- file I/O is the caller's responsibility.
 
     Args:
         config: A pre-built SystemConfig instance.
         data: List of EvaluationData conversations to evaluate.
         output_dir: Optional override for the output directory.
+        original_data_path: Path to the original evaluation data file.
+            Required for saving amended data when agents are enabled.
+        dataset_metadata: Optional dataset-level metadata to preserve in
+            amended output files.
 
     Returns:
         List of EvaluationResult objects (one per metric per turn/conversation).
@@ -61,7 +70,11 @@ def evaluate(
     loader = ConfigLoader.from_config(config)
     pipeline = EvaluationPipeline(loader, output_dir)
     try:
-        return pipeline.run_evaluation(data)
+        return pipeline.run_evaluation(
+            data,
+            original_data_path=original_data_path,
+            dataset_metadata=dataset_metadata,
+        )
     finally:
         pipeline.close()
 

src/lightspeed_evaluation/core/models/__init__.py

@@ -14,13 +14,16 @@
     AttachmentData,
 )
 from lightspeed_evaluation.core.models.data import (
+    ConversationMetadata,
+    DatasetMetadata,
     EvaluationData,
     EvaluationRequest,
     EvaluationResult,
     EvaluationScope,
     JudgeScore,
     MetricResult,
     TurnData,
+    TurnMetadata,
 )
 from lightspeed_evaluation.core.models.llm import (
     EmbeddingConfig,
@@ -61,7 +64,10 @@
     "ProposalAgentConfig",
     # Data models
     "TurnData",
+    "TurnMetadata",
     "EvaluationData",
+    "ConversationMetadata",
+    "DatasetMetadata",
     "EvaluationRequest",
     "JudgeScore",
     "MetricResult",

src/lightspeed_evaluation/core/models/data.py

@@ -12,6 +12,99 @@
 logger = logging.getLogger(__name__)
 
 
+class TurnMetadata(BaseModel):
+    """Optional user-defined metadata for a single turn."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    complexity: Optional[str] = Field(
+        default=None,
+        description="Complexity level (e.g. Simple, Moderate, Complex)",
+    )
+    data_source: Optional[str] = Field(
+        default=None,
+        description="Data source (e.g. Human-written, Production log, Synthetic)",
+    )
+    human_verified: Optional[bool] = Field(
+        default=None, description="Whether a domain expert verified this turn"
+    )
+    verified_by: Optional[str] = Field(default=None, description="Verifier name or ID")
+    negative_type: Optional[str] = Field(
+        default=None,
+        description="Negative scenario type (e.g. Out-of-scope, Adversarial, Ambiguous)",
+    )
+    persona: Optional[str] = Field(
+        default=None,
+        description="User persona represented (e.g. developer, admin, beginner)",
+    )
+    date_created: Optional[str] = Field(
+        default=None, description="When the sample was created (e.g. 2025-06-15)"
+    )
+    additional_metadata: Optional[dict[str, Any]] = Field(
+        default=None, description="Arbitrary key-value pairs for extra metadata"
+    )
+
+
+class ConversationMetadata(BaseModel):
+    """Optional user-defined metadata for a conversation group."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    scenario_category: Optional[str] = Field(
+        default=None,
+        description="Scenario category (e.g. Core/Happy path, Edge Case, Negative)",
+    )
+    use_case: Optional[str] = Field(
+        default=None,
+        description="System capability (e.g. RAG, Agent/Tools)",
+    )
+    interaction_type: Optional[str] = Field(
+        default=None,
+        description="Interaction type (e.g. Single-turn, Multi-turn)",
+    )
+    topic: Optional[str] = Field(
+        default=None,
+        description="Domain subject area (e.g. networking, storage)",
+    )
+    jtbd_reference: Optional[str] = Field(
+        default=None, description="Jobs-to-be-done reference (Job/Task)"
+    )
+    notes: Optional[str] = Field(
+        default=None, description="Free-text notes about the conversation"
+    )
+    additional_metadata: Optional[dict[str, Any]] = Field(
+        default=None, description="Arbitrary key-value pairs for extra metadata"
+    )
+
+
+class DatasetMetadata(BaseModel):
+    """Optional user-defined metadata for the entire evaluation dataset."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    team_product: Optional[str] = Field(
+        default=None, description="Owning team or product (with contact details)"
+    )
+    dataset_version: Optional[str] = Field(
+        default=None, description="Dataset version for tracking iterations"
+    )
+    pii_confirmed_removed: Optional[bool] = Field(
+        default=None, description="Whether PII has been confirmed removed"
+    )
+    generation_tools: Optional[list[str]] = Field(
+        default=None, description="Tools used for synthetic data generation"
+    )
+    llms_used: Optional[list[str]] = Field(
+        default=None, description="LLMs used in the generation pipeline"
+    )
+    last_updated: Optional[str] = Field(
+        default=None, description="Date the dataset was last updated (e.g. 2025-06-15)"
+    )
+    additional_metadata: Optional[dict[str, Any]] = Field(
+        default=None, description="Arbitrary key-value pairs for extra metadata"
+    )
+
+
 def _validate_and_deduplicate_metrics(
     metrics: list[str], metric_type: str = "metric"
 ) -> list[str]:
@@ -39,6 +132,10 @@ class TurnData(StreamingMetricsMixin):
     model_config = ConfigDict(extra="forbid")
 
     turn_id: str = Field(..., min_length=1, description="Turn ID (alphanumeric)")
+    metadata: Optional[TurnMetadata] = Field(
+        default=None,
+        description="User-defined metadata for traceability and quality grading",
+    )
     query: Optional[str] = Field(
         default=None,
         min_length=1,
@@ -428,6 +525,10 @@ class EvaluationData(BaseModel):
     conversation_group_id: str = Field(
         ..., min_length=1, description="Unique conversation group identifier"
     )
+    metadata: Optional[ConversationMetadata] = Field(
+        default=None,
+        description="User-defined metadata for traceability and quality grading",
+    )
     description: Optional[str] = Field(
         default=None,
         min_length=1,

src/lightspeed_evaluation/core/output/data_persistence.py

@@ -2,21 +2,28 @@
 
 from datetime import UTC, datetime
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
 import yaml
 
 from lightspeed_evaluation.core.constants import DEFAULT_OUTPUT_DIR
 from lightspeed_evaluation.core.models import EvaluationData
+from lightspeed_evaluation.core.models.data import DatasetMetadata
 
 
-# Use caching
 def save_evaluation_data(
     evaluation_data: list[EvaluationData],
     original_data_path: str,
     output_dir: str = DEFAULT_OUTPUT_DIR,
+    dataset_metadata: Optional[DatasetMetadata] = None,
 ) -> Optional[str]:
-    """Save amended evaluation data to output directory with timestamp."""
+    """Save amended evaluation data to output directory with timestamp.
+
+    When *dataset_metadata* is provided the file is written in the dict
+    format (``metadata`` + ``conversations`` keys) so that dataset-level
+    metadata is preserved across amend cycles.  Without metadata the
+    original list format is used for backward compatibility.
+    """
     original_path = Path(original_data_path)
     amended_data_path = None
 
@@ -33,10 +40,21 @@ def save_evaluation_data(
             / f"{original_path.stem}_amended_{timestamp}{original_path.suffix}"
         )
 
+        conversations = [
+            conv_data.model_dump(mode="json") for conv_data in evaluation_data
+        ]
+
+        output_data: Any = conversations
+        if dataset_metadata is not None:
+            output_data = {
+                "metadata": dataset_metadata.model_dump(mode="json", exclude_none=True),
+                "conversations": conversations,
+            }
+
         # Save amended data to output directory
         with open(amended_data_path, "w", encoding="utf-8") as f:
             yaml.dump(
-                [conv_data.model_dump(mode="json") for conv_data in evaluation_data],
+                output_data,
                 f,
                 default_flow_style=False,
                 sort_keys=False,