feat(EvalSchema): updating eval schema

Specifically updating evaluation and evaluationSet schema to adhere to agent definitions.

# Conflicts:
#	pyproject.toml

Author: akshaylive

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.67"
+version = "2.1.68"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

src/uipath/_cli/_evals/_models/_evaluation_set.py

@@ -5,6 +5,10 @@
 from pydantic.alias_generators import to_camel
 
 
+class EvaluationSimulationTool(BaseModel):
+    name: str = Field(..., alias="name")
+
+
 class EvaluationItem(BaseModel):
     """Individual evaluation item within an evaluation set."""
 
@@ -14,15 +18,19 @@ class EvaluationItem(BaseModel):
     name: str
     inputs: Dict[str, Any]
     expected_output: Dict[str, Any]
-    expected_agent_behavior: str = ""
-    simulation_instructions: str = ""
-    simulate_input: bool = False
-    input_generation_instructions: str = ""
-    simulate_tools: bool = False
-    tools_to_simulate: List[str] = Field(default_factory=list)
-    eval_set_id: str
-    created_at: str
-    updated_at: str
+    expected_agent_behavior: str = Field(default="", alias="expectedAgentBehavior")
+    simulation_instructions: str = Field(default="", alias="simulationInstructions")
+    simulate_input: bool = Field(default=False, alias="simulateInput")
+    input_generation_instructions: str = Field(
+        default="", alias="inputGenerationInstructions"
+    )
+    simulate_tools: bool = Field(default=False, alias="simulateTools")
+    tools_to_simulate: List[EvaluationSimulationTool] = Field(
+        default_factory=list, alias="toolsToSimulate"
+    )
+    eval_set_id: str = Field(alias="evalSetId")
+    created_at: str = Field(alias="createdAt")
+    updated_at: str = Field(alias="updatedAt")
 
 
 class EvaluationSet(BaseModel):
@@ -31,15 +39,17 @@ class EvaluationSet(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
 
     id: str
-    file_name: str
+    file_name: str = Field(..., alias="fileName")
     evaluator_refs: List[str] = Field(default_factory=list)
     evaluations: List[EvaluationItem] = Field(default_factory=list)
     name: str
-    batch_size: int = 10
-    timeout_minutes: int = 20
-    model_settings: List[Dict[str, Any]] = Field(default_factory=list)
-    created_at: str
-    updated_at: str
+    batch_size: int = Field(10, alias="batch_size")
+    timeout_minutes: int = Field(default=20, alias="timeoutMinutes")
+    model_settings: List[Dict[str, Any]] = Field(
+        default_factory=list, alias="modelSettings"
+    )
+    created_at: str = Field(alias="createdAt")
+    updated_at: str = Field(alias="updatedAt")
 
     def extract_selected_evals(self, eval_ids) -> None:
         selected_evals: list[EvaluationItem] = []

src/uipath/_cli/_evals/_models/_evaluator.py

@@ -0,0 +1,61 @@
+from typing import Annotated, Any, Union
+
+from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
+
+from uipath.eval.models.models import EvaluatorCategory, EvaluatorType
+
+
+class EvaluatorBaseParams(BaseModel):
+    """Parameters for initializing the base evaluator."""
+
+    id: str
+    name: str
+    description: str
+    category: EvaluatorCategory = Field(..., alias="category")
+    evaluator_type: EvaluatorType = Field(..., alias="type")
+    created_at: str = Field(..., alias="createdAt")
+    updated_at: str = Field(..., alias="updatedAt")
+    target_output_key: str = Field(..., alias="targetOutputKey")
+    file_name: str = Field(..., alias="fileName")
+
+
+class LLMEvaluatorParams(EvaluatorBaseParams):
+    prompt: str = Field(..., alias="prompt")
+    model: str = Field(..., alias="model")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+class UnknownEvaluatorParams(EvaluatorBaseParams):
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+def evaluator_discriminator(data: Any) -> str:
+    if isinstance(data, dict):
+        category = data.get("category")
+        evaluator_type = data.get("type")
+        if (
+            category == EvaluatorCategory.LlmAsAJudge
+            or evaluator_type == EvaluatorType.Trajectory
+        ):
+            return "LLMEvaluatorParams"
+    return "UnknownEvaluatorParams"
+
+
+Evaluator = Annotated[
+    Union[
+        Annotated[
+            LLMEvaluatorParams,
+            Tag("LLMEvaluatorParams"),
+        ],
+        Annotated[
+            UnknownEvaluatorParams,
+            Tag("UnknownEvaluatorParams"),
+        ],
+    ],
+    Field(discriminator=Discriminator(evaluator_discriminator)),
+]

src/uipath/_cli/cli_pull.py

@@ -112,7 +112,7 @@ async def download_folder_files(
             if local_hash != remote_hash:
                 styled_path = click.style(str(file_path), fg="cyan")
                 console.warning(f"File {styled_path}" + " differs from remote version.")
-                response = click.prompt("Do you want to override it? (y/n)", type=str)
+                response = click.prompt("Do you want to overwrite it? (y/n)", type=str)
                 if response.lower() == "y":
                     with open(local_path, "w", encoding="utf-8", newline="\n") as f:
                         f.write(remote_content)

src/uipath/agent/_utils.py

@@ -31,6 +31,52 @@ async def load_agent_definition(project_id: str):
         await get_file(project_structure, PurePath("agent.json"), studio_client)
     ).json()
 
+    evaluators = []
+    try:
+        evaluators_path = resolve_path(
+            project_structure, PurePath("evals", "evaluators")
+        )
+        if isinstance(evaluators_path, ProjectFolder):
+            for file in evaluators_path.files:
+                evaluators.append(
+                    (
+                        await get_file(
+                            evaluators_path, PurePath(file.name), studio_client
+                        )
+                    ).json()
+                )
+        else:
+            logger.warning(
+                "Unable to read evaluators from project. Defaulting to empty evaluators."
+            )
+    except Exception:
+        logger.warning(
+            "Unable to read evaluators from project. Defaulting to empty evaluators."
+        )
+
+    evaluation_sets = []
+    try:
+        evaluation_sets_path = resolve_path(
+            project_structure, PurePath("evals", "eval-sets")
+        )
+        if isinstance(evaluation_sets_path, ProjectFolder):
+            for file in evaluation_sets_path.files:
+                evaluation_sets.append(
+                    (
+                        await get_file(
+                            evaluation_sets_path, PurePath(file.name), studio_client
+                        )
+                    ).json()
+                )
+        else:
+            logger.warning(
+                "Unable to read eval-sets from project. Defaulting to empty eval-sets."
+            )
+    except Exception:
+        logger.warning(
+            "Unable to read eval-sets from project. Defaulting to empty eval-sets."
+        )
+
     resolved_path = resolve_path(project_structure, PurePath("resources"))
     if isinstance(resolved_path, ProjectFolder):
         resource_folders = resolved_path.folders
@@ -50,6 +96,8 @@ async def load_agent_definition(project_id: str):
         "id": project_id,
         "name": project_structure.name,
         "resources": resources,
+        "evaluators": evaluators,
+        "evaluationSets": evaluation_sets,
         **agent,
     }
     return TypeAdapter(AgentDefinition).validate_python(agent_definition)

src/uipath/agent/models/agent.py

@@ -5,6 +5,8 @@
 
 from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
 
+from uipath._cli._evals._models._evaluation_set import EvaluationSet
+from uipath._cli._evals._models._evaluator import Evaluator
 from uipath.models import Connection
 
 
@@ -307,6 +309,12 @@ class BaseAgentDefinition(BaseModel):
     resources: List[AgentResourceConfig] = Field(
         ..., description="List of tools, context, and escalation resources"
     )
+    evaluation_sets: List[EvaluationSet] = Field(
+        ...,
+        alias="evaluationSets",
+        description="List of agent evaluation sets",
+    )
+    evaluators: List[Evaluator] = Field(..., description="List of agent evaluators")
 
     model_config = ConfigDict(
         validate_by_name=True, validate_by_alias=True, extra="allow"