From 46331b7af52801d7499c13d281f2c4cee34223ac Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Mon, 14 Apr 2025 16:57:34 -0700 Subject: [PATCH 1/3] Adding sample for evaluation --- .../azure/ai/projects/onedp/models/_patch.py | 5 +- .../onedp/models/_patch_evaluations.py | 17 ++++ .../samples/evaluation/sample_evaluations.py | 83 +++++++++++++++++++ .../sample_data_evaluation.jsonl | 1 + 4 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch_evaluations.py create mode 100644 sdk/ai/azure-ai-projects-onedp/samples/evaluation/sample_evaluations.py create mode 100644 sdk/ai/azure-ai-projects-onedp/samples/evaluation/samples_folder/sample_data_evaluation.jsonl diff --git a/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch.py b/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch.py index f7dd32510333..03a12558c643 100644 --- a/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch.py +++ b/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch.py @@ -7,8 +7,11 @@ Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize """ from typing import List +from ._patch_evaluations import EvaluationMetrics -__all__: List[str] = [] # Add all objects you want publicly available to users at this package level +__all__: List[str] = [ + "EvaluationMetrics", +] # Add all objects you want publicly available to users at this package level def patch_sdk(): diff --git a/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch_evaluations.py b/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch_evaluations.py new file mode 100644 index 000000000000..225576c5f3ec --- /dev/null +++ b/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch_evaluations.py @@ -0,0 +1,17 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Customize generated code here. + +Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize +""" +from enum import Enum + +class EvaluationMetrics(str, Enum): + Relevance = "relevance" + HateUnfairness = "hate_unfairness" + Violence = "violence" + Groundedness = "groundedness" + GroundednessPro = "groundedness_pro" diff --git a/sdk/ai/azure-ai-projects-onedp/samples/evaluation/sample_evaluations.py b/sdk/ai/azure-ai-projects-onedp/samples/evaluation/sample_evaluations.py new file mode 100644 index 000000000000..228919e220cb --- /dev/null +++ b/sdk/ai/azure-ai-projects-onedp/samples/evaluation/sample_evaluations.py @@ -0,0 +1,83 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an AIProjectClient, this sample demonstrates how to use the synchronous + `.evaluations` methods to create, get and list evaluations. + +USAGE: + python sample_evaluations.py + + Before running the sample: + + pip install azure-ai-projects azure-identity + + Set these environment variables with your own values: + 1) PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your + Azure AI Foundry project. + 2) DATASET_NAME - Required. The name of the Dataset to create and use in this sample. +""" + +import os +from azure.identity import DefaultAzureCredential +from azure.ai.projects.onedp import AIProjectClient +from azure.ai.projects.onedp.models import Evaluation, InputDataset, EvaluatorConfiguration, EvaluationMetrics +from dotenv import load_dotenv + +load_dotenv() + +endpoint = os.environ["PROJECT_ENDPOINT"] +dataset_name = os.environ["DATASET_NAME"] + +with AIProjectClient( + endpoint=endpoint, + credential=DefaultAzureCredential(exclude_interactive_browser_credential=False), +) as project_client: + + # [START evaluations_sample] + print( + "Upload a single file and create a new Dataset to reference the file. Here we explicitly specify the dataset version." + ) + # dataset: DatasetVersion = project_client.datasets.upload_file_and_create( + # name=dataset_name, + # version="1", + # file="./samples_folder/sample_data_evaluation.jsonl", + # ) + # print(dataset) + + print("Create an evaluation") + # evaluation = Evaluation( + # display_name="Sample Evaluation", + # # Support format is : azureai://accounts/ilmatfdp492025-resource/projects/ilmatfdp492025/data/eval-4-11/versions/1 + # data=InputDataset(id="azureml://locations/centraluseuap/workspaces/abc/data/abc/versions/11"), + # evaluators={ + # "relevance": EvaluatorConfiguration( + # id=f"aiservices:{EvaluationMetrics.Relevance.value}", + # # id="azureml://registries/azureml/models/Retrieval-Evaluator/versions/4", + # # either client or service (TBD) resolves to azureml://registries/azureml/models/Retrieval-Evaluator/versions/... + # init_params={ + # "deployment_name": "gpt-4o", + # }, + # ), + # "hate_unfairness": EvaluatorConfiguration( + # # id=f"aiservices:{EvaluationMetrics.HateUnfairness.value}", + # id="azureml://registries/azureml/models/Retrieval-Evaluator/versions/4", + # # either client or service (TBD) resolves to azureml://registries/azureml/models/Hate-Unfairness-Evaluator/versions/... + # init_params={ + # "azure_ai_project": endpoint, + # }, + # ), + # }, + # ) + # + # evaluation_respone = project_client.evaluations.create_run(evaluation) + + print("Get evaluation") + # get_evaluation_response = project_client.evaluations.get(evaluation_respone.id) + # print(get_evaluation_response) + + # [END evaluations_sample] diff --git a/sdk/ai/azure-ai-projects-onedp/samples/evaluation/samples_folder/sample_data_evaluation.jsonl b/sdk/ai/azure-ai-projects-onedp/samples/evaluation/samples_folder/sample_data_evaluation.jsonl new file mode 100644 index 000000000000..c2187bb5add8 --- /dev/null +++ b/sdk/ai/azure-ai-projects-onedp/samples/evaluation/samples_folder/sample_data_evaluation.jsonl @@ -0,0 +1 @@ +{"query": "What is capital of France?", "context": "France is in Europe", "response": "Paris is the capital of France.", "ground_truth": "Paris is the capital of France."} \ No newline at end of file From 911842abb12606d4a9eae295b53b66212cf6f99f Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Mon, 14 Apr 2025 17:43:06 -0700 Subject: [PATCH 2/3] Fixing spell check errors --- .vscode/cspell.json | 7 +++++++ .../samples/evaluation/sample_evaluations.py | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index c4aedd3a0853..d1cdeda583a1 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -1361,6 +1361,13 @@ "azureopenai" ] }, + { + "filename": "sdk/ai/azure-ai-projects-onedp/**", + "words": [ + "aiservices", + "azureai", + ] + }, { "filename": "sdk/ai/azure-ai-inference/**", "words": [ diff --git a/sdk/ai/azure-ai-projects-onedp/samples/evaluation/sample_evaluations.py b/sdk/ai/azure-ai-projects-onedp/samples/evaluation/sample_evaluations.py index 228919e220cb..f4c33f93f8d4 100644 --- a/sdk/ai/azure-ai-projects-onedp/samples/evaluation/sample_evaluations.py +++ b/sdk/ai/azure-ai-projects-onedp/samples/evaluation/sample_evaluations.py @@ -52,7 +52,6 @@ print("Create an evaluation") # evaluation = Evaluation( # display_name="Sample Evaluation", - # # Support format is : azureai://accounts/ilmatfdp492025-resource/projects/ilmatfdp492025/data/eval-4-11/versions/1 # data=InputDataset(id="azureml://locations/centraluseuap/workspaces/abc/data/abc/versions/11"), # evaluators={ # "relevance": EvaluatorConfiguration( From 171ea6d43d0715796d8ebd3282fb3851b12b152c Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Tue, 15 Apr 2025 00:19:09 -0700 Subject: [PATCH 3/3] Fixing pylint issues --- .../projects/onedp/models/_patch_evaluations.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch_evaluations.py b/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch_evaluations.py index 225576c5f3ec..df5067c9a3ac 100644 --- a/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch_evaluations.py +++ b/sdk/ai/azure-ai-projects-onedp/azure/ai/projects/onedp/models/_patch_evaluations.py @@ -9,9 +9,12 @@ """ from enum import Enum -class EvaluationMetrics(str, Enum): - Relevance = "relevance" - HateUnfairness = "hate_unfairness" - Violence = "violence" - Groundedness = "groundedness" - GroundednessPro = "groundedness_pro" +from azure.core import CaseInsensitiveEnumMeta + + +class EvaluationMetrics(str, Enum, metaclass=CaseInsensitiveEnumMeta): + RELEVANCE = "relevance" + HATE_UNFAIRNESS = "hate_unfairness" + VIOLENCE = "violence" + GROUNDEDNESS = "groundedness" + GROUNDEDNESS_PRO = "groundedness_pro"