Evaluation: Track evaluator usage through user agent (#38600)

* track evaluator usage through user agent

* fix black

* fix the tests

* fix the type hint

Author: ninghu

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py

@@ -72,18 +72,21 @@ def get_formatted_template(data: dict, annotation_task: str) -> str:
     return user_text.replace("'", '\\"')
 
 
-def get_common_headers(token: str) -> Dict:
+def get_common_headers(token: str, evaluator_name: Optional[str] = None) -> Dict:
     """Get common headers for the HTTP request
 
     :param token: The Azure authentication token.
     :type token: str
+    :param evaluator_name: The evaluator name. Default is None.
+    :type evaluator_name: str
     :return: The common headers.
     :rtype: Dict
     """
+    user_agent = f"{USER_AGENT} (type=evaluator; subtype={evaluator_name})" if evaluator_name else USER_AGENT
     return {
         "Authorization": f"Bearer {token}",
         "Content-Type": "application/json",
-        "User-Agent": USER_AGENT,
+        "User-Agent": user_agent,
         # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
         # https://github.com/encode/httpx/discussions/2959
         "Connection": "close",
@@ -175,7 +178,9 @@ def generate_payload(normalized_user_text: str, metric: str, annotation_task: st
     )
 
 
-async def submit_request(data: dict, metric: str, rai_svc_url: str, token: str, annotation_task: str) -> str:
+async def submit_request(
+    data: dict, metric: str, rai_svc_url: str, token: str, annotation_task: str, evaluator_name: str
+) -> str:
     """Submit request to Responsible AI service for evaluation and return operation ID
 
     :param data: The data to evaluate.
@@ -188,14 +193,16 @@ async def submit_request(data: dict, metric: str, rai_svc_url: str, token: str,
     :type token: str
     :param annotation_task: The annotation task to use.
     :type annotation_task: str
+    :param evaluator_name: The evaluator name.
+    :type evaluator_name: str
     :return: The operation ID.
     :rtype: str
     """
     normalized_user_text = get_formatted_template(data, annotation_task)
     payload = generate_payload(normalized_user_text, metric, annotation_task=annotation_task)
 
     url = rai_svc_url + "/submitannotation"
-    headers = get_common_headers(token)
+    headers = get_common_headers(token, evaluator_name)
 
     async with get_async_http_client_with_timeout() as client:
         http_response = await client.post(url, json=payload, headers=headers)
@@ -493,24 +500,26 @@ async def evaluate_with_rai_service(
     credential: TokenCredential,
     annotation_task: str = Tasks.CONTENT_HARM,
     metric_display_name=None,
+    evaluator_name=None,
 ) -> Dict[str, Union[str, float]]:
-    """ "Evaluate the content safety of the response using Responsible AI service
+    """Evaluate the content safety of the response using Responsible AI service
 
-       :param data: The data to evaluate.
-       :type data: dict
-       :param metric_name: The evaluation metric to use.
-       :type metric_name: str
-       :param project_scope: The Azure AI project scope details.
-       :type project_scope: Dict
-       :param credential: The Azure authentication credential.
-       :type credential:
-    ~azure.core.credentials.TokenCredential
-       :param annotation_task: The annotation task to use.
-       :type annotation_task: str
-       :param metric_display_name: The display name of metric to use.
-       :type metric_display_name: str
-       :return: The parsed annotation result.
-       :rtype: Dict[str, Union[str, float]]
+    :param data: The data to evaluate.
+    :type data: dict
+    :param metric_name: The evaluation metric to use.
+    :type metric_name: str
+    :param project_scope: The Azure AI project scope details.
+    :type project_scope: Dict
+    :param credential: The Azure authentication credential.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param annotation_task: The annotation task to use.
+    :type annotation_task: str
+    :param metric_display_name: The display name of metric to use.
+    :type metric_display_name: str
+    :param evaluator_name: The evaluator name to use.
+    :type evaluator_name: str
+    :return: The parsed annotation result.
+    :rtype: Dict[str, Union[str, float]]
     """
 
     # Get RAI service URL from discovery service and check service availability
@@ -519,7 +528,7 @@ async def evaluate_with_rai_service(
     await ensure_service_availability(rai_svc_url, token, annotation_task)
 
     # Submit annotation request and fetch result
-    operation_id = await submit_request(data, metric_name, rai_svc_url, token, annotation_task)
+    operation_id = await submit_request(data, metric_name, rai_svc_url, token, annotation_task, evaluator_name)
     annotation_response = cast(List[Dict], await fetch_result(operation_id, rai_svc_url, credential, token))
     result = parse_response(annotation_response, metric_name, metric_display_name)
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -47,10 +47,12 @@ def __init__(self, *, result_key: str, prompty_file: str, model_config: dict, ev
         self._prompty_file = prompty_file
         super().__init__(eval_last_turn=eval_last_turn)
 
+        subclass_name = self.__class__.__name__
+        user_agent = f"{USER_AGENT} (type=evaluator subtype={subclass_name})"
         prompty_model_config = construct_prompty_model_config(
             validate_model_config(model_config),
             self._DEFAULT_OPEN_API_VERSION,
-            USER_AGENT,
+            user_agent,
         )
 
         self._flow = AsyncPrompty.load(source=prompty_file, model=prompty_model_config)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py

@@ -111,6 +111,7 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, T]:
             project_scope=self._azure_ai_project,
             credential=self._credential,
             annotation_task=self._get_task(),
+            evaluator_name=self.__class__.__name__,
         )
 
     def _get_task(self):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py

@@ -25,7 +25,7 @@
 from ._utils import JsonLineChatProtocol
 
 
-USER_AGENT += " (type=simulator subtype=Simulator)"
+USER_AGENT += " (type=simulator; subtype=Simulator)"
 
 
 @experimental

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_content_safety_rai_script.py

@@ -178,6 +178,7 @@ async def test_submit_request(self, client_mock):
             rai_svc_url="www.notarealurl.com",
             token="dummy",
             annotation_task=Tasks.CONTENT_HARM,
+            evaluator_name="dummy-evaluator",
         )
         assert result == "dummy-operation-id"
 
@@ -198,6 +199,7 @@ async def test_submit_request_not_found(self, client_mock):
                 rai_svc_url="www.notarealurl.com",
                 token="dummy",
                 annotation_task=Tasks.CONTENT_HARM,
+                evaluator_name="dummy-evaluator",
             )
         assert "Operation returned an invalid status '404 Not Found'" in str(exc_info._excinfo[1])