Conversation turn fix (Azure#34886)

* Reverting models to make sure calls to the simulator work

* quotes

* Spellcheck fixes

* ignore the models for doc generation

* Fixed the quotes on f strings

* pylint skip file

* Support for summarization

* Adding a limit of 2 conversation turns for all but conversation simulators

* exclude synthetic from mypy

* Another lint fix

* Skip the file causing linting issues

* Bugfix on output to json_qa_lines and empty response from callbacks

* Skip pylint

* Add if/else on message to eval json util

* max conversation turns defaulted to 1 and multiplied by 2 for coversation simulations

* Param rename

* Changed the usage of max_conversation_turns in the tests

Author: nagkumar91

sdk/ai/azure-ai-generative/azure/ai/generative/synthetic/simulator/simulator/simulator.py

@@ -215,14 +215,14 @@ def _join_conversation_starter(self, parameters, to_join):
     async def simulate_async(
         self,
         template: "Template",
-        max_conversation_turns: int = 2,
+        max_conversation_turns: int = 1,
         parameters: Optional[List[dict]] = None,
         jailbreak: bool = False,
         api_call_retry_limit: int = 3,
         api_call_retry_sleep_sec: int = 1,  # pylint: disable=unused-argument
         api_call_delay_sec: float = 0,
         concurrent_async_task: int = 3,
-        simulation_result_limit: int = 3,
+        max_simulation_results: int = 3,
     ):
         """Asynchronously simulate conversations using the provided template and parameters
 
@@ -245,8 +245,8 @@ async def simulate_async(
         :paramtype api_call_delay_sec: float, optional
         :keyword concurrent_async_task: The maximum number of asynchronous tasks to run concurrently. Defaults to 3.
         :paramtype concurrent_async_task: int, optional
-        :keyword simulation_result_limit: The maximum number of simulation results to return. Defaults to 3.
-        :paramtype simulation_result_limit: int, optional
+        :keyword max_simulation_results: The maximum number of simulation results to return. Defaults to 3.
+        :paramtype max_simulation_results: int, optional
 
         :return: A list of dictionaries containing the simulation results.
         :rtype: List[Dict]
@@ -267,6 +267,8 @@ async def simulate_async(
             )
         if "conversation" not in template.template_name:
             max_conversation_turns = 2
+        else:
+            max_conversation_turns = max_conversation_turns * 2
         if template.content_harm:
             self._ensure_service_dependencies()
             self.adversarial = True
@@ -283,15 +285,15 @@ async def simulate_async(
         tasks = []
         total_tasks = sum(len(t.template_parameters) for t in templates)
 
-        if simulation_result_limit > total_tasks and self.adversarial:
+        if max_simulation_results > total_tasks and self.adversarial:
             logger.warning(
                 "Cannot provide %s results due to maximum number of adversarial simulations that can be generated: %s."
                 "\n %s simulations will be generated.",
-                simulation_result_limit,
+                max_simulation_results,
                 total_tasks,
                 total_tasks,
             )
-        total_tasks = min(total_tasks, simulation_result_limit)
+        total_tasks = min(total_tasks, max_simulation_results)
         progress_bar = tqdm(
             total=total_tasks,
             desc="generating simulations",
@@ -321,10 +323,10 @@ async def simulate_async(
                     )
                 )
 
-                if len(tasks) >= simulation_result_limit:
+                if len(tasks) >= max_simulation_results:
                     break
 
-            if len(tasks) >= simulation_result_limit:
+            if len(tasks) >= max_simulation_results:
                 break
 
         sim_results = []

sdk/ai/azure-ai-generative/tests/simulator/unittests/test_simulator.py

@@ -108,7 +108,7 @@ def test_simulator_returns_formatted_conversations(self, _, simulate_conversatio
         conv = simulator.simulate(
             template=conv_template,
             parameters=[task_parameters],
-            max_conversation_turns=2)
+            max_conversation_turns=1)
 
         expected_keys = set(["messages", "$schema", "template_parameters"])
         assert issubclass(type(conv), list)
@@ -165,7 +165,7 @@ def test_simulator_from_openai_callback(self, to_chat_completion_model, mock_con
         conv = sim.simulate(
             template=conv_template,
             parameters=[task_parameters],
-            max_conversation_turns=2)
+            max_conversation_turns=1)
 
         oai_mock.assert_called_once()
         assert(len(conv) == 1)
@@ -201,7 +201,7 @@ async def callback(cm):
         conv = sim.simulate(
             template=conv_template,
             parameters=[task_parameters],
-            max_conversation_turns=2)
+            max_conversation_turns=1)
 
         assert(len(conv) == 1)
         assert(conv[0]["messages"][1]["content"] == content)
@@ -226,7 +226,7 @@ async def callback(cm):
         conv = sim.simulate(
             template=conv_template,
             parameters=[task_parameters],
-            max_conversation_turns=2)
+            max_conversation_turns=1)
 
         assert(len(conv) == 1)
         assert(conv[0]["messages"][1]["content"] == content)
@@ -242,7 +242,7 @@ def test_simulator_throws_expected_error_from_incorrect_template_type(self, mock
             asyncio.set_event_loop(loop)
             loop.run_until_complete(simulator.simulate_async(
                 template="wrong template type",
-                max_conversation_turns=2,
+                max_conversation_turns=1,
                 parameters=[task_parameters]
                 )
             )
@@ -274,4 +274,4 @@ def test_simulator_throws_expected_error_from_unset_ai_client_or_connection(self
             )
 
         assert(str(all_none_exc_info.value).startswith("One and only one of the parameters [ai_client, simulator_connection]"))
-        assert(str(all_set_exc_info.value).startswith("One and only one of the parameters [ai_client, simulator_connection]"))
+        assert(str(all_set_exc_info.value).startswith("One and only one of the parameters [ai_client, simulator_connection]"))