[DRAFT][EXPERIMENTAL] FEAT: Proposing a new design for Orchestration of Attacks

pyrit/common/default_values.py

@@ -3,6 +3,7 @@
 
 import logging
 import os
+from typing import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -34,7 +35,7 @@ def get_required_value(*, env_var_name: str, passed_value: str) -> str:
     raise ValueError(f"Environment variable {env_var_name} is required")
 
 
-def get_non_required_value(*, env_var_name: str, passed_value: str) -> str:
+def get_non_required_value(*, env_var_name: str, passed_value: Optional[str] = None) -> str:
     """
     Gets a non-required value from an environment variable or a passed value,
     prefering the passed value.

pyrit/common/path.py

@@ -28,6 +28,7 @@ def in_git_repo() -> bool:
 SCALES_PATH = pathlib.Path(DATASETS_PATH, "score", "scales").resolve()
 
 RED_TEAM_ORCHESTRATOR_PATH = pathlib.Path(DATASETS_PATH, "orchestrators", "red_teaming").resolve()
+TREE_OF_ATTACHS_ORCHESTRATOR_PATH = pathlib.Path(DATASETS_PATH, "orchestrators", "tree_of_attacks").resolve()
 
 # Points to the root of the project
 HOME_PATH = pathlib.Path(PYRIT_PATH, "..").resolve()

pyrit/models/prompt_request_piece.py

@@ -64,7 +64,7 @@ def __init__(
         converter_identifiers: Optional[List[Dict[str, str]]] = None,
         prompt_target_identifier: Optional[Dict[str, str]] = None,
         orchestrator_identifier: Optional[Dict[str, str]] = None,
-        scorer_identifier: Dict[str, str] = None,
+        scorer_identifier: Optional[Dict[str, str]] = None,
         original_value_data_type: PromptDataType = "text",
         converted_value_data_type: PromptDataType = "text",
         response_error: PromptResponseError = "none",
@@ -154,6 +154,18 @@ def to_prompt_request_response(self) -> "PromptRequestResponse":  # type: ignore
         from pyrit.models.prompt_request_response import PromptRequestResponse
 
         return PromptRequestResponse([self])  # noqa F821
+
+    def has_error(self) -> bool:
+        """
+        Check if the prompt request piece has an error.
+        """
+        return self.response_error != "none"
+
+    def is_blocked(self) -> bool:
+        """
+        Check if the prompt request piece is blocked.
+        """
+        return self.response_error == "blocked"
 
     def to_dict(self) -> dict:
         return {

pyrit/models/prompt_request_response.py

@@ -53,6 +53,16 @@ def validate(self):
             elif role != request_piece.role:
                 raise ValueError("Inconsistent roles within the same prompt request response entry.")
 
+    def get_piece(self, n: int = 0) -> PromptRequestPiece:
+        """Return the nth request piece."""
+        if len(self.request_pieces) == 0:
+            raise ValueError("Empty request pieces.")
+
+        if n >= len(self.request_pieces):
+            raise IndexError(f"No request piece at index {n}.")
+
+        return self.request_pieces[n]
+
     def __str__(self):
         ret = ""
         for request_piece in self.request_pieces:

pyrit/orchestrator/multi_turn/tree_of_attacks_node.py

@@ -43,7 +43,7 @@ def __init__(
         adversarial_chat_system_seed_prompt: SeedPrompt,
         desired_response_prefix: str,
         objective_scorer: Scorer,
-        on_topic_scorer: Scorer,
+        on_topic_scorer: Optional[Scorer] = None,
         prompt_converters: list[PromptConverter],
         orchestrator_id: dict[str, str],
         memory_labels: Optional[dict[str, str]] = None,

pyrit/orchestratorv3/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.

pyrit/orchestratorv3/base/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.

pyrit/orchestratorv3/base/attack_executor.py

@@ -0,0 +1,81 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import asyncio
+from typing import List, TypeVar
+
+from pyrit.orchestratorv3.base.attack_strategy import AttackStrategy
+from pyrit.orchestratorv3.base.core import (
+    MultiTurnAttackContext,
+    MultiTurnAttackResult,
+)
+
+MT_Context = TypeVar("MT_Context", bound=MultiTurnAttackContext)
+MT_Result = TypeVar("MT_Result", bound=MultiTurnAttackResult)
+
+
+class AttackExecutor:
+    """
+    Provides different type of attack execution patterns (e.g. we could add execution with timeouts, etc.)
+    """
+
+    async def execute_parallel(
+        self, *, strategy: AttackStrategy[MT_Context, MT_Result], contexts: List[MT_Context], max_concurrency: int = 5
+    ) -> List[MT_Result]:
+        """
+        Execute multiple strategies in parallel with concurrency control.
+
+        Args:
+            strategy: The attack strategy to use
+            contexts: List of contexts to execute the strategy against
+            max_concurrency: Maximum number of concurrent executions
+
+        Returns:
+            List of attack results
+        """
+        semaphore = asyncio.Semaphore(max_concurrency)
+
+        # annonymous function to execute strategy with semaphore
+        async def execute_with_semaphore(strategy: AttackStrategy[MT_Context, MT_Result], ctx: MT_Context) -> MT_Result:
+            async with semaphore:
+                return await strategy.execute(context=ctx)
+
+        tasks = [execute_with_semaphore(strategy, ctx) for ctx in contexts]
+
+        return await asyncio.gather(*tasks)
+
+    async def execute_multi_objective_attack(
+        self,
+        strategy: AttackStrategy[MT_Context, MT_Result],
+        context_template: MT_Context,
+        objectives: List[str],
+        max_concurrency: int = 5,
+    ) -> List[MT_Result]:
+        """
+        Execute multi-turn attacks with multiple objectives against the same target in parallel
+
+        This is a simplified method designed for the common case of running the same strategy
+        with multiple different objectives. Works with any context type that extends
+        MultiTurnAttackContext.
+
+        Args:
+            strategy: The attack strategy to use for all objectives
+            context_template: Template context to duplicate for each objective
+            objectives: List of objectives to attack
+            max_concurrency: Maximum number of concurrent executions
+
+        Returns:
+            List of attack results
+        """
+        contexts = []
+
+        for objective in objectives:
+            # Create a deep copy of the context using its duplicate method
+            context = context_template.duplicate()
+            # Set the new objective (all multi-turn contexts have objectives)
+            context.objective = objective
+            contexts.append(context)
+
+        # Run strategies in parallel
+        results = await self.execute_parallel(strategy=strategy, contexts=contexts, max_concurrency=max_concurrency)
+        return results

pyrit/orchestratorv3/base/attack_pipeline.py

@@ -0,0 +1,192 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import asyncio
+from enum import Enum
+from typing import List, TypeVar
+
+from pyrit.common.logger import logger
+from pyrit.orchestratorv3.base.attack_strategy import AttackStrategy
+from pyrit.orchestratorv3.base.core import (
+    AttackContext,
+    AttackResult,
+)
+
+T_Context = TypeVar("T_Context", bound=AttackContext)
+T_Result = TypeVar("T_Result", bound=AttackResult)
+
+
+class ResultSelectionStrategy(Enum):
+    """Strategies for selecting a result from parallel executions."""
+
+    # First result that achieved objective
+    FIRST_SUCCESS = "first_success"
+    # Result with the highest score
+    BEST_SCORE = "best_score"
+
+
+class AttackPipeline(AttackStrategy[T_Context, T_Result]):
+    """
+    Executes multiple strategies in parallel with proper context isolation
+
+    This class allows running multiple attack strategies at the same time against
+    the same context (each with its own duplicate), and then selecting the
+    result based on a configurable selection strategy.
+    """
+
+    def __init__(
+        self,
+        *,
+        strategies: List[AttackStrategy[T_Context, T_Result]],
+        name: str = "parallel_attack_pipeline",
+        max_concurrency: int = 5,
+        result_selection: ResultSelectionStrategy = ResultSelectionStrategy.FIRST_SUCCESS,
+    ):
+        """
+        Initialize the pipeline with a list of strategies
+
+        Args:
+            strategies: List of strategies to execute in parallel
+            name: Name identifier for this pipeline
+            max_concurrency: Maximum number of strategies to run concurrently
+            result_selection: How to select the final result from multiple parallel executions
+        """
+        super().__init__(logger=logger)
+
+        if not strategies:
+            raise ValueError("Pipeline contains no strategies")
+
+        self._strategies = strategies
+        self._name = name
+        self._max_concurrency = max_concurrency
+        self._result_selection = result_selection
+
+    async def execute(self, *, context: T_Context) -> T_Result:
+        """
+        Execute all strategies in parallel (duplicating context for each)
+
+        Args:
+            context: The initial context
+
+        Returns:
+            Result from the strategy with the best outcome based on selection criteria
+        """
+        self._logger.info(f"Starting parallel attack pipeline '{self._name}' with {len(self._strategies)} strategies")
+
+        # Create semaphore for concurrency control
+        semaphore = asyncio.Semaphore(self._max_concurrency)
+
+        async def execute_strategy_with_semaphore(
+            *, strategy: AttackStrategy[T_Context, T_Result], ctx: T_Context
+        ) -> T_Result:
+            """Execute a strategy with semaphore-based concurrency control"""
+            async with semaphore:
+                self._logger.info(f"Executing strategy: {strategy.__class__.__name__}")
+                return await strategy.execute(context=ctx)
+
+        # Create tasks for each strategy with its own duplicate context
+        tasks = []
+        for strategy in self._strategies:
+            strategy_context = context.duplicate()
+            tasks.append(execute_strategy_with_semaphore(strategy=strategy, ctx=strategy_context))
+
+        # Execute all strategies in parallel and wait for all results
+        self._logger.info(f"Executing {len(tasks)} strategies in parallel (max concurrency: {self._max_concurrency})")
+        results = await asyncio.gather(*tasks)
+
+        # Select result based on configured selection strategy
+        final_result = await self._select_result(results=results)
+        return final_result
+
+    async def _select_result(self, *, results: List[T_Result]) -> T_Result:
+        """
+        Select the result to return based on the configured selection strategy
+
+        Args:
+            results: List of results from all strategies
+
+        Returns:
+            The selected result based on the configured strategy
+        """
+        if not results:
+            raise ValueError("No results available from execution")
+
+        # If there is only one result, return it
+        if len(results) == 1:
+            return results[0]
+
+        # Select appropriate result-selection strategy
+        match self._result_selection:
+            case ResultSelectionStrategy.FIRST_SUCCESS:
+                return await self._select_first_success_result(results=results)
+            case ResultSelectionStrategy.BEST_SCORE:
+                return await self._select_best_score_result(results=results)
+            case _:
+                self._logger.warning(f"Unknown result selection strategy: {self._result_selection}, using first result")
+                return results[0]
+
+    async def _select_first_success_result(self, *, results: List[T_Result]) -> T_Result:
+        """
+        Select the first result that achieved its objective
+
+        Args:
+            results: List of results from all strategies
+
+        Returns:
+            The first successful result or the first result if none were successful
+        """
+        # For multi-turn results, check achieved_objective
+        if all(hasattr(r, "achieved_objective") for r in results):
+            successful_results = [r for r in results if getattr(r, "achieved_objective", False)]
+            if successful_results:
+                self._logger.info(f"Found {len(successful_results)} successful results, returning the first one")
+                return successful_results[0]
+
+        # If no successful results or not multi-turn, return the first result
+        self._logger.info("No successful results found, returning the first result")
+        return results[0]
+
+    async def _select_best_score_result(self, *, results: List[T_Result]) -> T_Result:
+        """
+        Select the result with the highest score
+
+        Args:
+            results: List of results from all strategies
+
+        Returns:
+            The result with the highest score or falls back to first_success if no scores
+        """
+        # For scored results, select the one with highest score
+        results_with_scores = []
+        for r in results:
+            if hasattr(r, "last_score") and getattr(r, "last_score") is not None:
+                last_score = getattr(r, "last_score")
+                # Check if score has a numeric value
+                if hasattr(last_score, "value") and isinstance(last_score.value, (int, float)):
+                    results_with_scores.append((r, last_score.value))
+
+        if results_with_scores:
+            # Sort by score (descending) and return the highest
+            results_with_scores.sort(key=lambda x: x[1], reverse=True)
+            best_result, best_score = results_with_scores[0]
+            self._logger.info(f"Selected result with highest score: {best_score}")
+            return best_result
+
+        # If no scores available, fall back to first_success strategy
+        self._logger.warning("No scores available, falling back to first_success selection")
+        return await self._select_first_success_result(results=results)
+
+    # The base AttackStrategy methods still need to be implemented
+    # but theyn will never be called due to execute() override
+
+    async def _setup(self, *, context: T_Context) -> None:
+        """This is not used in the pipeline."""
+        pass
+
+    async def _perform_attack(self, *, context: T_Context) -> T_Result:
+        """This is not used in the pipeline."""
+        raise NotImplementedError("This method should never be called - execute() is overridden")
+
+    async def _teardown(self, *, context: T_Context) -> None:
+        """This is not used in the pipeline."""
+        pass