[DRAFT] FEAT support adversarial_chat and scoring in scanner to enable automated multi-turn-orchestrators

pyrit/cli/__main__.py

@@ -77,11 +77,11 @@ async def validate_config_and_run_async(config: Dict[str, Any], memory_labels: O
     objective_target = validate_target(config, target_key="objective_target")
     prompt_converters: list[PromptConverter] = []
     # prompt_converters = validate_converters(config)
-    scorer = None
-    # TODO: need to find a solution for single/multiple scorers and scoring_targets
-    # scorers = validate_scorers(config)
     adversarial_chat = None
-    # adversarial_chat = validate_adversarial_chat(config)
+    if "adversarial_chat" in config:
+        adversarial_chat = validate_target(config, target_key="adversarial_chat")
+    scoring_target = validate_scoring_target(config, adversarial_chat=adversarial_chat)
+    objective_scorer = validate_objective_scorer(config, scoring_target=scoring_target)
 
     orchestrators = []
     for scenario_config in scenarios:
@@ -91,7 +91,8 @@ async def validate_config_and_run_async(config: Dict[str, Any], memory_labels: O
                 objective_target=objective_target,
                 adversarial_chat=adversarial_chat,
                 prompt_converters=prompt_converters,
-                scorer=scorer,
+                scoring_target=scoring_target,
+                objective_scorer=objective_scorer,
             )
         )
 
@@ -130,7 +131,8 @@ def validate_scenario(
     objective_target: PromptTarget,
     adversarial_chat: Optional[PromptChatTarget] = None,
     prompt_converters: Optional[List[PromptConverter]] = None,
-    scorer: Optional[Scorer] = None,
+    scoring_target: Optional[PromptChatTarget] = None,
+    objective_scorer: Optional[Scorer] = None,
 ) -> Orchestrator:
     if "type" not in scenario_config:
         raise KeyError("Scenario must contain a 'type' key.")
@@ -150,7 +152,7 @@ def validate_scenario(
 
         # Some orchestrator arguments have their own configuration since they
         # are more complex. They are passed in as args to this function.
-        complex_arg_names = ["objective_target", "adversarial_chat", "prompt_converters", "scorer"]
+        complex_arg_names = ["objective_target", "adversarial_chat", "prompt_converters", "scoring_target", "objective_scorer"]
         for complex_arg_name in complex_arg_names:
             if complex_arg_name in scenario_args:
                 raise ValueError(
@@ -167,7 +169,7 @@ def validate_scenario(
 
         orchestrator = orchestrator_class(**scenario_args)
     except Exception as ex:
-        raise ValueError(f"Failed to validate scenario {scenario_type}") from ex
+        raise ValueError(f"Failed to validate scenario {scenario_type}: {ex}") from ex
     return orchestrator
 
 
@@ -207,6 +209,44 @@ def validate_target(config: Dict[str, Any], target_key: str) -> PromptTarget:
     return target
 
 
+def validate_scoring_target(config: Dict[str, Any], adversarial_chat: Optional[PromptChatTarget]) -> PromptChatTarget | None:
+    if "scoring" not in config:
+        return None
+    scoring_config = config["scoring"]
+
+    # If a scoring_target has been configured use it.
+    # Otherwise, use the adversarial_chat target for scoring.
+    if "scoring_target" in scoring_config:
+        return validate_target(scoring_config, target_key="scoring_target")
+    return adversarial_chat
+
+
+def validate_objective_scorer(config: Dict[str, Any], scoring_target: Optional[PromptChatTarget]) -> Scorer | None:
+    if "scoring" not in config:
+        return None
+    scoring_config = config["scoring"]
+    if "objective_scorer" not in scoring_config:
+        return None
+
+    scorer_args = deepcopy(scoring_config["objective_scorer"])
+
+    if "type" not in scorer_args:
+        raise KeyError("Scorer definition must contain a 'type' key.")
+
+    scorer_type = scorer_args.pop("type")
+
+    try:
+        scorer_module = import_module("pyrit.score")
+        scorer_class = getattr(scorer_module, scorer_type)
+    except Exception as ex:
+        raise RuntimeError(f"Failed to import target {scorer_type} from pyrit.score") from ex
+
+    if scoring_target and "chat_target" in inspect.signature(scorer_class.__init__).parameters:
+        scorer_args["chat_target"] = scoring_target
+
+    return scorer_class(**scorer_args)
+
+
 def main(args=None):
     parsed_args = parse_args(args)
     config_file = parsed_args.config_file

scanner_configurations/basic_multi_turn_attack.yaml

@@ -0,0 +1,33 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "RedTeamingOrchestrator"
+  # - type: "CrescendoOrchestrator"
+  # - type: "TreeOfAttacksWithPruningOrchestrator"
+  #   depth: 2
+objective_target:
+  type: "OpenAIChatTarget" # "AzureMLChatTarget" | "HuggingFaceEndpointTarget" | ...
+  # endpoint_env_variable: # in case one wants to use multiple you need to specify the env vars - not yet supported
+  # api_key_env_variable:
+  # any arg for targets can be listed here:
+  # deployment_name_env_variable:
+  # headers:
+  is_azure_target: true
+# converters:
+#   - type: "Base64Converter"
+#   - type: "LeetspeakConverter"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+  is_azure_target: true
+scoring:
+  # scoring_target is optional. If a target is required but not provided, the adversarial_chat will be used for scoring
+  # scoring_target:
+  #   type: "OpenAIChatTarget"
+  objective_scorer:
+    type: "SelfAskRefusalScorer"
+memory_labels:
+  operator: roakey
+  operation: op_trash_panda
+execution_settings:
+  type: local # or "azureml"
+  # parallel_nodes: 4 # how many scenarios to execute in parallel

scanner_configurations/prompt_send.yaml

@@ -2,8 +2,6 @@ datasets:
   - ./pyrit/datasets/seed_prompts/illegal.prompt
 scenarios:
   - type: "PromptSendingOrchestrator"
-  # - type: "CrescendoOrchestrator"
-  # - type: "TreeOfAttackWithPruningOrchestrator"
 objective_target:
   type: "OpenAIChatTarget" # "AzureMLChatTarget" | "HuggingFaceEndpointTarget" | ...
   # endpoint_env_variable: # in case one wants to use multiple you need to specify the env vars - not yet supported
@@ -15,14 +13,11 @@ objective_target:
 # converters:
 #   - type: "Base64Converter"
 #   - type: "LeetspeakConverter"
-# adversarial_chat:
-#   type: "AzureMLChatTarget"
-#   ...
 # scoring:
-#   scorer: ...
+#   objective_scorer: ...
 memory_labels:
-  operator: romanlutz
-  operation: scanner_setup_jan25
+  operator: roakey
+  operation: op_trash_panda
 execution_settings:
   type: local # or "azureml"
   # parallel_nodes: 4 # how many scenarios to execute in parallel

tests/unit/cli/mixed_multiple_orchestrators_args_success.yaml

@@ -0,0 +1,28 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "PromptSendingOrchestrator"
+  - type: "RedTeamingOrchestrator"
+  - type: "CrescendoOrchestrator"
+    max_turns: 5
+    max_backtracks: 3
+  - type: "CrescendoOrchestrator"
+    max_turns: 10
+    max_backtracks: 10
+  - type: "TreeOfAttacksWithPruningOrchestrator"
+    depth: 10
+    width: 7
+    branching_factor: 8
+  - type: "TreeOfAttacksWithPruningOrchestrator"
+    depth: 5
+    width: 3
+    branching_factor: 6
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  scoring_target:
+    type: "OpenAIChatTarget"
+  objective_scorer:
+    type: "SelfAskRefusalScorer"

tests/unit/cli/multi_turn_crescendo_args_success.yaml

@@ -0,0 +1,13 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "CrescendoOrchestrator"
+    max_turns: 8
+    max_backtracks: 4
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  scoring_target:
+    type: "OpenAIChatTarget"

tests/unit/cli/multi_turn_crescendo_success.yaml

@@ -0,0 +1,11 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "CrescendoOrchestrator"
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  scoring_target:
+    type: "OpenAIChatTarget"

tests/unit/cli/multi_turn_crescendo_wrong_arg.yaml

@@ -0,0 +1,13 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "CrescendoOrchestrator"
+    max_turns: 8
+    wrong_arg: "wrong"
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  scoring_target:
+    type: "OpenAIChatTarget"

tests/unit/cli/multi_turn_multiple_orchestrators_args_success.yaml

@@ -0,0 +1,27 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "RedTeamingOrchestrator"
+  - type: "CrescendoOrchestrator"
+    max_turns: 5
+    max_backtracks: 3
+  - type: "CrescendoOrchestrator"
+    max_turns: 10
+    max_backtracks: 10
+  - type: "TreeOfAttacksWithPruningOrchestrator"
+    depth: 10
+    width: 7
+    branching_factor: 8
+  - type: "TreeOfAttacksWithPruningOrchestrator"
+    depth: 5
+    width: 3
+    branching_factor: 6
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  scoring_target:
+    type: "OpenAIChatTarget"
+  objective_scorer:
+    type: "SelfAskRefusalScorer"

tests/unit/cli/multi_turn_rto_args_success.yaml

@@ -0,0 +1,11 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "RedTeamingOrchestrator"
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  objective_scorer:
+    type: "SelfAskRefusalScorer"

tests/unit/cli/multi_turn_rto_success.yaml

@@ -0,0 +1,11 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "RedTeamingOrchestrator"
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  objective_scorer:
+    type: "SelfAskRefusalScorer"

tests/unit/cli/multi_turn_rto_wrong_arg.yaml

@@ -0,0 +1,12 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "RedTeamingOrchestrator"
+    wrong_arg: "wrong"
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  objective_scorer:
+    type: "SelfAskRefusalScorer"

tests/unit/cli/multi_turn_tap_args_success.yaml

@@ -0,0 +1,14 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "TreeOfAttacksWithPruningOrchestrator"
+    depth: 10
+    width: 7
+    branching_factor: 8
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  scoring_target:
+    type: "OpenAIChatTarget"

tests/unit/cli/multi_turn_tap_success.yaml

@@ -0,0 +1,11 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "TreeOfAttacksWithPruningOrchestrator"
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  scoring_target:
+    type: "OpenAIChatTarget"

tests/unit/cli/multi_turn_tap_wrong_arg.yaml

@@ -0,0 +1,14 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "TreeOfAttacksWithPruningOrchestrator"
+    depth: 10
+    width: 7
+    wrong_arg: "wrong"
+objective_target:
+  type: "OpenAIChatTarget"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+scoring:
+  scoring_target:
+    type: "OpenAIChatTarget"

tests/unit/cli/multi_turn_template.yaml

@@ -0,0 +1,32 @@
+datasets:
+  - ./pyrit/datasets/seed_prompts/illegal.prompt
+scenarios:
+  - type: "RedTeamingOrchestrator"
+  # - type: "CrescendoOrchestrator"
+  # - type: "TreeOfAttacksWithPruningOrchestrator"
+  #   depth: 2
+objective_target:
+  type: "OpenAIChatTarget" # "AzureMLChatTarget" | "HuggingFaceEndpointTarget" | ...
+  # endpoint_env_variable: # in case one wants to use multiple you need to specify the env vars - not yet supported
+  # api_key_env_variable:
+  # any arg for targets can be listed here:
+  # deployment_name_env_variable:
+  # headers:
+# converters:
+#   - type: "Base64Converter"
+#   - type: "LeetspeakConverter"
+adversarial_chat:
+  type: "OpenAIChatTarget"
+  is_azure_target: true
+scoring:
+  # scoring_target is optional. If a target is required but not provided, the adversarial_chat will be used for scoring
+  # scoring_target:
+  #   type: "OpenAIChatTarget"
+  objective_scorer:
+    type: "SelfAskRefusalScorer"
+memory_labels:
+  operator: roakey
+  operation: op_trash_panda
+execution_settings:
+  type: local # or "azureml"
+  # parallel_nodes: 4 # how many scenarios to execute in parallel

tests/unit/cli/prompt_send_no_objective_target.yaml

@@ -1,4 +1,4 @@
 datasets:
   - ./pyrit/datasets/seed_prompts/illegal.prompt
 scenarios:
-  - type: send_prompts
+  - type: "PromptSendingOrchestrator"

tests/unit/cli/prompt_send_no_objective_target_type.yaml

@@ -1,5 +1,5 @@
 datasets:
   - ./pyrit/datasets/seed_prompts/illegal.prompt
 scenarios:
-  - type: send_prompts
+  - type: "PromptSendingOrchestrator"
 objective_target: