Scorer composition (PR C, 3/3): complete LLM rollout with typed rubrics

pyrit/executor/attack/multi_turn/crescendo.py

@@ -194,7 +194,7 @@ def __init__(
             # Use provided adversarial chat target for scoring
             scoring_chat_target = attack_adversarial_config.target
             objective_scorer = FloatScaleThresholdScorer(
-                scorer=SelfAskScaleScorer(
+                scorer=SelfAskScaleScorer.from_scale_arguments(
                     chat_target=scoring_chat_target,
                     scale_arguments_path=SelfAskScaleScorer.ScalePaths.TASK_ACHIEVED_SCALE.value,
                     system_prompt_path=SelfAskScaleScorer.SystemPaths.RED_TEAMER_SYSTEM_PROMPT.value,

pyrit/executor/attack/multi_turn/tree_of_attacks.py

@@ -1511,7 +1511,7 @@ def __init__(
                 is_objective_required=True,
             )
             default_scorer = FloatScaleThresholdScorer(
-                scorer=SelfAskScaleScorer(
+                scorer=SelfAskScaleScorer.from_scale_arguments(
                     chat_target=self._adversarial_chat,
                     scale_arguments_path=SelfAskScaleScorer.ScalePaths.TASK_ACHIEVED_SCALE.value,
                     validator=scorer_validator,

pyrit/executor/benchmark/fairness_bias.py

@@ -76,9 +76,9 @@ class FairnessBiasBenchmark(Strategy[FairnessBiasBenchmarkContext, AttackResult]
     - Optionally, a custom objective (if not provided, a default will be generated)
 
     Example:
-        scorer = SelfAskCategoryScorer(
+        scorer = SelfAskCategoryScorer.from_content_classifier(
+            chat_target=target,
             content_classifier_path="path/to/classifier.yaml",
-            chat_target=target
         )
         benchmark = FairnessBiasBenchmark(
             objective_target=target,

pyrit/executor/promptgen/fuzzer/fuzzer.py

@@ -577,11 +577,7 @@ def with_default_scorer(
             FuzzerGenerator: A configured FuzzerGenerator instance with default scoring.
         """
         # Create default scorer using the provided scoring target
-        scale_scorer = SelfAskScaleScorer(
-            chat_target=scoring_target,
-            scale_arguments_path=SelfAskScaleScorer.ScalePaths.TREE_OF_ATTACKS_SCALE.value,
-            system_prompt_path=SelfAskScaleScorer.SystemPaths.GENERAL_SYSTEM_PROMPT.value,
-        )
+        scale_scorer = SelfAskScaleScorer(chat_target=scoring_target)
 
         objective_scorer = FloatScaleThresholdScorer(
             scorer=scale_scorer,

pyrit/score/__init__.py

@@ -19,11 +19,23 @@
     FloatScaleScorerByCategory,
 )
 from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer
-from pyrit.score.float_scale.insecure_code_scorer import InsecureCodeScorer
+from pyrit.score.float_scale.insecure_code_scorer import (
+    InsecureCodeScorer,
+    render_insecure_code_system_prompt,
+)
 from pyrit.score.float_scale.plagiarism_scorer import PlagiarismMetric, PlagiarismScorer
 from pyrit.score.float_scale.self_ask_general_float_scale_scorer import SelfAskGeneralFloatScaleScorer
-from pyrit.score.float_scale.self_ask_likert_scorer import LikertScaleEvalFiles, LikertScalePaths, SelfAskLikertScorer
-from pyrit.score.float_scale.self_ask_scale_scorer import SelfAskScaleScorer
+from pyrit.score.float_scale.self_ask_likert_scorer import (
+    LikertScaleEvalFiles,
+    LikertScalePaths,
+    SelfAskLikertScorer,
+    render_likert_system_prompt,
+)
+from pyrit.score.float_scale.self_ask_scale_scorer import (
+    SelfAskScaleScorer,
+    load_scale_arguments,
+    render_scale_system_prompt,
+)
 from pyrit.score.response_handler import (
     CallableResponseHandler,
     JsonSchemaResponseHandler,
@@ -66,7 +78,11 @@
 from pyrit.score.true_false.regex.static_prompt_injection_scorer import StaticPromptInjectionScorer
 from pyrit.score.true_false.regex.xss_output_scorer import XSSOutputScorer
 from pyrit.score.true_false.regex.xxe_output_scorer import XXEOutputScorer
-from pyrit.score.true_false.self_ask_category_scorer import ContentClassifierPaths, SelfAskCategoryScorer
+from pyrit.score.true_false.self_ask_category_scorer import (
+    ContentClassifierPaths,
+    SelfAskCategoryScorer,
+    render_category_system_prompt,
+)
 from pyrit.score.true_false.self_ask_general_true_false_scorer import SelfAskGeneralTrueFalseScorer
 from pyrit.score.true_false.self_ask_question_answer_scorer import SelfAskQuestionAnswerScorer
 from pyrit.score.true_false.self_ask_refusal_scorer import RefusalScorerPaths, SelfAskRefusalScorer
@@ -157,6 +173,7 @@ def __getattr__(name: str) -> object:
     "LDAPInjectionOutputScorer",
     "LikertScaleEvalFiles",
     "LikertScalePaths",
+    "load_scale_arguments",
     "MarkdownInjectionScorer",
     "MethKeywordScorer",
     "MetricsType",
@@ -172,6 +189,10 @@ def __getattr__(name: str) -> object:
     "QuestionAnswerScorer",
     "RegexScorer",
     "RegistryUpdateBehavior",
+    "render_category_system_prompt",
+    "render_insecure_code_system_prompt",
+    "render_likert_system_prompt",
+    "render_scale_system_prompt",
     "render_true_false_system_prompt",
     "ResponseHandler",
     "Scorer",

pyrit/score/conversation_scorer.py

@@ -182,7 +182,7 @@ def create_conversation_scorer(
         ValueError: If the scorer is not an instance of FloatScaleScorer or TrueFalseScorer.
 
     Example:
-        >>> float_scorer = SelfAskLikertScorer(chat_target=target, likert_scale=scale)
+        >>> float_scorer = SelfAskLikertScorer.from_likert_scale(chat_target=target, likert_scale=scale)
         >>> conversation_scorer = create_conversation_scorer(scorer=float_scorer)
         >>> isinstance(conversation_scorer, FloatScaleScorer)  # True
         >>> isinstance(conversation_scorer, ConversationScorer)  # True

pyrit/score/float_scale/insecure_code_scorer.py

@@ -1,24 +1,60 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+from collections.abc import Sequence
 from pathlib import Path
 
 from pyrit.common import verify_and_resolve_path
 from pyrit.common.path import SCORER_SEED_PROMPT_PATH
-from pyrit.exceptions.exception_classes import InvalidJsonException
-from pyrit.models import ComponentIdentifier, MessagePiece, Score, SeedPrompt
+from pyrit.models import ComponentIdentifier, JsonSchemaDefinition, MessagePiece, Score, SeedPrompt
 from pyrit.prompt_target import CHAT_TARGET_REQUIREMENTS, PromptTarget
 from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer
+from pyrit.score.llm_scoring import _run_llm_scoring_async
+from pyrit.score.response_handler import JsonSchemaResponseHandler, ResponseHandler
 from pyrit.score.scorer_prompt_validator import ScorerPromptValidator
 
+_DEFAULT_INSECURE_CODE_SYSTEM_PROMPT_PATH = SCORER_SEED_PROMPT_PATH / "insecure_code" / "system_prompt.yaml"
+_DEFAULT_HARM_CATEGORY = "security"
+
+
+def render_insecure_code_system_prompt(
+    *,
+    harm_categories: str = _DEFAULT_HARM_CATEGORY,
+    template_path: str | Path | None = None,
+) -> SeedPrompt:
+    """
+    Render an insecure-code scoring system prompt from a template.
+
+    Loads the templated system-prompt ``SeedPrompt`` (defaulting to the bundled
+    ``insecure_code/system_prompt.yaml``) and renders it with the given harm categories. The
+    returned ``SeedPrompt`` is a copy whose ``value`` is the rendered text; the template's other
+    fields (notably ``response_json_schema``) are preserved so schema forwarding keeps working.
+
+    Args:
+        harm_categories (str): The harm category rendered into the template. Defaults to "security".
+        template_path (str | Path | None): Path to the system-prompt template YAML. Defaults to the
+            bundled insecure-code system prompt.
+
+    Returns:
+        SeedPrompt: A rendered copy of the template with its ``value`` populated.
+    """
+    resolved_path = verify_and_resolve_path(
+        template_path if template_path else _DEFAULT_INSECURE_CODE_SYSTEM_PROMPT_PATH
+    )
+    template = SeedPrompt.from_yaml_file(resolved_path)
+    rendered_value = template.render_template_value(harm_categories=harm_categories)
+    return template.model_copy(update={"value": rendered_value})
+
 
 class InsecureCodeScorer(FloatScaleScorer):
     """
     A scorer that uses an LLM to evaluate code snippets for potential security vulnerabilities.
 
     This scorer is intended for generated-code evaluation scenarios where the response to score is
     source code or a code-like snippet, such as insecure-coding parity checks across vulnerability
-    scanners. Configuration is loaded from a YAML file for dynamic prompts and instructions.
+    scanners. It holds a chat ``chat_target``, a ``system_prompt`` (a rendered or static
+    ``SeedPrompt``, a plain ``str``, or ``None`` for the default insecure-code rubric), and a
+    ``response_handler`` that turns the target's raw output into a float-scale score.
     """
 
     _DEFAULT_VALIDATOR: ScorerPromptValidator = ScorerPromptValidator(supported_data_types=["text"])
@@ -27,39 +63,64 @@ class InsecureCodeScorer(FloatScaleScorer):
     def __init__(
         self,
         *,
-        chat_target: PromptTarget,
-        system_prompt_path: str | Path | None = None,
+        chat_target: PromptTarget | None = None,
+        system_prompt: SeedPrompt | str | None = None,
+        response_handler: ResponseHandler | None = None,
+        score_category: Sequence[str] | str | None = None,
         validator: ScorerPromptValidator | None = None,
     ) -> None:
         """
         Initialize the Insecure Code Scorer.
 
         Args:
-            chat_target (PromptTarget): The target to use for scoring code security.
-            system_prompt_path (str | Path | None): Path to the YAML file containing the system prompt.
-                Defaults to the default insecure code scoring prompt if not provided.
-            validator (ScorerPromptValidator | None): Custom validator for the scorer. Defaults to None.
+            chat_target (PromptTarget | None): The chat target used for scoring.
+            system_prompt (SeedPrompt | str | None): The scoring system prompt. A ``SeedPrompt``
+                (e.g. rendered via ``render_insecure_code_system_prompt``) is used verbatim and may
+                carry a ``response_json_schema``; a ``str`` is used as-is; ``None`` falls back to the
+                default insecure-code rubric. Defaults to None.
+            response_handler (ResponseHandler | None): Parser for the target's raw output. Defaults
+                to ``JsonSchemaResponseHandler``.
+            score_category (Sequence[str] | str | None): The category to attach to scores. Defaults
+                to "security".
+            validator (ScorerPromptValidator | None): Custom validator for the scorer. Defaults to
+                None.
+
+        Raises:
+            ValueError: If ``chat_target`` is not provided.
         """
+        if chat_target is None:
+            raise ValueError("A chat_target must be provided.")
+
         super().__init__(validator=validator or self._DEFAULT_VALIDATOR, chat_target=chat_target)
 
         self._prompt_target = chat_target
 
-        if not system_prompt_path:
-            system_prompt_path = SCORER_SEED_PROMPT_PATH / "insecure_code" / "system_prompt.yaml"
-
-        self._system_prompt_path: Path = verify_and_resolve_path(system_prompt_path)
-
-        # Load the system prompt template as a SeedPrompt object
-        scoring_instructions_template = SeedPrompt.from_yaml_file(self._system_prompt_path)
+        rendered_value, schema = self._resolve_system_prompt(system_prompt)
+        self._system_prompt = rendered_value
+        # When the caller does not supply a response handler, the default JSON handler carries the
+        # schema (if any) declared by the system prompt and enforces the numeric score contract, so
+        # the round-trip forwards the schema to the scoring target. A caller-supplied handler owns
+        # its own response contract.
+        self._response_handler = response_handler or JsonSchemaResponseHandler(
+            response_schema=schema, numeric_value=True
+        )
 
-        # Define the harm category
-        self._harm_category = "security"
+        self._score_category: Sequence[str] | str = (
+            score_category if score_category is not None else _DEFAULT_HARM_CATEGORY
+        )
 
-        # Render the system prompt with the harm category
-        self._system_prompt = scoring_instructions_template.render_template_value(harm_categories=self._harm_category)
-        # Optional JSON schema embedded in the system prompt YAML. Forwarded to the scoring
-        # target, which enforces it natively when supported or omits it via normalization.
-        self._response_json_schema = scoring_instructions_template.response_json_schema
+    @staticmethod
+    def _resolve_system_prompt(
+        system_prompt: SeedPrompt | str | None,
+    ) -> tuple[str, JsonSchemaDefinition | None]:
+        if system_prompt is None:
+            rendered = render_insecure_code_system_prompt()
+            return rendered.value, rendered.response_json_schema
+        if isinstance(system_prompt, SeedPrompt):
+            return system_prompt.value, system_prompt.response_json_schema
+        if isinstance(system_prompt, str):
+            return system_prompt, None
+        raise TypeError("system_prompt must be a SeedPrompt, str, or None.")
 
     def _build_identifier(self) -> ComponentIdentifier:
         """
@@ -71,7 +132,7 @@ def _build_identifier(self) -> ComponentIdentifier:
         return self._create_identifier(
             params={
                 "system_prompt_template": self._system_prompt,
-                "response_json_schema": self._response_json_schema,
+                "response_json_schema": self._response_handler.response_schema,
             },
             prompt_target=self._prompt_target.get_identifier(),
         )
@@ -88,31 +149,23 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st
             list[Score]: A list containing a single Score object.
 
         Raises:
-            InvalidJsonException: If the expected 'score_value' key is missing in the response.
+            InvalidJsonException: If the response is not valid JSON or the score value is not a float.
         """
-        # Use _score_value_with_llm to interact with the LLM and retrieve an UnvalidatedScore
-        unvalidated_score = await self._score_value_with_llm_async(
-            prompt_target=self._prompt_target,
+        unvalidated_score = await _run_llm_scoring_async(
+            chat_target=self._prompt_target,
             system_prompt=self._system_prompt,
-            message_value=message_piece.original_value,
-            message_data_type=message_piece.converted_value_data_type,
+            response_handler=self._response_handler,
+            value=message_piece.original_value,
+            data_type=message_piece.converted_value_data_type,
             scored_prompt_id=message_piece.id,
-            category=self._harm_category,
+            scorer_identifier=self.get_identifier(),
+            category=self._score_category,
             objective=objective,
-            response_json_schema=self._response_json_schema,
-            numeric_value=True,
         )
 
-        # Modify the UnvalidatedScore parsing to check for 'score_value'
-        try:
-            # Attempt to use score_value if available
-            raw_score_value = float(unvalidated_score.raw_score_value)
-        except KeyError:
-            raise InvalidJsonException(message="Expected 'score_value' key missing in the JSON response") from None
-
         # Convert UnvalidatedScore to Score, applying scaling and metadata
         score = unvalidated_score.to_score(
-            score_value=str(self.scale_value_float(raw_score_value, 0, 1)),
+            score_value=str(self.scale_value_float(float(unvalidated_score.raw_score_value), 0, 1)),
             score_type="float_scale",
         )