From 02c8a71fe32d8d5d666df7ba554ab045cf33b34e Mon Sep 17 00:00:00 2001 From: hannahwestra25 Date: Tue, 23 Jun 2026 21:07:58 -0400 Subject: [PATCH 1/8] Fix crash when scorer LLM response is blocked by content filter When the scorer's own LLM call is blocked by Azure content filtering, the response contains only an error piece with no text piece. The next() call in _score_value_with_llm_async raises StopIteration, which becomes a RuntimeError inside the async coroutine. Add an explicit check for all-blocked response pieces before the next() call and raise a descriptive BadRequestException instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyrit/score/scorer.py | 11 +++++++++++ tests/unit/score/test_scorer.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index 031c7c1553..522cba80c9 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -17,6 +17,7 @@ ) from pyrit.exceptions import ( + BadRequestException, InvalidJsonException, PyritException, pyrit_json_retry, @@ -781,6 +782,16 @@ async def _score_value_with_llm_async( response_json: str = "" try: + # Check if the scorer's own LLM response was blocked by content filtering + if all(piece.is_blocked() for piece in response[0].message_pieces): + raise BadRequestException( + message=( + f"The scorer's LLM request was blocked by content filtering while scoring " + f"prompt ID: {scored_prompt_id}. Consider using a scorer endpoint with " + f"content filtering disabled for red-teaming workflows." + ) + ) + # Get the text piece which contains the JSON response containing the score_value and rationale from the LLM text_piece = next( piece for piece in response[0].message_pieces if piece.converted_value_data_type == "text" diff --git a/tests/unit/score/test_scorer.py b/tests/unit/score/test_scorer.py index 6491ccefeb..9afde12082 100644 --- a/tests/unit/score/test_scorer.py +++ b/tests/unit/score/test_scorer.py @@ -1548,6 +1548,39 @@ async def test_score_value_with_llm_skips_reasoning_piece(good_json): assert result.score_rationale == "Valid response" +async def test_score_value_with_llm_raises_when_scorer_response_blocked(): + """When the scorer's own LLM response is blocked by content filtering, raise BadRequestException.""" + from pyrit.exceptions import BadRequestException + + chat_target = MagicMock(PromptTarget) + chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") + + blocked_piece = MessagePiece( + role="assistant", + original_value="", + original_value_data_type="error", + converted_value="", + converted_value_data_type="error", + conversation_id="test-convo", + response_error="blocked", + ) + blocked_response = Message(message_pieces=[blocked_piece]) + chat_target.send_prompt_async = AsyncMock(return_value=[blocked_response]) + + scorer = MockScorer() + + with pytest.raises(BadRequestException, match="blocked by content filtering"): + await scorer._score_value_with_llm_async( + prompt_target=chat_target, + system_prompt="system_prompt", + message_value="message_value", + message_data_type="text", + scored_prompt_id="test-prompt-id", + category="category", + objective="task", + ) + + # ── Helpers for score_blocked_content tests ────────────────────────────────── From 7e697729a272f7107e87f6ae611eb2f069f40ed3 Mon Sep 17 00:00:00 2001 From: hannahwestra25 Date: Tue, 23 Jun 2026 21:19:10 -0400 Subject: [PATCH 2/8] Add BadRequestException to docstring Raises section --- pyrit/score/scorer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index 522cba80c9..950c867b29 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -730,6 +730,7 @@ async def _score_value_with_llm_async( score_value still needs to be normalized and validated. Raises: + BadRequestException: If the scorer's LLM response is blocked by content filtering. ValueError: If required keys are missing from the response or if the response format is invalid. InvalidJsonException: If the response is not valid JSON. Exception: For other unexpected errors during scoring. From 99d5840ec431affef73dad686c54660ce6a6f226 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 30 Jun 2026 16:31:11 -0700 Subject: [PATCH 3/8] Clarify content-filter block message and document all() check Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyrit/score/scorer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index 07b9e3c440..0a58ec6265 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -783,11 +783,14 @@ async def _score_value_with_llm_async( response_json: str = "" try: - # Check if the scorer's own LLM response was blocked by content filtering + # Check if the scorer's own LLM response was blocked by content filtering. + # Require every piece to be blocked (all) rather than any: a partially blocked + # response may still carry a usable text piece, so we only bail out when there + # is no salvageable content to parse. if all(piece.is_blocked() for piece in response[0].message_pieces): raise BadRequestException( message=( - f"The scorer's LLM request was blocked by content filtering while scoring " + f"The scorer's LLM response was blocked by content filtering while scoring " f"prompt ID: {scored_prompt_id}. Consider using a scorer endpoint with " f"content filtering disabled for red-teaming workflows." ) From 0aa44b909015e429be0b2bbbcb79e43ff9f53b14 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Wed, 1 Jul 2026 04:56:35 -0700 Subject: [PATCH 4/8] Correct blocked-response comment per review A content-filter block always yields a single fully-blocked error piece, so the previous comment's claim about a salvageable text piece was inaccurate. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyrit/score/scorer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index 0a58ec6265..e04c09b5a8 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -783,10 +783,8 @@ async def _score_value_with_llm_async( response_json: str = "" try: - # Check if the scorer's own LLM response was blocked by content filtering. - # Require every piece to be blocked (all) rather than any: a partially blocked - # response may still carry a usable text piece, so we only bail out when there - # is no salvageable content to parse. + # A content-filter block yields a single error piece with no parseable text piece, + # so raise a clear error here instead of failing on the missing text piece below. if all(piece.is_blocked() for piece in response[0].message_pieces): raise BadRequestException( message=( From 5287319faff47dc4b1c6d68c1246c7d8ae101ae1 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Fri, 10 Jul 2026 14:02:59 -0700 Subject: [PATCH 5/8] Make scorer content-filter block configurable via raise_if_scorer_blocks When the scorer's own LLM response is content-filtered, the transport now raises a dedicated ScorerLLMResponseBlockedException instead of a generic BadRequestException. The raise-vs-fallback policy lives in the Scorer (per framework.md): score_async catches the exception and, when raise_if_scorer_blocks is False, returns the scorer's type default (False / 0.0) via _build_fallback_score instead of raising. Defaults to raising. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyrit/exceptions/__init__.py | 2 + pyrit/exceptions/exception_classes.py | 15 ++ .../azure_content_filter_scorer.py | 11 +- pyrit/score/float_scale/float_scale_scorer.py | 14 +- pyrit/score/llm_scoring.py | 14 +- pyrit/score/scorer.py | 39 +++- pyrit/score/true_false/true_false_scorer.py | 14 +- .../score/test_conversation_history_scorer.py | 4 +- tests/unit/score/test_scorer.py | 182 +++++++++++++++++- 9 files changed, 277 insertions(+), 18 deletions(-) diff --git a/pyrit/exceptions/__init__.py b/pyrit/exceptions/__init__.py index 5b4948a628..6d889ab95d 100644 --- a/pyrit/exceptions/__init__.py +++ b/pyrit/exceptions/__init__.py @@ -12,6 +12,7 @@ MissingPromptPlaceholderException, PyritException, RateLimitException, + ScorerLLMResponseBlockedException, get_retry_max_num_attempts, handle_bad_request_exception, pyrit_custom_result_retry, @@ -60,6 +61,7 @@ "RateLimitException", "remove_markdown_json", "RetryCollector", + "ScorerLLMResponseBlockedException", "set_execution_context", "set_retry_collector", "execution_context", diff --git a/pyrit/exceptions/exception_classes.py b/pyrit/exceptions/exception_classes.py index 5b4f3de72f..04043a2381 100644 --- a/pyrit/exceptions/exception_classes.py +++ b/pyrit/exceptions/exception_classes.py @@ -205,6 +205,21 @@ def __init__(self, *, status_code: int = 204, message: str = "No Content") -> No super().__init__(status_code=status_code, message=message) +class ScorerLLMResponseBlockedException(BadRequestException): + """Exception raised when a scorer's own LLM response is blocked by content filtering.""" + + def __init__(self, *, status_code: int = 400, message: str = "Scorer LLM response blocked") -> None: + """ + Initialize a scorer-response-blocked exception. + + Args: + status_code (int): Status code for the error. + message (str): Error message. + + """ + super().__init__(status_code=status_code, message=message) + + class InvalidJsonException(PyritException): """Exception class for blocked content errors.""" diff --git a/pyrit/score/float_scale/azure_content_filter_scorer.py b/pyrit/score/float_scale/azure_content_filter_scorer.py index 89cd2597cc..b4f4bfd84a 100644 --- a/pyrit/score/float_scale/azure_content_filter_scorer.py +++ b/pyrit/score/float_scale/azure_content_filter_scorer.py @@ -340,7 +340,9 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st for result in aggregated_results ] - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: """ Build one neutral ``0.0`` fallback score per configured harm category. @@ -357,6 +359,8 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l Args: message (Message): The message whose first piece is inspected for status. objective (str | None): The objective associated with this scoring call. + scorer_response_blocked (bool): When True, the scorer's own LLM response was + blocked by content filtering; reflected in the rationale. Returns: list[Score]: One ``0.0`` ``float_scale`` score per configured harm category, @@ -370,7 +374,10 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l if piece_id is None: raise ValueError("Cannot create score: message piece has no id or original_prompt_id") - if first_piece.is_blocked(): + if scorer_response_blocked: + status = "The scorer's own LLM response was blocked by content filtering (raise_if_scorer_blocks is False)" + description = "Scorer response blocked; returning 0.0 per configured category." + elif first_piece.is_blocked(): status = ( "The request was blocked by the target (score_blocked_content is False or no partial content available)" ) diff --git a/pyrit/score/float_scale/float_scale_scorer.py b/pyrit/score/float_scale/float_scale_scorer.py index 6f304c2e8f..538ea44d1c 100644 --- a/pyrit/score/float_scale/float_scale_scorer.py +++ b/pyrit/score/float_scale/float_scale_scorer.py @@ -49,7 +49,9 @@ def __init__(self, *, validator: ScorerPromptValidator, chat_target: PromptTarge """ super().__init__(validator=validator, chat_target=chat_target) - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: """ Build a single-element list containing a neutral ``0.0`` score when no pieces could be scored. @@ -59,6 +61,8 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l Args: message (Message): The message whose first piece is inspected for status. objective (str | None): The objective associated with this scoring call. + scorer_response_blocked (bool): When True, the scorer's own LLM response was + blocked by content filtering; reflected in the rationale. Returns: list[Score]: A single-element list containing a ``0.0`` ``float_scale`` score @@ -72,7 +76,13 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l if piece_id is None: raise ValueError("Cannot create score: message piece has no id or original_prompt_id") - if first_piece.is_blocked(): + if scorer_response_blocked: + rationale = ( + "The scorer's own LLM response was blocked by content filtering " + "(raise_if_scorer_blocks is False); returning 0.0." + ) + description = "Scorer response blocked; returning 0.0." + elif first_piece.is_blocked(): rationale = ( "The request was blocked by the target " "(score_blocked_content is False or no partial content available); returning 0.0." diff --git a/pyrit/score/llm_scoring.py b/pyrit/score/llm_scoring.py index 14d3387871..8d4d1c50c7 100644 --- a/pyrit/score/llm_scoring.py +++ b/pyrit/score/llm_scoring.py @@ -6,7 +6,7 @@ import uuid from typing import TYPE_CHECKING, Any -from pyrit.exceptions import BadRequestException, pyrit_json_retry +from pyrit.exceptions import ScorerLLMResponseBlockedException, pyrit_json_retry from pyrit.models import JSON_SCHEMA_METADATA_KEY, Message, MessagePiece if TYPE_CHECKING: @@ -75,7 +75,9 @@ async def _run_llm_scoring_async( normalized and validated by the caller. Raises: - BadRequestException: If the scoring target's response is blocked by content filtering. + ScorerLLMResponseBlockedException: If the scoring target's response is blocked by + content filtering. The calling ``Scorer`` decides whether to raise or return a + default score (see ``Scorer.raise_if_scorer_blocks``). InvalidJsonException: If the response is not valid JSON, is missing required keys, or fails the handler's value validation. Exception: For other unexpected errors during scoring. @@ -130,10 +132,12 @@ async def _run_llm_scoring_async( except Exception as ex: raise Exception(f"Error scoring prompt with original prompt ID: {scored_prompt_id}") from ex - # A content-filter block yields a single error piece with no parseable text piece, - # so raise a clear error here instead of failing on the missing text piece below. + # A content-filter block yields a single error piece with no parseable text piece. + # Surfacing this as a dedicated exception keeps the transport free of policy: the + # Scorer decides whether to raise or fall back to a default score + # (see Scorer.raise_if_scorer_blocks). Not retried by @pyrit_json_retry. if all(piece.is_blocked() for piece in response[0].message_pieces): - raise BadRequestException( + raise ScorerLLMResponseBlockedException( message=( f"The scorer's LLM response was blocked by content filtering while scoring " f"prompt ID: {scored_prompt_id}. Consider using a scorer endpoint with " diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index 226eb44cd8..30f0d86047 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -14,7 +14,7 @@ cast, ) -from pyrit.exceptions import PyritException +from pyrit.exceptions import PyritException, ScorerLLMResponseBlockedException from pyrit.memory import CentralMemory, MemoryInterface from pyrit.models import ( ChatMessageRole, @@ -81,6 +81,15 @@ class Scorer(Identifiable, abc.ABC): #: (Chat Completions API) and ``OpenAIResponseTarget`` (Responses API). score_blocked_content: bool = False + #: Controls what happens when the scorer's *own* LLM response is blocked by content + #: filtering (common in red-teaming, since the scorer's rationale quotes harmful content). + #: When True (default), scoring raises ``ScorerLLMResponseBlockedException`` — a blocked + #: scorer endpoint is treated as a real error. When False, scoring returns the scorer's + #: type default instead (False for true/false scorers, 0.0 for float-scale). This is + #: distinct from ``score_blocked_content``, which concerns the target-under-test response. + #: Set this on scorer instances before use. Defaults to True. + raise_if_scorer_blocks: bool = True + def __init_subclass__(cls, **kwargs: Any) -> None: """ Enforce the keyword-only constructor contract on subclasses. @@ -228,6 +237,8 @@ async def score_async( list[Score]: A list of Score objects representing the results. Raises: + ScorerLLMResponseBlockedException: If the scorer's own LLM response is blocked by + content filtering and ``raise_if_scorer_blocks`` is True (the default). PyritException: If scoring raises a PyRIT exception (re-raised with enhanced context). RuntimeError: If scoring raises a non-PyRIT exception (wrapped with scorer context). """ @@ -259,6 +270,25 @@ async def score_async( scoring_message, objective=objective, ) + except ScorerLLMResponseBlockedException as e: + # The scorer's own LLM response was content-filtered. By default this is a real + # error and re-raised; when raise_if_scorer_blocks is False, fall back to the + # scorer's type default (False / 0.0) instead. The decision lives here in the + # Scorer, not the transport (see doc/code/framework.md). + if self.raise_if_scorer_blocks: + e.message = f"Error in scorer {self.__class__.__name__}: {e.message}" + e.args = (f"Status Code: {e.status_code}, Message: {e.message}",) + raise + logger.info( + "Scorer %s LLM response was blocked by content filtering; " + "returning default score (raise_if_scorer_blocks=False).", + self.__class__.__name__, + ) + scores = self._build_fallback_score( + message=scoring_message, + objective=objective, + scorer_response_blocked=True, + ) except PyritException as e: # Re-raise PyRIT exceptions with enhanced context while preserving type for retry decorators e.message = f"Error in scorer {self.__class__.__name__}: {e.message}" @@ -401,7 +431,9 @@ def _get_supported_pieces(self, message: Message) -> list[MessagePiece]: ] @abstractmethod - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: """ Return neutral fallback ``Score`` objects when ``_score_async`` produced no scores. @@ -420,6 +452,9 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l Args: message (Message): The (possibly substituted) message that was scored. objective (str | None): The objective associated with this scoring call. + scorer_response_blocked (bool): When True, the fallback was triggered because the + scorer's *own* LLM response was blocked by content filtering (not the + target-under-test). Subclasses should reflect this in the rationale. Returns: list[Score]: One or more fallback scores. Must not be empty. diff --git a/pyrit/score/true_false/true_false_scorer.py b/pyrit/score/true_false/true_false_scorer.py index 30f4ab21d5..c3bb3399a7 100644 --- a/pyrit/score/true_false/true_false_scorer.py +++ b/pyrit/score/true_false/true_false_scorer.py @@ -160,7 +160,9 @@ async def _score_async(self, message: Message, *, objective: str | None = None) ) ] - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: """ Build a single-element list containing a ``false`` score when no pieces could be scored. @@ -170,6 +172,8 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l Args: message (Message): The message whose first piece is inspected for status. objective (str | None): The objective associated with this scoring call. + scorer_response_blocked (bool): When True, the scorer's own LLM response was + blocked by content filtering; reflected in the rationale. Returns: list[Score]: A single-element list containing a ``false`` ``true_false`` score @@ -183,7 +187,13 @@ def _build_fallback_score(self, *, message: Message, objective: str | None) -> l if piece_id is None: raise ValueError("Cannot create score: message piece has no id or original_prompt_id") - if first_piece.is_blocked(): + if scorer_response_blocked: + rationale = ( + "The scorer's own LLM response was blocked by content filtering " + "(raise_if_scorer_blocks is False); returning false." + ) + description = "Scorer response blocked; returning false." + elif first_piece.is_blocked(): rationale = ( "The request was blocked by the target " "(score_blocked_content is False or no partial content available); returning false." diff --git a/tests/unit/score/test_conversation_history_scorer.py b/tests/unit/score/test_conversation_history_scorer.py index ddfa663c79..0fa339b40e 100644 --- a/tests/unit/score/test_conversation_history_scorer.py +++ b/tests/unit/score/test_conversation_history_scorer.py @@ -68,7 +68,9 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: st def validate_return_scores(self, scores: list[Score]): pass - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: return [ Score( score_value="false", diff --git a/tests/unit/score/test_scorer.py b/tests/unit/score/test_scorer.py index 62f81b712c..19a7992b80 100644 --- a/tests/unit/score/test_scorer.py +++ b/tests/unit/score/test_scorer.py @@ -14,6 +14,7 @@ from pyrit.models import ComponentIdentifier, Message, MessagePiece, Score from pyrit.prompt_target import PromptTarget from pyrit.score import ( + FloatScaleScorer, Scorer, ScorerPromptValidator, TrueFalseScorer, @@ -143,7 +144,9 @@ def validate_return_scores(self, scores: list[Score]): for score in scores: assert 0 <= float(score.score_value) <= 1 - def _build_fallback_score(self, *, message: Message, objective: str | None) -> list[Score]: + def _build_fallback_score( + self, *, message: Message, objective: str | None, scorer_response_blocked: bool = False + ) -> list[Score]: return [ Score( score_value="0.0", @@ -1551,8 +1554,8 @@ async def test_score_value_with_llm_skips_reasoning_piece(good_json): async def test_score_value_with_llm_raises_when_scorer_response_blocked(): - """When the scorer's own LLM response is blocked by content filtering, raise BadRequestException.""" - from pyrit.exceptions import BadRequestException + """When the scorer's own LLM response is blocked, the transport raises ScorerLLMResponseBlockedException.""" + from pyrit.exceptions import ScorerLLMResponseBlockedException chat_target = MagicMock(PromptTarget) chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") @@ -1571,7 +1574,7 @@ async def test_score_value_with_llm_raises_when_scorer_response_blocked(): scorer = MockScorer() - with pytest.raises(BadRequestException, match="blocked by content filtering"): + with pytest.raises(ScorerLLMResponseBlockedException, match="blocked by content filtering"): await scorer._score_value_with_llm_async( prompt_target=chat_target, system_prompt="system_prompt", @@ -1582,6 +1585,177 @@ async def test_score_value_with_llm_raises_when_scorer_response_blocked(): objective="task", ) + # A blocked response is a terminal condition, not a transient JSON error: it must not retry. + assert chat_target.send_prompt_async.call_count == 1 + + +# ── Axis B: the scorer's own LLM response is blocked (raise_if_scorer_blocks) ───────────── + + +class _ForwarderTrueFalseScorer(TrueFalseScorer): + """TrueFalseScorer whose piece scoring goes through the base ``_score_value_with_llm_async`` forwarder.""" + + def __init__(self, *, chat_target: PromptTarget) -> None: + super().__init__(validator=DummyValidator()) + self._prompt_target = chat_target + self._system_prompt = "system" + + def _build_identifier(self) -> ComponentIdentifier: + return self._create_identifier() + + async def _score_piece_async(self, message_piece: MessagePiece, *, objective: str | None = None) -> list[Score]: + unvalidated = await self._score_value_with_llm_async( + prompt_target=self._prompt_target, + system_prompt=self._system_prompt, + message_value=message_piece.converted_value, + message_data_type="text", + scored_prompt_id=message_piece.id, + objective=objective, + ) + return [unvalidated.to_score(score_value=unvalidated.raw_score_value, score_type="true_false")] + + +class _DirectTransportTrueFalseScorer(TrueFalseScorer): + """TrueFalseScorer that calls ``_run_llm_scoring_async`` directly, like SelfAskTrueFalseScorer.""" + + def __init__(self, *, chat_target: PromptTarget) -> None: + from pyrit.score import JsonSchemaResponseHandler + + super().__init__(validator=DummyValidator()) + self._prompt_target = chat_target + self._system_prompt = "system" + self._response_handler = JsonSchemaResponseHandler() + + def _build_identifier(self) -> ComponentIdentifier: + return self._create_identifier() + + async def _score_piece_async(self, message_piece: MessagePiece, *, objective: str | None = None) -> list[Score]: + from pyrit.score.llm_scoring import _run_llm_scoring_async + + unvalidated = await _run_llm_scoring_async( + chat_target=self._prompt_target, + system_prompt=self._system_prompt, + response_handler=self._response_handler, + value=message_piece.converted_value, + data_type="text", + scored_prompt_id=message_piece.id, + scorer_identifier=self.get_identifier(), + objective=objective, + ) + return [unvalidated.to_score(score_value=unvalidated.raw_score_value, score_type="true_false")] + + +class _ForwarderFloatScaleScorer(FloatScaleScorer): + """FloatScaleScorer whose piece scoring goes through the base ``_score_value_with_llm_async`` forwarder.""" + + def __init__(self, *, chat_target: PromptTarget) -> None: + super().__init__(validator=DummyValidator()) + self._prompt_target = chat_target + self._system_prompt = "system" + + def _build_identifier(self) -> ComponentIdentifier: + return self._create_identifier() + + async def _score_piece_async(self, message_piece: MessagePiece, *, objective: str | None = None) -> list[Score]: + unvalidated = await self._score_value_with_llm_async( + prompt_target=self._prompt_target, + system_prompt=self._system_prompt, + message_value=message_piece.converted_value, + message_data_type="text", + scored_prompt_id=message_piece.id, + objective=objective, + numeric_value=True, + ) + return [unvalidated.to_score(score_value=unvalidated.raw_score_value, score_type="float_scale")] + + +def _make_scorer_blocking_target() -> MagicMock: + """A chat target mock whose response is fully blocked by content filtering.""" + chat_target = MagicMock(PromptTarget) + chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") + chat_target.set_system_prompt = MagicMock() + blocked_piece = MessagePiece( + role="assistant", + original_value="", + original_value_data_type="error", + converted_value="", + converted_value_data_type="error", + conversation_id="scorer-convo", + response_error="blocked", + ) + chat_target.send_prompt_async = AsyncMock(return_value=[Message(message_pieces=[blocked_piece])]) + return chat_target + + +def _make_normal_input_message() -> Message: + """A normal (non-blocked) message to be scored.""" + return Message( + message_pieces=[ + MessagePiece( + role="assistant", + original_value="some response to score", + converted_value="some response to score", + original_value_data_type="text", + converted_value_data_type="text", + conversation_id="input-convo", + ) + ] + ) + + +@pytest.mark.usefixtures("patch_central_database") +class TestScorerResponseBlocked: + """Axis B: behavior when the scorer's own LLM response is content-filtered.""" + + async def test_raises_by_default(self): + from pyrit.exceptions import ScorerLLMResponseBlockedException + + scorer = _ForwarderTrueFalseScorer(chat_target=_make_scorer_blocking_target()) + + with pytest.raises(ScorerLLMResponseBlockedException, match="blocked by content filtering"): + await scorer.score_async(_make_normal_input_message()) + + async def test_returns_false_when_flag_disabled(self): + target = _make_scorer_blocking_target() + scorer = _ForwarderTrueFalseScorer(chat_target=target) + scorer.raise_if_scorer_blocks = False + + scores = await scorer.score_async(_make_normal_input_message()) + + assert len(scores) == 1 + assert scores[0].score_value == "false" + assert "blocked by content filtering" in scores[0].score_rationale + # Blocked is terminal: no retry storm. + assert target.send_prompt_async.call_count == 1 + + async def test_returns_zero_for_float_scale_when_flag_disabled(self): + scorer = _ForwarderFloatScaleScorer(chat_target=_make_scorer_blocking_target()) + scorer.raise_if_scorer_blocks = False + + scores = await scorer.score_async(_make_normal_input_message()) + + assert len(scores) == 1 + assert scores[0].score_value == "0.0" + assert "blocked by content filtering" in scores[0].score_rationale + + async def test_direct_transport_caller_raises_by_default(self): + from pyrit.exceptions import ScorerLLMResponseBlockedException + + scorer = _DirectTransportTrueFalseScorer(chat_target=_make_scorer_blocking_target()) + + with pytest.raises(ScorerLLMResponseBlockedException, match="blocked by content filtering"): + await scorer.score_async(_make_normal_input_message()) + + async def test_direct_transport_caller_returns_false_when_flag_disabled(self): + scorer = _DirectTransportTrueFalseScorer(chat_target=_make_scorer_blocking_target()) + scorer.raise_if_scorer_blocks = False + + scores = await scorer.score_async(_make_normal_input_message()) + + assert len(scores) == 1 + assert scores[0].score_value == "false" + assert "blocked by content filtering" in scores[0].score_rationale + # ── Helpers for score_blocked_content tests ────────────────────────────────── From 24ec7a4efcb149eeee97fb4370814675176270ed Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Fri, 10 Jul 2026 14:20:53 -0700 Subject: [PATCH 6/8] Align scorer-blocked terminology and decouple transport docstring from policy Use 'scorer's LLM response' consistently in llm_scoring.py and drop the Scorer.raise_if_scorer_blocks reference from the transport docstring/comment so the transport doesn't name the Scorer's policy attribute. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyrit/score/llm_scoring.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyrit/score/llm_scoring.py b/pyrit/score/llm_scoring.py index 8d4d1c50c7..12aa37b8ef 100644 --- a/pyrit/score/llm_scoring.py +++ b/pyrit/score/llm_scoring.py @@ -75,9 +75,9 @@ async def _run_llm_scoring_async( normalized and validated by the caller. Raises: - ScorerLLMResponseBlockedException: If the scoring target's response is blocked by - content filtering. The calling ``Scorer`` decides whether to raise or return a - default score (see ``Scorer.raise_if_scorer_blocks``). + ScorerLLMResponseBlockedException: If the scorer's LLM response is blocked by + content filtering. The transport only surfaces the condition; the calling + ``Scorer`` owns the policy for whether to raise or return a default score. InvalidJsonException: If the response is not valid JSON, is missing required keys, or fails the handler's value validation. Exception: For other unexpected errors during scoring. @@ -134,8 +134,8 @@ async def _run_llm_scoring_async( # A content-filter block yields a single error piece with no parseable text piece. # Surfacing this as a dedicated exception keeps the transport free of policy: the - # Scorer decides whether to raise or fall back to a default score - # (see Scorer.raise_if_scorer_blocks). Not retried by @pyrit_json_retry. + # calling Scorer decides whether to raise or fall back to a default score. + # Not retried by @pyrit_json_retry. if all(piece.is_blocked() for piece in response[0].message_pieces): raise ScorerLLMResponseBlockedException( message=( From 2ad33ccb1956e4daf55a64f06035713c8536b9ea Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Fri, 10 Jul 2026 14:32:43 -0700 Subject: [PATCH 7/8] Guard scorer scoring on missing text piece, not all-pieces-blocked The crash this PR fixes is a StopIteration when no text piece exists in the scorer's response. The previous guard keyed on 'all pieces blocked', which left two gaps: an empty piece list spuriously raised (all() over empty is True), and a non-blocked response with no text piece fell through and crashed again. Resolve the text piece safely and branch on why it's missing: a content-filter block raises ScorerLLMResponseBlockedException (routed through the Scorer's raise_if_scorer_blocks policy), while any other no-text response raises EmptyResponseException so a genuinely empty/malformed response surfaces as a real error instead of being silently converted to a default score. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyrit/score/llm_scoring.py | 37 +++++++++++++++++++++------------ tests/unit/score/test_scorer.py | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/pyrit/score/llm_scoring.py b/pyrit/score/llm_scoring.py index 12aa37b8ef..1575d2edd9 100644 --- a/pyrit/score/llm_scoring.py +++ b/pyrit/score/llm_scoring.py @@ -6,7 +6,7 @@ import uuid from typing import TYPE_CHECKING, Any -from pyrit.exceptions import ScorerLLMResponseBlockedException, pyrit_json_retry +from pyrit.exceptions import EmptyResponseException, ScorerLLMResponseBlockedException, pyrit_json_retry from pyrit.models import JSON_SCHEMA_METADATA_KEY, Message, MessagePiece if TYPE_CHECKING: @@ -78,6 +78,8 @@ async def _run_llm_scoring_async( ScorerLLMResponseBlockedException: If the scorer's LLM response is blocked by content filtering. The transport only surfaces the condition; the calling ``Scorer`` owns the policy for whether to raise or return a default score. + EmptyResponseException: If the scorer's LLM response contains no text piece to parse + and was not blocked (e.g. an empty or malformed response). InvalidJsonException: If the response is not valid JSON, is missing required keys, or fails the handler's value validation. Exception: For other unexpected errors during scoring. @@ -132,22 +134,31 @@ async def _run_llm_scoring_async( except Exception as ex: raise Exception(f"Error scoring prompt with original prompt ID: {scored_prompt_id}") from ex - # A content-filter block yields a single error piece with no parseable text piece. - # Surfacing this as a dedicated exception keeps the transport free of policy: the - # calling Scorer decides whether to raise or fall back to a default score. - # Not retried by @pyrit_json_retry. - if all(piece.is_blocked() for piece in response[0].message_pieces): - raise ScorerLLMResponseBlockedException( + # Resolve the text piece that holds the JSON response (score_value + rationale). When it's + # absent the scorer produced no parseable output. Guard on the actual failure (no text + # piece) rather than "all pieces blocked" so every no-text shape is handled instead of + # only the fully-blocked one. A content-filter block surfaces as a dedicated exception + # (the calling Scorer owns whether to raise or fall back); any other no-text response is a + # genuine empty/malformed error. Neither is retried by @pyrit_json_retry. + text_piece = next( + (piece for piece in response[0].message_pieces if piece.converted_value_data_type == "text"), None + ) + if text_piece is None: + if any(piece.is_blocked() for piece in response[0].message_pieces): + raise ScorerLLMResponseBlockedException( + message=( + f"The scorer's LLM response was blocked by content filtering while scoring " + f"prompt ID: {scored_prompt_id}. Consider using a scorer endpoint with " + f"content filtering disabled for red-teaming workflows." + ) + ) + raise EmptyResponseException( message=( - f"The scorer's LLM response was blocked by content filtering while scoring " - f"prompt ID: {scored_prompt_id}. Consider using a scorer endpoint with " - f"content filtering disabled for red-teaming workflows." + f"The scorer's LLM response contained no text to parse while scoring " + f"prompt ID: {scored_prompt_id}." ) ) - # Get the text piece which contains the JSON response containing the score_value and rationale from the LLM - text_piece = next(piece for piece in response[0].message_pieces if piece.converted_value_data_type == "text") - return response_handler.parse( response_text=text_piece.converted_value, scorer_identifier=scorer_identifier, diff --git a/tests/unit/score/test_scorer.py b/tests/unit/score/test_scorer.py index 19a7992b80..2acde6de7c 100644 --- a/tests/unit/score/test_scorer.py +++ b/tests/unit/score/test_scorer.py @@ -1589,6 +1589,42 @@ async def test_score_value_with_llm_raises_when_scorer_response_blocked(): assert chat_target.send_prompt_async.call_count == 1 +async def test_score_value_with_llm_raises_empty_response_when_no_text_piece(): + """A no-text response that wasn't content-filtered raises EmptyResponseException, not blocked.""" + from pyrit.exceptions import EmptyResponseException + + chat_target = MagicMock(PromptTarget) + chat_target.get_identifier.return_value = get_mock_target_identifier("MockChatTarget") + + # An error piece that is NOT flagged as blocked (e.g. a flaky/empty response) and no text piece. + non_text_piece = MessagePiece( + role="assistant", + original_value="", + original_value_data_type="error", + converted_value="", + converted_value_data_type="error", + conversation_id="test-convo", + response_error="unknown", + ) + chat_target.send_prompt_async = AsyncMock(return_value=[Message(message_pieces=[non_text_piece])]) + + scorer = MockScorer() + + with pytest.raises(EmptyResponseException, match="no text to parse"): + await scorer._score_value_with_llm_async( + prompt_target=chat_target, + system_prompt="system_prompt", + message_value="message_value", + message_data_type="text", + scored_prompt_id="test-prompt-id", + category="category", + objective="task", + ) + + # No parseable text is terminal here, not a transient JSON error: it must not retry. + assert chat_target.send_prompt_async.call_count == 1 + + # ── Axis B: the scorer's own LLM response is blocked (raise_if_scorer_blocks) ───────────── From eadc65011c00f672f8558828d7d6d355fe3d1bdc Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Fri, 10 Jul 2026 14:52:59 -0700 Subject: [PATCH 8/8] Apply ruff format after merge Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyrit/score/llm_scoring.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyrit/score/llm_scoring.py b/pyrit/score/llm_scoring.py index 1575d2edd9..78183f0a74 100644 --- a/pyrit/score/llm_scoring.py +++ b/pyrit/score/llm_scoring.py @@ -154,8 +154,7 @@ async def _run_llm_scoring_async( ) raise EmptyResponseException( message=( - f"The scorer's LLM response contained no text to parse while scoring " - f"prompt ID: {scored_prompt_id}." + f"The scorer's LLM response contained no text to parse while scoring prompt ID: {scored_prompt_id}." ) )