FIX: warn instead of raising on reasoning-model token truncation (Chat and Responses targets)

pyrit/prompt_target/common/chat_completions_response_parser.py

@@ -16,6 +16,8 @@
 from collections.abc import Mapping
 from typing import Any
 
+from openai.types.chat import ChatCompletion
+
 from pyrit.exceptions import (
     EmptyResponseException,
     PyritException,
@@ -37,6 +39,21 @@
 DEFAULT_VALID_FINISH_REASONS: frozenset[str] = frozenset({"stop", "length", "content_filter", "tool_calls"})
 
 
+def get_finish_reason(*, response: ChatCompletion) -> str | None:
+    """
+    Extract the first choice's ``finish_reason`` from a Chat Completions response.
+
+    Args:
+        response (ChatCompletion): The Chat Completions response object.
+
+    Returns:
+        str | None: The first choice's ``finish_reason``, or None when there are no choices.
+    """
+    if not response.choices:
+        return None
+    return response.choices[0].finish_reason
+
+
 def detect_response_content(message: Any) -> tuple[bool, bool, bool]:
     """
     Detect which content types are present in a Chat Completions ``message``.

pyrit/prompt_target/common/utils.py

@@ -6,6 +6,7 @@
 from typing import Any
 
 from pyrit.exceptions import PyritException
+from pyrit.models import Message, MessagePiece, construct_response_from_request
 
 
 def validate_temperature(temperature: float | None) -> None:
@@ -57,3 +58,27 @@ async def set_max_rpm_async(*args: Any, **kwargs: Any) -> Any:
         return await func(*args, **kwargs)
 
     return set_max_rpm_async
+
+
+def build_empty_truncated_response(*, request: MessagePiece) -> Message:
+    """
+    Build a graceful empty response for a token-limit-truncated model response.
+
+    A response truncated at the token limit (Chat Completions ``finish_reason == "length"`` or the
+    Responses API ``status == "incomplete"`` with ``reason == "max_output_tokens"``) may legitimately
+    contain no visible content. Callers gate this on their own truncation check (for example a
+    target's ``_is_truncated_response``); returning an empty ``error="empty"`` text response lets the
+    run continue instead of raising.
+
+    Args:
+        request (MessagePiece): The originating request piece.
+
+    Returns:
+        Message: An empty text response marked with ``error="empty"``.
+    """
+    return construct_response_from_request(
+        request=request,
+        response_text_pieces=[""],
+        response_type="text",
+        error="empty",
+    )

pyrit/prompt_target/openai/openai_chat_target.py

@@ -5,6 +5,8 @@
 from collections.abc import MutableSequence
 from typing import Any
 
+from openai.types.chat import ChatCompletion
+
 from pyrit.common import forward_init_parameters
 from pyrit.exceptions import (
     EmptyResponseException,
@@ -28,13 +30,15 @@
     capture_token_usage,
     detect_response_content,
     extract_partial_content,
+    get_finish_reason,
     is_content_filter_response,
     save_audio_response_async,
     validate_chat_completion_response,
 )
 from pyrit.prompt_target.common.target_capabilities import TargetCapabilities
 from pyrit.prompt_target.common.target_configuration import TargetConfiguration
 from pyrit.prompt_target.common.utils import (
+    build_empty_truncated_response,
     limit_requests_per_minute,
     validate_temperature,
     validate_top_p,
@@ -271,7 +275,7 @@ def _extract_partial_content(self, response: Any) -> str | None:
         """
         return extract_partial_content(response)
 
-    def _validate_response(self, response: Any, request: MessagePiece) -> Message | None:
+    def _validate_response(self, response: ChatCompletion, request: MessagePiece) -> None:
         """
         Validate a Chat Completions API response for errors.
 
@@ -280,19 +284,53 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message |
         - Invalid finish_reason
         - At least one valid response type (text content, audio, or tool_calls)
 
+        A ``finish_reason == "length"`` (token-limit truncation) response is treated as valid, with a
+        warning, so that ``_construct_message_from_response_async`` can preserve any partial content
+        or fall back to a graceful empty response. Genuinely empty responses (no truncation) are
+        raised so the retry logic can attempt to get a complete response. Content filter responses
+        are handled separately by ``_check_content_filter``.
+
         Args:
             response: The ChatCompletion response from OpenAI SDK.
             request: The original request MessagePiece.
 
-        Returns:
-            None if valid, does not return Message for content filter (handled by _check_content_filter).
-
         Raises:
             PyritException: For unexpected response structures or finish reasons.
-            EmptyResponseException: When the API returns an empty response.
+            EmptyResponseException: When the API returns an empty response that was not caused by
+                token-limit truncation.
         """
+        # Token-limit truncation is handled before the shared validator, which would otherwise raise
+        # EmptyResponseException on a validly truncated but empty response. Reasoning models can spend
+        # the whole budget on hidden reasoning before emitting a visible answer, and a low limit may be
+        # deliberate, so warn instead of raising and let construction preserve any partial content or
+        # fall back to a graceful empty response.
+        if self._is_truncated_response(response):
+            logger.warning(
+                "The response was truncated because it reached the token limit (finish_reason='length'). "
+                "Reasoning models consume tokens on hidden reasoning in addition to the visible answer, so a "
+                "low max_completion_tokens can truncate or empty the response. Increase max_completion_tokens "
+                "if you expected complete content."
+            )
+            return
+
+        # Genuinely empty responses (no truncation) raise so the retry logic can attempt to get a
+        # complete response.
         validate_chat_completion_response(response=response)
-        return None
+
+    def _is_truncated_response(self, response: ChatCompletion) -> bool:
+        """
+        Return True if the response was cut off by the token limit.
+
+        The Chat Completions API signals token-limit truncation via ``finish_reason == "length"``
+        on the first choice.
+
+        Args:
+            response: A ChatCompletion response from the OpenAI SDK.
+
+        Returns:
+            bool: True if the response was truncated at the token limit, False otherwise.
+        """
+        return get_finish_reason(response=response) == "length"
 
     def _detect_response_content(self, message: Any) -> tuple[bool, bool, bool]:
         """
@@ -337,7 +375,7 @@ def _should_skip_sending_audio(
             prefer_transcript_for_history=prefer_transcript_for_history,
         )
 
-    async def _construct_message_from_response_async(self, response: Any, request: MessagePiece) -> Message:
+    async def _construct_message_from_response_async(self, response: ChatCompletion, request: MessagePiece) -> Message:
         """
         Construct a Message from a ChatCompletion response.
 
@@ -354,16 +392,30 @@ async def _construct_message_from_response_async(self, response: Any, request: M
             Message: Constructed message with one or more MessagePiece entries.
 
         Raises:
-            EmptyResponseException: If the response contains no content, audio, or tool calls.
+            EmptyResponseException: If a non-truncated response contains no content, audio, or tool
+                calls. A truncated (``finish_reason == "length"``) response with no content instead
+                yields a graceful empty piece so the run continues. Truncated responses set
+                ``prompt_metadata["truncated"] = True`` on the first piece.
         """
         audio_format = self._audio_response_config.audio_format if self._audio_response_config else "wav"
+        truncated = self._is_truncated_response(response)
         pieces = await build_response_pieces_async(response=response, request=request, audio_format=audio_format)
 
         if not pieces:
+            # A truncated (finish_reason == "length") response may legitimately produce no content;
+            # return a graceful empty piece so the run continues. Validation already raised for
+            # genuinely empty (non-truncated) responses.
+            if truncated:
+                empty_message = build_empty_truncated_response(request=request)
+                capture_token_usage(pieces=empty_message.message_pieces, response=response)
+                empty_message.message_pieces[0].prompt_metadata["truncated"] = True
+                return empty_message
             raise EmptyResponseException(message="Failed to extract any response content.")
 
         # Capture token usage from the API response and store in the first piece's metadata
         capture_token_usage(pieces=pieces, response=response)
+        if truncated:
+            pieces[0].prompt_metadata["truncated"] = True
 
         return Message(message_pieces=pieces)