diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 85c232ad3f4..5e16563ae26 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -931,7 +931,14 @@ def _map_openai_like_exception( ) -_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)") +_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS: Final = ( + re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)"), + re.compile(r"Input length \((\d+)\) exceeds model\\?'s maximum context length \((\d+)\)"), + re.compile(r"The engine prompt length (\d+) exceeds the max_model_len (\d+)"), +) +_BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN: Final = re.compile( + r"maximum context length is (\d+) tokens\. However, you requested (\d+) output tokens" +) _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN: Final = re.compile( r"""\\?["']code\\?["']\s*:\s*\\?["']context_length_exceeded\\?["']""" ) @@ -960,16 +967,26 @@ def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> b def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: """ - Mantle reports context overflow as a validation_error carrying the token counts, or as - OpenAI's context_length_exceeded code, either in a 400 body or in a streamed error event - with no error type. Clients such as Claude Code only treat "prompt is too long" as - overflow (litellm/litellm#36546). + Mantle reports context overflow as a validation_error carrying the token counts (worded + differently for gpt-oss, and again on its Responses API), as OpenAI's context_length_exceeded + code, or in a wording the other providers share, either in a 400 body or in a streamed error + event with no error type. Clients such as Claude Code only treat "prompt is too long" as overflow + (litellm/litellm#36546). qwen's wording caps its input count at the window minus the requested + output, so only the output budget says whether a shorter prompt could fit at all; when it cannot, + the overflow keeps Mantle's own message so the client is not told to shorten the prompt. """ - match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str) + matches: Final = (pattern.search(error_str) for pattern in _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERNS) + match: Final = next((found for found in matches if found is not None), None) if match is not None: prompt_tokens, max_tokens = match.groups() return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum" - if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search(error_str) is not None: + output_budget: Final = _BEDROCK_MANTLE_OUTPUT_BUDGET_PATTERN.search(error_str) + if output_budget is not None: + context_window, requested_output_tokens = (int(group) for group in output_budget.groups()) + return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE if requested_output_tokens < context_window else error_str + if _BEDROCK_MANTLE_CONTEXT_LENGTH_EXCEEDED_CODE_PATTERN.search( + error_str + ) is not None or ExceptionCheckers.is_error_str_context_window_exceeded(error_str): return _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE return None diff --git a/litellm/router.py b/litellm/router.py index 535072d5fcf..67b7400f411 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -680,6 +680,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_FALLBACK_CHAIN_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str]], ...]) _DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter( Mapping[str, object] | GenericLiteLLMParams ) @@ -860,6 +861,25 @@ def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error return has_generated_content or not error.is_pre_first_chunk +def _anthropic_stream_error_skips_recovery( + error: Exception, has_generated_content: bool, holds_for_overflow_only: bool +) -> bool: + """ + A held stream whose only possible recovery is a context window fallback lets every other failure through + with its held frames, as it would have reached the client with no hold. + """ + from litellm.exceptions import MidStreamFallbackError + + if isinstance(error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( + has_generated_content, error + ): + return True + if not holds_for_overflow_only: + return False + underlying: Final = error.original_exception if isinstance(error, MidStreamFallbackError) else error + return not isinstance(underlying, litellm.ContextWindowExceededError) + + def _stream_raised_error_status(error: Exception) -> int | None: raw_status: Final = getattr(error, "status_code", None) if isinstance(raw_status, int): @@ -873,9 +893,15 @@ def _stream_raised_error_status(error: Exception) -> int | None: def _stream_fallback_error_for_raised( error: Exception, model: str, has_generated_content: bool, llm_provider: str = "anthropic" ) -> "MidStreamFallbackError | None": - """The pre-stream retry rule (408, 409, 429, 5xx); None means the raise propagates unchanged.""" + """ + The pre-stream retry rule (408, 409, 429, 5xx), plus a context window overflow, which the fallback chain + sends to context_window_fallbacks the way it does before a stream opens; None means the raise propagates + unchanged. + """ if has_generated_content: return None + if isinstance(error, litellm.ContextWindowExceededError): + return _stream_pre_content_error(error, model, llm_provider) status_code: Final = _stream_raised_error_status(error) if status_code is None: return _stream_pre_content_error(error, model, llm_provider) @@ -1848,7 +1874,9 @@ class Router(DeprecatedRouterMembers): else: self._core_fallbacks = [{"*": _fallbacks}] - self._core_context_window_fallbacks = context_window_fallbacks or litellm.context_window_fallbacks + self._core_context_window_fallbacks: list[dict[str, list[str]]] | None = ( # mutable-ok: resolver contract + context_window_fallbacks or litellm.context_window_fallbacks + ) _content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks self.validate_fallbacks(fallback_param=_content_policy_fallbacks) @@ -6927,9 +6955,14 @@ class Router(DeprecatedRouterMembers): # to take over there is nothing to buffer for, so every frame, # including pings and provider error frames, is forwarded live. model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group + can_retry_or_fall_back: Final = self._anthropic_messages_stream_can_retry( + initial_kwargs + ) or self._stream_can_fall_back(model, initial_kwargs) + holds_for_overflow_only: Final = not can_retry_or_fall_back and self._stream_can_fall_back_on_overflow( + model, initial_kwargs + ) has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over - self._anthropic_messages_stream_can_retry(initial_kwargs) - or self._stream_can_fall_back(model, initial_kwargs) + can_retry_or_fall_back or holds_for_overflow_only ) buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline try: @@ -6991,6 +7024,7 @@ class Router(DeprecatedRouterMembers): async for item in self._aanthropic_messages_recover_stream_error( stream_error, has_generated_content, + holds_for_overflow_only, buffered_lifecycle_chunks, model, initial_kwargs, @@ -7011,6 +7045,7 @@ class Router(DeprecatedRouterMembers): self, stream_error: Exception, has_generated_content: bool, + holds_for_overflow_only: bool, buffered_lifecycle_chunks: tuple[bytes, ...], model: str, initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it @@ -7020,12 +7055,10 @@ class Router(DeprecatedRouterMembers): """Turns a source-iterator failure into a fallback attempt or the error reaching the caller.""" from litellm.exceptions import MidStreamFallbackError - if isinstance(stream_error, MidStreamFallbackError) and _anthropic_stream_should_decline_fallback( - has_generated_content, stream_error - ): + if _anthropic_stream_error_skips_recovery(stream_error, has_generated_content, holds_for_overflow_only): for buffered_chunk in buffered_lifecycle_chunks: yield buffered_chunk - if stream_error.original_exception is not None: + if isinstance(stream_error, MidStreamFallbackError) and stream_error.original_exception is not None: raise stream_error.original_exception from stream_error raise stream_error fallback_error: Final = ( @@ -7171,7 +7204,7 @@ class Router(DeprecatedRouterMembers): context_window_fallbacks=_request_fallback_list( kwargs, "context_window_fallbacks", - cast( # cast-ok: untyped router attribute + cast( # cast-ok: list is invariant and the helper takes list[object] "list[object] | None", self._core_context_window_fallbacks ), ), @@ -10375,6 +10408,27 @@ class Router(DeprecatedRouterMembers): ) ) + def _stream_can_fall_back_on_overflow(self, model_group: str, kwargs: Mapping[str, object]) -> bool: + """Whether a context window overflow before content could still reach a fallback, which context_window_fallbacks + can serve as well as every route _stream_can_fall_back counts. A chain in a shape other than group-keyed lists + errs toward True, as _stream_can_fall_back does for non-standard fallbacks.""" + if fallbacks_disabled_for_request(kwargs): + return False + if self._stream_can_fall_back(model_group, kwargs): + return True + configured: Final = kwargs.get("context_window_fallbacks", self._core_context_window_fallbacks) + if not configured: + return False + try: + chain: Final = _FALLBACK_CHAIN_ADAPTER.validate_python(configured) + except ValidationError: + return True + resolved, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=chain, + lookup_groups=fallback_lookup_groups(kwargs, model_group), + ) + return has_unattempted_fallback_target(resolved, kwargs) + def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: """ Whether async_function_with_fallbacks_common_utils could still route a diff --git a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py index 23b75a5a8cf..3053c52f740 100644 --- a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py @@ -409,14 +409,11 @@ def test_context_window_fallback_fires_for_the_overflow_on_streamed_chat_complet _assert_fell_back(cwf_rig, "overflow", prompt, text) -def test_context_window_fallback_is_not_consulted_on_streamed_messages(cwf_rig: Rig) -> None: +def test_context_window_fallback_fires_for_the_overflow_on_streamed_messages(cwf_rig: Rig) -> None: prompt: Final = _prompt("cwf messages stream") response, text = _consumed(cwf_rig.gateway, "/v1/messages", _messages(cwf_rig.models["cwf"], prompt, True)) assert response.status_code == 200, text - error: Final = _only_error(text) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(cwf_rig, "overflow", prompt) + _assert_fell_back(cwf_rig, "overflow", prompt, text) def test_context_window_fallback_fires_on_streamed_responses(cwf_rig: Rig) -> None: @@ -454,16 +451,11 @@ def test_plain_fallback_fires_for_the_overflow_on_streamed_chat_completions(fb_r _assert_fell_back(fb_rig, "overflow", prompt, text) -def test_streamed_messages_overflow_on_a_plain_fallback_deployment_returns_400_without_falling_back( - fb_rig: Rig, -) -> None: +def test_plain_fallback_fires_for_the_overflow_on_streamed_messages(fb_rig: Rig) -> None: prompt: Final = _prompt("fb messages stream") response, text = _consumed(fb_rig.gateway, "/v1/messages", _messages(fb_rig.models["fb"], prompt, True)) - assert response.status_code == 400, text - error: Final = object_value(_JSON_OBJECT.validate_json(text)["error"]) - assert error["type"] == "invalid_request_error", text - assert _GENERIC in str(error["message"]), text - _assert_no_fallback(fb_rig, "overflow", prompt) + assert response.status_code == 200, text + _assert_fell_back(fb_rig, "overflow", prompt, text) def test_plain_fallback_fires_on_streamed_responses(fb_rig: Rig) -> None: diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 2b1e4a5eed3..be4927cf26f 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import httpx import openai import pytest +from _pytest.mark.structures import ParameterSet from fastapi import HTTPException from openai import AsyncOpenAI, OpenAI @@ -1754,6 +1755,145 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): assert "prompt is too long: 1055489 tokens > 1050000 maximum" in excinfo.value.message +_GPT_OSS_OVERFLOW_EVENT: Final = ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), ' + 'message: "Input length (360068) exceeds model\'s maximum context length (131072).", param: None } }' +) + + +@pytest.mark.parametrize( + "original_exception", + [ + pytest.param( + BaseLLMException( + status_code=400, + message=json.dumps( + { + "error": { + "code": "validation_error", + "message": _GPT_OSS_OVERFLOW_EVENT, + "param": None, + "type": "invalid_request_error", + } + } + ), + ), + id="400-body", + ), + pytest.param(OpenAIError(status_code=503, message=_GPT_OSS_OVERFLOW_EVENT), id="streamed-event"), + pytest.param( + BaseLLMException( + status_code=400, message=f"Error code: 400 - { {'error': {'message': _GPT_OSS_OVERFLOW_EVENT}}!r}" + ), + id="python-repr", + ), + ], +) +def test_bedrock_mantle_gpt_oss_input_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="openai.gpt-oss-20b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.response.status_code == 400 + assert "prompt is too long: 360068 tokens > 131072 maximum" in excinfo.value.message + + +def test_bedrock_mantle_gpt_oss_streamed_responses_overflow_event_maps_to_context_window_exceeded() -> None: + from litellm.responses.streaming_iterator import map_stream_error_to_exception + + mapped_exception: Final = map_stream_error_to_exception( + { + "code": "invalid_prompt", + "message": ( + 'ErrorEvent { error: APIError { type: "invalid_request_error", code: Some(400), message: ' + '"The engine prompt length 165072 exceeds the max_model_len 131072. Please reduce prompt.", ' + 'param: Some("input") } }' + ), + }, + model="openai.gpt-oss-20b", + custom_llm_provider="bedrock_mantle", + ) + + assert isinstance(mapped_exception, litellm.ContextWindowExceededError) + assert mapped_exception.status_code == 400 + assert "prompt is too long: 165072 tokens > 131072 maximum" in mapped_exception.message + + +def _qwen_overflow_event(requested_output_tokens: int) -> str: + counted_input_tokens: Final = 32768 - requested_output_tokens + 1 + return ( + 'ErrorEvent { error: APIError { type: "BadRequestError", code: Some(400), message: "This model\'s maximum ' + f"context length is 32768 tokens. However, you requested {requested_output_tokens} output tokens and your " + f"prompt contains at least {counted_input_tokens} input tokens, for a total of at least 32769 tokens. Please " + "reduce the length of the input prompt or the number of requested output tokens. " + f'(parameter=input_tokens, value={counted_input_tokens})", param: None }} }}' + ) + + +def _mantle_overflow_serializations(event: str) -> tuple[ParameterSet, ...]: + return ( + pytest.param( + BaseLLMException( + status_code=400, + message=json.dumps( + { + "error": { + "code": "validation_error", + "message": event, + "param": None, + "type": "invalid_request_error", + } + } + ), + ), + id="400-body", + ), + pytest.param(OpenAIError(status_code=503, message=event), id="streamed-event"), + pytest.param( + BaseLLMException(status_code=400, message=f"Error code: 400 - { {'error': {'message': event}}!r}"), + id="python-repr", + ), + ) + + +@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(16))) +def test_bedrock_mantle_qwen_maximum_context_length_overflow_maps_to_context_window_exceeded( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.response.status_code == 400 + assert "prompt is too long" in excinfo.value.message + + +@pytest.mark.parametrize("original_exception", _mantle_overflow_serializations(_qwen_overflow_event(32768))) +def test_bedrock_mantle_qwen_output_budget_filling_the_window_keeps_mantles_message( + original_exception: BaseLLMException, +) -> None: + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="qwen.qwen3-32b", + original_exception=original_exception, + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.status_code == 400 + assert "prompt is too long" not in excinfo.value.message + assert "you requested 32768 output tokens" in excinfo.value.message + + def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 7ac27e22a93..147880ee6b8 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -19265,6 +19265,105 @@ async def test_anthropic_messages_retry_raising_a_context_window_error_takes_the assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" +def _anthropic_messages_overflow_before_content() -> _AnthropicMessagesRaisingByteStream: + return _AnthropicMessagesRaisingByteStream( + [_anthropic_messages_message_start_chunk()], + litellm.ContextWindowExceededError( + message="prompt is too long: 9 tokens > 8 maximum", llm_provider="bedrock_mantle", model="glm" + ), + ) + + +@pytest.mark.asyncio +async def test_anthropic_messages_stream_overflow_before_content_takes_the_context_window_fallback() -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + _anthropic_messages_overflow_before_content, + lambda: _AnthropicMessagesFakeByteStream( + [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + ), + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + body: Final = [chunk async for chunk in stream] + + assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("from cw")] + assert [model for model, _, _ in provider.calls][-1] == "anthropic/cw-model" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs"), + [ + pytest.param(None, {}, id="no-fallback-configured"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, id="fallbacks-disabled-for-the-request"), + ], +) +async def test_anthropic_messages_stream_overflow_surfaces_when_no_fallback_can_take_over( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + provider: Final = _AnthropicMessagesScriptedProvider(_anthropic_messages_overflow_before_content) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider, **request_kwargs) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], litellm.ContextWindowExceededError) + assert len(provider.calls) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("raised", "expected_error"), + [ + pytest.param( + litellm.ServiceUnavailableError(message="unavailable", llm_provider="bedrock_mantle", model="glm"), + litellm.ServiceUnavailableError, + id="raised-503", + ), + pytest.param( + litellm.BadRequestError(message="bad request", llm_provider="bedrock_mantle", model="glm"), + litellm.BadRequestError, + id="raised-400", + ), + pytest.param(_anthropic_messages_transport_drop(), litellm.APIConnectionError, id="transport-drop"), + ], +) +async def test_anthropic_messages_stream_context_window_only_group_forwards_other_errors_after_message_start( + raised: Exception, expected_error: type[Exception] +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=[{"glm": ["cw"]}]) + provider: Final = _AnthropicMessagesScriptedProvider( + lambda: _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised) + ) + + stream: Final = await _anthropic_messages_stream_through_router(router, provider) + outcome: Final = [chunk async for chunk in _events_until_error(stream)] + + assert outcome[:-1] == [_anthropic_messages_message_start_chunk()] + assert isinstance(outcome[-1], expected_error) + assert len(provider.calls) == 1 + + +@pytest.mark.parametrize( + ("context_window_fallbacks", "request_kwargs", "expected"), + [ + pytest.param([{"glm": ["cw"]}], {}, True, id="router-context-window-fallback"), + pytest.param(None, {"context_window_fallbacks": [{"glm": ["cw"]}]}, True, id="request-context-window-fallback"), + pytest.param(None, {}, False, id="no-fallback-configured"), + pytest.param([{"glm": ["cw"]}], {"disable_fallbacks": True}, False, id="fallbacks-disabled-for-the-request"), + pytest.param(None, {"context_window_fallbacks": ["cw"]}, True, id="request-chain-in-a-non-standard-shape"), + ], +) +def test_stream_can_fall_back_on_overflow_counts_context_window_fallbacks( + context_window_fallbacks: list[dict[str, list[str]]] | None, request_kwargs: dict[str, object], expected: bool +) -> None: + router: Final = _anthropic_messages_retry_router(num_retries=0, context_window_fallbacks=context_window_fallbacks) + + assert router._stream_can_fall_back_on_overflow("glm", {"model": "glm", **request_kwargs}) is expected + + def test_anthropic_messages_retry_budget_precedence_direct_call(): """A retry policy naming the error class outranks the request's num_retries, which outranks the routed deployment's, which outranks the router's; num_retries=0 on the request turns a policy off too.""" @@ -20144,7 +20243,14 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_ with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt: recovery = router._aanthropic_messages_recover_stream_error( - declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + declined, + True, + False, + buffered, + "primary", + {"model": "primary"}, + _anthropic_messages_make_wrapper(), + AsyncMock(), ) with pytest.raises(BedrockError) as exc_info: await drain(recovery) @@ -20165,7 +20271,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt): recovery = router._aanthropic_messages_recover_stream_error( - raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() + raised, False, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() ) collected = [chunk async for chunk in recovery] assert collected == [b"fallback"]