From f9116cc3f1a689c3a3fcd1d3fcd7b029beddc5cd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:27:37 -0700 Subject: [PATCH] fix(router): guard SDK text_format, narrow forced tool_choice, read Claude base_model, and cap Responses continuation output Cut Responses streams that asked for structured output through the SDK's text_format now surface the primary's error instead of stitching two models' JSON. Only a required or named tool_choice counts as forced, so allowed_tools in auto mode and hosted tool choices still continue. A Claude deployment named only through base_model keeps its prefill continuation, and the continuation's max_output_tokens is lowered by what the cut stream already emitted. --- litellm/router.py | 20 ++++++- .../continuation_prefill_check.py | 32 ++++++++---- .../router_code_coverage.py | 1 + .../base_llm/responses/test_transformation.py | 52 ++++++++++++++++++- .../test_continuation_prefill_check.py | 25 ++++++++- 5 files changed, 115 insertions(+), 15 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 6b36ff824b5..40b555cffbd 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -454,6 +454,7 @@ from litellm.utils import ( provider_rejectable_params, resolve_api_key_secret_reference, set_live_deployment_replay, + token_counter, ) from .router_utils.pattern_match_deployments import PatternMatchRouter @@ -3719,7 +3720,8 @@ class Router: ) -> Mapping[str, object]: """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies.""" deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} - defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None} + default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self.default_litellm_params) + defaults: Final = {key: value for key, value in default_params.items() if value is not None} deployment_tool_choice: Final = ( {"tool_choice": deployment_params["tool_choice"]} if deployment_params.get("tool_choice") is not None @@ -4034,6 +4036,18 @@ class Router: return earlier.extended(e.generated_content) return ResponsesContinuation(request_kwargs.get("input"), e.generated_content) + @staticmethod + def _responses_continuation_ceilings( + e: "MidStreamFallbackError", request_kwargs: Mapping[str, object] + ) -> Mapping[str, object]: + """The caller's output ceilings less what the cut stream already emitted, raising the primary's error once + nothing is left.""" + emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content) + reduced: Final = continuation_output_ceilings(request_kwargs, emitted_tokens) + if reduced is None: + Router._raise_original_mid_stream_error(e) + return reduced + async def _aresponses_fallback_attempt( self, e: "MidStreamFallbackError", @@ -4053,6 +4067,9 @@ class Router: from litellm.exceptions import MidStreamFallbackError continuation: Final = self._responses_continuation(e, source_iterator, initial_kwargs) + reduced_ceilings: Final[Mapping[str, object]] = ( + Router._responses_continuation_ceilings(e, initial_kwargs) if continuation is not None else {} + ) seam: Final = ResponsesContinuationSeam(continuation.emitted_text if continuation is not None else "") partial_usage: Final = Router._extract_partial_responses_usage(source_iterator) fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted @@ -4072,6 +4089,7 @@ class Router: self._ageneric_api_call_with_fallbacks_responses_attempt ) if continuation is not None: + initial_kwargs.update(reduced_ceilings) initial_kwargs[RESPONSES_CONTINUATION_KWARG] = continuation # rebind-ok: shaped per fallback deployment initial_kwargs["input"] = responses_continuation_input(continuation) # rebind-ok: fallback hop input # The Responses-API path stores observability metadata diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index 6811d9dc79d..fc360a9df7b 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -28,7 +28,8 @@ MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker() _STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) _UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"}) -_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens") +_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens") +_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"}) RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation" @@ -39,30 +40,39 @@ def _str_keyed(value: object) -> Mapping[str, object]: return _NO_FIELDS -def deployment_supports_prefill(deployment: object) -> bool: +def _underlying_model(deployment: object) -> str | None: deployment_map: Final = _str_keyed(deployment) model_info: Final = _str_keyed(deployment_map.get("model_info")) - declared: Final = model_info.get("supports_assistant_prefill") - if isinstance(declared, bool): - return declared litellm_params: Final = _str_keyed(deployment_map.get("litellm_params")) candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model")) - model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) + return next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) + + +def deployment_supports_prefill(deployment: object) -> bool: + declared: Final = _str_keyed(_str_keyed(deployment).get("model_info")).get("supports_assistant_prefill") + if isinstance(declared, bool): + return declared + model: Final = _underlying_model(deployment) return model is not None and supports_assistant_prefill(model=model) +def _forces_tool_call(tool_choice: object) -> bool: + choice: Final = _str_keyed(tool_choice) + mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode") + return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES + + def constrains_continuation(params: Mapping[str, object]) -> bool: sent: Final = {**params, **_str_keyed(params.get("extra_body"))} choice_count: Final = sent.get("n") response_format: Final = sent.get("response_format") text_format: Final = _str_keyed(sent.get("text")).get("format") - tool_choice: Final = sent.get("tool_choice") return ( (isinstance(choice_count, int) and choice_count > 1) or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT) or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT) - or tool_choice == "required" - or isinstance(tool_choice, Mapping) + or sent.get("text_format") is not None + or _forces_tool_call(sent.get("tool_choice")) or sent.get("merge_reasoning_content_in_choices") is True ) @@ -280,8 +290,8 @@ class ResponsesContinuation: def deployment_takes_responses_prefill(deployment: object) -> bool: """Claude takes a trailing assistant turn as prefill with no `prefix` flag, which a Responses input item cannot carry.""" - model: Final = _str_keyed(_str_keyed(deployment).get("litellm_params")).get("model") - return isinstance(model, str) and "claude" in model.lower() and deployment_supports_prefill(deployment) + model: Final = _underlying_model(deployment) + return model is not None and "claude" in model.lower() and deployment_supports_prefill(deployment) def _continuation_request(emitted_text: str) -> Mapping[str, object]: diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 2f460e29a06..f76193f9a89 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -156,6 +156,7 @@ ignored_function_names = [ "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py + "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py ] diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index 9f18b377770..d1992e106d8 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -11,6 +11,7 @@ import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.router import GenericLiteLLMParams from litellm import Router +from pydantic import BaseModel, ConfigDict from litellm.constants import STREAM_SSE_DONE_STRING from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig @@ -1490,6 +1491,12 @@ async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cann router.discard() +class _Numbers(BaseModel): + model_config = ConfigDict(frozen=True) + + numbers: tuple[int, ...] + + @pytest.mark.asyncio @pytest.mark.parametrize( "constraint", @@ -1497,8 +1504,9 @@ async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cann {"text": {"format": {"type": "json_schema", "name": "numbers", "strict": True, "schema": {"type": "object"}}}}, {"text": {"format": {"type": "json_object"}}}, {"tool_choice": "required", "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}]}, + {"text_format": _Numbers}, ], - ids=["json_schema", "json_object", "required_tool"], + ids=["json_schema", "json_object", "required_tool", "sdk_text_format"], ) async def test_responses_fallback_surfaces_the_primary_error_once_constrained_output_was_streamed( constraint: dict[str, object], @@ -1525,6 +1533,48 @@ async def test_responses_fallback_surfaces_the_primary_error_once_constrained_ou router.discard() +@pytest.mark.asyncio +async def test_responses_fallback_continues_within_the_callers_output_ceiling( + claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses(model="primary", input="Count to 5.", stream=True, max_output_tokens=100) + events: Final = tuple([event async for event in stream]) + assert _streamed_text(events) == "1, 2, 3, 4, 5" + assert json.loads(fallback.calls.last.request.content)["max_tokens"] == 100 - litellm.token_counter( + text="1, 2, " + ) + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_used_the_output_ceiling( + claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses( + model="primary", input="Count to 5.", stream=True, max_output_tokens=litellm.token_counter(text="1, 2, ") + ) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert fallback.call_count == 0 + finally: + router.discard() + + @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio async def test_aresponses_client_error_event_skips_fallback(): diff --git a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py index dd12ad0bf2b..4492f8512ae 100644 --- a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py @@ -322,8 +322,25 @@ def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left( ({"text": {"verbosity": "low"}}, False), ({"tool_choice": "required"}, True), ({"tool_choice": "auto"}, False), + ({"tool_choice": {"type": "function", "name": "lookup"}}, True), + ({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True), + ({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False), + ({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False), + ({"tool_choice": {"type": "web_search_preview"}}, False), + ], + ids=[ + "json_schema", + "json_object", + "text_format", + "verbosity_only", + "required_tool", + "auto_tool", + "forced_function", + "allowed_tools_required", + "allowed_tools_auto", + "chat_allowed_tools_auto", + "hosted_tool", ], - ids=["json_schema", "json_object", "text_format", "verbosity_only", "required_tool", "auto_tool"], ) def test_constrains_continuation_reads_the_responses_text_format(sent, expected): assert constrains_continuation(sent) is expected @@ -335,8 +352,12 @@ def test_constrains_continuation_reads_the_responses_text_format(sent, expected) (_deployment(CLAUDE_PREFILL_MODEL, "a"), True), (_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), False), (_deployment(PREFILL_MODEL, "c"), False), + ( + {"litellm_params": {"model": "anthropic/support-model-v1"}, "model_info": {"id": "d", "base_model": CLAUDE_PREFILL_MODEL}}, + True, + ), ], - ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude"], + ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude", "claude_named_by_base_model"], ) def test_deployment_takes_responses_prefill_only_on_claude_with_prefill(deployment, expected): assert deployment_takes_responses_prefill(deployment) is expected