fix(router): guard SDK text_format, narrow forced tool_choice, read Claude base_model, and cap Responses continuation output

Cut Responses streams that asked for structured output through the SDK's text_format now surface the primary's
error instead of stitching two models' JSON. Only a required or named tool_choice counts as forced, so allowed_tools
in auto mode and hosted tool choices still continue. A Claude deployment named only through base_model keeps its
prefill continuation, and the continuation's max_output_tokens is lowered by what the cut stream already emitted.
This commit is contained in:
mateo-berri 2026-10-20 20:27:37 -07:00
parent ccda4e11ab
commit f9116cc3f1
5 changed files with 115 additions and 15 deletions

View file

@ -454,6 +454,7 @@ from litellm.utils import (
provider_rejectable_params,
resolve_api_key_secret_reference,
set_live_deployment_replay,
token_counter,
)
from .router_utils.pattern_match_deployments import PatternMatchRouter
@ -3719,7 +3720,8 @@ class Router:
) -> Mapping[str, object]:
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None}
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self.default_litellm_params)
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
deployment_tool_choice: Final = (
{"tool_choice": deployment_params["tool_choice"]}
if deployment_params.get("tool_choice") is not None
@ -4034,6 +4036,18 @@ class Router:
return earlier.extended(e.generated_content)
return ResponsesContinuation(request_kwargs.get("input"), e.generated_content)
@staticmethod
def _responses_continuation_ceilings(
e: "MidStreamFallbackError", request_kwargs: Mapping[str, object]
) -> Mapping[str, object]:
"""The caller's output ceilings less what the cut stream already emitted, raising the primary's error once
nothing is left."""
emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content)
reduced: Final = continuation_output_ceilings(request_kwargs, emitted_tokens)
if reduced is None:
Router._raise_original_mid_stream_error(e)
return reduced
async def _aresponses_fallback_attempt(
self,
e: "MidStreamFallbackError",
@ -4053,6 +4067,9 @@ class Router:
from litellm.exceptions import MidStreamFallbackError
continuation: Final = self._responses_continuation(e, source_iterator, initial_kwargs)
reduced_ceilings: Final[Mapping[str, object]] = (
Router._responses_continuation_ceilings(e, initial_kwargs) if continuation is not None else {}
)
seam: Final = ResponsesContinuationSeam(continuation.emitted_text if continuation is not None else "")
partial_usage: Final = Router._extract_partial_responses_usage(source_iterator)
fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted
@ -4072,6 +4089,7 @@ class Router:
self._ageneric_api_call_with_fallbacks_responses_attempt
)
if continuation is not None:
initial_kwargs.update(reduced_ceilings)
initial_kwargs[RESPONSES_CONTINUATION_KWARG] = continuation # rebind-ok: shaped per fallback deployment
initial_kwargs["input"] = responses_continuation_input(continuation) # rebind-ok: fallback hop input
# The Responses-API path stores observability metadata

View file

@ -28,7 +28,8 @@ MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker()
_STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"})
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens")
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens")
_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"})
RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation"
@ -39,30 +40,39 @@ def _str_keyed(value: object) -> Mapping[str, object]:
return _NO_FIELDS
def deployment_supports_prefill(deployment: object) -> bool:
def _underlying_model(deployment: object) -> str | None:
deployment_map: Final = _str_keyed(deployment)
model_info: Final = _str_keyed(deployment_map.get("model_info"))
declared: Final = model_info.get("supports_assistant_prefill")
if isinstance(declared, bool):
return declared
litellm_params: Final = _str_keyed(deployment_map.get("litellm_params"))
candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model"))
model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
return next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
def deployment_supports_prefill(deployment: object) -> bool:
declared: Final = _str_keyed(_str_keyed(deployment).get("model_info")).get("supports_assistant_prefill")
if isinstance(declared, bool):
return declared
model: Final = _underlying_model(deployment)
return model is not None and supports_assistant_prefill(model=model)
def _forces_tool_call(tool_choice: object) -> bool:
choice: Final = _str_keyed(tool_choice)
mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode")
return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES
def constrains_continuation(params: Mapping[str, object]) -> bool:
sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
choice_count: Final = sent.get("n")
response_format: Final = sent.get("response_format")
text_format: Final = _str_keyed(sent.get("text")).get("format")
tool_choice: Final = sent.get("tool_choice")
return (
(isinstance(choice_count, int) and choice_count > 1)
or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT)
or tool_choice == "required"
or isinstance(tool_choice, Mapping)
or sent.get("text_format") is not None
or _forces_tool_call(sent.get("tool_choice"))
or sent.get("merge_reasoning_content_in_choices") is True
)
@ -280,8 +290,8 @@ class ResponsesContinuation:
def deployment_takes_responses_prefill(deployment: object) -> bool:
"""Claude takes a trailing assistant turn as prefill with no `prefix` flag, which a Responses input item cannot
carry."""
model: Final = _str_keyed(_str_keyed(deployment).get("litellm_params")).get("model")
return isinstance(model, str) and "claude" in model.lower() and deployment_supports_prefill(deployment)
model: Final = _underlying_model(deployment)
return model is not None and "claude" in model.lower() and deployment_supports_prefill(deployment)
def _continuation_request(emitted_text: str) -> Mapping[str, object]:

View file

@ -156,6 +156,7 @@ ignored_function_names = [
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
"_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
"_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
]

View file

@ -11,6 +11,7 @@ import litellm
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.router import GenericLiteLLMParams
from litellm import Router
from pydantic import BaseModel, ConfigDict
from litellm.constants import STREAM_SSE_DONE_STRING
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
@ -1490,6 +1491,12 @@ async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cann
router.discard()
class _Numbers(BaseModel):
model_config = ConfigDict(frozen=True)
numbers: tuple[int, ...]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"constraint",
@ -1497,8 +1504,9 @@ async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cann
{"text": {"format": {"type": "json_schema", "name": "numbers", "strict": True, "schema": {"type": "object"}}}},
{"text": {"format": {"type": "json_object"}}},
{"tool_choice": "required", "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}]},
{"text_format": _Numbers},
],
ids=["json_schema", "json_object", "required_tool"],
ids=["json_schema", "json_object", "required_tool", "sdk_text_format"],
)
async def test_responses_fallback_surfaces_the_primary_error_once_constrained_output_was_streamed(
constraint: dict[str, object],
@ -1525,6 +1533,48 @@ async def test_responses_fallback_surfaces_the_primary_error_once_constrained_ou
router.discard()
@pytest.mark.asyncio
async def test_responses_fallback_continues_within_the_callers_output_ceiling(
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream("3, 4, 5")
)
router: Final = _primary_with_fallback(claude_with_prefill)
try:
stream: Final = await router.aresponses(model="primary", input="Count to 5.", stream=True, max_output_tokens=100)
events: Final = tuple([event async for event in stream])
assert _streamed_text(events) == "1, 2, 3, 4, 5"
assert json.loads(fallback.calls.last.request.content)["max_tokens"] == 100 - litellm.token_counter(
text="1, 2, "
)
finally:
router.discard()
@pytest.mark.asyncio
async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_used_the_output_ceiling(
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream("3, 4, 5")
)
router: Final = _primary_with_fallback(claude_with_prefill)
try:
stream: Final = await router.aresponses(
model="primary", input="Count to 5.", stream=True, max_output_tokens=litellm.token_counter(text="1, 2, ")
)
with pytest.raises(litellm.InternalServerError, match="interrupted"):
[event async for event in stream]
assert fallback.call_count == 0
finally:
router.discard()
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
@pytest.mark.asyncio
async def test_aresponses_client_error_event_skips_fallback():

View file

@ -322,8 +322,25 @@ def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(
({"text": {"verbosity": "low"}}, False),
({"tool_choice": "required"}, True),
({"tool_choice": "auto"}, False),
({"tool_choice": {"type": "function", "name": "lookup"}}, True),
({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True),
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False),
({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False),
({"tool_choice": {"type": "web_search_preview"}}, False),
],
ids=[
"json_schema",
"json_object",
"text_format",
"verbosity_only",
"required_tool",
"auto_tool",
"forced_function",
"allowed_tools_required",
"allowed_tools_auto",
"chat_allowed_tools_auto",
"hosted_tool",
],
ids=["json_schema", "json_object", "text_format", "verbosity_only", "required_tool", "auto_tool"],
)
def test_constrains_continuation_reads_the_responses_text_format(sent, expected):
assert constrains_continuation(sent) is expected
@ -335,8 +352,12 @@ def test_constrains_continuation_reads_the_responses_text_format(sent, expected)
(_deployment(CLAUDE_PREFILL_MODEL, "a"), True),
(_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), False),
(_deployment(PREFILL_MODEL, "c"), False),
(
{"litellm_params": {"model": "anthropic/support-model-v1"}, "model_info": {"id": "d", "base_model": CLAUDE_PREFILL_MODEL}},
True,
),
],
ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude"],
ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude", "claude_named_by_base_model"],
)
def test_deployment_takes_responses_prefill_only_on_claude_with_prefill(deployment, expected):
assert deployment_takes_responses_prefill(deployment) is expected