mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-19 09:53:33 +00:00
fix(router): guard SDK text_format, narrow forced tool_choice, read Claude base_model, and cap Responses continuation output
Cut Responses streams that asked for structured output through the SDK's text_format now surface the primary's error instead of stitching two models' JSON. Only a required or named tool_choice counts as forced, so allowed_tools in auto mode and hosted tool choices still continue. A Claude deployment named only through base_model keeps its prefill continuation, and the continuation's max_output_tokens is lowered by what the cut stream already emitted.
This commit is contained in:
parent
ccda4e11ab
commit
f9116cc3f1
5 changed files with 115 additions and 15 deletions
|
|
@ -454,6 +454,7 @@ from litellm.utils import (
|
|||
provider_rejectable_params,
|
||||
resolve_api_key_secret_reference,
|
||||
set_live_deployment_replay,
|
||||
token_counter,
|
||||
)
|
||||
|
||||
from .router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
|
@ -3719,7 +3720,8 @@ class Router:
|
|||
) -> Mapping[str, object]:
|
||||
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
|
||||
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
|
||||
defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None}
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self.default_litellm_params)
|
||||
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
|
||||
deployment_tool_choice: Final = (
|
||||
{"tool_choice": deployment_params["tool_choice"]}
|
||||
if deployment_params.get("tool_choice") is not None
|
||||
|
|
@ -4034,6 +4036,18 @@ class Router:
|
|||
return earlier.extended(e.generated_content)
|
||||
return ResponsesContinuation(request_kwargs.get("input"), e.generated_content)
|
||||
|
||||
@staticmethod
|
||||
def _responses_continuation_ceilings(
|
||||
e: "MidStreamFallbackError", request_kwargs: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
"""The caller's output ceilings less what the cut stream already emitted, raising the primary's error once
|
||||
nothing is left."""
|
||||
emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content)
|
||||
reduced: Final = continuation_output_ceilings(request_kwargs, emitted_tokens)
|
||||
if reduced is None:
|
||||
Router._raise_original_mid_stream_error(e)
|
||||
return reduced
|
||||
|
||||
async def _aresponses_fallback_attempt(
|
||||
self,
|
||||
e: "MidStreamFallbackError",
|
||||
|
|
@ -4053,6 +4067,9 @@ class Router:
|
|||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
continuation: Final = self._responses_continuation(e, source_iterator, initial_kwargs)
|
||||
reduced_ceilings: Final[Mapping[str, object]] = (
|
||||
Router._responses_continuation_ceilings(e, initial_kwargs) if continuation is not None else {}
|
||||
)
|
||||
seam: Final = ResponsesContinuationSeam(continuation.emitted_text if continuation is not None else "")
|
||||
partial_usage: Final = Router._extract_partial_responses_usage(source_iterator)
|
||||
fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted
|
||||
|
|
@ -4072,6 +4089,7 @@ class Router:
|
|||
self._ageneric_api_call_with_fallbacks_responses_attempt
|
||||
)
|
||||
if continuation is not None:
|
||||
initial_kwargs.update(reduced_ceilings)
|
||||
initial_kwargs[RESPONSES_CONTINUATION_KWARG] = continuation # rebind-ok: shaped per fallback deployment
|
||||
initial_kwargs["input"] = responses_continuation_input(continuation) # rebind-ok: fallback hop input
|
||||
# The Responses-API path stores observability metadata
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker()
|
|||
_STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"})
|
||||
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens")
|
||||
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens")
|
||||
_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"})
|
||||
RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation"
|
||||
|
||||
|
||||
|
|
@ -39,30 +40,39 @@ def _str_keyed(value: object) -> Mapping[str, object]:
|
|||
return _NO_FIELDS
|
||||
|
||||
|
||||
def deployment_supports_prefill(deployment: object) -> bool:
|
||||
def _underlying_model(deployment: object) -> str | None:
|
||||
deployment_map: Final = _str_keyed(deployment)
|
||||
model_info: Final = _str_keyed(deployment_map.get("model_info"))
|
||||
declared: Final = model_info.get("supports_assistant_prefill")
|
||||
if isinstance(declared, bool):
|
||||
return declared
|
||||
litellm_params: Final = _str_keyed(deployment_map.get("litellm_params"))
|
||||
candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model"))
|
||||
model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
|
||||
return next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
|
||||
|
||||
|
||||
def deployment_supports_prefill(deployment: object) -> bool:
|
||||
declared: Final = _str_keyed(_str_keyed(deployment).get("model_info")).get("supports_assistant_prefill")
|
||||
if isinstance(declared, bool):
|
||||
return declared
|
||||
model: Final = _underlying_model(deployment)
|
||||
return model is not None and supports_assistant_prefill(model=model)
|
||||
|
||||
|
||||
def _forces_tool_call(tool_choice: object) -> bool:
|
||||
choice: Final = _str_keyed(tool_choice)
|
||||
mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode")
|
||||
return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES
|
||||
|
||||
|
||||
def constrains_continuation(params: Mapping[str, object]) -> bool:
|
||||
sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
|
||||
choice_count: Final = sent.get("n")
|
||||
response_format: Final = sent.get("response_format")
|
||||
text_format: Final = _str_keyed(sent.get("text")).get("format")
|
||||
tool_choice: Final = sent.get("tool_choice")
|
||||
return (
|
||||
(isinstance(choice_count, int) and choice_count > 1)
|
||||
or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
|
||||
or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT)
|
||||
or tool_choice == "required"
|
||||
or isinstance(tool_choice, Mapping)
|
||||
or sent.get("text_format") is not None
|
||||
or _forces_tool_call(sent.get("tool_choice"))
|
||||
or sent.get("merge_reasoning_content_in_choices") is True
|
||||
)
|
||||
|
||||
|
|
@ -280,8 +290,8 @@ class ResponsesContinuation:
|
|||
def deployment_takes_responses_prefill(deployment: object) -> bool:
|
||||
"""Claude takes a trailing assistant turn as prefill with no `prefix` flag, which a Responses input item cannot
|
||||
carry."""
|
||||
model: Final = _str_keyed(_str_keyed(deployment).get("litellm_params")).get("model")
|
||||
return isinstance(model, str) and "claude" in model.lower() and deployment_supports_prefill(deployment)
|
||||
model: Final = _underlying_model(deployment)
|
||||
return model is not None and "claude" in model.lower() and deployment_supports_prefill(deployment)
|
||||
|
||||
|
||||
def _continuation_request(emitted_text: str) -> Mapping[str, object]:
|
||||
|
|
|
|||
|
|
@ -156,6 +156,7 @@ ignored_function_names = [
|
|||
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
|
||||
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
|
||||
"_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
|
||||
"_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import litellm
|
|||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm import Router
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from litellm.constants import STREAM_SSE_DONE_STRING
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
|
|
@ -1490,6 +1491,12 @@ async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cann
|
|||
router.discard()
|
||||
|
||||
|
||||
class _Numbers(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
numbers: tuple[int, ...]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"constraint",
|
||||
|
|
@ -1497,8 +1504,9 @@ async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cann
|
|||
{"text": {"format": {"type": "json_schema", "name": "numbers", "strict": True, "schema": {"type": "object"}}}},
|
||||
{"text": {"format": {"type": "json_object"}}},
|
||||
{"tool_choice": "required", "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}]},
|
||||
{"text_format": _Numbers},
|
||||
],
|
||||
ids=["json_schema", "json_object", "required_tool"],
|
||||
ids=["json_schema", "json_object", "required_tool", "sdk_text_format"],
|
||||
)
|
||||
async def test_responses_fallback_surfaces_the_primary_error_once_constrained_output_was_streamed(
|
||||
constraint: dict[str, object],
|
||||
|
|
@ -1525,6 +1533,48 @@ async def test_responses_fallback_surfaces_the_primary_error_once_constrained_ou
|
|||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_continues_within_the_callers_output_ceiling(
|
||||
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream("3, 4, 5")
|
||||
)
|
||||
router: Final = _primary_with_fallback(claude_with_prefill)
|
||||
try:
|
||||
stream: Final = await router.aresponses(model="primary", input="Count to 5.", stream=True, max_output_tokens=100)
|
||||
events: Final = tuple([event async for event in stream])
|
||||
assert _streamed_text(events) == "1, 2, 3, 4, 5"
|
||||
assert json.loads(fallback.calls.last.request.content)["max_tokens"] == 100 - litellm.token_counter(
|
||||
text="1, 2, "
|
||||
)
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_used_the_output_ceiling(
|
||||
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream("3, 4, 5")
|
||||
)
|
||||
router: Final = _primary_with_fallback(claude_with_prefill)
|
||||
try:
|
||||
stream: Final = await router.aresponses(
|
||||
model="primary", input="Count to 5.", stream=True, max_output_tokens=litellm.token_counter(text="1, 2, ")
|
||||
)
|
||||
with pytest.raises(litellm.InternalServerError, match="interrupted"):
|
||||
[event async for event in stream]
|
||||
assert fallback.call_count == 0
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_client_error_event_skips_fallback():
|
||||
|
|
|
|||
|
|
@ -322,8 +322,25 @@ def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(
|
|||
({"text": {"verbosity": "low"}}, False),
|
||||
({"tool_choice": "required"}, True),
|
||||
({"tool_choice": "auto"}, False),
|
||||
({"tool_choice": {"type": "function", "name": "lookup"}}, True),
|
||||
({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True),
|
||||
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False),
|
||||
({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False),
|
||||
({"tool_choice": {"type": "web_search_preview"}}, False),
|
||||
],
|
||||
ids=[
|
||||
"json_schema",
|
||||
"json_object",
|
||||
"text_format",
|
||||
"verbosity_only",
|
||||
"required_tool",
|
||||
"auto_tool",
|
||||
"forced_function",
|
||||
"allowed_tools_required",
|
||||
"allowed_tools_auto",
|
||||
"chat_allowed_tools_auto",
|
||||
"hosted_tool",
|
||||
],
|
||||
ids=["json_schema", "json_object", "text_format", "verbosity_only", "required_tool", "auto_tool"],
|
||||
)
|
||||
def test_constrains_continuation_reads_the_responses_text_format(sent, expected):
|
||||
assert constrains_continuation(sent) is expected
|
||||
|
|
@ -335,8 +352,12 @@ def test_constrains_continuation_reads_the_responses_text_format(sent, expected)
|
|||
(_deployment(CLAUDE_PREFILL_MODEL, "a"), True),
|
||||
(_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), False),
|
||||
(_deployment(PREFILL_MODEL, "c"), False),
|
||||
(
|
||||
{"litellm_params": {"model": "anthropic/support-model-v1"}, "model_info": {"id": "d", "base_model": CLAUDE_PREFILL_MODEL}},
|
||||
True,
|
||||
),
|
||||
],
|
||||
ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude"],
|
||||
ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude", "claude_named_by_base_model"],
|
||||
)
|
||||
def test_deployment_takes_responses_prefill_only_on_claude_with_prefill(deployment, expected):
|
||||
assert deployment_takes_responses_prefill(deployment) is expected
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue