mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(bedrock): keep streams bridged back from Responses working and keep a non-list include on Converse
A chat stream that the Responses bridge routes back to chat for a provider with no Responses config has no httpx response, so reading provider headers from it raised and the stream failed. A top-level or extra_body include that is not a list now keeps Converse, so it answers the way main does
This commit is contained in:
parent
29f49eb3a2
commit
229dbaffbb
4 changed files with 75 additions and 7 deletions
|
|
@ -11,13 +11,12 @@ from typing import (
|
|||
cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict
|
||||
)
|
||||
|
||||
import httpx
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
|
||||
from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse
|
||||
|
||||
|
||||
|
|
@ -34,10 +33,14 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
|
|||
client: ReadOnly[object]
|
||||
|
||||
|
||||
def _provider_response_headers(stream: object) -> "httpx.Headers | None":
|
||||
def _provider_response_headers(stream: object) -> httpx.Headers | None:
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
|
||||
return stream.response.headers if isinstance(stream, BaseResponsesAPIStreamingIterator) else None
|
||||
match stream:
|
||||
case BaseResponsesAPIStreamingIterator(response=httpx.Response(headers=headers)):
|
||||
return headers
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
|
||||
|
|
|
|||
|
|
@ -1258,8 +1258,10 @@ def _extra_body_bridgeable(extra_body: object) -> bool:
|
|||
if not isinstance(extra_body, dict):
|
||||
return False
|
||||
validated: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body)
|
||||
return validated.keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES and isinstance(
|
||||
validated.get("store", False), bool
|
||||
return (
|
||||
validated.keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES
|
||||
and isinstance(validated.get("store", False), bool)
|
||||
and isinstance(validated.get("include"), (list, tuple, NoneType))
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1298,6 +1300,7 @@ def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) -
|
|||
and not _sets_a_chat_param_the_responses_bridge_drops(request_params)
|
||||
and isinstance(request_params.get("reasoning_effort"), (str, NoneType))
|
||||
and isinstance(request_params.get("store"), (bool, NoneType))
|
||||
and isinstance(request_params.get("include"), (list, tuple, NoneType))
|
||||
and _bedrock_serves_responses(model)
|
||||
)
|
||||
|
||||
|
|
@ -1316,7 +1319,7 @@ def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, objec
|
|||
extra_body: Final = optional_params.get("extra_body")
|
||||
extra_body_params: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) if isinstance(extra_body, dict) else {}
|
||||
include: Final = extra_body_params.get("include", optional_params.get("include"))
|
||||
requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else ()
|
||||
requested: Final = _INCLUDE_ADAPTER.validate_python(include or ())
|
||||
return {
|
||||
**optional_params,
|
||||
**(
|
||||
|
|
|
|||
|
|
@ -10,6 +10,9 @@ from litellm.completion_extras.litellm_responses_transformation.handler import (
|
|||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
|
||||
LiteLLMCompletionStreamingIterator,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
|
@ -266,6 +269,54 @@ def test_completion_streams_completed_model_response():
|
|||
)
|
||||
|
||||
|
||||
|
||||
def _chat_stream_behind_responses() -> LiteLLMCompletionStreamingIterator:
|
||||
return LiteLLMCompletionStreamingIterator(
|
||||
model="claude-haiku-5-5",
|
||||
litellm_custom_stream_wrapper=litellm.completion(
|
||||
model="anthropic/claude-haiku-5-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
mock_response="pong",
|
||||
stream=True,
|
||||
),
|
||||
request_input="hi",
|
||||
responses_api_request={},
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
|
||||
def test_completion_streams_a_chat_stream_bridged_back_from_responses():
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
bridge.transformation_handler,
|
||||
"transform_request",
|
||||
return_value={"model": "claude-haiku-5-5", "input": "hi"},
|
||||
),
|
||||
patch("litellm.responses", return_value=_chat_stream_behind_responses()),
|
||||
):
|
||||
result = bridge.completion(**_bridge_kwargs(stream=True))
|
||||
|
||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in result) == "pong"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_streams_a_chat_stream_bridged_back_from_responses():
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
bridge.transformation_handler,
|
||||
"transform_request",
|
||||
return_value={"model": "claude-haiku-5-5", "input": "hi"},
|
||||
),
|
||||
patch("litellm.aresponses", new=AsyncMock(return_value=_chat_stream_behind_responses())),
|
||||
):
|
||||
result = await bridge.acompletion(**_bridge_kwargs(stream=True))
|
||||
|
||||
assert "".join([chunk.choices[0].delta.content or "" async for chunk in result]) == "pong"
|
||||
|
||||
_PROVIDER_NATIVE_MODEL_CASES = [
|
||||
("perplexity", "perplexity/kimi-k3", "perplexity/kimi-k3"),
|
||||
("perplexity", "openai/gpt-5.2", "openai/gpt-5.2"),
|
||||
|
|
|
|||
|
|
@ -615,6 +615,17 @@ def _weather_tool_named(name):
|
|||
),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"top_k": 5}}, "converse"),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": "store=false"}, "converse"),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "include": ["file_search_call.results"]}, "responses"),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "include": "file_search_call.results"}, "converse"),
|
||||
(
|
||||
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": ["file_search_call.results"]}},
|
||||
"responses",
|
||||
),
|
||||
(
|
||||
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": "file_search_call.results"}},
|
||||
"converse",
|
||||
),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": {"a": 1}}}, "converse"),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "seed": 7}, "converse"),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logit_bias": {"50256": -100}}, "converse"),
|
||||
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "web_search_options": {}}, "converse"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue