fix(bedrock): keep streams bridged back from Responses working and keep a non-list include on Converse

A chat stream that the Responses bridge routes back to chat for a provider with no Responses config has no httpx response, so reading provider headers from it raised and the stream failed. A top-level or extra_body include that is not a list now keeps Converse, so it answers the way main does
This commit is contained in:
mateo-berri 2026-10-21 01:24:30 -07:00
parent 29f49eb3a2
commit 229dbaffbb
4 changed files with 75 additions and 7 deletions

View file

@ -11,13 +11,12 @@ from typing import (
cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict
)
import httpx
from typing_extensions import ReadOnly, TypedDict
from litellm.types.llms.openai import ResponsesAPIResponse
if TYPE_CHECKING:
import httpx
from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse
@ -34,10 +33,14 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
client: ReadOnly[object]
def _provider_response_headers(stream: object) -> "httpx.Headers | None":
def _provider_response_headers(stream: object) -> httpx.Headers | None:
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
return stream.response.headers if isinstance(stream, BaseResponsesAPIStreamingIterator) else None
match stream:
case BaseResponsesAPIStreamingIterator(response=httpx.Response(headers=headers)):
return headers
case _:
return None
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:

View file

@ -1258,8 +1258,10 @@ def _extra_body_bridgeable(extra_body: object) -> bool:
if not isinstance(extra_body, dict):
return False
validated: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body)
return validated.keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES and isinstance(
validated.get("store", False), bool
return (
validated.keys() <= _EXTRA_BODY_KEYS_THE_RESPONSES_BRIDGE_CARRIES
and isinstance(validated.get("store", False), bool)
and isinstance(validated.get("include"), (list, tuple, NoneType))
)
@ -1298,6 +1300,7 @@ def _bridgeable_to_responses(model: str, request_params: Mapping[str, object]) -
and not _sets_a_chat_param_the_responses_bridge_drops(request_params)
and isinstance(request_params.get("reasoning_effort"), (str, NoneType))
and isinstance(request_params.get("store"), (bool, NoneType))
and isinstance(request_params.get("include"), (list, tuple, NoneType))
and _bedrock_serves_responses(model)
)
@ -1316,7 +1319,7 @@ def bedrock_responses_bridge_optional_params(optional_params: Mapping[str, objec
extra_body: Final = optional_params.get("extra_body")
extra_body_params: Final = _EXTRA_BODY_ADAPTER.validate_python(extra_body) if isinstance(extra_body, dict) else {}
include: Final = extra_body_params.get("include", optional_params.get("include"))
requested: Final = _INCLUDE_ADAPTER.validate_python(include) if isinstance(include, (list, tuple)) else ()
requested: Final = _INCLUDE_ADAPTER.validate_python(include or ())
return {
**optional_params,
**(

View file

@ -10,6 +10,9 @@ from litellm.completion_extras.litellm_responses_transformation.handler import (
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse
@ -266,6 +269,54 @@ def test_completion_streams_completed_model_response():
)
def _chat_stream_behind_responses() -> LiteLLMCompletionStreamingIterator:
return LiteLLMCompletionStreamingIterator(
model="claude-haiku-5-5",
litellm_custom_stream_wrapper=litellm.completion(
model="anthropic/claude-haiku-5-5",
messages=[{"role": "user", "content": "hi"}],
mock_response="pong",
stream=True,
),
request_input="hi",
responses_api_request={},
custom_llm_provider="anthropic",
)
def test_completion_streams_a_chat_stream_bridged_back_from_responses():
bridge = ResponsesToCompletionBridgeHandler()
with (
patch.object(
bridge.transformation_handler,
"transform_request",
return_value={"model": "claude-haiku-5-5", "input": "hi"},
),
patch("litellm.responses", return_value=_chat_stream_behind_responses()),
):
result = bridge.completion(**_bridge_kwargs(stream=True))
assert "".join(chunk.choices[0].delta.content or "" for chunk in result) == "pong"
@pytest.mark.asyncio
async def test_acompletion_streams_a_chat_stream_bridged_back_from_responses():
bridge = ResponsesToCompletionBridgeHandler()
with (
patch.object(
bridge.transformation_handler,
"transform_request",
return_value={"model": "claude-haiku-5-5", "input": "hi"},
),
patch("litellm.aresponses", new=AsyncMock(return_value=_chat_stream_behind_responses())),
):
result = await bridge.acompletion(**_bridge_kwargs(stream=True))
assert "".join([chunk.choices[0].delta.content or "" async for chunk in result]) == "pong"
_PROVIDER_NATIVE_MODEL_CASES = [
("perplexity", "perplexity/kimi-k3", "perplexity/kimi-k3"),
("perplexity", "openai/gpt-5.2", "openai/gpt-5.2"),

View file

@ -615,6 +615,17 @@ def _weather_tool_named(name):
),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"top_k": 5}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": "store=false"}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "include": ["file_search_call.results"]}, "responses"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "include": "file_search_call.results"}, "converse"),
(
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": ["file_search_call.results"]}},
"responses",
),
(
{"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": "file_search_call.results"}},
"converse",
),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "extra_body": {"include": {"a": 1}}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "seed": 7}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "logit_bias": {"50256": -100}}, "converse"),
({"tools": [GET_WEATHER_TOOL], "reasoning_effort": "low", "web_search_options": {}}, "converse"),