diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4a10db31606..e1152c9ee68 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -920,7 +920,7 @@ class BaseLLMHTTPHandler: api_key=api_key, model=model, optional_params=optional_params, - stream=stream, + stream=False if fake_stream else stream, litellm_params=litellm_params, ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 76c0095c2f8..0e947c6905a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4301,6 +4301,7 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( "supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output", + "supports_bedrock_native_structured_output_by_default", "supports_pdf_input", "supports_continuation", "reasoning_modes", diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index dd0e5762845..302ce9904fe 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -4890,6 +4890,54 @@ async def test_json_mode_stream_reaches_the_iterator_of_a_config_without_a_reque ] +class _StreamRouteConfig(_TransformRecordingConfig): + def __init__(self): + super().__init__(transform_async=False) + + def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None): + return f"{api_base}/{'stream' if stream else 'answer'}" + + +def _answers_only_the_non_streaming_route(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/answer"): + return httpx.Response(200, json=json.loads(request.content)) + return httpx.Response(403, json={"message": "not authorized to stream"}) + + +def _fake_streamed_completion(acompletion: bool, client: HTTPHandler | AsyncHTTPHandler): + return BaseLLMHTTPHandler().completion( + model="stub-model", + messages=[{"role": "user", "content": "hi"}], + api_base="https://llm.example/v1/chat", + custom_llm_provider="openai", + model_response=ModelResponse(), + encoding=None, + logging_obj=_streaming_logging_obj(), + optional_params={"fake_stream": True}, + timeout=10.0, + litellm_params={}, + acompletion=acompletion, + stream=True, + client=client, + provider_config=_StreamRouteConfig(), + ) + + +async def test_a_fake_streamed_completion_is_sent_to_the_non_streaming_route(): + sync_client: Final = HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_answers_only_the_non_streaming_route)) + ) + async_client: Final = AsyncHTTPHandler() + async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_answers_only_the_non_streaming_route)) + + sync_chunks: Final = list(_fake_streamed_completion(acompletion=False, client=sync_client)) + async_chunks: Final = [ + chunk async for chunk in await _fake_streamed_completion(acompletion=True, client=async_client) + ] + + assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync") + + class _ImageEditRecordingConfig(BaseImageEditConfig): def __init__(self): self.transform_calls = [] diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index c0c8a0655fe..cd71c3f9bee 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -542,7 +542,13 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): @pytest.mark.parametrize( "flag", - ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params", "supports_continuation"], + [ + "supports_regex_lookaround", + "supports_assistant_prefill", + "supports_sampling_params", + "supports_continuation", + "supports_bedrock_native_structured_output_by_default", + ], ) def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: """A deployment's capability override must not land on the shared