diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index fe665628f06..df82891f789 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -897,7 +897,7 @@ class BaseLLMHTTPHandler: fake_stream = ( fake_stream - or optional_params.pop("fake_stream", False) + or optional_params.pop("fake_stream", False) is True or provider_config.should_fake_stream(model=model, custom_llm_provider=custom_llm_provider, stream=stream) ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ee08580a024..7f48d77ff09 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16746,8 +16746,14 @@ def _enrich_model_info_with_litellm_data( if k not in stamped_model_info or (stamped_model_info[k] is None and k in discovered_model_info) }, } + native_output_info: Final = { + **merged_model_info, + "supports_bedrock_native_structured_output_by_default": litellm_model_info.get( + "supports_bedrock_native_structured_output_by_default" + ), + } model["model_info"] = ( - {**merged_model_info, "supports_native_structured_output": native_structured_output_support(merged_model_info)} + {**merged_model_info, "supports_native_structured_output": native_structured_output_support(native_output_info)} if "supports_native_structured_output" in merged_model_info and "supports_native_structured_output" not in stamped_model_info else merged_model_info diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index a9b067cedef..a758f4d4250 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -4890,17 +4890,19 @@ async def test_json_mode_stream_reaches_the_iterator_of_a_config_without_a_reque ] -class _StreamRouteConfig(_TransformRecordingConfig): +class _PlainStreamRouteConfig(_TransformRecordingConfig): def __init__(self): super().__init__(transform_async=False) + def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None): + return f"{api_base}/{'stream' if stream else 'answer'}" + + +class _StreamRouteConfig(_PlainStreamRouteConfig): @property def has_custom_stream_wrapper(self) -> bool: return True - def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None): - return f"{api_base}/{'stream' if stream else 'answer'}" - def _answers_only_the_non_streaming_route(request: httpx.Request) -> httpx.Response: if request.url.path.endswith("/answer"): @@ -4908,7 +4910,12 @@ def _answers_only_the_non_streaming_route(request: httpx.Request) -> httpx.Respo return httpx.Response(403, json={"message": "not authorized to stream"}) -def _fake_streamed_completion(acompletion: bool, client: HTTPHandler | AsyncHTTPHandler): +def _fake_streamed_completion( + acompletion: bool, + client: HTTPHandler | AsyncHTTPHandler, + fake_stream: object = True, + provider_config: BaseConfig | None = None, +): return BaseLLMHTTPHandler().completion( model="stub-model", messages=[{"role": "user", "content": "hi"}], @@ -4917,13 +4924,13 @@ def _fake_streamed_completion(acompletion: bool, client: HTTPHandler | AsyncHTTP model_response=ModelResponse(), encoding=None, logging_obj=_streaming_logging_obj(), - optional_params={"fake_stream": True}, + optional_params={"fake_stream": fake_stream}, timeout=10.0, litellm_params={}, acompletion=acompletion, stream=True, client=client, - provider_config=_StreamRouteConfig(), + provider_config=provider_config or _StreamRouteConfig(), ) @@ -4942,6 +4949,33 @@ async def test_a_fake_streamed_completion_is_sent_to_the_non_streaming_route(): assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync") +def _answers_only_the_streaming_route(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/stream"): + return _sse_echoing_transformed_by(request) + return httpx.Response(403, json={"message": "only streaming is allowed"}) + + +@pytest.mark.parametrize("fake_stream", ["true", 1], ids=["string", "int"]) +async def test_a_fake_stream_value_other_than_true_still_streams_from_the_streaming_route(fake_stream: object): + sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_answers_only_the_streaming_route))) + async_client: Final = AsyncHTTPHandler() + async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_answers_only_the_streaming_route)) + + sync_chunks: Final = list( + _fake_streamed_completion( + acompletion=False, client=sync_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig() + ) + ) + async_chunks: Final = [ + chunk + async for chunk in await _fake_streamed_completion( + acompletion=True, client=async_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig() + ) + ] + + assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync") + + class _ImageEditRecordingConfig(BaseImageEditConfig): def __init__(self): self.transform_calls = [] diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index ea182140616..79f3e697a4f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -513,6 +513,28 @@ def test_model_info_reports_a_deployment_bedrock_native_output_opt_out( assert info["supports_native_structured_output"] is native +@pytest.mark.parametrize( + ("model", "by_default"), + [("bedrock/us.anthropic.claude-opus-4-7", True), ("bedrock/us.anthropic.claude-sonnet-5", False)], + ids=["set-on-a-model-the-cost-map-leaves-on-the-json-tool", "unset-on-a-model-the-cost-map-makes-native"], +) +def test_model_info_ignores_a_deployment_by_default_key_the_router_reads_only_from_the_cost_map( + local_model_cost_map: None, model: str, by_default: bool +) -> None: + def reported_native(deployment_model_info: dict[str, object]) -> object: + return proxy_server._enrich_model_info_with_litellm_data( + { + "model_name": "claude", + "litellm_params": {"model": model}, + "model_info": {"id": "claude-deployment", **deployment_model_info}, + } + )["model_info"]["supports_native_structured_output"] + + assert reported_native({"supports_bedrock_native_structured_output_by_default": by_default}) is reported_native( + {} + ) + + def test_model_info_id_lookup_reports_the_same_cost_as_the_list( client: TestClient, auth_as: Callable[[], AbstractContextManager[object]],