fix(bedrock): count only a true fake_stream and keep the deployment by-default key out of /model/info

A fake_stream sent as "true" or 1 picked the non-streaming URL but kept the streaming reader, so Bedrock invoke streams answered 500 after three billed calls. Only a real True counts now, as on main.

/model/info read supports_bedrock_native_structured_output_by_default from the deployment, while routing reads it only from the cost map, so it reported native output for deployments that use the JSON tool.

litellm/llms/custom_httpx/llm_http_handler.py

@@ -897,7 +897,7 @@ class BaseLLMHTTPHandler:
 
         fake_stream = (
             fake_stream
-            or optional_params.pop("fake_stream", False)
+            or optional_params.pop("fake_stream", False) is True
             or provider_config.should_fake_stream(model=model, custom_llm_provider=custom_llm_provider, stream=stream)
         )
 

litellm/proxy/proxy_server.py

@@ -16746,8 +16746,14 @@ def _enrich_model_info_with_litellm_data(
             if k not in stamped_model_info or (stamped_model_info[k] is None and k in discovered_model_info)
         },
     }
+    native_output_info: Final = {
+        **merged_model_info,
+        "supports_bedrock_native_structured_output_by_default": litellm_model_info.get(
+            "supports_bedrock_native_structured_output_by_default"
+        ),
+    }
     model["model_info"] = (
-        {**merged_model_info, "supports_native_structured_output": native_structured_output_support(merged_model_info)}
+        {**merged_model_info, "supports_native_structured_output": native_structured_output_support(native_output_info)}
         if "supports_native_structured_output" in merged_model_info
         and "supports_native_structured_output" not in stamped_model_info
         else merged_model_info

tests/unit/llms/custom_httpx/test_llm_http_handler.py

@@ -4890,17 +4890,19 @@ async def test_json_mode_stream_reaches_the_iterator_of_a_config_without_a_reque
     ]
 
 
-class _StreamRouteConfig(_TransformRecordingConfig):
+class _PlainStreamRouteConfig(_TransformRecordingConfig):
     def __init__(self):
         super().__init__(transform_async=False)
 
+    def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None):
+        return f"{api_base}/{'stream' if stream else 'answer'}"
+
+
+class _StreamRouteConfig(_PlainStreamRouteConfig):
     @property
     def has_custom_stream_wrapper(self) -> bool:
         return True
 
-    def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None):
-        return f"{api_base}/{'stream' if stream else 'answer'}"
-
 
 def _answers_only_the_non_streaming_route(request: httpx.Request) -> httpx.Response:
     if request.url.path.endswith("/answer"):
@@ -4908,7 +4910,12 @@ def _answers_only_the_non_streaming_route(request: httpx.Request) -> httpx.Respo
     return httpx.Response(403, json={"message": "not authorized to stream"})
 
 
-def _fake_streamed_completion(acompletion: bool, client: HTTPHandler | AsyncHTTPHandler):
+def _fake_streamed_completion(
+    acompletion: bool,
+    client: HTTPHandler | AsyncHTTPHandler,
+    fake_stream: object = True,
+    provider_config: BaseConfig | None = None,
+):
     return BaseLLMHTTPHandler().completion(
         model="stub-model",
         messages=[{"role": "user", "content": "hi"}],
@@ -4917,13 +4924,13 @@ def _fake_streamed_completion(acompletion: bool, client: HTTPHandler | AsyncHTTP
         model_response=ModelResponse(),
         encoding=None,
         logging_obj=_streaming_logging_obj(),
-        optional_params={"fake_stream": True},
+        optional_params={"fake_stream": fake_stream},
         timeout=10.0,
         litellm_params={},
         acompletion=acompletion,
         stream=True,
         client=client,
-        provider_config=_StreamRouteConfig(),
+        provider_config=provider_config or _StreamRouteConfig(),
     )
 
 
@@ -4942,6 +4949,33 @@ async def test_a_fake_streamed_completion_is_sent_to_the_non_streaming_route():
     assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync")
 
 
+def _answers_only_the_streaming_route(request: httpx.Request) -> httpx.Response:
+    if request.url.path.endswith("/stream"):
+        return _sse_echoing_transformed_by(request)
+    return httpx.Response(403, json={"message": "only streaming is allowed"})
+
+
+@pytest.mark.parametrize("fake_stream", ["true", 1], ids=["string", "int"])
+async def test_a_fake_stream_value_other_than_true_still_streams_from_the_streaming_route(fake_stream: object):
+    sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_answers_only_the_streaming_route)))
+    async_client: Final = AsyncHTTPHandler()
+    async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_answers_only_the_streaming_route))
+
+    sync_chunks: Final = list(
+        _fake_streamed_completion(
+            acompletion=False, client=sync_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig()
+        )
+    )
+    async_chunks: Final = [
+        chunk
+        async for chunk in await _fake_streamed_completion(
+            acompletion=True, client=async_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig()
+        )
+    ]
+
+    assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync")
+
+
 class _ImageEditRecordingConfig(BaseImageEditConfig):
     def __init__(self):
         self.transform_calls = []

tests/unit/proxy/proxy_server/test_routes_model_info.py

@@ -513,6 +513,28 @@ def test_model_info_reports_a_deployment_bedrock_native_output_opt_out(
     assert info["supports_native_structured_output"] is native
 
 
+@pytest.mark.parametrize(
+    ("model", "by_default"),
+    [("bedrock/us.anthropic.claude-opus-4-7", True), ("bedrock/us.anthropic.claude-sonnet-5", False)],
+    ids=["set-on-a-model-the-cost-map-leaves-on-the-json-tool", "unset-on-a-model-the-cost-map-makes-native"],
+)
+def test_model_info_ignores_a_deployment_by_default_key_the_router_reads_only_from_the_cost_map(
+    local_model_cost_map: None, model: str, by_default: bool
+) -> None:
+    def reported_native(deployment_model_info: dict[str, object]) -> object:
+        return proxy_server._enrich_model_info_with_litellm_data(
+            {
+                "model_name": "claude",
+                "litellm_params": {"model": model},
+                "model_info": {"id": "claude-deployment", **deployment_model_info},
+            }
+        )["model_info"]["supports_native_structured_output"]
+
+    assert reported_native({"supports_bedrock_native_structured_output_by_default": by_default}) is reported_native(
+        {}
+    )
+
+
 def test_model_info_id_lookup_reports_the_same_cost_as_the_list(
     client: TestClient,
     auth_as: Callable[[], AbstractContextManager[object]],