fix(proxy): forward Bedrock event-stream content-type on unbuffered passthrough

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-07-17 19:33:28 +00:00
parent ae92e511f1
commit 371fa670d6
2 changed files with 65 additions and 4 deletions

View file

@ -1766,6 +1766,7 @@ class ProxyBaseLLMRequestProcessing:
return StreamingResponse(
content=generator, # type: ignore[arg-type]
status_code=status.HTTP_200_OK,
media_type=self._passthrough_event_stream_media_type(),
headers=custom_headers,
)
else:
@ -2216,10 +2217,15 @@ class ProxyBaseLLMRequestProcessing:
def _passthrough_event_stream_media_type(self) -> Optional[str]:
"""
Content-type for a buffered passthrough event-stream response, resolved
from the provider handler so the proxy stays provider-agnostic. Mirrors
the upstream content-type the non-streaming path forwards, since the
buffered streaming generator carries no headers of its own.
Content-type for a passthrough event-stream response, resolved from the
provider handler so the proxy stays provider-agnostic. Mirrors the
upstream content-type the non-streaming path forwards, since the
streaming generator carries no headers of its own. Used for both the
buffered (guardrail-rewritten) and the unbuffered relay paths so
clients that enforce the event-stream content-type (e.g. Claude Code on
Bedrock invoke-with-response-stream) see the correct header instead of
Starlette's application/octet-stream default. Returns None for providers
with no event-stream media type, leaving the response default unchanged.
"""
from litellm.llms.pass_through.guardrail_translation.handler import (
LlmPassthroughRouteHandler,

View file

@ -4137,6 +4137,61 @@ class TestAllmPassthroughStreamingProviderGate:
assert streamed == chunks
mock_handler.assert_not_awaited()
@pytest.mark.asyncio
async def test_bedrock_invoke_stream_sets_event_stream_content_type(self, monkeypatch):
"""
Regression for LIT-4561. The unbuffered Bedrock event-stream relay
(invoke-with-response-stream, no post-call guardrail rewriting) must set
content-type: application/vnd.amazon.eventstream instead of leaving it to
Starlette's application/octet-stream default, which trips Claude Code's
content-type guard added in 2.1.208
"""
processing_obj = self._build_processing_obj(
"bedrock", "model/us.anthropic.claude-sonnet-4-20250514-v1:0/invoke-with-response-stream"
)
chunks = [b"raw-1", b"raw-2"]
with patch.object(
ProxyBaseLLMRequestProcessing,
"_has_post_call_guardrails",
return_value=False,
), patch.object(
ProxyBaseLLMRequestProcessing,
"_has_post_call_guardrails_for_passthrough",
return_value=False,
):
result = await self._run(processing_obj, monkeypatch, chunks)
assert isinstance(result, StreamingResponse)
assert result.media_type == "application/vnd.amazon.eventstream"
assert result.headers["content-type"] == "application/vnd.amazon.eventstream"
streamed = [chunk async for chunk in result.body_iterator]
assert streamed == chunks
@pytest.mark.asyncio
async def test_non_bedrock_stream_keeps_default_content_type(self, monkeypatch):
"""
A provider with no registered event-stream media type must not have one
forced onto its unbuffered stream, so the response default is unchanged
"""
processing_obj = self._build_processing_obj("anthropic")
chunks = [b"chunk-1", b"chunk-2"]
with patch.object(
ProxyBaseLLMRequestProcessing,
"_has_post_call_guardrails",
return_value=False,
), patch.object(
ProxyBaseLLMRequestProcessing,
"_has_post_call_guardrails_for_passthrough",
return_value=False,
):
result = await self._run(processing_obj, monkeypatch, chunks)
assert isinstance(result, StreamingResponse)
assert result.media_type is None
assert result.headers.get("content-type") != "application/vnd.amazon.eventstream"
class TestResponseCostHeaderForTypedDictResponses:
"""