mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
fix(proxy): forward Bedrock event-stream content-type on unbuffered passthrough
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
ae92e511f1
commit
371fa670d6
2 changed files with 65 additions and 4 deletions
|
|
@ -1766,6 +1766,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
return StreamingResponse(
|
||||
content=generator, # type: ignore[arg-type]
|
||||
status_code=status.HTTP_200_OK,
|
||||
media_type=self._passthrough_event_stream_media_type(),
|
||||
headers=custom_headers,
|
||||
)
|
||||
else:
|
||||
|
|
@ -2216,10 +2217,15 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
def _passthrough_event_stream_media_type(self) -> Optional[str]:
|
||||
"""
|
||||
Content-type for a buffered passthrough event-stream response, resolved
|
||||
from the provider handler so the proxy stays provider-agnostic. Mirrors
|
||||
the upstream content-type the non-streaming path forwards, since the
|
||||
buffered streaming generator carries no headers of its own.
|
||||
Content-type for a passthrough event-stream response, resolved from the
|
||||
provider handler so the proxy stays provider-agnostic. Mirrors the
|
||||
upstream content-type the non-streaming path forwards, since the
|
||||
streaming generator carries no headers of its own. Used for both the
|
||||
buffered (guardrail-rewritten) and the unbuffered relay paths so
|
||||
clients that enforce the event-stream content-type (e.g. Claude Code on
|
||||
Bedrock invoke-with-response-stream) see the correct header instead of
|
||||
Starlette's application/octet-stream default. Returns None for providers
|
||||
with no event-stream media type, leaving the response default unchanged.
|
||||
"""
|
||||
from litellm.llms.pass_through.guardrail_translation.handler import (
|
||||
LlmPassthroughRouteHandler,
|
||||
|
|
|
|||
|
|
@ -4137,6 +4137,61 @@ class TestAllmPassthroughStreamingProviderGate:
|
|||
assert streamed == chunks
|
||||
mock_handler.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_invoke_stream_sets_event_stream_content_type(self, monkeypatch):
|
||||
"""
|
||||
Regression for LIT-4561. The unbuffered Bedrock event-stream relay
|
||||
(invoke-with-response-stream, no post-call guardrail rewriting) must set
|
||||
content-type: application/vnd.amazon.eventstream instead of leaving it to
|
||||
Starlette's application/octet-stream default, which trips Claude Code's
|
||||
content-type guard added in 2.1.208
|
||||
"""
|
||||
processing_obj = self._build_processing_obj(
|
||||
"bedrock", "model/us.anthropic.claude-sonnet-4-20250514-v1:0/invoke-with-response-stream"
|
||||
)
|
||||
chunks = [b"raw-1", b"raw-2"]
|
||||
|
||||
with patch.object(
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
"_has_post_call_guardrails",
|
||||
return_value=False,
|
||||
), patch.object(
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
"_has_post_call_guardrails_for_passthrough",
|
||||
return_value=False,
|
||||
):
|
||||
result = await self._run(processing_obj, monkeypatch, chunks)
|
||||
|
||||
assert isinstance(result, StreamingResponse)
|
||||
assert result.media_type == "application/vnd.amazon.eventstream"
|
||||
assert result.headers["content-type"] == "application/vnd.amazon.eventstream"
|
||||
streamed = [chunk async for chunk in result.body_iterator]
|
||||
assert streamed == chunks
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_bedrock_stream_keeps_default_content_type(self, monkeypatch):
|
||||
"""
|
||||
A provider with no registered event-stream media type must not have one
|
||||
forced onto its unbuffered stream, so the response default is unchanged
|
||||
"""
|
||||
processing_obj = self._build_processing_obj("anthropic")
|
||||
chunks = [b"chunk-1", b"chunk-2"]
|
||||
|
||||
with patch.object(
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
"_has_post_call_guardrails",
|
||||
return_value=False,
|
||||
), patch.object(
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
"_has_post_call_guardrails_for_passthrough",
|
||||
return_value=False,
|
||||
):
|
||||
result = await self._run(processing_obj, monkeypatch, chunks)
|
||||
|
||||
assert isinstance(result, StreamingResponse)
|
||||
assert result.media_type is None
|
||||
assert result.headers.get("content-type") != "application/vnd.amazon.eventstream"
|
||||
|
||||
|
||||
class TestResponseCostHeaderForTypedDictResponses:
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue