scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

fix(bigmodel): keep context-window and content-policy errors on streamed Responses replays

Browse source
This commit is contained in:
mateo-berri 2026-10-28 13:20:56 -07:00
parent d1c71dca9e
commit 9fb6a00829
2 changed files with 60 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -3037,6 +3037,17 @@ class BaseLLMHTTPHandler:
},
)
replay_as_stream: Final = partial(
MockResponsesAPIStreamingIterator,
model=model,
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
request_data=request_context,
call_type=CallTypes.responses.value,
)
try:
if is_stream_request:
response = sync_httpx_client.post(
@ -3047,12 +3058,14 @@ class BaseLLMHTTPHandler:
**body_kwargs,
)
if caller_requested_stream:
if fake_stream is True or (
if fake_stream is True:
return replay_as_stream(response=response)
if not (
responses_api_provider_config.may_answer_stream_with_json()
and _is_sized_json_body(response)
and _is_json_document(response.read())
):
return MockResponsesAPIStreamingIterator(
return SyncResponsesAPIStreamingIterator(
response=response,
model=model,
logging_obj=logging_obj,
@ -3062,17 +3075,6 @@ class BaseLLMHTTPHandler:
request_data=request_context,
call_type=CallTypes.responses.value,
)
return SyncResponsesAPIStreamingIterator(
response=response,
model=model,
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
request_data=request_context,
call_type=CallTypes.responses.value,
)
response.read()
else:
response = sync_httpx_client.post(
@ -3087,6 +3089,9 @@ class BaseLLMHTTPHandler:
provider_config=responses_api_provider_config,
)
if is_stream_request and caller_requested_stream:
return replay_as_stream(response=response)
initial_response: Final = responses_api_provider_config.transform_response_api_response(
model=model,
raw_response=response,
@ -3242,6 +3247,17 @@ class BaseLLMHTTPHandler:
},
)
replay_as_stream: Final = partial(
MockResponsesAPIStreamingIterator,
model=model,
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
request_data=request_context,
call_type=CallTypes.responses.value,
)
try:
if is_stream_request:
response = await async_httpx_client.post(
@ -3254,12 +3270,14 @@ class BaseLLMHTTPHandler:
)
if caller_requested_stream:
if fake_stream is True or (
if fake_stream is True:
return replay_as_stream(response=response)
if not (
responses_api_provider_config.may_answer_stream_with_json()
and _is_sized_json_body(response)
and _is_json_document(await response.aread())
):
return MockResponsesAPIStreamingIterator(
return ResponsesAPIStreamingIterator(
response=response,
model=model,
logging_obj=logging_obj,
@ -3269,17 +3287,6 @@ class BaseLLMHTTPHandler:
request_data=request_context,
call_type=CallTypes.responses.value,
)
return ResponsesAPIStreamingIterator(
response=response,
model=model,
logging_obj=logging_obj,
responses_api_provider_config=responses_api_provider_config,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
request_data=request_context,
call_type=CallTypes.responses.value,
)
await response.aread()
else:
response = await async_httpx_client.post(
@ -3296,6 +3303,9 @@ class BaseLLMHTTPHandler:
provider_config=responses_api_provider_config,
)
if is_stream_request and caller_requested_stream:
return replay_as_stream(response=response)
initial_response: Final = responses_api_provider_config.transform_response_api_response(
model=model,
raw_response=response,

24
tests/unit/llms/bigmodel/test_bigmodel_provider.py
View file

@ -573,6 +573,30 @@ def test_prompt_too_long_and_unsafe_content_raise_the_errors_fallbacks_catch(
assert raised.value.status_code == 400
@pytest.mark.parametrize(
("code", "error_class"),
((1261, litellm.ContextWindowExceededError), (1301, litellm.ContentPolicyViolationError)),
)
@pytest.mark.parametrize("is_async", (False, True))
@pytest.mark.asyncio
@respx.mock
async def test_streamed_prompt_too_long_and_unsafe_content_raise_the_errors_fallbacks_catch(
code: int, error_class: type[Exception], is_async: bool
) -> None:
respx.post(f"{BASE}/api/v1/responses").respond(json={"code": code, "msg": "rejected by BigModel", "success": False})
params: Final = {"model": f"bigmodel/{MODEL}", "input": "Hello", "stream": True}
async def call() -> None:
if is_async:
await litellm.aresponses(**params)
return
litellm.responses(**params)
with pytest.raises(error_class, match="rejected by BigModel") as raised:
await call()
assert raised.value.status_code == 400
@pytest.mark.parametrize("route", ("responses", "messages"))
@pytest.mark.asyncio
@respx.mock