mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
fix(headroom): leave background responses requests uncompressed and unconverted
This commit is contained in:
parent
de643c028f
commit
45f44fea60
2 changed files with 48 additions and 1 deletions
|
|
@ -728,6 +728,10 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
verbose_proxy_logger.debug("Headroom: %s header set; skipping compression", BYPASS_HEADER)
|
||||
return inputs
|
||||
|
||||
if request_data.get("background"):
|
||||
verbose_proxy_logger.debug("Headroom: background request; skipping compression")
|
||||
return inputs
|
||||
|
||||
structured_messages: Final = inputs.get("structured_messages")
|
||||
if not _is_object_list(structured_messages) or not structured_messages:
|
||||
return inputs
|
||||
|
|
@ -831,7 +835,7 @@ class HeadroomGuardrail(CustomGuardrail):
|
|||
effective: Final = base_result if base_result is not None else kwargs
|
||||
if call_type not in _STREAM_CONVERTIBLE_CALL_TYPES:
|
||||
return base_result
|
||||
if not effective.get("stream"):
|
||||
if not effective.get("stream") or effective.get("background"):
|
||||
return base_result
|
||||
if not has_headroom_retrieve_tool(effective.get("tools")):
|
||||
return base_result
|
||||
|
|
|
|||
|
|
@ -193,6 +193,33 @@ async def test_apply_guardrail_compresses_and_returns_structured_messages(
|
|||
assert "headroom" in _applied_guardrails(request_data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_guardrail_leaves_background_requests_uncompressed(
|
||||
guardrail: HeadroomGuardrail,
|
||||
):
|
||||
inputs = GenericGuardrailAPIInputs(
|
||||
texts=["A" * 5000],
|
||||
structured_messages=ORIGINAL_MESSAGES,
|
||||
)
|
||||
request_data = {"model": "gpt-4o", "background": True}
|
||||
|
||||
with patch.object(
|
||||
guardrail.async_handler,
|
||||
"post",
|
||||
new_callable=AsyncMock,
|
||||
return_value=_make_compress_response(COMPRESSED_MESSAGES),
|
||||
) as post:
|
||||
result = await guardrail.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=request_data,
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert result is inputs
|
||||
post.assert_not_awaited()
|
||||
assert _recorded_guardrail_entries(request_data) == []
|
||||
|
||||
|
||||
def _recorded_guardrail_response(request_data: dict) -> dict:
|
||||
entries = request_data["metadata"]["standard_logging_guardrail_information"]
|
||||
assert len(entries) == 1
|
||||
|
|
@ -2074,6 +2101,22 @@ async def test_pre_call_deployment_hook_converts_stream_only_for_ccr_chat_comple
|
|||
assert kwargs["stream"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_deployment_hook_leaves_background_streams_alone(guardrail: HeadroomGuardrail):
|
||||
kwargs = {
|
||||
"model": "gpt-4o",
|
||||
"stream": True,
|
||||
"background": True,
|
||||
"tools": [_responses_retrieve_tool_definition()],
|
||||
}
|
||||
|
||||
result = await guardrail.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.aresponses)
|
||||
|
||||
assert result is kwargs
|
||||
assert HEADROOM_CONVERTED_STREAM_KEY not in kwargs
|
||||
assert kwargs["stream"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_deployment_hook_still_compresses_for_deployment_level_configs(
|
||||
guardrail: HeadroomGuardrail,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue