mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
fix(headroom): fake-stream converted sync /v1/responses calls too
The sync response_api_handler agentic branch returned the completed ResponsesAPIResponse for a request the Headroom guardrail had converted from streaming, so litellm.responses(stream=True) handed callers a non-iterable object. Wrap it in the same fake stream the async path uses
This commit is contained in:
parent
201ada9982
commit
1695b7f7b1
2 changed files with 64 additions and 2 deletions
|
|
@ -2744,6 +2744,7 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
|
||||
if self._has_agentic_completion_hook(logging_obj):
|
||||
agentic_kwargs: Final = dict(litellm_params)
|
||||
final_response: Final = run_async_function(
|
||||
self._call_agentic_completion_hooks,
|
||||
response=initial_response,
|
||||
|
|
@ -2754,10 +2755,19 @@ class BaseLLMHTTPHandler:
|
|||
logging_obj=logging_obj,
|
||||
stream=False,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=dict(litellm_params),
|
||||
kwargs=agentic_kwargs,
|
||||
api_surface="responses",
|
||||
)
|
||||
return final_response if final_response is not None else initial_response
|
||||
result: Final = final_response if final_response is not None else initial_response
|
||||
if converted_stream_requested(agentic_kwargs) and not agentic_kwargs.get("_agentic_loop_depth"):
|
||||
return self._wrap_responses_response_as_fake_stream(
|
||||
result=result,
|
||||
model=model,
|
||||
responses_api_provider_config=responses_api_provider_config,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
return result
|
||||
|
||||
return initial_response
|
||||
|
||||
|
|
|
|||
|
|
@ -2241,6 +2241,58 @@ async def test_streaming_responses_resolves_ccr_retrieval_end_to_end(
|
|||
assert not any(key.startswith("_headroom_interception") for key in followup_body)
|
||||
|
||||
|
||||
def test_sync_streaming_responses_resolves_ccr_retrieval_end_to_end(
|
||||
guardrail: HeadroomGuardrail,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""The synchronous responses() path converts the stream the same way, so it
|
||||
must hand back a stream iterator with the resolved answer rather than the
|
||||
completed response object."""
|
||||
original_content = "the full uncompressed document"
|
||||
final_answer = "the document says hello"
|
||||
guardrail._issued_hashes_by_call_id["ccr-call-id"] = (
|
||||
frozenset({CCR_HASH}),
|
||||
time.monotonic() + 999,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail])
|
||||
upstream = respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
side_effect=[
|
||||
httpx.Response(200, json=_openai_responses_retrieve_call_payload()),
|
||||
httpx.Response(200, json=_openai_responses_text_payload(final_answer)),
|
||||
]
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
guardrail.async_handler,
|
||||
"get",
|
||||
new_callable=AsyncMock,
|
||||
return_value=_make_retrieve_response(original_content),
|
||||
) as mock_get:
|
||||
response = litellm.responses(
|
||||
model="openai/gpt-4o",
|
||||
input=[{"role": "user", "content": f"summarize hash={CCR_HASH}"}],
|
||||
tools=[_responses_retrieve_tool_definition()],
|
||||
stream=True,
|
||||
litellm_call_id="ccr-call-id",
|
||||
)
|
||||
events = list(response)
|
||||
|
||||
streamed_text = "".join(
|
||||
getattr(event, "delta", "") for event in events if getattr(event, "type", None) == "response.output_text.delta"
|
||||
)
|
||||
assert streamed_text == final_answer
|
||||
assert not any(
|
||||
getattr(getattr(event, "item", None), "type", None) == "function_call" for event in events
|
||||
)
|
||||
mock_get.assert_called_once()
|
||||
assert len(upstream.calls) == 2
|
||||
assert not json.loads(upstream.calls[1].request.content).get("stream")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LIT-5018: the turn the model is being asked to act on is never compressed.
|
||||
#
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue