From 4a537e2c19da060321df8cf93a7d4268492a9f0f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:23:37 -0700 Subject: [PATCH] fix(proxy): emit SSE keepalives on queue, rag, azure passthrough, usage chat and policy enrich streams (#39273) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../policy_endpoints/endpoints.py | 15 +- .../usage_endpoints/ai_usage_chat.py | 4 +- .../usage_endpoints/endpoints.py | 19 ++- .../llm_passthrough_endpoints.py | 137 +++++++++++------- litellm/proxy/proxy_server.py | 33 +++-- litellm/proxy/rag_endpoints/endpoints.py | 88 ++++++----- .../policy_endpoints/test_endpoints.py | 68 +++++++++ .../usage_endpoints/test_ai_usage_chat.py | 58 ++++++++ .../test_llm_pass_through_endpoints.py | 90 ++++++++++++ .../proxy_server/test_streaming_helpers.py | 89 ++++++++++++ .../proxy/rag_endpoints/test_rag_endpoints.py | 95 +++++++++++- 11 files changed, 588 insertions(+), 108 deletions(-) diff --git a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py index f58f3722741..69356922ea1 100644 --- a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py @@ -12,7 +12,7 @@ All /policy management endpoints import copy import json import os -from collections.abc import AsyncIterator +from collections.abc import AsyncGenerator, AsyncIterator from typing import TYPE_CHECKING, Final, Literal, cast from fastapi import APIRouter, Depends, HTTPException, Request @@ -20,6 +20,7 @@ from fastapi.responses import Response, StreamingResponse from pydantic import BaseModel, Field from typing_extensions import TypedDict +import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ( COMPETITOR_LLM_TEMPERATURE, @@ -32,6 +33,10 @@ from litellm.llms.openai.chat.guardrail_translation.handler import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.sse_keepalive import ( + SSE_COMMENT_PING, + wrap_sse_stream_with_keepalive_pings, +) from litellm.proxy.guardrails.guardrail_hooks.custom_code import ( RESPONSE_REJECTION_GUARDRAIL_CODE, CustomCodeGuardrail, @@ -811,7 +816,7 @@ async def _stream_competitor_events( llm_enrichment: dict, brand_name: str, model: str, -) -> AsyncIterator[str]: +) -> AsyncGenerator[str, None]: """Stream competitor names as SSE events, then emit a final 'done' event.""" competitors: Final[list[str]] = list(data.competitors or []) @@ -883,7 +888,11 @@ async def enrich_policy_template_stream( model: Final = data.model or DEFAULT_COMPETITOR_DISCOVERY_MODEL return StreamingResponse( - _stream_competitor_events(data, template, llm_enrichment, brand_name, model), + wrap_sse_stream_with_keepalive_pings( + _stream_competitor_events(data, template, llm_enrichment, brand_name, model), + ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, + ping_chunk=SSE_COMMENT_PING, + ), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index 9d5ddda017a..7ef496f94e3 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -4,7 +4,7 @@ usage/spend data by querying the aggregated daily activity endpoints. """ import json -from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence from datetime import date from typing import Any, Final, Literal, Protocol, cast, overload @@ -543,7 +543,7 @@ async def stream_usage_ai_chat( model: str | None = None, user_id: str | None = None, is_admin: bool = False, -) -> AsyncIterator[str]: +) -> AsyncGenerator[str, None]: """Stream SSE events: status → tool_call → chunk → done.""" resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages diff --git a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py index b7b0ae2d8e5..d1e92d0c7e4 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py @@ -10,8 +10,13 @@ from fastapi import APIRouter, Depends, Request from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field +import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.sse_keepalive import ( + SSE_COMMENT_PING, + wrap_sse_stream_with_keepalive_pings, +) router: Final = APIRouter() @@ -56,11 +61,15 @@ async def usage_ai_chat( messages: Final = [{"role": m.role, "content": m.content} for m in data.messages] return StreamingResponse( - stream_usage_ai_chat( - messages=messages, - model=data.model, - user_id=user_id, - is_admin=is_admin, + wrap_sse_stream_with_keepalive_pings( + stream_usage_ai_chat( + messages=messages, + model=data.model, + user_id=user_id, + is_admin=is_admin, + ), + ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, + ping_chunk=SSE_COMMENT_PING, ), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 688123c9d41..6b1d6405a6a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -9,10 +9,11 @@ Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. from __future__ import annotations import hmac +import inspect import json import os import re -from collections.abc import Callable, Mapping +from collections.abc import AsyncGenerator, Callable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, cast @@ -32,6 +33,7 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.passthrough.main import AsyncPassthroughStreamingResponse from litellm.proxy._types import * from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks @@ -40,6 +42,7 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth, user_api_key_auth_websocket, ) +from litellm.proxy.common_request_processing import open_sse_before_first_byte from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_get_request_headers, @@ -47,6 +50,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( get_form_data, get_request_body, ) +from litellm.proxy.common_utils.sse_keepalive import ( + wrap_passthrough_sse_bytes_with_keepalive_pings, +) from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( HttpPassThroughEndpointHelpers, @@ -1478,6 +1484,74 @@ def is_azure_ai_search_service_level_index_create(method: str, endpoint: str) -> return path == "indexes" or path.endswith("/indexes") +async def _relay_upstream_bytes(upstream: AsyncGenerator[bytes, bytes]) -> AsyncGenerator[bytes, None]: + try: + async for chunk in upstream: + yield chunk + finally: + await upstream.aclose() + + +async def _relay_azure_router_model( + llm_router: litellm.Router, + model: str, + endpoint: str, + request: Request, + request_body: Mapping[str, object], + is_streaming_request: bool, + user_api_key_dict: UserAPIKeyAuth, +) -> Response: + result: Final = await llm_router.allm_passthrough_route( + model=model, + method=request.method, + endpoint=endpoint, + request_query_params=request.query_params, + request_headers=_safe_get_request_headers(request), + stream=is_streaming_request, + content=None, + data=None, + files=None, + json=(request_body if request.headers.get("content-type") == "application/json" else None), + params=None, + headers=None, + cookies=None, + litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict), + ) + + if not is_streaming_request: + upstream: Final = cast(httpx.Response, result) + return Response( + content=await upstream.aread(), + status_code=upstream.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers(headers=upstream.headers, custom_headers=None), + ) + + if inspect.isasyncgen(result): + sse_headers: Final = {"content-type": "text/event-stream"} + return StreamingResponse( + content=wrap_passthrough_sse_bytes_with_keepalive_pings( + stream=_relay_upstream_bytes(result), + ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, + upstream_headers=sse_headers, + ), + status_code=200, + headers=sse_headers, + ) + + upstream_stream: Final = cast(AsyncPassthroughStreamingResponse, result) + return StreamingResponse( + content=wrap_passthrough_sse_bytes_with_keepalive_pings( + stream=_relay_upstream_bytes(upstream_stream), + ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, + upstream_headers=upstream_stream.headers, + ), + status_code=upstream_stream.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=upstream_stream.headers, custom_headers=None + ), + ) + + @router.api_route( "/azure_ai/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], @@ -1528,55 +1602,18 @@ async def azure_proxy_route( if is_router_model: request_body = await get_request_body(request) is_streaming_request = is_passthrough_request_streaming(request_body) - result = await llm_router.allm_passthrough_route( - model=part, - method=request.method, - endpoint=endpoint, - request_query_params=request.query_params, - request_headers=_safe_get_request_headers(request), - stream=is_streaming_request, - content=None, - data=None, - files=None, - json=(request_body if request.headers.get("content-type") == "application/json" else None), - params=None, - headers=None, - cookies=None, - litellm_metadata=get_passthrough_router_request_metadata(user_api_key_dict), - ) - - if is_streaming_request: - # Check if result is an async generator (from _async_streaming) - import inspect - - if inspect.isasyncgen(result): - # Result is already an async generator, use it directly - return StreamingResponse( - content=result, - status_code=200, - headers={"content-type": "text/event-stream"}, - ) - else: - # Result is an httpx.Response, use aiter_bytes() - result = cast(httpx.Response, result) - return StreamingResponse( - content=result.aiter_bytes(), - status_code=result.status_code, - headers=HttpPassThroughEndpointHelpers.get_response_headers( - headers=result.headers, - custom_headers=None, - ), - ) - - # Non-streaming response - result = cast(httpx.Response, result) - content = await result.aread() - return Response( - content=content, - status_code=result.status_code, - headers=HttpPassThroughEndpointHelpers.get_response_headers( - headers=result.headers, - custom_headers=None, + return await open_sse_before_first_byte( + _relay_azure_router_model( + llm_router=llm_router, + model=part, + endpoint=endpoint, + request=request, + request_body=request_body, + is_streaming_request=is_streaming_request, + user_api_key_dict=user_api_key_dict, + ), + ping_interval_seconds=( + litellm.sse_keepalive_ping_interval_seconds if is_streaming_request else None ), ) elif is_vector_store_index: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 83f63c15529..1f39a78e12a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15238,20 +15238,33 @@ async def async_queue_request( if llm_router is None: raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) - - response: Final = await llm_router.schedule_acompletion(**data) + router: Final = llm_router if "stream" in data and data["stream"] is True: # use generate_responses to stream responses - return StreamingResponse( - async_data_generator( - user_api_key_dict=user_api_key_dict, - response=response, - request_data=data, - request=request, - ), - media_type="text/event-stream", + + async def produce_queue_stream() -> StreamingResponse: + return StreamingResponse( + async_data_generator( + user_api_key_dict=user_api_key_dict, + response=await router.schedule_acompletion(**data), + request_data=data, + request=request, + ), + media_type="text/event-stream", + ) + + async def audit_late_failure(exc: Exception) -> HTTPException | None: + return await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, original_exception=exc, request_data=data + ) + + return await open_sse_before_first_byte( + produce_queue_stream(), + ping_interval_seconds=ttft_keepalive_interval(data, router), + on_late_failure=audit_late_failure, ) + response: Final = await router.schedule_acompletion(**data) fastapi_response.headers.update({"x-litellm-priority": str(data["priority"])}) return response except Exception as e: diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index c8c6c505375..d6a402e1860 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -23,11 +23,14 @@ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( LiteLLM_ManagedVectorStore, ) -from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.proxy._types import * from litellm.proxy.auth.auth_utils import is_request_body_safe from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth -from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.common_request_processing import ( + ProxyBaseLLMRequestProcessing, + open_sse_before_first_byte, + ttft_keepalive_interval, +) from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_get_request_headers, @@ -48,6 +51,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, ) from litellm.repositories.table_repositories import ManagedVectorStoresRepository +from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -756,43 +760,53 @@ async def rag_query( merged_retrieval_config.get("custom_llm_provider"), ) - # Call query - response: Final = await litellm.aquery( - model=model, - messages=messages, - retrieval_config=merged_retrieval_config, - vector_store_params=store_data, - rerank=rerank, - stream=stream, - router=llm_router, - **request_data, - ) - - hidden_params: Final = getattr(response, "_hidden_params", {}) or {} - custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( - user_api_key_dict=user_api_key_dict, - call_id=hidden_params.get("litellm_call_id", None) or "", - model_id=hidden_params.get("model_id", None) or "", - cache_key=hidden_params.get("cache_key", None) or "", - api_base=hidden_params.get("api_base", None) or "", - version=version, - response_cost=hidden_params.get("response_cost", None), - request_data=request_data, - ) - - if isinstance(response, CustomStreamWrapper): - return StreamingResponse( - select_data_generator( - response=response, - user_api_key_dict=user_api_key_dict, - request_data=request_data, - request=request, - ), - media_type="text/event-stream", - headers=custom_headers, + async def query() -> ModelResponse: + return await litellm.aquery( + model=model, + messages=messages, + retrieval_config=merged_retrieval_config, + vector_store_params=store_data, + rerank=rerank, + stream=stream, + router=llm_router, + **request_data, ) - fastapi_response.headers.update(custom_headers) + def custom_headers_for(response: ModelResponse) -> Mapping[str, str]: + hidden_params: Final = getattr(response, "_hidden_params", {}) or {} + return ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=hidden_params.get("litellm_call_id", None) or "", + model_id=hidden_params.get("model_id", None) or "", + cache_key=hidden_params.get("cache_key", None) or "", + api_base=hidden_params.get("api_base", None) or "", + version=version, + response_cost=hidden_params.get("response_cost", None), + request_data=request_data, + ) + + if stream: + + async def produce_stream() -> StreamingResponse: + response: Final = await query() + return StreamingResponse( + select_data_generator( + response=response, + user_api_key_dict=user_api_key_dict, + request_data=request_data, + request=request, + ), + media_type="text/event-stream", + headers=custom_headers_for(response), + ) + + return await open_sse_before_first_byte( + produce_stream(), + ping_interval_seconds=ttft_keepalive_interval(data, llm_router), + ) + + response: Final = await query() + fastapi_response.headers.update(custom_headers_for(response)) return response except HTTPException: diff --git a/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_endpoints.py b/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_endpoints.py index 4e063dd0c5b..86f9aeafc08 100644 --- a/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/policy_endpoints/test_endpoints.py @@ -225,3 +225,71 @@ def test_compute_overall_action_all_passed(): def test_compute_overall_action_empty(): assert _compute_overall_action([]) == "passed" + + +class TestEnrichPolicyTemplateStreamKeepalive: + async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]: + import asyncio + from unittest.mock import MagicMock + + import litellm + import litellm.proxy.management_endpoints.policy_endpoints.endpoints as policy_endpoints + import litellm.proxy.proxy_server as proxy_server + from fastapi.responses import StreamingResponse + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.policy_endpoints.endpoints import ( + EnrichTemplateRequest, + enrich_policy_template_stream, + ) + + monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", interval) + + async def _name_chunks(): + await asyncio.sleep(delay) + chunk = MagicMock() + chunk.choices = [MagicMock()] + chunk.choices[0].delta.content = "Rival Air\n" + yield chunk + + class SlowRouter: + async def acompletion(self, **kwargs): + return _name_chunks() + + async def _no_variations(competitors, model): + return {} + + monkeypatch.setattr(proxy_server, "llm_router", SlowRouter()) + monkeypatch.setattr(policy_endpoints, "_generate_competitor_variations", _no_variations) + + response = await enrich_policy_template_stream( + data=EnrichTemplateRequest( + template_id="competitor-mention-detection", + parameters={"brand_name": "Acme"}, + model="gpt-5.4-mini", + ), + request=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert isinstance(response, StreamingResponse) + chunks = [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator] + return chunks, dict(response.headers) + + @pytest.mark.asyncio + async def test_endpoint_pings_while_competitor_discovery_is_still_running(self, monkeypatch): + chunks, headers = await self._collect_endpoint_body(monkeypatch, interval=0.05) + + assert headers["content-type"].startswith("text/event-stream") + assert headers["cache-control"] == "no-cache" + assert headers["x-accel-buffering"] == "no" + assert chunks[0] == b": ping\n\n" + assert chunks.count(b": ping\n\n") >= 3 + assert b'data: {"type": "competitor", "name": "Rival Air"}\n\n' in chunks + assert chunks[-1].startswith(b'data: {"type": "done"') + + @pytest.mark.asyncio + async def test_endpoint_stream_is_untouched_while_keepalives_are_unconfigured(self, monkeypatch): + chunks, _ = await self._collect_endpoint_body(monkeypatch, interval=None, delay=0.15) + + assert b": ping\n\n" not in chunks + assert chunks[0] == b'data: {"type": "competitor", "name": "Rival Air"}\n\n' + assert chunks[-1].startswith(b'data: {"type": "done"') diff --git a/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py index 3a32b3cc128..e5616a1975d 100644 --- a/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ b/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py @@ -466,3 +466,61 @@ class TestUsageAiChatServiceAccountGuard: is_admin=False, ) assert "Endpoint-level guard missing" in str(exc_info.value) + + +class TestUsageAiChatKeepalive: + async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]: + import asyncio + + import litellm + from fastapi.responses import StreamingResponse + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( + ChatMessage, + UsageAIChatRequest, + usage_ai_chat, + ) + + monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", interval) + + async def slow_acompletion(**kwargs): + await asyncio.sleep(delay) + response = MagicMock() + response.choices = [MagicMock()] + response.choices[0].message.tool_calls = None + response.choices[0].message.content = "Total spend is $50.25" + return response + + with patch( # test-quality-ok: the stream calls the module-level litellm.acompletion directly; no injection seam + "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", + new=AsyncMock(side_effect=slow_acompletion), + ): + response = await usage_ai_chat( + data=UsageAIChatRequest(messages=[ChatMessage(role="user", content="hi")], model="gpt-4o-mini"), + request=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert isinstance(response, StreamingResponse) + chunks = [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator] + return chunks, dict(response.headers) + + @pytest.mark.asyncio + async def test_endpoint_pings_while_the_planning_completion_is_still_running(self, monkeypatch): + chunks, headers = await self._collect_endpoint_body(monkeypatch, interval=0.05) + + assert headers["content-type"].startswith("text/event-stream") + assert headers["cache-control"] == "no-cache" + assert headers["x-accel-buffering"] == "no" + assert chunks[0].startswith(b'data: {"type": "status"') + assert chunks[1] == b": ping\n\n" + assert chunks.count(b": ping\n\n") >= 3 + assert b'"content": "Total spend is $50.25"' in b"".join(chunks) + assert chunks[-1] == b'data: {"type": "done"}\n\n' + + @pytest.mark.asyncio + async def test_endpoint_stream_is_untouched_while_keepalives_are_unconfigured(self, monkeypatch): + chunks, _ = await self._collect_endpoint_body(monkeypatch, interval=None, delay=0.15) + + assert b": ping\n\n" not in chunks + assert chunks[0].startswith(b'data: {"type": "status"') + assert chunks[-1] == b'data: {"type": "done"}\n\n' diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 5154f738e9a..acb45038df0 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -5116,3 +5116,93 @@ class TestAzureRouterModelStreamingDispatch: assert result.status_code == 200 body = b"".join([chunk async for chunk in result.body_iterator]) assert body == upstream_body + + +class TestAzureRouterModelStreamingKeepalive: + async def _dispatch(self, monkeypatch, interval, headers_delay=0.0, body_delay=0.0) -> StreamingResponse: + import asyncio + + import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.passthrough.main import AsyncPassthroughStreamingResponse + + monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", interval) + + class _StallingBody(httpx.AsyncByteStream): + async def __aiter__(self): + await asyncio.sleep(body_delay) + yield b"data: hello\n\n" + + async def _upstream_response() -> httpx.Response: + await asyncio.sleep(headers_delay) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream", "x-upstream": "kept"}, + stream=_StallingBody(), + request=httpx.Request("POST", "https://my-azure.openai.azure.com/openai/deployments/gpt-5/x"), + ) + + logging_obj = MagicMock() + logging_obj.async_flush_passthrough_collected_chunks = AsyncMock() + + class StreamingRouter: + async def allm_passthrough_route(self, **kwargs): + return await AsyncPassthroughStreamingResponse( + response=_upstream_response(), + litellm_logging_obj=logging_obj, + provider_config=MagicMock(), + ) + + async def fake_get_request_body(_request): + return {"model": "gpt-5", "stream": True} + + monkeypatch.setattr(proxy_server, "llm_router", StreamingRouter()) + monkeypatch.setattr(ep, "get_request_body", fake_get_request_body) + monkeypatch.setattr(ep, "is_passthrough_request_using_router_model", lambda *a, **k: True) + + request = MagicMock(spec=Request) + request.method = "POST" + request.headers = {"content-type": "application/json"} + request.query_params = {} + + result = await azure_proxy_route( + endpoint="openai/deployments/gpt-5/chat/completions", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"), + ) + assert isinstance(result, StreamingResponse) + return result + + @pytest.mark.asyncio + async def test_pings_while_upstream_headers_are_still_pending(self, monkeypatch): + result = await self._dispatch(monkeypatch, interval=0.05, headers_delay=0.3) + + chunks = [chunk async for chunk in result.body_iterator] + + assert result.status_code == 200 + assert result.headers["x-accel-buffering"] == "no" + assert chunks[0] == b": ping\n\n" + assert chunks.count(b": ping\n\n") >= 3 + assert b"".join(chunks).endswith(b"data: hello\n\n") + + @pytest.mark.asyncio + async def test_pings_while_upstream_body_is_still_pending(self, monkeypatch): + result = await self._dispatch(monkeypatch, interval=0.05, body_delay=0.3) + + chunks = [chunk async for chunk in result.body_iterator] + + assert result.status_code == 200 + assert result.headers["x-upstream"] == "kept" + assert chunks[0] == b": ping\n\n" + assert chunks.count(b": ping\n\n") >= 3 + assert chunks[-1] == b"data: hello\n\n" + + @pytest.mark.asyncio + async def test_relays_upstream_bytes_untouched_while_keepalives_are_unconfigured(self, monkeypatch): + result = await self._dispatch(monkeypatch, interval=None, headers_delay=0.15, body_delay=0.15) + + chunks = [chunk async for chunk in result.body_iterator] + + assert result.headers["x-upstream"] == "kept" + assert chunks == [b"data: hello\n\n"] diff --git a/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py b/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py index fdaad567f95..87e10ce7e8d 100644 --- a/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py @@ -1819,3 +1819,92 @@ async def test_run_thread_stream_is_untouched_while_keepalives_are_unconfigured( assert not any(chunk.startswith(": ping") for chunk in chunks) assert chunks[-1] == "data: [DONE]\n\n" + + +# --------------------------------------------------------------------------- +# async_queue_request: SSE keepalives during the time-to-first-token +# --------------------------------------------------------------------------- + + +async def _queue_streaming(monkeypatch, interval, delay=0.3, fails_with=None): + _patch_logging_flags(monkeypatch) + monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", interval) + + router = MagicMock() + router.get_model_list.return_value = [] + + async def _schedule_after_the_scheduler_queue_drains(**kwargs): + await asyncio.sleep(delay) + if fails_with is not None: + raise fails_with + return _async_iter([_simple_chunk(content="queued reply")]) + + router.schedule_acompletion = _schedule_after_the_scheduler_queue_drains + monkeypatch.setattr(ps, "llm_router", router) + + request = MagicMock() + request.url = "http://testserver/queue/chat/completions" + request.method = "POST" + request.headers = {} + request.json = AsyncMock( + return_value={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "priority": 0, + "stream": True, + } + ) + request.is_disconnected = AsyncMock(return_value=False) + + return await ps.async_queue_request( + request=request, + fastapi_response=Response(), + user_api_key_dict=_user_auth(), + ) + + +@pytest.mark.asyncio +async def test_queue_request_pings_while_the_scheduler_is_still_waiting(monkeypatch): + response = await _queue_streaming(monkeypatch, interval=0.05) + + assert isinstance(response, StreamingResponse) + assert response.headers["x-accel-buffering"] == "no" + chunks = [chunk async for chunk in response.body_iterator] + + assert chunks[0] == b": ping\n\n" + assert chunks.count(b": ping\n\n") >= 3 + assert b'"content":"queued reply"' in chunks[-2] + assert chunks[-1] == b"data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_queue_request_audits_a_failure_that_arrives_after_the_first_ping(monkeypatch): + audited = [] + + async def _record_failure(*, user_api_key_dict, original_exception, request_data, **kwargs): + audited.append(original_exception) + return None + + monkeypatch.setattr(ps.proxy_logging_obj, "post_call_failure_hook", _record_failure) + + boom = RuntimeError("scheduler died after the wire was already open") + response = await _queue_streaming(monkeypatch, interval=0.05, fails_with=boom) + + assert isinstance(response, StreamingResponse) + chunks = [chunk async for chunk in response.body_iterator] + + assert chunks[0] == b": ping\n\n" + assert audited == [boom] + assert json.loads(chunks[-2].removeprefix(b"data: "))["error"]["code"] == "500" + assert chunks[-1] == b"data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_queue_request_stream_is_untouched_while_keepalives_are_unconfigured(monkeypatch): + response = await _queue_streaming(monkeypatch, interval=None, delay=0.15) + + assert isinstance(response, StreamingResponse) + chunks = [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator] + + assert not any(chunk.startswith(b": ping") for chunk in chunks) + assert chunks[-1] == b"data: [DONE]\n\n" diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index a176e91eaa4..832435711c6 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -11,7 +11,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.proxy_server import app @@ -324,6 +323,100 @@ def test_rag_query_stream_returns_event_stream(client_internal_user): assert "data: [DONE]" in response.text +def test_rag_query_stream_pings_while_retrieval_is_still_running(client_internal_user, monkeypatch): + import asyncio + + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "sse_keepalive_ping_interval_seconds", 0.05) + + async def slow_aquery(**kwargs): + await asyncio.sleep(0.3) + return await litellm_module.acompletion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "What is the codename?"}], + mock_response="The codename is AZURE-FALCON-42.", + stream=True, + api_key="test-key", + ) + + with ( + patch( # test-quality-ok: the handler calls the module-level litellm.aquery directly; no injection seam + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new=AsyncMock(side_effect=slow_aquery), + ), + patch("litellm.vector_store_registry", None), # test-quality-ok: proxy module global, no injection seam + patch("litellm.proxy.proxy_server.prisma_client", None), # test-quality-ok: proxy module global, no injection seam + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "What is the codename?"}], + "retrieval_config": { + "vector_store_id": "vs_test_123", + "custom_llm_provider": "openai", + }, + "stream": True, + }, + ) + + assert response.status_code == 200, response.text + assert response.headers.get("content-type", "").startswith("text/event-stream") + assert response.headers["x-accel-buffering"] == "no" + assert response.text.startswith(": ping\n\n") + assert response.text.count(": ping\n\n") >= 3 + assert '"object":"chat.completion.chunk"' in response.text + assert response.text.endswith("data: [DONE]\n\n") + + +def test_rag_query_stream_keeps_response_headers_when_retrieval_beats_the_keepalive( + client_internal_user, monkeypatch +): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "sse_keepalive_ping_interval_seconds", 5) + + async def fast_aquery(**kwargs): + response = await litellm_module.acompletion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "What is the codename?"}], + mock_response="The codename is AZURE-FALCON-42.", + stream=True, + api_key="test-key", + ) + response._hidden_params["response_cost"] = 3.45e-06 + return response + + with ( + patch( # test-quality-ok: the handler calls the module-level litellm.aquery directly; no injection seam + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new=AsyncMock(side_effect=fast_aquery), + ), + patch("litellm.vector_store_registry", None), # test-quality-ok: proxy module global, no injection seam + patch("litellm.proxy.proxy_server.prisma_client", None), # test-quality-ok: proxy module global, no injection seam + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "What is the codename?"}], + "retrieval_config": { + "vector_store_id": "vs_test_123", + "custom_llm_provider": "openai", + }, + "stream": True, + }, + ) + + assert response.status_code == 200, response.text + assert response.headers.get("content-type", "").startswith("text/event-stream") + assert response.headers.get("x-litellm-response-cost") == "3.45e-06" + assert not response.text.startswith(": ping") + assert '"object":"chat.completion.chunk"' in response.text + assert response.text.endswith("data: [DONE]\n\n") + + def test_rag_query_merges_managed_store_params(client_internal_user): """ Regression: /v1/rag/query must consult the managed vector store registry