From 580b9266a11d8db9646a6151c6f04e4f6e2f5d13 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 31 Oct 2026 14:43:24 -0700 Subject: [PATCH 1/4] feat(chat): resume replies after browser disconnects (#49461) * feat(chat): resume background MCP runs after browser disconnects * feat(chat): background ordinary text conversations * feat(chat): configure background replies from Settings * fix(chat): address background response review findings * test(sdk): finish the HTTP smoke stream explicitly * fix(chat): preserve background response timing metrics * test(router): terminate the scripted completion stream --- litellm/proxy/proxy_server.py | 51 ++-- .../response_polling/background_streaming.py | 36 ++- .../proxy/response_polling/polling_handler.py | 18 +- .../proxy_setting_endpoints.py | 95 ++++++++ .../model_management_endpoints.py | 3 + tests/base_sdk_tests/check_sdk_http.py | 4 +- .../test_response_polling_handler.py | 68 ++++++ tests/unit/proxy/test_proxy_server.py | 85 +++++++ .../test_proxy_setting_endpoints.py | 111 +++++++++ tests/unit/test_router/core/test_endpoints.py | 1 + ui/litellm-dashboard/eslint-suppressions.json | 5 - .../chat/page.background.integration.test.tsx | 229 ++++++++++++++++++ .../src/app/chat/page.integration.test.tsx | 2 +- ui/litellm-dashboard/src/app/chat/page.tsx | 87 ++++++- ...ackgroundChatSettings.integration.test.tsx | 98 ++++++++ .../UISettings/BackgroundChatSettings.tsx | 73 ++++++ .../UISettings/UISettings.test.tsx | 2 + .../AdminSettings/UISettings/UISettings.tsx | 5 +- .../src/components/chat/ChatShell.tsx | 3 - .../components/chat/backgroundChat.test.ts | 177 ++++++++++++++ .../src/components/chat/backgroundChat.ts | 138 +++++++++++ .../src/components/chat/runChatRequest.ts | 11 + .../src/components/chat/types.ts | 9 + .../src/components/chat/useBackgroundChat.ts | 62 +++++ .../src/components/llm_calls/fetch_models.tsx | 3 + .../components/llm_calls/responses_api.tsx | 17 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 130 ++++++++++ 27 files changed, 1480 insertions(+), 43 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx create mode 100644 ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts create mode 100644 ui/litellm-dashboard/src/components/chat/backgroundChat.ts create mode 100644 ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b5f4a36f138..d8a57d3e9a1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3070,6 +3070,29 @@ redis_usage_cache: RedisCache | None = None # redis cache used for tracking spe polling_via_cache_enabled: Literal["all"] | list[str] | bool = False native_background_mode: list[str] = [] # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache + + +class _ResponseBackgroundSettings(BaseModel): + model_config = ConfigDict(frozen=True) + + polling_via_cache: Literal["all"] | tuple[str, ...] | bool = False + native_background_mode: tuple[str, ...] = () + ttl: int = 3600 + + +def _apply_background_response_settings(value: SettingsJsonValue) -> None: + global polling_via_cache_enabled, native_background_mode, polling_cache_ttl + + responses: Final = _as_settings_mapping(value) + settings: Final = _ResponseBackgroundSettings.model_validate(responses.get("background_mode") or {}) + polling: Final = settings.polling_via_cache + polling_via_cache_enabled, native_background_mode, polling_cache_ttl = ( # rebind-ok: Refresh runtime config. + list(polling) if isinstance(polling, tuple) else polling, + list(settings.native_background_mode), + settings.ttl, + ) + + user_custom_auth = None user_custom_key_generate = None # Sentinel: prevents PKCE-no-Redis advisory from re-logging on config hot-reload. @@ -6932,20 +6955,7 @@ class ProxyConfig: # see usage here: https://docs.litellm.ai/docs/proxy/caching pass elif key == "responses": - # Initialize global polling via cache settings - global polling_via_cache_enabled, native_background_mode, polling_cache_ttl - background_mode = value.get("background_mode", {}) - polling_via_cache_enabled = background_mode.get("polling_via_cache", False) - native_background_mode = background_mode.get("native_background_mode", []) - polling_cache_ttl = background_mode.get("ttl", 3600) - verbose_proxy_logger.debug( - "%s Initialized polling via cache: enabled=%s, native_background_mode=%s, ttl=%s%s", - blue_color_code, - polling_via_cache_enabled, - native_background_mode, - polling_cache_ttl, - reset_color_code, - ) + _apply_background_response_settings(value) elif key == "max_ui_session_budget": litellm.max_ui_session_budget = float(value) if value is not None else None verbose_proxy_logger.debug( @@ -8653,6 +8663,7 @@ class ProxyConfig: def _apply_litellm_settings_db_values(self, db_values: Mapping[str, SettingsJsonValue]) -> None: self.litellm_settings.apply_db_row("litellm_settings", db_values) + _apply_background_response_settings(self.litellm_settings.get("responses", {})) for key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES: if key in db_values and (value := self.litellm_settings.get(key)) is not None: setattr(litellm, key, value) @@ -18110,6 +18121,8 @@ async def model_deprecations( def get_model_group_info( llm_router: Router, all_models_str: Sequence[str], model_group: str | None, team_id: str | None = None ) -> list[ModelGroupInfoProxy]: + from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request + model_groups: Final[list[ModelGroupInfoProxy]] = [] unique_models: Final = tuple(dict.fromkeys(all_models_str)) @@ -18119,6 +18132,14 @@ def get_model_group_info( if model_group is not None and model_group != model: continue + supports_background_chat = should_use_polling_for_request( + background_mode=True, + polling_via_cache_enabled=polling_via_cache_enabled, + redis_cache=redis_usage_cache, + model=model, + llm_router=llm_router, + native_background_mode=native_background_mode, + ) _model_group_info = registry.group(model, team_id=team_id) if _model_group_info is not None: @@ -18126,6 +18147,7 @@ def get_model_group_info( ModelGroupInfoProxy.model_validate( { **_model_group_info.model_dump(), + "supports_background_chat": supports_background_chat, **model_group_custom_fields( registry.deployments( resolve_model_group_alias(registry.aliases(), model) or model, @@ -18140,6 +18162,7 @@ def get_model_group_info( model_group_info = ModelGroupInfoProxy( model_group=model, providers=[], + supports_background_chat=supports_background_chat, ) model_groups.append(model_group_info) diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 1eb76f8f558..574152048d4 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming import asyncio import json from collections.abc import AsyncIterable, Callable, Mapping, Sequence +from time import perf_counter from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable from fastapi import Request, Response @@ -128,6 +129,16 @@ async def background_streaming_task( https://platform.openai.com/docs/api-reference/responses/object """ + started_at: Final = perf_counter() + first_token_at: float | None = None # rebind-ok: first content event closes the TTFT interval + finished_at: float | None = None # rebind-ok: terminal event closes the execution interval + + def timing_snapshot() -> Mapping[str, float]: + return { + **({"time_to_first_token_ms": (first_token_at - started_at) * 1000} if first_token_at is not None else {}), + **({"total_latency_ms": (finished_at - started_at) * 1000} if finished_at is not None else {}), + } + try: verbose_proxy_logger.info("Starting background streaming for %s", polling_id) @@ -191,7 +202,7 @@ async def background_streaming_task( incomplete_details_data = None state_dirty = False # Track if state needs to be synced - last_update_time = asyncio.get_event_loop().time() + last_update_time = perf_counter() UPDATE_INTERVAL: Final = 0.150 # 150ms batching interval # Track the terminal event from the stream (may not be "completed") @@ -210,13 +221,14 @@ async def background_streaming_task( """Flush accumulated state to Redis if interval elapsed or forced""" nonlocal state_dirty, last_update_time - current_time: Final = asyncio.get_event_loop().time() + current_time: Final = perf_counter() if state_dirty and (force or (current_time - last_update_time) >= UPDATE_INTERVAL): # Convert output_items dict to list for update output_list: Final = list(output_items.values()) await polling_handler.update_state( polling_id=polling_id, output=output_list, + timing=timing_snapshot(), ) state_dirty = False last_update_time = current_time @@ -242,6 +254,20 @@ async def background_streaming_task( event: _StreamEvent = _StreamEventParser.parse(chunk_data) event_type = event.get("type", "") + if ( + first_token_at is None + and event_type + in ( + "response.output_text.delta", + "response.reasoning_text.delta", + "response.reasoning_summary_text.delta", + "response.reasoning.delta", + ) + and event.get("delta") + ): + first_token_at = perf_counter() + state_dirty = True + # Process different event types based on OpenAI streaming spec if event_type == "response.output_item.added": # New output item added @@ -334,6 +360,7 @@ async def background_streaming_task( ): # Terminal event - extract all ResponsesAPIResponse fields # https://platform.openai.com/docs/api-reference/responses-streaming + finished_at = perf_counter() response_data = event.get("response", {}) terminal_status = response_data.get( "status", @@ -384,10 +411,13 @@ async def background_streaming_task( # Use the terminal status from the stream, default to "completed" final_status: Final = terminal_status or "completed" + if finished_at is None: + finished_at = perf_counter() # rebind-ok: close the interval when the stream has no terminal event await polling_handler.update_state( polling_id=polling_id, status=final_status, + timing=timing_snapshot(), usage=usage_data, error=terminal_error, reasoning=reasoning_data, @@ -417,6 +447,7 @@ async def background_streaming_task( ) except Exception as e: + finished_at = perf_counter() # rebind-ok: capture execution time at failure verbose_proxy_logger.error("Error in background streaming task for %s: %s", polling_id, e) import traceback @@ -425,6 +456,7 @@ async def background_streaming_task( await polling_handler.update_state( polling_id=polling_id, status="failed", + timing=timing_snapshot(), error={ "type": "internal_error", "message": str(e), diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index fe224a52cfa..d66773cd048 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -3,9 +3,9 @@ Response Polling Handler for Background Responses with Cache """ import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime, timezone -from typing import Final +from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter @@ -15,6 +15,9 @@ from litellm._uuid import uuid4 from litellm.caching.redis_cache import RedisCache from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus +if TYPE_CHECKING: + from litellm.router import Router + _RESPONSE_POLLING_TARGET: Final = "response_polling" _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _JSON_LIST: Final = TypeAdapter(list[object]) @@ -116,6 +119,7 @@ class ResponsePollingHandler: parallel_tool_calls: bool | None = None, user: str | None = None, store: bool | None = None, + timing: Mapping[str, float] | None = None, ) -> None: """ Update the polling state in Redis @@ -144,6 +148,7 @@ class ResponsePollingHandler: parallel_tool_calls: Whether parallel tool calls are enabled user: User identifier store: Whether to store the response + timing: Server execution timings in milliseconds """ if not self.redis_cache: return @@ -167,6 +172,9 @@ class ResponsePollingHandler: if output is not None: state["output"] = output + if timing is not None: + state["litellm_response_timing"] = timing + # Update usage if usage: state["usage"] = usage @@ -264,10 +272,10 @@ class ResponsePollingHandler: def should_use_polling_for_request( background_mode: bool, - polling_via_cache_enabled, # Can be False, "all", or List[str] - redis_cache, # RedisCache or None + polling_via_cache_enabled: bool | str | Sequence[str], + redis_cache: RedisCache | None, model: str, - llm_router, # Router instance or None + llm_router: "Router | None", native_background_mode: list[str] | None = None, # List of models that should use native background mode ) -> bool: """ diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 0efd4e4c619..e5348684d78 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -216,6 +216,20 @@ class _SettingsWithSchema(LiteLLMBaseModel): field_schema: dict[str, object] +class BackgroundChatSettings(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + enabled: bool = Field(description="Enable cache-backed background Responses for Chat and API requests.") + + +class BackgroundChatSettingsResponse(BackgroundChatSettings): + model_config = ConfigDict(extra="forbid", frozen=True) + + redis_configured: bool = Field(description="Whether this proxy has coordination Redis configured.") + managed_by_config: bool = Field(description="Whether the responses configuration is owned by the config file.") + can_save: bool = Field(description="Whether database-backed configuration updates are available.") + + class SSOSettingsResponse(SettingsResponse): """Response model for SSO settings""" @@ -1815,6 +1829,87 @@ async def sync_ui_settings_to_general_settings( return apply_runtime_general_settings_flags(parsed) +def _background_chat_settings_response(config: Mapping[str, object]) -> BackgroundChatSettingsResponse: + from litellm.proxy import proxy_server + + settings: Final = _as_settings_section(config.get("litellm_settings")) + responses: Final = _as_settings_section(settings.get("responses")) + background: Final = _as_settings_section(responses.get("background_mode")) + polling: Final = background.get("polling_via_cache", False) + return BackgroundChatSettingsResponse( + enabled=bool(polling), + redis_configured=proxy_server.redis_usage_cache is not None, + managed_by_config=proxy_server.proxy_config.litellm_settings.owned_by_config("responses"), + can_save=proxy_server.prisma_client is not None and proxy_server.store_model_in_db is True, + ) + + +@router.get( + "/get/background_chat_settings", + tags=["Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=BackgroundChatSettingsResponse, +) +async def get_background_chat_settings() -> BackgroundChatSettingsResponse: + """Get saved background-response settings and the prerequisites for changing them.""" + from litellm.proxy.proxy_server import proxy_config + + config: Final = _as_settings_section(_DECODED_JSON.validate_python(await proxy_config.get_config())) + return _background_chat_settings_response(config) + + +@router.patch( + "/update/background_chat_settings", + tags=["Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=BackgroundChatSettingsResponse, +) +async def update_background_chat_settings( + settings: BackgroundChatSettings, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> BackgroundChatSettingsResponse: + """Save background-response polling settings for the proxy configuration refresh.""" + from litellm.proxy.proxy_server import ( + create_config_audit_log, + prisma_client, + proxy_config, + redis_usage_cache, + store_model_in_db, + ) + + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException(status_code=403, detail="Only proxy admins can update background chat settings.") + if prisma_client is None or store_model_in_db is not True: + raise HTTPException(status_code=400, detail="Enable database-backed configuration to save this setting.") + if settings.enabled and redis_usage_cache is None: + raise HTTPException(status_code=400, detail="Configure coordination Redis and restart the proxy first.") + + raw_config: Final = await proxy_config.get_config() + config: Final = _as_settings_section(_DECODED_JSON.validate_python(raw_config)) + litellm_settings: Final = _as_settings_section(config.get("litellm_settings")) + responses: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(litellm_settings.get("responses"))) + background: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(responses.get("background_mode"))) + updated_responses: Final[Mapping[str, JsonValue]] = { + **responses, + "background_mode": {**background, "polling_via_cache": "all" if settings.enabled else False}, + } + proxy_config.reject_config_owned_writes( + section_name="litellm_settings", changed_keys={"responses": dict(updated_responses)} + ) + raw_config["litellm_settings"] = {**litellm_settings, "responses": dict(updated_responses)} + await proxy_config.save_config(new_config=config) + asyncio.create_task( + create_config_audit_log( + param_name="responses.background_mode.polling_via_cache", + action="updated", + before_value=background.get("polling_via_cache", False), + after_value="all" if settings.enabled else False, + user_api_key_dict=user_api_key_dict, + ) + ) + return _background_chat_settings_response(config) + + @router.get( "/get/ui_settings", tags=["UI Settings"], diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py index f032bb20c2a..b12745679df 100644 --- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py @@ -35,6 +35,9 @@ class PatchModelRequest(updateDeployment): class ModelGroupInfoProxy(ModelGroupInfo): + supports_background_chat: bool = Field( + default=False, description="Whether cache-backed background Responses runs are enabled for this model group" + ) is_public_model_group: bool = Field(default=False) health_status: str | None = Field(default=None) health_response_time: float | None = Field(default=None) diff --git a/tests/base_sdk_tests/check_sdk_http.py b/tests/base_sdk_tests/check_sdk_http.py index f7a1e30ac99..16ee4824f2d 100644 --- a/tests/base_sdk_tests/check_sdk_http.py +++ b/tests/base_sdk_tests/check_sdk_http.py @@ -70,7 +70,9 @@ def enqueue(body: object, status: int = 200) -> None: def enqueue_stream() -> None: chunk: Final = {**CHAT, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "pong"}}]} - RESPONSES.put((200, f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode())) + terminal: Final = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]} + stream: Final = f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(terminal)}\n\ndata: [DONE]\n\n" + RESPONSES.put((200, stream.encode())) def bedrock_event(event: str, payload: object) -> bytes: diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 087b273548a..d6df7a6c69a 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1917,3 +1917,71 @@ async def test_create_initial_state_rejects_request_metadata_that_is_not_an_obje assert [(error["loc"], error["type"], repr(error["input"])) for error in rejected.value.errors()] == [ (("metadata",), "dict_type", repr(metadata)) ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "delta_type", + [ + "response.output_text.delta", + "response.reasoning_text.delta", + "response.reasoning_summary_text.delta", + "response.reasoning.delta", + None, + ], +) +async def test_background_timings_survive_retrieval_after_completion(delta_type): + from litellm.caching.redis_cache import RedisCache + from litellm.proxy.response_polling.background_streaming import background_streaming_task + + cache = {} + redis = AsyncMock(spec=RedisCache) + redis.async_get_cache.side_effect = lambda key: cache.get(key) + + async def save_cache(key, value, **kwargs): + cache[key] = value + + redis.async_set_cache.side_effect = save_cache + handler = ResponsePollingHandler(redis_cache=redis) + await handler.create_initial_state("litellm_poll_timing", {"model": "gpt-4o"}) + clock = [100.0] + + async def stream(): + clock[0] = 101.0 + yield 'data: {"type":"response.output_text.delta","delta":""}\n\n' + yield 'data: {"type":"response.mcp_call.in_progress"}\n\n' + if delta_type: + clock[0] = 102.0 + yield f"data: {json.dumps({'type': delta_type, 'delta': 'Hello'})}\n\n" + partial = await handler.get_state("litellm_poll_timing") + assert partial is not None + assert partial["status"] == "in_progress" + assert partial["litellm_response_timing"] == {"time_to_first_token_ms": 2000.0} + clock[0] = 110.0 + yield 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}\n\n' + clock[0] = 5000.0 + yield "data: [DONE]\n\n" + + from starlette.responses import StreamingResponse + + with ( + patch( + "litellm.proxy.response_polling.background_streaming.perf_counter", + create=True, + side_effect=lambda: clock[0], + ), + patch( + # test-quality-ok: processor is constructed inside the task; supply the upstream SSE boundary like sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as processor, + ): + processor.return_value.base_process_llm_request = AsyncMock(return_value=StreamingResponse(stream())) + await background_streaming_task(**_make_background_streaming_kwargs("litellm_poll_timing", handler)) + state = await handler.get_state("litellm_poll_timing") + assert state is not None + expected = {"total_latency_ms": 10000.0} + if delta_type: + expected["time_to_first_token_ms"] = 2000.0 + assert state["litellm_response_timing"] == expected + clock[0] = 10000.0 + assert (await handler.get_state("litellm_poll_timing"))["litellm_response_timing"] == expected diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index 6e494f0b24b..e4af36d99f2 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -4561,3 +4561,88 @@ async def test_text_completion_provider_rejection_captures_unknown_usage( assert envelope["usage"] is None assert envelope["usage_source"] == "unknown" assert envelope["pricing"]["reported_cost"] is None + + +@pytest.mark.parametrize( + "polling,has_redis,native,expected", + [ + (False, True, False, False), + ("all", False, False, False), + ("all", True, True, False), + ("all", True, False, True), + (["openai"], True, False, True), + (["anthropic"], True, False, False), + ], +) +def test_model_groups_advertise_cache_backed_background_chat( + monkeypatch: pytest.MonkeyPatch, + polling: bool | str | list[str], + has_redis: bool, + native: bool, + expected: bool, +) -> None: + from litellm.caching.redis_cache import RedisCache + from litellm.proxy import proxy_server + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "research-chat", + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "fake-test-key"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", polling) + monkeypatch.setattr( + proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379) if has_redis else None + ) + monkeypatch.setattr(proxy_server, "native_background_mode", ["research-chat"] if native else []) + groups: Final = proxy_server.get_model_group_info(router, ["research-chat"], None) + assert groups[0].supports_background_chat is expected + + +@pytest.mark.parametrize("config_owned", [False, True]) +def test_background_response_settings_refresh_and_respect_config( + monkeypatch: pytest.MonkeyPatch, + config_owned: bool, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False) + monkeypatch.setattr(proxy_server, "native_background_mode", []) + monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600) + config: Final = proxy_server.ProxyConfig() + if config_owned: + config.litellm_settings.load_yaml( + { + "responses": { + "background_mode": { + "polling_via_cache": ["openai"], + "native_background_mode": ["config-native"], + "ttl": 900, + } + }, + } + ) + config._apply_litellm_settings_db_values( + { + "responses": { + "background_mode": { + "polling_via_cache": "all", + "native_background_mode": ["db-native"], + "ttl": 7200, + } + }, + } + ) + assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else "all") + assert proxy_server.native_background_mode == (["config-native"] if config_owned else ["db-native"]) + assert proxy_server.polling_cache_ttl == (900 if config_owned else 7200) + config._apply_litellm_settings_db_values( + { + "responses": {"background_mode": {"polling_via_cache": False}}, + } + ) + assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else False) + assert proxy_server.native_background_mode == (["config-native"] if config_owned else []) + assert proxy_server.polling_cache_ttl == (900 if config_owned else 3600) diff --git a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 4360655c8ca..ea39d5d59e2 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -4488,3 +4488,114 @@ def test_usage_default_range_reports_config_precedence(mock_auth, monkeypatch: p assert response.status_code == 200, response.text assert response.json()["values"]["usage_default_time_range"] == "month_to_date" assert response.json()["source"]["usage_default_time_range"] == "config" + + +@pytest.fixture +def background_chat_config(monkeypatch: pytest.MonkeyPatch): + from copy import deepcopy + from unittest.mock import AsyncMock, MagicMock + + from litellm.caching.redis_cache import RedisCache + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.config_resolvers import SettingsStore + + config = { + "litellm_settings": { + "callbacks": ["prometheus"], + "responses": { + "background_mode": { + "polling_via_cache": False, + "ttl": 7200, + "native_background_mode": ["native-model"], + }, + }, + } + } + store = SettingsStore("litellm_settings") + monkeypatch.setattr(proxy_server.proxy_config, "litellm_settings", store) + monkeypatch.setattr(proxy_server.proxy_config, "_settings_stores", {"litellm_settings": store}) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379)) + monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False) + monkeypatch.setattr(proxy_server, "native_background_mode", []) + monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600) + monkeypatch.setattr(proxy_server.proxy_config, "get_config", AsyncMock(side_effect=lambda: deepcopy(config))) + + async def save_config(*, new_config): + config.clear() + config.update(deepcopy(new_config)) + + save = AsyncMock(side_effect=save_config) + audit = AsyncMock() + monkeypatch.setattr(proxy_server.proxy_config, "save_config", save) + monkeypatch.setattr(proxy_server, "create_config_audit_log", audit) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + yield config, save, audit + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_background_chat_settings_persist_preserving_existing_options(background_chat_config, monkeypatch): + from litellm.proxy import proxy_server + + config, _, _ = background_chat_config + response = client.patch("/update/background_chat_settings", json={"enabled": True}) + assert response.status_code == 200 + assert response.json()["enabled"] is True + assert proxy_server.polling_via_cache_enabled is False + assert config["litellm_settings"]["responses"]["background_mode"] == { + "polling_via_cache": "all", + "ttl": 7200, + "native_background_mode": ["native-model"], + } + assert config["litellm_settings"]["callbacks"] == ["prometheus"] + assert client.get("/get/background_chat_settings").json()["enabled"] is True + proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"]) + assert proxy_server.polling_via_cache_enabled == "all" + disabled = client.patch("/update/background_chat_settings", json={"enabled": False}) + assert disabled.status_code == 200 + assert disabled.json()["enabled"] is False + proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"]) + assert proxy_server.polling_via_cache_enabled is False + + +@pytest.mark.parametrize("blocker", ["role", "redis", "config", "persistence"]) +def test_background_chat_settings_reject_unavailable_updates(background_chat_config, monkeypatch, blocker): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + config, save, _ = background_chat_config + if blocker == "role": + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="viewer", user_role=LitellmUserRoles.INTERNAL_USER + ) + elif blocker == "redis": + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + assert client.get("/get/background_chat_settings").json()["redis_configured"] is False + elif blocker == "config": + proxy_server.proxy_config.litellm_settings.load_yaml(config["litellm_settings"]) + assert client.get("/get/background_chat_settings").json()["managed_by_config"] is True + else: + monkeypatch.setattr(proxy_server, "store_model_in_db", False) + assert client.get("/get/background_chat_settings").json()["can_save"] is False + response = client.patch("/update/background_chat_settings", json={"enabled": True}) + assert response.status_code == (403 if blocker == "role" else 400) + save.assert_not_awaited() + assert config["litellm_settings"]["responses"]["background_mode"]["polling_via_cache"] is False + + +def test_background_chat_settings_failed_save_does_not_enable_runtime(background_chat_config): + from litellm.proxy import proxy_server + + _, save, audit = background_chat_config + save.side_effect = RuntimeError("Database write failed") + with pytest.raises(RuntimeError, match="Database write failed"): + client.patch("/update/background_chat_settings", json={"enabled": True}) + assert client.get("/get/background_chat_settings").json()["enabled"] is False + assert proxy_server.polling_via_cache_enabled is False + audit.assert_not_awaited() diff --git a/tests/unit/test_router/core/test_endpoints.py b/tests/unit/test_router/core/test_endpoints.py index a3cffe8278a..b2ad602c4c6 100644 --- a/tests/unit/test_router/core/test_endpoints.py +++ b/tests/unit/test_router/core/test_endpoints.py @@ -414,6 +414,7 @@ def _scripted_stream(*texts: str) -> Callable[..., CustomStreamWrapper]: async def chunks() -> AsyncIterator[ModelResponseStream]: for text in texts: yield ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}]) + yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}]) logging_obj: Final = LiteLLMLogging( model="d1", diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index c14f3e6f036..a4e0d59d73d 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1142,11 +1142,6 @@ "count": 1 } }, - "src/app/chat/page.tsx": { - "no-nested-ternary": { - "count": 1 - } - }, "src/app/login/LoginPage.tsx": { "local/no-complex-jsx-arrow": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx new file mode 100644 index 00000000000..53be87ce30c --- /dev/null +++ b/ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx @@ -0,0 +1,229 @@ +import { act, fireEvent, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { ChatShellProvider } from "@/contexts/ChatShellContext"; +import type { Conversation } from "@/components/chat/types"; +import ChatConversationPage from "./page"; + +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn(), replace: vi.fn() }) })); + +const storageKey = "litellm_chat_history_v1:background-user"; +const requests: Record[] = []; +let supported: boolean; +let status: "in_progress" | "completed" | "cancelled"; +let retrievalStatus: number; +let cancellations: number; +let acceptance: Promise | undefined; +const saved = (): Conversation[] => JSON.parse(localStorage.getItem(storageKey) ?? "[]"); +const result = () => ({ + id: "litellm_poll_research", + status, + usage: status === "completed" ? { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost: "0.125" } : undefined, + litellm_response_timing: + status === "completed" ? { time_to_first_token_ms: 250, total_latency_ms: 12500 } : undefined, + output: + status === "completed" + ? [ + { + type: "mcp_call", + id: "call_inventory", + name: "lookup_inventory", + arguments: "{}", + output: "12 widgets", + server_label: "inventory", + }, + { type: "message", content: [{ type: "output_text", text: "There are 12 widgets in stock." }] }, + ].filter((item) => item.type !== "mcp_call" || Boolean(requests.at(-1)?.tools)) + : [], +}); + +function renderChat(id?: string) { + return renderWithProviders( + + + , + { searchParams: id ? { id } : {} }, + ); +} + +beforeEach(() => { + localStorage.clear(); + testQueryClient.clear(); + requests.length = 0; + cancellations = 0; + acceptance = undefined; + supported = true; + retrievalStatus = 200; + status = "in_progress"; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const request = + input instanceof Request ? input : new Request(new URL(String(input), window.location.origin), init); + const path = new URL(request.url).pathname; + if (path.endsWith("/model_group/info")) + return Response.json({ + data: [{ model_group: "research-chat", mode: "chat", supports_background_chat: supported }], + }); + if (path.endsWith("/v1/mcp/server")) return Response.json([{ server_id: "inventory", server_name: "inventory" }]); + if (path.endsWith("/mcp-rest/tools/list")) return Response.json({ tools: [{ name: "lookup_inventory" }] }); + if (path.endsWith("/responses")) { + const body = JSON.parse(await request.text()); + requests.push(body); + if (body.background) { + await acceptance; + request.signal.throwIfAborted(); + return Response.json({ ...result(), status: "queued" }); + } + return new Response( + 'data: {"type":"response.output_text.delta","delta":"Foreground answer"}\n\ndata: {"type":"response.completed","response":{"id":"resp_foreground","output":[]}}\n\n', + { headers: { "Content-Type": "text/event-stream" } }, + ); + } + if (path.endsWith("/litellm_poll_research/cancel")) { + cancellations++; + status = "cancelled"; + return Response.json(result()); + } + if (path.endsWith("/litellm_poll_research")) + return Response.json(retrievalStatus === 200 ? result() : { error: { message: "Unavailable" } }, { + status: retrievalStatus, + }); + return Response.json({}, { status: 404 }); + }), + ); +}); +afterEach(() => vi.unstubAllGlobals()); + +async function startResearch(withTools = true) { + const user = userEvent.setup(); + await screen.findByRole("button", { name: "research-chat" }); + if (withTools) { + await user.click(screen.getByRole("button", { name: "Connect tools" })); + await user.click(await screen.findByRole("switch")); + await user.keyboard("{Escape}"); + } + fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), { + target: { value: "Check inventory" }, + }); + await user.click(screen.getByRole("button", { name: "Send", exact: true })); +} + +it("should retrieve the completed tool run after closing and reopening without submitting it again", async () => { + const first = renderChat(); + await startResearch(); + await screen.findByText("Running in the background. You can close this tab and return to this chat."); + await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research")); + const id = saved()[0].id; + first.unmount(); + expect(cancellations).toBe(0); + status = "completed"; + renderChat(id); + expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible(); + expect(screen.getByRole("button", { name: "lookup_inventory" })).toBeVisible(); + expect(screen.getByLabelText(/^Cost:/)).toHaveTextContent("$0.125"); + expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s"); + expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s"); + await waitFor(() => + expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }), + ); + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + background: true, + stream: false, + tools: [{ type: "mcp", server_label: "inventory" }], + }); + expect(screen.queryByRole("button", { name: "Stop request" })).not.toBeInTheDocument(); + fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), { + target: { value: "What was the result?" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Send", exact: true })); + await waitFor(() => expect(requests).toHaveLength(2)); + expect(requests[1]).toMatchObject({ background: true, stream: false }); + expect(requests[1]).not.toHaveProperty("previous_response_id"); + expect(requests[1].input).toEqual( + expect.arrayContaining([{ type: "function_call_output", call_id: "call_inventory", output: "12 widgets" }]), + ); +}); + +it("should continue an ordinary chat without MCP tools after closing and reopening", async () => { + const first = renderChat(); + await startResearch(false); + await screen.findByText("Running in the background. You can close this tab and return to this chat."); + await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research")); + expect(requests[0]).toMatchObject({ background: true, stream: false }); + expect(requests[0]).not.toHaveProperty("tools"); + const id = saved()[0].id; + first.unmount(); + status = "completed"; + renderChat(id); + expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible(); + expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s"); + expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s"); + await waitFor(() => + expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }), + ); + expect(requests).toHaveLength(1); + expect(cancellations).toBe(0); +}); + +it("should cancel a background run only when Stop is clicked", async () => { + renderChat(); + await startResearch(); + await screen.findByText("Running in the background. You can close this tab and return to this chat."); + fireEvent.click(screen.getByRole("button", { name: "Stop request" })); + expect(await screen.findByText("Cancellation requested")).toBeVisible(); + expect(cancellations).toBe(1); +}); + +it("should retain ordinary streaming when background support is unavailable", async () => { + supported = false; + renderChat(); + await startResearch(); + expect(await screen.findByText("Foreground answer")).toBeVisible(); + expect(requests[0]).toMatchObject({ stream: true }); + expect(requests[0]).not.toHaveProperty("background"); + expect(screen.getByText("Keep this tab open. Background runs are not available for this chat.")).toBeVisible(); +}); + +it("should let the user reconnect after an authentication failure without resubmitting work", async () => { + const first = renderChat(); + await startResearch(); + await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research")); + const id = saved()[0].id; + first.unmount(); + retrievalStatus = 401; + renderChat(id); + await screen.findByRole("button", { name: "Reconnect" }); + retrievalStatus = 200; + status = "completed"; + fireEvent.click(screen.getByRole("button", { name: "Reconnect" })); + expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible(); + expect(requests).toHaveLength(1); +}); + +it.each([false, true])( + "should honor Stop before a background submission returns its ID (tools=%s)", + async (withTools) => { + let accept!: () => void; + acceptance = new Promise((resolve) => { + accept = resolve; + }); + renderChat(); + await startResearch(withTools); + await waitFor(() => expect(requests).toHaveLength(1)); + fireEvent.click(screen.getByRole("button", { name: "Stop request" })); + await act(async () => accept()); + expect(await screen.findByText("Cancellation requested")).toBeVisible(); + expect(cancellations).toBe(1); + expect(requests).toHaveLength(1); + await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.status).toBe("cancelled")); + }, +); diff --git a/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx index 6b333750b78..5ea5db56920 100644 --- a/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx @@ -64,7 +64,7 @@ vi.mock("@/contexts/ChatShellContext", () => ({ }, })); -const ONE_TURN_ARG_COUNT = 25; +const ONE_TURN_ARG_COUNT = 28; const ON_TIMING_DATA_INDEX = 7; const ON_USAGE_DATA_INDEX = 8; const ON_TOTAL_LATENCY_INDEX = 24; diff --git a/ui/litellm-dashboard/src/app/chat/page.tsx b/ui/litellm-dashboard/src/app/chat/page.tsx index 85fc7133215..757bf010fd9 100644 --- a/ui/litellm-dashboard/src/app/chat/page.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.tsx @@ -15,6 +15,7 @@ import type { AssistantMessageUpdate, ChatAttachment, ChatMessage } from "@/comp import ChatModelPicker, { isChatModel, modelChatMode, type ChatMode } from "@/components/chat/ChatModelPicker"; import MoyaiChatComposer from "@/components/chat/moyai/MoyaiChatComposer"; import MoyaiChatHome from "@/components/chat/moyai/MoyaiChatHome"; +import { useBackgroundChat } from "@/components/chat/useBackgroundChat"; import { runChatRequest } from "@/components/chat/runChatRequest"; import { readChatAttachment, MAX_CHAT_ATTACHMENTS, CHAT_ATTACHMENT_ACCEPT } from "@/components/chat/attachmentHelpers"; import ChatFileChip from "@/components/chat/ChatFileChip"; @@ -102,7 +103,30 @@ export default function ChatConversationPage() { const responsesSessionGenerationRef = useRef(0); const previousResponsesConversationRef = useRef(activeConversationId); const [prevConversationIdForSessionReset, setPrevConversationIdForSessionReset] = useState(activeConversationId); - const [isStreaming, setIsStreaming] = useState(false); + const [isRequestInFlight, setIsStreaming] = useState(false); + const [cancelOnAcceptance, setCancelOnAcceptance] = useState(null); + const backgroundRequestRef = useRef(false); + const lastMessage = activeConversation?.messages.at(-1); + const background = useBackgroundChat( + activeConversationId, + lastMessage?.backgroundRun, + accessToken, + updateLastAssistantMessage, + ); + const { pending: backgroundPending, cancelling: backgroundCancelling, cancel: cancelBackground } = background; + useEffect(() => { + if (!backgroundPending || backgroundCancelling || !activeConversationId) return; + if (cancelOnAcceptance !== activeConversationId) return; + void cancelBackground().then(() => { + setCancelOnAcceptance((current) => (current === activeConversationId ? null : current)); + }); + }, [activeConversationId, backgroundPending, backgroundCancelling, cancelBackground, cancelOnAcceptance]); + + const isStreaming = isRequestInFlight || background.pending; + const supportsBackground = models.some( + (model) => model.model_group === selectedModel && model.supports_background_chat, + ); + const useBackground = mode === "chat" && supportsBackground && !storageUnavailable; const [inputText, setInputText] = useState(""); const [mcpPopoverOpen, setMcpPopoverOpen] = useState(false); const [storageBannerDismissed, setStorageBannerDismissed] = useState(false); @@ -264,6 +288,7 @@ export default function ChatConversationPage() { setAttachmentError(null); setIsStreaming(true); + setCancelOnAcceptance(null); abortControllerRef.current = new AbortController(); requestConversationRef.current = convId; @@ -297,12 +322,14 @@ export default function ChatConversationPage() { signal: requestController.signal, messages: history, previousResponseId, + background: requestMode === "chat" && useBackground, mcpServers: selectedMCPServers, onResponseId: (id: string | null) => { if (responsesSessionGenerationRef.current === sessionGeneration) setResponsesSessionId(id); }, onUpdate: (updates: AssistantMessageUpdate) => updateLastAssistantMessage(convId, updates), }; + backgroundRequestRef.current = request.background; try { await runChatRequest(request); } finally { @@ -326,13 +353,22 @@ export default function ChatConversationPage() { updateLastAssistantMessage, isStreaming, responsesSessionId, + useBackground, setConversationIdInUrl, ], ); - const handleStop = useCallback(() => { + const handleStop = () => { + if (background.pending) { + void background.cancel(); + return; + } + if (backgroundRequestRef.current) { + setCancelOnAcceptance(activeConversationId); + return; + } abortControllerRef.current?.abort(); - }, []); + }; const handleEditAndResend = useCallback( (messageId: string, newContent: string) => { @@ -403,7 +439,21 @@ export default function ChatConversationPage() { const showBlankState = !activeConversation || activeConversation.messages.length === 0; + const backgroundHint = () => { + if (background.cancelling || (isRequestInFlight && cancelOnAcceptance === activeConversationId)) + return "Requesting cancellation…"; + if (background.pending && storageUnavailable) + return "Keep this tab open. This run could not be saved in this browser."; + if (background.pending) return "Running in the background. You can close this tab and return to this chat."; + if (useBackground) return null; + return "Keep this tab open. Background runs are not available for this chat."; + }; + const inputBar = (inConversation: boolean) => { + const hint = mode === "chat" || background.pending ? backgroundHint() : null; + const chatPlaceholder = inConversation + ? "Send a message..." + : "Describe a change, paste an error, or ask a question…"; const preparing = isLoadingModels || isReadingAttachments; const submitDisabled = !hasMessageContent(inputText, attachments, mode) || !selectedModel || preparing; return ( @@ -424,13 +474,7 @@ export default function ChatConversationPage() { onChange={setInputText} onSubmit={() => void handleSend(inputText)} onCancel={handleStop} - placeholder={ - mode === "image" - ? "Describe the image you want to create…" - : inConversation - ? "Send a message..." - : "Describe a change, paste an error, or ask a question…" - } + placeholder={mode === "image" ? "Describe the image you want to create…" : chatPlaceholder} isLoading={isStreaming} submitLabel={mode === "image" ? "Generate image" : "Send"} submitDisabled={submitDisabled} @@ -536,6 +580,29 @@ export default function ChatConversationPage() { : "No chat models are available for your account."}

)} + {background.connectionError && ( +
+ {background.connectionError} + +
+ )} + {lastMessage?.backgroundError && ( +

+ {lastMessage.backgroundError} +

+ )} + {lastMessage?.backgroundRun?.status === "cancelled" && ( +

+ Cancellation requested +

+ )} + {hint && ( +

+ {hint} +

+ )}

{mode === "image" ? "Describe an image · Enter to generate" : "Drop files or paste images · Enter to send"}

diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx new file mode 100644 index 00000000000..311094909ce --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx @@ -0,0 +1,98 @@ +import { fireEvent, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import BackgroundChatSettings from "./BackgroundChatSettings"; + +let settings: { + enabled: boolean; + redis_configured: boolean; + managed_by_config: boolean; + can_save: boolean; +}; +let failLoad: boolean; +let failSave: boolean; +const updates: unknown[] = []; + +beforeEach(() => { + testQueryClient.clear(); + settings = { + enabled: false, + redis_configured: true, + managed_by_config: false, + can_save: true, + }; + failLoad = false; + failSave = false; + updates.length = 0; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const request = + input instanceof Request ? input : new Request(new URL(String(input), window.location.origin), init); + if (request.method === "PATCH") { + const body = await request.json(); + updates.push(body); + if (failSave) return Response.json({ detail: "Save failed" }, { status: 500 }); + settings = { ...settings, enabled: body.enabled }; + } else if (failLoad) return Response.json({ detail: "Unavailable" }, { status: 503 }); + return Response.json(settings); + }), + ); +}); +afterEach(() => vi.unstubAllGlobals()); + +it("saves background execution next to its explanation and retains it after reopening", async () => { + const first = renderWithProviders(); + fireEvent.click(await screen.findByRole("switch", { name: "Continue chats in the background" })); + await waitFor(() => expect(screen.getByRole("switch")).toBeChecked()); + expect(updates).toEqual([{ enabled: true }]); + expect(screen.getByRole("switch")).toBeChecked(); + first.unmount(); + testQueryClient.clear(); + renderWithProviders(); + expect(await screen.findByRole("switch")).toBeChecked(); + fireEvent.click(screen.getByRole("switch")); + await waitFor(() => expect(screen.getByRole("switch")).not.toBeChecked()); + expect(updates).toEqual([{ enabled: true }, { enabled: false }]); +}); + +it("links to Redis setup and prevents enabling without Redis", async () => { + settings.redis_configured = false; + renderWithProviders(); + expect(await screen.findByRole("switch")).toHaveAttribute("aria-disabled", "true"); + expect(screen.getByRole("link", { name: /Configure it in Caching/ })).toHaveAttribute( + "href", + expect.stringMatching(/\/caching$/), + ); + expect(updates).toHaveLength(0); +}); + +it.each(["managed_by_config", "can_save"] as const)( + "explains a setting that cannot be edited because of %s", + async (field) => { + settings[field] = field === "managed_by_config"; + renderWithProviders(); + expect(await screen.findByRole("switch")).toHaveAttribute("aria-disabled", "true"); + const message = + field === "managed_by_config" ? /Managed by your config file/ : /Enable database-backed configuration/; + expect(screen.getByText(message)).toBeVisible(); + }, +); + +it("keeps a failed save off", async () => { + failSave = true; + renderWithProviders(); + fireEvent.click(await screen.findByRole("switch")); + await waitFor(() => expect(updates).toHaveLength(1)); + expect(screen.getByRole("switch")).not.toBeChecked(); +}); + +it("offers retry for a settings fetch failure instead of claiming Redis is missing", async () => { + failLoad = true; + renderWithProviders(); + await screen.findByRole("button", { name: "Retry" }); + expect(screen.queryByRole("link")).not.toBeInTheDocument(); + failLoad = false; + fireEvent.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByRole("switch")).not.toHaveAttribute("aria-disabled", "true"); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx new file mode 100644 index 00000000000..428edfff86d --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx @@ -0,0 +1,73 @@ +import { $api } from "@/lib/http/api"; +import { toast } from "@/lib/toast"; +import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Switch } from "@/components/ui/switch"; +import { uiHref } from "@/utils/uiHref"; + +export default function BackgroundChatSettings() { + const query = $api.useQuery("get", "/get/background_chat_settings"); + const update = $api.useMutation("patch", "/update/background_chat_settings", { + onSuccess: () => { + void query.refetch(); + toast.success("Background chat settings saved."); + }, + onError: (error) => toast.fromError(error), + }); + + if (query.isPending) return ; + if (query.isError || !query.data) + return ( +
+ Could not load background chat settings. + +
+ ); + + const settings = query.data; + const readOnly = settings.managed_by_config || !settings.can_save; + const needsRedis = !settings.redis_configured && !settings.enabled; + const disabled = update.isPending || readOnly || needsRedis; + + return ( +
+ update.mutate({ body: { enabled } })} + aria-label="Continue chats in the background" + aria-describedby="background-chat-description" + /> +
+

Continue chats in the background

+

+ Keep replies and tool runs going after users close the browser. Return to the same browser to see the result. +

+ {!settings.redis_configured && ( +

+ Requires Redis.{" "} + + Configure it in Caching → Coordination Redis + + , then restart the proxy. +

+ )} + {settings.managed_by_config && ( +

+ Managed by your config file. Update its background polling setting to change this. +

+ )} + {!settings.can_save && !settings.managed_by_config && ( +

Enable database-backed configuration to save this setting.

+ )} + {!readOnly && ( +

+ Changes apply automatically when the proxy refreshes its configuration. +

+ )} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx index 2aa5c738880..c2bfd6076d4 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx @@ -5,6 +5,8 @@ import userEvent from "@testing-library/user-event"; import UISettings from "./UISettings"; import { toast } from "@/lib/toast"; +vi.mock("./BackgroundChatSettings", () => ({ default: () => null })); + const mockUseAuthorized = vi.hoisted(() => vi.fn()); const mockUseUISettings = vi.hoisted(() => vi.fn()); const mockUseUpdateUISettings = vi.hoisted(() => vi.fn()); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx index 3103851f280..08f345af928 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx @@ -16,6 +16,7 @@ import { relativeTimeOptions } from "@/components/shared/date_range_presets"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Label } from "@/components/ui/label"; import PageVisibilitySettings from "./PageVisibilitySettings"; +import BackgroundChatSettings from "./BackgroundChatSettings"; interface SettingRowProps { ariaLabel: string; @@ -429,13 +430,15 @@ export default function UISettings() { disabled={isUpdating} onCheckedChange={handleToggleEnableChatUI} ariaLabel={enableChatUIProperty?.description ?? "Enable Chat page"} - label="[BETA] Enable Chat page (page will refresh)" + label="Enable Chat page (page will refresh)" description={ enableChatUIProperty?.description ?? "If enabled, shows the Chat page in the UI sidebar, letting users chat with an LLM and connect their own MCP server credentials via OAuth." } /> + +
diff --git a/ui/litellm-dashboard/src/components/chat/ChatShell.tsx b/ui/litellm-dashboard/src/components/chat/ChatShell.tsx index 280e2d10b8c..03d91bb13bc 100644 --- a/ui/litellm-dashboard/src/components/chat/ChatShell.tsx +++ b/ui/litellm-dashboard/src/components/chat/ChatShell.tsx @@ -192,9 +192,6 @@ const ChatShell: React.FC<{ children: React.ReactNode }> = ({ children }) => { ))} -

- Chat is a pre-v0 preview and may change unexpectedly. Do not use in production. -

diff --git a/ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts b/ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts new file mode 100644 index 00000000000..fd3d51883ff --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts @@ -0,0 +1,177 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { Response as ModelResponse } from "openai/resources/responses/responses"; +import { backgroundResponseUpdate, cancelBackgroundChat, watchBackgroundChat } from "./backgroundChat"; + +vi.mock("@/components/networking", () => ({ getGatewayBaseUrl: () => "http://localhost:4000/v1" })); + +afterEach(() => vi.unstubAllGlobals()); + +it("should restore text, reasoning, tool results and usage from a completed response", () => { + const response = { + id: "litellm_poll_example", + status: "completed", + output: [ + { type: "reasoning", summary: [{ type: "summary_text", text: "Check the inventory" }] }, + { + type: "mcp_call", + id: "call_lookup", + name: "lookup", + arguments: "{}", + output: "12 widgets", + server_label: "inventory", + }, + { type: "message", content: [{ type: "output_text", text: "There are 12 widgets." }] }, + ], + usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, output_tokens_details: { reasoning_tokens: 2 } }, + } as ModelResponse; + const update = backgroundResponseUpdate(response); + expect(update.content).toBe("There are 12 widgets."); + expect(update.reasoningContent).toBe("Check the inventory"); + expect(update.mcpEvents?.[0].item?.output).toBe("12 widgets"); + const usage = { promptTokens: 4, completionTokens: 9, totalTokens: 13, reasoningTokens: 2 }; + expect(update.usage).toMatchObject(usage); + expect(update.backgroundRun).toEqual({ id: response.id, status: "completed" }); +}); + +describe("background response lifecycle", () => { + it("should detach polling without cancelling the server run", async () => { + const controller = new AbortController(); + const fetch = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) => + Response.json({ id: "litellm_poll_example", status: "in_progress", output: [] }), + ); + vi.stubGlobal("fetch", fetch); + const onUpdate = vi.fn(() => controller.abort()); + const polling = { + accessToken: "test-token", + responseId: "litellm_poll_example", + signal: controller.signal, + onUpdate, + }; + await watchBackgroundChat(polling); + expect(onUpdate).toHaveBeenCalledOnce(); + expect(fetch).toHaveBeenCalledOnce(); + expect(fetch.mock.calls[0][1]?.method).toBe("GET"); + }); + + it("should mark an expired run unavailable without clearing its partial answer", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ error: { message: "Expired" } }, { status: 404 })), + ); + const onUpdate = vi.fn(); + const polling = { + accessToken: "test-token", + responseId: "litellm_poll_expired", + signal: new AbortController().signal, + onUpdate, + }; + await watchBackgroundChat(polling); + expect(onUpdate).toHaveBeenCalledWith({ + backgroundRun: { id: "litellm_poll_expired", status: "expired" }, + backgroundError: "This saved run is no longer available. Its result may have expired on the server.", + }); + }); + + it("should cancel explicitly through the server and preserve returned partial output", async () => { + const fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const request = input instanceof Request ? input : new Request(input, init); + expect(request.method).toBe("POST"); + expect(new URL(request.url).pathname).toBe("/v1/responses/litellm_poll_example/cancel"); + return Response.json({ + id: "litellm_poll_example", + status: "cancelled", + output: [{ type: "message", content: [{ type: "output_text", text: "Partial answer" }] }], + }); + }); + vi.stubGlobal("fetch", fetch); + expect(await cancelBackgroundChat("test-token", "litellm_poll_example")).toMatchObject({ + content: "Partial answer", + backgroundRun: { id: "litellm_poll_example", status: "cancelled" }, + }); + }); +}); + +it.each([ + [0.125, 0.125], + ["0.125", 0.125], + [0, 0], + ["invalid", undefined], +])("should preserve valid returned cost %s", (cost, expected) => { + const response = { + id: "litellm_poll_cost", + status: "completed", + output: [], + usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost }, + } as ModelResponse; + expect(backgroundResponseUpdate(response).usage?.cost).toBe(expected); +}); + +it("should keep polling through unfinished output items until the answer and tools arrive", async () => { + let polls = 0; + vi.stubGlobal( + "fetch", + vi.fn(async () => { + polls++; + return Response.json({ + id: "litellm_poll_partial", + status: polls === 1 ? "in_progress" : "completed", + output: + polls === 1 + ? [ + { type: "message", id: "message_1" }, + { type: "mcp_list_tools", id: "tools_1", server_label: "inventory" }, + ] + : [ + { type: "message", content: [{ type: "output_text", text: "12 widgets" }] }, + { type: "mcp_list_tools", server_label: "inventory", tools: [{ name: "lookup", input_schema: {} }] }, + ], + }); + }), + ); + const onUpdate = vi.fn(); + const polling = { + accessToken: "test-token", + responseId: "litellm_poll_partial", + signal: new AbortController().signal, + onUpdate, + }; + await watchBackgroundChat(polling); + expect(polls).toBe(2); + expect(onUpdate.mock.lastCall?.[0]).toMatchObject({ + content: "12 widgets", + backgroundRun: { status: "completed" }, + mcpEvents: [{ item: { tools: [{ name: "lookup" }] } }], + }); +}); + +it("should restore server timings from polling without counting time away from the chat", async () => { + const response = { + id: "litellm_poll_timing", + status: "completed", + created_at: 1000, + output: [], + litellm_response_timing: { time_to_first_token_ms: 250, total_latency_ms: 12500 }, + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json(response)), + ); + const onUpdate = vi.fn(); + const polling = { + accessToken: "test-token", + responseId: "litellm_poll_timing", + signal: new AbortController().signal, + onUpdate, + }; + await watchBackgroundChat(polling); + expect(onUpdate).toHaveBeenCalledWith(expect.objectContaining({ timeToFirstToken: 250, totalLatency: 12500 })); +}); + +it.each(["queued", "completed"] as const)( + "should not invent timings for a %s response without measurements", + (status) => { + const update = backgroundResponseUpdate({ id: "litellm_poll_legacy", status, output: [] } as ModelResponse); + expect(update).not.toHaveProperty("timeToFirstToken"); + expect(update).not.toHaveProperty("totalLatency"); + }, +); diff --git a/ui/litellm-dashboard/src/components/chat/backgroundChat.ts b/ui/litellm-dashboard/src/components/chat/backgroundChat.ts new file mode 100644 index 00000000000..177d25f2e16 --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat/backgroundChat.ts @@ -0,0 +1,138 @@ +import OpenAI from "openai"; +import type { Response } from "openai/resources/responses/responses"; +import { getGatewayBaseUrl } from "@/components/networking"; +import { parseUsageCost } from "@/components/llm_calls/usage_cost"; +import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; +import type { AssistantMessageUpdate, ChatBackgroundRun, MCPEvent } from "./types"; + +export const isBackgroundRunPending = (run?: ChatBackgroundRun): boolean => + run?.status === "queued" || run?.status === "in_progress"; + +type TimedResponse = Response & { + litellm_response_timing?: { + time_to_first_token_ms?: number; + total_latency_ms?: number; + }; +}; + +export function backgroundResponseUpdate(response: TimedResponse): AssistantMessageUpdate { + const output = response.output ?? []; + const content = output + .filter((item) => item.type === "message") + .flatMap((item) => item.content ?? []) + .map((part) => (part.type === "output_text" ? part.text : part.refusal)) + .join(""); + const reasoningContent = output + .filter((item) => item.type === "reasoning") + .flatMap((item) => item.summary ?? []) + .map((part) => part.text) + .join("\n\n"); + const mcpEvents: MCPEvent[] = output.flatMap((item) => { + if (item.type === "mcp_call") { + return [{ type: "response.output_item.done", item: { ...item, output: item.output ?? undefined } }]; + } + if (item.type === "mcp_list_tools") { + return [ + { + type: "response.output_item.done", + item: { + ...item, + tools: (item.tools ?? []).map((tool) => ({ + name: tool.name, + input_schema: tool.input_schema, + description: tool.description ?? "", + })), + }, + }, + ]; + } + return []; + }); + const status = response.status ?? "in_progress"; + const errors: Partial> = { + failed: response.error?.message || "The background run failed.", + incomplete: "The run ended before completing its answer.", + }; + const backgroundError = errors[status]; + const cost = response.usage && "cost" in response.usage ? parseUsageCost(response.usage.cost) : undefined; + const timing = response.litellm_response_timing; + return { + content, + reasoningContent, + mcpEvents, + backgroundRun: { id: response.id, status }, + backgroundError, + ...(timing?.time_to_first_token_ms !== undefined && { timeToFirstToken: timing.time_to_first_token_ms }), + ...(timing?.total_latency_ms !== undefined && { totalLatency: timing.total_latency_ms }), + ...(response.usage && { + usage: { + promptTokens: response.usage.input_tokens, + completionTokens: response.usage.output_tokens, + totalTokens: response.usage.total_tokens, + reasoningTokens: response.usage.output_tokens_details?.reasoning_tokens, + ...extractPromptCacheTokens(response.usage), + ...(cost !== undefined ? { cost } : {}), + }, + }), + }; +} + +const backgroundClient = (accessToken: string) => + new OpenAI({ + apiKey: accessToken, + baseURL: getGatewayBaseUrl(), + dangerouslyAllowBrowser: true, + }); + +function waitForPoll(signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const abort = () => { + clearTimeout(timer); + reject(signal.reason); + }; + const timer = setTimeout(() => { + signal.removeEventListener("abort", abort); + resolve(); + }, 1000); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + }); +} + +export async function watchBackgroundChat({ + accessToken, + responseId, + signal, + onUpdate, +}: { + accessToken: string; + responseId: string; + signal: AbortSignal; + onUpdate: (update: AssistantMessageUpdate) => void; +}): Promise { + const client = backgroundClient(accessToken); + while (!signal.aborted) { + try { + const response = await client.responses.retrieve(responseId, { stream: false }, { signal }); + if (signal.aborted) return; + onUpdate(backgroundResponseUpdate(response)); + if (!isBackgroundRunPending({ id: response.id, status: response.status ?? "in_progress" })) return; + await waitForPoll(signal); + } catch (error) { + if (signal.aborted) return; + if (error instanceof OpenAI.APIError && error.status === 404) { + onUpdate({ + backgroundRun: { id: responseId, status: "expired" }, + backgroundError: "This saved run is no longer available. Its result may have expired on the server.", + }); + return; + } + throw error; + } + } +} + +export async function cancelBackgroundChat(accessToken: string, responseId: string): Promise { + const response = await backgroundClient(accessToken).responses.cancel(responseId); + return backgroundResponseUpdate(response); +} diff --git a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts index 157363d2461..9e8016b608e 100644 --- a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts +++ b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts @@ -1,5 +1,6 @@ import OpenAI from "openai"; import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api"; +import { backgroundResponseUpdate } from "./backgroundChat"; import { generateChatImages } from "./imageHelpers"; import { toResponsesMessages } from "./attachmentHelpers"; import type { AssistantMessageUpdate, ChatMessage, MCPEvent } from "./types"; @@ -15,6 +16,7 @@ interface ChatRequestOptions { "role" | "content" | "attachments" | "generatedImages" | "imageGeneration" | "mcpEvents" >[]; previousResponseId?: string | null; + background?: boolean; mcpServers?: string[]; onResponseId: (id: string | null) => void; onUpdate: (update: AssistantMessageUpdate) => void; @@ -43,6 +45,7 @@ export async function runChatRequest({ signal, messages, previousResponseId, + background = false, mcpServers, onResponseId, onUpdate, @@ -94,6 +97,14 @@ export async function runChatRequest({ undefined, true, (totalLatency) => onUpdate({ totalLatency }), + undefined, + undefined, + background + ? (response) => { + onResponseId(null); + onUpdate(backgroundResponseUpdate(response)); + } + : undefined, ); signal.throwIfAborted(); } catch (error: unknown) { diff --git a/ui/litellm-dashboard/src/components/chat/types.ts b/ui/litellm-dashboard/src/components/chat/types.ts index 25b232e40c4..e98b61e2fe4 100644 --- a/ui/litellm-dashboard/src/components/chat/types.ts +++ b/ui/litellm-dashboard/src/components/chat/types.ts @@ -25,6 +25,11 @@ export interface ChatImageGeneration { prompt: string; } +export interface ChatBackgroundRun { + id: string; + status: "queued" | "in_progress" | "completed" | "failed" | "cancelled" | "incomplete" | "expired"; +} + export interface ChatMessage { id: string; role: "user" | "assistant" | "tool"; @@ -33,6 +38,8 @@ export interface ChatMessage { generatedImages?: ChatImageAttachment[]; imageGeneration?: ChatImageGeneration; reasoningContent?: string; + backgroundRun?: ChatBackgroundRun; + backgroundError?: string; /** MCP tool events that occurred during this assistant turn, in order. */ mcpEvents?: MCPEvent[]; toolName?: string; @@ -55,6 +62,8 @@ export type AssistantMessageUpdate = Partial< | "usage" | "generatedImages" | "imageGeneration" + | "backgroundRun" + | "backgroundError" > >; diff --git a/ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts b/ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts new file mode 100644 index 00000000000..c0d1c43bcde --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts @@ -0,0 +1,62 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { cancelBackgroundChat, isBackgroundRunPending, watchBackgroundChat } from "./backgroundChat"; +import type { AssistantMessageUpdate, ChatBackgroundRun } from "./types"; + +export function useBackgroundChat( + conversationId: string | null, + run: ChatBackgroundRun | undefined, + accessToken: string, + updateMessage: (conversationId: string, update: AssistantMessageUpdate) => void, +) { + const [failure, setFailure] = useState<{ responseId: string; retry: number; message: string } | null>(null); + const [retry, setRetry] = useState(0); + const [cancellingId, setCancellingId] = useState(null); + const controllerRef = useRef(null); + const pending = isBackgroundRunPending(run); + const responseId = run?.id; + const cancelling = Boolean(responseId && cancellingId === responseId); + const connectionError = + pending && failure?.responseId === responseId && failure?.retry === retry ? failure.message : null; + + useEffect(() => { + if (!conversationId || !responseId || !pending) return; + const controller = new AbortController(); + controllerRef.current = controller; + const polling = { + accessToken, + responseId, + signal: controller.signal, + onUpdate: (update: AssistantMessageUpdate) => updateMessage(conversationId, update), + }; + void watchBackgroundChat(polling).catch(() => { + if (!controller.signal.aborted) + setFailure({ responseId, retry, message: "Connection lost. Your run may still be running on the server." }); + }); + return () => { + controller.abort(); + if (controllerRef.current === controller) controllerRef.current = null; + }; + }, [conversationId, responseId, pending, accessToken, updateMessage, retry]); + + const reconnect = useCallback(() => setRetry((value) => value + 1), []); + + const cancel = useCallback(async () => { + if (!conversationId || !responseId || cancelling) return; + const controller = controllerRef.current; + controller?.abort(); + setCancellingId(responseId); + try { + const update = await cancelBackgroundChat(accessToken, responseId); + updateMessage(conversationId, update); + if (controllerRef.current === controller) reconnect(); + } catch { + if (controllerRef.current === controller) { + setFailure({ responseId, retry, message: "Could not stop the run. Reconnect and try Stop again." }); + } + } finally { + setCancellingId(null); + } + }, [conversationId, responseId, accessToken, cancelling, reconnect, retry, updateMessage]); + + return { pending, connectionError, cancelling, reconnect, cancel }; +} diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx index d38db631c6e..cac91cf92d4 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx @@ -9,6 +9,7 @@ export interface ModelGroup { mode?: string; supports_reasoning?: boolean; supports_fast_mode?: boolean; + supports_background_chat?: boolean; supported_reasoning_efforts?: string[] | null; } @@ -20,6 +21,7 @@ interface AvailableModel { mode?: string | null; supports_reasoning?: boolean | null; supports_fast_mode?: boolean | null; + supports_background_chat?: boolean | null; supported_reasoning_efforts?: string[] | null; } @@ -30,6 +32,7 @@ const toModelGroup = (item: AvailableModel): ModelGroup => { ...(item.providers && { providers: item.providers }), ...(item.mode && { mode: item.mode }), ...(item.supports_reasoning === true && { supports_reasoning: true }), + ...(item.supports_background_chat === true && { supports_background_chat: true }), ...(item.supports_fast_mode === true && { supports_fast_mode: true }), ...(item.supported_reasoning_efforts !== undefined && { supported_reasoning_efforts: item.supported_reasoning_efforts, diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index ce1a8aab0c5..ef737c22cdb 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -1,6 +1,10 @@ import { observeResponseHeaders, type ResponseHeadersListener } from "@/lib/http/observeResponseHeaders"; import openai from "openai"; -import type { ResponseFunctionToolCall, ResponseInputItem } from "openai/resources/responses/responses"; +import type { + Response as OpenAIResponse, + ResponseFunctionToolCall, + ResponseInputItem, +} from "openai/resources/responses/responses"; import { MessageType } from "../chat_ui/types"; import { TokenUsage } from "../chat_ui/ResponseMetrics"; import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking"; @@ -110,6 +114,7 @@ export async function makeOpenAIResponsesRequest( onTotalLatency?: (latency: number) => void, customHeaders?: CustomHeaders, onResponseHeaders?: ResponseHeadersListener, + onBackgroundResponse?: (response: OpenAIResponse) => void, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -239,6 +244,16 @@ export async function makeOpenAIResponsesRequest( ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), }; + if (onBackgroundResponse) { + const response = await client.post("/responses", { + body: { ...requestBody, background: true, stream: false }, + signal, + maxRetries: 0, + }); + onBackgroundResponse(response); + return response; + } + // Create request to OpenAI responses API // Use 'any' type to avoid TypeScript issues with the experimental API const response = streamingEnabled diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e80df24a818..0dce821f862 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5849,6 +5849,26 @@ export interface paths { patch?: never; trace?: never; }; + "/get/background_chat_settings": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Background Chat Settings + * @description Get saved background-response settings and the prerequisites for changing them. + */ + get: operations["get_background_chat_settings_get_background_chat_settings_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/get/config/callbacks": { parameters: { query?: never; @@ -18240,6 +18260,26 @@ export interface paths { patch: operations["typesafe_proxy_route_typesafe__endpoint__patch"]; trace?: never; }; + "/update/background_chat_settings": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** + * Update Background Chat Settings + * @description Save background-response polling settings for the proxy configuration refresh. + */ + patch: operations["update_background_chat_settings_update_background_chat_settings_patch"]; + trace?: never; + }; "/update/default_team_settings": { parameters: { query?: never; @@ -27157,6 +27197,37 @@ export interface components { } & { [key: string]: unknown; }; + /** BackgroundChatSettings */ + BackgroundChatSettings: { + /** + * Enabled + * @description Enable cache-backed background Responses for Chat and API requests. + */ + enabled: boolean; + }; + /** BackgroundChatSettingsResponse */ + BackgroundChatSettingsResponse: { + /** + * Can Save + * @description Whether database-backed configuration updates are available. + */ + can_save: boolean; + /** + * Enabled + * @description Enable cache-backed background Responses for Chat and API requests. + */ + enabled: boolean; + /** + * Managed By Config + * @description Whether the responses configuration is owned by the config file. + */ + managed_by_config: boolean; + /** + * Redis Configured + * @description Whether this proxy has coordination Redis configured. + */ + redis_configured: boolean; + }; /** BaseLitellmParams */ BaseLitellmParams: { /** @@ -38804,6 +38875,12 @@ export interface components { supported_openai_params: string[] | null; /** Supported Reasoning Efforts */ supported_reasoning_efforts?: string[] | null; + /** + * Supports Background Chat + * @description Whether cache-backed background Responses runs are enabled for this model group + * @default false + */ + supports_background_chat: boolean; /** * Supports Fast Mode * @default false @@ -60982,6 +61059,26 @@ export interface operations { }; }; }; + get_background_chat_settings_get_background_chat_settings_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BackgroundChatSettingsResponse"]; + }; + }; + }; + }; get_config_get_config_callbacks_get: { parameters: { query?: never; @@ -76302,6 +76399,39 @@ export interface operations { }; }; }; + update_background_chat_settings_update_background_chat_settings_patch: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BackgroundChatSettings"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BackgroundChatSettingsResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; update_default_team_settings_update_default_team_settings_patch: { parameters: { query?: never; From 6aa546303e09326372f42488ddc01c0883344aae Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:48:56 +0000 Subject: [PATCH 2/4] docs(pr-template): wrap the agent-drafted summary in make-me-human-readable tags (#49494) * docs(pr-template): wrap the agent-drafted summary in make-me-human-readable tags Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(pr-template): keep the make-me-human-readable tags visible in rendered descriptions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(pr-template): mark the agent-drafted summary with a visible Make me human-readable warning block Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Revert "docs(pr-template): mark the agent-drafted summary with a visible Make me human-readable warning block" This reverts commit 4eb7f666a451cc9a88c63ab50b84d0eb4998f660. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/PULL_REQUEST_TEMPLATE/general.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/general.md b/.github/PULL_REQUEST_TEMPLATE/general.md index f6ce687a552..aafd968ee0f 100644 --- a/.github/PULL_REQUEST_TEMPLATE/general.md +++ b/.github/PULL_REQUEST_TEMPLATE/general.md @@ -9,10 +9,16 @@ +`` + +`` +
Details From da5ea5966797499dc6aa22ad1d89df1a012c27ae Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 31 Oct 2026 15:09:15 -0700 Subject: [PATCH 3/4] refactor(ui): remove dead API calls from networking.tsx (#49507) Deletes 17 exported calls that nothing references anymore. The Old Usage page removal left 10 behind, the move of agent traces to the Lens service left 5, and modelInfoV1Call and getUISettings lost their last callers in earlier refactors Seven names that are only used inside the file stop being exported. buildModelGroupTestRequest stays exported because networking.test.ts calls it directly Also drops the two test mocks that still stubbed deleted calls and prunes the lower eslint suppression counts for the file --- ui/litellm-dashboard/eslint-suppressions.json | 4 +- .../components/SidebarProvider.test.tsx | 2 +- .../model_info_view.integration.test.tsx | 23 -- .../src/components/networking.tsx | 352 +----------------- 4 files changed, 10 insertions(+), 371 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index a4e0d59d73d..d4ac60e501b 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1732,10 +1732,10 @@ "count": 5 }, "no-restricted-syntax": { - "count": 139 + "count": 133 }, "prefer-const": { - "count": 28 + "count": 26 } }, "src/components/object_permissions_view.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx index a3d75df9606..fa4b48a5a9d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx @@ -9,7 +9,7 @@ const { getSettings } = vi.hoisted(() => ({ getSettings: vi.fn().mockResolvedValue({ values: { enable_projects_ui: true } }), })); -vi.mock("@/components/networking", () => ({ getUiSettings: getSettings, getUISettings: getSettings })); +vi.mock("@/components/networking", () => ({ getUiSettings: getSettings })); vi.mock("@/components/leftnav", () => ({ default: ({ enableProjectsUI }: ComponentProps) => ( diff --git a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx index f93e01eb583..32e590017b0 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx @@ -19,7 +19,6 @@ vi.mock("../../utils/dataUtils", () => ({ })); vi.mock("./networking", () => ({ - modelInfoV1Call: vi.fn(), credentialGetCall: vi.fn(), credentialListCall: vi.fn(), getGuardrailsList: vi.fn(), @@ -56,7 +55,6 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () => })); const mockToast = vi.mocked(toast); -const mockModelInfoV1Call = vi.mocked(networking.modelInfoV1Call); const mockCredentialGetCall = vi.mocked(networking.credentialGetCall); const mockCredentialListCall = vi.mocked(networking.credentialListCall); const mockGetGuardrailsList = vi.mocked(networking.getGuardrailsList); @@ -134,10 +132,6 @@ describe("ModelInfoView", () => { error: null, }); - mockModelInfoV1Call.mockResolvedValue({ - data: [defaultModelData], - }); - mockCredentialGetCall.mockResolvedValue({ credential_name: "test-credential", credential_values: {}, @@ -394,7 +388,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, mode: "chat", key: "anthropic.claude-haiku-4-5" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [displayedModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [displayedModel] }); render(, { wrapper }); await waitFor(() => { @@ -869,7 +862,6 @@ describe("ModelInfoView", () => { const renderWithPtuModel = () => { mockUseModelsInfo.mockReturnValue({ data: { data: [ptuModelData] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [ptuModelData] }); return render(, { wrapper }); }; @@ -967,7 +959,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, team_id: "team-1", input_cost_per_token: 0.0000003 }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [seededModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [seededModel] }); const user = userEvent.setup(); render(, { wrapper }); @@ -1008,7 +999,6 @@ describe("ModelInfoView", () => { model_info: { ...ptuModelData.model_info, cost_per_ptu_per_hour: null, ptu_effective_from: null }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [halfSetPtuModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [halfSetPtuModel] }); const user = userEvent.setup(); render(, { wrapper }); @@ -1057,7 +1047,6 @@ describe("ModelInfoView", () => { model_info: { ...ptuModelData.model_info, ptu_effective_from: null, ptu_effective_to: null }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [undatedPtuModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [undatedPtuModel] }); const user = userEvent.setup(); render(, { wrapper }); @@ -1201,7 +1190,6 @@ describe("ModelInfoView", () => { isLoading: false, error: null, }); - mockModelInfoV1Call.mockResolvedValue({ data: [maskedModelData] }); const user = userEvent.setup(); render(, { wrapper }); @@ -1508,7 +1496,6 @@ describe("ModelInfoView", () => { beforeEach(() => { mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] }); }); const readRawJson = async (user: ReturnType) => { @@ -1555,7 +1542,6 @@ describe("ModelInfoView", () => { it("shows Not Set and no team_alias for a model without a team", async () => { mockUseModelsInfo.mockReturnValue({ data: { data: [defaultModelData] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [defaultModelData] }); mockUseTeams.mockReturnValue({ data: [{ team_id: "team-1", team_alias: "alpha" }], isLoading: false, @@ -1849,7 +1835,6 @@ describe("ModelInfoView", () => { beforeEach(() => { mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheWriteTiers] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [withCacheWriteTiers] }); }); it("shows both configured cache write rates per million tokens", async () => { @@ -1926,7 +1911,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, team_id: "team-1" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] }); const user = userEvent.setup(); await enterEditMode(user); @@ -1952,7 +1936,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, team_id: "team-1" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] }); const user = userEvent.setup(); await enterEditMode(user); @@ -1992,7 +1975,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, team_id: "team-1" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] }); const user = userEvent.setup(); render(, { wrapper }); await user.click(await screen.findByRole("button", { name: /edit settings/i })); @@ -2069,7 +2051,6 @@ describe("ModelInfoView", () => { litellm_params: litellmParamsWithoutCredential, }; mockUseModelsInfo.mockReturnValue({ data: { data: [modelWithoutCredential] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [modelWithoutCredential] }); const user = userEvent.setup(); await enterEditMode(user); @@ -2150,7 +2131,6 @@ describe("ModelInfoView", () => { litellm_params: { ...defaultModelData.litellm_params, model: "openai/gpt-4*" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [wildcard] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [wildcard] }); mockUseModelHub.mockReturnValue({ data: { data: [{ model_group: "openai/gpt-4o", providers: ["openai"] }] }, isLoading: false, @@ -2207,7 +2187,6 @@ describe("ModelInfoView", () => { litellm_params: { ...defaultModelData.litellm_params, cache_control_injection_points: points }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [data] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [data] }); }; it("omits the key when the deployment has none and the operator leaves the toggle alone", async () => { @@ -2330,7 +2309,6 @@ describe("ModelInfoView", () => { litellm_params: { ...defaultModelData.litellm_params, cache_read_input_token_cost: 0.000001 }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheRead] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [withCacheRead] }); const user = userEvent.setup(); await enterEditMode(user); setInputCost("2"); @@ -2365,7 +2343,6 @@ describe("ModelInfoView", () => { }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [resolved] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [resolved] }); const user = userEvent.setup(); await enterEditMode(user); setInputCost("5"); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index c13052be245..994c2948346 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -48,7 +48,7 @@ export const getAutoRouterClassifierDefaultPromptCall = async ( } }; -export type AssembledPromptTierSource = +type AssembledPromptTierSource = | { tierDefinitions: { name: string; description?: string }[] } | { tierLabels?: Record; classificationRubric?: string }; @@ -127,16 +127,6 @@ import type { SubscriptionUsageResponse, } from "@/app/(dashboard)/usage/_components/components/SubscriptionCost/types"; import type { RoutingDecision } from "./logs/detail/RoutingDecisionCard"; -import type { - SpanDetail, - SpanErrorPage, - SpanErrorQuery, - SpanQuery, - Trace, - TraceDetailQuery, - TraceListQuery, - TracePage, -} from "@litellm/lens-ui"; import { createApiClient, deriveErrorMessage, @@ -541,7 +531,7 @@ export const apiClient = createApiClient({ onError: handleError, }); -export const gatewayApiClient = createApiClient({ +const gatewayApiClient = createApiClient({ getBaseUrl: getGatewayBaseUrl, getAuthHeaderName: getGlobalLitellmHeaderName, onError: handleError, @@ -1725,37 +1715,6 @@ export const modelInfoCall = async ( } }; -export const modelInfoV1Call = async (accessToken: string, modelId: string) => { - /** - * Get all models on proxy - */ - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/v1/model/info` : `/v1/model/info`; - url += `?litellm_model_id=${modelId}`; - - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - export const modelHubPublicModelsCall = async () => { const url = proxyBaseUrl ? `${proxyBaseUrl}/public/model_hub` : `/public/model_hub`; const response = await fetch(url, { @@ -1906,66 +1865,6 @@ export const modelAvailableCall = async ( } }; -export const teamSpendLogsCall = async (accessToken: string) => { - try { - const data = await apiClient.get(`/global/spend/teams`, { accessToken }); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const tagsSpendLogsCall = async ( - accessToken: string, - startTime: string | undefined, - endTime: string | undefined, - tags: string[] | undefined, -) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/tags` : `/global/spend/tags`; - - if (startTime && endTime) { - url = `${url}?start_date=${startTime}&end_date=${endTime}`; - } - - // if tags, convert the list to a comma separated string - if (tags) { - url += `&tags=${tags.join(",")}`; - } - - const response = await fetch(`${url}`, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const allTagNamesCall = async (accessToken: string) => { - try { - const data = await apiClient.get(`/global/spend/all_tag_names`, { accessToken }); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - export const userFilterUICall = async (accessToken: string, params: URLSearchParams) => { try { return await apiClient.get(`/user/filter/ui`, { @@ -2081,214 +1980,6 @@ export const uiSpendLogsCall = async ({ } }; -/** - * Agent tracing. All three respond 501 `{detail}` when `general_settings.tracing` is not - * configured; callers can detect that through the thrown `ApiError`'s `status`. - */ -export const agentTraceListCall = async ({ - accessToken, - startMs, - endMs, - cursor, -}: { - accessToken: string; - startMs: number; - endMs: number; - cursor?: string | null; -}): Promise => { - const query = { start_ms: startMs, end_ms: endMs, cursor: cursor ?? undefined } satisfies TraceListQuery; - return apiClient.get(`/v1/traces`, { accessToken, query }); -}; - -export const sendOtlpTraceCall = async (accessToken: string, exportRequest: object): Promise => - apiClient.post(`/v1/traces`, { accessToken, body: exportRequest }); - -export const agentTraceCall = async ( - accessToken: string, - traceId: string, - traceRef?: string, - cursor?: string | null, -): Promise => - apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}`, { - accessToken, - query: { trace_ref: traceRef || undefined, cursor: cursor ?? undefined, page_size: 200 } satisfies TraceDetailQuery, - }); - -export const agentTraceSpanCall = async ( - accessToken: string, - traceId: string, - spanId: string, - traceRef?: string, -): Promise => - apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, { - accessToken, - query: { trace_ref: traceRef || undefined } satisfies SpanQuery, - }); - -export const agentTraceSpanErrorCall = async ( - accessToken: string, - traceId: string, - spanId: string, - options: { traceRef?: string; cursor?: string | null }, -): Promise => - apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}/error`, { - accessToken, - query: { trace_ref: options.traceRef || undefined, cursor: options.cursor || undefined } satisfies SpanErrorQuery, - }); - -export const adminSpendLogsCall = async (accessToken: string) => { - try { - const data = await apiClient.get(`/global/spend/logs`, { accessToken }); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const adminTopKeysCall = async (accessToken: string) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/keys?limit=5` : `/global/spend/keys?limit=5`; - - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const adminTopEndUsersCall = async ( - accessToken: string, - keyToken: string | null, - startTime: string | undefined, - endTime: string | undefined, -) => { - try { - const body = keyToken - ? { api_key: keyToken, startTime: startTime, endTime: endTime } - : { startTime: startTime, endTime: endTime }; - - const data = await apiClient.post(`/global/spend/end_users`, { accessToken, body }); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const adminspendByProvider = async ( - accessToken: string, - startTime: string | undefined, - endTime: string | undefined, -) => { - try { - const data = await apiClient.get(`/global/spend/provider`, { - accessToken, - query: { - ...(startTime && endTime ? { start_date: startTime, end_date: endTime } : {}), - }, - }); - return data; - } catch (error) { - console.error("Failed to fetch spend data:", error); - throw error; - } -}; - -export const adminGlobalActivity = async ( - accessToken: string, - startTime: string | undefined, - endTime: string | undefined, -) => { - try { - const data = await apiClient.get(`/global/activity`, { - accessToken, - query: startTime && endTime ? { start_date: startTime, end_date: endTime } : undefined, - }); - return data; - } catch (error) { - console.error("Failed to fetch spend data:", error); - throw error; - } -}; - -export const adminGlobalActivityPerModel = async ( - accessToken: string, - startTime: string | undefined, - endTime: string | undefined, -) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/global/activity/model` : `/global/activity/model`; - - if (startTime && endTime) { - url += `?start_date=${startTime}&end_date=${endTime}`; - } - - const requestOptions = { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - }, - }; - - const response = await fetch(url, requestOptions); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to fetch spend data:", error); - throw error; - } -}; - -export const adminTopModelsCall = async (accessToken: string) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/models?limit=5` : `/global/spend/models?limit=5`; - - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - export const testConnectionRequest = async ( accessToken: string, litellm_params: Record, @@ -2421,7 +2112,7 @@ export const testAutoRouterRouting = async ( } }; -export interface ComplexityRouterConfigValidation { +interface ComplexityRouterConfigValidation { valid: boolean; error?: string | null; } @@ -2733,7 +2424,7 @@ export interface UserConnectionStartResponse { flow_handle: string; } -export type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected"; +type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected"; export interface UserConnectionPollResponse { status: UserConnectionPollStatus; @@ -3697,35 +3388,6 @@ export const getProxyUISettings = async (accessToken: string) => { } }; -export const getUISettings = async (accessToken: string) => { - /** - * Get UI-specific configuration flags from the database - */ - try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/get/ui_settings` : `/get/ui_settings`; - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - console.error("Failed to get UI settings:", errorMessage); - return null; - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to get UI settings:", error); - return null; - } -}; - export const getMCPSemanticFilterSettings = async (accessToken: string) => { /** * Get MCP semantic filter configuration @@ -4685,12 +4347,12 @@ export const createAgentCall = async (accessToken: string, agentData: any) => { export type DiscoveredAgentCard = Agent["agent_card_params"]; -export type DiscoverAgentCardResponse = Omit & { +type DiscoverAgentCardResponse = Omit & { agent_card: DiscoveredAgentCard; }; export type DiscoveryMode = components["schemas"]["DiscoveryMode"]; -export type DiscoverAgentCardOptions = Partial< +type DiscoverAgentCardOptions = Partial< Pick >; @@ -5613,7 +5275,7 @@ export const vectorStoreListCall = async ( } }; -export interface IndexesListResponse { +interface IndexesListResponse { object: string; data: VectorStoreIndex[]; } From e708343309981ec61d4d51b7e8b0c1f1d0156ffb Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 31 Oct 2026 15:14:33 -0700 Subject: [PATCH 4/4] test: deflake the json accumulator drain test and the armed routing prefetch test (#49505) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_request_redis_batch_pre_call.py | 6 +++- .../test_json_fragment_accumulator.py | 34 ++++--------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/tests/unit/caching/test_request_redis_batch_pre_call.py b/tests/unit/caching/test_request_redis_batch_pre_call.py index 37169e13a0b..94f19dbdc7c 100644 --- a/tests/unit/caching/test_request_redis_batch_pre_call.py +++ b/tests/unit/caching/test_request_redis_batch_pre_call.py @@ -290,7 +290,11 @@ def _router(redis_cache: FakeRedisCache, routing_strategy: str = "usage-based-ro @pytest.mark.asyncio -async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own(): +async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own(monkeypatch): + monkeypatch.setattr( + "litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime", + lambda: datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc), + ) client = FakeClient(_lua_ok_replies) redis_cache = FakeRedisCache(client) router = _router(redis_cache) diff --git a/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py b/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py index f27a5fd2307..212dff79daf 100644 --- a/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py +++ b/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py @@ -1,6 +1,5 @@ import json import time -import tracemalloc from collections.abc import Iterator from typing import Final from unittest.mock import patch @@ -193,34 +192,15 @@ def _streamed(fragments: tuple[str, ...]) -> JSONFragmentAccumulator: return accumulator -def _drain_peak_bytes(fragments: tuple[str, ...]) -> int: - accumulator: Final = _streamed(fragments) - first_found, _ = accumulator.pop_next_value() - assert first_found is True - already_tracing: Final = tracemalloc.is_tracing() - if not already_tracing: - tracemalloc.start() - tracemalloc.reset_peak() - baseline, _ = tracemalloc.get_traced_memory() - try: - drained: Final = sum(1 for _ in _popped_values(accumulator)) - _, peak = tracemalloc.get_traced_memory() - finally: - if not already_tracing: - tracemalloc.stop() - assert drained == len(fragments) - 1 - return peak - baseline - - -def test_draining_many_concatenated_values_never_copies_the_remainder(): +def test_draining_many_concatenated_values_yields_every_value_in_order(): value: Final = '{"text": "' + "x" * 20 + '"}' fragments: Final = tuple(value if index % 2 else f" {value}" for index in range(40_000)) - payload_bytes: Final = sum(len(fragment) for fragment in fragments) - peak_bytes: Final = _drain_peak_bytes(fragments) - assert peak_bytes < payload_bytes // 4, ( - f"draining {len(fragments)} values allocated a peak of {peak_bytes} bytes over a {payload_bytes} byte payload, " - "copying the remainder on every pop?" - ) + accumulator: Final = _streamed(fragments) + + drained: Final = tuple(_popped_values(accumulator)) + + assert drained == (json.loads(value),) * len(fragments) + assert accumulator.pop_next_value() == (False, None) def test_could_close_json_after_many_blank_fragments_is_not_quadratic():