scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-02 07:13:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

Merge remote-tracking branch 'origin/main' into litellm_router_stream_test_finish_reason

Browse source
This commit is contained in:
Moe Khalil 2026-10-31 22:15:46 +00:00
commit 8b55c6b1ff
32 changed files with 1508 additions and 443 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.github/PULL_REQUEST_TEMPLATE/general.md vendored
View file

@ -9,10 +9,16 @@
<!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
plain sentences proves the author understands what it does, and human-written technical English is far easier
for a reviewer to parse than AI-written English
An AI agent may draft a starting point here, but the author replaces it with their own handwritten version
An AI agent may draft a starting point here, but always wraps its draft in the make-me-human-readable tags below,
backticks included so the tags stay visible on GitHub, so nobody mistakes it for the author's words. The author rewrites it in their own words and deletes the tags.
Tags still present at review time mean the summary is AI-written and has not been rewritten yet
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
No bullets, tables, or code names here. Everything else goes inside the Details block below -->
`<make-me-human-readable>`
<blah>
`</make-me-human-readable>`
<details>
<summary>Details</summary>

51
litellm/proxy/proxy_server.py
View file

@ -3070,6 +3070,29 @@ redis_usage_cache: RedisCache | None = None # redis cache used for tracking spe
polling_via_cache_enabled: Literal["all"] | list[str] | bool = False
native_background_mode: list[str] = [] # Models that should use native provider background mode instead of polling
polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache
class _ResponseBackgroundSettings(BaseModel):
model_config = ConfigDict(frozen=True)
polling_via_cache: Literal["all"] | tuple[str, ...] | bool = False
native_background_mode: tuple[str, ...] = ()
ttl: int = 3600
def _apply_background_response_settings(value: SettingsJsonValue) -> None:
global polling_via_cache_enabled, native_background_mode, polling_cache_ttl
responses: Final = _as_settings_mapping(value)
settings: Final = _ResponseBackgroundSettings.model_validate(responses.get("background_mode") or {})
polling: Final = settings.polling_via_cache
polling_via_cache_enabled, native_background_mode, polling_cache_ttl = ( # rebind-ok: Refresh runtime config.
list(polling) if isinstance(polling, tuple) else polling,
list(settings.native_background_mode),
settings.ttl,
)
user_custom_auth = None
user_custom_key_generate = None
# Sentinel: prevents PKCE-no-Redis advisory from re-logging on config hot-reload.
@ -6932,20 +6955,7 @@ class ProxyConfig:
# see usage here: https://docs.litellm.ai/docs/proxy/caching
pass
elif key == "responses":
# Initialize global polling via cache settings
global polling_via_cache_enabled, native_background_mode, polling_cache_ttl
background_mode = value.get("background_mode", {})
polling_via_cache_enabled = background_mode.get("polling_via_cache", False)
native_background_mode = background_mode.get("native_background_mode", [])
polling_cache_ttl = background_mode.get("ttl", 3600)
verbose_proxy_logger.debug(
"%s Initialized polling via cache: enabled=%s, native_background_mode=%s, ttl=%s%s",
blue_color_code,
polling_via_cache_enabled,
native_background_mode,
polling_cache_ttl,
reset_color_code,
)
_apply_background_response_settings(value)
elif key == "max_ui_session_budget":
litellm.max_ui_session_budget = float(value) if value is not None else None
verbose_proxy_logger.debug(
@ -8653,6 +8663,7 @@ class ProxyConfig:
def _apply_litellm_settings_db_values(self, db_values: Mapping[str, SettingsJsonValue]) -> None:
self.litellm_settings.apply_db_row("litellm_settings", db_values)
_apply_background_response_settings(self.litellm_settings.get("responses", {}))
for key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
if key in db_values and (value := self.litellm_settings.get(key)) is not None:
setattr(litellm, key, value)
@ -18110,6 +18121,8 @@ async def model_deprecations(
def get_model_group_info(
llm_router: Router, all_models_str: Sequence[str], model_group: str | None, team_id: str | None = None
) -> list[ModelGroupInfoProxy]:
from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request
model_groups: Final[list[ModelGroupInfoProxy]] = []
unique_models: Final = tuple(dict.fromkeys(all_models_str))
@ -18119,6 +18132,14 @@ def get_model_group_info(
if model_group is not None and model_group != model:
continue
supports_background_chat = should_use_polling_for_request(
background_mode=True,
polling_via_cache_enabled=polling_via_cache_enabled,
redis_cache=redis_usage_cache,
model=model,
llm_router=llm_router,
native_background_mode=native_background_mode,
)
_model_group_info = registry.group(model, team_id=team_id)
if _model_group_info is not None:
@ -18126,6 +18147,7 @@ def get_model_group_info(
ModelGroupInfoProxy.model_validate(
{
**_model_group_info.model_dump(),
"supports_background_chat": supports_background_chat,
**model_group_custom_fields(
registry.deployments(
resolve_model_group_alias(registry.aliases(), model) or model,
@ -18140,6 +18162,7 @@ def get_model_group_info(
model_group_info = ModelGroupInfoProxy(
model_group=model,
providers=[],
supports_background_chat=supports_background_chat,
)
model_groups.append(model_group_info)

36
litellm/proxy/response_polling/background_streaming.py
View file

@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming
import asyncio
import json
from collections.abc import AsyncIterable, Callable, Mapping, Sequence
from time import perf_counter
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable
from fastapi import Request, Response
@ -128,6 +129,16 @@ async def background_streaming_task(
https://platform.openai.com/docs/api-reference/responses/object
"""
started_at: Final = perf_counter()
first_token_at: float | None = None # rebind-ok: first content event closes the TTFT interval
finished_at: float | None = None # rebind-ok: terminal event closes the execution interval
def timing_snapshot() -> Mapping[str, float]:
return {
**({"time_to_first_token_ms": (first_token_at - started_at) * 1000} if first_token_at is not None else {}),
**({"total_latency_ms": (finished_at - started_at) * 1000} if finished_at is not None else {}),
}
try:
verbose_proxy_logger.info("Starting background streaming for %s", polling_id)
@ -191,7 +202,7 @@ async def background_streaming_task(
incomplete_details_data = None
state_dirty = False # Track if state needs to be synced
last_update_time = asyncio.get_event_loop().time()
last_update_time = perf_counter()
UPDATE_INTERVAL: Final = 0.150 # 150ms batching interval
# Track the terminal event from the stream (may not be "completed")
@ -210,13 +221,14 @@ async def background_streaming_task(
"""Flush accumulated state to Redis if interval elapsed or forced"""
nonlocal state_dirty, last_update_time
current_time: Final = asyncio.get_event_loop().time()
current_time: Final = perf_counter()
if state_dirty and (force or (current_time - last_update_time) >= UPDATE_INTERVAL):
# Convert output_items dict to list for update
output_list: Final = list(output_items.values())
await polling_handler.update_state(
polling_id=polling_id,
output=output_list,
timing=timing_snapshot(),
)
state_dirty = False
last_update_time = current_time
@ -242,6 +254,20 @@ async def background_streaming_task(
event: _StreamEvent = _StreamEventParser.parse(chunk_data)
event_type = event.get("type", "")
if (
first_token_at is None
and event_type
in (
"response.output_text.delta",
"response.reasoning_text.delta",
"response.reasoning_summary_text.delta",
"response.reasoning.delta",
)
and event.get("delta")
):
first_token_at = perf_counter()
state_dirty = True
# Process different event types based on OpenAI streaming spec
if event_type == "response.output_item.added":
# New output item added
@ -334,6 +360,7 @@ async def background_streaming_task(
):
# Terminal event - extract all ResponsesAPIResponse fields
# https://platform.openai.com/docs/api-reference/responses-streaming
finished_at = perf_counter()
response_data = event.get("response", {})
terminal_status = response_data.get(
"status",
@ -384,10 +411,13 @@ async def background_streaming_task(
# Use the terminal status from the stream, default to "completed"
final_status: Final = terminal_status or "completed"
if finished_at is None:
finished_at = perf_counter() # rebind-ok: close the interval when the stream has no terminal event
await polling_handler.update_state(
polling_id=polling_id,
status=final_status,
timing=timing_snapshot(),
usage=usage_data,
error=terminal_error,
reasoning=reasoning_data,
@ -417,6 +447,7 @@ async def background_streaming_task(
)
except Exception as e:
finished_at = perf_counter() # rebind-ok: capture execution time at failure
verbose_proxy_logger.error("Error in background streaming task for %s: %s", polling_id, e)
import traceback
@ -425,6 +456,7 @@ async def background_streaming_task(
await polling_handler.update_state(
polling_id=polling_id,
status="failed",
timing=timing_snapshot(),
error={
"type": "internal_error",
"message": str(e),

18
litellm/proxy/response_polling/polling_handler.py
View file

@ -3,9 +3,9 @@ Response Polling Handler for Background Responses with Cache
"""
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from typing import Final
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter
@ -15,6 +15,9 @@ from litellm._uuid import uuid4
from litellm.caching.redis_cache import RedisCache
from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus
if TYPE_CHECKING:
from litellm.router import Router
_RESPONSE_POLLING_TARGET: Final = "response_polling"
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
_JSON_LIST: Final = TypeAdapter(list[object])
@ -116,6 +119,7 @@ class ResponsePollingHandler:
parallel_tool_calls: bool | None = None,
user: str | None = None,
store: bool | None = None,
timing: Mapping[str, float] | None = None,
) -> None:
"""
Update the polling state in Redis
@ -144,6 +148,7 @@ class ResponsePollingHandler:
parallel_tool_calls: Whether parallel tool calls are enabled
user: User identifier
store: Whether to store the response
timing: Server execution timings in milliseconds
"""
if not self.redis_cache:
return
@ -167,6 +172,9 @@ class ResponsePollingHandler:
if output is not None:
state["output"] = output
if timing is not None:
state["litellm_response_timing"] = timing
# Update usage
if usage:
state["usage"] = usage
@ -264,10 +272,10 @@ class ResponsePollingHandler:
def should_use_polling_for_request(
background_mode: bool,
polling_via_cache_enabled, # Can be False, "all", or List[str]
redis_cache, # RedisCache or None
polling_via_cache_enabled: bool | str | Sequence[str],
redis_cache: RedisCache | None,
model: str,
llm_router, # Router instance or None
llm_router: "Router | None",
native_background_mode: list[str] | None = None, # List of models that should use native background mode
) -> bool:
"""

95
litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
View file

@ -216,6 +216,20 @@ class _SettingsWithSchema(LiteLLMBaseModel):
field_schema: dict[str, object]
class BackgroundChatSettings(LiteLLMBaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
enabled: bool = Field(description="Enable cache-backed background Responses for Chat and API requests.")
class BackgroundChatSettingsResponse(BackgroundChatSettings):
model_config = ConfigDict(extra="forbid", frozen=True)
redis_configured: bool = Field(description="Whether this proxy has coordination Redis configured.")
managed_by_config: bool = Field(description="Whether the responses configuration is owned by the config file.")
can_save: bool = Field(description="Whether database-backed configuration updates are available.")
class SSOSettingsResponse(SettingsResponse):
"""Response model for SSO settings"""
@ -1815,6 +1829,87 @@ async def sync_ui_settings_to_general_settings(
return apply_runtime_general_settings_flags(parsed)
def _background_chat_settings_response(config: Mapping[str, object]) -> BackgroundChatSettingsResponse:
from litellm.proxy import proxy_server
settings: Final = _as_settings_section(config.get("litellm_settings"))
responses: Final = _as_settings_section(settings.get("responses"))
background: Final = _as_settings_section(responses.get("background_mode"))
polling: Final = background.get("polling_via_cache", False)
return BackgroundChatSettingsResponse(
enabled=bool(polling),
redis_configured=proxy_server.redis_usage_cache is not None,
managed_by_config=proxy_server.proxy_config.litellm_settings.owned_by_config("responses"),
can_save=proxy_server.prisma_client is not None and proxy_server.store_model_in_db is True,
)
@router.get(
"/get/background_chat_settings",
tags=["Settings"],
dependencies=[Depends(user_api_key_auth)],
response_model=BackgroundChatSettingsResponse,
)
async def get_background_chat_settings() -> BackgroundChatSettingsResponse:
"""Get saved background-response settings and the prerequisites for changing them."""
from litellm.proxy.proxy_server import proxy_config
config: Final = _as_settings_section(_DECODED_JSON.validate_python(await proxy_config.get_config()))
return _background_chat_settings_response(config)
@router.patch(
"/update/background_chat_settings",
tags=["Settings"],
dependencies=[Depends(user_api_key_auth)],
response_model=BackgroundChatSettingsResponse,
)
async def update_background_chat_settings(
settings: BackgroundChatSettings,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> BackgroundChatSettingsResponse:
"""Save background-response polling settings for the proxy configuration refresh."""
from litellm.proxy.proxy_server import (
create_config_audit_log,
prisma_client,
proxy_config,
redis_usage_cache,
store_model_in_db,
)
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(status_code=403, detail="Only proxy admins can update background chat settings.")
if prisma_client is None or store_model_in_db is not True:
raise HTTPException(status_code=400, detail="Enable database-backed configuration to save this setting.")
if settings.enabled and redis_usage_cache is None:
raise HTTPException(status_code=400, detail="Configure coordination Redis and restart the proxy first.")
raw_config: Final = await proxy_config.get_config()
config: Final = _as_settings_section(_DECODED_JSON.validate_python(raw_config))
litellm_settings: Final = _as_settings_section(config.get("litellm_settings"))
responses: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(litellm_settings.get("responses")))
background: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(responses.get("background_mode")))
updated_responses: Final[Mapping[str, JsonValue]] = {
**responses,
"background_mode": {**background, "polling_via_cache": "all" if settings.enabled else False},
}
proxy_config.reject_config_owned_writes(
section_name="litellm_settings", changed_keys={"responses": dict(updated_responses)}
)
raw_config["litellm_settings"] = {**litellm_settings, "responses": dict(updated_responses)}
await proxy_config.save_config(new_config=config)
asyncio.create_task(
create_config_audit_log(
param_name="responses.background_mode.polling_via_cache",
action="updated",
before_value=background.get("polling_via_cache", False),
after_value="all" if settings.enabled else False,
user_api_key_dict=user_api_key_dict,
)
)
return _background_chat_settings_response(config)
@router.get(
"/get/ui_settings",
tags=["UI Settings"],

3
litellm/types/proxy/management_endpoints/model_management_endpoints.py
View file

@ -35,6 +35,9 @@ class PatchModelRequest(updateDeployment):
class ModelGroupInfoProxy(ModelGroupInfo):
supports_background_chat: bool = Field(
default=False, description="Whether cache-backed background Responses runs are enabled for this model group"
)
is_public_model_group: bool = Field(default=False)
health_status: str | None = Field(default=None)
health_response_time: float | None = Field(default=None)

4
tests/base_sdk_tests/check_sdk_http.py
View file

@ -70,7 +70,9 @@ def enqueue(body: object, status: int = 200) -> None:
def enqueue_stream() -> None:
chunk: Final = {**CHAT, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "pong"}}]}
RESPONSES.put((200, f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode()))
terminal: Final = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
stream: Final = f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(terminal)}\n\ndata: [DONE]\n\n"
RESPONSES.put((200, stream.encode()))
def bedrock_event(event: str, payload: object) -> bytes:

6
tests/unit/caching/test_request_redis_batch_pre_call.py
View file

@ -290,7 +290,11 @@ def _router(redis_cache: FakeRedisCache, routing_strategy: str = "usage-based-ro
@pytest.mark.asyncio
async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own():
async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own(monkeypatch):
monkeypatch.setattr(
"litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime",
lambda: datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc),
)
client = FakeClient(_lua_ok_replies)
redis_cache = FakeRedisCache(client)
router = _router(redis_cache)

34
tests/unit/litellm_core_utils/test_json_fragment_accumulator.py
View file

@ -1,6 +1,5 @@
import json
import time
import tracemalloc
from collections.abc import Iterator
from typing import Final
from unittest.mock import patch
@ -193,34 +192,15 @@ def _streamed(fragments: tuple[str, ...]) -> JSONFragmentAccumulator:
return accumulator
def _drain_peak_bytes(fragments: tuple[str, ...]) -> int:
accumulator: Final = _streamed(fragments)
first_found, _ = accumulator.pop_next_value()
assert first_found is True
already_tracing: Final = tracemalloc.is_tracing()
if not already_tracing:
tracemalloc.start()
tracemalloc.reset_peak()
baseline, _ = tracemalloc.get_traced_memory()
try:
drained: Final = sum(1 for _ in _popped_values(accumulator))
_, peak = tracemalloc.get_traced_memory()
finally:
if not already_tracing:
tracemalloc.stop()
assert drained == len(fragments) - 1
return peak - baseline
def test_draining_many_concatenated_values_never_copies_the_remainder():
def test_draining_many_concatenated_values_yields_every_value_in_order():
value: Final = '{"text": "' + "x" * 20 + '"}'
fragments: Final = tuple(value if index % 2 else f" {value}" for index in range(40_000))
payload_bytes: Final = sum(len(fragment) for fragment in fragments)
peak_bytes: Final = _drain_peak_bytes(fragments)
assert peak_bytes < payload_bytes // 4, (
f"draining {len(fragments)} values allocated a peak of {peak_bytes} bytes over a {payload_bytes} byte payload, "
"copying the remainder on every pop?"
)
accumulator: Final = _streamed(fragments)
drained: Final = tuple(_popped_values(accumulator))
assert drained == (json.loads(value),) * len(fragments)
assert accumulator.pop_next_value() == (False, None)
def test_could_close_json_after_many_blank_fragments_is_not_quadratic():

68
tests/unit/proxy/response_polling/test_response_polling_handler.py
View file

@ -1917,3 +1917,71 @@ async def test_create_initial_state_rejects_request_metadata_that_is_not_an_obje
assert [(error["loc"], error["type"], repr(error["input"])) for error in rejected.value.errors()] == [
(("metadata",), "dict_type", repr(metadata))
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"delta_type",
[
"response.output_text.delta",
"response.reasoning_text.delta",
"response.reasoning_summary_text.delta",
"response.reasoning.delta",
None,
],
)
async def test_background_timings_survive_retrieval_after_completion(delta_type):
from litellm.caching.redis_cache import RedisCache
from litellm.proxy.response_polling.background_streaming import background_streaming_task
cache = {}
redis = AsyncMock(spec=RedisCache)
redis.async_get_cache.side_effect = lambda key: cache.get(key)
async def save_cache(key, value, **kwargs):
cache[key] = value
redis.async_set_cache.side_effect = save_cache
handler = ResponsePollingHandler(redis_cache=redis)
await handler.create_initial_state("litellm_poll_timing", {"model": "gpt-4o"})
clock = [100.0]
async def stream():
clock[0] = 101.0
yield 'data: {"type":"response.output_text.delta","delta":""}\n\n'
yield 'data: {"type":"response.mcp_call.in_progress"}\n\n'
if delta_type:
clock[0] = 102.0
yield f"data: {json.dumps({'type': delta_type, 'delta': 'Hello'})}\n\n"
partial = await handler.get_state("litellm_poll_timing")
assert partial is not None
assert partial["status"] == "in_progress"
assert partial["litellm_response_timing"] == {"time_to_first_token_ms": 2000.0}
clock[0] = 110.0
yield 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}\n\n'
clock[0] = 5000.0
yield "data: [DONE]\n\n"
from starlette.responses import StreamingResponse
with (
patch(
"litellm.proxy.response_polling.background_streaming.perf_counter",
create=True,
side_effect=lambda: clock[0],
),
patch(
# test-quality-ok: processor is constructed inside the task; supply the upstream SSE boundary like sibling tests
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as processor,
):
processor.return_value.base_process_llm_request = AsyncMock(return_value=StreamingResponse(stream()))
await background_streaming_task(**_make_background_streaming_kwargs("litellm_poll_timing", handler))
state = await handler.get_state("litellm_poll_timing")
assert state is not None
expected = {"total_latency_ms": 10000.0}
if delta_type:
expected["time_to_first_token_ms"] = 2000.0
assert state["litellm_response_timing"] == expected
clock[0] = 10000.0
assert (await handler.get_state("litellm_poll_timing"))["litellm_response_timing"] == expected

85
tests/unit/proxy/test_proxy_server.py
View file

@ -4561,3 +4561,88 @@ async def test_text_completion_provider_rejection_captures_unknown_usage(
assert envelope["usage"] is None
assert envelope["usage_source"] == "unknown"
assert envelope["pricing"]["reported_cost"] is None
@pytest.mark.parametrize(
"polling,has_redis,native,expected",
[
(False, True, False, False),
("all", False, False, False),
("all", True, True, False),
("all", True, False, True),
(["openai"], True, False, True),
(["anthropic"], True, False, False),
],
)
def test_model_groups_advertise_cache_backed_background_chat(
monkeypatch: pytest.MonkeyPatch,
polling: bool | str | list[str],
has_redis: bool,
native: bool,
expected: bool,
) -> None:
from litellm.caching.redis_cache import RedisCache
from litellm.proxy import proxy_server
router: Final = litellm.Router(
model_list=[
{
"model_name": "research-chat",
"litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "fake-test-key"},
}
]
)
monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", polling)
monkeypatch.setattr(
proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379) if has_redis else None
)
monkeypatch.setattr(proxy_server, "native_background_mode", ["research-chat"] if native else [])
groups: Final = proxy_server.get_model_group_info(router, ["research-chat"], None)
assert groups[0].supports_background_chat is expected
@pytest.mark.parametrize("config_owned", [False, True])
def test_background_response_settings_refresh_and_respect_config(
monkeypatch: pytest.MonkeyPatch,
config_owned: bool,
) -> None:
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False)
monkeypatch.setattr(proxy_server, "native_background_mode", [])
monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600)
config: Final = proxy_server.ProxyConfig()
if config_owned:
config.litellm_settings.load_yaml(
{
"responses": {
"background_mode": {
"polling_via_cache": ["openai"],
"native_background_mode": ["config-native"],
"ttl": 900,
}
},
}
)
config._apply_litellm_settings_db_values(
{
"responses": {
"background_mode": {
"polling_via_cache": "all",
"native_background_mode": ["db-native"],
"ttl": 7200,
}
},
}
)
assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else "all")
assert proxy_server.native_background_mode == (["config-native"] if config_owned else ["db-native"])
assert proxy_server.polling_cache_ttl == (900 if config_owned else 7200)
config._apply_litellm_settings_db_values(
{
"responses": {"background_mode": {"polling_via_cache": False}},
}
)
assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else False)
assert proxy_server.native_background_mode == (["config-native"] if config_owned else [])
assert proxy_server.polling_cache_ttl == (900 if config_owned else 3600)

111
tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
View file

@ -4488,3 +4488,114 @@ def test_usage_default_range_reports_config_precedence(mock_auth, monkeypatch: p
assert response.status_code == 200, response.text
assert response.json()["values"]["usage_default_time_range"] == "month_to_date"
assert response.json()["source"]["usage_default_time_range"] == "config"
@pytest.fixture
def background_chat_config(monkeypatch: pytest.MonkeyPatch):
from copy import deepcopy
from unittest.mock import AsyncMock, MagicMock
from litellm.caching.redis_cache import RedisCache
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.config_resolvers import SettingsStore
config = {
"litellm_settings": {
"callbacks": ["prometheus"],
"responses": {
"background_mode": {
"polling_via_cache": False,
"ttl": 7200,
"native_background_mode": ["native-model"],
},
},
}
}
store = SettingsStore("litellm_settings")
monkeypatch.setattr(proxy_server.proxy_config, "litellm_settings", store)
monkeypatch.setattr(proxy_server.proxy_config, "_settings_stores", {"litellm_settings": store})
monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
monkeypatch.setattr(proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379))
monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False)
monkeypatch.setattr(proxy_server, "native_background_mode", [])
monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600)
monkeypatch.setattr(proxy_server.proxy_config, "get_config", AsyncMock(side_effect=lambda: deepcopy(config)))
async def save_config(*, new_config):
config.clear()
config.update(deepcopy(new_config))
save = AsyncMock(side_effect=save_config)
audit = AsyncMock()
monkeypatch.setattr(proxy_server.proxy_config, "save_config", save)
monkeypatch.setattr(proxy_server, "create_config_audit_log", audit)
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
)
yield config, save, audit
app.dependency_overrides.pop(user_api_key_auth, None)
def test_background_chat_settings_persist_preserving_existing_options(background_chat_config, monkeypatch):
from litellm.proxy import proxy_server
config, _, _ = background_chat_config
response = client.patch("/update/background_chat_settings", json={"enabled": True})
assert response.status_code == 200
assert response.json()["enabled"] is True
assert proxy_server.polling_via_cache_enabled is False
assert config["litellm_settings"]["responses"]["background_mode"] == {
"polling_via_cache": "all",
"ttl": 7200,
"native_background_mode": ["native-model"],
}
assert config["litellm_settings"]["callbacks"] == ["prometheus"]
assert client.get("/get/background_chat_settings").json()["enabled"] is True
proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"])
assert proxy_server.polling_via_cache_enabled == "all"
disabled = client.patch("/update/background_chat_settings", json={"enabled": False})
assert disabled.status_code == 200
assert disabled.json()["enabled"] is False
proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"])
assert proxy_server.polling_via_cache_enabled is False
@pytest.mark.parametrize("blocker", ["role", "redis", "config", "persistence"])
def test_background_chat_settings_reject_unavailable_updates(background_chat_config, monkeypatch, blocker):
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
config, save, _ = background_chat_config
if blocker == "role":
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_id="viewer", user_role=LitellmUserRoles.INTERNAL_USER
)
elif blocker == "redis":
monkeypatch.setattr(proxy_server, "redis_usage_cache", None)
assert client.get("/get/background_chat_settings").json()["redis_configured"] is False
elif blocker == "config":
proxy_server.proxy_config.litellm_settings.load_yaml(config["litellm_settings"])
assert client.get("/get/background_chat_settings").json()["managed_by_config"] is True
else:
monkeypatch.setattr(proxy_server, "store_model_in_db", False)
assert client.get("/get/background_chat_settings").json()["can_save"] is False
response = client.patch("/update/background_chat_settings", json={"enabled": True})
assert response.status_code == (403 if blocker == "role" else 400)
save.assert_not_awaited()
assert config["litellm_settings"]["responses"]["background_mode"]["polling_via_cache"] is False
def test_background_chat_settings_failed_save_does_not_enable_runtime(background_chat_config):
from litellm.proxy import proxy_server
_, save, audit = background_chat_config
save.side_effect = RuntimeError("Database write failed")
with pytest.raises(RuntimeError, match="Database write failed"):
client.patch("/update/background_chat_settings", json={"enabled": True})
assert client.get("/get/background_chat_settings").json()["enabled"] is False
assert proxy_server.polling_via_cache_enabled is False
audit.assert_not_awaited()

9
ui/litellm-dashboard/eslint-suppressions.json
View file

@ -1142,11 +1142,6 @@
"count": 1
}
},
"src/app/chat/page.tsx": {
"no-nested-ternary": {
"count": 1
}
},
"src/app/login/LoginPage.tsx": {
"local/no-complex-jsx-arrow": {
"count": 1
@ -1737,10 +1732,10 @@
"count": 5
},
"no-restricted-syntax": {
"count": 139
"count": 133
},
"prefer-const": {
"count": 28
"count": 26
}
},
"src/components/object_permissions_view.tsx": {

2
ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx
View file

@ -9,7 +9,7 @@ const { getSettings } = vi.hoisted(() => ({
getSettings: vi.fn().mockResolvedValue({ values: { enable_projects_ui: true } }),
}));
vi.mock("@/components/networking", () => ({ getUiSettings: getSettings, getUISettings: getSettings }));
vi.mock("@/components/networking", () => ({ getUiSettings: getSettings }));
vi.mock("@/components/leftnav", () => ({
default: ({ enableProjectsUI }: ComponentProps<typeof Sidebar>) => (
<nav>{enableProjectsUI && <a href="/projects">Projects</a>}</nav>

229
ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx Normal file
View file

@ -0,0 +1,229 @@
import { act, fireEvent, screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { renderWithProviders, testQueryClient } from "@/../tests/test-utils";
import { ChatShellProvider } from "@/contexts/ChatShellContext";
import type { Conversation } from "@/components/chat/types";
import ChatConversationPage from "./page";
vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn(), replace: vi.fn() }) }));
const storageKey = "litellm_chat_history_v1:background-user";
const requests: Record<string, unknown>[] = [];
let supported: boolean;
let status: "in_progress" | "completed" | "cancelled";
let retrievalStatus: number;
let cancellations: number;
let acceptance: Promise<void> | undefined;
const saved = (): Conversation[] => JSON.parse(localStorage.getItem(storageKey) ?? "[]");
const result = () => ({
id: "litellm_poll_research",
status,
usage: status === "completed" ? { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost: "0.125" } : undefined,
litellm_response_timing:
status === "completed" ? { time_to_first_token_ms: 250, total_latency_ms: 12500 } : undefined,
output:
status === "completed"
? [
{
type: "mcp_call",
id: "call_inventory",
name: "lookup_inventory",
arguments: "{}",
output: "12 widgets",
server_label: "inventory",
},
{ type: "message", content: [{ type: "output_text", text: "There are 12 widgets in stock." }] },
].filter((item) => item.type !== "mcp_call" || Boolean(requests.at(-1)?.tools))
: [],
});
function renderChat(id?: string) {
return renderWithProviders(
<ChatShellProvider
accessToken="test-token"
userId="background-user"
userEmail="researcher@example.com"
userRole="Internal Viewer"
premiumUser={false}
>
<ChatConversationPage />
</ChatShellProvider>,
{ searchParams: id ? { id } : {} },
);
}
beforeEach(() => {
localStorage.clear();
testQueryClient.clear();
requests.length = 0;
cancellations = 0;
acceptance = undefined;
supported = true;
retrievalStatus = 200;
status = "in_progress";
vi.stubGlobal(
"fetch",
vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
const request =
input instanceof Request ? input : new Request(new URL(String(input), window.location.origin), init);
const path = new URL(request.url).pathname;
if (path.endsWith("/model_group/info"))
return Response.json({
data: [{ model_group: "research-chat", mode: "chat", supports_background_chat: supported }],
});
if (path.endsWith("/v1/mcp/server")) return Response.json([{ server_id: "inventory", server_name: "inventory" }]);
if (path.endsWith("/mcp-rest/tools/list")) return Response.json({ tools: [{ name: "lookup_inventory" }] });
if (path.endsWith("/responses")) {
const body = JSON.parse(await request.text());
requests.push(body);
if (body.background) {
await acceptance;
request.signal.throwIfAborted();
return Response.json({ ...result(), status: "queued" });
}
return new Response(
'data: {"type":"response.output_text.delta","delta":"Foreground answer"}\n\ndata: {"type":"response.completed","response":{"id":"resp_foreground","output":[]}}\n\n',
{ headers: { "Content-Type": "text/event-stream" } },
);
}
if (path.endsWith("/litellm_poll_research/cancel")) {
cancellations++;
status = "cancelled";
return Response.json(result());
}
if (path.endsWith("/litellm_poll_research"))
return Response.json(retrievalStatus === 200 ? result() : { error: { message: "Unavailable" } }, {
status: retrievalStatus,
});
return Response.json({}, { status: 404 });
}),
);
});
afterEach(() => vi.unstubAllGlobals());
async function startResearch(withTools = true) {
const user = userEvent.setup();
await screen.findByRole("button", { name: "research-chat" });
if (withTools) {
await user.click(screen.getByRole("button", { name: "Connect tools" }));
await user.click(await screen.findByRole("switch"));
await user.keyboard("{Escape}");
}
fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), {
target: { value: "Check inventory" },
});
await user.click(screen.getByRole("button", { name: "Send", exact: true }));
}
it("should retrieve the completed tool run after closing and reopening without submitting it again", async () => {
const first = renderChat();
await startResearch();
await screen.findByText("Running in the background. You can close this tab and return to this chat.");
await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
const id = saved()[0].id;
first.unmount();
expect(cancellations).toBe(0);
status = "completed";
renderChat(id);
expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible();
expect(screen.getByRole("button", { name: "lookup_inventory" })).toBeVisible();
expect(screen.getByLabelText(/^Cost:/)).toHaveTextContent("$0.125");
expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s");
expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s");
await waitFor(() =>
expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }),
);
expect(requests).toHaveLength(1);
expect(requests[0]).toMatchObject({
background: true,
stream: false,
tools: [{ type: "mcp", server_label: "inventory" }],
});
expect(screen.queryByRole("button", { name: "Stop request" })).not.toBeInTheDocument();
fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), {
target: { value: "What was the result?" },
});
fireEvent.click(screen.getByRole("button", { name: "Send", exact: true }));
await waitFor(() => expect(requests).toHaveLength(2));
expect(requests[1]).toMatchObject({ background: true, stream: false });
expect(requests[1]).not.toHaveProperty("previous_response_id");
expect(requests[1].input).toEqual(
expect.arrayContaining([{ type: "function_call_output", call_id: "call_inventory", output: "12 widgets" }]),
);
});
it("should continue an ordinary chat without MCP tools after closing and reopening", async () => {
const first = renderChat();
await startResearch(false);
await screen.findByText("Running in the background. You can close this tab and return to this chat.");
await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
expect(requests[0]).toMatchObject({ background: true, stream: false });
expect(requests[0]).not.toHaveProperty("tools");
const id = saved()[0].id;
first.unmount();
status = "completed";
renderChat(id);
expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible();
expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s");
expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s");
await waitFor(() =>
expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }),
);
expect(requests).toHaveLength(1);
expect(cancellations).toBe(0);
});
it("should cancel a background run only when Stop is clicked", async () => {
renderChat();
await startResearch();
await screen.findByText("Running in the background. You can close this tab and return to this chat.");
fireEvent.click(screen.getByRole("button", { name: "Stop request" }));
expect(await screen.findByText("Cancellation requested")).toBeVisible();
expect(cancellations).toBe(1);
});
it("should retain ordinary streaming when background support is unavailable", async () => {
supported = false;
renderChat();
await startResearch();
expect(await screen.findByText("Foreground answer")).toBeVisible();
expect(requests[0]).toMatchObject({ stream: true });
expect(requests[0]).not.toHaveProperty("background");
expect(screen.getByText("Keep this tab open. Background runs are not available for this chat.")).toBeVisible();
});
it("should let the user reconnect after an authentication failure without resubmitting work", async () => {
const first = renderChat();
await startResearch();
await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
const id = saved()[0].id;
first.unmount();
retrievalStatus = 401;
renderChat(id);
await screen.findByRole("button", { name: "Reconnect" });
retrievalStatus = 200;
status = "completed";
fireEvent.click(screen.getByRole("button", { name: "Reconnect" }));
expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible();
expect(requests).toHaveLength(1);
});
it.each([false, true])(
"should honor Stop before a background submission returns its ID (tools=%s)",
async (withTools) => {
let accept!: () => void;
acceptance = new Promise<void>((resolve) => {
accept = resolve;
});
renderChat();
await startResearch(withTools);
await waitFor(() => expect(requests).toHaveLength(1));
fireEvent.click(screen.getByRole("button", { name: "Stop request" }));
await act(async () => accept());
expect(await screen.findByText("Cancellation requested")).toBeVisible();
expect(cancellations).toBe(1);
expect(requests).toHaveLength(1);
await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.status).toBe("cancelled"));
},
);

2
ui/litellm-dashboard/src/app/chat/page.integration.test.tsx
View file

@ -64,7 +64,7 @@ vi.mock("@/contexts/ChatShellContext", () => ({
},
}));
const ONE_TURN_ARG_COUNT = 25;
const ONE_TURN_ARG_COUNT = 28;
const ON_TIMING_DATA_INDEX = 7;
const ON_USAGE_DATA_INDEX = 8;
const ON_TOTAL_LATENCY_INDEX = 24;

87
ui/litellm-dashboard/src/app/chat/page.tsx
View file

@ -15,6 +15,7 @@ import type { AssistantMessageUpdate, ChatAttachment, ChatMessage } from "@/comp
import ChatModelPicker, { isChatModel, modelChatMode, type ChatMode } from "@/components/chat/ChatModelPicker";
import MoyaiChatComposer from "@/components/chat/moyai/MoyaiChatComposer";
import MoyaiChatHome from "@/components/chat/moyai/MoyaiChatHome";
import { useBackgroundChat } from "@/components/chat/useBackgroundChat";
import { runChatRequest } from "@/components/chat/runChatRequest";
import { readChatAttachment, MAX_CHAT_ATTACHMENTS, CHAT_ATTACHMENT_ACCEPT } from "@/components/chat/attachmentHelpers";
import ChatFileChip from "@/components/chat/ChatFileChip";
@ -102,7 +103,30 @@ export default function ChatConversationPage() {
const responsesSessionGenerationRef = useRef(0);
const previousResponsesConversationRef = useRef(activeConversationId);
const [prevConversationIdForSessionReset, setPrevConversationIdForSessionReset] = useState(activeConversationId);
const [isStreaming, setIsStreaming] = useState(false);
const [isRequestInFlight, setIsStreaming] = useState(false);
const [cancelOnAcceptance, setCancelOnAcceptance] = useState<string | null>(null);
const backgroundRequestRef = useRef(false);
const lastMessage = activeConversation?.messages.at(-1);
const background = useBackgroundChat(
activeConversationId,
lastMessage?.backgroundRun,
accessToken,
updateLastAssistantMessage,
);
const { pending: backgroundPending, cancelling: backgroundCancelling, cancel: cancelBackground } = background;
useEffect(() => {
if (!backgroundPending || backgroundCancelling || !activeConversationId) return;
if (cancelOnAcceptance !== activeConversationId) return;
void cancelBackground().then(() => {
setCancelOnAcceptance((current) => (current === activeConversationId ? null : current));
});
}, [activeConversationId, backgroundPending, backgroundCancelling, cancelBackground, cancelOnAcceptance]);
const isStreaming = isRequestInFlight || background.pending;
const supportsBackground = models.some(
(model) => model.model_group === selectedModel && model.supports_background_chat,
);
const useBackground = mode === "chat" && supportsBackground && !storageUnavailable;
const [inputText, setInputText] = useState("");
const [mcpPopoverOpen, setMcpPopoverOpen] = useState(false);
const [storageBannerDismissed, setStorageBannerDismissed] = useState(false);
@ -264,6 +288,7 @@ export default function ChatConversationPage() {
setAttachmentError(null);
setIsStreaming(true);
setCancelOnAcceptance(null);
abortControllerRef.current = new AbortController();
requestConversationRef.current = convId;
@ -297,12 +322,14 @@ export default function ChatConversationPage() {
signal: requestController.signal,
messages: history,
previousResponseId,
background: requestMode === "chat" && useBackground,
mcpServers: selectedMCPServers,
onResponseId: (id: string | null) => {
if (responsesSessionGenerationRef.current === sessionGeneration) setResponsesSessionId(id);
},
onUpdate: (updates: AssistantMessageUpdate) => updateLastAssistantMessage(convId, updates),
};
backgroundRequestRef.current = request.background;
try {
await runChatRequest(request);
} finally {
@ -326,13 +353,22 @@ export default function ChatConversationPage() {
updateLastAssistantMessage,
isStreaming,
responsesSessionId,
useBackground,
setConversationIdInUrl,
],
);
const handleStop = useCallback(() => {
const handleStop = () => {
if (background.pending) {
void background.cancel();
return;
}
if (backgroundRequestRef.current) {
setCancelOnAcceptance(activeConversationId);
return;
}
abortControllerRef.current?.abort();
}, []);
};
const handleEditAndResend = useCallback(
(messageId: string, newContent: string) => {
@ -403,7 +439,21 @@ export default function ChatConversationPage() {
const showBlankState = !activeConversation || activeConversation.messages.length === 0;
const backgroundHint = () => {
if (background.cancelling || (isRequestInFlight && cancelOnAcceptance === activeConversationId))
return "Requesting cancellation…";
if (background.pending && storageUnavailable)
return "Keep this tab open. This run could not be saved in this browser.";
if (background.pending) return "Running in the background. You can close this tab and return to this chat.";
if (useBackground) return null;
return "Keep this tab open. Background runs are not available for this chat.";
};
const inputBar = (inConversation: boolean) => {
const hint = mode === "chat" || background.pending ? backgroundHint() : null;
const chatPlaceholder = inConversation
? "Send a message..."
: "Describe a change, paste an error, or ask a question…";
const preparing = isLoadingModels || isReadingAttachments;
const submitDisabled = !hasMessageContent(inputText, attachments, mode) || !selectedModel || preparing;
return (
@ -424,13 +474,7 @@ export default function ChatConversationPage() {
onChange={setInputText}
onSubmit={() => void handleSend(inputText)}
onCancel={handleStop}
placeholder={
mode === "image"
? "Describe the image you want to create…"
: inConversation
? "Send a message..."
: "Describe a change, paste an error, or ask a question…"
}
placeholder={mode === "image" ? "Describe the image you want to create…" : chatPlaceholder}
isLoading={isStreaming}
submitLabel={mode === "image" ? "Generate image" : "Send"}
submitDisabled={submitDisabled}
@ -536,6 +580,29 @@ export default function ChatConversationPage() {
: "No chat models are available for your account."}
</p>
)}
{background.connectionError && (
<div role="alert" className="mt-2 flex items-center gap-2 text-sm text-destructive">
<span>{background.connectionError}</span>
<Button variant="ghost" size="sm" onClick={background.reconnect}>
Reconnect
</Button>
</div>
)}
{lastMessage?.backgroundError && (
<p role="alert" className="mt-2 text-sm text-destructive">
{lastMessage.backgroundError}
</p>
)}
{lastMessage?.backgroundRun?.status === "cancelled" && (
<p role="status" className="mt-2 text-xs text-muted-foreground">
Cancellation requested
</p>
)}
{hint && (
<p role="status" className="mt-2 text-xs text-muted-foreground">
{hint}
</p>
)}
<p className="moyai-chat-home-composer-hint">
{mode === "image" ? "Describe an image · Enter to generate" : "Drop files or paste images · Enter to send"}
</p>

98
ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx Normal file
View file

@ -0,0 +1,98 @@
import { fireEvent, screen, waitFor } from "@testing-library/react";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { renderWithProviders, testQueryClient } from "@/../tests/test-utils";
import BackgroundChatSettings from "./BackgroundChatSettings";
let settings: {
enabled: boolean;
redis_configured: boolean;
managed_by_config: boolean;
can_save: boolean;
};
let failLoad: boolean;
let failSave: boolean;
const updates: unknown[] = [];
beforeEach(() => {
testQueryClient.clear();
settings = {
enabled: false,
redis_configured: true,
managed_by_config: false,
can_save: true,
};
failLoad = false;
failSave = false;
updates.length = 0;
vi.stubGlobal(
"fetch",
vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
const request =
input instanceof Request ? input : new Request(new URL(String(input), window.location.origin), init);
if (request.method === "PATCH") {
const body = await request.json();
updates.push(body);
if (failSave) return Response.json({ detail: "Save failed" }, { status: 500 });
settings = { ...settings, enabled: body.enabled };
} else if (failLoad) return Response.json({ detail: "Unavailable" }, { status: 503 });
return Response.json(settings);
}),
);
});
afterEach(() => vi.unstubAllGlobals());
it("saves background execution next to its explanation and retains it after reopening", async () => {
const first = renderWithProviders(<BackgroundChatSettings />);
fireEvent.click(await screen.findByRole("switch", { name: "Continue chats in the background" }));
await waitFor(() => expect(screen.getByRole("switch")).toBeChecked());
expect(updates).toEqual([{ enabled: true }]);
expect(screen.getByRole("switch")).toBeChecked();
first.unmount();
testQueryClient.clear();
renderWithProviders(<BackgroundChatSettings />);
expect(await screen.findByRole("switch")).toBeChecked();
fireEvent.click(screen.getByRole("switch"));
await waitFor(() => expect(screen.getByRole("switch")).not.toBeChecked());
expect(updates).toEqual([{ enabled: true }, { enabled: false }]);
});
it("links to Redis setup and prevents enabling without Redis", async () => {
settings.redis_configured = false;
renderWithProviders(<BackgroundChatSettings />);
expect(await screen.findByRole("switch")).toHaveAttribute("aria-disabled", "true");
expect(screen.getByRole("link", { name: /Configure it in Caching/ })).toHaveAttribute(
"href",
expect.stringMatching(/\/caching$/),
);
expect(updates).toHaveLength(0);
});
it.each(["managed_by_config", "can_save"] as const)(
"explains a setting that cannot be edited because of %s",
async (field) => {
settings[field] = field === "managed_by_config";
renderWithProviders(<BackgroundChatSettings />);
expect(await screen.findByRole("switch")).toHaveAttribute("aria-disabled", "true");
const message =
field === "managed_by_config" ? /Managed by your config file/ : /Enable database-backed configuration/;
expect(screen.getByText(message)).toBeVisible();
},
);
it("keeps a failed save off", async () => {
failSave = true;
renderWithProviders(<BackgroundChatSettings />);
fireEvent.click(await screen.findByRole("switch"));
await waitFor(() => expect(updates).toHaveLength(1));
expect(screen.getByRole("switch")).not.toBeChecked();
});
it("offers retry for a settings fetch failure instead of claiming Redis is missing", async () => {
failLoad = true;
renderWithProviders(<BackgroundChatSettings />);
await screen.findByRole("button", { name: "Retry" });
expect(screen.queryByRole("link")).not.toBeInTheDocument();
failLoad = false;
fireEvent.click(screen.getByRole("button", { name: "Retry" }));
expect(await screen.findByRole("switch")).not.toHaveAttribute("aria-disabled", "true");
});

73
ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx Normal file
View file

@ -0,0 +1,73 @@
import { $api } from "@/lib/http/api";
import { toast } from "@/lib/toast";
import { Button } from "@/components/ui/button";
import { Skeleton } from "@/components/ui/skeleton";
import { Switch } from "@/components/ui/switch";
import { uiHref } from "@/utils/uiHref";
export default function BackgroundChatSettings() {
const query = $api.useQuery("get", "/get/background_chat_settings");
const update = $api.useMutation("patch", "/update/background_chat_settings", {
onSuccess: () => {
void query.refetch();
toast.success("Background chat settings saved.");
},
onError: (error) => toast.fromError(error),
});
if (query.isPending) return <Skeleton className="h-14 w-full" />;
if (query.isError || !query.data)
return (
<div role="alert" className="text-sm text-destructive">
Could not load background chat settings.
<Button variant="ghost" size="sm" onClick={() => void query.refetch()}>
Retry
</Button>
</div>
);
const settings = query.data;
const readOnly = settings.managed_by_config || !settings.can_save;
const needsRedis = !settings.redis_configured && !settings.enabled;
const disabled = update.isPending || readOnly || needsRedis;
return (
<div className="flex items-start gap-3">
<Switch
checked={settings.enabled}
disabled={disabled}
onCheckedChange={(enabled) => update.mutate({ body: { enabled } })}
aria-label="Continue chats in the background"
aria-describedby="background-chat-description"
/>
<div className="space-y-1">
<p className="text-sm font-medium text-foreground">Continue chats in the background</p>
<p id="background-chat-description" className="text-sm text-muted-foreground">
Keep replies and tool runs going after users close the browser. Return to the same browser to see the result.
</p>
{!settings.redis_configured && (
<p className="text-sm text-muted-foreground">
Requires Redis.{" "}
<a className="text-primary underline" href={uiHref("/caching")}>
Configure it in Caching → Coordination Redis
</a>
, then restart the proxy.
</p>
)}
{settings.managed_by_config && (
<p className="text-sm text-muted-foreground">
Managed by your config file. Update its background polling setting to change this.
</p>
)}
{!settings.can_save && !settings.managed_by_config && (
<p className="text-sm text-muted-foreground">Enable database-backed configuration to save this setting.</p>
)}
{!readOnly && (
<p className="text-sm text-muted-foreground">
Changes apply automatically when the proxy refreshes its configuration.
</p>
)}
</div>
</div>
);
}

2
ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx
View file

@ -5,6 +5,8 @@ import userEvent from "@testing-library/user-event";
import UISettings from "./UISettings";
import { toast } from "@/lib/toast";
vi.mock("./BackgroundChatSettings", () => ({ default: () => null }));
const mockUseAuthorized = vi.hoisted(() => vi.fn());
const mockUseUISettings = vi.hoisted(() => vi.fn());
const mockUseUpdateUISettings = vi.hoisted(() => vi.fn());

5
ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx
View file

@ -16,6 +16,7 @@ import { relativeTimeOptions } from "@/components/shared/date_range_presets";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Label } from "@/components/ui/label";
import PageVisibilitySettings from "./PageVisibilitySettings";
import BackgroundChatSettings from "./BackgroundChatSettings";
interface SettingRowProps {
ariaLabel: string;
@ -429,13 +430,15 @@ export default function UISettings() {
disabled={isUpdating}
onCheckedChange={handleToggleEnableChatUI}
ariaLabel={enableChatUIProperty?.description ?? "Enable Chat page"}
label="[BETA] Enable Chat page (page will refresh)"
label="Enable Chat page (page will refresh)"
description={
enableChatUIProperty?.description ??
"If enabled, shows the Chat page in the UI sidebar, letting users chat with an LLM and connect their own MCP server credentials via OAuth."
}
/>
<BackgroundChatSettings />
<Separator />
<div className="flex items-start justify-between gap-3">
<div className="space-y-1">

3
ui/litellm-dashboard/src/components/chat/ChatShell.tsx
View file

@ -192,9 +192,6 @@ const ChatShell: React.FC<{ children: React.ReactNode }> = ({ children }) => {
</DropdownMenuItem>
))}
<DropdownMenuSeparator />
<p className="px-2 py-1 text-xs text-muted-foreground">
Chat is a pre-v0 preview and may change unexpectedly. Do not use in production.
</p>
<DropdownMenuItem
render={
<a href="https://github.com/BerriAI/litellm/discussions/32085" target="_blank" rel="noreferrer" />

177
ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts Normal file
View file

@ -0,0 +1,177 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Response as ModelResponse } from "openai/resources/responses/responses";
import { backgroundResponseUpdate, cancelBackgroundChat, watchBackgroundChat } from "./backgroundChat";
vi.mock("@/components/networking", () => ({ getGatewayBaseUrl: () => "http://localhost:4000/v1" }));
afterEach(() => vi.unstubAllGlobals());
it("should restore text, reasoning, tool results and usage from a completed response", () => {
const response = {
id: "litellm_poll_example",
status: "completed",
output: [
{ type: "reasoning", summary: [{ type: "summary_text", text: "Check the inventory" }] },
{
type: "mcp_call",
id: "call_lookup",
name: "lookup",
arguments: "{}",
output: "12 widgets",
server_label: "inventory",
},
{ type: "message", content: [{ type: "output_text", text: "There are 12 widgets." }] },
],
usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, output_tokens_details: { reasoning_tokens: 2 } },
} as ModelResponse;
const update = backgroundResponseUpdate(response);
expect(update.content).toBe("There are 12 widgets.");
expect(update.reasoningContent).toBe("Check the inventory");
expect(update.mcpEvents?.[0].item?.output).toBe("12 widgets");
const usage = { promptTokens: 4, completionTokens: 9, totalTokens: 13, reasoningTokens: 2 };
expect(update.usage).toMatchObject(usage);
expect(update.backgroundRun).toEqual({ id: response.id, status: "completed" });
});
describe("background response lifecycle", () => {
it("should detach polling without cancelling the server run", async () => {
const controller = new AbortController();
const fetch = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) =>
Response.json({ id: "litellm_poll_example", status: "in_progress", output: [] }),
);
vi.stubGlobal("fetch", fetch);
const onUpdate = vi.fn(() => controller.abort());
const polling = {
accessToken: "test-token",
responseId: "litellm_poll_example",
signal: controller.signal,
onUpdate,
};
await watchBackgroundChat(polling);
expect(onUpdate).toHaveBeenCalledOnce();
expect(fetch).toHaveBeenCalledOnce();
expect(fetch.mock.calls[0][1]?.method).toBe("GET");
});
it("should mark an expired run unavailable without clearing its partial answer", async () => {
vi.stubGlobal(
"fetch",
vi.fn(async () => Response.json({ error: { message: "Expired" } }, { status: 404 })),
);
const onUpdate = vi.fn();
const polling = {
accessToken: "test-token",
responseId: "litellm_poll_expired",
signal: new AbortController().signal,
onUpdate,
};
await watchBackgroundChat(polling);
expect(onUpdate).toHaveBeenCalledWith({
backgroundRun: { id: "litellm_poll_expired", status: "expired" },
backgroundError: "This saved run is no longer available. Its result may have expired on the server.",
});
});
it("should cancel explicitly through the server and preserve returned partial output", async () => {
const fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
const request = input instanceof Request ? input : new Request(input, init);
expect(request.method).toBe("POST");
expect(new URL(request.url).pathname).toBe("/v1/responses/litellm_poll_example/cancel");
return Response.json({
id: "litellm_poll_example",
status: "cancelled",
output: [{ type: "message", content: [{ type: "output_text", text: "Partial answer" }] }],
});
});
vi.stubGlobal("fetch", fetch);
expect(await cancelBackgroundChat("test-token", "litellm_poll_example")).toMatchObject({
content: "Partial answer",
backgroundRun: { id: "litellm_poll_example", status: "cancelled" },
});
});
});
it.each([
[0.125, 0.125],
["0.125", 0.125],
[0, 0],
["invalid", undefined],
])("should preserve valid returned cost %s", (cost, expected) => {
const response = {
id: "litellm_poll_cost",
status: "completed",
output: [],
usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost },
} as ModelResponse;
expect(backgroundResponseUpdate(response).usage?.cost).toBe(expected);
});
it("should keep polling through unfinished output items until the answer and tools arrive", async () => {
let polls = 0;
vi.stubGlobal(
"fetch",
vi.fn(async () => {
polls++;
return Response.json({
id: "litellm_poll_partial",
status: polls === 1 ? "in_progress" : "completed",
output:
polls === 1
? [
{ type: "message", id: "message_1" },
{ type: "mcp_list_tools", id: "tools_1", server_label: "inventory" },
]
: [
{ type: "message", content: [{ type: "output_text", text: "12 widgets" }] },
{ type: "mcp_list_tools", server_label: "inventory", tools: [{ name: "lookup", input_schema: {} }] },
],
});
}),
);
const onUpdate = vi.fn();
const polling = {
accessToken: "test-token",
responseId: "litellm_poll_partial",
signal: new AbortController().signal,
onUpdate,
};
await watchBackgroundChat(polling);
expect(polls).toBe(2);
expect(onUpdate.mock.lastCall?.[0]).toMatchObject({
content: "12 widgets",
backgroundRun: { status: "completed" },
mcpEvents: [{ item: { tools: [{ name: "lookup" }] } }],
});
});
it("should restore server timings from polling without counting time away from the chat", async () => {
const response = {
id: "litellm_poll_timing",
status: "completed",
created_at: 1000,
output: [],
litellm_response_timing: { time_to_first_token_ms: 250, total_latency_ms: 12500 },
};
vi.stubGlobal(
"fetch",
vi.fn(async () => Response.json(response)),
);
const onUpdate = vi.fn();
const polling = {
accessToken: "test-token",
responseId: "litellm_poll_timing",
signal: new AbortController().signal,
onUpdate,
};
await watchBackgroundChat(polling);
expect(onUpdate).toHaveBeenCalledWith(expect.objectContaining({ timeToFirstToken: 250, totalLatency: 12500 }));
});
it.each(["queued", "completed"] as const)(
"should not invent timings for a %s response without measurements",
(status) => {
const update = backgroundResponseUpdate({ id: "litellm_poll_legacy", status, output: [] } as ModelResponse);
expect(update).not.toHaveProperty("timeToFirstToken");
expect(update).not.toHaveProperty("totalLatency");
},
);

138
ui/litellm-dashboard/src/components/chat/backgroundChat.ts Normal file
View file

@ -0,0 +1,138 @@
import OpenAI from "openai";
import type { Response } from "openai/resources/responses/responses";
import { getGatewayBaseUrl } from "@/components/networking";
import { parseUsageCost } from "@/components/llm_calls/usage_cost";
import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
import type { AssistantMessageUpdate, ChatBackgroundRun, MCPEvent } from "./types";
export const isBackgroundRunPending = (run?: ChatBackgroundRun): boolean =>
run?.status === "queued" || run?.status === "in_progress";
type TimedResponse = Response & {
litellm_response_timing?: {
time_to_first_token_ms?: number;
total_latency_ms?: number;
};
};
export function backgroundResponseUpdate(response: TimedResponse): AssistantMessageUpdate {
const output = response.output ?? [];
const content = output
.filter((item) => item.type === "message")
.flatMap((item) => item.content ?? [])
.map((part) => (part.type === "output_text" ? part.text : part.refusal))
.join("");
const reasoningContent = output
.filter((item) => item.type === "reasoning")
.flatMap((item) => item.summary ?? [])
.map((part) => part.text)
.join("\n\n");
const mcpEvents: MCPEvent[] = output.flatMap<MCPEvent>((item) => {
if (item.type === "mcp_call") {
return [{ type: "response.output_item.done", item: { ...item, output: item.output ?? undefined } }];
}
if (item.type === "mcp_list_tools") {
return [
{
type: "response.output_item.done",
item: {
...item,
tools: (item.tools ?? []).map((tool) => ({
name: tool.name,
input_schema: tool.input_schema,
description: tool.description ?? "",
})),
},
},
];
}
return [];
});
const status = response.status ?? "in_progress";
const errors: Partial<Record<ChatBackgroundRun["status"], string>> = {
failed: response.error?.message || "The background run failed.",
incomplete: "The run ended before completing its answer.",
};
const backgroundError = errors[status];
const cost = response.usage && "cost" in response.usage ? parseUsageCost(response.usage.cost) : undefined;
const timing = response.litellm_response_timing;
return {
content,
reasoningContent,
mcpEvents,
backgroundRun: { id: response.id, status },
backgroundError,
...(timing?.time_to_first_token_ms !== undefined && { timeToFirstToken: timing.time_to_first_token_ms }),
...(timing?.total_latency_ms !== undefined && { totalLatency: timing.total_latency_ms }),
...(response.usage && {
usage: {
promptTokens: response.usage.input_tokens,
completionTokens: response.usage.output_tokens,
totalTokens: response.usage.total_tokens,
reasoningTokens: response.usage.output_tokens_details?.reasoning_tokens,
...extractPromptCacheTokens(response.usage),
...(cost !== undefined ? { cost } : {}),
},
}),
};
}
const backgroundClient = (accessToken: string) =>
new OpenAI({
apiKey: accessToken,
baseURL: getGatewayBaseUrl(),
dangerouslyAllowBrowser: true,
});
function waitForPoll(signal: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
const abort = () => {
clearTimeout(timer);
reject(signal.reason);
};
const timer = setTimeout(() => {
signal.removeEventListener("abort", abort);
resolve();
}, 1000);
signal.addEventListener("abort", abort, { once: true });
if (signal.aborted) abort();
});
}
export async function watchBackgroundChat({
accessToken,
responseId,
signal,
onUpdate,
}: {
accessToken: string;
responseId: string;
signal: AbortSignal;
onUpdate: (update: AssistantMessageUpdate) => void;
}): Promise<void> {
const client = backgroundClient(accessToken);
while (!signal.aborted) {
try {
const response = await client.responses.retrieve(responseId, { stream: false }, { signal });
if (signal.aborted) return;
onUpdate(backgroundResponseUpdate(response));
if (!isBackgroundRunPending({ id: response.id, status: response.status ?? "in_progress" })) return;
await waitForPoll(signal);
} catch (error) {
if (signal.aborted) return;
if (error instanceof OpenAI.APIError && error.status === 404) {
onUpdate({
backgroundRun: { id: responseId, status: "expired" },
backgroundError: "This saved run is no longer available. Its result may have expired on the server.",
});
return;
}
throw error;
}
}
}
export async function cancelBackgroundChat(accessToken: string, responseId: string): Promise<AssistantMessageUpdate> {
const response = await backgroundClient(accessToken).responses.cancel(responseId);
return backgroundResponseUpdate(response);
}

11
ui/litellm-dashboard/src/components/chat/runChatRequest.ts
View file

@ -1,5 +1,6 @@
import OpenAI from "openai";
import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api";
import { backgroundResponseUpdate } from "./backgroundChat";
import { generateChatImages } from "./imageHelpers";
import { toResponsesMessages } from "./attachmentHelpers";
import type { AssistantMessageUpdate, ChatMessage, MCPEvent } from "./types";
@ -15,6 +16,7 @@ interface ChatRequestOptions {
"role" | "content" | "attachments" | "generatedImages" | "imageGeneration" | "mcpEvents"
>[];
previousResponseId?: string | null;
background?: boolean;
mcpServers?: string[];
onResponseId: (id: string | null) => void;
onUpdate: (update: AssistantMessageUpdate) => void;
@ -43,6 +45,7 @@ export async function runChatRequest({
signal,
messages,
previousResponseId,
background = false,
mcpServers,
onResponseId,
onUpdate,
@ -94,6 +97,14 @@ export async function runChatRequest({
undefined,
true,
(totalLatency) => onUpdate({ totalLatency }),
undefined,
undefined,
background
? (response) => {
onResponseId(null);
onUpdate(backgroundResponseUpdate(response));
}
: undefined,
);
signal.throwIfAborted();
} catch (error: unknown) {

9
ui/litellm-dashboard/src/components/chat/types.ts
View file

@ -25,6 +25,11 @@ export interface ChatImageGeneration {
prompt: string;
}
export interface ChatBackgroundRun {
id: string;
status: "queued" | "in_progress" | "completed" | "failed" | "cancelled" | "incomplete" | "expired";
}
export interface ChatMessage {
id: string;
role: "user" | "assistant" | "tool";
@ -33,6 +38,8 @@ export interface ChatMessage {
generatedImages?: ChatImageAttachment[];
imageGeneration?: ChatImageGeneration;
reasoningContent?: string;
backgroundRun?: ChatBackgroundRun;
backgroundError?: string;
/** MCP tool events that occurred during this assistant turn, in order. */
mcpEvents?: MCPEvent[];
toolName?: string;
@ -55,6 +62,8 @@ export type AssistantMessageUpdate = Partial<
| "usage"
| "generatedImages"
| "imageGeneration"
| "backgroundRun"
| "backgroundError"
>
>;

62
ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts Normal file
View file

@ -0,0 +1,62 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { cancelBackgroundChat, isBackgroundRunPending, watchBackgroundChat } from "./backgroundChat";
import type { AssistantMessageUpdate, ChatBackgroundRun } from "./types";
export function useBackgroundChat(
conversationId: string | null,
run: ChatBackgroundRun | undefined,
accessToken: string,
updateMessage: (conversationId: string, update: AssistantMessageUpdate) => void,
) {
const [failure, setFailure] = useState<{ responseId: string; retry: number; message: string } | null>(null);
const [retry, setRetry] = useState(0);
const [cancellingId, setCancellingId] = useState<string | null>(null);
const controllerRef = useRef<AbortController | null>(null);
const pending = isBackgroundRunPending(run);
const responseId = run?.id;
const cancelling = Boolean(responseId && cancellingId === responseId);
const connectionError =
pending && failure?.responseId === responseId && failure?.retry === retry ? failure.message : null;
useEffect(() => {
if (!conversationId || !responseId || !pending) return;
const controller = new AbortController();
controllerRef.current = controller;
const polling = {
accessToken,
responseId,
signal: controller.signal,
onUpdate: (update: AssistantMessageUpdate) => updateMessage(conversationId, update),
};
void watchBackgroundChat(polling).catch(() => {
if (!controller.signal.aborted)
setFailure({ responseId, retry, message: "Connection lost. Your run may still be running on the server." });
});
return () => {
controller.abort();
if (controllerRef.current === controller) controllerRef.current = null;
};
}, [conversationId, responseId, pending, accessToken, updateMessage, retry]);
const reconnect = useCallback(() => setRetry((value) => value + 1), []);
const cancel = useCallback(async () => {
if (!conversationId || !responseId || cancelling) return;
const controller = controllerRef.current;
controller?.abort();
setCancellingId(responseId);
try {
const update = await cancelBackgroundChat(accessToken, responseId);
updateMessage(conversationId, update);
if (controllerRef.current === controller) reconnect();
} catch {
if (controllerRef.current === controller) {
setFailure({ responseId, retry, message: "Could not stop the run. Reconnect and try Stop again." });
}
} finally {
setCancellingId(null);
}
}, [conversationId, responseId, accessToken, cancelling, reconnect, retry, updateMessage]);
return { pending, connectionError, cancelling, reconnect, cancel };
}

3
ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
View file

@ -9,6 +9,7 @@ export interface ModelGroup {
mode?: string;
supports_reasoning?: boolean;
supports_fast_mode?: boolean;
supports_background_chat?: boolean;
supported_reasoning_efforts?: string[] | null;
}
@ -20,6 +21,7 @@ interface AvailableModel {
mode?: string | null;
supports_reasoning?: boolean | null;
supports_fast_mode?: boolean | null;
supports_background_chat?: boolean | null;
supported_reasoning_efforts?: string[] | null;
}
@ -30,6 +32,7 @@ const toModelGroup = (item: AvailableModel): ModelGroup => {
...(item.providers && { providers: item.providers }),
...(item.mode && { mode: item.mode }),
...(item.supports_reasoning === true && { supports_reasoning: true }),
...(item.supports_background_chat === true && { supports_background_chat: true }),
...(item.supports_fast_mode === true && { supports_fast_mode: true }),
...(item.supported_reasoning_efforts !== undefined && {
supported_reasoning_efforts: item.supported_reasoning_efforts,

17
ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
View file

@ -1,6 +1,10 @@
import { observeResponseHeaders, type ResponseHeadersListener } from "@/lib/http/observeResponseHeaders";
import openai from "openai";
import type { ResponseFunctionToolCall, ResponseInputItem } from "openai/resources/responses/responses";
import type {
Response as OpenAIResponse,
ResponseFunctionToolCall,
ResponseInputItem,
} from "openai/resources/responses/responses";
import { MessageType } from "../chat_ui/types";
import { TokenUsage } from "../chat_ui/ResponseMetrics";
import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking";
@ -110,6 +114,7 @@ export async function makeOpenAIResponsesRequest(
onTotalLatency?: (latency: number) => void,
customHeaders?: CustomHeaders,
onResponseHeaders?: ResponseHeadersListener,
onBackgroundResponse?: (response: OpenAIResponse) => void,
) {
if (!accessToken) {
throw new Error("Virtual Key is required");
@ -239,6 +244,16 @@ export async function makeOpenAIResponsesRequest(
...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
};
if (onBackgroundResponse) {
const response = await client.post<OpenAIResponse>("/responses", {
body: { ...requestBody, background: true, stream: false },
signal,
maxRetries: 0,
});
onBackgroundResponse(response);
return response;
}
// Create request to OpenAI responses API
// Use 'any' type to avoid TypeScript issues with the experimental API
const response = streamingEnabled

23
ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx
View file

@ -19,7 +19,6 @@ vi.mock("../../utils/dataUtils", () => ({
}));
vi.mock("./networking", () => ({
modelInfoV1Call: vi.fn(),
credentialGetCall: vi.fn(),
credentialListCall: vi.fn(),
getGuardrailsList: vi.fn(),
@ -56,7 +55,6 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () =>
}));
const mockToast = vi.mocked(toast);
const mockModelInfoV1Call = vi.mocked(networking.modelInfoV1Call);
const mockCredentialGetCall = vi.mocked(networking.credentialGetCall);
const mockCredentialListCall = vi.mocked(networking.credentialListCall);
const mockGetGuardrailsList = vi.mocked(networking.getGuardrailsList);
@ -134,10 +132,6 @@ describe("ModelInfoView", () => {
error: null,
});
mockModelInfoV1Call.mockResolvedValue({
data: [defaultModelData],
});
mockCredentialGetCall.mockResolvedValue({
credential_name: "test-credential",
credential_values: {},
@ -394,7 +388,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, mode: "chat", key: "anthropic.claude-haiku-4-5" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [displayedModel] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [displayedModel] });
render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
await waitFor(() => {
@ -869,7 +862,6 @@ describe("ModelInfoView", () => {
const renderWithPtuModel = () => {
mockUseModelsInfo.mockReturnValue({ data: { data: [ptuModelData] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [ptuModelData] });
return render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
};
@ -967,7 +959,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, team_id: "team-1", input_cost_per_token: 0.0000003 },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [seededModel] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [seededModel] });
const user = userEvent.setup();
render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
@ -1008,7 +999,6 @@ describe("ModelInfoView", () => {
model_info: { ...ptuModelData.model_info, cost_per_ptu_per_hour: null, ptu_effective_from: null },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [halfSetPtuModel] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [halfSetPtuModel] });
const user = userEvent.setup();
render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
@ -1057,7 +1047,6 @@ describe("ModelInfoView", () => {
model_info: { ...ptuModelData.model_info, ptu_effective_from: null, ptu_effective_to: null },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [undatedPtuModel] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [undatedPtuModel] });
const user = userEvent.setup();
render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
@ -1201,7 +1190,6 @@ describe("ModelInfoView", () => {
isLoading: false,
error: null,
});
mockModelInfoV1Call.mockResolvedValue({ data: [maskedModelData] });
const user = userEvent.setup();
render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
@ -1508,7 +1496,6 @@ describe("ModelInfoView", () => {
beforeEach(() => {
mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
});
const readRawJson = async (user: ReturnType<typeof userEvent.setup>) => {
@ -1555,7 +1542,6 @@ describe("ModelInfoView", () => {
it("shows Not Set and no team_alias for a model without a team", async () => {
mockUseModelsInfo.mockReturnValue({ data: { data: [defaultModelData] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [defaultModelData] });
mockUseTeams.mockReturnValue({
data: [{ team_id: "team-1", team_alias: "alpha" }],
isLoading: false,
@ -1849,7 +1835,6 @@ describe("ModelInfoView", () => {
beforeEach(() => {
mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheWriteTiers] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [withCacheWriteTiers] });
});
it("shows both configured cache write rates per million tokens", async () => {
@ -1926,7 +1911,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, team_id: "team-1" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
const user = userEvent.setup();
await enterEditMode(user);
@ -1952,7 +1936,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, team_id: "team-1" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
const user = userEvent.setup();
await enterEditMode(user);
@ -1992,7 +1975,6 @@ describe("ModelInfoView", () => {
model_info: { ...defaultModelData.model_info, team_id: "team-1" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
const user = userEvent.setup();
render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} userRole={userRole} />, { wrapper });
await user.click(await screen.findByRole("button", { name: /edit settings/i }));
@ -2069,7 +2051,6 @@ describe("ModelInfoView", () => {
litellm_params: litellmParamsWithoutCredential,
};
mockUseModelsInfo.mockReturnValue({ data: { data: [modelWithoutCredential] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [modelWithoutCredential] });
const user = userEvent.setup();
await enterEditMode(user);
@ -2150,7 +2131,6 @@ describe("ModelInfoView", () => {
litellm_params: { ...defaultModelData.litellm_params, model: "openai/gpt-4*" },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [wildcard] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [wildcard] });
mockUseModelHub.mockReturnValue({
data: { data: [{ model_group: "openai/gpt-4o", providers: ["openai"] }] },
isLoading: false,
@ -2207,7 +2187,6 @@ describe("ModelInfoView", () => {
litellm_params: { ...defaultModelData.litellm_params, cache_control_injection_points: points },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [data] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [data] });
};
it("omits the key when the deployment has none and the operator leaves the toggle alone", async () => {
@ -2330,7 +2309,6 @@ describe("ModelInfoView", () => {
litellm_params: { ...defaultModelData.litellm_params, cache_read_input_token_cost: 0.000001 },
};
mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheRead] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [withCacheRead] });
const user = userEvent.setup();
await enterEditMode(user);
setInputCost("2");
@ -2365,7 +2343,6 @@ describe("ModelInfoView", () => {
},
};
mockUseModelsInfo.mockReturnValue({ data: { data: [resolved] }, isLoading: false, error: null });
mockModelInfoV1Call.mockResolvedValue({ data: [resolved] });
const user = userEvent.setup();
await enterEditMode(user);
setInputCost("5");

352
ui/litellm-dashboard/src/components/networking.tsx
View file

@ -48,7 +48,7 @@ export const getAutoRouterClassifierDefaultPromptCall = async (
}
};
export type AssembledPromptTierSource =
type AssembledPromptTierSource =
| { tierDefinitions: { name: string; description?: string }[] }
| { tierLabels?: Record<string, string>; classificationRubric?: string };
@ -127,16 +127,6 @@ import type {
SubscriptionUsageResponse,
} from "@/app/(dashboard)/usage/_components/components/SubscriptionCost/types";
import type { RoutingDecision } from "./logs/detail/RoutingDecisionCard";
import type {
SpanDetail,
SpanErrorPage,
SpanErrorQuery,
SpanQuery,
Trace,
TraceDetailQuery,
TraceListQuery,
TracePage,
} from "@litellm/lens-ui";
import {
createApiClient,
deriveErrorMessage,
@ -541,7 +531,7 @@ export const apiClient = createApiClient({
onError: handleError,
});
export const gatewayApiClient = createApiClient({
const gatewayApiClient = createApiClient({
getBaseUrl: getGatewayBaseUrl,
getAuthHeaderName: getGlobalLitellmHeaderName,
onError: handleError,
@ -1725,37 +1715,6 @@ export const modelInfoCall = async (
}
};
export const modelInfoV1Call = async (accessToken: string, modelId: string) => {
/**
* Get all models on proxy
*/
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/v1/model/info` : `/v1/model/info`;
url += `?litellm_model_id=${modelId}`;
const response = await fetch(url, {
method: "GET",
headers: {
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
const errorData = await response.json();
const errorMessage = deriveErrorMessage(errorData);
handleError(errorMessage);
throw new Error(errorMessage);
}
const data = await response.json();
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const modelHubPublicModelsCall = async () => {
const url = proxyBaseUrl ? `${proxyBaseUrl}/public/model_hub` : `/public/model_hub`;
const response = await fetch(url, {
@ -1906,66 +1865,6 @@ export const modelAvailableCall = async (
}
};
export const teamSpendLogsCall = async (accessToken: string) => {
try {
const data = await apiClient.get(`/global/spend/teams`, { accessToken });
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const tagsSpendLogsCall = async (
accessToken: string,
startTime: string | undefined,
endTime: string | undefined,
tags: string[] | undefined,
) => {
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/tags` : `/global/spend/tags`;
if (startTime && endTime) {
url = `${url}?start_date=${startTime}&end_date=${endTime}`;
}
// if tags, convert the list to a comma separated string
if (tags) {
url += `&tags=${tags.join(",")}`;
}
const response = await fetch(`${url}`, {
method: "GET",
headers: {
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
const errorData = await response.json();
const errorMessage = deriveErrorMessage(errorData);
handleError(errorMessage);
throw new Error(errorMessage);
}
const data = await response.json();
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const allTagNamesCall = async (accessToken: string) => {
try {
const data = await apiClient.get(`/global/spend/all_tag_names`, { accessToken });
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const userFilterUICall = async (accessToken: string, params: URLSearchParams) => {
try {
return await apiClient.get(`/user/filter/ui`, {
@ -2081,214 +1980,6 @@ export const uiSpendLogsCall = async ({
}
};
/**
* Agent tracing. All three respond 501 `{detail}` when `general_settings.tracing` is not
* configured; callers can detect that through the thrown `ApiError`'s `status`.
*/
export const agentTraceListCall = async ({
accessToken,
startMs,
endMs,
cursor,
}: {
accessToken: string;
startMs: number;
endMs: number;
cursor?: string | null;
}): Promise<TracePage> => {
const query = { start_ms: startMs, end_ms: endMs, cursor: cursor ?? undefined } satisfies TraceListQuery;
return apiClient.get<TracePage>(`/v1/traces`, { accessToken, query });
};
export const sendOtlpTraceCall = async (accessToken: string, exportRequest: object): Promise<void> =>
apiClient.post(`/v1/traces`, { accessToken, body: exportRequest });
export const agentTraceCall = async (
accessToken: string,
traceId: string,
traceRef?: string,
cursor?: string | null,
): Promise<Trace> =>
apiClient.get<Trace>(`/v1/traces/${encodeURIComponent(traceId)}`, {
accessToken,
query: { trace_ref: traceRef || undefined, cursor: cursor ?? undefined, page_size: 200 } satisfies TraceDetailQuery,
});
export const agentTraceSpanCall = async (
accessToken: string,
traceId: string,
spanId: string,
traceRef?: string,
): Promise<SpanDetail> =>
apiClient.get<SpanDetail>(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, {
accessToken,
query: { trace_ref: traceRef || undefined } satisfies SpanQuery,
});
export const agentTraceSpanErrorCall = async (
accessToken: string,
traceId: string,
spanId: string,
options: { traceRef?: string; cursor?: string | null },
): Promise<SpanErrorPage> =>
apiClient.get<SpanErrorPage>(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}/error`, {
accessToken,
query: { trace_ref: options.traceRef || undefined, cursor: options.cursor || undefined } satisfies SpanErrorQuery,
});
export const adminSpendLogsCall = async (accessToken: string) => {
try {
const data = await apiClient.get(`/global/spend/logs`, { accessToken });
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const adminTopKeysCall = async (accessToken: string) => {
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/keys?limit=5` : `/global/spend/keys?limit=5`;
const response = await fetch(url, {
method: "GET",
headers: {
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
const errorData = await response.json();
const errorMessage = deriveErrorMessage(errorData);
handleError(errorMessage);
throw new Error(errorMessage);
}
const data = await response.json();
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const adminTopEndUsersCall = async (
accessToken: string,
keyToken: string | null,
startTime: string | undefined,
endTime: string | undefined,
) => {
try {
const body = keyToken
? { api_key: keyToken, startTime: startTime, endTime: endTime }
: { startTime: startTime, endTime: endTime };
const data = await apiClient.post(`/global/spend/end_users`, { accessToken, body });
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const adminspendByProvider = async (
accessToken: string,
startTime: string | undefined,
endTime: string | undefined,
) => {
try {
const data = await apiClient.get(`/global/spend/provider`, {
accessToken,
query: {
...(startTime && endTime ? { start_date: startTime, end_date: endTime } : {}),
},
});
return data;
} catch (error) {
console.error("Failed to fetch spend data:", error);
throw error;
}
};
export const adminGlobalActivity = async (
accessToken: string,
startTime: string | undefined,
endTime: string | undefined,
) => {
try {
const data = await apiClient.get(`/global/activity`, {
accessToken,
query: startTime && endTime ? { start_date: startTime, end_date: endTime } : undefined,
});
return data;
} catch (error) {
console.error("Failed to fetch spend data:", error);
throw error;
}
};
export const adminGlobalActivityPerModel = async (
accessToken: string,
startTime: string | undefined,
endTime: string | undefined,
) => {
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/global/activity/model` : `/global/activity/model`;
if (startTime && endTime) {
url += `?start_date=${startTime}&end_date=${endTime}`;
}
const requestOptions = {
method: "GET",
headers: {
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
},
};
const response = await fetch(url, requestOptions);
if (!response.ok) {
const errorData = await response.json();
const errorMessage = deriveErrorMessage(errorData);
handleError(errorMessage);
throw new Error(errorMessage);
}
const data = await response.json();
return data;
} catch (error) {
console.error("Failed to fetch spend data:", error);
throw error;
}
};
export const adminTopModelsCall = async (accessToken: string) => {
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/models?limit=5` : `/global/spend/models?limit=5`;
const response = await fetch(url, {
method: "GET",
headers: {
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
const errorData = await response.json();
const errorMessage = deriveErrorMessage(errorData);
handleError(errorMessage);
throw new Error(errorMessage);
}
const data = await response.json();
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const testConnectionRequest = async (
accessToken: string,
litellm_params: Record<string, any>,
@ -2421,7 +2112,7 @@ export const testAutoRouterRouting = async (
}
};
export interface ComplexityRouterConfigValidation {
interface ComplexityRouterConfigValidation {
valid: boolean;
error?: string | null;
}
@ -2733,7 +2424,7 @@ export interface UserConnectionStartResponse {
flow_handle: string;
}
export type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected";
type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected";
export interface UserConnectionPollResponse {
status: UserConnectionPollStatus;
@ -3697,35 +3388,6 @@ export const getProxyUISettings = async (accessToken: string) => {
}
};
export const getUISettings = async (accessToken: string) => {
/**
* Get UI-specific configuration flags from the database
*/
try {
const url = proxyBaseUrl ? `${proxyBaseUrl}/get/ui_settings` : `/get/ui_settings`;
const response = await fetch(url, {
method: "GET",
headers: {
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
const errorData = await response.json();
const errorMessage = deriveErrorMessage(errorData);
console.error("Failed to get UI settings:", errorMessage);
return null;
}
const data = await response.json();
return data;
} catch (error) {
console.error("Failed to get UI settings:", error);
return null;
}
};
export const getMCPSemanticFilterSettings = async (accessToken: string) => {
/**
* Get MCP semantic filter configuration
@ -4685,12 +4347,12 @@ export const createAgentCall = async (accessToken: string, agentData: any) => {
export type DiscoveredAgentCard = Agent["agent_card_params"];
export type DiscoverAgentCardResponse = Omit<components["schemas"]["DiscoverAgentResponse"], "agent_card"> & {
type DiscoverAgentCardResponse = Omit<components["schemas"]["DiscoverAgentResponse"], "agent_card"> & {
agent_card: DiscoveredAgentCard;
};
export type DiscoveryMode = components["schemas"]["DiscoveryMode"];
export type DiscoverAgentCardOptions = Partial<
type DiscoverAgentCardOptions = Partial<
Pick<components["schemas"]["DiscoverAgentRequest"], "discovery_mode" | "params">
>;
@ -5613,7 +5275,7 @@ export const vectorStoreListCall = async (
}
};
export interface IndexesListResponse {
interface IndexesListResponse {
object: string;
data: VectorStoreIndex[];
}

130
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -5849,6 +5849,26 @@ export interface paths {
patch?: never;
trace?: never;
};
"/get/background_chat_settings": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Get Background Chat Settings
* @description Get saved background-response settings and the prerequisites for changing them.
*/
get: operations["get_background_chat_settings_get_background_chat_settings_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/get/config/callbacks": {
parameters: {
query?: never;
@ -18240,6 +18260,26 @@ export interface paths {
patch: operations["typesafe_proxy_route_typesafe__endpoint__patch"];
trace?: never;
};
"/update/background_chat_settings": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
/**
* Update Background Chat Settings
* @description Save background-response polling settings for the proxy configuration refresh.
*/
patch: operations["update_background_chat_settings_update_background_chat_settings_patch"];
trace?: never;
};
"/update/default_team_settings": {
parameters: {
query?: never;
@ -27157,6 +27197,37 @@ export interface components {
} & {
[key: string]: unknown;
};
/** BackgroundChatSettings */
BackgroundChatSettings: {
/**
* Enabled
* @description Enable cache-backed background Responses for Chat and API requests.
*/
enabled: boolean;
};
/** BackgroundChatSettingsResponse */
BackgroundChatSettingsResponse: {
/**
* Can Save
* @description Whether database-backed configuration updates are available.
*/
can_save: boolean;
/**
* Enabled
* @description Enable cache-backed background Responses for Chat and API requests.
*/
enabled: boolean;
/**
* Managed By Config
* @description Whether the responses configuration is owned by the config file.
*/
managed_by_config: boolean;
/**
* Redis Configured
* @description Whether this proxy has coordination Redis configured.
*/
redis_configured: boolean;
};
/** BaseLitellmParams */
BaseLitellmParams: {
/**
@ -38804,6 +38875,12 @@ export interface components {
supported_openai_params: string[] | null;
/** Supported Reasoning Efforts */
supported_reasoning_efforts?: string[] | null;
/**
* Supports Background Chat
* @description Whether cache-backed background Responses runs are enabled for this model group
* @default false
*/
supports_background_chat: boolean;
/**
* Supports Fast Mode
* @default false
@ -60982,6 +61059,26 @@ export interface operations {
};
};
};
get_background_chat_settings_get_background_chat_settings_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["BackgroundChatSettingsResponse"];
};
};
};
};
get_config_get_config_callbacks_get: {
parameters: {
query?: never;
@ -76302,6 +76399,39 @@ export interface operations {
};
};
};
update_background_chat_settings_update_background_chat_settings_patch: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["BackgroundChatSettings"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["BackgroundChatSettingsResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
update_default_team_settings_update_default_team_settings_patch: {
parameters: {
query?: never;