Merge remote-tracking branch 'origin/main' into litellm_a2a_own_call_id

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -9,10 +9,16 @@
 <!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
      plain sentences proves the author understands what it does, and human-written technical English is far easier
      for a reviewer to parse than AI-written English
-     An AI agent may draft a starting point here, but the author replaces it with their own handwritten version
+     An AI agent may draft a starting point here, but always wraps its draft in the make-me-human-readable tags below,
+     backticks included so the tags stay visible on GitHub, so nobody mistakes it for the author's words. The author rewrites it in their own words and deletes the tags.
+     Tags still present at review time mean the summary is AI-written and has not been rewritten yet
      Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
      No bullets, tables, or code names here. Everything else goes inside the Details block below -->
 
+`<make-me-human-readable>`
+<blah>
+`</make-me-human-readable>`
+
 <details>
 <summary>Details</summary>
 

litellm/integrations/anthropic_cache_control_hook.py

@@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     tool_call_ids_answered_by_client,
     tool_call_ids_answered_in,
 )
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field
 from litellm.llms.anthropic.common_utils import (
     is_claude_code_one_shot_subagent_request,
     supports_anthropic_cache_control,
@@ -128,7 +129,7 @@ def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
         return False
     return (
         not isinstance(fields.get("content"), list)
-        or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
+        or string_field(fields, "role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
     )
 
 

litellm/litellm_core_utils/core_helpers.py

@@ -315,6 +315,8 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
     "model_context_window_exceeded": "length",
     # Cohere
     "COMPLETE": "stop",
+    "STOP_SEQUENCE": "stop",
+    "TOOL_CALL": "tool_calls",
     "ERROR_TOXIC": "content_filter",
     "ERROR": "stop",
     # HuggingFace / Together AI

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -8,7 +8,7 @@ import io
 import json
 import mimetypes
 import re
-from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from graphlib import CycleError, TopologicalSorter
 from itertools import chain, groupby, islice
@@ -3235,3 +3235,15 @@ def text_completion_prompt_to_messages(prompt: object) -> tuple[AllMessageValues
     if string_entries and len(string_entries) == len(entries):
         return tuple(ChatCompletionUserMessage(role="user", content=entry) for entry in string_entries)
     raise ValueError(f"`prompt` must be a non-empty string or a non-empty list of strings. Got: {prompt_type_name}.")
+
+
+def is_present_non_string(value: object) -> bool:
+    return value is not None and not isinstance(value, str)
+
+
+def invalid_type_message(field_path: str, value: object) -> str:
+    return f"Invalid type for '{field_path}': expected a string, got {type(value).__name__}"
+
+
+def is_unhashable_value_behind(error: TypeError, value: object) -> bool:
+    return not isinstance(value, Hashable) and f"unhashable type: '{type(value).__name__}'" in str(error)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     parts_of,
     place_mid_conversation_system,
     split_leading_system_run,
+    string_field,
 )
 from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
 from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value
@@ -2023,7 +2024,7 @@ def _server_tool_answers_replaced_after(messages: Sequence[object], index: int)
     following: Final = _messages_until_the_next_assistant(messages, index)
     a_user_turn_remains: Final = any(
         message_field(message, "role") in ("user", "tool", "function")
-        and message_field(message, "tool_call_id") not in kept_server_result_ids
+        and string_field(message, "tool_call_id") not in kept_server_result_ids
         for message in following
     )
     return _tool_call_ids_in_tool_messages(following) & kept_server_result_ids if a_user_turn_remains else frozenset()

litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py

@@ -85,6 +85,11 @@ def message_field(message: object, key: str) -> object:
     return mapping.get(key) if mapping is not None else getattr(message, key, None)
 
 
+def string_field(message: object, key: str) -> str | None:
+    value: Final = message_field(message, key)
+    return value if isinstance(value, str) else None
+
+
 def is_system_message(message: object) -> bool:
     return message_field(message, "role") == "system"
 

litellm/llms/anthropic/chat/transformation.py

@@ -2,7 +2,9 @@ import copy
 import json
 import re
 import time
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from functools import partial
+from itertools import chain
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast
 
@@ -27,7 +29,10 @@ from litellm.constants import (
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     DEFS_MAX_INLINED_BYTES,
+    invalid_type_message,
     is_object_schema,
+    is_present_non_string,
+    is_unhashable_value_behind,
     sanitize_input_schema_for_anthropic,
     schema_fragment_ref,
     schema_grammar_branch_count,
@@ -45,6 +50,8 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
     inline_remote_image_urls,
 )
 from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
+    message_field,
+    parts_of,
     place_mid_conversation_system,
     split_leading_system_run,
 )
@@ -179,6 +186,76 @@ NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24
 NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048
 
 
+def _tool_call_ids_failing(
+    index: int, tool_calls: object, fails: Callable[[object], bool]
+) -> Iterator[tuple[str, object]]:
+    ids: Final = ((position, message_field(tool_call, "id")) for position, tool_call in enumerate(parts_of(tool_calls)))
+    return (
+        (f"messages[{index}].tool_calls[{position}].id", tool_call_id)
+        for position, tool_call_id in ids
+        if fails(tool_call_id)
+    )
+
+
+def _roles_failing(messages: Sequence[object], fails: Callable[[object], bool]) -> Iterator[tuple[str, object]]:
+    roles: Final = ((index, message_field(message, "role")) for index, message in enumerate(messages))
+    return ((f"messages[{index}].role", role) for index, role in roles if fails(role))
+
+
+def _is_non_empty_non_string(value: object) -> bool:
+    return bool(value) and is_present_non_string(value)
+
+
+def _assistant_tool_call_ids_failing(
+    messages: Sequence[object], fails: Callable[[object], bool]
+) -> Iterator[tuple[str, object]]:
+    assistant_turns: Final = (
+        (index, message) for index, message in enumerate(messages) if message_field(message, "role") == "assistant"
+    )
+    return chain.from_iterable(
+        _tool_call_ids_failing(index, message_field(message, "tool_calls"), fails) for index, message in assistant_turns
+    )
+
+
+def _tool_answer_ids_failing(
+    messages: Sequence[object], fails: Callable[[object], bool]
+) -> Iterator[tuple[str, object]]:
+    answers: Final = (
+        (index, message_field(message, "tool_call_id"))
+        for index, message in enumerate(messages)
+        if message_field(message, "role") in ("tool", "function")
+    )
+    return ((f"messages[{index}].tool_call_id", tool_call_id) for index, tool_call_id in answers if fails(tool_call_id))
+
+
+def _is_non_string_value_behind(error: AttributeError, value: object) -> bool:
+    return is_present_non_string(value) and value is error.obj
+
+
+def _fields_behind(messages: Sequence[object], error: Exception) -> Iterable[tuple[str, object]]:
+    match error:
+        case TypeError():
+            caused_error: Final = partial(is_unhashable_value_behind, error)
+            return chain(
+                _assistant_tool_call_ids_failing(messages, caused_error),
+                _tool_answer_ids_failing(messages, caused_error),
+                _roles_failing(messages, caused_error),
+            )
+        case AttributeError():
+            return _assistant_tool_call_ids_failing(messages, partial(_is_non_string_value_behind, error))
+        case _:
+            return ()
+
+
+def non_string_message_field_error(messages: Sequence[object], error: Exception) -> AnthropicError | None:
+    malformed_field: Final = next(
+        chain(_roles_failing(messages, _is_non_empty_non_string), _fields_behind(messages, error)), None
+    )
+    if malformed_field is None:
+        return None
+    return AnthropicError(status_code=400, message=invalid_type_message(*malformed_field))
+
+
 def _tool_type(tool: object) -> str | None:
     match tool:
         case {"type": str() as tool_type}:
@@ -2370,12 +2447,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         # Handling anthropic API Prompt Caching
         if len(anthropic_system_message_list) > 0:
             optional_params["system"] = anthropic_system_message_list
-        conversation: Final = place_mid_conversation_system(
-            later_messages,
-            supports_mid_conversation_system=supports_mid_conversation_system(
-                model=model, custom_llm_provider=self.custom_llm_provider
-            ),
-        )
+        try:
+            conversation: Final = place_mid_conversation_system(
+                later_messages,
+                supports_mid_conversation_system=supports_mid_conversation_system(
+                    model=model, custom_llm_provider=self.custom_llm_provider
+                ),
+            )
+        except TypeError as error:
+            field_error: Final = non_string_message_field_error(messages, error)
+            if field_error is None:
+                raise
+            raise field_error from error
         # Format rest of message according to anthropic guidelines
         try:
             anthropic_messages = anthropic_messages_pt(
@@ -2385,7 +2468,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 drop_params=drop_params,
             )
         except Exception as e:
-            raise AnthropicError(
+            conversion_field_error: Final = non_string_message_field_error(messages, e)
+            raise conversion_field_error or AnthropicError(
                 status_code=400,
                 message=f"{e}\nReceived Messages={request_messages}",
             )  # don't use verbose_logger.exception, if exception is raised

litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py

@@ -74,6 +74,7 @@ class _ClearToolUsesFields(BaseModel):
     clear_tool_inputs: bool | tuple[str, ...] | None = None
 
 
+_BLOCK: Final = TypeAdapter(Mapping[str, object])
 _BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
 # api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently
 _API_MESSAGES: Final = MappingProxyType(
@@ -317,12 +318,10 @@ def _clear_tool_results(
         new_blocks: list[object] = []
         mutated = False
         for block in content:
-            if (
-                isinstance(block, dict)
-                and block.get("type") == "tool_result"
-                and block.get("tool_use_id") in ids_to_clear
-            ):
-                new_blocks.append(_cleared_result(block))
+            if (parsed := _parse(_BLOCK, block)) is not None and _block_id(
+                parsed, "tool_result", "tool_use_id"
+            ) in ids_to_clear:
+                new_blocks.append(_cleared_result(parsed))
                 mutated = True
                 cleared += 1
             else:

litellm/llms/base_llm/files/batch_records.py

@@ -19,6 +19,8 @@ def responses_batch_body_to_chat_body(
     openai_request_body: Mapping[str, object],
     custom_llm_provider: str | None = None,
     drop_params: bool | None = None,
+    *,
+    caller_input: str | ResponseInputParam | None = None,
 ) -> dict[str, object]:  # mutable-ok: provider transforms take the bridged chat body as a plain dict
     """
     Rewrite the body of an OpenAI `/v1/responses` batch record as a Chat Completions body.
@@ -52,5 +54,6 @@ def responses_batch_body_to_chat_body(
         responses_api_request=responses_request,
         custom_llm_provider=custom_llm_provider,
         drop_params=drop_params,
+        caller_input=caller_input,
         metadata=openai_request_body.get("metadata"),
     )

litellm/llms/bedrock/files/transformation.py

@@ -42,6 +42,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_data,
     text_completion_prompt_to_messages,
 )
+from litellm.llms.anthropic.common_utils import AnthropicError
 from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
@@ -347,7 +348,14 @@ class _BatchInputLine:
     record: _OpenAIBatchRecord
 
 
-_BATCH_RECORD_CONVERSION_ERRORS: Final = (AttributeError, LookupError, TypeError, ValueError, BadRequestError)
+_BATCH_RECORD_CONVERSION_ERRORS: Final = (
+    AttributeError,
+    LookupError,
+    TypeError,
+    ValueError,
+    BadRequestError,
+    AnthropicError,
+)
 
 
 def _batch_line_error(line_number: int, problem: str) -> BedrockError:

litellm/llms/cohere/common_utils.py

@@ -3,6 +3,7 @@ from typing import Final, Literal
 
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.cohere import CohereV2StreamedToolCall
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import (
     ChatCompletionToolCallChunk,
@@ -222,6 +223,7 @@ class CohereV2ModelResponseIterator:
         self.content_blocks: list = []
         self.tool_index = -1
         self.json_mode = json_mode
+        self.tool_call_indexes_with_arguments: frozenset[int] = frozenset()
 
     def _parse_content_delta(self, chunk: dict) -> str:
         """Parse content-delta chunks to extract text."""
@@ -234,20 +236,23 @@ class CohereV2ModelResponseIterator:
             return content
         return ""
 
-    def _parse_tool_call_delta(self, chunk: dict) -> ChatCompletionToolCallChunk | None:
-        """Parse tool-call-delta chunks to extract tool calls."""
-        delta: Final = chunk.get("delta", {})
-        tool_calls: Final = delta.get("tool_calls", [])
-        if tool_calls:
-            return {
-                "id": tool_calls[0].get("id", ""),
-                "type": "function",
-                "function": {
-                    "name": tool_calls[0].get("name", ""),
-                    "arguments": tool_calls[0].get("arguments", ""),
-                },
-            }
-        return None
+    def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk:
+        """Parse tool-call-start and tool-call-delta chunks to extract tool calls."""
+        tool_call: Final = CohereV2StreamedToolCall.model_validate(chunk)
+        if tool_call.arguments:
+            self.tool_call_indexes_with_arguments = self.tool_call_indexes_with_arguments | {index}
+        return {
+            "id": tool_call.id,
+            "type": "function",
+            "function": {"name": tool_call.name, "arguments": tool_call.arguments},
+            "index": index,
+        }
+
+    def _parse_tool_call_end(self, index: int) -> ChatCompletionToolCallChunk | None:
+        """Close a tool call that streamed no arguments with an empty JSON object, as OpenAI streams one."""
+        if index in self.tool_call_indexes_with_arguments:
+            return None
+        return {"id": None, "type": "function", "function": {"name": None, "arguments": "{}"}, "index": index}
 
     def _parse_tool_plan_delta(self, chunk: dict) -> dict | None:
         """Parse tool-plan-delta events to extract tool plan."""
@@ -301,7 +306,8 @@ class CohereV2ModelResponseIterator:
 
         v2 format:
         - Content: chunk.type == "content-delta" -> chunk.delta.message.content.text
-        - Tool calls: chunk.type == "tool-call-delta" -> chunk.delta.tool_calls
+        - Tool calls: chunk.type in ("tool-call-start", "tool-call-delta") -> chunk.delta.message.tool_calls
+        - Tool call end: chunk.type == "tool-call-end" -> "{}" arguments when the call streamed none
         - Tool plan: chunk.event == "tool-plan-delta" -> chunk.data.delta.message.tool_plan
         - Citations: chunk.event == "citation-start" -> chunk.data.delta.message.citations
         - Finish: chunk.event == "message-end" -> chunk.data.delta.finish_reason
@@ -321,8 +327,12 @@ class CohereV2ModelResponseIterator:
             # Handle different chunk types
             if chunk_type == "content-delta":
                 text = self._parse_content_delta(chunk)
-            elif chunk_type == "tool-call-delta":
-                tool_use = self._parse_tool_call_delta(chunk)
+            elif chunk_type in ("tool-call-start", "tool-call-delta", "tool-call-end"):
+                tool_use = (
+                    self._parse_tool_call_end(index)
+                    if chunk_type == "tool-call-end"
+                    else self._parse_tool_call_delta(chunk, index)
+                )
             elif event_type == "tool-plan-delta":
                 provider_specific_fields = self._parse_tool_plan_delta(chunk)
             elif event_type == "citation-start":

litellm/proxy/proxy_server.py

@@ -3070,6 +3070,29 @@ redis_usage_cache: RedisCache | None = None  # redis cache used for tracking spe
 polling_via_cache_enabled: Literal["all"] | list[str] | bool = False
 native_background_mode: list[str] = []  # Models that should use native provider background mode instead of polling
 polling_cache_ttl: int = 3600  # Default 1 hour TTL for polling cache
+
+
+class _ResponseBackgroundSettings(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    polling_via_cache: Literal["all"] | tuple[str, ...] | bool = False
+    native_background_mode: tuple[str, ...] = ()
+    ttl: int = 3600
+
+
+def _apply_background_response_settings(value: SettingsJsonValue) -> None:
+    global polling_via_cache_enabled, native_background_mode, polling_cache_ttl
+
+    responses: Final = _as_settings_mapping(value)
+    settings: Final = _ResponseBackgroundSettings.model_validate(responses.get("background_mode") or {})
+    polling: Final = settings.polling_via_cache
+    polling_via_cache_enabled, native_background_mode, polling_cache_ttl = (  # rebind-ok: Refresh runtime config.
+        list(polling) if isinstance(polling, tuple) else polling,
+        list(settings.native_background_mode),
+        settings.ttl,
+    )
+
+
 user_custom_auth = None
 user_custom_key_generate = None
 # Sentinel: prevents PKCE-no-Redis advisory from re-logging on config hot-reload.
@@ -6932,20 +6955,7 @@ class ProxyConfig:
                     # see usage here: https://docs.litellm.ai/docs/proxy/caching
                     pass
                 elif key == "responses":
-                    # Initialize global polling via cache settings
-                    global polling_via_cache_enabled, native_background_mode, polling_cache_ttl
-                    background_mode = value.get("background_mode", {})
-                    polling_via_cache_enabled = background_mode.get("polling_via_cache", False)
-                    native_background_mode = background_mode.get("native_background_mode", [])
-                    polling_cache_ttl = background_mode.get("ttl", 3600)
-                    verbose_proxy_logger.debug(
-                        "%s Initialized polling via cache: enabled=%s, native_background_mode=%s, ttl=%s%s",
-                        blue_color_code,
-                        polling_via_cache_enabled,
-                        native_background_mode,
-                        polling_cache_ttl,
-                        reset_color_code,
-                    )
+                    _apply_background_response_settings(value)
                 elif key == "max_ui_session_budget":
                     litellm.max_ui_session_budget = float(value) if value is not None else None
                     verbose_proxy_logger.debug(
@@ -8653,6 +8663,7 @@ class ProxyConfig:
 
     def _apply_litellm_settings_db_values(self, db_values: Mapping[str, SettingsJsonValue]) -> None:
         self.litellm_settings.apply_db_row("litellm_settings", db_values)
+        _apply_background_response_settings(self.litellm_settings.get("responses", {}))
         for key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES:
             if key in db_values and (value := self.litellm_settings.get(key)) is not None:
                 setattr(litellm, key, value)
@@ -18110,6 +18121,8 @@ async def model_deprecations(
 def get_model_group_info(
     llm_router: Router, all_models_str: Sequence[str], model_group: str | None, team_id: str | None = None
 ) -> list[ModelGroupInfoProxy]:
+    from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request
+
     model_groups: Final[list[ModelGroupInfoProxy]] = []
 
     unique_models: Final = tuple(dict.fromkeys(all_models_str))
@@ -18119,6 +18132,14 @@ def get_model_group_info(
         if model_group is not None and model_group != model:
             continue
 
+        supports_background_chat = should_use_polling_for_request(
+            background_mode=True,
+            polling_via_cache_enabled=polling_via_cache_enabled,
+            redis_cache=redis_usage_cache,
+            model=model,
+            llm_router=llm_router,
+            native_background_mode=native_background_mode,
+        )
         _model_group_info = registry.group(model, team_id=team_id)
 
         if _model_group_info is not None:
@@ -18126,6 +18147,7 @@ def get_model_group_info(
                 ModelGroupInfoProxy.model_validate(
                     {
                         **_model_group_info.model_dump(),
+                        "supports_background_chat": supports_background_chat,
                         **model_group_custom_fields(
                             registry.deployments(
                                 resolve_model_group_alias(registry.aliases(), model) or model,
@@ -18140,6 +18162,7 @@ def get_model_group_info(
             model_group_info = ModelGroupInfoProxy(
                 model_group=model,
                 providers=[],
+                supports_background_chat=supports_background_chat,
             )
             model_groups.append(model_group_info)
 

litellm/proxy/response_polling/background_streaming.py

@@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming
 import asyncio
 import json
 from collections.abc import AsyncIterable, Callable, Mapping, Sequence
+from time import perf_counter
 from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable
 
 from fastapi import Request, Response
@@ -128,6 +129,16 @@ async def background_streaming_task(
     https://platform.openai.com/docs/api-reference/responses/object
     """
 
+    started_at: Final = perf_counter()
+    first_token_at: float | None = None  # rebind-ok: first content event closes the TTFT interval
+    finished_at: float | None = None  # rebind-ok: terminal event closes the execution interval
+
+    def timing_snapshot() -> Mapping[str, float]:
+        return {
+            **({"time_to_first_token_ms": (first_token_at - started_at) * 1000} if first_token_at is not None else {}),
+            **({"total_latency_ms": (finished_at - started_at) * 1000} if finished_at is not None else {}),
+        }
+
     try:
         verbose_proxy_logger.info("Starting background streaming for %s", polling_id)
 
@@ -191,7 +202,7 @@ async def background_streaming_task(
         incomplete_details_data = None
 
         state_dirty = False  # Track if state needs to be synced
-        last_update_time = asyncio.get_event_loop().time()
+        last_update_time = perf_counter()
         UPDATE_INTERVAL: Final = 0.150  # 150ms batching interval
 
         # Track the terminal event from the stream (may not be "completed")
@@ -210,13 +221,14 @@ async def background_streaming_task(
             """Flush accumulated state to Redis if interval elapsed or forced"""
             nonlocal state_dirty, last_update_time
 
-            current_time: Final = asyncio.get_event_loop().time()
+            current_time: Final = perf_counter()
             if state_dirty and (force or (current_time - last_update_time) >= UPDATE_INTERVAL):
                 # Convert output_items dict to list for update
                 output_list: Final = list(output_items.values())
                 await polling_handler.update_state(
                     polling_id=polling_id,
                     output=output_list,
+                    timing=timing_snapshot(),
                 )
                 state_dirty = False
                 last_update_time = current_time
@@ -242,6 +254,20 @@ async def background_streaming_task(
                         event: _StreamEvent = _StreamEventParser.parse(chunk_data)
                         event_type = event.get("type", "")
 
+                        if (
+                            first_token_at is None
+                            and event_type
+                            in (
+                                "response.output_text.delta",
+                                "response.reasoning_text.delta",
+                                "response.reasoning_summary_text.delta",
+                                "response.reasoning.delta",
+                            )
+                            and event.get("delta")
+                        ):
+                            first_token_at = perf_counter()
+                            state_dirty = True
+
                         # Process different event types based on OpenAI streaming spec
                         if event_type == "response.output_item.added":
                             # New output item added
@@ -334,6 +360,7 @@ async def background_streaming_task(
                         ):
                             # Terminal event - extract all ResponsesAPIResponse fields
                             # https://platform.openai.com/docs/api-reference/responses-streaming
+                            finished_at = perf_counter()
                             response_data = event.get("response", {})
                             terminal_status = response_data.get(
                                 "status",
@@ -384,10 +411,13 @@ async def background_streaming_task(
 
         # Use the terminal status from the stream, default to "completed"
         final_status: Final = terminal_status or "completed"
+        if finished_at is None:
+            finished_at = perf_counter()  # rebind-ok: close the interval when the stream has no terminal event
 
         await polling_handler.update_state(
             polling_id=polling_id,
             status=final_status,
+            timing=timing_snapshot(),
             usage=usage_data,
             error=terminal_error,
             reasoning=reasoning_data,
@@ -417,6 +447,7 @@ async def background_streaming_task(
         )
 
     except Exception as e:
+        finished_at = perf_counter()  # rebind-ok: capture execution time at failure
         verbose_proxy_logger.error("Error in background streaming task for %s: %s", polling_id, e)
         import traceback
 
@@ -425,6 +456,7 @@ async def background_streaming_task(
         await polling_handler.update_state(
             polling_id=polling_id,
             status="failed",
+            timing=timing_snapshot(),
             error={
                 "type": "internal_error",
                 "message": str(e),

litellm/proxy/response_polling/polling_handler.py

@@ -3,9 +3,9 @@ Response Polling Handler for Background Responses with Cache
 """
 
 import json
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 from pydantic import TypeAdapter
 
@@ -15,6 +15,9 @@ from litellm._uuid import uuid4
 from litellm.caching.redis_cache import RedisCache
 from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus
 
+if TYPE_CHECKING:
+    from litellm.router import Router
+
 _RESPONSE_POLLING_TARGET: Final = "response_polling"
 _JSON_OBJECT: Final = TypeAdapter(dict[str, object])
 _JSON_LIST: Final = TypeAdapter(list[object])
@@ -116,6 +119,7 @@ class ResponsePollingHandler:
         parallel_tool_calls: bool | None = None,
         user: str | None = None,
         store: bool | None = None,
+        timing: Mapping[str, float] | None = None,
     ) -> None:
         """
         Update the polling state in Redis
@@ -144,6 +148,7 @@ class ResponsePollingHandler:
             parallel_tool_calls: Whether parallel tool calls are enabled
             user: User identifier
             store: Whether to store the response
+            timing: Server execution timings in milliseconds
         """
         if not self.redis_cache:
             return
@@ -167,6 +172,9 @@ class ResponsePollingHandler:
         if output is not None:
             state["output"] = output
 
+        if timing is not None:
+            state["litellm_response_timing"] = timing
+
         # Update usage
         if usage:
             state["usage"] = usage
@@ -264,10 +272,10 @@ class ResponsePollingHandler:
 
 def should_use_polling_for_request(
     background_mode: bool,
-    polling_via_cache_enabled,  # Can be False, "all", or List[str]
-    redis_cache,  # RedisCache or None
+    polling_via_cache_enabled: bool | str | Sequence[str],
+    redis_cache: RedisCache | None,
     model: str,
-    llm_router,  # Router instance or None
+    llm_router: "Router | None",
     native_background_mode: list[str] | None = None,  # List of models that should use native background mode
 ) -> bool:
     """

litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py

@@ -216,6 +216,20 @@ class _SettingsWithSchema(LiteLLMBaseModel):
     field_schema: dict[str, object]
 
 
+class BackgroundChatSettings(LiteLLMBaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    enabled: bool = Field(description="Enable cache-backed background Responses for Chat and API requests.")
+
+
+class BackgroundChatSettingsResponse(BackgroundChatSettings):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    redis_configured: bool = Field(description="Whether this proxy has coordination Redis configured.")
+    managed_by_config: bool = Field(description="Whether the responses configuration is owned by the config file.")
+    can_save: bool = Field(description="Whether database-backed configuration updates are available.")
+
+
 class SSOSettingsResponse(SettingsResponse):
     """Response model for SSO settings"""
 
@@ -1815,6 +1829,87 @@ async def sync_ui_settings_to_general_settings(
     return apply_runtime_general_settings_flags(parsed)
 
 
+def _background_chat_settings_response(config: Mapping[str, object]) -> BackgroundChatSettingsResponse:
+    from litellm.proxy import proxy_server
+
+    settings: Final = _as_settings_section(config.get("litellm_settings"))
+    responses: Final = _as_settings_section(settings.get("responses"))
+    background: Final = _as_settings_section(responses.get("background_mode"))
+    polling: Final = background.get("polling_via_cache", False)
+    return BackgroundChatSettingsResponse(
+        enabled=bool(polling),
+        redis_configured=proxy_server.redis_usage_cache is not None,
+        managed_by_config=proxy_server.proxy_config.litellm_settings.owned_by_config("responses"),
+        can_save=proxy_server.prisma_client is not None and proxy_server.store_model_in_db is True,
+    )
+
+
+@router.get(
+    "/get/background_chat_settings",
+    tags=["Settings"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=BackgroundChatSettingsResponse,
+)
+async def get_background_chat_settings() -> BackgroundChatSettingsResponse:
+    """Get saved background-response settings and the prerequisites for changing them."""
+    from litellm.proxy.proxy_server import proxy_config
+
+    config: Final = _as_settings_section(_DECODED_JSON.validate_python(await proxy_config.get_config()))
+    return _background_chat_settings_response(config)
+
+
+@router.patch(
+    "/update/background_chat_settings",
+    tags=["Settings"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=BackgroundChatSettingsResponse,
+)
+async def update_background_chat_settings(
+    settings: BackgroundChatSettings,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+) -> BackgroundChatSettingsResponse:
+    """Save background-response polling settings for the proxy configuration refresh."""
+    from litellm.proxy.proxy_server import (
+        create_config_audit_log,
+        prisma_client,
+        proxy_config,
+        redis_usage_cache,
+        store_model_in_db,
+    )
+
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise HTTPException(status_code=403, detail="Only proxy admins can update background chat settings.")
+    if prisma_client is None or store_model_in_db is not True:
+        raise HTTPException(status_code=400, detail="Enable database-backed configuration to save this setting.")
+    if settings.enabled and redis_usage_cache is None:
+        raise HTTPException(status_code=400, detail="Configure coordination Redis and restart the proxy first.")
+
+    raw_config: Final = await proxy_config.get_config()
+    config: Final = _as_settings_section(_DECODED_JSON.validate_python(raw_config))
+    litellm_settings: Final = _as_settings_section(config.get("litellm_settings"))
+    responses: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(litellm_settings.get("responses")))
+    background: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(responses.get("background_mode")))
+    updated_responses: Final[Mapping[str, JsonValue]] = {
+        **responses,
+        "background_mode": {**background, "polling_via_cache": "all" if settings.enabled else False},
+    }
+    proxy_config.reject_config_owned_writes(
+        section_name="litellm_settings", changed_keys={"responses": dict(updated_responses)}
+    )
+    raw_config["litellm_settings"] = {**litellm_settings, "responses": dict(updated_responses)}
+    await proxy_config.save_config(new_config=config)
+    asyncio.create_task(
+        create_config_audit_log(
+            param_name="responses.background_mode.polling_via_cache",
+            action="updated",
+            before_value=background.get("polling_via_cache", False),
+            after_value="all" if settings.enabled else False,
+            user_api_key_dict=user_api_key_dict,
+        )
+    )
+    return _background_chat_settings_response(config)
+
+
 @router.get(
     "/get/ui_settings",
     tags=["UI Settings"],

litellm/responses/litellm_completion_transformation/count_request.py

@@ -128,6 +128,15 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object], drop
     return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)}
 
 
+def _caller_input(responses_request: Mapping[str, object]) -> ResponseInputParam | None:
+    responses_input: Final = responses_request.get("input")
+    return (
+        cast(ResponseInputParam, responses_input)  # cast-ok: forwarded as sent, as inference does
+        if isinstance(responses_input, list)
+        else None
+    )
+
+
 def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]:
     tools: Final = responses_request.get("tools")
     if tools is None:
@@ -200,6 +209,7 @@ async def chat_count_request(
                 ),
                 custom_llm_provider,
                 policy.drop_params,
+                caller_input=_caller_input(responses_request),
             ),
             model,
             custom_llm_provider,

litellm/responses/litellm_completion_transformation/handler.py

@@ -4,6 +4,7 @@ Handler for transforming responses api requests to litellm.completion requests
 
 from collections.abc import Coroutine, Mapping, Sequence
 from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Final
 
 from pydantic import TypeAdapter
@@ -32,6 +33,13 @@ from litellm.types.llms.openai import (
 from litellm.types.utils import ModelResponse
 
 _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
+_BRIDGE_TRANSFORM_KEYWORDS: Final = frozenset({"caller_input", "names_non_string_fields"})
+
+
+def _without_bridge_transform_keywords(request_kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    return MappingProxyType(
+        {key: value for key, value in request_kwargs.items() if key not in _BRIDGE_TRANSFORM_KEYWORDS}
+    )
 
 
 def _chat_verbosity(responses_api_request: Mapping[str, object]) -> Mapping[str, object]:
@@ -95,6 +103,7 @@ class LiteLLMCompletionTransformationHandler:
                 previous_response_id,
                 hoisted,
                 model=model,
+                caller_input=input,
                 responses_api_request=responses_api_request,
                 custom_llm_provider=custom_llm_provider,
                 stream=stream,
@@ -108,11 +117,13 @@ class LiteLLMCompletionTransformationHandler:
             **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
                 model=model,
                 input=bridged_input,
+                caller_input=input,
+                names_non_string_fields=True,
                 responses_api_request=bridged_request,
                 custom_llm_provider=custom_llm_provider,
                 stream=stream,
                 extra_headers=extra_headers,
-                **kwargs,
+                **_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)),
             ),
             **_chat_verbosity(bridged_request),
         }
@@ -163,6 +174,7 @@ class LiteLLMCompletionTransformationHandler:
         /,
         *,
         model: str,
+        caller_input: str | ResponseInputParam,
         responses_api_request: ResponsesAPIOptionalRequestParams,
         custom_llm_provider: str | None,
         stream: bool | None,
@@ -186,11 +198,13 @@ class LiteLLMCompletionTransformationHandler:
                 **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
                     model=model,
                     input=bridged.input,
+                    caller_input=caller_input,
+                    names_non_string_fields=True,
                     responses_api_request=bridged.responses_api_request,
                     custom_llm_provider=custom_llm_provider,
                     stream=stream,
                     extra_headers=extra_headers,
-                    **request_kwargs,
+                    **_without_bridge_transform_keywords(request_kwargs),
                 ),
                 **_chat_verbosity(bridged.responses_api_request),
             },

litellm/responses/litellm_completion_transformation/transformation.py

@@ -45,6 +45,7 @@ from litellm.constants import (
     REDACTED_BY_LITELLM,
     REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
 )
+from litellm.exceptions import BadRequestError
 from litellm.integrations.anthropic_cache_control_hook import (
     AnthropicCacheControlHook,
     block_with_cache_control,
@@ -65,6 +66,12 @@ from litellm.litellm_core_utils.hidden_params import (
 )
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    invalid_type_message,
+    is_present_non_string,
+    is_unhashable_value_behind,
+)
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
 from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
 from litellm.llms.anthropic.messages.native.utils import anthropic_stop_reason, openai_chat_refusal_text
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@@ -268,6 +275,7 @@ _IMAGE_URL_ADAPTER: Final = TypeAdapter(ChatCompletionImageUrlObject)
 _FILE_PART_ADAPTER: Final = TypeAdapter(_ResponsesFilePart)
 _MessageT = TypeVar("_MessageT")
 _MediaT: Final = TypeVar("_MediaT")
+_InputItemT: Final = TypeVar("_InputItemT")
 _CACHED_CONTENT_ADAPTER: Final = TypeAdapter(ChatCompletionCachedContent)
 _DICT_ITEMS_LIST_ADAPTER: Final = TypeAdapter(list[dict[object, object]])
 _TEXT_ADAPTER: Final = TypeAdapter(str)
@@ -445,6 +453,86 @@ TOOL_CALL_OUTPUT_ITEM_TYPES: Final = frozenset(
 )
 
 
+def _read_positions(
+    input_items: Sequence[_InputItemT], read_input: str | ResponseInputParam
+) -> Iterator[tuple[int, _InputItemT]]:
+    read_ids: Final = frozenset(map(id, parts_of(read_input)))
+    return ((index, item) for index, item in enumerate(input_items) if id(item) in read_ids)
+
+
+def _first_invalid_input_type(fields: Iterator[tuple[str, object]]) -> BadRequestError | None:
+    first: Final = next(fields, None)
+    return None if first is None else _invalid_input_type(*first)
+
+
+def _is_tool_output_item(item: object) -> bool:
+    item_type: Final = message_field(item, "type")
+    return isinstance(item_type, str) and item_type in TOOL_CALL_OUTPUT_ITEM_TYPES
+
+
+def _converts_before(
+    convert_input: Callable[[str | ResponseInputParam], object], sent_input: str | ResponseInputParam, position: int
+) -> bool:
+    try:
+        convert_input(sent_input[:position])
+    except TypeError:
+        return False
+    return True
+
+
+def _unhashable_tool_output_call_id_error(
+    input: str | ResponseInputParam,
+    read_input: str | ResponseInputParam,
+    error: TypeError,
+    converts_before: Callable[[int], bool],
+) -> BadRequestError | None:
+    call_ids: Final = (
+        (f"input[{index}].call_id", message_field(item, "call_id"), item)
+        for index, item in _read_positions(parts_of(input), read_input)
+        if _is_tool_output_item(item)
+    )
+    first: Final = next((candidate for candidate in call_ids if is_unhashable_value_behind(error, candidate[1])), None)
+    if first is None:
+        return None
+    path, call_id, item = first
+    read_position: Final = next(
+        position for position, read_item in enumerate(parts_of(read_input)) if read_item is item
+    )
+    return _invalid_input_type(path, call_id) if converts_before(read_position) else None
+
+
+def _carries_non_string_field(message: object) -> bool:
+    tool_call_ids: Final = (
+        message_field(tool_call, "id") for tool_call in parts_of(message_field(message, "tool_calls"))
+    )
+    return is_present_non_string(message_field(message, "role")) or any(map(is_present_non_string, tool_call_ids))
+
+
+def _non_string_item_fields(
+    item: Mapping[str, object], convert_item: Callable[[Mapping[str, object]], Sequence[object]]
+) -> Iterator[tuple[str, object]]:
+    messages: Final = convert_item(item)
+    tool_call_id_field: Final = "call_id" if item.get("call_id") else "id"
+    roles: Final = (("role", message_field(message, "role")) for message in messages)
+    tool_calls: Final = chain.from_iterable(parts_of(message_field(message, "tool_calls")) for message in messages)
+    tool_call_ids: Final = ((tool_call_id_field, message_field(tool_call, "id")) for tool_call in tool_calls)
+    return ((field, value) for field, value in chain(roles, tool_call_ids) if is_present_non_string(value))
+
+
+def _indexed_non_string_item_fields(
+    input: ResponseInputParam,
+    read_input: str | ResponseInputParam,
+    convert_item: Callable[[Mapping[str, object]], Sequence[object]],
+) -> Iterator[tuple[str, object]]:
+    for index, item in _read_positions(input, read_input):
+        for field, value in _non_string_item_fields(item, convert_item):
+            yield f"input[{index}].{field}", value
+
+
+def _invalid_input_type(field_path: str, value: object) -> BadRequestError:
+    return BadRequestError(message=invalid_type_message(field_path, value), model="", llm_provider="")
+
+
 def _input_item_role(input_item: Mapping[str, object]) -> str:
     return cast(str, input_item.get("role") or "user")  # cast-ok: client-supplied role forwarded verbatim, unvalidated
 
@@ -1101,6 +1189,9 @@ class LiteLLMCompletionResponsesConfig:
         stream: bool | None = None,
         extra_headers: Mapping[str, object] | None = None,
         drop_params: object = None,
+        *,
+        caller_input: str | ResponseInputParam | None = None,
+        names_non_string_fields: bool = False,
         **kwargs,
     ) -> dict:
         """
@@ -1149,6 +1240,7 @@ class LiteLLMCompletionResponsesConfig:
         native_compaction: Final = _replays_compaction_block(
             request_input, context_management
         ) and provider_takes_compaction_blocks(model, custom_llm_provider)
+        numbered_input: Final = request_input if caller_input is None else caller_input
         bridged_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
             input=request_input,
             responses_api_request=responses_api_request,
@@ -1157,7 +1249,28 @@ class LiteLLMCompletionResponsesConfig:
             tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
             native_compaction=native_compaction,
             only_since_compaction=not native_compaction,
+            caller_input=numbered_input,
         )
+        malformed_field_error: Final = (
+            _first_invalid_input_type(
+                _indexed_non_string_item_fields(
+                    numbered_input,
+                    _input_to_read(request_input, not native_compaction, native_compaction),
+                    partial(
+                        LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message,
+                        replay_reasoning=True,
+                        native_compaction=native_compaction,
+                    ),
+                )
+            )
+            if names_non_string_fields
+            and isinstance(numbered_input, list)
+            and any(map(_carries_non_string_field, bridged_messages))
+            and provider_builds_anthropic_request(model, custom_llm_provider)
+            else None
+        )
+        if malformed_field_error is not None:
+            raise malformed_field_error
 
         litellm_completion_request: dict = {
             "messages": (
@@ -1221,6 +1334,7 @@ class LiteLLMCompletionResponsesConfig:
         tool_call_provider_fields: bool = False,
         native_compaction: bool = False,
         only_since_compaction: bool = False,
+        caller_input: str | ResponseInputParam | None = None,
     ) -> list[
         AllMessageValues
         | GenericChatCompletionMessage
@@ -1274,13 +1388,27 @@ class LiteLLMCompletionResponsesConfig:
 
         dumped_input: Final = with_compaction_items_as_params(input)
         read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction)
-        messages.extend(
-            LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
-                input=_without_item_keys(read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)),
-                replay_reasoning=replay_reasoning,
-                native_compaction=native_compaction,
-            )
+        sent_input: Final = _without_item_keys(
+            read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)
         )
+        convert_input: Final = partial(
+            LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message,
+            replay_reasoning=replay_reasoning,
+            native_compaction=native_compaction,
+        )
+        try:
+            converted: Final = convert_input(sent_input)
+        except TypeError as error:
+            call_id_error: Final = _unhashable_tool_output_call_id_error(
+                dumped_input if caller_input is None else caller_input,
+                read_input,
+                error,
+                partial(_converts_before, convert_input, sent_input),
+            )
+            if call_id_error is None:
+                raise
+            raise call_id_error from error
+        messages.extend(converted)
 
         return messages
 

litellm/types/llms/cohere.py

@@ -1,5 +1,6 @@
 from typing import Literal
 
+from pydantic import AliasPath, BaseModel, ConfigDict, Field
 from typing_extensions import Required, TypedDict
 
 
@@ -98,3 +99,15 @@ class CohereV2ChatResponse(TypedDict):
     message: CohereV2ChatResponseMessage
     usage: CohereV2ChatResponseUsage
     logprobs: CohereV2ChatResponseLogProbs
+
+
+class CohereV2StreamedToolCall(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    id: str | None = Field(default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "id"))
+    name: str | None = Field(
+        default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "function", "name")
+    )
+    arguments: str = Field(
+        default="", validation_alias=AliasPath("delta", "message", "tool_calls", "function", "arguments")
+    )

litellm/types/proxy/management_endpoints/model_management_endpoints.py

@@ -35,6 +35,9 @@ class PatchModelRequest(updateDeployment):
 
 
 class ModelGroupInfoProxy(ModelGroupInfo):
+    supports_background_chat: bool = Field(
+        default=False, description="Whether cache-backed background Responses runs are enabled for this model group"
+    )
     is_public_model_group: bool = Field(default=False)
     health_status: str | None = Field(default=None)
     health_response_time: float | None = Field(default=None)

tests/base_sdk_tests/check_sdk_http.py

@@ -70,7 +70,9 @@ def enqueue(body: object, status: int = 200) -> None:
 
 def enqueue_stream() -> None:
     chunk: Final = {**CHAT, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "pong"}}]}
-    RESPONSES.put((200, f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode()))
+    terminal: Final = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
+    stream: Final = f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(terminal)}\n\ndata: [DONE]\n\n"
+    RESPONSES.put((200, stream.encode()))
 
 
 def bedrock_event(event: str, payload: object) -> bytes:

tests/unit/caching/test_request_redis_batch_pre_call.py

@@ -290,7 +290,11 @@ def _router(redis_cache: FakeRedisCache, routing_strategy: str = "usage-based-ro
 
 
 @pytest.mark.asyncio
-async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own():
+async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own(monkeypatch):
+    monkeypatch.setattr(
+        "litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime",
+        lambda: datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc),
+    )
     client = FakeClient(_lua_ok_replies)
     redis_cache = FakeRedisCache(client)
     router = _router(redis_cache)

tests/unit/integrations/test_anthropic_cache_control_hook.py

@@ -5042,3 +5042,50 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi
         {"type": "ephemeral"},
         {"type": "ephemeral"},
     ]
+
+
+def _search_conversation_with(follow_up: dict[str, object]) -> list[AllMessageValues]:
+    return [
+        cast(AllMessageValues, {"role": "user", "content": "Weather in Paris?"}),
+        _replayed_search_call(_SEARCH_CALL),
+        cast(AllMessageValues, follow_up),
+        cast(AllMessageValues, _GO_ON),
+    ]
+
+
+def _cache_marks_after_injection(messages: list[AllMessageValues]) -> tuple[int, tuple[int, ...]]:
+    _, processed, _ = AnthropicCacheControlHook().get_chat_completion_prompt(
+        model="anthropic/claude-opus-5-5",
+        messages=messages,
+        non_default_params={"cache_control_injection_points": [{"location": "message", "role": "user"}]},
+        prompt_id=None,
+        prompt_variables=None,
+        dynamic_callback_params={},
+    )
+    return (
+        AnthropicCacheControlHook.count_request_cache_breakpoints(messages),
+        tuple(_marks_in(message) for message in processed),
+    )
+
+
+@pytest.mark.parametrize(
+    ("malformed", "well_formed"),
+    [
+        pytest.param(
+            {**_SEARCH_ANSWER, "tool_call_id": {"id": "srvtoolu_01Search"}},
+            {**_SEARCH_ANSWER, "tool_call_id": "toolu_01Unrelated"},
+            id="object_tool_call_id",
+        ),
+        pytest.param(
+            {**_SEARCH_ANSWER, "tool_call_id": ["srvtoolu_01Search"]},
+            {**_SEARCH_ANSWER, "tool_call_id": "toolu_01Unrelated"},
+            id="list_tool_call_id",
+        ),
+    ],
+)
+def test_cache_control_hook_reads_a_non_string_answer_id_as_answering_no_tool_call(
+    malformed: dict[str, object], well_formed: dict[str, object]
+):
+    assert _cache_marks_after_injection(_search_conversation_with(malformed)) == _cache_marks_after_injection(
+        _search_conversation_with(well_formed)
+    )

tests/unit/litellm_core_utils/test_core_helpers.py

@@ -319,6 +319,12 @@ class TestMapFinishReasonCohere:
     def test_error(self):
         assert map_finish_reason("ERROR") == "stop"
 
+    def test_tool_call(self):
+        assert map_finish_reason("TOOL_CALL") == "tool_calls"
+
+    def test_stop_sequence(self):
+        assert map_finish_reason("STOP_SEQUENCE") == "stop"
+
 
 class TestMapFinishReasonHuggingFace:
     def test_eos_token(self):

tests/unit/litellm_core_utils/test_json_fragment_accumulator.py

@@ -1,6 +1,5 @@
 import json
 import time
-import tracemalloc
 from collections.abc import Iterator
 from typing import Final
 from unittest.mock import patch
@@ -193,34 +192,15 @@ def _streamed(fragments: tuple[str, ...]) -> JSONFragmentAccumulator:
     return accumulator
 
 
-def _drain_peak_bytes(fragments: tuple[str, ...]) -> int:
-    accumulator: Final = _streamed(fragments)
-    first_found, _ = accumulator.pop_next_value()
-    assert first_found is True
-    already_tracing: Final = tracemalloc.is_tracing()
-    if not already_tracing:
-        tracemalloc.start()
-    tracemalloc.reset_peak()
-    baseline, _ = tracemalloc.get_traced_memory()
-    try:
-        drained: Final = sum(1 for _ in _popped_values(accumulator))
-        _, peak = tracemalloc.get_traced_memory()
-    finally:
-        if not already_tracing:
-            tracemalloc.stop()
-    assert drained == len(fragments) - 1
-    return peak - baseline
-
-
-def test_draining_many_concatenated_values_never_copies_the_remainder():
+def test_draining_many_concatenated_values_yields_every_value_in_order():
     value: Final = '{"text": "' + "x" * 20 + '"}'
     fragments: Final = tuple(value if index % 2 else f" {value}" for index in range(40_000))
-    payload_bytes: Final = sum(len(fragment) for fragment in fragments)
-    peak_bytes: Final = _drain_peak_bytes(fragments)
-    assert peak_bytes < payload_bytes // 4, (
-        f"draining {len(fragments)} values allocated a peak of {peak_bytes} bytes over a {payload_bytes} byte payload, "
-        "copying the remainder on every pop?"
-    )
+    accumulator: Final = _streamed(fragments)
+
+    drained: Final = tuple(_popped_values(accumulator))
+
+    assert drained == (json.loads(value),) * len(fragments)
+    assert accumulator.pop_next_value() == (False, None)
 
 
 def test_could_close_json_after_many_blank_fragments_is_not_quadratic():

tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -25,7 +25,7 @@ from litellm.constants import (
 from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
 from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
 from litellm.llms.anthropic.chat import ModelResponseIterator
-from litellm.llms.anthropic.common_utils import process_anthropic_headers
+from litellm.llms.anthropic.common_utils import AnthropicError, process_anthropic_headers
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema
 from litellm.llms.anthropic.messages.native.transformation import (
@@ -9922,6 +9922,382 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
     assert litellm_params["metadata"] == {}
 
 
+_KEPT_SEARCH_CALL: Final = {
+    "role": "assistant",
+    "content": None,
+    "tool_calls": [
+        {
+            "id": "srvtoolu_01Search",
+            "type": "function",
+            "function": {"name": "tool_search_tool_bm25", "arguments": "{}"},
+        }
+    ],
+    "provider_specific_fields": {
+        "tool_results": [
+            {
+                "type": "tool_search_tool_result",
+                "tool_use_id": "srvtoolu_01Search",
+                "content": {"type": "tool_search_tool_search_result", "tool_references": []},
+            }
+        ]
+    },
+}
+_WEATHER_CALL: Final = {
+    "role": "assistant",
+    "content": None,
+    "tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}],
+}
+_ASK: Final = {"role": "user", "content": "Weather in Paris?"}
+
+
+def _search_answered_with(tool_call_id: object) -> list[dict[str, object]]:
+    return [
+        _ASK,
+        _KEPT_SEARCH_CALL,
+        {"role": "tool", "tool_call_id": tool_call_id, "content": "found get_weather"},
+        {"role": "user", "content": "Go on"},
+    ]
+
+
+def _orphan_answer_with(tool_call_id: object) -> list[dict[str, object]]:
+    return [
+        _ASK,
+        {"role": "assistant", "content": "Checking"},
+        {"role": "tool", "tool_call_id": tool_call_id, "content": "sunny"},
+        {"role": "user", "content": "Go on"},
+    ]
+
+
+@pytest.mark.parametrize(
+    ("messages", "field_path", "type_name"),
+    [
+        pytest.param(
+            [
+                {"role": "user", "content": "Hi"},
+                {"role": "assistant", "content": "Hello"},
+                {"role": ["user"], "content": "Go on"},
+                {"role": "system", "content": "Answer in one sentence."},
+            ],
+            "messages[2].role",
+            "list",
+            id="list_role_before_a_later_system_message",
+        ),
+        pytest.param(
+            [
+                {"role": "user", "content": "Hi"},
+                {"role": "assistant", "content": "Hello"},
+                {"role": ["user"], "content": "Go on"},
+            ],
+            "messages[2].role",
+            "list",
+            id="list_role",
+        ),
+        pytest.param(
+            [
+                {"role": "user", "content": "Hi"},
+                {"role": "assistant", "content": "Hello"},
+                {"role": 123, "content": "Go on"},
+                {"role": "system", "content": "Answer in one sentence."},
+            ],
+            "messages[2].role",
+            "int",
+            id="int_role_before_a_later_system_message",
+        ),
+        pytest.param(
+            [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": True, "content": "Go on"}],
+            "messages[3].role",
+            "bool",
+            id="bool_role_after_an_orphan_object_tool_call_id",
+        ),
+        pytest.param(
+            _search_answered_with({"id": "srvtoolu_01Search"}),
+            "messages[2].tool_call_id",
+            "dict",
+            id="object_tool_call_id_after_a_kept_server_result",
+        ),
+        pytest.param(
+            _search_answered_with(["srvtoolu_01Search"]),
+            "messages[2].tool_call_id",
+            "list",
+            id="list_tool_call_id_after_a_kept_server_result",
+        ),
+        pytest.param(
+            [_ASK, _WEATHER_CALL, {"role": "tool", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"}],
+            "messages[2].tool_call_id",
+            "dict",
+            id="object_tool_call_id_answering_a_client_tool_call",
+        ),
+        pytest.param(
+            [
+                _ASK,
+                _WEATHER_CALL,
+                {"role": "function", "name": "get_weather", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"},
+            ],
+            "messages[2].tool_call_id",
+            "dict",
+            id="object_tool_call_id_on_a_function_message",
+        ),
+        pytest.param(
+            [
+                _ASK,
+                {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]},
+                {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"},
+            ],
+            "messages[1].tool_calls[0].id",
+            "dict",
+            id="object_assistant_tool_call_id",
+        ),
+        pytest.param(
+            [
+                _ASK,
+                {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": 7}]},
+                {"role": "tool", "tool_call_id": "7", "content": "sunny"},
+            ],
+            "messages[1].tool_calls[0].id",
+            "int",
+            id="int_assistant_tool_call_id",
+        ),
+        pytest.param(
+            [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": ["user"], "content": "Go on"}],
+            "messages[3].role",
+            "list",
+            id="list_role_after_an_orphan_object_tool_call_id",
+        ),
+        pytest.param(
+            [
+                *_orphan_answer_with({"id": "toolu_01"})[:3],
+                {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]},
+                {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"},
+            ],
+            "messages[3].tool_calls[0].id",
+            "dict",
+            id="object_assistant_tool_call_id_after_an_orphan_object_tool_call_id",
+        ),
+        pytest.param(
+            [
+                {**_ASK, "tool_call_id": {"id": "stray"}},
+                _WEATHER_CALL,
+                {"role": "tool", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"},
+            ],
+            "messages[2].tool_call_id",
+            "dict",
+            id="object_tool_call_id_after_a_stray_id_on_a_user_message",
+        ),
+        pytest.param(
+            [
+                {**_ASK, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "stray"}}]},
+                {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]},
+                {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"},
+            ],
+            "messages[1].tool_calls[0].id",
+            "dict",
+            id="object_assistant_tool_call_id_after_stray_tool_calls_on_a_user_message",
+        ),
+        pytest.param(
+            [_ASK, {"role": [], "content": "ok"}, {"role": "system", "content": "Be terse."}],
+            "messages[1].role",
+            "list",
+            id="empty_list_role_before_a_system_message",
+        ),
+        pytest.param(
+            [_ASK, {"role": {}, "content": "ok"}, {"role": "user", "content": "again"}],
+            "messages[1].role",
+            "dict",
+            id="empty_object_role",
+        ),
+    ],
+)
+@pytest.mark.parametrize("modify_params", [False, True])
+@pytest.mark.parametrize("model", [UNFLAGGED_CLAUDE, FLAGGED_CLAUDE])
+def test_chat_transform_request_answers_400_naming_a_non_string_role_or_tool_call_id(
+    local_model_cost_map,
+    monkeypatch: pytest.MonkeyPatch,
+    messages: list[dict[str, object]],
+    field_path: str,
+    type_name: str,
+    modify_params: bool,
+    model: str,
+):
+    monkeypatch.setattr(litellm, "modify_params", modify_params)
+
+    with pytest.raises(AnthropicError) as raised:
+        _chat_request(AnthropicConfig(), model, copy.deepcopy(messages))
+
+    assert (raised.value.status_code, raised.value.message) == (
+        400,
+        f"Invalid type for '{field_path}': expected a string, got {type_name}",
+    )
+
+
+@pytest.mark.parametrize(("role", "type_name"), [(["user"], "list"), ({"name": "user"}, "dict"), ([], "list")])
+def test_chat_transform_request_names_a_non_string_role_on_a_cache_marked_message_under_drop_params(
+    local_model_cost_map, role: object, type_name: str
+):
+    marked: Final = {"role": role, "content": [{"type": "text", "text": "ok"}], "cache_control": {"type": "ephemeral"}}
+
+    with pytest.raises(AnthropicError) as raised:
+        AnthropicConfig().transform_request(
+            model=UNFLAGGED_CLAUDE,
+            messages=[_ASK, marked],
+            optional_params={},
+            litellm_params={"drop_params": True},
+            headers={},
+        )
+
+    assert (raised.value.status_code, raised.value.message) == (
+        400,
+        f"Invalid type for 'messages[1].role': expected a string, got {type_name}",
+    )
+
+
+@pytest.mark.parametrize(
+    ("malformed", "well_formed", "modify_params"),
+    [
+        pytest.param(
+            [{**_ASK, "tool_call_id": {"id": "stray"}}], [_ASK], False, id="stray_tool_call_id_on_a_user_message"
+        ),
+        pytest.param(
+            [{**_ASK, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}],
+            [_ASK],
+            False,
+            id="stray_tool_calls_on_a_user_message",
+        ),
+        pytest.param(
+            _orphan_answer_with({"id": "toolu_01"}),
+            _orphan_answer_with("toolu_01"),
+            True,
+            id="orphan_answer_dropped_under_modify_params",
+        ),
+    ],
+)
+def test_chat_transform_request_keeps_converting_a_non_string_id_it_never_reads(
+    local_model_cost_map,
+    monkeypatch: pytest.MonkeyPatch,
+    malformed: list[dict[str, object]],
+    well_formed: list[dict[str, object]],
+    modify_params: bool,
+):
+    monkeypatch.setattr(litellm, "modify_params", modify_params)
+
+    converted: Final = _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, copy.deepcopy(malformed))
+
+    assert (
+        converted["messages"]
+        == _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, copy.deepcopy(well_formed))["messages"]
+    )
+
+
+def test_chat_transform_request_keeps_a_conversion_error_that_a_dropped_orphan_answer_did_not_cause(
+    local_model_cost_map, monkeypatch: pytest.MonkeyPatch
+):
+    monkeypatch.setattr(litellm, "modify_params", True)
+
+    with pytest.raises(AnthropicError) as raised:
+        _chat_request(
+            AnthropicConfig(),
+            UNFLAGGED_CLAUDE,
+            [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": "bogus", "content": "Go on"}],
+        )
+
+    assert raised.value.status_code == 400
+    assert "Invalid Message passed in {'role': 'bogus'" in raised.value.message
+
+
+@pytest.mark.parametrize(
+    ("later_turn", "real_cause"),
+    [
+        pytest.param(
+            [{"role": "user", "content": ["hi", "there"]}],
+            "'str' object has no attribute 'get'",
+            id="string_content_parts",
+        ),
+        pytest.param(
+            [
+                {
+                    **_WEATHER_CALL,
+                    "tool_calls": [
+                        {
+                            **_WEATHER_CALL["tool_calls"][0],
+                            "function": {"name": "get_weather", "arguments": {"city": "Paris"}},
+                        }
+                    ],
+                },
+                {"role": "tool", "tool_call_id": "toolu_01", "content": "rain"},
+            ],
+            "'dict' object has no attribute 'strip'",
+            id="object_tool_call_arguments",
+        ),
+        pytest.param(
+            [{"role": "user", "content": [{"type": "image_url", "image_url": ["x"]}]}],
+            "list indices must be integers or slices, not str",
+            id="list_image_url",
+        ),
+    ],
+)
+@pytest.mark.parametrize("dropped_answer_id", [7, {"id": "toolu_01"}], ids=["int", "object"])
+def test_chat_transform_request_never_blames_a_dropped_tool_answer_id_for_another_failure(
+    local_model_cost_map,
+    monkeypatch: pytest.MonkeyPatch,
+    later_turn: list[dict[str, object]],
+    real_cause: str,
+    dropped_answer_id: object,
+):
+    monkeypatch.setattr(litellm, "modify_params", True)
+
+    with pytest.raises(AnthropicError) as raised:
+        _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, [*_orphan_answer_with(dropped_answer_id)[:3], *later_turn])
+
+    assert raised.value.status_code == 400
+    assert raised.value.message.startswith(real_cause)
+
+
+def test_chat_transform_request_never_blames_a_numeric_assistant_tool_call_id_for_an_earlier_failure(
+    local_model_cost_map,
+):
+    with pytest.raises(AnthropicError) as raised:
+        _chat_request(
+            AnthropicConfig(),
+            UNFLAGGED_CLAUDE,
+            [
+                {"role": "user", "content": ["hi", "there"]},
+                {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [{"id": 5, "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}],
+                },
+                {"role": "tool", "tool_call_id": "5", "content": "rain"},
+            ],
+        )
+
+    assert raised.value.status_code == 400
+    assert raised.value.message.startswith("'str' object has no attribute 'get'")
+
+
+@pytest.mark.parametrize("empty_role", [0, False])
+def test_chat_transform_request_never_blames_an_empty_role_for_another_failure(local_model_cost_map, empty_role):
+    list_part_type_after_an_empty_role: Final = [
+        _ASK,
+        {"role": empty_role, "content": "ok"},
+        {"role": "user", "content": [{"type": ["text"], "text": "again"}]},
+        {"role": "system", "content": "Be terse."},
+    ]
+
+    with pytest.raises(TypeError):
+        _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, list_part_type_after_an_empty_role)
+
+
+def test_chat_transform_request_names_no_field_for_a_failure_outside_roles_and_tool_call_ids(local_model_cost_map):
+    list_part_type: Final = [
+        _ASK,
+        {"role": "assistant", "content": "ok"},
+        {"role": "user", "content": [{"type": ["text"], "text": "again"}]},
+        {"role": "system", "content": "Be terse."},
+    ]
+
+    with pytest.raises(TypeError):
+        _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, list_part_type)
+
+
 def _five_cache_hint_conversation() -> list[dict[str, object]]:
     return [
         {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},

tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py

@@ -614,9 +614,18 @@ def test_clear_tool_inputs_list_clears_only_the_named_tools():
     }
 
 
+def _history_with_first_block_field(index: int, field: str, value: object) -> list[dict[str, object]]:
+    history: Final = _make_history(n_pairs=4)
+    return [
+        {**message, "content": [{**message["content"][0], field: value}, *message["content"][1:]]}
+        if position == index
+        else message
+        for position, message in enumerate(history)
+    ]
+
+
 def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject():
-    messages = _make_history(n_pairs=4)
-    messages[1]["content"][0]["id"] = ["toolu_00"]
+    messages: Final = _history_with_first_block_field(1, "id", ["toolu_00"])
     new_messages, applied = apply_clear_tool_uses_20250919(
         model=MODEL,
         messages=messages,
@@ -635,6 +644,24 @@ def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject():
     assert (new_messages[3]["content"][0]["input"], new_messages[5]["content"][0]["input"]) == ({}, {})
 
 
+def test_a_tool_result_with_a_non_string_tool_use_id_is_left_for_the_provider_to_reject():
+    messages: Final = _history_with_first_block_field(2, "tool_use_id", ["toolu_00"])
+    new_messages, applied = apply_clear_tool_uses_20250919(
+        model=MODEL,
+        messages=messages,
+        tools=None,
+        system=None,
+        edit_spec={
+            "type": "clear_tool_uses_20250919",
+            "trigger": {"type": "tool_uses", "value": 0},
+            "keep": {"type": "tool_uses", "value": 1},
+        },
+    )
+    assert applied is not None
+    assert new_messages[2] == messages[2]
+    assert _cleared_ids(new_messages[3:]) == {"toolu_01", "toolu_02"}
+
+
 async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicContextManagementError:
     with pytest.raises(AnthropicContextManagementError) as raised:
         await apply_context_management(

tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py

@@ -4472,6 +4472,17 @@ def test_file_content_conversion_keeps_a_line_whose_json_holds_unicode_line_sepa
     assert bedrock_record["modelInput"]["messages"] == [{"role": "user", "content": [{"text": text}]}]
 
 
+_CLAUDE_BODY_WITH_A_LIST_ROLE_BEFORE_A_SYSTEM_MESSAGE: Final = {
+    "model": "bedrock/us.anthropic.claude-haiku-5-5",
+    "max_tokens": 20,
+    "messages": [
+        {"role": "user", "content": "hi"},
+        {"role": ["user"], "content": "ok"},
+        {"role": "system", "content": "Be terse."},
+    ],
+}
+
+
 def _batch_line(body: object) -> str:
     return json.dumps({**_VALID_BATCH_RECORD, "body": body})
 
@@ -4516,6 +4527,11 @@ def _batch_line_with_body(field: str, value: object) -> str:
             "Batch input file line 2 could not be converted to a Bedrock batch record",
             id="string-messages",
         ),
+        pytest.param(
+            f"{_VALID_BATCH_LINE}\n{_batch_line(_CLAUDE_BODY_WITH_A_LIST_ROLE_BEFORE_A_SYSTEM_MESSAGE)}\n".encode(),
+            "Batch input file line 2 could not be converted to a Bedrock batch record",
+            id="claude-list-role-before-a-system-message",
+        ),
         pytest.param(
             f"{_VALID_BATCH_LINE}\n{'[' * 1_000_000}{']' * 1_000_000}\n".encode(),
             "Batch input file line 2 could not be parsed",

tests/unit/llms/cohere/chat/test_cohere_transformation.py

@@ -1,3 +1,4 @@
+import itertools
 import json
 from collections.abc import Callable, Iterator
 from typing import Final
@@ -653,9 +654,7 @@ async def test_cohere_v2_annotations(respx_mock: respx.MockRouter) -> None:
 def test_cohere_v2_parameter_mapping(respx_mock: respx.MockRouter) -> None:
     messages: Final = [{"role": "user", "content": "Generate a creative story."}]
     route: Final = respx_mock.post(COHERE_V2_CHAT_URL).mock(
-        return_value=httpx.Response(
-            200, json=_cohere_v2_response({"content": [{"type": "text", "text": "A story."}]})
-        )
+        return_value=httpx.Response(200, json=_cohere_v2_response({"content": [{"type": "text", "text": "A story."}]}))
     )
 
     litellm.completion(
@@ -705,3 +704,123 @@ def test_cohere_v2_error_handling(respx_mock: respx.MockRouter) -> None:
     assert error.value.status_code == 400
     assert error.value.llm_provider == "cohere"
     assert "invalid request: model 'invalid-model' not found" in error.value.message
+
+
+def _cohere_v2_tool_call_events(index: int, call_id: str, city: str) -> tuple[dict[str, object], ...]:
+    return (
+        {
+            "type": "tool-call-start",
+            "index": index,
+            "delta": {
+                "message": {
+                    "tool_calls": {
+                        "id": call_id,
+                        "type": "function",
+                        "function": {"name": "get_weather", "arguments": ""},
+                    }
+                }
+            },
+        },
+        {
+            "type": "tool-call-delta",
+            "index": index,
+            "delta": {"message": {"tool_calls": {"function": {"arguments": '{"city": '}}}},
+        },
+        {
+            "type": "tool-call-delta",
+            "index": index,
+            "delta": {"message": {"tool_calls": {"function": {"arguments": json.dumps(city) + "}"}}}},
+        },
+        {"type": "tool-call-end", "index": index},
+    )
+
+
+@pytest.mark.usefixtures("_cohere_httpx_transport")
+@pytest.mark.respx(assert_all_called=True)
+async def test_cohere_v2_stream_returns_parallel_tool_calls(respx_mock: respx.MockRouter) -> None:
+    stream_events: Final = (
+        {"type": "message-start", "id": "c0a2b6e1", "delta": {"message": {"role": "assistant"}}},
+        *_cohere_v2_tool_call_events(0, "get_weather_paris", "Paris"),
+        *_cohere_v2_tool_call_events(1, "get_weather_tokyo", "Tokyo"),
+        {
+            "type": "message-end",
+            "delta": {"finish_reason": "TOOL_CALL", "usage": {"tokens": {"input_tokens": 40, "output_tokens": 30}}},
+        },
+    )
+    respx_mock.post(COHERE_V2_CHAT_URL).mock(
+        return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode())
+    )
+
+    response: Final = await litellm.acompletion(
+        model="cohere_chat/v2/command-a-plus-05-2026",
+        messages=[{"role": "user", "content": "What is the weather in Paris and in Tokyo?"}],
+        tools=[
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
+                },
+            }
+        ],
+        stream=True,
+        api_key="cohere-test-key",
+    )
+    chunks: Final = [chunk async for chunk in response]
+    rebuilt: Final = litellm.stream_chunk_builder(chunks)
+
+    assert rebuilt is not None
+    assert [
+        (call.id, call.function.name, json.loads(call.function.arguments))
+        for call in rebuilt.choices[0].message.tool_calls
+    ] == [
+        ("get_weather_paris", "get_weather", {"city": "Paris"}),
+        ("get_weather_tokyo", "get_weather", {"city": "Tokyo"}),
+    ]
+    assert chunks[-1].choices[0].finish_reason == "tool_calls"
+
+
+@pytest.mark.usefixtures("_cohere_httpx_transport")
+@pytest.mark.respx(assert_all_called=True)
+async def test_cohere_v2_stream_closes_argument_less_tool_call_with_empty_object(respx_mock: respx.MockRouter) -> None:
+    stream_events: Final = (
+        {"type": "message-start", "id": "5d1e9f20", "delta": {"message": {"role": "assistant"}}},
+        {
+            "type": "tool-call-start",
+            "index": 0,
+            "delta": {
+                "message": {
+                    "tool_calls": {
+                        "id": "get_server_time_0",
+                        "type": "function",
+                        "function": {"name": "get_server_time", "arguments": ""},
+                    }
+                }
+            },
+        },
+        {"type": "tool-call-end", "index": 0},
+        {"type": "message-end", "delta": {"finish_reason": "TOOL_CALL"}},
+    )
+    respx_mock.post(COHERE_V2_CHAT_URL).mock(
+        return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode())
+    )
+
+    response: Final = await litellm.acompletion(
+        model="cohere_chat/v2/command-a-plus-05-2026",
+        messages=[{"role": "user", "content": "What time is it on the server?"}],
+        tools=[
+            {
+                "type": "function",
+                "function": {"name": "get_server_time", "parameters": {"type": "object", "properties": {}}},
+            }
+        ],
+        stream=True,
+        api_key="cohere-test-key",
+    )
+    deltas_per_chunk: Final = [chunk.choices[0].delta.tool_calls or () async for chunk in response]
+    streamed_calls: Final = tuple(itertools.chain.from_iterable(deltas_per_chunk))
+
+    assert [(call.id, call.function.name) for call in streamed_calls if call.id] == [
+        ("get_server_time_0", "get_server_time")
+    ]
+    assert json.loads("".join(call.function.arguments for call in streamed_calls)) == {}

tests/unit/proxy/response_polling/test_response_polling_handler.py

@@ -1917,3 +1917,71 @@ async def test_create_initial_state_rejects_request_metadata_that_is_not_an_obje
     assert [(error["loc"], error["type"], repr(error["input"])) for error in rejected.value.errors()] == [
         (("metadata",), "dict_type", repr(metadata))
     ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "delta_type",
+    [
+        "response.output_text.delta",
+        "response.reasoning_text.delta",
+        "response.reasoning_summary_text.delta",
+        "response.reasoning.delta",
+        None,
+    ],
+)
+async def test_background_timings_survive_retrieval_after_completion(delta_type):
+    from litellm.caching.redis_cache import RedisCache
+    from litellm.proxy.response_polling.background_streaming import background_streaming_task
+
+    cache = {}
+    redis = AsyncMock(spec=RedisCache)
+    redis.async_get_cache.side_effect = lambda key: cache.get(key)
+
+    async def save_cache(key, value, **kwargs):
+        cache[key] = value
+
+    redis.async_set_cache.side_effect = save_cache
+    handler = ResponsePollingHandler(redis_cache=redis)
+    await handler.create_initial_state("litellm_poll_timing", {"model": "gpt-4o"})
+    clock = [100.0]
+
+    async def stream():
+        clock[0] = 101.0
+        yield 'data: {"type":"response.output_text.delta","delta":""}\n\n'
+        yield 'data: {"type":"response.mcp_call.in_progress"}\n\n'
+        if delta_type:
+            clock[0] = 102.0
+            yield f"data: {json.dumps({'type': delta_type, 'delta': 'Hello'})}\n\n"
+            partial = await handler.get_state("litellm_poll_timing")
+            assert partial is not None
+            assert partial["status"] == "in_progress"
+            assert partial["litellm_response_timing"] == {"time_to_first_token_ms": 2000.0}
+        clock[0] = 110.0
+        yield 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}\n\n'
+        clock[0] = 5000.0
+        yield "data: [DONE]\n\n"
+
+    from starlette.responses import StreamingResponse
+
+    with (
+        patch(
+            "litellm.proxy.response_polling.background_streaming.perf_counter",
+            create=True,
+            side_effect=lambda: clock[0],
+        ),
+        patch(
+            # test-quality-ok: processor is constructed inside the task; supply the upstream SSE boundary like sibling tests
+            "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
+        ) as processor,
+    ):
+        processor.return_value.base_process_llm_request = AsyncMock(return_value=StreamingResponse(stream()))
+        await background_streaming_task(**_make_background_streaming_kwargs("litellm_poll_timing", handler))
+        state = await handler.get_state("litellm_poll_timing")
+        assert state is not None
+        expected = {"total_latency_ms": 10000.0}
+        if delta_type:
+            expected["time_to_first_token_ms"] = 2000.0
+        assert state["litellm_response_timing"] == expected
+        clock[0] = 10000.0
+        assert (await handler.get_state("litellm_poll_timing"))["litellm_response_timing"] == expected

tests/unit/proxy/test_proxy_server.py

@@ -4561,3 +4561,88 @@ async def test_text_completion_provider_rejection_captures_unknown_usage(
     assert envelope["usage"] is None
     assert envelope["usage_source"] == "unknown"
     assert envelope["pricing"]["reported_cost"] is None
+
+
+@pytest.mark.parametrize(
+    "polling,has_redis,native,expected",
+    [
+        (False, True, False, False),
+        ("all", False, False, False),
+        ("all", True, True, False),
+        ("all", True, False, True),
+        (["openai"], True, False, True),
+        (["anthropic"], True, False, False),
+    ],
+)
+def test_model_groups_advertise_cache_backed_background_chat(
+    monkeypatch: pytest.MonkeyPatch,
+    polling: bool | str | list[str],
+    has_redis: bool,
+    native: bool,
+    expected: bool,
+) -> None:
+    from litellm.caching.redis_cache import RedisCache
+    from litellm.proxy import proxy_server
+
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": "research-chat",
+                "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "fake-test-key"},
+            }
+        ]
+    )
+    monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", polling)
+    monkeypatch.setattr(
+        proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379) if has_redis else None
+    )
+    monkeypatch.setattr(proxy_server, "native_background_mode", ["research-chat"] if native else [])
+    groups: Final = proxy_server.get_model_group_info(router, ["research-chat"], None)
+    assert groups[0].supports_background_chat is expected
+
+
+@pytest.mark.parametrize("config_owned", [False, True])
+def test_background_response_settings_refresh_and_respect_config(
+    monkeypatch: pytest.MonkeyPatch,
+    config_owned: bool,
+) -> None:
+    from litellm.proxy import proxy_server
+
+    monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False)
+    monkeypatch.setattr(proxy_server, "native_background_mode", [])
+    monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600)
+    config: Final = proxy_server.ProxyConfig()
+    if config_owned:
+        config.litellm_settings.load_yaml(
+            {
+                "responses": {
+                    "background_mode": {
+                        "polling_via_cache": ["openai"],
+                        "native_background_mode": ["config-native"],
+                        "ttl": 900,
+                    }
+                },
+            }
+        )
+    config._apply_litellm_settings_db_values(
+        {
+            "responses": {
+                "background_mode": {
+                    "polling_via_cache": "all",
+                    "native_background_mode": ["db-native"],
+                    "ttl": 7200,
+                }
+            },
+        }
+    )
+    assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else "all")
+    assert proxy_server.native_background_mode == (["config-native"] if config_owned else ["db-native"])
+    assert proxy_server.polling_cache_ttl == (900 if config_owned else 7200)
+    config._apply_litellm_settings_db_values(
+        {
+            "responses": {"background_mode": {"polling_via_cache": False}},
+        }
+    )
+    assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else False)
+    assert proxy_server.native_background_mode == (["config-native"] if config_owned else [])
+    assert proxy_server.polling_cache_ttl == (900 if config_owned else 3600)

tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py

@@ -4488,3 +4488,114 @@ def test_usage_default_range_reports_config_precedence(mock_auth, monkeypatch: p
     assert response.status_code == 200, response.text
     assert response.json()["values"]["usage_default_time_range"] == "month_to_date"
     assert response.json()["source"]["usage_default_time_range"] == "config"
+
+
+@pytest.fixture
+def background_chat_config(monkeypatch: pytest.MonkeyPatch):
+    from copy import deepcopy
+    from unittest.mock import AsyncMock, MagicMock
+
+    from litellm.caching.redis_cache import RedisCache
+    from litellm.proxy import proxy_server
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+    from litellm.proxy.config_resolvers import SettingsStore
+
+    config = {
+        "litellm_settings": {
+            "callbacks": ["prometheus"],
+            "responses": {
+                "background_mode": {
+                    "polling_via_cache": False,
+                    "ttl": 7200,
+                    "native_background_mode": ["native-model"],
+                },
+            },
+        }
+    }
+    store = SettingsStore("litellm_settings")
+    monkeypatch.setattr(proxy_server.proxy_config, "litellm_settings", store)
+    monkeypatch.setattr(proxy_server.proxy_config, "_settings_stores", {"litellm_settings": store})
+    monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379))
+    monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False)
+    monkeypatch.setattr(proxy_server, "native_background_mode", [])
+    monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600)
+    monkeypatch.setattr(proxy_server.proxy_config, "get_config", AsyncMock(side_effect=lambda: deepcopy(config)))
+
+    async def save_config(*, new_config):
+        config.clear()
+        config.update(deepcopy(new_config))
+
+    save = AsyncMock(side_effect=save_config)
+    audit = AsyncMock()
+    monkeypatch.setattr(proxy_server.proxy_config, "save_config", save)
+    monkeypatch.setattr(proxy_server, "create_config_audit_log", audit)
+    app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
+        user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
+    )
+    yield config, save, audit
+    app.dependency_overrides.pop(user_api_key_auth, None)
+
+
+def test_background_chat_settings_persist_preserving_existing_options(background_chat_config, monkeypatch):
+    from litellm.proxy import proxy_server
+
+    config, _, _ = background_chat_config
+    response = client.patch("/update/background_chat_settings", json={"enabled": True})
+    assert response.status_code == 200
+    assert response.json()["enabled"] is True
+    assert proxy_server.polling_via_cache_enabled is False
+    assert config["litellm_settings"]["responses"]["background_mode"] == {
+        "polling_via_cache": "all",
+        "ttl": 7200,
+        "native_background_mode": ["native-model"],
+    }
+    assert config["litellm_settings"]["callbacks"] == ["prometheus"]
+    assert client.get("/get/background_chat_settings").json()["enabled"] is True
+    proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"])
+    assert proxy_server.polling_via_cache_enabled == "all"
+    disabled = client.patch("/update/background_chat_settings", json={"enabled": False})
+    assert disabled.status_code == 200
+    assert disabled.json()["enabled"] is False
+    proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"])
+    assert proxy_server.polling_via_cache_enabled is False
+
+
+@pytest.mark.parametrize("blocker", ["role", "redis", "config", "persistence"])
+def test_background_chat_settings_reject_unavailable_updates(background_chat_config, monkeypatch, blocker):
+    from litellm.proxy import proxy_server
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    config, save, _ = background_chat_config
+    if blocker == "role":
+        app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
+            user_id="viewer", user_role=LitellmUserRoles.INTERNAL_USER
+        )
+    elif blocker == "redis":
+        monkeypatch.setattr(proxy_server, "redis_usage_cache", None)
+        assert client.get("/get/background_chat_settings").json()["redis_configured"] is False
+    elif blocker == "config":
+        proxy_server.proxy_config.litellm_settings.load_yaml(config["litellm_settings"])
+        assert client.get("/get/background_chat_settings").json()["managed_by_config"] is True
+    else:
+        monkeypatch.setattr(proxy_server, "store_model_in_db", False)
+        assert client.get("/get/background_chat_settings").json()["can_save"] is False
+    response = client.patch("/update/background_chat_settings", json={"enabled": True})
+    assert response.status_code == (403 if blocker == "role" else 400)
+    save.assert_not_awaited()
+    assert config["litellm_settings"]["responses"]["background_mode"]["polling_via_cache"] is False
+
+
+def test_background_chat_settings_failed_save_does_not_enable_runtime(background_chat_config):
+    from litellm.proxy import proxy_server
+
+    _, save, audit = background_chat_config
+    save.side_effect = RuntimeError("Database write failed")
+    with pytest.raises(RuntimeError, match="Database write failed"):
+        client.patch("/update/background_chat_settings", json={"enabled": True})
+    assert client.get("/get/background_chat_settings").json()["enabled"] is False
+    assert proxy_server.polling_via_cache_enabled is False
+    audit.assert_not_awaited()

tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py

@@ -1598,6 +1598,53 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search
     assert prisma_client.db.queries == 1
 
 
+@pytest.mark.asyncio
+async def test_a_codex_additional_tools_item_sends_claude_its_tools_but_not_its_text() -> None:
+    anthropic: Final = _RecordingRawAnthropicMessages()
+
+    await litellm.aresponses(
+        model=_MODEL,
+        input=[
+            {"type": "additional_tools", "role": "developer", "content": "Reply PINEAPPLE.", "tools": [_SHELL_TOOL]},
+            {"role": "user", "content": "Hi"},
+        ],
+        api_key="sk-ant-fake",
+        client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)),
+    )
+
+    sent: Final = json.loads(anthropic.body)
+    assert [tool["name"] for tool in sent["tools"]] == ["shell"]
+    assert "PINEAPPLE" not in json.dumps(sent)
+
+
+@pytest.mark.asyncio
+async def test_previous_response_id_names_a_malformed_item_after_a_codex_additional_tools_item_by_its_index() -> None:
+    weather_tool: Final = {
+        "type": "function",
+        "name": "get_weather",
+        "parameters": {"type": "object", "properties": {}},
+    }
+
+    with (
+        patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SEARCH_TURN, _CALENDAR_TURN])),
+        pytest.raises(litellm.BadRequestError) as raised,
+    ):
+        await litellm.aresponses(
+            model=_MODEL,
+            previous_response_id="chatcmpl-calendar-turn",
+            input=[
+                {"type": "additional_tools", "role": "developer", "tools": [weather_tool]},
+                {"type": "function_call_output", "call_id": "toolu_standup", "output": "Created Standup."},
+                {"role": ["user"], "content": "Now create another event titled Retro."},
+            ],
+            tools=[_CODEX_TOOL_SEARCH, _SHELL_TOOL],
+            api_key="sk-ant-fake",
+            client=AsyncHTTPHandler(transport=httpx.MockTransport(_RecordingRawAnthropicMessages())),
+        )
+
+    assert "'input[2].role'" in raised.value.message
+
+
 _REFUSAL_EXPLANATION: Final = ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"]["explanation"]
 _ANSWER_BEFORE_REFUSAL: Final = {"type": "text", "text": "Here is the start."}
 
@@ -1808,7 +1855,11 @@ _RAN_SEARCH_RESULT: Final = {
         {"type": "web_search_result", "url": "https://tokyo.example", "title": "Tokyo", "encrypted_content": "e"}
     ],
 }
-_THINKING_BEFORE_SEARCHES: Final = {"type": "thinking", "thinking": "Find each sister city in turn.", "signature": "sig"}
+_THINKING_BEFORE_SEARCHES: Final = {
+    "type": "thinking",
+    "thinking": "Find each sister city in turn.",
+    "signature": "sig",
+}
 _TEXT_BEFORE_PENDING_SEARCH: Final = {"type": "text", "text": "Tokyo pairs with Paris. Checking Paris next."}
 _PENDING_SEARCH: Final = {
     "type": "server_tool_use",

tests/unit/responses/litellm_completion_transformation/test_count_request.py

@@ -86,7 +86,13 @@ async def test_claude_count_request_counts_additional_tools_input_items_as_tools
     policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset())
 
     hoisted: Final = await _claude_count(
-        {"input": [{"type": "additional_tools", "tools": [_WEATHER_TOOL]}, _QUESTION]}, policy
+        {
+            "input": [
+                {"type": "additional_tools", "role": "developer", "content": "Reply PINEAPPLE.", "tools": [_WEATHER_TOOL]},
+                _QUESTION,
+            ]
+        },
+        policy,
     )
 
     assert isinstance(hoisted, ChatCountRequest)
@@ -129,6 +135,17 @@ async def test_claude_count_request_rejects_a_malformed_tool_flag_without_drop_p
     assert "defer_loading" in flagged.message
 
 
+@pytest.mark.asyncio
+async def test_claude_count_request_leaves_a_non_string_role_for_the_count_endpoint_to_judge() -> None:
+    counted: Final = await _claude_count(
+        {"input": [{"type": "additional_tools", "tools": [_WEATHER_TOOL]}, _QUESTION, {"role": ["user"], "content": "?"}]},
+        DropPolicy(drop_params=False, additional_drop_params=frozenset()),
+    )
+
+    assert isinstance(counted, ChatCountRequest)
+    assert [message.get("role") for message in counted.messages] == ["user", ["user"]]
+
+
 @pytest.mark.asyncio
 async def test_claude_count_request_leaves_out_tools_the_deployment_drops() -> None:
     policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset({"tools"}))

tests/unit/responses/litellm_completion_transformation/test_handler.py

@@ -246,6 +246,26 @@ async def test_bridged_turn_with_a_client_field_named_previous_response_client_i
     assert isinstance(response, ResponsesAPIResponse)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("client_field", ["caller_input", "names_non_string_fields"])
+async def test_bridged_turn_with_a_client_field_named_like_a_bridge_keyword_reaches_the_provider(client_field: str):
+    provider: Final = _RecordingAnthropicHandler(_ANTHROPIC_MESSAGE_PAYLOAD)
+    client: Final = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
+
+    response = await litellm.aresponses(
+        model="azure_ai/claude-sonnet-4-6",
+        api_base="https://fake-foundry-resource.services.ai.azure.com",
+        api_key="fake-api-key",
+        input="Double it",
+        client=client,
+        **{client_field: "from the client body"},
+    )
+
+    assert provider.request_body is not None, "the bridged turn never reached the provider"
+    assert isinstance(response, ResponsesAPIResponse)
+
+
 class _StoredTurns:
     def __init__(self, rows: tuple[Mapping[str, object], ...]) -> None:
         self.rows: Final = rows
@@ -310,3 +330,32 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h
         ("assistant", json.dumps([{"type": "text", "text": "Noted."}])),
         ("user", json.dumps([{"type": "text", "text": "Double it"}])),
     ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("client_field", ["caller_input", "names_non_string_fields"])
+async def test_bridged_follow_up_after_a_client_tool_search_with_a_client_field_named_like_a_bridge_keyword_reaches_the_provider(
+    client_field: str,
+):
+    previous_response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id(
+        custom_llm_provider="azure_ai", model_id="deployment-a", response_id="chatcmpl-turn-one"
+    )
+    stored_turns: Final = _StoredTurns((_stored_mcp_turn(previous_response_id),))
+    provider: Final = _RecordingAnthropicHandler(_ANTHROPIC_MESSAGE_PAYLOAD)
+    client: Final = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
+
+    with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored_turns)):
+        response = await litellm.aresponses(
+            model="azure_ai/claude-sonnet-4-6",
+            api_base="https://fake-foundry-resource.services.ai.azure.com",
+            api_key="fake-api-key",
+            input="Double it",
+            tools=[{"type": "tool_search", "execution": "client"}],
+            previous_response_id=previous_response_id,
+            client=client,
+            **{client_field: "from the client body"},
+        )
+
+    assert provider.request_body is not None, "the bridged turn never reached the provider"
+    assert isinstance(response, ResponsesAPIResponse)

tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py

@@ -18,6 +18,7 @@ from openai.types.responses.response_function_web_search import (
 from pydantic import TypeAdapter, ValidationError
 
 import litellm
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field
 from litellm.constants import CALLER_SESSION_ID_METADATA_KEY
 from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
@@ -7189,3 +7190,207 @@ async def test_bridge_partial_response_on_disconnect_bills_only_the_input_since_
     assert await _input_tokens_billed_on_disconnect(
         cast(ResponseInputParam, [*history_it_replaces, *since_compaction])
     ) == await _input_tokens_billed_on_disconnect(since_compaction)
+
+
+_ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-5-5"
+_WEATHER_ASK: Final = {"role": "user", "content": "Weather in Paris?"}
+
+
+def _weather_call(**fields: object) -> dict[str, object]:
+    return {"type": "function_call", "name": "get_weather", "arguments": "{}", **fields}
+
+
+def _bridge_to(model: str, history: list[dict[str, object]], **request: object) -> dict:
+    return LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
+        model=model,
+        input=cast(ResponseInputParam, history),
+        responses_api_request=cast(ResponsesAPIOptionalRequestParams, request),
+        names_non_string_fields=True,
+    )
+
+
+@pytest.mark.parametrize("output_type", ["function_call_output", "custom_tool_call_output"])
+@pytest.mark.parametrize(("call_id", "type_name"), [({"id": "call_weather"}, "dict"), (["call_weather"], "list")])
+def test_a_tool_output_with_an_unhashable_call_id_is_a_400_naming_its_input_item(
+    output_type: str, call_id: object, type_name: str
+) -> None:
+    history: Final = cast(
+        ResponseInputParam,
+        [_WEATHER_ASK, _weather_call(call_id="call_weather"), {"type": output_type, "call_id": call_id, "output": "sunny"}],
+    )
+
+    with pytest.raises(litellm.BadRequestError) as raised:
+        LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+            input=history, responses_api_request={}
+        )
+
+    assert raised.value.message == (
+        f"litellm.BadRequestError: Invalid type for 'input[2].call_id': expected a string, got {type_name}"
+    )
+
+
+def test_a_malformed_call_id_after_a_compaction_item_is_named_by_its_index_in_the_request() -> None:
+    history: Final = cast(
+        ResponseInputParam,
+        [
+            {"role": "user", "content": "Plan a Tokyo trip."},
+            {"role": "assistant", "content": "Day one, Asakusa."},
+            {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)},
+            _weather_call(call_id="call_weather"),
+            {"type": "function_call_output", "call_id": {"id": "call_weather"}, "output": "sunny"},
+        ],
+    )
+
+    with pytest.raises(litellm.BadRequestError) as raised:
+        LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+            input=history, responses_api_request={}, only_since_compaction=True
+        )
+
+    assert "'input[4].call_id'" in raised.value.message
+
+
+@pytest.mark.parametrize("call_id", [7, {"id": "call_weather"}, ["call_weather"]], ids=["int", "object", "list"])
+def test_a_conversion_failure_elsewhere_is_never_blamed_on_a_call_id(call_id: object) -> None:
+    history: Final = cast(
+        ResponseInputParam,
+        [
+            _WEATHER_ASK,
+            {"type": ["message"], "role": "user", "content": "Go on"},
+            {"type": "function_call_output", "call_id": call_id, "output": "sunny"},
+        ],
+    )
+
+    with pytest.raises((TypeError, litellm.BadRequestError)) as raised:
+        LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+            input=history, responses_api_request={}
+        )
+
+    assert "call_id" not in str(raised.value)
+
+
+def test_a_tool_output_with_an_int_call_id_converts_like_its_string_form() -> None:
+    def converted(call_id: object) -> list[object]:
+        return list(
+            LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+                input=cast(
+                    ResponseInputParam,
+                    [_WEATHER_ASK, {"type": "function_call_output", "call_id": call_id, "output": "sunny"}],
+                ),
+                responses_api_request={},
+            )
+        )
+
+    assert converted(7) == converted("7")
+
+
+def test_a_non_string_role_still_converts_for_a_target_that_judges_it_itself() -> None:
+    history: Final = [{"role": "user", "content": "Hi"}, {"role": ["user"], "content": "Go on"}]
+
+    shared: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+        input=cast(ResponseInputParam, history), responses_api_request={}
+    )
+    bridged: Final = _bridge_to("gemini/gemini-3.8-flash", history)
+
+    assert [message_field(message, "role") for message in shared] == ["user", ["user"]]
+    assert [message_field(message, "role") for message in bridged["messages"]] == ["user", ["user"]]
+
+
+@pytest.mark.parametrize(
+    ("history", "field_path", "type_name"),
+    [
+        pytest.param(
+            [{"role": "user", "content": "Hi"}, {"role": ["user"], "content": "Go on"}],
+            "input[1].role",
+            "list",
+            id="list_role",
+        ),
+        pytest.param(
+            [{"role": "user", "content": "Hi"}, {"role": {"name": "user"}, "content": "Go on"}],
+            "input[1].role",
+            "dict",
+            id="dict_role",
+        ),
+        pytest.param(
+            [{"role": "user", "content": "Hi"}, {"role": 7, "content": "Go on"}], "input[1].role", "int", id="int_role"
+        ),
+        pytest.param(
+            [_WEATHER_ASK, _weather_call(call_id={"id": "call_weather"})],
+            "input[1].call_id",
+            "dict",
+            id="dict_function_call_id",
+        ),
+        pytest.param(
+            [_WEATHER_ASK, _weather_call(call_id=7), {"type": "function_call_output", "call_id": "7", "output": "sunny"}],
+            "input[1].call_id",
+            "int",
+            id="int_function_call_id",
+        ),
+        pytest.param(
+            [_WEATHER_ASK, _weather_call(id=["fc_weather"])], "input[1].id", "list", id="list_function_call_item_id"
+        ),
+        pytest.param(
+            [
+                {"role": "user", "content": "Plan a Tokyo trip."},
+                {"role": "assistant", "content": "Day one, Asakusa."},
+                {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)},
+                {"role": ["user"], "content": "And Osaka?"},
+            ],
+            "input[3].role",
+            "list",
+            id="list_role_after_a_compaction_item",
+        ),
+    ],
+)
+def test_a_non_string_field_bridged_to_anthropic_is_a_400_naming_its_input_item(
+    history: list[dict[str, object]], field_path: str, type_name: str
+) -> None:
+    with pytest.raises(litellm.BadRequestError) as raised:
+        _bridge_to(_ANTHROPIC_MODEL, history, instructions="Be terse.")
+
+    assert raised.value.message == (
+        f"litellm.BadRequestError: Invalid type for '{field_path}': expected a string, got {type_name}"
+    )
+
+
+def test_a_non_string_role_on_an_item_that_never_reads_it_still_bridges_to_anthropic() -> None:
+    history: Final = [
+        _WEATHER_ASK,
+        _weather_call(call_id="call_weather", role=["x"]),
+        {"type": "function_call_output", "call_id": "call_weather", "output": "sunny", "role": ["x"]},
+    ]
+
+    bridged: Final = _bridge_to(_ANTHROPIC_MODEL, history)
+
+    assert [message_field(message, "role") for message in bridged["messages"]] == ["user", "assistant", "tool"]
+
+
+@pytest.mark.parametrize(
+    ("history", "field_path"),
+    [
+        pytest.param([_WEATHER_ASK, {"role": ["user"], "content": "Go on"}], "input[2].role", id="role"),
+        pytest.param(
+            [
+                _WEATHER_ASK,
+                _weather_call(call_id="call_weather"),
+                {"type": "function_call_output", "call_id": {"id": "call_weather"}, "output": "sunny"},
+            ],
+            "input[3].call_id",
+            id="tool_output_call_id",
+        ),
+    ],
+)
+def test_a_malformed_item_after_a_codex_additional_tools_item_is_named_by_its_index_in_the_request(
+    history: list[dict[str, object]], field_path: str
+) -> None:
+    tools_item: Final = {
+        "type": "additional_tools",
+        "role": "developer",
+        "tools": [{"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}],
+    }
+
+    with pytest.raises(litellm.BadRequestError) as raised:
+        litellm.responses(
+            model=_ANTHROPIC_MODEL, api_key="sk-ant-test", input=cast(ResponseInputParam, [tools_item, *history])
+        )
+
+    assert f"'{field_path}'" in raised.value.message

tests/unit/test_router/core/test_endpoints.py

@@ -414,6 +414,7 @@ def _scripted_stream(*texts: str) -> Callable[..., CustomStreamWrapper]:
         async def chunks() -> AsyncIterator[ModelResponseStream]:
             for text in texts:
                 yield ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}])
+            yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}])
 
         logging_obj: Final = LiteLLMLogging(
             model="d1",

ui/litellm-dashboard/eslint-suppressions.json

@@ -1142,11 +1142,6 @@
       "count": 1
     }
   },
-  "src/app/chat/page.tsx": {
-    "no-nested-ternary": {
-      "count": 1
-    }
-  },
   "src/app/login/LoginPage.tsx": {
     "local/no-complex-jsx-arrow": {
       "count": 1
@@ -1737,10 +1732,10 @@
       "count": 5
     },
     "no-restricted-syntax": {
-      "count": 139
+      "count": 133
     },
     "prefer-const": {
-      "count": 28
+      "count": 26
     }
   },
   "src/components/object_permissions_view.tsx": {

ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx

@@ -9,7 +9,7 @@ const { getSettings } = vi.hoisted(() => ({
   getSettings: vi.fn().mockResolvedValue({ values: { enable_projects_ui: true } }),
 }));
 
-vi.mock("@/components/networking", () => ({ getUiSettings: getSettings, getUISettings: getSettings }));
+vi.mock("@/components/networking", () => ({ getUiSettings: getSettings }));
 vi.mock("@/components/leftnav", () => ({
   default: ({ enableProjectsUI }: ComponentProps<typeof Sidebar>) => (
     <nav>{enableProjectsUI && <a href="/projects">Projects</a>}</nav>

ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx

@@ -0,0 +1,229 @@
+import { act, fireEvent, screen, waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import { renderWithProviders, testQueryClient } from "@/../tests/test-utils";
+import { ChatShellProvider } from "@/contexts/ChatShellContext";
+import type { Conversation } from "@/components/chat/types";
+import ChatConversationPage from "./page";
+
+vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn(), replace: vi.fn() }) }));
+
+const storageKey = "litellm_chat_history_v1:background-user";
+const requests: Record<string, unknown>[] = [];
+let supported: boolean;
+let status: "in_progress" | "completed" | "cancelled";
+let retrievalStatus: number;
+let cancellations: number;
+let acceptance: Promise<void> | undefined;
+const saved = (): Conversation[] => JSON.parse(localStorage.getItem(storageKey) ?? "[]");
+const result = () => ({
+  id: "litellm_poll_research",
+  status,
+  usage: status === "completed" ? { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost: "0.125" } : undefined,
+  litellm_response_timing:
+    status === "completed" ? { time_to_first_token_ms: 250, total_latency_ms: 12500 } : undefined,
+  output:
+    status === "completed"
+      ? [
+          {
+            type: "mcp_call",
+            id: "call_inventory",
+            name: "lookup_inventory",
+            arguments: "{}",
+            output: "12 widgets",
+            server_label: "inventory",
+          },
+          { type: "message", content: [{ type: "output_text", text: "There are 12 widgets in stock." }] },
+        ].filter((item) => item.type !== "mcp_call" || Boolean(requests.at(-1)?.tools))
+      : [],
+});
+
+function renderChat(id?: string) {
+  return renderWithProviders(
+    <ChatShellProvider
+      accessToken="test-token"
+      userId="background-user"
+      userEmail="researcher@example.com"
+      userRole="Internal Viewer"
+      premiumUser={false}
+    >
+      <ChatConversationPage />
+    </ChatShellProvider>,
+    { searchParams: id ? { id } : {} },
+  );
+}
+
+beforeEach(() => {
+  localStorage.clear();
+  testQueryClient.clear();
+  requests.length = 0;
+  cancellations = 0;
+  acceptance = undefined;
+  supported = true;
+  retrievalStatus = 200;
+  status = "in_progress";
+  vi.stubGlobal(
+    "fetch",
+    vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+      const request =
+        input instanceof Request ? input : new Request(new URL(String(input), window.location.origin), init);
+      const path = new URL(request.url).pathname;
+      if (path.endsWith("/model_group/info"))
+        return Response.json({
+          data: [{ model_group: "research-chat", mode: "chat", supports_background_chat: supported }],
+        });
+      if (path.endsWith("/v1/mcp/server")) return Response.json([{ server_id: "inventory", server_name: "inventory" }]);
+      if (path.endsWith("/mcp-rest/tools/list")) return Response.json({ tools: [{ name: "lookup_inventory" }] });
+      if (path.endsWith("/responses")) {
+        const body = JSON.parse(await request.text());
+        requests.push(body);
+        if (body.background) {
+          await acceptance;
+          request.signal.throwIfAborted();
+          return Response.json({ ...result(), status: "queued" });
+        }
+        return new Response(
+          'data: {"type":"response.output_text.delta","delta":"Foreground answer"}\n\ndata: {"type":"response.completed","response":{"id":"resp_foreground","output":[]}}\n\n',
+          { headers: { "Content-Type": "text/event-stream" } },
+        );
+      }
+      if (path.endsWith("/litellm_poll_research/cancel")) {
+        cancellations++;
+        status = "cancelled";
+        return Response.json(result());
+      }
+      if (path.endsWith("/litellm_poll_research"))
+        return Response.json(retrievalStatus === 200 ? result() : { error: { message: "Unavailable" } }, {
+          status: retrievalStatus,
+        });
+      return Response.json({}, { status: 404 });
+    }),
+  );
+});
+afterEach(() => vi.unstubAllGlobals());
+
+async function startResearch(withTools = true) {
+  const user = userEvent.setup();
+  await screen.findByRole("button", { name: "research-chat" });
+  if (withTools) {
+    await user.click(screen.getByRole("button", { name: "Connect tools" }));
+    await user.click(await screen.findByRole("switch"));
+    await user.keyboard("{Escape}");
+  }
+  fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), {
+    target: { value: "Check inventory" },
+  });
+  await user.click(screen.getByRole("button", { name: "Send", exact: true }));
+}
+
+it("should retrieve the completed tool run after closing and reopening without submitting it again", async () => {
+  const first = renderChat();
+  await startResearch();
+  await screen.findByText("Running in the background. You can close this tab and return to this chat.");
+  await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
+  const id = saved()[0].id;
+  first.unmount();
+  expect(cancellations).toBe(0);
+  status = "completed";
+  renderChat(id);
+  expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible();
+  expect(screen.getByRole("button", { name: "lookup_inventory" })).toBeVisible();
+  expect(screen.getByLabelText(/^Cost:/)).toHaveTextContent("$0.125");
+  expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s");
+  expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s");
+  await waitFor(() =>
+    expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }),
+  );
+  expect(requests).toHaveLength(1);
+  expect(requests[0]).toMatchObject({
+    background: true,
+    stream: false,
+    tools: [{ type: "mcp", server_label: "inventory" }],
+  });
+  expect(screen.queryByRole("button", { name: "Stop request" })).not.toBeInTheDocument();
+  fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), {
+    target: { value: "What was the result?" },
+  });
+  fireEvent.click(screen.getByRole("button", { name: "Send", exact: true }));
+  await waitFor(() => expect(requests).toHaveLength(2));
+  expect(requests[1]).toMatchObject({ background: true, stream: false });
+  expect(requests[1]).not.toHaveProperty("previous_response_id");
+  expect(requests[1].input).toEqual(
+    expect.arrayContaining([{ type: "function_call_output", call_id: "call_inventory", output: "12 widgets" }]),
+  );
+});
+
+it("should continue an ordinary chat without MCP tools after closing and reopening", async () => {
+  const first = renderChat();
+  await startResearch(false);
+  await screen.findByText("Running in the background. You can close this tab and return to this chat.");
+  await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
+  expect(requests[0]).toMatchObject({ background: true, stream: false });
+  expect(requests[0]).not.toHaveProperty("tools");
+  const id = saved()[0].id;
+  first.unmount();
+  status = "completed";
+  renderChat(id);
+  expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible();
+  expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s");
+  expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s");
+  await waitFor(() =>
+    expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }),
+  );
+  expect(requests).toHaveLength(1);
+  expect(cancellations).toBe(0);
+});
+
+it("should cancel a background run only when Stop is clicked", async () => {
+  renderChat();
+  await startResearch();
+  await screen.findByText("Running in the background. You can close this tab and return to this chat.");
+  fireEvent.click(screen.getByRole("button", { name: "Stop request" }));
+  expect(await screen.findByText("Cancellation requested")).toBeVisible();
+  expect(cancellations).toBe(1);
+});
+
+it("should retain ordinary streaming when background support is unavailable", async () => {
+  supported = false;
+  renderChat();
+  await startResearch();
+  expect(await screen.findByText("Foreground answer")).toBeVisible();
+  expect(requests[0]).toMatchObject({ stream: true });
+  expect(requests[0]).not.toHaveProperty("background");
+  expect(screen.getByText("Keep this tab open. Background runs are not available for this chat.")).toBeVisible();
+});
+
+it("should let the user reconnect after an authentication failure without resubmitting work", async () => {
+  const first = renderChat();
+  await startResearch();
+  await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research"));
+  const id = saved()[0].id;
+  first.unmount();
+  retrievalStatus = 401;
+  renderChat(id);
+  await screen.findByRole("button", { name: "Reconnect" });
+  retrievalStatus = 200;
+  status = "completed";
+  fireEvent.click(screen.getByRole("button", { name: "Reconnect" }));
+  expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible();
+  expect(requests).toHaveLength(1);
+});
+
+it.each([false, true])(
+  "should honor Stop before a background submission returns its ID (tools=%s)",
+  async (withTools) => {
+    let accept!: () => void;
+    acceptance = new Promise<void>((resolve) => {
+      accept = resolve;
+    });
+    renderChat();
+    await startResearch(withTools);
+    await waitFor(() => expect(requests).toHaveLength(1));
+    fireEvent.click(screen.getByRole("button", { name: "Stop request" }));
+    await act(async () => accept());
+    expect(await screen.findByText("Cancellation requested")).toBeVisible();
+    expect(cancellations).toBe(1);
+    expect(requests).toHaveLength(1);
+    await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.status).toBe("cancelled"));
+  },
+);

ui/litellm-dashboard/src/app/chat/page.integration.test.tsx

@@ -64,7 +64,7 @@ vi.mock("@/contexts/ChatShellContext", () => ({
   },
 }));
 
-const ONE_TURN_ARG_COUNT = 25;
+const ONE_TURN_ARG_COUNT = 28;
 const ON_TIMING_DATA_INDEX = 7;
 const ON_USAGE_DATA_INDEX = 8;
 const ON_TOTAL_LATENCY_INDEX = 24;

ui/litellm-dashboard/src/app/chat/page.tsx

@@ -15,6 +15,7 @@ import type { AssistantMessageUpdate, ChatAttachment, ChatMessage } from "@/comp
 import ChatModelPicker, { isChatModel, modelChatMode, type ChatMode } from "@/components/chat/ChatModelPicker";
 import MoyaiChatComposer from "@/components/chat/moyai/MoyaiChatComposer";
 import MoyaiChatHome from "@/components/chat/moyai/MoyaiChatHome";
+import { useBackgroundChat } from "@/components/chat/useBackgroundChat";
 import { runChatRequest } from "@/components/chat/runChatRequest";
 import { readChatAttachment, MAX_CHAT_ATTACHMENTS, CHAT_ATTACHMENT_ACCEPT } from "@/components/chat/attachmentHelpers";
 import ChatFileChip from "@/components/chat/ChatFileChip";
@@ -102,7 +103,30 @@ export default function ChatConversationPage() {
   const responsesSessionGenerationRef = useRef(0);
   const previousResponsesConversationRef = useRef(activeConversationId);
   const [prevConversationIdForSessionReset, setPrevConversationIdForSessionReset] = useState(activeConversationId);
-  const [isStreaming, setIsStreaming] = useState(false);
+  const [isRequestInFlight, setIsStreaming] = useState(false);
+  const [cancelOnAcceptance, setCancelOnAcceptance] = useState<string | null>(null);
+  const backgroundRequestRef = useRef(false);
+  const lastMessage = activeConversation?.messages.at(-1);
+  const background = useBackgroundChat(
+    activeConversationId,
+    lastMessage?.backgroundRun,
+    accessToken,
+    updateLastAssistantMessage,
+  );
+  const { pending: backgroundPending, cancelling: backgroundCancelling, cancel: cancelBackground } = background;
+  useEffect(() => {
+    if (!backgroundPending || backgroundCancelling || !activeConversationId) return;
+    if (cancelOnAcceptance !== activeConversationId) return;
+    void cancelBackground().then(() => {
+      setCancelOnAcceptance((current) => (current === activeConversationId ? null : current));
+    });
+  }, [activeConversationId, backgroundPending, backgroundCancelling, cancelBackground, cancelOnAcceptance]);
+
+  const isStreaming = isRequestInFlight || background.pending;
+  const supportsBackground = models.some(
+    (model) => model.model_group === selectedModel && model.supports_background_chat,
+  );
+  const useBackground = mode === "chat" && supportsBackground && !storageUnavailable;
   const [inputText, setInputText] = useState("");
   const [mcpPopoverOpen, setMcpPopoverOpen] = useState(false);
   const [storageBannerDismissed, setStorageBannerDismissed] = useState(false);
@@ -264,6 +288,7 @@ export default function ChatConversationPage() {
       setAttachmentError(null);
 
       setIsStreaming(true);
+      setCancelOnAcceptance(null);
       abortControllerRef.current = new AbortController();
       requestConversationRef.current = convId;
 
@@ -297,12 +322,14 @@ export default function ChatConversationPage() {
         signal: requestController.signal,
         messages: history,
         previousResponseId,
+        background: requestMode === "chat" && useBackground,
         mcpServers: selectedMCPServers,
         onResponseId: (id: string | null) => {
           if (responsesSessionGenerationRef.current === sessionGeneration) setResponsesSessionId(id);
         },
         onUpdate: (updates: AssistantMessageUpdate) => updateLastAssistantMessage(convId, updates),
       };
+      backgroundRequestRef.current = request.background;
       try {
         await runChatRequest(request);
       } finally {
@@ -326,13 +353,22 @@ export default function ChatConversationPage() {
       updateLastAssistantMessage,
       isStreaming,
       responsesSessionId,
+      useBackground,
       setConversationIdInUrl,
     ],
   );
 
-  const handleStop = useCallback(() => {
+  const handleStop = () => {
+    if (background.pending) {
+      void background.cancel();
+      return;
+    }
+    if (backgroundRequestRef.current) {
+      setCancelOnAcceptance(activeConversationId);
+      return;
+    }
     abortControllerRef.current?.abort();
-  }, []);
+  };
 
   const handleEditAndResend = useCallback(
     (messageId: string, newContent: string) => {
@@ -403,7 +439,21 @@ export default function ChatConversationPage() {
 
   const showBlankState = !activeConversation || activeConversation.messages.length === 0;
 
+  const backgroundHint = () => {
+    if (background.cancelling || (isRequestInFlight && cancelOnAcceptance === activeConversationId))
+      return "Requesting cancellation…";
+    if (background.pending && storageUnavailable)
+      return "Keep this tab open. This run could not be saved in this browser.";
+    if (background.pending) return "Running in the background. You can close this tab and return to this chat.";
+    if (useBackground) return null;
+    return "Keep this tab open. Background runs are not available for this chat.";
+  };
+
   const inputBar = (inConversation: boolean) => {
+    const hint = mode === "chat" || background.pending ? backgroundHint() : null;
+    const chatPlaceholder = inConversation
+      ? "Send a message..."
+      : "Describe a change, paste an error, or ask a question…";
     const preparing = isLoadingModels || isReadingAttachments;
     const submitDisabled = !hasMessageContent(inputText, attachments, mode) || !selectedModel || preparing;
     return (
@@ -424,13 +474,7 @@ export default function ChatConversationPage() {
           onChange={setInputText}
           onSubmit={() => void handleSend(inputText)}
           onCancel={handleStop}
-          placeholder={
-            mode === "image"
-              ? "Describe the image you want to create…"
-              : inConversation
-                ? "Send a message..."
-                : "Describe a change, paste an error, or ask a question…"
-          }
+          placeholder={mode === "image" ? "Describe the image you want to create…" : chatPlaceholder}
           isLoading={isStreaming}
           submitLabel={mode === "image" ? "Generate image" : "Send"}
           submitDisabled={submitDisabled}
@@ -536,6 +580,29 @@ export default function ChatConversationPage() {
               : "No chat models are available for your account."}
           </p>
         )}
+        {background.connectionError && (
+          <div role="alert" className="mt-2 flex items-center gap-2 text-sm text-destructive">
+            <span>{background.connectionError}</span>
+            <Button variant="ghost" size="sm" onClick={background.reconnect}>
+              Reconnect
+            </Button>
+          </div>
+        )}
+        {lastMessage?.backgroundError && (
+          <p role="alert" className="mt-2 text-sm text-destructive">
+            {lastMessage.backgroundError}
+          </p>
+        )}
+        {lastMessage?.backgroundRun?.status === "cancelled" && (
+          <p role="status" className="mt-2 text-xs text-muted-foreground">
+            Cancellation requested
+          </p>
+        )}
+        {hint && (
+          <p role="status" className="mt-2 text-xs text-muted-foreground">
+            {hint}
+          </p>
+        )}
         <p className="moyai-chat-home-composer-hint">
           {mode === "image" ? "Describe an image · Enter to generate" : "Drop files or paste images · Enter to send"}
         </p>

ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx

@@ -0,0 +1,98 @@
+import { fireEvent, screen, waitFor } from "@testing-library/react";
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import { renderWithProviders, testQueryClient } from "@/../tests/test-utils";
+import BackgroundChatSettings from "./BackgroundChatSettings";
+
+let settings: {
+  enabled: boolean;
+  redis_configured: boolean;
+  managed_by_config: boolean;
+  can_save: boolean;
+};
+let failLoad: boolean;
+let failSave: boolean;
+const updates: unknown[] = [];
+
+beforeEach(() => {
+  testQueryClient.clear();
+  settings = {
+    enabled: false,
+    redis_configured: true,
+    managed_by_config: false,
+    can_save: true,
+  };
+  failLoad = false;
+  failSave = false;
+  updates.length = 0;
+  vi.stubGlobal(
+    "fetch",
+    vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+      const request =
+        input instanceof Request ? input : new Request(new URL(String(input), window.location.origin), init);
+      if (request.method === "PATCH") {
+        const body = await request.json();
+        updates.push(body);
+        if (failSave) return Response.json({ detail: "Save failed" }, { status: 500 });
+        settings = { ...settings, enabled: body.enabled };
+      } else if (failLoad) return Response.json({ detail: "Unavailable" }, { status: 503 });
+      return Response.json(settings);
+    }),
+  );
+});
+afterEach(() => vi.unstubAllGlobals());
+
+it("saves background execution next to its explanation and retains it after reopening", async () => {
+  const first = renderWithProviders(<BackgroundChatSettings />);
+  fireEvent.click(await screen.findByRole("switch", { name: "Continue chats in the background" }));
+  await waitFor(() => expect(screen.getByRole("switch")).toBeChecked());
+  expect(updates).toEqual([{ enabled: true }]);
+  expect(screen.getByRole("switch")).toBeChecked();
+  first.unmount();
+  testQueryClient.clear();
+  renderWithProviders(<BackgroundChatSettings />);
+  expect(await screen.findByRole("switch")).toBeChecked();
+  fireEvent.click(screen.getByRole("switch"));
+  await waitFor(() => expect(screen.getByRole("switch")).not.toBeChecked());
+  expect(updates).toEqual([{ enabled: true }, { enabled: false }]);
+});
+
+it("links to Redis setup and prevents enabling without Redis", async () => {
+  settings.redis_configured = false;
+  renderWithProviders(<BackgroundChatSettings />);
+  expect(await screen.findByRole("switch")).toHaveAttribute("aria-disabled", "true");
+  expect(screen.getByRole("link", { name: /Configure it in Caching/ })).toHaveAttribute(
+    "href",
+    expect.stringMatching(/\/caching$/),
+  );
+  expect(updates).toHaveLength(0);
+});
+
+it.each(["managed_by_config", "can_save"] as const)(
+  "explains a setting that cannot be edited because of %s",
+  async (field) => {
+    settings[field] = field === "managed_by_config";
+    renderWithProviders(<BackgroundChatSettings />);
+    expect(await screen.findByRole("switch")).toHaveAttribute("aria-disabled", "true");
+    const message =
+      field === "managed_by_config" ? /Managed by your config file/ : /Enable database-backed configuration/;
+    expect(screen.getByText(message)).toBeVisible();
+  },
+);
+
+it("keeps a failed save off", async () => {
+  failSave = true;
+  renderWithProviders(<BackgroundChatSettings />);
+  fireEvent.click(await screen.findByRole("switch"));
+  await waitFor(() => expect(updates).toHaveLength(1));
+  expect(screen.getByRole("switch")).not.toBeChecked();
+});
+
+it("offers retry for a settings fetch failure instead of claiming Redis is missing", async () => {
+  failLoad = true;
+  renderWithProviders(<BackgroundChatSettings />);
+  await screen.findByRole("button", { name: "Retry" });
+  expect(screen.queryByRole("link")).not.toBeInTheDocument();
+  failLoad = false;
+  fireEvent.click(screen.getByRole("button", { name: "Retry" }));
+  expect(await screen.findByRole("switch")).not.toHaveAttribute("aria-disabled", "true");
+});

ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx

@@ -0,0 +1,73 @@
+import { $api } from "@/lib/http/api";
+import { toast } from "@/lib/toast";
+import { Button } from "@/components/ui/button";
+import { Skeleton } from "@/components/ui/skeleton";
+import { Switch } from "@/components/ui/switch";
+import { uiHref } from "@/utils/uiHref";
+
+export default function BackgroundChatSettings() {
+  const query = $api.useQuery("get", "/get/background_chat_settings");
+  const update = $api.useMutation("patch", "/update/background_chat_settings", {
+    onSuccess: () => {
+      void query.refetch();
+      toast.success("Background chat settings saved.");
+    },
+    onError: (error) => toast.fromError(error),
+  });
+
+  if (query.isPending) return <Skeleton className="h-14 w-full" />;
+  if (query.isError || !query.data)
+    return (
+      <div role="alert" className="text-sm text-destructive">
+        Could not load background chat settings.
+        <Button variant="ghost" size="sm" onClick={() => void query.refetch()}>
+          Retry
+        </Button>
+      </div>
+    );
+
+  const settings = query.data;
+  const readOnly = settings.managed_by_config || !settings.can_save;
+  const needsRedis = !settings.redis_configured && !settings.enabled;
+  const disabled = update.isPending || readOnly || needsRedis;
+
+  return (
+    <div className="flex items-start gap-3">
+      <Switch
+        checked={settings.enabled}
+        disabled={disabled}
+        onCheckedChange={(enabled) => update.mutate({ body: { enabled } })}
+        aria-label="Continue chats in the background"
+        aria-describedby="background-chat-description"
+      />
+      <div className="space-y-1">
+        <p className="text-sm font-medium text-foreground">Continue chats in the background</p>
+        <p id="background-chat-description" className="text-sm text-muted-foreground">
+          Keep replies and tool runs going after users close the browser. Return to the same browser to see the result.
+        </p>
+        {!settings.redis_configured && (
+          <p className="text-sm text-muted-foreground">
+            Requires Redis.{" "}
+            <a className="text-primary underline" href={uiHref("/caching")}>
+              Configure it in Caching → Coordination Redis
+            </a>
+            , then restart the proxy.
+          </p>
+        )}
+        {settings.managed_by_config && (
+          <p className="text-sm text-muted-foreground">
+            Managed by your config file. Update its background polling setting to change this.
+          </p>
+        )}
+        {!settings.can_save && !settings.managed_by_config && (
+          <p className="text-sm text-muted-foreground">Enable database-backed configuration to save this setting.</p>
+        )}
+        {!readOnly && (
+          <p className="text-sm text-muted-foreground">
+            Changes apply automatically when the proxy refreshes its configuration.
+          </p>
+        )}
+      </div>
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx

@@ -5,6 +5,8 @@ import userEvent from "@testing-library/user-event";
 import UISettings from "./UISettings";
 import { toast } from "@/lib/toast";
 
+vi.mock("./BackgroundChatSettings", () => ({ default: () => null }));
+
 const mockUseAuthorized = vi.hoisted(() => vi.fn());
 const mockUseUISettings = vi.hoisted(() => vi.fn());
 const mockUseUpdateUISettings = vi.hoisted(() => vi.fn());

ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx

@@ -16,6 +16,7 @@ import { relativeTimeOptions } from "@/components/shared/date_range_presets";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { Label } from "@/components/ui/label";
 import PageVisibilitySettings from "./PageVisibilitySettings";
+import BackgroundChatSettings from "./BackgroundChatSettings";
 
 interface SettingRowProps {
   ariaLabel: string;
@@ -429,13 +430,15 @@ export default function UISettings() {
               disabled={isUpdating}
               onCheckedChange={handleToggleEnableChatUI}
               ariaLabel={enableChatUIProperty?.description ?? "Enable Chat page"}
-              label="[BETA] Enable Chat page (page will refresh)"
+              label="Enable Chat page (page will refresh)"
               description={
                 enableChatUIProperty?.description ??
                 "If enabled, shows the Chat page in the UI sidebar, letting users chat with an LLM and connect their own MCP server credentials via OAuth."
               }
             />
 
+            <BackgroundChatSettings />
+
             <Separator />
             <div className="flex items-start justify-between gap-3">
               <div className="space-y-1">

ui/litellm-dashboard/src/components/chat/ChatShell.tsx

@@ -192,9 +192,6 @@ const ChatShell: React.FC<{ children: React.ReactNode }> = ({ children }) => {
               </DropdownMenuItem>
             ))}
             <DropdownMenuSeparator />
-            <p className="px-2 py-1 text-xs text-muted-foreground">
-              Chat is a pre-v0 preview and may change unexpectedly. Do not use in production.
-            </p>
             <DropdownMenuItem
               render={
                 <a href="https://github.com/BerriAI/litellm/discussions/32085" target="_blank" rel="noreferrer" />

ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts

@@ -0,0 +1,177 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { Response as ModelResponse } from "openai/resources/responses/responses";
+import { backgroundResponseUpdate, cancelBackgroundChat, watchBackgroundChat } from "./backgroundChat";
+
+vi.mock("@/components/networking", () => ({ getGatewayBaseUrl: () => "http://localhost:4000/v1" }));
+
+afterEach(() => vi.unstubAllGlobals());
+
+it("should restore text, reasoning, tool results and usage from a completed response", () => {
+  const response = {
+    id: "litellm_poll_example",
+    status: "completed",
+    output: [
+      { type: "reasoning", summary: [{ type: "summary_text", text: "Check the inventory" }] },
+      {
+        type: "mcp_call",
+        id: "call_lookup",
+        name: "lookup",
+        arguments: "{}",
+        output: "12 widgets",
+        server_label: "inventory",
+      },
+      { type: "message", content: [{ type: "output_text", text: "There are 12 widgets." }] },
+    ],
+    usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, output_tokens_details: { reasoning_tokens: 2 } },
+  } as ModelResponse;
+  const update = backgroundResponseUpdate(response);
+  expect(update.content).toBe("There are 12 widgets.");
+  expect(update.reasoningContent).toBe("Check the inventory");
+  expect(update.mcpEvents?.[0].item?.output).toBe("12 widgets");
+  const usage = { promptTokens: 4, completionTokens: 9, totalTokens: 13, reasoningTokens: 2 };
+  expect(update.usage).toMatchObject(usage);
+  expect(update.backgroundRun).toEqual({ id: response.id, status: "completed" });
+});
+
+describe("background response lifecycle", () => {
+  it("should detach polling without cancelling the server run", async () => {
+    const controller = new AbortController();
+    const fetch = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) =>
+      Response.json({ id: "litellm_poll_example", status: "in_progress", output: [] }),
+    );
+    vi.stubGlobal("fetch", fetch);
+    const onUpdate = vi.fn(() => controller.abort());
+    const polling = {
+      accessToken: "test-token",
+      responseId: "litellm_poll_example",
+      signal: controller.signal,
+      onUpdate,
+    };
+    await watchBackgroundChat(polling);
+    expect(onUpdate).toHaveBeenCalledOnce();
+    expect(fetch).toHaveBeenCalledOnce();
+    expect(fetch.mock.calls[0][1]?.method).toBe("GET");
+  });
+
+  it("should mark an expired run unavailable without clearing its partial answer", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => Response.json({ error: { message: "Expired" } }, { status: 404 })),
+    );
+    const onUpdate = vi.fn();
+    const polling = {
+      accessToken: "test-token",
+      responseId: "litellm_poll_expired",
+      signal: new AbortController().signal,
+      onUpdate,
+    };
+    await watchBackgroundChat(polling);
+    expect(onUpdate).toHaveBeenCalledWith({
+      backgroundRun: { id: "litellm_poll_expired", status: "expired" },
+      backgroundError: "This saved run is no longer available. Its result may have expired on the server.",
+    });
+  });
+
+  it("should cancel explicitly through the server and preserve returned partial output", async () => {
+    const fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+      const request = input instanceof Request ? input : new Request(input, init);
+      expect(request.method).toBe("POST");
+      expect(new URL(request.url).pathname).toBe("/v1/responses/litellm_poll_example/cancel");
+      return Response.json({
+        id: "litellm_poll_example",
+        status: "cancelled",
+        output: [{ type: "message", content: [{ type: "output_text", text: "Partial answer" }] }],
+      });
+    });
+    vi.stubGlobal("fetch", fetch);
+    expect(await cancelBackgroundChat("test-token", "litellm_poll_example")).toMatchObject({
+      content: "Partial answer",
+      backgroundRun: { id: "litellm_poll_example", status: "cancelled" },
+    });
+  });
+});
+
+it.each([
+  [0.125, 0.125],
+  ["0.125", 0.125],
+  [0, 0],
+  ["invalid", undefined],
+])("should preserve valid returned cost %s", (cost, expected) => {
+  const response = {
+    id: "litellm_poll_cost",
+    status: "completed",
+    output: [],
+    usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost },
+  } as ModelResponse;
+  expect(backgroundResponseUpdate(response).usage?.cost).toBe(expected);
+});
+
+it("should keep polling through unfinished output items until the answer and tools arrive", async () => {
+  let polls = 0;
+  vi.stubGlobal(
+    "fetch",
+    vi.fn(async () => {
+      polls++;
+      return Response.json({
+        id: "litellm_poll_partial",
+        status: polls === 1 ? "in_progress" : "completed",
+        output:
+          polls === 1
+            ? [
+                { type: "message", id: "message_1" },
+                { type: "mcp_list_tools", id: "tools_1", server_label: "inventory" },
+              ]
+            : [
+                { type: "message", content: [{ type: "output_text", text: "12 widgets" }] },
+                { type: "mcp_list_tools", server_label: "inventory", tools: [{ name: "lookup", input_schema: {} }] },
+              ],
+      });
+    }),
+  );
+  const onUpdate = vi.fn();
+  const polling = {
+    accessToken: "test-token",
+    responseId: "litellm_poll_partial",
+    signal: new AbortController().signal,
+    onUpdate,
+  };
+  await watchBackgroundChat(polling);
+  expect(polls).toBe(2);
+  expect(onUpdate.mock.lastCall?.[0]).toMatchObject({
+    content: "12 widgets",
+    backgroundRun: { status: "completed" },
+    mcpEvents: [{ item: { tools: [{ name: "lookup" }] } }],
+  });
+});
+
+it("should restore server timings from polling without counting time away from the chat", async () => {
+  const response = {
+    id: "litellm_poll_timing",
+    status: "completed",
+    created_at: 1000,
+    output: [],
+    litellm_response_timing: { time_to_first_token_ms: 250, total_latency_ms: 12500 },
+  };
+  vi.stubGlobal(
+    "fetch",
+    vi.fn(async () => Response.json(response)),
+  );
+  const onUpdate = vi.fn();
+  const polling = {
+    accessToken: "test-token",
+    responseId: "litellm_poll_timing",
+    signal: new AbortController().signal,
+    onUpdate,
+  };
+  await watchBackgroundChat(polling);
+  expect(onUpdate).toHaveBeenCalledWith(expect.objectContaining({ timeToFirstToken: 250, totalLatency: 12500 }));
+});
+
+it.each(["queued", "completed"] as const)(
+  "should not invent timings for a %s response without measurements",
+  (status) => {
+    const update = backgroundResponseUpdate({ id: "litellm_poll_legacy", status, output: [] } as ModelResponse);
+    expect(update).not.toHaveProperty("timeToFirstToken");
+    expect(update).not.toHaveProperty("totalLatency");
+  },
+);

ui/litellm-dashboard/src/components/chat/backgroundChat.ts

@@ -0,0 +1,138 @@
+import OpenAI from "openai";
+import type { Response } from "openai/resources/responses/responses";
+import { getGatewayBaseUrl } from "@/components/networking";
+import { parseUsageCost } from "@/components/llm_calls/usage_cost";
+import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
+import type { AssistantMessageUpdate, ChatBackgroundRun, MCPEvent } from "./types";
+
+export const isBackgroundRunPending = (run?: ChatBackgroundRun): boolean =>
+  run?.status === "queued" || run?.status === "in_progress";
+
+type TimedResponse = Response & {
+  litellm_response_timing?: {
+    time_to_first_token_ms?: number;
+    total_latency_ms?: number;
+  };
+};
+
+export function backgroundResponseUpdate(response: TimedResponse): AssistantMessageUpdate {
+  const output = response.output ?? [];
+  const content = output
+    .filter((item) => item.type === "message")
+    .flatMap((item) => item.content ?? [])
+    .map((part) => (part.type === "output_text" ? part.text : part.refusal))
+    .join("");
+  const reasoningContent = output
+    .filter((item) => item.type === "reasoning")
+    .flatMap((item) => item.summary ?? [])
+    .map((part) => part.text)
+    .join("\n\n");
+  const mcpEvents: MCPEvent[] = output.flatMap<MCPEvent>((item) => {
+    if (item.type === "mcp_call") {
+      return [{ type: "response.output_item.done", item: { ...item, output: item.output ?? undefined } }];
+    }
+    if (item.type === "mcp_list_tools") {
+      return [
+        {
+          type: "response.output_item.done",
+          item: {
+            ...item,
+            tools: (item.tools ?? []).map((tool) => ({
+              name: tool.name,
+              input_schema: tool.input_schema,
+              description: tool.description ?? "",
+            })),
+          },
+        },
+      ];
+    }
+    return [];
+  });
+  const status = response.status ?? "in_progress";
+  const errors: Partial<Record<ChatBackgroundRun["status"], string>> = {
+    failed: response.error?.message || "The background run failed.",
+    incomplete: "The run ended before completing its answer.",
+  };
+  const backgroundError = errors[status];
+  const cost = response.usage && "cost" in response.usage ? parseUsageCost(response.usage.cost) : undefined;
+  const timing = response.litellm_response_timing;
+  return {
+    content,
+    reasoningContent,
+    mcpEvents,
+    backgroundRun: { id: response.id, status },
+    backgroundError,
+    ...(timing?.time_to_first_token_ms !== undefined && { timeToFirstToken: timing.time_to_first_token_ms }),
+    ...(timing?.total_latency_ms !== undefined && { totalLatency: timing.total_latency_ms }),
+    ...(response.usage && {
+      usage: {
+        promptTokens: response.usage.input_tokens,
+        completionTokens: response.usage.output_tokens,
+        totalTokens: response.usage.total_tokens,
+        reasoningTokens: response.usage.output_tokens_details?.reasoning_tokens,
+        ...extractPromptCacheTokens(response.usage),
+        ...(cost !== undefined ? { cost } : {}),
+      },
+    }),
+  };
+}
+
+const backgroundClient = (accessToken: string) =>
+  new OpenAI({
+    apiKey: accessToken,
+    baseURL: getGatewayBaseUrl(),
+    dangerouslyAllowBrowser: true,
+  });
+
+function waitForPoll(signal: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const abort = () => {
+      clearTimeout(timer);
+      reject(signal.reason);
+    };
+    const timer = setTimeout(() => {
+      signal.removeEventListener("abort", abort);
+      resolve();
+    }, 1000);
+    signal.addEventListener("abort", abort, { once: true });
+    if (signal.aborted) abort();
+  });
+}
+
+export async function watchBackgroundChat({
+  accessToken,
+  responseId,
+  signal,
+  onUpdate,
+}: {
+  accessToken: string;
+  responseId: string;
+  signal: AbortSignal;
+  onUpdate: (update: AssistantMessageUpdate) => void;
+}): Promise<void> {
+  const client = backgroundClient(accessToken);
+  while (!signal.aborted) {
+    try {
+      const response = await client.responses.retrieve(responseId, { stream: false }, { signal });
+      if (signal.aborted) return;
+      onUpdate(backgroundResponseUpdate(response));
+      if (!isBackgroundRunPending({ id: response.id, status: response.status ?? "in_progress" })) return;
+      await waitForPoll(signal);
+    } catch (error) {
+      if (signal.aborted) return;
+      if (error instanceof OpenAI.APIError && error.status === 404) {
+        onUpdate({
+          backgroundRun: { id: responseId, status: "expired" },
+          backgroundError: "This saved run is no longer available. Its result may have expired on the server.",
+        });
+        return;
+      }
+      throw error;
+    }
+  }
+}
+
+export async function cancelBackgroundChat(accessToken: string, responseId: string): Promise<AssistantMessageUpdate> {
+  const response = await backgroundClient(accessToken).responses.cancel(responseId);
+  return backgroundResponseUpdate(response);
+}

ui/litellm-dashboard/src/components/chat/runChatRequest.ts

@@ -1,5 +1,6 @@
 import OpenAI from "openai";
 import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api";
+import { backgroundResponseUpdate } from "./backgroundChat";
 import { generateChatImages } from "./imageHelpers";
 import { toResponsesMessages } from "./attachmentHelpers";
 import type { AssistantMessageUpdate, ChatMessage, MCPEvent } from "./types";
@@ -15,6 +16,7 @@ interface ChatRequestOptions {
     "role" | "content" | "attachments" | "generatedImages" | "imageGeneration" | "mcpEvents"
   >[];
   previousResponseId?: string | null;
+  background?: boolean;
   mcpServers?: string[];
   onResponseId: (id: string | null) => void;
   onUpdate: (update: AssistantMessageUpdate) => void;
@@ -43,6 +45,7 @@ export async function runChatRequest({
   signal,
   messages,
   previousResponseId,
+  background = false,
   mcpServers,
   onResponseId,
   onUpdate,
@@ -94,6 +97,14 @@ export async function runChatRequest({
       undefined,
       true,
       (totalLatency) => onUpdate({ totalLatency }),
+      undefined,
+      undefined,
+      background
+        ? (response) => {
+            onResponseId(null);
+            onUpdate(backgroundResponseUpdate(response));
+          }
+        : undefined,
     );
     signal.throwIfAborted();
   } catch (error: unknown) {

ui/litellm-dashboard/src/components/chat/types.ts

@@ -25,6 +25,11 @@ export interface ChatImageGeneration {
   prompt: string;
 }
 
+export interface ChatBackgroundRun {
+  id: string;
+  status: "queued" | "in_progress" | "completed" | "failed" | "cancelled" | "incomplete" | "expired";
+}
+
 export interface ChatMessage {
   id: string;
   role: "user" | "assistant" | "tool";
@@ -33,6 +38,8 @@ export interface ChatMessage {
   generatedImages?: ChatImageAttachment[];
   imageGeneration?: ChatImageGeneration;
   reasoningContent?: string;
+  backgroundRun?: ChatBackgroundRun;
+  backgroundError?: string;
   /** MCP tool events that occurred during this assistant turn, in order. */
   mcpEvents?: MCPEvent[];
   toolName?: string;
@@ -55,6 +62,8 @@ export type AssistantMessageUpdate = Partial<
     | "usage"
     | "generatedImages"
     | "imageGeneration"
+    | "backgroundRun"
+    | "backgroundError"
   >
 >;
 

ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts

@@ -0,0 +1,62 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+import { cancelBackgroundChat, isBackgroundRunPending, watchBackgroundChat } from "./backgroundChat";
+import type { AssistantMessageUpdate, ChatBackgroundRun } from "./types";
+
+export function useBackgroundChat(
+  conversationId: string | null,
+  run: ChatBackgroundRun | undefined,
+  accessToken: string,
+  updateMessage: (conversationId: string, update: AssistantMessageUpdate) => void,
+) {
+  const [failure, setFailure] = useState<{ responseId: string; retry: number; message: string } | null>(null);
+  const [retry, setRetry] = useState(0);
+  const [cancellingId, setCancellingId] = useState<string | null>(null);
+  const controllerRef = useRef<AbortController | null>(null);
+  const pending = isBackgroundRunPending(run);
+  const responseId = run?.id;
+  const cancelling = Boolean(responseId && cancellingId === responseId);
+  const connectionError =
+    pending && failure?.responseId === responseId && failure?.retry === retry ? failure.message : null;
+
+  useEffect(() => {
+    if (!conversationId || !responseId || !pending) return;
+    const controller = new AbortController();
+    controllerRef.current = controller;
+    const polling = {
+      accessToken,
+      responseId,
+      signal: controller.signal,
+      onUpdate: (update: AssistantMessageUpdate) => updateMessage(conversationId, update),
+    };
+    void watchBackgroundChat(polling).catch(() => {
+      if (!controller.signal.aborted)
+        setFailure({ responseId, retry, message: "Connection lost. Your run may still be running on the server." });
+    });
+    return () => {
+      controller.abort();
+      if (controllerRef.current === controller) controllerRef.current = null;
+    };
+  }, [conversationId, responseId, pending, accessToken, updateMessage, retry]);
+
+  const reconnect = useCallback(() => setRetry((value) => value + 1), []);
+
+  const cancel = useCallback(async () => {
+    if (!conversationId || !responseId || cancelling) return;
+    const controller = controllerRef.current;
+    controller?.abort();
+    setCancellingId(responseId);
+    try {
+      const update = await cancelBackgroundChat(accessToken, responseId);
+      updateMessage(conversationId, update);
+      if (controllerRef.current === controller) reconnect();
+    } catch {
+      if (controllerRef.current === controller) {
+        setFailure({ responseId, retry, message: "Could not stop the run. Reconnect and try Stop again." });
+      }
+    } finally {
+      setCancellingId(null);
+    }
+  }, [conversationId, responseId, accessToken, cancelling, reconnect, retry, updateMessage]);
+
+  return { pending, connectionError, cancelling, reconnect, cancel };
+}

ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx

@@ -9,6 +9,7 @@ export interface ModelGroup {
   mode?: string;
   supports_reasoning?: boolean;
   supports_fast_mode?: boolean;
+  supports_background_chat?: boolean;
   supported_reasoning_efforts?: string[] | null;
 }
 
@@ -20,6 +21,7 @@ interface AvailableModel {
   mode?: string | null;
   supports_reasoning?: boolean | null;
   supports_fast_mode?: boolean | null;
+  supports_background_chat?: boolean | null;
   supported_reasoning_efforts?: string[] | null;
 }
 
@@ -30,6 +32,7 @@ const toModelGroup = (item: AvailableModel): ModelGroup => {
     ...(item.providers && { providers: item.providers }),
     ...(item.mode && { mode: item.mode }),
     ...(item.supports_reasoning === true && { supports_reasoning: true }),
+    ...(item.supports_background_chat === true && { supports_background_chat: true }),
     ...(item.supports_fast_mode === true && { supports_fast_mode: true }),
     ...(item.supported_reasoning_efforts !== undefined && {
       supported_reasoning_efforts: item.supported_reasoning_efforts,

ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx

@@ -1,6 +1,10 @@
 import { observeResponseHeaders, type ResponseHeadersListener } from "@/lib/http/observeResponseHeaders";
 import openai from "openai";
-import type { ResponseFunctionToolCall, ResponseInputItem } from "openai/resources/responses/responses";
+import type {
+  Response as OpenAIResponse,
+  ResponseFunctionToolCall,
+  ResponseInputItem,
+} from "openai/resources/responses/responses";
 import { MessageType } from "../chat_ui/types";
 import { TokenUsage } from "../chat_ui/ResponseMetrics";
 import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking";
@@ -110,6 +114,7 @@ export async function makeOpenAIResponsesRequest(
   onTotalLatency?: (latency: number) => void,
   customHeaders?: CustomHeaders,
   onResponseHeaders?: ResponseHeadersListener,
+  onBackgroundResponse?: (response: OpenAIResponse) => void,
 ) {
   if (!accessToken) {
     throw new Error("Virtual Key is required");
@@ -239,6 +244,16 @@ export async function makeOpenAIResponsesRequest(
       ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
     };
 
+    if (onBackgroundResponse) {
+      const response = await client.post<OpenAIResponse>("/responses", {
+        body: { ...requestBody, background: true, stream: false },
+        signal,
+        maxRetries: 0,
+      });
+      onBackgroundResponse(response);
+      return response;
+    }
+
     // Create request to OpenAI responses API
     // Use 'any' type to avoid TypeScript issues with the experimental API
     const response = streamingEnabled

ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx

@@ -19,7 +19,6 @@ vi.mock("../../utils/dataUtils", () => ({
 }));
 
 vi.mock("./networking", () => ({
-  modelInfoV1Call: vi.fn(),
   credentialGetCall: vi.fn(),
   credentialListCall: vi.fn(),
   getGuardrailsList: vi.fn(),
@@ -56,7 +55,6 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () =>
 }));
 
 const mockToast = vi.mocked(toast);
-const mockModelInfoV1Call = vi.mocked(networking.modelInfoV1Call);
 const mockCredentialGetCall = vi.mocked(networking.credentialGetCall);
 const mockCredentialListCall = vi.mocked(networking.credentialListCall);
 const mockGetGuardrailsList = vi.mocked(networking.getGuardrailsList);
@@ -134,10 +132,6 @@ describe("ModelInfoView", () => {
       error: null,
     });
 
-    mockModelInfoV1Call.mockResolvedValue({
-      data: [defaultModelData],
-    });
-
     mockCredentialGetCall.mockResolvedValue({
       credential_name: "test-credential",
       credential_values: {},
@@ -394,7 +388,6 @@ describe("ModelInfoView", () => {
       model_info: { ...defaultModelData.model_info, mode: "chat", key: "anthropic.claude-haiku-4-5" },
     };
     mockUseModelsInfo.mockReturnValue({ data: { data: [displayedModel] }, isLoading: false, error: null });
-    mockModelInfoV1Call.mockResolvedValue({ data: [displayedModel] });
     render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
 
     await waitFor(() => {
@@ -869,7 +862,6 @@ describe("ModelInfoView", () => {
 
     const renderWithPtuModel = () => {
       mockUseModelsInfo.mockReturnValue({ data: { data: [ptuModelData] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [ptuModelData] });
       return render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
     };
 
@@ -967,7 +959,6 @@ describe("ModelInfoView", () => {
         model_info: { ...defaultModelData.model_info, team_id: "team-1", input_cost_per_token: 0.0000003 },
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [seededModel] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [seededModel] });
       const user = userEvent.setup();
       render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
 
@@ -1008,7 +999,6 @@ describe("ModelInfoView", () => {
         model_info: { ...ptuModelData.model_info, cost_per_ptu_per_hour: null, ptu_effective_from: null },
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [halfSetPtuModel] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [halfSetPtuModel] });
       const user = userEvent.setup();
       render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
 
@@ -1057,7 +1047,6 @@ describe("ModelInfoView", () => {
         model_info: { ...ptuModelData.model_info, ptu_effective_from: null, ptu_effective_to: null },
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [undatedPtuModel] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [undatedPtuModel] });
       const user = userEvent.setup();
       render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
 
@@ -1201,7 +1190,6 @@ describe("ModelInfoView", () => {
       isLoading: false,
       error: null,
     });
-    mockModelInfoV1Call.mockResolvedValue({ data: [maskedModelData] });
 
     const user = userEvent.setup();
     render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} />, { wrapper });
@@ -1508,7 +1496,6 @@ describe("ModelInfoView", () => {
 
     beforeEach(() => {
       mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
     });
 
     const readRawJson = async (user: ReturnType<typeof userEvent.setup>) => {
@@ -1555,7 +1542,6 @@ describe("ModelInfoView", () => {
 
     it("shows Not Set and no team_alias for a model without a team", async () => {
       mockUseModelsInfo.mockReturnValue({ data: { data: [defaultModelData] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [defaultModelData] });
       mockUseTeams.mockReturnValue({
         data: [{ team_id: "team-1", team_alias: "alpha" }],
         isLoading: false,
@@ -1849,7 +1835,6 @@ describe("ModelInfoView", () => {
 
       beforeEach(() => {
         mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheWriteTiers] }, isLoading: false, error: null });
-        mockModelInfoV1Call.mockResolvedValue({ data: [withCacheWriteTiers] });
       });
 
       it("shows both configured cache write rates per million tokens", async () => {
@@ -1926,7 +1911,6 @@ describe("ModelInfoView", () => {
         model_info: { ...defaultModelData.model_info, team_id: "team-1" },
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
       const user = userEvent.setup();
       await enterEditMode(user);
 
@@ -1952,7 +1936,6 @@ describe("ModelInfoView", () => {
         model_info: { ...defaultModelData.model_info, team_id: "team-1" },
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
       const user = userEvent.setup();
       await enterEditMode(user);
 
@@ -1992,7 +1975,6 @@ describe("ModelInfoView", () => {
         model_info: { ...defaultModelData.model_info, team_id: "team-1" },
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] });
       const user = userEvent.setup();
       render(<ModelInfoView {...DEFAULT_ADMIN_PROPS} userRole={userRole} />, { wrapper });
       await user.click(await screen.findByRole("button", { name: /edit settings/i }));
@@ -2069,7 +2051,6 @@ describe("ModelInfoView", () => {
         litellm_params: litellmParamsWithoutCredential,
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [modelWithoutCredential] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [modelWithoutCredential] });
       const user = userEvent.setup();
       await enterEditMode(user);
 
@@ -2150,7 +2131,6 @@ describe("ModelInfoView", () => {
         litellm_params: { ...defaultModelData.litellm_params, model: "openai/gpt-4*" },
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [wildcard] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [wildcard] });
       mockUseModelHub.mockReturnValue({
         data: { data: [{ model_group: "openai/gpt-4o", providers: ["openai"] }] },
         isLoading: false,
@@ -2207,7 +2187,6 @@ describe("ModelInfoView", () => {
           litellm_params: { ...defaultModelData.litellm_params, cache_control_injection_points: points },
         };
         mockUseModelsInfo.mockReturnValue({ data: { data: [data] }, isLoading: false, error: null });
-        mockModelInfoV1Call.mockResolvedValue({ data: [data] });
       };
 
       it("omits the key when the deployment has none and the operator leaves the toggle alone", async () => {
@@ -2330,7 +2309,6 @@ describe("ModelInfoView", () => {
         litellm_params: { ...defaultModelData.litellm_params, cache_read_input_token_cost: 0.000001 },
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheRead] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [withCacheRead] });
       const user = userEvent.setup();
       await enterEditMode(user);
       setInputCost("2");
@@ -2365,7 +2343,6 @@ describe("ModelInfoView", () => {
         },
       };
       mockUseModelsInfo.mockReturnValue({ data: { data: [resolved] }, isLoading: false, error: null });
-      mockModelInfoV1Call.mockResolvedValue({ data: [resolved] });
       const user = userEvent.setup();
       await enterEditMode(user);
       setInputCost("5");

ui/litellm-dashboard/src/components/networking.tsx

@@ -48,7 +48,7 @@ export const getAutoRouterClassifierDefaultPromptCall = async (
   }
 };
 
-export type AssembledPromptTierSource =
+type AssembledPromptTierSource =
   | { tierDefinitions: { name: string; description?: string }[] }
   | { tierLabels?: Record<string, string>; classificationRubric?: string };
 
@@ -127,16 +127,6 @@ import type {
   SubscriptionUsageResponse,
 } from "@/app/(dashboard)/usage/_components/components/SubscriptionCost/types";
 import type { RoutingDecision } from "./logs/detail/RoutingDecisionCard";
-import type {
-  SpanDetail,
-  SpanErrorPage,
-  SpanErrorQuery,
-  SpanQuery,
-  Trace,
-  TraceDetailQuery,
-  TraceListQuery,
-  TracePage,
-} from "@litellm/lens-ui";
 import {
   createApiClient,
   deriveErrorMessage,
@@ -541,7 +531,7 @@ export const apiClient = createApiClient({
   onError: handleError,
 });
 
-export const gatewayApiClient = createApiClient({
+const gatewayApiClient = createApiClient({
   getBaseUrl: getGatewayBaseUrl,
   getAuthHeaderName: getGlobalLitellmHeaderName,
   onError: handleError,
@@ -1725,37 +1715,6 @@ export const modelInfoCall = async (
   }
 };
 
-export const modelInfoV1Call = async (accessToken: string, modelId: string) => {
-  /**
-   * Get all models on proxy
-   */
-  try {
-    let url = proxyBaseUrl ? `${proxyBaseUrl}/v1/model/info` : `/v1/model/info`;
-    url += `?litellm_model_id=${modelId}`;
-
-    const response = await fetch(url, {
-      method: "GET",
-      headers: {
-        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
-        "Content-Type": "application/json",
-      },
-    });
-
-    if (!response.ok) {
-      const errorData = await response.json();
-      const errorMessage = deriveErrorMessage(errorData);
-      handleError(errorMessage);
-      throw new Error(errorMessage);
-    }
-
-    const data = await response.json();
-    return data;
-  } catch (error) {
-    console.error("Failed to create key:", error);
-    throw error;
-  }
-};
-
 export const modelHubPublicModelsCall = async () => {
   const url = proxyBaseUrl ? `${proxyBaseUrl}/public/model_hub` : `/public/model_hub`;
   const response = await fetch(url, {
@@ -1906,66 +1865,6 @@ export const modelAvailableCall = async (
   }
 };
 
-export const teamSpendLogsCall = async (accessToken: string) => {
-  try {
-    const data = await apiClient.get(`/global/spend/teams`, { accessToken });
-    return data;
-  } catch (error) {
-    console.error("Failed to create key:", error);
-    throw error;
-  }
-};
-
-export const tagsSpendLogsCall = async (
-  accessToken: string,
-  startTime: string | undefined,
-  endTime: string | undefined,
-  tags: string[] | undefined,
-) => {
-  try {
-    let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/tags` : `/global/spend/tags`;
-
-    if (startTime && endTime) {
-      url = `${url}?start_date=${startTime}&end_date=${endTime}`;
-    }
-
-    // if tags, convert the list to a comma separated string
-    if (tags) {
-      url += `&tags=${tags.join(",")}`;
-    }
-
-    const response = await fetch(`${url}`, {
-      method: "GET",
-      headers: {
-        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
-        "Content-Type": "application/json",
-      },
-    });
-    if (!response.ok) {
-      const errorData = await response.json();
-      const errorMessage = deriveErrorMessage(errorData);
-      handleError(errorMessage);
-      throw new Error(errorMessage);
-    }
-
-    const data = await response.json();
-    return data;
-  } catch (error) {
-    console.error("Failed to create key:", error);
-    throw error;
-  }
-};
-
-export const allTagNamesCall = async (accessToken: string) => {
-  try {
-    const data = await apiClient.get(`/global/spend/all_tag_names`, { accessToken });
-    return data;
-  } catch (error) {
-    console.error("Failed to create key:", error);
-    throw error;
-  }
-};
-
 export const userFilterUICall = async (accessToken: string, params: URLSearchParams) => {
   try {
     return await apiClient.get(`/user/filter/ui`, {
@@ -2081,214 +1980,6 @@ export const uiSpendLogsCall = async ({
   }
 };
 
-/**
- * Agent tracing. All three respond 501 `{detail}` when `general_settings.tracing` is not
- * configured; callers can detect that through the thrown `ApiError`'s `status`.
- */
-export const agentTraceListCall = async ({
-  accessToken,
-  startMs,
-  endMs,
-  cursor,
-}: {
-  accessToken: string;
-  startMs: number;
-  endMs: number;
-  cursor?: string | null;
-}): Promise<TracePage> => {
-  const query = { start_ms: startMs, end_ms: endMs, cursor: cursor ?? undefined } satisfies TraceListQuery;
-  return apiClient.get<TracePage>(`/v1/traces`, { accessToken, query });
-};
-
-export const sendOtlpTraceCall = async (accessToken: string, exportRequest: object): Promise<void> =>
-  apiClient.post(`/v1/traces`, { accessToken, body: exportRequest });
-
-export const agentTraceCall = async (
-  accessToken: string,
-  traceId: string,
-  traceRef?: string,
-  cursor?: string | null,
-): Promise<Trace> =>
-  apiClient.get<Trace>(`/v1/traces/${encodeURIComponent(traceId)}`, {
-    accessToken,
-    query: { trace_ref: traceRef || undefined, cursor: cursor ?? undefined, page_size: 200 } satisfies TraceDetailQuery,
-  });
-
-export const agentTraceSpanCall = async (
-  accessToken: string,
-  traceId: string,
-  spanId: string,
-  traceRef?: string,
-): Promise<SpanDetail> =>
-  apiClient.get<SpanDetail>(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, {
-    accessToken,
-    query: { trace_ref: traceRef || undefined } satisfies SpanQuery,
-  });
-
-export const agentTraceSpanErrorCall = async (
-  accessToken: string,
-  traceId: string,
-  spanId: string,
-  options: { traceRef?: string; cursor?: string | null },
-): Promise<SpanErrorPage> =>
-  apiClient.get<SpanErrorPage>(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}/error`, {
-    accessToken,
-    query: { trace_ref: options.traceRef || undefined, cursor: options.cursor || undefined } satisfies SpanErrorQuery,
-  });
-
-export const adminSpendLogsCall = async (accessToken: string) => {
-  try {
-    const data = await apiClient.get(`/global/spend/logs`, { accessToken });
-    return data;
-  } catch (error) {
-    console.error("Failed to create key:", error);
-    throw error;
-  }
-};
-
-export const adminTopKeysCall = async (accessToken: string) => {
-  try {
-    let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/keys?limit=5` : `/global/spend/keys?limit=5`;
-
-    const response = await fetch(url, {
-      method: "GET",
-      headers: {
-        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
-        "Content-Type": "application/json",
-      },
-    });
-    if (!response.ok) {
-      const errorData = await response.json();
-      const errorMessage = deriveErrorMessage(errorData);
-      handleError(errorMessage);
-      throw new Error(errorMessage);
-    }
-
-    const data = await response.json();
-    return data;
-  } catch (error) {
-    console.error("Failed to create key:", error);
-    throw error;
-  }
-};
-
-export const adminTopEndUsersCall = async (
-  accessToken: string,
-  keyToken: string | null,
-  startTime: string | undefined,
-  endTime: string | undefined,
-) => {
-  try {
-    const body = keyToken
-      ? { api_key: keyToken, startTime: startTime, endTime: endTime }
-      : { startTime: startTime, endTime: endTime };
-
-    const data = await apiClient.post(`/global/spend/end_users`, { accessToken, body });
-    return data;
-  } catch (error) {
-    console.error("Failed to create key:", error);
-    throw error;
-  }
-};
-
-export const adminspendByProvider = async (
-  accessToken: string,
-  startTime: string | undefined,
-  endTime: string | undefined,
-) => {
-  try {
-    const data = await apiClient.get(`/global/spend/provider`, {
-      accessToken,
-      query: {
-        ...(startTime && endTime ? { start_date: startTime, end_date: endTime } : {}),
-      },
-    });
-    return data;
-  } catch (error) {
-    console.error("Failed to fetch spend data:", error);
-    throw error;
-  }
-};
-
-export const adminGlobalActivity = async (
-  accessToken: string,
-  startTime: string | undefined,
-  endTime: string | undefined,
-) => {
-  try {
-    const data = await apiClient.get(`/global/activity`, {
-      accessToken,
-      query: startTime && endTime ? { start_date: startTime, end_date: endTime } : undefined,
-    });
-    return data;
-  } catch (error) {
-    console.error("Failed to fetch spend data:", error);
-    throw error;
-  }
-};
-
-export const adminGlobalActivityPerModel = async (
-  accessToken: string,
-  startTime: string | undefined,
-  endTime: string | undefined,
-) => {
-  try {
-    let url = proxyBaseUrl ? `${proxyBaseUrl}/global/activity/model` : `/global/activity/model`;
-
-    if (startTime && endTime) {
-      url += `?start_date=${startTime}&end_date=${endTime}`;
-    }
-
-    const requestOptions = {
-      method: "GET",
-      headers: {
-        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
-      },
-    };
-
-    const response = await fetch(url, requestOptions);
-
-    if (!response.ok) {
-      const errorData = await response.json();
-      const errorMessage = deriveErrorMessage(errorData);
-      handleError(errorMessage);
-      throw new Error(errorMessage);
-    }
-
-    const data = await response.json();
-    return data;
-  } catch (error) {
-    console.error("Failed to fetch spend data:", error);
-    throw error;
-  }
-};
-
-export const adminTopModelsCall = async (accessToken: string) => {
-  try {
-    let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/models?limit=5` : `/global/spend/models?limit=5`;
-
-    const response = await fetch(url, {
-      method: "GET",
-      headers: {
-        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
-        "Content-Type": "application/json",
-      },
-    });
-    if (!response.ok) {
-      const errorData = await response.json();
-      const errorMessage = deriveErrorMessage(errorData);
-      handleError(errorMessage);
-      throw new Error(errorMessage);
-    }
-
-    const data = await response.json();
-    return data;
-  } catch (error) {
-    console.error("Failed to create key:", error);
-    throw error;
-  }
-};
-
 export const testConnectionRequest = async (
   accessToken: string,
   litellm_params: Record<string, any>,
@@ -2421,7 +2112,7 @@ export const testAutoRouterRouting = async (
   }
 };
 
-export interface ComplexityRouterConfigValidation {
+interface ComplexityRouterConfigValidation {
   valid: boolean;
   error?: string | null;
 }
@@ -2733,7 +2424,7 @@ export interface UserConnectionStartResponse {
   flow_handle: string;
 }
 
-export type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected";
+type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected";
 
 export interface UserConnectionPollResponse {
   status: UserConnectionPollStatus;
@@ -3697,35 +3388,6 @@ export const getProxyUISettings = async (accessToken: string) => {
   }
 };
 
-export const getUISettings = async (accessToken: string) => {
-  /**
-   * Get UI-specific configuration flags from the database
-   */
-  try {
-    const url = proxyBaseUrl ? `${proxyBaseUrl}/get/ui_settings` : `/get/ui_settings`;
-    const response = await fetch(url, {
-      method: "GET",
-      headers: {
-        [globalLitellmHeaderName]: `Bearer ${accessToken}`,
-        "Content-Type": "application/json",
-      },
-    });
-
-    if (!response.ok) {
-      const errorData = await response.json();
-      const errorMessage = deriveErrorMessage(errorData);
-      console.error("Failed to get UI settings:", errorMessage);
-      return null;
-    }
-
-    const data = await response.json();
-    return data;
-  } catch (error) {
-    console.error("Failed to get UI settings:", error);
-    return null;
-  }
-};
-
 export const getMCPSemanticFilterSettings = async (accessToken: string) => {
   /**
    * Get MCP semantic filter configuration
@@ -4685,12 +4347,12 @@ export const createAgentCall = async (accessToken: string, agentData: any) => {
 
 export type DiscoveredAgentCard = Agent["agent_card_params"];
 
-export type DiscoverAgentCardResponse = Omit<components["schemas"]["DiscoverAgentResponse"], "agent_card"> & {
+type DiscoverAgentCardResponse = Omit<components["schemas"]["DiscoverAgentResponse"], "agent_card"> & {
   agent_card: DiscoveredAgentCard;
 };
 
 export type DiscoveryMode = components["schemas"]["DiscoveryMode"];
-export type DiscoverAgentCardOptions = Partial<
+type DiscoverAgentCardOptions = Partial<
   Pick<components["schemas"]["DiscoverAgentRequest"], "discovery_mode" | "params">
 >;
 
@@ -5613,7 +5275,7 @@ export const vectorStoreListCall = async (
   }
 };
 
-export interface IndexesListResponse {
+interface IndexesListResponse {
   object: string;
   data: VectorStoreIndex[];
 }

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -5849,6 +5849,26 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/get/background_chat_settings": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /**
+         * Get Background Chat Settings
+         * @description Get saved background-response settings and the prerequisites for changing them.
+         */
+        get: operations["get_background_chat_settings_get_background_chat_settings_get"];
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/get/config/callbacks": {
         parameters: {
             query?: never;
@@ -18240,6 +18260,26 @@ export interface paths {
         patch: operations["typesafe_proxy_route_typesafe__endpoint__patch"];
         trace?: never;
     };
+    "/update/background_chat_settings": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        /**
+         * Update Background Chat Settings
+         * @description Save background-response polling settings for the proxy configuration refresh.
+         */
+        patch: operations["update_background_chat_settings_update_background_chat_settings_patch"];
+        trace?: never;
+    };
     "/update/default_team_settings": {
         parameters: {
             query?: never;
@@ -27157,6 +27197,37 @@ export interface components {
         } & {
             [key: string]: unknown;
         };
+        /** BackgroundChatSettings */
+        BackgroundChatSettings: {
+            /**
+             * Enabled
+             * @description Enable cache-backed background Responses for Chat and API requests.
+             */
+            enabled: boolean;
+        };
+        /** BackgroundChatSettingsResponse */
+        BackgroundChatSettingsResponse: {
+            /**
+             * Can Save
+             * @description Whether database-backed configuration updates are available.
+             */
+            can_save: boolean;
+            /**
+             * Enabled
+             * @description Enable cache-backed background Responses for Chat and API requests.
+             */
+            enabled: boolean;
+            /**
+             * Managed By Config
+             * @description Whether the responses configuration is owned by the config file.
+             */
+            managed_by_config: boolean;
+            /**
+             * Redis Configured
+             * @description Whether this proxy has coordination Redis configured.
+             */
+            redis_configured: boolean;
+        };
         /** BaseLitellmParams */
         BaseLitellmParams: {
             /**
@@ -38804,6 +38875,12 @@ export interface components {
             supported_openai_params: string[] | null;
             /** Supported Reasoning Efforts */
             supported_reasoning_efforts?: string[] | null;
+            /**
+             * Supports Background Chat
+             * @description Whether cache-backed background Responses runs are enabled for this model group
+             * @default false
+             */
+            supports_background_chat: boolean;
             /**
              * Supports Fast Mode
              * @default false
@@ -60982,6 +61059,26 @@ export interface operations {
             };
         };
     };
+    get_background_chat_settings_get_background_chat_settings_get: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["BackgroundChatSettingsResponse"];
+                };
+            };
+        };
+    };
     get_config_get_config_callbacks_get: {
         parameters: {
             query?: never;
@@ -76302,6 +76399,39 @@ export interface operations {
             };
         };
     };
+    update_background_chat_settings_update_background_chat_settings_patch: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["BackgroundChatSettings"];
+            };
+        };
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["BackgroundChatSettingsResponse"];
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
     update_default_team_settings_update_default_team_settings_patch: {
         parameters: {
             query?: never;