From f7b2cd0000945fe2e5f2d494fb0e1026b82bba31 Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:13:24 -0700 Subject: [PATCH 1/6] fix(anthropic): answer 400 naming a non-string role or tool call id instead of 500 (#49237) * fix(anthropic): answer 400 naming a non-string role or tool call id instead of 500 A list role or an object or list tool_call_id crashed the chat conversion with "unhashable type". Depending on the step that hit it, callers got a 500 or a 400 that never named the field. A failed conversion now answers 400 naming the first non-string role or tool call id, and the prompt caching hook reads a non-string answer id as matching no tool call. * fix(anthropic): name the field the conversion failed on, and answer 400 for a non-string Responses call_id The chat field check now runs only when the conversion raised a type error, and scans roles first, then assistant tool call ids, then tool answer ids, so an orphan tool answer that modify_params drops is no longer blamed. A tool output call_id that is not a string on /v1/responses answers 400 naming input[N].call_id, and the clear_tool_uses polyfill leaves a non-string tool_use_id for the provider * fix(anthropic): name a non-string role whatever error the conversion raised An int or bool role fails the conversion with an invalid message error, not a type error, so it lost the named 400. Roles are now named for any conversion failure, while tool call ids stay limited to type errors * fix(responses): name a non-string message item role by its input index * fix(responses): name a malformed input field only where litellm reads it, by the caller's index A tool output call_id is named only when it is an object or a list and the conversion fails, so an orphan numeric id still converts and another field's failure is never blamed on it. On the bridge to Claude a non-string role or function_call call_id is named by its input item, counting items before a compaction item. A numeric assistant tool call id on chat is named too. * fix(anthropic): name a tool id only for the error it causes and count Codex tool items in input indexes A numeric tool answer id that modify_params drops was blamed for any later type or attribute error, so the 400 hid the real cause. Tool ids are now named only for the error their value raises, and a hashable answer id never. The /v1/responses bridge and its input token count now read the input the caller sent, with only additional_tools hoisted into tools, so input[N] counts those items too. * fix(anthropic): name a tool id only for the error its own value causes and keep Codex tool item text out of the request * fix(responses): name a tool output call_id only when the items before it convert * fix(responses): name a non-string input field only on inference so token counting and batch records keep their answer * fix(responses): let a client field named caller_input or names_non_string_fields reach the bridge * test(anthropic): build the non-string id histories without editing them in place * fix(anthropic): never name an empty role for another field's failure * fix(anthropic): name an empty list or object role when the conversion fails on it * fix(anthropic): name a list role under the drop_params cache cap and keep the Bedrock batch line prefix --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../anthropic_cache_control_hook.py | 3 +- .../prompt_templates/common_utils.py | 14 +- .../prompt_templates/factory.py | 3 +- .../mid_conversation_system.py | 5 + litellm/llms/anthropic/chat/transformation.py | 100 ++++- .../editors/clear_tool_uses.py | 11 +- litellm/llms/base_llm/files/batch_records.py | 3 + litellm/llms/bedrock/files/transformation.py | 10 +- .../count_request.py | 10 + .../handler.py | 18 +- .../transformation.py | 140 ++++++- .../test_anthropic_cache_control_hook.py | 47 +++ .../test_anthropic_chat_transformation.py | 378 +++++++++++++++++- .../test_clear_tool_uses.py | 31 +- .../test_bedrock_files_transformation.py | 16 + .../test_anthropic_responses_bridge.py | 53 ++- .../test_count_request.py | 19 +- .../test_handler.py | 49 +++ .../test_litellm_completion_responses.py | 205 ++++++++++ 19 files changed, 1084 insertions(+), 31 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index fa519740e88..c05407b7972 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( tool_call_ids_answered_by_client, tool_call_ids_answered_in, ) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field from litellm.llms.anthropic.common_utils import ( is_claude_code_one_shot_subagent_request, supports_anthropic_cache_control, @@ -128,7 +129,7 @@ def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool: return False return ( not isinstance(fields.get("content"), list) - or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT + or string_field(fields, "role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT ) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 5fbd787b3b6..938ab585b72 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -8,7 +8,7 @@ import io import json import mimetypes import re -from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from graphlib import CycleError, TopologicalSorter from itertools import chain, groupby, islice @@ -3235,3 +3235,15 @@ def text_completion_prompt_to_messages(prompt: object) -> tuple[AllMessageValues if string_entries and len(string_entries) == len(entries): return tuple(ChatCompletionUserMessage(role="user", content=entry) for entry in string_entries) raise ValueError(f"`prompt` must be a non-empty string or a non-empty list of strings. Got: {prompt_type_name}.") + + +def is_present_non_string(value: object) -> bool: + return value is not None and not isinstance(value, str) + + +def invalid_type_message(field_path: str, value: object) -> str: + return f"Invalid type for '{field_path}': expected a string, got {type(value).__name__}" + + +def is_unhashable_value_behind(error: TypeError, value: object) -> bool: + return not isinstance(value, Hashable) and f"unhashable type: '{type(value).__name__}'" in str(error) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 320b4df8192..98348e434b2 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import parts_of, place_mid_conversation_system, split_leading_system_run, + string_field, ) from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value @@ -2023,7 +2024,7 @@ def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) following: Final = _messages_until_the_next_assistant(messages, index) a_user_turn_remains: Final = any( message_field(message, "role") in ("user", "tool", "function") - and message_field(message, "tool_call_id") not in kept_server_result_ids + and string_field(message, "tool_call_id") not in kept_server_result_ids for message in following ) return _tool_call_ids_in_tool_messages(following) & kept_server_result_ids if a_user_turn_remains else frozenset() diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index 3f956b0614c..ec6de4e206d 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -85,6 +85,11 @@ def message_field(message: object, key: str) -> object: return mapping.get(key) if mapping is not None else getattr(message, key, None) +def string_field(message: object, key: str) -> str | None: + value: Final = message_field(message, key) + return value if isinstance(value, str) else None + + def is_system_message(message: object) -> bool: return message_field(message, "role") == "system" diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index f03004cec15..f198d76f72a 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2,7 +2,9 @@ import copy import json import re import time -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from functools import partial +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast @@ -27,7 +29,10 @@ from litellm.constants import ( from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( DEFS_MAX_INLINED_BYTES, + invalid_type_message, is_object_schema, + is_present_non_string, + is_unhashable_value_behind, sanitize_input_schema_for_anthropic, schema_fragment_ref, schema_grammar_branch_count, @@ -45,6 +50,8 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( inline_remote_image_urls, ) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( + message_field, + parts_of, place_mid_conversation_system, split_leading_system_run, ) @@ -179,6 +186,76 @@ NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24 NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048 +def _tool_call_ids_failing( + index: int, tool_calls: object, fails: Callable[[object], bool] +) -> Iterator[tuple[str, object]]: + ids: Final = ((position, message_field(tool_call, "id")) for position, tool_call in enumerate(parts_of(tool_calls))) + return ( + (f"messages[{index}].tool_calls[{position}].id", tool_call_id) + for position, tool_call_id in ids + if fails(tool_call_id) + ) + + +def _roles_failing(messages: Sequence[object], fails: Callable[[object], bool]) -> Iterator[tuple[str, object]]: + roles: Final = ((index, message_field(message, "role")) for index, message in enumerate(messages)) + return ((f"messages[{index}].role", role) for index, role in roles if fails(role)) + + +def _is_non_empty_non_string(value: object) -> bool: + return bool(value) and is_present_non_string(value) + + +def _assistant_tool_call_ids_failing( + messages: Sequence[object], fails: Callable[[object], bool] +) -> Iterator[tuple[str, object]]: + assistant_turns: Final = ( + (index, message) for index, message in enumerate(messages) if message_field(message, "role") == "assistant" + ) + return chain.from_iterable( + _tool_call_ids_failing(index, message_field(message, "tool_calls"), fails) for index, message in assistant_turns + ) + + +def _tool_answer_ids_failing( + messages: Sequence[object], fails: Callable[[object], bool] +) -> Iterator[tuple[str, object]]: + answers: Final = ( + (index, message_field(message, "tool_call_id")) + for index, message in enumerate(messages) + if message_field(message, "role") in ("tool", "function") + ) + return ((f"messages[{index}].tool_call_id", tool_call_id) for index, tool_call_id in answers if fails(tool_call_id)) + + +def _is_non_string_value_behind(error: AttributeError, value: object) -> bool: + return is_present_non_string(value) and value is error.obj + + +def _fields_behind(messages: Sequence[object], error: Exception) -> Iterable[tuple[str, object]]: + match error: + case TypeError(): + caused_error: Final = partial(is_unhashable_value_behind, error) + return chain( + _assistant_tool_call_ids_failing(messages, caused_error), + _tool_answer_ids_failing(messages, caused_error), + _roles_failing(messages, caused_error), + ) + case AttributeError(): + return _assistant_tool_call_ids_failing(messages, partial(_is_non_string_value_behind, error)) + case _: + return () + + +def non_string_message_field_error(messages: Sequence[object], error: Exception) -> AnthropicError | None: + malformed_field: Final = next( + chain(_roles_failing(messages, _is_non_empty_non_string), _fields_behind(messages, error)), None + ) + if malformed_field is None: + return None + return AnthropicError(status_code=400, message=invalid_type_message(*malformed_field)) + + def _tool_type(tool: object) -> str | None: match tool: case {"type": str() as tool_type}: @@ -2370,12 +2447,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Handling anthropic API Prompt Caching if len(anthropic_system_message_list) > 0: optional_params["system"] = anthropic_system_message_list - conversation: Final = place_mid_conversation_system( - later_messages, - supports_mid_conversation_system=supports_mid_conversation_system( - model=model, custom_llm_provider=self.custom_llm_provider - ), - ) + try: + conversation: Final = place_mid_conversation_system( + later_messages, + supports_mid_conversation_system=supports_mid_conversation_system( + model=model, custom_llm_provider=self.custom_llm_provider + ), + ) + except TypeError as error: + field_error: Final = non_string_message_field_error(messages, error) + if field_error is None: + raise + raise field_error from error # Format rest of message according to anthropic guidelines try: anthropic_messages = anthropic_messages_pt( @@ -2385,7 +2468,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): drop_params=drop_params, ) except Exception as e: - raise AnthropicError( + conversion_field_error: Final = non_string_message_field_error(messages, e) + raise conversion_field_error or AnthropicError( status_code=400, message=f"{e}\nReceived Messages={request_messages}", ) # don't use verbose_logger.exception, if exception is raised diff --git a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py index fabc112d443..f1d50fb75e2 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py @@ -74,6 +74,7 @@ class _ClearToolUsesFields(BaseModel): clear_tool_inputs: bool | tuple[str, ...] | None = None +_BLOCK: Final = TypeAdapter(Mapping[str, object]) _BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) # api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently _API_MESSAGES: Final = MappingProxyType( @@ -317,12 +318,10 @@ def _clear_tool_results( new_blocks: list[object] = [] mutated = False for block in content: - if ( - isinstance(block, dict) - and block.get("type") == "tool_result" - and block.get("tool_use_id") in ids_to_clear - ): - new_blocks.append(_cleared_result(block)) + if (parsed := _parse(_BLOCK, block)) is not None and _block_id( + parsed, "tool_result", "tool_use_id" + ) in ids_to_clear: + new_blocks.append(_cleared_result(parsed)) mutated = True cleared += 1 else: diff --git a/litellm/llms/base_llm/files/batch_records.py b/litellm/llms/base_llm/files/batch_records.py index 070cf73cd57..87ac4bd2b93 100644 --- a/litellm/llms/base_llm/files/batch_records.py +++ b/litellm/llms/base_llm/files/batch_records.py @@ -19,6 +19,8 @@ def responses_batch_body_to_chat_body( openai_request_body: Mapping[str, object], custom_llm_provider: str | None = None, drop_params: bool | None = None, + *, + caller_input: str | ResponseInputParam | None = None, ) -> dict[str, object]: # mutable-ok: provider transforms take the bridged chat body as a plain dict """ Rewrite the body of an OpenAI `/v1/responses` batch record as a Chat Completions body. @@ -52,5 +54,6 @@ def responses_batch_body_to_chat_body( responses_api_request=responses_request, custom_llm_provider=custom_llm_provider, drop_params=drop_params, + caller_input=caller_input, metadata=openai_request_body.get("metadata"), ) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 0624c2c65db..dcdd9e39c20 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -42,6 +42,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, text_completion_prompt_to_messages, ) +from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body @@ -347,7 +348,14 @@ class _BatchInputLine: record: _OpenAIBatchRecord -_BATCH_RECORD_CONVERSION_ERRORS: Final = (AttributeError, LookupError, TypeError, ValueError, BadRequestError) +_BATCH_RECORD_CONVERSION_ERRORS: Final = ( + AttributeError, + LookupError, + TypeError, + ValueError, + BadRequestError, + AnthropicError, +) def _batch_line_error(line_number: int, problem: str) -> BedrockError: diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index a24a26aac42..0b769326012 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -128,6 +128,15 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object], drop return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)} +def _caller_input(responses_request: Mapping[str, object]) -> ResponseInputParam | None: + responses_input: Final = responses_request.get("input") + return ( + cast(ResponseInputParam, responses_input) # cast-ok: forwarded as sent, as inference does + if isinstance(responses_input, list) + else None + ) + + def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]: tools: Final = responses_request.get("tools") if tools is None: @@ -200,6 +209,7 @@ async def chat_count_request( ), custom_llm_provider, policy.drop_params, + caller_input=_caller_input(responses_request), ), model, custom_llm_provider, diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 04fe027cbcc..802ed6bf9c5 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -4,6 +4,7 @@ Handler for transforming responses api requests to litellm.completion requests from collections.abc import Coroutine, Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Final from pydantic import TypeAdapter @@ -32,6 +33,13 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ModelResponse _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_BRIDGE_TRANSFORM_KEYWORDS: Final = frozenset({"caller_input", "names_non_string_fields"}) + + +def _without_bridge_transform_keywords(request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType( + {key: value for key, value in request_kwargs.items() if key not in _BRIDGE_TRANSFORM_KEYWORDS} + ) def _chat_verbosity(responses_api_request: Mapping[str, object]) -> Mapping[str, object]: @@ -95,6 +103,7 @@ class LiteLLMCompletionTransformationHandler: previous_response_id, hoisted, model=model, + caller_input=input, responses_api_request=responses_api_request, custom_llm_provider=custom_llm_provider, stream=stream, @@ -108,11 +117,13 @@ class LiteLLMCompletionTransformationHandler: **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged_input, + caller_input=input, + names_non_string_fields=True, responses_api_request=bridged_request, custom_llm_provider=custom_llm_provider, stream=stream, extra_headers=extra_headers, - **kwargs, + **_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)), ), **_chat_verbosity(bridged_request), } @@ -163,6 +174,7 @@ class LiteLLMCompletionTransformationHandler: /, *, model: str, + caller_input: str | ResponseInputParam, responses_api_request: ResponsesAPIOptionalRequestParams, custom_llm_provider: str | None, stream: bool | None, @@ -186,11 +198,13 @@ class LiteLLMCompletionTransformationHandler: **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged.input, + caller_input=caller_input, + names_non_string_fields=True, responses_api_request=bridged.responses_api_request, custom_llm_provider=custom_llm_provider, stream=stream, extra_headers=extra_headers, - **request_kwargs, + **_without_bridge_transform_keywords(request_kwargs), ), **_chat_verbosity(bridged.responses_api_request), }, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 17ae9f2e656..0f4ed0611be 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -45,6 +45,7 @@ from litellm.constants import ( REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, ) +from litellm.exceptions import BadRequestError from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, block_with_cache_control, @@ -65,6 +66,12 @@ from litellm.litellm_core_utils.hidden_params import ( ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + invalid_type_message, + is_present_non_string, + is_unhashable_value_behind, +) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control from litellm.llms.anthropic.messages.native.utils import anthropic_stop_reason, openai_chat_refusal_text from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM @@ -268,6 +275,7 @@ _IMAGE_URL_ADAPTER: Final = TypeAdapter(ChatCompletionImageUrlObject) _FILE_PART_ADAPTER: Final = TypeAdapter(_ResponsesFilePart) _MessageT = TypeVar("_MessageT") _MediaT: Final = TypeVar("_MediaT") +_InputItemT: Final = TypeVar("_InputItemT") _CACHED_CONTENT_ADAPTER: Final = TypeAdapter(ChatCompletionCachedContent) _DICT_ITEMS_LIST_ADAPTER: Final = TypeAdapter(list[dict[object, object]]) _TEXT_ADAPTER: Final = TypeAdapter(str) @@ -445,6 +453,86 @@ TOOL_CALL_OUTPUT_ITEM_TYPES: Final = frozenset( ) +def _read_positions( + input_items: Sequence[_InputItemT], read_input: str | ResponseInputParam +) -> Iterator[tuple[int, _InputItemT]]: + read_ids: Final = frozenset(map(id, parts_of(read_input))) + return ((index, item) for index, item in enumerate(input_items) if id(item) in read_ids) + + +def _first_invalid_input_type(fields: Iterator[tuple[str, object]]) -> BadRequestError | None: + first: Final = next(fields, None) + return None if first is None else _invalid_input_type(*first) + + +def _is_tool_output_item(item: object) -> bool: + item_type: Final = message_field(item, "type") + return isinstance(item_type, str) and item_type in TOOL_CALL_OUTPUT_ITEM_TYPES + + +def _converts_before( + convert_input: Callable[[str | ResponseInputParam], object], sent_input: str | ResponseInputParam, position: int +) -> bool: + try: + convert_input(sent_input[:position]) + except TypeError: + return False + return True + + +def _unhashable_tool_output_call_id_error( + input: str | ResponseInputParam, + read_input: str | ResponseInputParam, + error: TypeError, + converts_before: Callable[[int], bool], +) -> BadRequestError | None: + call_ids: Final = ( + (f"input[{index}].call_id", message_field(item, "call_id"), item) + for index, item in _read_positions(parts_of(input), read_input) + if _is_tool_output_item(item) + ) + first: Final = next((candidate for candidate in call_ids if is_unhashable_value_behind(error, candidate[1])), None) + if first is None: + return None + path, call_id, item = first + read_position: Final = next( + position for position, read_item in enumerate(parts_of(read_input)) if read_item is item + ) + return _invalid_input_type(path, call_id) if converts_before(read_position) else None + + +def _carries_non_string_field(message: object) -> bool: + tool_call_ids: Final = ( + message_field(tool_call, "id") for tool_call in parts_of(message_field(message, "tool_calls")) + ) + return is_present_non_string(message_field(message, "role")) or any(map(is_present_non_string, tool_call_ids)) + + +def _non_string_item_fields( + item: Mapping[str, object], convert_item: Callable[[Mapping[str, object]], Sequence[object]] +) -> Iterator[tuple[str, object]]: + messages: Final = convert_item(item) + tool_call_id_field: Final = "call_id" if item.get("call_id") else "id" + roles: Final = (("role", message_field(message, "role")) for message in messages) + tool_calls: Final = chain.from_iterable(parts_of(message_field(message, "tool_calls")) for message in messages) + tool_call_ids: Final = ((tool_call_id_field, message_field(tool_call, "id")) for tool_call in tool_calls) + return ((field, value) for field, value in chain(roles, tool_call_ids) if is_present_non_string(value)) + + +def _indexed_non_string_item_fields( + input: ResponseInputParam, + read_input: str | ResponseInputParam, + convert_item: Callable[[Mapping[str, object]], Sequence[object]], +) -> Iterator[tuple[str, object]]: + for index, item in _read_positions(input, read_input): + for field, value in _non_string_item_fields(item, convert_item): + yield f"input[{index}].{field}", value + + +def _invalid_input_type(field_path: str, value: object) -> BadRequestError: + return BadRequestError(message=invalid_type_message(field_path, value), model="", llm_provider="") + + def _input_item_role(input_item: Mapping[str, object]) -> str: return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated @@ -1101,6 +1189,9 @@ class LiteLLMCompletionResponsesConfig: stream: bool | None = None, extra_headers: Mapping[str, object] | None = None, drop_params: object = None, + *, + caller_input: str | ResponseInputParam | None = None, + names_non_string_fields: bool = False, **kwargs, ) -> dict: """ @@ -1149,6 +1240,7 @@ class LiteLLMCompletionResponsesConfig: native_compaction: Final = _replays_compaction_block( request_input, context_management ) and provider_takes_compaction_blocks(model, custom_llm_provider) + numbered_input: Final = request_input if caller_input is None else caller_input bridged_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=request_input, responses_api_request=responses_api_request, @@ -1157,7 +1249,28 @@ class LiteLLMCompletionResponsesConfig: tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs), native_compaction=native_compaction, only_since_compaction=not native_compaction, + caller_input=numbered_input, ) + malformed_field_error: Final = ( + _first_invalid_input_type( + _indexed_non_string_item_fields( + numbered_input, + _input_to_read(request_input, not native_compaction, native_compaction), + partial( + LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message, + replay_reasoning=True, + native_compaction=native_compaction, + ), + ) + ) + if names_non_string_fields + and isinstance(numbered_input, list) + and any(map(_carries_non_string_field, bridged_messages)) + and provider_builds_anthropic_request(model, custom_llm_provider) + else None + ) + if malformed_field_error is not None: + raise malformed_field_error litellm_completion_request: dict = { "messages": ( @@ -1221,6 +1334,7 @@ class LiteLLMCompletionResponsesConfig: tool_call_provider_fields: bool = False, native_compaction: bool = False, only_since_compaction: bool = False, + caller_input: str | ResponseInputParam | None = None, ) -> list[ AllMessageValues | GenericChatCompletionMessage @@ -1274,13 +1388,27 @@ class LiteLLMCompletionResponsesConfig: dumped_input: Final = with_compaction_items_as_params(input) read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction) - messages.extend( - LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( - input=_without_item_keys(read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)), - replay_reasoning=replay_reasoning, - native_compaction=native_compaction, - ) + sent_input: Final = _without_item_keys( + read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields) ) + convert_input: Final = partial( + LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message, + replay_reasoning=replay_reasoning, + native_compaction=native_compaction, + ) + try: + converted: Final = convert_input(sent_input) + except TypeError as error: + call_id_error: Final = _unhashable_tool_output_call_id_error( + dumped_input if caller_input is None else caller_input, + read_input, + error, + partial(_converts_before, convert_input, sent_input), + ) + if call_id_error is None: + raise + raise call_id_error from error + messages.extend(converted) return messages diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 05a63030c69..778c8dfe2f5 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -5042,3 +5042,50 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi {"type": "ephemeral"}, {"type": "ephemeral"}, ] + + +def _search_conversation_with(follow_up: dict[str, object]) -> list[AllMessageValues]: + return [ + cast(AllMessageValues, {"role": "user", "content": "Weather in Paris?"}), + _replayed_search_call(_SEARCH_CALL), + cast(AllMessageValues, follow_up), + cast(AllMessageValues, _GO_ON), + ] + + +def _cache_marks_after_injection(messages: list[AllMessageValues]) -> tuple[int, tuple[int, ...]]: + _, processed, _ = AnthropicCacheControlHook().get_chat_completion_prompt( + model="anthropic/claude-opus-5-5", + messages=messages, + non_default_params={"cache_control_injection_points": [{"location": "message", "role": "user"}]}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return ( + AnthropicCacheControlHook.count_request_cache_breakpoints(messages), + tuple(_marks_in(message) for message in processed), + ) + + +@pytest.mark.parametrize( + ("malformed", "well_formed"), + [ + pytest.param( + {**_SEARCH_ANSWER, "tool_call_id": {"id": "srvtoolu_01Search"}}, + {**_SEARCH_ANSWER, "tool_call_id": "toolu_01Unrelated"}, + id="object_tool_call_id", + ), + pytest.param( + {**_SEARCH_ANSWER, "tool_call_id": ["srvtoolu_01Search"]}, + {**_SEARCH_ANSWER, "tool_call_id": "toolu_01Unrelated"}, + id="list_tool_call_id", + ), + ], +) +def test_cache_control_hook_reads_a_non_string_answer_id_as_answering_no_tool_call( + malformed: dict[str, object], well_formed: dict[str, object] +): + assert _cache_marks_after_injection(_search_conversation_with(malformed)) == _cache_marks_after_injection( + _search_conversation_with(well_formed) + ) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index c8a2f783809..636e23168cf 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -25,7 +25,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.llms.anthropic.chat import ModelResponseIterator -from litellm.llms.anthropic.common_utils import process_anthropic_headers +from litellm.llms.anthropic.common_utils import AnthropicError, process_anthropic_headers from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema from litellm.llms.anthropic.messages.native.transformation import ( @@ -9922,6 +9922,382 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert litellm_params["metadata"] == {} +_KEPT_SEARCH_CALL: Final = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "tool_search_tool_bm25", "arguments": "{}"}, + } + ], + "provider_specific_fields": { + "tool_results": [ + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01Search", + "content": {"type": "tool_search_tool_search_result", "tool_references": []}, + } + ] + }, +} +_WEATHER_CALL: Final = { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}], +} +_ASK: Final = {"role": "user", "content": "Weather in Paris?"} + + +def _search_answered_with(tool_call_id: object) -> list[dict[str, object]]: + return [ + _ASK, + _KEPT_SEARCH_CALL, + {"role": "tool", "tool_call_id": tool_call_id, "content": "found get_weather"}, + {"role": "user", "content": "Go on"}, + ] + + +def _orphan_answer_with(tool_call_id: object) -> list[dict[str, object]]: + return [ + _ASK, + {"role": "assistant", "content": "Checking"}, + {"role": "tool", "tool_call_id": tool_call_id, "content": "sunny"}, + {"role": "user", "content": "Go on"}, + ] + + +@pytest.mark.parametrize( + ("messages", "field_path", "type_name"), + [ + pytest.param( + [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": ["user"], "content": "Go on"}, + {"role": "system", "content": "Answer in one sentence."}, + ], + "messages[2].role", + "list", + id="list_role_before_a_later_system_message", + ), + pytest.param( + [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": ["user"], "content": "Go on"}, + ], + "messages[2].role", + "list", + id="list_role", + ), + pytest.param( + [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": 123, "content": "Go on"}, + {"role": "system", "content": "Answer in one sentence."}, + ], + "messages[2].role", + "int", + id="int_role_before_a_later_system_message", + ), + pytest.param( + [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": True, "content": "Go on"}], + "messages[3].role", + "bool", + id="bool_role_after_an_orphan_object_tool_call_id", + ), + pytest.param( + _search_answered_with({"id": "srvtoolu_01Search"}), + "messages[2].tool_call_id", + "dict", + id="object_tool_call_id_after_a_kept_server_result", + ), + pytest.param( + _search_answered_with(["srvtoolu_01Search"]), + "messages[2].tool_call_id", + "list", + id="list_tool_call_id_after_a_kept_server_result", + ), + pytest.param( + [_ASK, _WEATHER_CALL, {"role": "tool", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"}], + "messages[2].tool_call_id", + "dict", + id="object_tool_call_id_answering_a_client_tool_call", + ), + pytest.param( + [ + _ASK, + _WEATHER_CALL, + {"role": "function", "name": "get_weather", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"}, + ], + "messages[2].tool_call_id", + "dict", + id="object_tool_call_id_on_a_function_message", + ), + pytest.param( + [ + _ASK, + {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ], + "messages[1].tool_calls[0].id", + "dict", + id="object_assistant_tool_call_id", + ), + pytest.param( + [ + _ASK, + {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": 7}]}, + {"role": "tool", "tool_call_id": "7", "content": "sunny"}, + ], + "messages[1].tool_calls[0].id", + "int", + id="int_assistant_tool_call_id", + ), + pytest.param( + [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": ["user"], "content": "Go on"}], + "messages[3].role", + "list", + id="list_role_after_an_orphan_object_tool_call_id", + ), + pytest.param( + [ + *_orphan_answer_with({"id": "toolu_01"})[:3], + {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ], + "messages[3].tool_calls[0].id", + "dict", + id="object_assistant_tool_call_id_after_an_orphan_object_tool_call_id", + ), + pytest.param( + [ + {**_ASK, "tool_call_id": {"id": "stray"}}, + _WEATHER_CALL, + {"role": "tool", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"}, + ], + "messages[2].tool_call_id", + "dict", + id="object_tool_call_id_after_a_stray_id_on_a_user_message", + ), + pytest.param( + [ + {**_ASK, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "stray"}}]}, + {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ], + "messages[1].tool_calls[0].id", + "dict", + id="object_assistant_tool_call_id_after_stray_tool_calls_on_a_user_message", + ), + pytest.param( + [_ASK, {"role": [], "content": "ok"}, {"role": "system", "content": "Be terse."}], + "messages[1].role", + "list", + id="empty_list_role_before_a_system_message", + ), + pytest.param( + [_ASK, {"role": {}, "content": "ok"}, {"role": "user", "content": "again"}], + "messages[1].role", + "dict", + id="empty_object_role", + ), + ], +) +@pytest.mark.parametrize("modify_params", [False, True]) +@pytest.mark.parametrize("model", [UNFLAGGED_CLAUDE, FLAGGED_CLAUDE]) +def test_chat_transform_request_answers_400_naming_a_non_string_role_or_tool_call_id( + local_model_cost_map, + monkeypatch: pytest.MonkeyPatch, + messages: list[dict[str, object]], + field_path: str, + type_name: str, + modify_params: bool, + model: str, +): + monkeypatch.setattr(litellm, "modify_params", modify_params) + + with pytest.raises(AnthropicError) as raised: + _chat_request(AnthropicConfig(), model, copy.deepcopy(messages)) + + assert (raised.value.status_code, raised.value.message) == ( + 400, + f"Invalid type for '{field_path}': expected a string, got {type_name}", + ) + + +@pytest.mark.parametrize(("role", "type_name"), [(["user"], "list"), ({"name": "user"}, "dict"), ([], "list")]) +def test_chat_transform_request_names_a_non_string_role_on_a_cache_marked_message_under_drop_params( + local_model_cost_map, role: object, type_name: str +): + marked: Final = {"role": role, "content": [{"type": "text", "text": "ok"}], "cache_control": {"type": "ephemeral"}} + + with pytest.raises(AnthropicError) as raised: + AnthropicConfig().transform_request( + model=UNFLAGGED_CLAUDE, + messages=[_ASK, marked], + optional_params={}, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert (raised.value.status_code, raised.value.message) == ( + 400, + f"Invalid type for 'messages[1].role': expected a string, got {type_name}", + ) + + +@pytest.mark.parametrize( + ("malformed", "well_formed", "modify_params"), + [ + pytest.param( + [{**_ASK, "tool_call_id": {"id": "stray"}}], [_ASK], False, id="stray_tool_call_id_on_a_user_message" + ), + pytest.param( + [{**_ASK, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}], + [_ASK], + False, + id="stray_tool_calls_on_a_user_message", + ), + pytest.param( + _orphan_answer_with({"id": "toolu_01"}), + _orphan_answer_with("toolu_01"), + True, + id="orphan_answer_dropped_under_modify_params", + ), + ], +) +def test_chat_transform_request_keeps_converting_a_non_string_id_it_never_reads( + local_model_cost_map, + monkeypatch: pytest.MonkeyPatch, + malformed: list[dict[str, object]], + well_formed: list[dict[str, object]], + modify_params: bool, +): + monkeypatch.setattr(litellm, "modify_params", modify_params) + + converted: Final = _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, copy.deepcopy(malformed)) + + assert ( + converted["messages"] + == _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, copy.deepcopy(well_formed))["messages"] + ) + + +def test_chat_transform_request_keeps_a_conversion_error_that_a_dropped_orphan_answer_did_not_cause( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "modify_params", True) + + with pytest.raises(AnthropicError) as raised: + _chat_request( + AnthropicConfig(), + UNFLAGGED_CLAUDE, + [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": "bogus", "content": "Go on"}], + ) + + assert raised.value.status_code == 400 + assert "Invalid Message passed in {'role': 'bogus'" in raised.value.message + + +@pytest.mark.parametrize( + ("later_turn", "real_cause"), + [ + pytest.param( + [{"role": "user", "content": ["hi", "there"]}], + "'str' object has no attribute 'get'", + id="string_content_parts", + ), + pytest.param( + [ + { + **_WEATHER_CALL, + "tool_calls": [ + { + **_WEATHER_CALL["tool_calls"][0], + "function": {"name": "get_weather", "arguments": {"city": "Paris"}}, + } + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "rain"}, + ], + "'dict' object has no attribute 'strip'", + id="object_tool_call_arguments", + ), + pytest.param( + [{"role": "user", "content": [{"type": "image_url", "image_url": ["x"]}]}], + "list indices must be integers or slices, not str", + id="list_image_url", + ), + ], +) +@pytest.mark.parametrize("dropped_answer_id", [7, {"id": "toolu_01"}], ids=["int", "object"]) +def test_chat_transform_request_never_blames_a_dropped_tool_answer_id_for_another_failure( + local_model_cost_map, + monkeypatch: pytest.MonkeyPatch, + later_turn: list[dict[str, object]], + real_cause: str, + dropped_answer_id: object, +): + monkeypatch.setattr(litellm, "modify_params", True) + + with pytest.raises(AnthropicError) as raised: + _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, [*_orphan_answer_with(dropped_answer_id)[:3], *later_turn]) + + assert raised.value.status_code == 400 + assert raised.value.message.startswith(real_cause) + + +def test_chat_transform_request_never_blames_a_numeric_assistant_tool_call_id_for_an_earlier_failure( + local_model_cost_map, +): + with pytest.raises(AnthropicError) as raised: + _chat_request( + AnthropicConfig(), + UNFLAGGED_CLAUDE, + [ + {"role": "user", "content": ["hi", "there"]}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": 5, "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "5", "content": "rain"}, + ], + ) + + assert raised.value.status_code == 400 + assert raised.value.message.startswith("'str' object has no attribute 'get'") + + +@pytest.mark.parametrize("empty_role", [0, False]) +def test_chat_transform_request_never_blames_an_empty_role_for_another_failure(local_model_cost_map, empty_role): + list_part_type_after_an_empty_role: Final = [ + _ASK, + {"role": empty_role, "content": "ok"}, + {"role": "user", "content": [{"type": ["text"], "text": "again"}]}, + {"role": "system", "content": "Be terse."}, + ] + + with pytest.raises(TypeError): + _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, list_part_type_after_an_empty_role) + + +def test_chat_transform_request_names_no_field_for_a_failure_outside_roles_and_tool_call_ids(local_model_cost_map): + list_part_type: Final = [ + _ASK, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": [{"type": ["text"], "text": "again"}]}, + {"role": "system", "content": "Be terse."}, + ] + + with pytest.raises(TypeError): + _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, list_part_type) + + def _five_cache_hint_conversation() -> list[dict[str, object]]: return [ {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, diff --git a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py index 4cbf4cfa5f6..70eddd9e16e 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py @@ -614,9 +614,18 @@ def test_clear_tool_inputs_list_clears_only_the_named_tools(): } +def _history_with_first_block_field(index: int, field: str, value: object) -> list[dict[str, object]]: + history: Final = _make_history(n_pairs=4) + return [ + {**message, "content": [{**message["content"][0], field: value}, *message["content"][1:]]} + if position == index + else message + for position, message in enumerate(history) + ] + + def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject(): - messages = _make_history(n_pairs=4) - messages[1]["content"][0]["id"] = ["toolu_00"] + messages: Final = _history_with_first_block_field(1, "id", ["toolu_00"]) new_messages, applied = apply_clear_tool_uses_20250919( model=MODEL, messages=messages, @@ -635,6 +644,24 @@ def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject(): assert (new_messages[3]["content"][0]["input"], new_messages[5]["content"][0]["input"]) == ({}, {}) +def test_a_tool_result_with_a_non_string_tool_use_id_is_left_for_the_provider_to_reject(): + messages: Final = _history_with_first_block_field(2, "tool_use_id", ["toolu_00"]) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + }, + ) + assert applied is not None + assert new_messages[2] == messages[2] + assert _cleared_ids(new_messages[3:]) == {"toolu_01", "toolu_02"} + + async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicContextManagementError: with pytest.raises(AnthropicContextManagementError) as raised: await apply_context_management( diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 9d9e39df944..512e1f325f4 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -4472,6 +4472,17 @@ def test_file_content_conversion_keeps_a_line_whose_json_holds_unicode_line_sepa assert bedrock_record["modelInput"]["messages"] == [{"role": "user", "content": [{"text": text}]}] +_CLAUDE_BODY_WITH_A_LIST_ROLE_BEFORE_A_SYSTEM_MESSAGE: Final = { + "model": "bedrock/us.anthropic.claude-haiku-5-5", + "max_tokens": 20, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": ["user"], "content": "ok"}, + {"role": "system", "content": "Be terse."}, + ], +} + + def _batch_line(body: object) -> str: return json.dumps({**_VALID_BATCH_RECORD, "body": body}) @@ -4516,6 +4527,11 @@ def _batch_line_with_body(field: str, value: object) -> str: "Batch input file line 2 could not be converted to a Bedrock batch record", id="string-messages", ), + pytest.param( + f"{_VALID_BATCH_LINE}\n{_batch_line(_CLAUDE_BODY_WITH_A_LIST_ROLE_BEFORE_A_SYSTEM_MESSAGE)}\n".encode(), + "Batch input file line 2 could not be converted to a Bedrock batch record", + id="claude-list-role-before-a-system-message", + ), pytest.param( f"{_VALID_BATCH_LINE}\n{'[' * 1_000_000}{']' * 1_000_000}\n".encode(), "Batch input file line 2 could not be parsed", diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index bc73ef8598a..5ae985ea7e5 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -1598,6 +1598,53 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search assert prisma_client.db.queries == 1 +@pytest.mark.asyncio +async def test_a_codex_additional_tools_item_sends_claude_its_tools_but_not_its_text() -> None: + anthropic: Final = _RecordingRawAnthropicMessages() + + await litellm.aresponses( + model=_MODEL, + input=[ + {"type": "additional_tools", "role": "developer", "content": "Reply PINEAPPLE.", "tools": [_SHELL_TOOL]}, + {"role": "user", "content": "Hi"}, + ], + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + sent: Final = json.loads(anthropic.body) + assert [tool["name"] for tool in sent["tools"]] == ["shell"] + assert "PINEAPPLE" not in json.dumps(sent) + + +@pytest.mark.asyncio +async def test_previous_response_id_names_a_malformed_item_after_a_codex_additional_tools_item_by_its_index() -> None: + weather_tool: Final = { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + } + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SEARCH_TURN, _CALENDAR_TURN])), + pytest.raises(litellm.BadRequestError) as raised, + ): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-calendar-turn", + input=[ + {"type": "additional_tools", "role": "developer", "tools": [weather_tool]}, + {"type": "function_call_output", "call_id": "toolu_standup", "output": "Created Standup."}, + {"role": ["user"], "content": "Now create another event titled Retro."}, + ], + tools=[_CODEX_TOOL_SEARCH, _SHELL_TOOL], + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_RecordingRawAnthropicMessages())), + ) + + assert "'input[2].role'" in raised.value.message + + _REFUSAL_EXPLANATION: Final = ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"]["explanation"] _ANSWER_BEFORE_REFUSAL: Final = {"type": "text", "text": "Here is the start."} @@ -1808,7 +1855,11 @@ _RAN_SEARCH_RESULT: Final = { {"type": "web_search_result", "url": "https://tokyo.example", "title": "Tokyo", "encrypted_content": "e"} ], } -_THINKING_BEFORE_SEARCHES: Final = {"type": "thinking", "thinking": "Find each sister city in turn.", "signature": "sig"} +_THINKING_BEFORE_SEARCHES: Final = { + "type": "thinking", + "thinking": "Find each sister city in turn.", + "signature": "sig", +} _TEXT_BEFORE_PENDING_SEARCH: Final = {"type": "text", "text": "Tokyo pairs with Paris. Checking Paris next."} _PENDING_SEARCH: Final = { "type": "server_tool_use", diff --git a/tests/unit/responses/litellm_completion_transformation/test_count_request.py b/tests/unit/responses/litellm_completion_transformation/test_count_request.py index 2080a091160..e0a5997eac7 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_count_request.py +++ b/tests/unit/responses/litellm_completion_transformation/test_count_request.py @@ -86,7 +86,13 @@ async def test_claude_count_request_counts_additional_tools_input_items_as_tools policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset()) hoisted: Final = await _claude_count( - {"input": [{"type": "additional_tools", "tools": [_WEATHER_TOOL]}, _QUESTION]}, policy + { + "input": [ + {"type": "additional_tools", "role": "developer", "content": "Reply PINEAPPLE.", "tools": [_WEATHER_TOOL]}, + _QUESTION, + ] + }, + policy, ) assert isinstance(hoisted, ChatCountRequest) @@ -129,6 +135,17 @@ async def test_claude_count_request_rejects_a_malformed_tool_flag_without_drop_p assert "defer_loading" in flagged.message +@pytest.mark.asyncio +async def test_claude_count_request_leaves_a_non_string_role_for_the_count_endpoint_to_judge() -> None: + counted: Final = await _claude_count( + {"input": [{"type": "additional_tools", "tools": [_WEATHER_TOOL]}, _QUESTION, {"role": ["user"], "content": "?"}]}, + DropPolicy(drop_params=False, additional_drop_params=frozenset()), + ) + + assert isinstance(counted, ChatCountRequest) + assert [message.get("role") for message in counted.messages] == ["user", ["user"]] + + @pytest.mark.asyncio async def test_claude_count_request_leaves_out_tools_the_deployment_drops() -> None: policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset({"tools"})) diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index d731bf5f41c..4794f3701c5 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -246,6 +246,26 @@ async def test_bridged_turn_with_a_client_field_named_previous_response_client_i assert isinstance(response, ResponsesAPIResponse) +@pytest.mark.asyncio +@pytest.mark.parametrize("client_field", ["caller_input", "names_non_string_fields"]) +async def test_bridged_turn_with_a_client_field_named_like_a_bridge_keyword_reaches_the_provider(client_field: str): + provider: Final = _RecordingAnthropicHandler(_ANTHROPIC_MESSAGE_PAYLOAD) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider)) + + response = await litellm.aresponses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + client=client, + **{client_field: "from the client body"}, + ) + + assert provider.request_body is not None, "the bridged turn never reached the provider" + assert isinstance(response, ResponsesAPIResponse) + + class _StoredTurns: def __init__(self, rows: tuple[Mapping[str, object], ...]) -> None: self.rows: Final = rows @@ -310,3 +330,32 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h ("assistant", json.dumps([{"type": "text", "text": "Noted."}])), ("user", json.dumps([{"type": "text", "text": "Double it"}])), ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("client_field", ["caller_input", "names_non_string_fields"]) +async def test_bridged_follow_up_after_a_client_tool_search_with_a_client_field_named_like_a_bridge_keyword_reaches_the_provider( + client_field: str, +): + previous_response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="azure_ai", model_id="deployment-a", response_id="chatcmpl-turn-one" + ) + stored_turns: Final = _StoredTurns((_stored_mcp_turn(previous_response_id),)) + provider: Final = _RecordingAnthropicHandler(_ANTHROPIC_MESSAGE_PAYLOAD) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider)) + + with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored_turns)): + response = await litellm.aresponses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + tools=[{"type": "tool_search", "execution": "client"}], + previous_response_id=previous_response_id, + client=client, + **{client_field: "from the client body"}, + ) + + assert provider.request_body is not None, "the bridged turn never reached the provider" + assert isinstance(response, ResponsesAPIResponse) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index bb1f5104200..b4cd606374a 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -18,6 +18,7 @@ from openai.types.responses.response_function_web_search import ( from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field from litellm.constants import CALLER_SESSION_ID_METADATA_KEY from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -7189,3 +7190,207 @@ async def test_bridge_partial_response_on_disconnect_bills_only_the_input_since_ assert await _input_tokens_billed_on_disconnect( cast(ResponseInputParam, [*history_it_replaces, *since_compaction]) ) == await _input_tokens_billed_on_disconnect(since_compaction) + + +_ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-5-5" +_WEATHER_ASK: Final = {"role": "user", "content": "Weather in Paris?"} + + +def _weather_call(**fields: object) -> dict[str, object]: + return {"type": "function_call", "name": "get_weather", "arguments": "{}", **fields} + + +def _bridge_to(model: str, history: list[dict[str, object]], **request: object) -> dict: + return LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input=cast(ResponseInputParam, history), + responses_api_request=cast(ResponsesAPIOptionalRequestParams, request), + names_non_string_fields=True, + ) + + +@pytest.mark.parametrize("output_type", ["function_call_output", "custom_tool_call_output"]) +@pytest.mark.parametrize(("call_id", "type_name"), [({"id": "call_weather"}, "dict"), (["call_weather"], "list")]) +def test_a_tool_output_with_an_unhashable_call_id_is_a_400_naming_its_input_item( + output_type: str, call_id: object, type_name: str +) -> None: + history: Final = cast( + ResponseInputParam, + [_WEATHER_ASK, _weather_call(call_id="call_weather"), {"type": output_type, "call_id": call_id, "output": "sunny"}], + ) + + with pytest.raises(litellm.BadRequestError) as raised: + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={} + ) + + assert raised.value.message == ( + f"litellm.BadRequestError: Invalid type for 'input[2].call_id': expected a string, got {type_name}" + ) + + +def test_a_malformed_call_id_after_a_compaction_item_is_named_by_its_index_in_the_request() -> None: + history: Final = cast( + ResponseInputParam, + [ + {"role": "user", "content": "Plan a Tokyo trip."}, + {"role": "assistant", "content": "Day one, Asakusa."}, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + _weather_call(call_id="call_weather"), + {"type": "function_call_output", "call_id": {"id": "call_weather"}, "output": "sunny"}, + ], + ) + + with pytest.raises(litellm.BadRequestError) as raised: + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={}, only_since_compaction=True + ) + + assert "'input[4].call_id'" in raised.value.message + + +@pytest.mark.parametrize("call_id", [7, {"id": "call_weather"}, ["call_weather"]], ids=["int", "object", "list"]) +def test_a_conversion_failure_elsewhere_is_never_blamed_on_a_call_id(call_id: object) -> None: + history: Final = cast( + ResponseInputParam, + [ + _WEATHER_ASK, + {"type": ["message"], "role": "user", "content": "Go on"}, + {"type": "function_call_output", "call_id": call_id, "output": "sunny"}, + ], + ) + + with pytest.raises((TypeError, litellm.BadRequestError)) as raised: + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={} + ) + + assert "call_id" not in str(raised.value) + + +def test_a_tool_output_with_an_int_call_id_converts_like_its_string_form() -> None: + def converted(call_id: object) -> list[object]: + return list( + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=cast( + ResponseInputParam, + [_WEATHER_ASK, {"type": "function_call_output", "call_id": call_id, "output": "sunny"}], + ), + responses_api_request={}, + ) + ) + + assert converted(7) == converted("7") + + +def test_a_non_string_role_still_converts_for_a_target_that_judges_it_itself() -> None: + history: Final = [{"role": "user", "content": "Hi"}, {"role": ["user"], "content": "Go on"}] + + shared: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=cast(ResponseInputParam, history), responses_api_request={} + ) + bridged: Final = _bridge_to("gemini/gemini-3.8-flash", history) + + assert [message_field(message, "role") for message in shared] == ["user", ["user"]] + assert [message_field(message, "role") for message in bridged["messages"]] == ["user", ["user"]] + + +@pytest.mark.parametrize( + ("history", "field_path", "type_name"), + [ + pytest.param( + [{"role": "user", "content": "Hi"}, {"role": ["user"], "content": "Go on"}], + "input[1].role", + "list", + id="list_role", + ), + pytest.param( + [{"role": "user", "content": "Hi"}, {"role": {"name": "user"}, "content": "Go on"}], + "input[1].role", + "dict", + id="dict_role", + ), + pytest.param( + [{"role": "user", "content": "Hi"}, {"role": 7, "content": "Go on"}], "input[1].role", "int", id="int_role" + ), + pytest.param( + [_WEATHER_ASK, _weather_call(call_id={"id": "call_weather"})], + "input[1].call_id", + "dict", + id="dict_function_call_id", + ), + pytest.param( + [_WEATHER_ASK, _weather_call(call_id=7), {"type": "function_call_output", "call_id": "7", "output": "sunny"}], + "input[1].call_id", + "int", + id="int_function_call_id", + ), + pytest.param( + [_WEATHER_ASK, _weather_call(id=["fc_weather"])], "input[1].id", "list", id="list_function_call_item_id" + ), + pytest.param( + [ + {"role": "user", "content": "Plan a Tokyo trip."}, + {"role": "assistant", "content": "Day one, Asakusa."}, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + {"role": ["user"], "content": "And Osaka?"}, + ], + "input[3].role", + "list", + id="list_role_after_a_compaction_item", + ), + ], +) +def test_a_non_string_field_bridged_to_anthropic_is_a_400_naming_its_input_item( + history: list[dict[str, object]], field_path: str, type_name: str +) -> None: + with pytest.raises(litellm.BadRequestError) as raised: + _bridge_to(_ANTHROPIC_MODEL, history, instructions="Be terse.") + + assert raised.value.message == ( + f"litellm.BadRequestError: Invalid type for '{field_path}': expected a string, got {type_name}" + ) + + +def test_a_non_string_role_on_an_item_that_never_reads_it_still_bridges_to_anthropic() -> None: + history: Final = [ + _WEATHER_ASK, + _weather_call(call_id="call_weather", role=["x"]), + {"type": "function_call_output", "call_id": "call_weather", "output": "sunny", "role": ["x"]}, + ] + + bridged: Final = _bridge_to(_ANTHROPIC_MODEL, history) + + assert [message_field(message, "role") for message in bridged["messages"]] == ["user", "assistant", "tool"] + + +@pytest.mark.parametrize( + ("history", "field_path"), + [ + pytest.param([_WEATHER_ASK, {"role": ["user"], "content": "Go on"}], "input[2].role", id="role"), + pytest.param( + [ + _WEATHER_ASK, + _weather_call(call_id="call_weather"), + {"type": "function_call_output", "call_id": {"id": "call_weather"}, "output": "sunny"}, + ], + "input[3].call_id", + id="tool_output_call_id", + ), + ], +) +def test_a_malformed_item_after_a_codex_additional_tools_item_is_named_by_its_index_in_the_request( + history: list[dict[str, object]], field_path: str +) -> None: + tools_item: Final = { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}], + } + + with pytest.raises(litellm.BadRequestError) as raised: + litellm.responses( + model=_ANTHROPIC_MODEL, api_key="sk-ant-test", input=cast(ResponseInputParam, [tools_item, *history]) + ) + + assert f"'{field_path}'" in raised.value.message From d2321581ecf8f4353d2cdf6bc3bdf730767bf906 Mon Sep 17 00:00:00 2001 From: alexcastgonz <82875882+pcbeingused333@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:14:53 -0400 Subject: [PATCH 2/6] fix(cohere): emit streamed tool calls from v2 tool-call-start and tool-call-delta chunks (#47794) * fix(cohere): emit streamed tool calls from v2 tool-call-start and tool-call-delta chunks * fix(cohere): type _parse_tool_call_delta without growing basedpyright counts The reassigned Final local and the untyped chunk lookups added basedpyright errors past the base ceiling. Narrow them with casts; per-rule counts for the file are now at or below the merge-base. Co-Authored-By: Claude Sonnet 5.5 * fix(cohere): parse streamed tool calls with a typed model and map Cohere TOOL_CALL to tool_calls Read the streamed tool call through a frozen pydantic model instead of casts, which the strict and type-discipline gates refuse. Map Cohere's TOOL_CALL and STOP_SEQUENCE finish reasons, so a stream that reads Cohere's message-end still ends tool_calls. The regression test now drives litellm.acompletion with two parallel streamed calls and checks the rebuilt calls and the final finish reason * fix(cohere): close a streamed tool call that sent no arguments with an empty JSON object --------- Co-authored-by: Claude Sonnet 5.5 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/core_helpers.py | 2 + litellm/llms/cohere/common_utils.py | 44 +++--- litellm/types/llms/cohere.py | 13 ++ .../litellm_core_utils/test_core_helpers.py | 6 + .../cohere/chat/test_cohere_transformation.py | 125 +++++++++++++++++- 5 files changed, 170 insertions(+), 20 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index fd6b6bb4a63..58189140a98 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -315,6 +315,8 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = { "model_context_window_exceeded": "length", # Cohere "COMPLETE": "stop", + "STOP_SEQUENCE": "stop", + "TOOL_CALL": "tool_calls", "ERROR_TOXIC": "content_filter", "ERROR": "stop", # HuggingFace / Together AI diff --git a/litellm/llms/cohere/common_utils.py b/litellm/llms/cohere/common_utils.py index c964e60fac7..65531451d52 100644 --- a/litellm/llms/cohere/common_utils.py +++ b/litellm/llms/cohere/common_utils.py @@ -3,6 +3,7 @@ from typing import Final, Literal from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.cohere import CohereV2StreamedToolCall from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ( ChatCompletionToolCallChunk, @@ -222,6 +223,7 @@ class CohereV2ModelResponseIterator: self.content_blocks: list = [] self.tool_index = -1 self.json_mode = json_mode + self.tool_call_indexes_with_arguments: frozenset[int] = frozenset() def _parse_content_delta(self, chunk: dict) -> str: """Parse content-delta chunks to extract text.""" @@ -234,20 +236,23 @@ class CohereV2ModelResponseIterator: return content return "" - def _parse_tool_call_delta(self, chunk: dict) -> ChatCompletionToolCallChunk | None: - """Parse tool-call-delta chunks to extract tool calls.""" - delta: Final = chunk.get("delta", {}) - tool_calls: Final = delta.get("tool_calls", []) - if tool_calls: - return { - "id": tool_calls[0].get("id", ""), - "type": "function", - "function": { - "name": tool_calls[0].get("name", ""), - "arguments": tool_calls[0].get("arguments", ""), - }, - } - return None + def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk: + """Parse tool-call-start and tool-call-delta chunks to extract tool calls.""" + tool_call: Final = CohereV2StreamedToolCall.model_validate(chunk) + if tool_call.arguments: + self.tool_call_indexes_with_arguments = self.tool_call_indexes_with_arguments | {index} + return { + "id": tool_call.id, + "type": "function", + "function": {"name": tool_call.name, "arguments": tool_call.arguments}, + "index": index, + } + + def _parse_tool_call_end(self, index: int) -> ChatCompletionToolCallChunk | None: + """Close a tool call that streamed no arguments with an empty JSON object, as OpenAI streams one.""" + if index in self.tool_call_indexes_with_arguments: + return None + return {"id": None, "type": "function", "function": {"name": None, "arguments": "{}"}, "index": index} def _parse_tool_plan_delta(self, chunk: dict) -> dict | None: """Parse tool-plan-delta events to extract tool plan.""" @@ -301,7 +306,8 @@ class CohereV2ModelResponseIterator: v2 format: - Content: chunk.type == "content-delta" -> chunk.delta.message.content.text - - Tool calls: chunk.type == "tool-call-delta" -> chunk.delta.tool_calls + - Tool calls: chunk.type in ("tool-call-start", "tool-call-delta") -> chunk.delta.message.tool_calls + - Tool call end: chunk.type == "tool-call-end" -> "{}" arguments when the call streamed none - Tool plan: chunk.event == "tool-plan-delta" -> chunk.data.delta.message.tool_plan - Citations: chunk.event == "citation-start" -> chunk.data.delta.message.citations - Finish: chunk.event == "message-end" -> chunk.data.delta.finish_reason @@ -321,8 +327,12 @@ class CohereV2ModelResponseIterator: # Handle different chunk types if chunk_type == "content-delta": text = self._parse_content_delta(chunk) - elif chunk_type == "tool-call-delta": - tool_use = self._parse_tool_call_delta(chunk) + elif chunk_type in ("tool-call-start", "tool-call-delta", "tool-call-end"): + tool_use = ( + self._parse_tool_call_end(index) + if chunk_type == "tool-call-end" + else self._parse_tool_call_delta(chunk, index) + ) elif event_type == "tool-plan-delta": provider_specific_fields = self._parse_tool_plan_delta(chunk) elif event_type == "citation-start": diff --git a/litellm/types/llms/cohere.py b/litellm/types/llms/cohere.py index 92d1a8a573d..e48ef73b134 100644 --- a/litellm/types/llms/cohere.py +++ b/litellm/types/llms/cohere.py @@ -1,5 +1,6 @@ from typing import Literal +from pydantic import AliasPath, BaseModel, ConfigDict, Field from typing_extensions import Required, TypedDict @@ -98,3 +99,15 @@ class CohereV2ChatResponse(TypedDict): message: CohereV2ChatResponseMessage usage: CohereV2ChatResponseUsage logprobs: CohereV2ChatResponseLogProbs + + +class CohereV2StreamedToolCall(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str | None = Field(default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "id")) + name: str | None = Field( + default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "function", "name") + ) + arguments: str = Field( + default="", validation_alias=AliasPath("delta", "message", "tool_calls", "function", "arguments") + ) diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 1a92aa7c2c1..b21e9eb8b5a 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -319,6 +319,12 @@ class TestMapFinishReasonCohere: def test_error(self): assert map_finish_reason("ERROR") == "stop" + def test_tool_call(self): + assert map_finish_reason("TOOL_CALL") == "tool_calls" + + def test_stop_sequence(self): + assert map_finish_reason("STOP_SEQUENCE") == "stop" + class TestMapFinishReasonHuggingFace: def test_eos_token(self): diff --git a/tests/unit/llms/cohere/chat/test_cohere_transformation.py b/tests/unit/llms/cohere/chat/test_cohere_transformation.py index 8d39c301029..d32b3bcbeca 100644 --- a/tests/unit/llms/cohere/chat/test_cohere_transformation.py +++ b/tests/unit/llms/cohere/chat/test_cohere_transformation.py @@ -1,3 +1,4 @@ +import itertools import json from collections.abc import Callable, Iterator from typing import Final @@ -653,9 +654,7 @@ async def test_cohere_v2_annotations(respx_mock: respx.MockRouter) -> None: def test_cohere_v2_parameter_mapping(respx_mock: respx.MockRouter) -> None: messages: Final = [{"role": "user", "content": "Generate a creative story."}] route: Final = respx_mock.post(COHERE_V2_CHAT_URL).mock( - return_value=httpx.Response( - 200, json=_cohere_v2_response({"content": [{"type": "text", "text": "A story."}]}) - ) + return_value=httpx.Response(200, json=_cohere_v2_response({"content": [{"type": "text", "text": "A story."}]})) ) litellm.completion( @@ -705,3 +704,123 @@ def test_cohere_v2_error_handling(respx_mock: respx.MockRouter) -> None: assert error.value.status_code == 400 assert error.value.llm_provider == "cohere" assert "invalid request: model 'invalid-model' not found" in error.value.message + + +def _cohere_v2_tool_call_events(index: int, call_id: str, city: str) -> tuple[dict[str, object], ...]: + return ( + { + "type": "tool-call-start", + "index": index, + "delta": { + "message": { + "tool_calls": { + "id": call_id, + "type": "function", + "function": {"name": "get_weather", "arguments": ""}, + } + } + }, + }, + { + "type": "tool-call-delta", + "index": index, + "delta": {"message": {"tool_calls": {"function": {"arguments": '{"city": '}}}}, + }, + { + "type": "tool-call-delta", + "index": index, + "delta": {"message": {"tool_calls": {"function": {"arguments": json.dumps(city) + "}"}}}}, + }, + {"type": "tool-call-end", "index": index}, + ) + + +@pytest.mark.usefixtures("_cohere_httpx_transport") +@pytest.mark.respx(assert_all_called=True) +async def test_cohere_v2_stream_returns_parallel_tool_calls(respx_mock: respx.MockRouter) -> None: + stream_events: Final = ( + {"type": "message-start", "id": "c0a2b6e1", "delta": {"message": {"role": "assistant"}}}, + *_cohere_v2_tool_call_events(0, "get_weather_paris", "Paris"), + *_cohere_v2_tool_call_events(1, "get_weather_tokyo", "Tokyo"), + { + "type": "message-end", + "delta": {"finish_reason": "TOOL_CALL", "usage": {"tokens": {"input_tokens": 40, "output_tokens": 30}}}, + }, + ) + respx_mock.post(COHERE_V2_CHAT_URL).mock( + return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode()) + ) + + response: Final = await litellm.acompletion( + model="cohere_chat/v2/command-a-plus-05-2026", + messages=[{"role": "user", "content": "What is the weather in Paris and in Tokyo?"}], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + } + ], + stream=True, + api_key="cohere-test-key", + ) + chunks: Final = [chunk async for chunk in response] + rebuilt: Final = litellm.stream_chunk_builder(chunks) + + assert rebuilt is not None + assert [ + (call.id, call.function.name, json.loads(call.function.arguments)) + for call in rebuilt.choices[0].message.tool_calls + ] == [ + ("get_weather_paris", "get_weather", {"city": "Paris"}), + ("get_weather_tokyo", "get_weather", {"city": "Tokyo"}), + ] + assert chunks[-1].choices[0].finish_reason == "tool_calls" + + +@pytest.mark.usefixtures("_cohere_httpx_transport") +@pytest.mark.respx(assert_all_called=True) +async def test_cohere_v2_stream_closes_argument_less_tool_call_with_empty_object(respx_mock: respx.MockRouter) -> None: + stream_events: Final = ( + {"type": "message-start", "id": "5d1e9f20", "delta": {"message": {"role": "assistant"}}}, + { + "type": "tool-call-start", + "index": 0, + "delta": { + "message": { + "tool_calls": { + "id": "get_server_time_0", + "type": "function", + "function": {"name": "get_server_time", "arguments": ""}, + } + } + }, + }, + {"type": "tool-call-end", "index": 0}, + {"type": "message-end", "delta": {"finish_reason": "TOOL_CALL"}}, + ) + respx_mock.post(COHERE_V2_CHAT_URL).mock( + return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode()) + ) + + response: Final = await litellm.acompletion( + model="cohere_chat/v2/command-a-plus-05-2026", + messages=[{"role": "user", "content": "What time is it on the server?"}], + tools=[ + { + "type": "function", + "function": {"name": "get_server_time", "parameters": {"type": "object", "properties": {}}}, + } + ], + stream=True, + api_key="cohere-test-key", + ) + deltas_per_chunk: Final = [chunk.choices[0].delta.tool_calls or () async for chunk in response] + streamed_calls: Final = tuple(itertools.chain.from_iterable(deltas_per_chunk)) + + assert [(call.id, call.function.name) for call in streamed_calls if call.id] == [ + ("get_server_time_0", "get_server_time") + ] + assert json.loads("".join(call.function.arguments for call in streamed_calls)) == {} From 580b9266a11d8db9646a6151c6f04e4f6e2f5d13 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 31 Oct 2026 14:43:24 -0700 Subject: [PATCH 3/6] feat(chat): resume replies after browser disconnects (#49461) * feat(chat): resume background MCP runs after browser disconnects * feat(chat): background ordinary text conversations * feat(chat): configure background replies from Settings * fix(chat): address background response review findings * test(sdk): finish the HTTP smoke stream explicitly * fix(chat): preserve background response timing metrics * test(router): terminate the scripted completion stream --- litellm/proxy/proxy_server.py | 51 ++-- .../response_polling/background_streaming.py | 36 ++- .../proxy/response_polling/polling_handler.py | 18 +- .../proxy_setting_endpoints.py | 95 ++++++++ .../model_management_endpoints.py | 3 + tests/base_sdk_tests/check_sdk_http.py | 4 +- .../test_response_polling_handler.py | 68 ++++++ tests/unit/proxy/test_proxy_server.py | 85 +++++++ .../test_proxy_setting_endpoints.py | 111 +++++++++ tests/unit/test_router/core/test_endpoints.py | 1 + ui/litellm-dashboard/eslint-suppressions.json | 5 - .../chat/page.background.integration.test.tsx | 229 ++++++++++++++++++ .../src/app/chat/page.integration.test.tsx | 2 +- ui/litellm-dashboard/src/app/chat/page.tsx | 87 ++++++- ...ackgroundChatSettings.integration.test.tsx | 98 ++++++++ .../UISettings/BackgroundChatSettings.tsx | 73 ++++++ .../UISettings/UISettings.test.tsx | 2 + .../AdminSettings/UISettings/UISettings.tsx | 5 +- .../src/components/chat/ChatShell.tsx | 3 - .../components/chat/backgroundChat.test.ts | 177 ++++++++++++++ .../src/components/chat/backgroundChat.ts | 138 +++++++++++ .../src/components/chat/runChatRequest.ts | 11 + .../src/components/chat/types.ts | 9 + .../src/components/chat/useBackgroundChat.ts | 62 +++++ .../src/components/llm_calls/fetch_models.tsx | 3 + .../components/llm_calls/responses_api.tsx | 17 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 130 ++++++++++ 27 files changed, 1480 insertions(+), 43 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx create mode 100644 ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts create mode 100644 ui/litellm-dashboard/src/components/chat/backgroundChat.ts create mode 100644 ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b5f4a36f138..d8a57d3e9a1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3070,6 +3070,29 @@ redis_usage_cache: RedisCache | None = None # redis cache used for tracking spe polling_via_cache_enabled: Literal["all"] | list[str] | bool = False native_background_mode: list[str] = [] # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache + + +class _ResponseBackgroundSettings(BaseModel): + model_config = ConfigDict(frozen=True) + + polling_via_cache: Literal["all"] | tuple[str, ...] | bool = False + native_background_mode: tuple[str, ...] = () + ttl: int = 3600 + + +def _apply_background_response_settings(value: SettingsJsonValue) -> None: + global polling_via_cache_enabled, native_background_mode, polling_cache_ttl + + responses: Final = _as_settings_mapping(value) + settings: Final = _ResponseBackgroundSettings.model_validate(responses.get("background_mode") or {}) + polling: Final = settings.polling_via_cache + polling_via_cache_enabled, native_background_mode, polling_cache_ttl = ( # rebind-ok: Refresh runtime config. + list(polling) if isinstance(polling, tuple) else polling, + list(settings.native_background_mode), + settings.ttl, + ) + + user_custom_auth = None user_custom_key_generate = None # Sentinel: prevents PKCE-no-Redis advisory from re-logging on config hot-reload. @@ -6932,20 +6955,7 @@ class ProxyConfig: # see usage here: https://docs.litellm.ai/docs/proxy/caching pass elif key == "responses": - # Initialize global polling via cache settings - global polling_via_cache_enabled, native_background_mode, polling_cache_ttl - background_mode = value.get("background_mode", {}) - polling_via_cache_enabled = background_mode.get("polling_via_cache", False) - native_background_mode = background_mode.get("native_background_mode", []) - polling_cache_ttl = background_mode.get("ttl", 3600) - verbose_proxy_logger.debug( - "%s Initialized polling via cache: enabled=%s, native_background_mode=%s, ttl=%s%s", - blue_color_code, - polling_via_cache_enabled, - native_background_mode, - polling_cache_ttl, - reset_color_code, - ) + _apply_background_response_settings(value) elif key == "max_ui_session_budget": litellm.max_ui_session_budget = float(value) if value is not None else None verbose_proxy_logger.debug( @@ -8653,6 +8663,7 @@ class ProxyConfig: def _apply_litellm_settings_db_values(self, db_values: Mapping[str, SettingsJsonValue]) -> None: self.litellm_settings.apply_db_row("litellm_settings", db_values) + _apply_background_response_settings(self.litellm_settings.get("responses", {})) for key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES: if key in db_values and (value := self.litellm_settings.get(key)) is not None: setattr(litellm, key, value) @@ -18110,6 +18121,8 @@ async def model_deprecations( def get_model_group_info( llm_router: Router, all_models_str: Sequence[str], model_group: str | None, team_id: str | None = None ) -> list[ModelGroupInfoProxy]: + from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request + model_groups: Final[list[ModelGroupInfoProxy]] = [] unique_models: Final = tuple(dict.fromkeys(all_models_str)) @@ -18119,6 +18132,14 @@ def get_model_group_info( if model_group is not None and model_group != model: continue + supports_background_chat = should_use_polling_for_request( + background_mode=True, + polling_via_cache_enabled=polling_via_cache_enabled, + redis_cache=redis_usage_cache, + model=model, + llm_router=llm_router, + native_background_mode=native_background_mode, + ) _model_group_info = registry.group(model, team_id=team_id) if _model_group_info is not None: @@ -18126,6 +18147,7 @@ def get_model_group_info( ModelGroupInfoProxy.model_validate( { **_model_group_info.model_dump(), + "supports_background_chat": supports_background_chat, **model_group_custom_fields( registry.deployments( resolve_model_group_alias(registry.aliases(), model) or model, @@ -18140,6 +18162,7 @@ def get_model_group_info( model_group_info = ModelGroupInfoProxy( model_group=model, providers=[], + supports_background_chat=supports_background_chat, ) model_groups.append(model_group_info) diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 1eb76f8f558..574152048d4 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming import asyncio import json from collections.abc import AsyncIterable, Callable, Mapping, Sequence +from time import perf_counter from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable from fastapi import Request, Response @@ -128,6 +129,16 @@ async def background_streaming_task( https://platform.openai.com/docs/api-reference/responses/object """ + started_at: Final = perf_counter() + first_token_at: float | None = None # rebind-ok: first content event closes the TTFT interval + finished_at: float | None = None # rebind-ok: terminal event closes the execution interval + + def timing_snapshot() -> Mapping[str, float]: + return { + **({"time_to_first_token_ms": (first_token_at - started_at) * 1000} if first_token_at is not None else {}), + **({"total_latency_ms": (finished_at - started_at) * 1000} if finished_at is not None else {}), + } + try: verbose_proxy_logger.info("Starting background streaming for %s", polling_id) @@ -191,7 +202,7 @@ async def background_streaming_task( incomplete_details_data = None state_dirty = False # Track if state needs to be synced - last_update_time = asyncio.get_event_loop().time() + last_update_time = perf_counter() UPDATE_INTERVAL: Final = 0.150 # 150ms batching interval # Track the terminal event from the stream (may not be "completed") @@ -210,13 +221,14 @@ async def background_streaming_task( """Flush accumulated state to Redis if interval elapsed or forced""" nonlocal state_dirty, last_update_time - current_time: Final = asyncio.get_event_loop().time() + current_time: Final = perf_counter() if state_dirty and (force or (current_time - last_update_time) >= UPDATE_INTERVAL): # Convert output_items dict to list for update output_list: Final = list(output_items.values()) await polling_handler.update_state( polling_id=polling_id, output=output_list, + timing=timing_snapshot(), ) state_dirty = False last_update_time = current_time @@ -242,6 +254,20 @@ async def background_streaming_task( event: _StreamEvent = _StreamEventParser.parse(chunk_data) event_type = event.get("type", "") + if ( + first_token_at is None + and event_type + in ( + "response.output_text.delta", + "response.reasoning_text.delta", + "response.reasoning_summary_text.delta", + "response.reasoning.delta", + ) + and event.get("delta") + ): + first_token_at = perf_counter() + state_dirty = True + # Process different event types based on OpenAI streaming spec if event_type == "response.output_item.added": # New output item added @@ -334,6 +360,7 @@ async def background_streaming_task( ): # Terminal event - extract all ResponsesAPIResponse fields # https://platform.openai.com/docs/api-reference/responses-streaming + finished_at = perf_counter() response_data = event.get("response", {}) terminal_status = response_data.get( "status", @@ -384,10 +411,13 @@ async def background_streaming_task( # Use the terminal status from the stream, default to "completed" final_status: Final = terminal_status or "completed" + if finished_at is None: + finished_at = perf_counter() # rebind-ok: close the interval when the stream has no terminal event await polling_handler.update_state( polling_id=polling_id, status=final_status, + timing=timing_snapshot(), usage=usage_data, error=terminal_error, reasoning=reasoning_data, @@ -417,6 +447,7 @@ async def background_streaming_task( ) except Exception as e: + finished_at = perf_counter() # rebind-ok: capture execution time at failure verbose_proxy_logger.error("Error in background streaming task for %s: %s", polling_id, e) import traceback @@ -425,6 +456,7 @@ async def background_streaming_task( await polling_handler.update_state( polling_id=polling_id, status="failed", + timing=timing_snapshot(), error={ "type": "internal_error", "message": str(e), diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index fe224a52cfa..d66773cd048 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -3,9 +3,9 @@ Response Polling Handler for Background Responses with Cache """ import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime, timezone -from typing import Final +from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter @@ -15,6 +15,9 @@ from litellm._uuid import uuid4 from litellm.caching.redis_cache import RedisCache from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStatus +if TYPE_CHECKING: + from litellm.router import Router + _RESPONSE_POLLING_TARGET: Final = "response_polling" _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _JSON_LIST: Final = TypeAdapter(list[object]) @@ -116,6 +119,7 @@ class ResponsePollingHandler: parallel_tool_calls: bool | None = None, user: str | None = None, store: bool | None = None, + timing: Mapping[str, float] | None = None, ) -> None: """ Update the polling state in Redis @@ -144,6 +148,7 @@ class ResponsePollingHandler: parallel_tool_calls: Whether parallel tool calls are enabled user: User identifier store: Whether to store the response + timing: Server execution timings in milliseconds """ if not self.redis_cache: return @@ -167,6 +172,9 @@ class ResponsePollingHandler: if output is not None: state["output"] = output + if timing is not None: + state["litellm_response_timing"] = timing + # Update usage if usage: state["usage"] = usage @@ -264,10 +272,10 @@ class ResponsePollingHandler: def should_use_polling_for_request( background_mode: bool, - polling_via_cache_enabled, # Can be False, "all", or List[str] - redis_cache, # RedisCache or None + polling_via_cache_enabled: bool | str | Sequence[str], + redis_cache: RedisCache | None, model: str, - llm_router, # Router instance or None + llm_router: "Router | None", native_background_mode: list[str] | None = None, # List of models that should use native background mode ) -> bool: """ diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 0efd4e4c619..e5348684d78 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -216,6 +216,20 @@ class _SettingsWithSchema(LiteLLMBaseModel): field_schema: dict[str, object] +class BackgroundChatSettings(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + enabled: bool = Field(description="Enable cache-backed background Responses for Chat and API requests.") + + +class BackgroundChatSettingsResponse(BackgroundChatSettings): + model_config = ConfigDict(extra="forbid", frozen=True) + + redis_configured: bool = Field(description="Whether this proxy has coordination Redis configured.") + managed_by_config: bool = Field(description="Whether the responses configuration is owned by the config file.") + can_save: bool = Field(description="Whether database-backed configuration updates are available.") + + class SSOSettingsResponse(SettingsResponse): """Response model for SSO settings""" @@ -1815,6 +1829,87 @@ async def sync_ui_settings_to_general_settings( return apply_runtime_general_settings_flags(parsed) +def _background_chat_settings_response(config: Mapping[str, object]) -> BackgroundChatSettingsResponse: + from litellm.proxy import proxy_server + + settings: Final = _as_settings_section(config.get("litellm_settings")) + responses: Final = _as_settings_section(settings.get("responses")) + background: Final = _as_settings_section(responses.get("background_mode")) + polling: Final = background.get("polling_via_cache", False) + return BackgroundChatSettingsResponse( + enabled=bool(polling), + redis_configured=proxy_server.redis_usage_cache is not None, + managed_by_config=proxy_server.proxy_config.litellm_settings.owned_by_config("responses"), + can_save=proxy_server.prisma_client is not None and proxy_server.store_model_in_db is True, + ) + + +@router.get( + "/get/background_chat_settings", + tags=["Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=BackgroundChatSettingsResponse, +) +async def get_background_chat_settings() -> BackgroundChatSettingsResponse: + """Get saved background-response settings and the prerequisites for changing them.""" + from litellm.proxy.proxy_server import proxy_config + + config: Final = _as_settings_section(_DECODED_JSON.validate_python(await proxy_config.get_config())) + return _background_chat_settings_response(config) + + +@router.patch( + "/update/background_chat_settings", + tags=["Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=BackgroundChatSettingsResponse, +) +async def update_background_chat_settings( + settings: BackgroundChatSettings, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> BackgroundChatSettingsResponse: + """Save background-response polling settings for the proxy configuration refresh.""" + from litellm.proxy.proxy_server import ( + create_config_audit_log, + prisma_client, + proxy_config, + redis_usage_cache, + store_model_in_db, + ) + + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException(status_code=403, detail="Only proxy admins can update background chat settings.") + if prisma_client is None or store_model_in_db is not True: + raise HTTPException(status_code=400, detail="Enable database-backed configuration to save this setting.") + if settings.enabled and redis_usage_cache is None: + raise HTTPException(status_code=400, detail="Configure coordination Redis and restart the proxy first.") + + raw_config: Final = await proxy_config.get_config() + config: Final = _as_settings_section(_DECODED_JSON.validate_python(raw_config)) + litellm_settings: Final = _as_settings_section(config.get("litellm_settings")) + responses: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(litellm_settings.get("responses"))) + background: Final = _UI_SETTINGS_OBJECT.validate_python(_as_settings_section(responses.get("background_mode"))) + updated_responses: Final[Mapping[str, JsonValue]] = { + **responses, + "background_mode": {**background, "polling_via_cache": "all" if settings.enabled else False}, + } + proxy_config.reject_config_owned_writes( + section_name="litellm_settings", changed_keys={"responses": dict(updated_responses)} + ) + raw_config["litellm_settings"] = {**litellm_settings, "responses": dict(updated_responses)} + await proxy_config.save_config(new_config=config) + asyncio.create_task( + create_config_audit_log( + param_name="responses.background_mode.polling_via_cache", + action="updated", + before_value=background.get("polling_via_cache", False), + after_value="all" if settings.enabled else False, + user_api_key_dict=user_api_key_dict, + ) + ) + return _background_chat_settings_response(config) + + @router.get( "/get/ui_settings", tags=["UI Settings"], diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py index f032bb20c2a..b12745679df 100644 --- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py @@ -35,6 +35,9 @@ class PatchModelRequest(updateDeployment): class ModelGroupInfoProxy(ModelGroupInfo): + supports_background_chat: bool = Field( + default=False, description="Whether cache-backed background Responses runs are enabled for this model group" + ) is_public_model_group: bool = Field(default=False) health_status: str | None = Field(default=None) health_response_time: float | None = Field(default=None) diff --git a/tests/base_sdk_tests/check_sdk_http.py b/tests/base_sdk_tests/check_sdk_http.py index f7a1e30ac99..16ee4824f2d 100644 --- a/tests/base_sdk_tests/check_sdk_http.py +++ b/tests/base_sdk_tests/check_sdk_http.py @@ -70,7 +70,9 @@ def enqueue(body: object, status: int = 200) -> None: def enqueue_stream() -> None: chunk: Final = {**CHAT, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "pong"}}]} - RESPONSES.put((200, f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode())) + terminal: Final = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]} + stream: Final = f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(terminal)}\n\ndata: [DONE]\n\n" + RESPONSES.put((200, stream.encode())) def bedrock_event(event: str, payload: object) -> bytes: diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 087b273548a..d6df7a6c69a 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1917,3 +1917,71 @@ async def test_create_initial_state_rejects_request_metadata_that_is_not_an_obje assert [(error["loc"], error["type"], repr(error["input"])) for error in rejected.value.errors()] == [ (("metadata",), "dict_type", repr(metadata)) ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "delta_type", + [ + "response.output_text.delta", + "response.reasoning_text.delta", + "response.reasoning_summary_text.delta", + "response.reasoning.delta", + None, + ], +) +async def test_background_timings_survive_retrieval_after_completion(delta_type): + from litellm.caching.redis_cache import RedisCache + from litellm.proxy.response_polling.background_streaming import background_streaming_task + + cache = {} + redis = AsyncMock(spec=RedisCache) + redis.async_get_cache.side_effect = lambda key: cache.get(key) + + async def save_cache(key, value, **kwargs): + cache[key] = value + + redis.async_set_cache.side_effect = save_cache + handler = ResponsePollingHandler(redis_cache=redis) + await handler.create_initial_state("litellm_poll_timing", {"model": "gpt-4o"}) + clock = [100.0] + + async def stream(): + clock[0] = 101.0 + yield 'data: {"type":"response.output_text.delta","delta":""}\n\n' + yield 'data: {"type":"response.mcp_call.in_progress"}\n\n' + if delta_type: + clock[0] = 102.0 + yield f"data: {json.dumps({'type': delta_type, 'delta': 'Hello'})}\n\n" + partial = await handler.get_state("litellm_poll_timing") + assert partial is not None + assert partial["status"] == "in_progress" + assert partial["litellm_response_timing"] == {"time_to_first_token_ms": 2000.0} + clock[0] = 110.0 + yield 'data: {"type":"response.completed","response":{"status":"completed","output":[]}}\n\n' + clock[0] = 5000.0 + yield "data: [DONE]\n\n" + + from starlette.responses import StreamingResponse + + with ( + patch( + "litellm.proxy.response_polling.background_streaming.perf_counter", + create=True, + side_effect=lambda: clock[0], + ), + patch( + # test-quality-ok: processor is constructed inside the task; supply the upstream SSE boundary like sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as processor, + ): + processor.return_value.base_process_llm_request = AsyncMock(return_value=StreamingResponse(stream())) + await background_streaming_task(**_make_background_streaming_kwargs("litellm_poll_timing", handler)) + state = await handler.get_state("litellm_poll_timing") + assert state is not None + expected = {"total_latency_ms": 10000.0} + if delta_type: + expected["time_to_first_token_ms"] = 2000.0 + assert state["litellm_response_timing"] == expected + clock[0] = 10000.0 + assert (await handler.get_state("litellm_poll_timing"))["litellm_response_timing"] == expected diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index 6e494f0b24b..e4af36d99f2 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -4561,3 +4561,88 @@ async def test_text_completion_provider_rejection_captures_unknown_usage( assert envelope["usage"] is None assert envelope["usage_source"] == "unknown" assert envelope["pricing"]["reported_cost"] is None + + +@pytest.mark.parametrize( + "polling,has_redis,native,expected", + [ + (False, True, False, False), + ("all", False, False, False), + ("all", True, True, False), + ("all", True, False, True), + (["openai"], True, False, True), + (["anthropic"], True, False, False), + ], +) +def test_model_groups_advertise_cache_backed_background_chat( + monkeypatch: pytest.MonkeyPatch, + polling: bool | str | list[str], + has_redis: bool, + native: bool, + expected: bool, +) -> None: + from litellm.caching.redis_cache import RedisCache + from litellm.proxy import proxy_server + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "research-chat", + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "fake-test-key"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", polling) + monkeypatch.setattr( + proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379) if has_redis else None + ) + monkeypatch.setattr(proxy_server, "native_background_mode", ["research-chat"] if native else []) + groups: Final = proxy_server.get_model_group_info(router, ["research-chat"], None) + assert groups[0].supports_background_chat is expected + + +@pytest.mark.parametrize("config_owned", [False, True]) +def test_background_response_settings_refresh_and_respect_config( + monkeypatch: pytest.MonkeyPatch, + config_owned: bool, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False) + monkeypatch.setattr(proxy_server, "native_background_mode", []) + monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600) + config: Final = proxy_server.ProxyConfig() + if config_owned: + config.litellm_settings.load_yaml( + { + "responses": { + "background_mode": { + "polling_via_cache": ["openai"], + "native_background_mode": ["config-native"], + "ttl": 900, + } + }, + } + ) + config._apply_litellm_settings_db_values( + { + "responses": { + "background_mode": { + "polling_via_cache": "all", + "native_background_mode": ["db-native"], + "ttl": 7200, + } + }, + } + ) + assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else "all") + assert proxy_server.native_background_mode == (["config-native"] if config_owned else ["db-native"]) + assert proxy_server.polling_cache_ttl == (900 if config_owned else 7200) + config._apply_litellm_settings_db_values( + { + "responses": {"background_mode": {"polling_via_cache": False}}, + } + ) + assert proxy_server.polling_via_cache_enabled == (["openai"] if config_owned else False) + assert proxy_server.native_background_mode == (["config-native"] if config_owned else []) + assert proxy_server.polling_cache_ttl == (900 if config_owned else 3600) diff --git a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 4360655c8ca..ea39d5d59e2 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -4488,3 +4488,114 @@ def test_usage_default_range_reports_config_precedence(mock_auth, monkeypatch: p assert response.status_code == 200, response.text assert response.json()["values"]["usage_default_time_range"] == "month_to_date" assert response.json()["source"]["usage_default_time_range"] == "config" + + +@pytest.fixture +def background_chat_config(monkeypatch: pytest.MonkeyPatch): + from copy import deepcopy + from unittest.mock import AsyncMock, MagicMock + + from litellm.caching.redis_cache import RedisCache + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.config_resolvers import SettingsStore + + config = { + "litellm_settings": { + "callbacks": ["prometheus"], + "responses": { + "background_mode": { + "polling_via_cache": False, + "ttl": 7200, + "native_background_mode": ["native-model"], + }, + }, + } + } + store = SettingsStore("litellm_settings") + monkeypatch.setattr(proxy_server.proxy_config, "litellm_settings", store) + monkeypatch.setattr(proxy_server.proxy_config, "_settings_stores", {"litellm_settings": store}) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "redis_usage_cache", RedisCache(host="localhost", port=6379)) + monkeypatch.setattr(proxy_server, "polling_via_cache_enabled", False) + monkeypatch.setattr(proxy_server, "native_background_mode", []) + monkeypatch.setattr(proxy_server, "polling_cache_ttl", 3600) + monkeypatch.setattr(proxy_server.proxy_config, "get_config", AsyncMock(side_effect=lambda: deepcopy(config))) + + async def save_config(*, new_config): + config.clear() + config.update(deepcopy(new_config)) + + save = AsyncMock(side_effect=save_config) + audit = AsyncMock() + monkeypatch.setattr(proxy_server.proxy_config, "save_config", save) + monkeypatch.setattr(proxy_server, "create_config_audit_log", audit) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + yield config, save, audit + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_background_chat_settings_persist_preserving_existing_options(background_chat_config, monkeypatch): + from litellm.proxy import proxy_server + + config, _, _ = background_chat_config + response = client.patch("/update/background_chat_settings", json={"enabled": True}) + assert response.status_code == 200 + assert response.json()["enabled"] is True + assert proxy_server.polling_via_cache_enabled is False + assert config["litellm_settings"]["responses"]["background_mode"] == { + "polling_via_cache": "all", + "ttl": 7200, + "native_background_mode": ["native-model"], + } + assert config["litellm_settings"]["callbacks"] == ["prometheus"] + assert client.get("/get/background_chat_settings").json()["enabled"] is True + proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"]) + assert proxy_server.polling_via_cache_enabled == "all" + disabled = client.patch("/update/background_chat_settings", json={"enabled": False}) + assert disabled.status_code == 200 + assert disabled.json()["enabled"] is False + proxy_server.proxy_config._apply_litellm_settings_db_values(config["litellm_settings"]) + assert proxy_server.polling_via_cache_enabled is False + + +@pytest.mark.parametrize("blocker", ["role", "redis", "config", "persistence"]) +def test_background_chat_settings_reject_unavailable_updates(background_chat_config, monkeypatch, blocker): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + config, save, _ = background_chat_config + if blocker == "role": + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="viewer", user_role=LitellmUserRoles.INTERNAL_USER + ) + elif blocker == "redis": + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + assert client.get("/get/background_chat_settings").json()["redis_configured"] is False + elif blocker == "config": + proxy_server.proxy_config.litellm_settings.load_yaml(config["litellm_settings"]) + assert client.get("/get/background_chat_settings").json()["managed_by_config"] is True + else: + monkeypatch.setattr(proxy_server, "store_model_in_db", False) + assert client.get("/get/background_chat_settings").json()["can_save"] is False + response = client.patch("/update/background_chat_settings", json={"enabled": True}) + assert response.status_code == (403 if blocker == "role" else 400) + save.assert_not_awaited() + assert config["litellm_settings"]["responses"]["background_mode"]["polling_via_cache"] is False + + +def test_background_chat_settings_failed_save_does_not_enable_runtime(background_chat_config): + from litellm.proxy import proxy_server + + _, save, audit = background_chat_config + save.side_effect = RuntimeError("Database write failed") + with pytest.raises(RuntimeError, match="Database write failed"): + client.patch("/update/background_chat_settings", json={"enabled": True}) + assert client.get("/get/background_chat_settings").json()["enabled"] is False + assert proxy_server.polling_via_cache_enabled is False + audit.assert_not_awaited() diff --git a/tests/unit/test_router/core/test_endpoints.py b/tests/unit/test_router/core/test_endpoints.py index a3cffe8278a..b2ad602c4c6 100644 --- a/tests/unit/test_router/core/test_endpoints.py +++ b/tests/unit/test_router/core/test_endpoints.py @@ -414,6 +414,7 @@ def _scripted_stream(*texts: str) -> Callable[..., CustomStreamWrapper]: async def chunks() -> AsyncIterator[ModelResponseStream]: for text in texts: yield ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}]) + yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}]) logging_obj: Final = LiteLLMLogging( model="d1", diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index c14f3e6f036..a4e0d59d73d 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1142,11 +1142,6 @@ "count": 1 } }, - "src/app/chat/page.tsx": { - "no-nested-ternary": { - "count": 1 - } - }, "src/app/login/LoginPage.tsx": { "local/no-complex-jsx-arrow": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx new file mode 100644 index 00000000000..53be87ce30c --- /dev/null +++ b/ui/litellm-dashboard/src/app/chat/page.background.integration.test.tsx @@ -0,0 +1,229 @@ +import { act, fireEvent, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { ChatShellProvider } from "@/contexts/ChatShellContext"; +import type { Conversation } from "@/components/chat/types"; +import ChatConversationPage from "./page"; + +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn(), replace: vi.fn() }) })); + +const storageKey = "litellm_chat_history_v1:background-user"; +const requests: Record[] = []; +let supported: boolean; +let status: "in_progress" | "completed" | "cancelled"; +let retrievalStatus: number; +let cancellations: number; +let acceptance: Promise | undefined; +const saved = (): Conversation[] => JSON.parse(localStorage.getItem(storageKey) ?? "[]"); +const result = () => ({ + id: "litellm_poll_research", + status, + usage: status === "completed" ? { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost: "0.125" } : undefined, + litellm_response_timing: + status === "completed" ? { time_to_first_token_ms: 250, total_latency_ms: 12500 } : undefined, + output: + status === "completed" + ? [ + { + type: "mcp_call", + id: "call_inventory", + name: "lookup_inventory", + arguments: "{}", + output: "12 widgets", + server_label: "inventory", + }, + { type: "message", content: [{ type: "output_text", text: "There are 12 widgets in stock." }] }, + ].filter((item) => item.type !== "mcp_call" || Boolean(requests.at(-1)?.tools)) + : [], +}); + +function renderChat(id?: string) { + return renderWithProviders( + + + , + { searchParams: id ? { id } : {} }, + ); +} + +beforeEach(() => { + localStorage.clear(); + testQueryClient.clear(); + requests.length = 0; + cancellations = 0; + acceptance = undefined; + supported = true; + retrievalStatus = 200; + status = "in_progress"; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const request = + input instanceof Request ? input : new Request(new URL(String(input), window.location.origin), init); + const path = new URL(request.url).pathname; + if (path.endsWith("/model_group/info")) + return Response.json({ + data: [{ model_group: "research-chat", mode: "chat", supports_background_chat: supported }], + }); + if (path.endsWith("/v1/mcp/server")) return Response.json([{ server_id: "inventory", server_name: "inventory" }]); + if (path.endsWith("/mcp-rest/tools/list")) return Response.json({ tools: [{ name: "lookup_inventory" }] }); + if (path.endsWith("/responses")) { + const body = JSON.parse(await request.text()); + requests.push(body); + if (body.background) { + await acceptance; + request.signal.throwIfAborted(); + return Response.json({ ...result(), status: "queued" }); + } + return new Response( + 'data: {"type":"response.output_text.delta","delta":"Foreground answer"}\n\ndata: {"type":"response.completed","response":{"id":"resp_foreground","output":[]}}\n\n', + { headers: { "Content-Type": "text/event-stream" } }, + ); + } + if (path.endsWith("/litellm_poll_research/cancel")) { + cancellations++; + status = "cancelled"; + return Response.json(result()); + } + if (path.endsWith("/litellm_poll_research")) + return Response.json(retrievalStatus === 200 ? result() : { error: { message: "Unavailable" } }, { + status: retrievalStatus, + }); + return Response.json({}, { status: 404 }); + }), + ); +}); +afterEach(() => vi.unstubAllGlobals()); + +async function startResearch(withTools = true) { + const user = userEvent.setup(); + await screen.findByRole("button", { name: "research-chat" }); + if (withTools) { + await user.click(screen.getByRole("button", { name: "Connect tools" })); + await user.click(await screen.findByRole("switch")); + await user.keyboard("{Escape}"); + } + fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), { + target: { value: "Check inventory" }, + }); + await user.click(screen.getByRole("button", { name: "Send", exact: true })); +} + +it("should retrieve the completed tool run after closing and reopening without submitting it again", async () => { + const first = renderChat(); + await startResearch(); + await screen.findByText("Running in the background. You can close this tab and return to this chat."); + await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research")); + const id = saved()[0].id; + first.unmount(); + expect(cancellations).toBe(0); + status = "completed"; + renderChat(id); + expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible(); + expect(screen.getByRole("button", { name: "lookup_inventory" })).toBeVisible(); + expect(screen.getByLabelText(/^Cost:/)).toHaveTextContent("$0.125"); + expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s"); + expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s"); + await waitFor(() => + expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }), + ); + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + background: true, + stream: false, + tools: [{ type: "mcp", server_label: "inventory" }], + }); + expect(screen.queryByRole("button", { name: "Stop request" })).not.toBeInTheDocument(); + fireEvent.change(screen.getByRole("textbox", { name: "Message", exact: true }), { + target: { value: "What was the result?" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Send", exact: true })); + await waitFor(() => expect(requests).toHaveLength(2)); + expect(requests[1]).toMatchObject({ background: true, stream: false }); + expect(requests[1]).not.toHaveProperty("previous_response_id"); + expect(requests[1].input).toEqual( + expect.arrayContaining([{ type: "function_call_output", call_id: "call_inventory", output: "12 widgets" }]), + ); +}); + +it("should continue an ordinary chat without MCP tools after closing and reopening", async () => { + const first = renderChat(); + await startResearch(false); + await screen.findByText("Running in the background. You can close this tab and return to this chat."); + await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research")); + expect(requests[0]).toMatchObject({ background: true, stream: false }); + expect(requests[0]).not.toHaveProperty("tools"); + const id = saved()[0].id; + first.unmount(); + status = "completed"; + renderChat(id); + expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible(); + expect(screen.getByLabelText(/^TTFT:/)).toHaveTextContent("0.25s"); + expect(screen.getByLabelText(/^Total Latency:/)).toHaveTextContent("12.50s"); + await waitFor(() => + expect(saved()[0]?.messages.at(-1)).toMatchObject({ timeToFirstToken: 250, totalLatency: 12500 }), + ); + expect(requests).toHaveLength(1); + expect(cancellations).toBe(0); +}); + +it("should cancel a background run only when Stop is clicked", async () => { + renderChat(); + await startResearch(); + await screen.findByText("Running in the background. You can close this tab and return to this chat."); + fireEvent.click(screen.getByRole("button", { name: "Stop request" })); + expect(await screen.findByText("Cancellation requested")).toBeVisible(); + expect(cancellations).toBe(1); +}); + +it("should retain ordinary streaming when background support is unavailable", async () => { + supported = false; + renderChat(); + await startResearch(); + expect(await screen.findByText("Foreground answer")).toBeVisible(); + expect(requests[0]).toMatchObject({ stream: true }); + expect(requests[0]).not.toHaveProperty("background"); + expect(screen.getByText("Keep this tab open. Background runs are not available for this chat.")).toBeVisible(); +}); + +it("should let the user reconnect after an authentication failure without resubmitting work", async () => { + const first = renderChat(); + await startResearch(); + await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.id).toBe("litellm_poll_research")); + const id = saved()[0].id; + first.unmount(); + retrievalStatus = 401; + renderChat(id); + await screen.findByRole("button", { name: "Reconnect" }); + retrievalStatus = 200; + status = "completed"; + fireEvent.click(screen.getByRole("button", { name: "Reconnect" })); + expect(await screen.findByText("There are 12 widgets in stock.")).toBeVisible(); + expect(requests).toHaveLength(1); +}); + +it.each([false, true])( + "should honor Stop before a background submission returns its ID (tools=%s)", + async (withTools) => { + let accept!: () => void; + acceptance = new Promise((resolve) => { + accept = resolve; + }); + renderChat(); + await startResearch(withTools); + await waitFor(() => expect(requests).toHaveLength(1)); + fireEvent.click(screen.getByRole("button", { name: "Stop request" })); + await act(async () => accept()); + expect(await screen.findByText("Cancellation requested")).toBeVisible(); + expect(cancellations).toBe(1); + expect(requests).toHaveLength(1); + await waitFor(() => expect(saved()[0]?.messages.at(-1)?.backgroundRun?.status).toBe("cancelled")); + }, +); diff --git a/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx index 6b333750b78..5ea5db56920 100644 --- a/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx @@ -64,7 +64,7 @@ vi.mock("@/contexts/ChatShellContext", () => ({ }, })); -const ONE_TURN_ARG_COUNT = 25; +const ONE_TURN_ARG_COUNT = 28; const ON_TIMING_DATA_INDEX = 7; const ON_USAGE_DATA_INDEX = 8; const ON_TOTAL_LATENCY_INDEX = 24; diff --git a/ui/litellm-dashboard/src/app/chat/page.tsx b/ui/litellm-dashboard/src/app/chat/page.tsx index 85fc7133215..757bf010fd9 100644 --- a/ui/litellm-dashboard/src/app/chat/page.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.tsx @@ -15,6 +15,7 @@ import type { AssistantMessageUpdate, ChatAttachment, ChatMessage } from "@/comp import ChatModelPicker, { isChatModel, modelChatMode, type ChatMode } from "@/components/chat/ChatModelPicker"; import MoyaiChatComposer from "@/components/chat/moyai/MoyaiChatComposer"; import MoyaiChatHome from "@/components/chat/moyai/MoyaiChatHome"; +import { useBackgroundChat } from "@/components/chat/useBackgroundChat"; import { runChatRequest } from "@/components/chat/runChatRequest"; import { readChatAttachment, MAX_CHAT_ATTACHMENTS, CHAT_ATTACHMENT_ACCEPT } from "@/components/chat/attachmentHelpers"; import ChatFileChip from "@/components/chat/ChatFileChip"; @@ -102,7 +103,30 @@ export default function ChatConversationPage() { const responsesSessionGenerationRef = useRef(0); const previousResponsesConversationRef = useRef(activeConversationId); const [prevConversationIdForSessionReset, setPrevConversationIdForSessionReset] = useState(activeConversationId); - const [isStreaming, setIsStreaming] = useState(false); + const [isRequestInFlight, setIsStreaming] = useState(false); + const [cancelOnAcceptance, setCancelOnAcceptance] = useState(null); + const backgroundRequestRef = useRef(false); + const lastMessage = activeConversation?.messages.at(-1); + const background = useBackgroundChat( + activeConversationId, + lastMessage?.backgroundRun, + accessToken, + updateLastAssistantMessage, + ); + const { pending: backgroundPending, cancelling: backgroundCancelling, cancel: cancelBackground } = background; + useEffect(() => { + if (!backgroundPending || backgroundCancelling || !activeConversationId) return; + if (cancelOnAcceptance !== activeConversationId) return; + void cancelBackground().then(() => { + setCancelOnAcceptance((current) => (current === activeConversationId ? null : current)); + }); + }, [activeConversationId, backgroundPending, backgroundCancelling, cancelBackground, cancelOnAcceptance]); + + const isStreaming = isRequestInFlight || background.pending; + const supportsBackground = models.some( + (model) => model.model_group === selectedModel && model.supports_background_chat, + ); + const useBackground = mode === "chat" && supportsBackground && !storageUnavailable; const [inputText, setInputText] = useState(""); const [mcpPopoverOpen, setMcpPopoverOpen] = useState(false); const [storageBannerDismissed, setStorageBannerDismissed] = useState(false); @@ -264,6 +288,7 @@ export default function ChatConversationPage() { setAttachmentError(null); setIsStreaming(true); + setCancelOnAcceptance(null); abortControllerRef.current = new AbortController(); requestConversationRef.current = convId; @@ -297,12 +322,14 @@ export default function ChatConversationPage() { signal: requestController.signal, messages: history, previousResponseId, + background: requestMode === "chat" && useBackground, mcpServers: selectedMCPServers, onResponseId: (id: string | null) => { if (responsesSessionGenerationRef.current === sessionGeneration) setResponsesSessionId(id); }, onUpdate: (updates: AssistantMessageUpdate) => updateLastAssistantMessage(convId, updates), }; + backgroundRequestRef.current = request.background; try { await runChatRequest(request); } finally { @@ -326,13 +353,22 @@ export default function ChatConversationPage() { updateLastAssistantMessage, isStreaming, responsesSessionId, + useBackground, setConversationIdInUrl, ], ); - const handleStop = useCallback(() => { + const handleStop = () => { + if (background.pending) { + void background.cancel(); + return; + } + if (backgroundRequestRef.current) { + setCancelOnAcceptance(activeConversationId); + return; + } abortControllerRef.current?.abort(); - }, []); + }; const handleEditAndResend = useCallback( (messageId: string, newContent: string) => { @@ -403,7 +439,21 @@ export default function ChatConversationPage() { const showBlankState = !activeConversation || activeConversation.messages.length === 0; + const backgroundHint = () => { + if (background.cancelling || (isRequestInFlight && cancelOnAcceptance === activeConversationId)) + return "Requesting cancellation…"; + if (background.pending && storageUnavailable) + return "Keep this tab open. This run could not be saved in this browser."; + if (background.pending) return "Running in the background. You can close this tab and return to this chat."; + if (useBackground) return null; + return "Keep this tab open. Background runs are not available for this chat."; + }; + const inputBar = (inConversation: boolean) => { + const hint = mode === "chat" || background.pending ? backgroundHint() : null; + const chatPlaceholder = inConversation + ? "Send a message..." + : "Describe a change, paste an error, or ask a question…"; const preparing = isLoadingModels || isReadingAttachments; const submitDisabled = !hasMessageContent(inputText, attachments, mode) || !selectedModel || preparing; return ( @@ -424,13 +474,7 @@ export default function ChatConversationPage() { onChange={setInputText} onSubmit={() => void handleSend(inputText)} onCancel={handleStop} - placeholder={ - mode === "image" - ? "Describe the image you want to create…" - : inConversation - ? "Send a message..." - : "Describe a change, paste an error, or ask a question…" - } + placeholder={mode === "image" ? "Describe the image you want to create…" : chatPlaceholder} isLoading={isStreaming} submitLabel={mode === "image" ? "Generate image" : "Send"} submitDisabled={submitDisabled} @@ -536,6 +580,29 @@ export default function ChatConversationPage() { : "No chat models are available for your account."}

)} + {background.connectionError && ( +
+ {background.connectionError} + +
+ )} + {lastMessage?.backgroundError && ( +

+ {lastMessage.backgroundError} +

+ )} + {lastMessage?.backgroundRun?.status === "cancelled" && ( +

+ Cancellation requested +

+ )} + {hint && ( +

+ {hint} +

+ )}

{mode === "image" ? "Describe an image · Enter to generate" : "Drop files or paste images · Enter to send"}

diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx new file mode 100644 index 00000000000..311094909ce --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.integration.test.tsx @@ -0,0 +1,98 @@ +import { fireEvent, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import BackgroundChatSettings from "./BackgroundChatSettings"; + +let settings: { + enabled: boolean; + redis_configured: boolean; + managed_by_config: boolean; + can_save: boolean; +}; +let failLoad: boolean; +let failSave: boolean; +const updates: unknown[] = []; + +beforeEach(() => { + testQueryClient.clear(); + settings = { + enabled: false, + redis_configured: true, + managed_by_config: false, + can_save: true, + }; + failLoad = false; + failSave = false; + updates.length = 0; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const request = + input instanceof Request ? input : new Request(new URL(String(input), window.location.origin), init); + if (request.method === "PATCH") { + const body = await request.json(); + updates.push(body); + if (failSave) return Response.json({ detail: "Save failed" }, { status: 500 }); + settings = { ...settings, enabled: body.enabled }; + } else if (failLoad) return Response.json({ detail: "Unavailable" }, { status: 503 }); + return Response.json(settings); + }), + ); +}); +afterEach(() => vi.unstubAllGlobals()); + +it("saves background execution next to its explanation and retains it after reopening", async () => { + const first = renderWithProviders(); + fireEvent.click(await screen.findByRole("switch", { name: "Continue chats in the background" })); + await waitFor(() => expect(screen.getByRole("switch")).toBeChecked()); + expect(updates).toEqual([{ enabled: true }]); + expect(screen.getByRole("switch")).toBeChecked(); + first.unmount(); + testQueryClient.clear(); + renderWithProviders(); + expect(await screen.findByRole("switch")).toBeChecked(); + fireEvent.click(screen.getByRole("switch")); + await waitFor(() => expect(screen.getByRole("switch")).not.toBeChecked()); + expect(updates).toEqual([{ enabled: true }, { enabled: false }]); +}); + +it("links to Redis setup and prevents enabling without Redis", async () => { + settings.redis_configured = false; + renderWithProviders(); + expect(await screen.findByRole("switch")).toHaveAttribute("aria-disabled", "true"); + expect(screen.getByRole("link", { name: /Configure it in Caching/ })).toHaveAttribute( + "href", + expect.stringMatching(/\/caching$/), + ); + expect(updates).toHaveLength(0); +}); + +it.each(["managed_by_config", "can_save"] as const)( + "explains a setting that cannot be edited because of %s", + async (field) => { + settings[field] = field === "managed_by_config"; + renderWithProviders(); + expect(await screen.findByRole("switch")).toHaveAttribute("aria-disabled", "true"); + const message = + field === "managed_by_config" ? /Managed by your config file/ : /Enable database-backed configuration/; + expect(screen.getByText(message)).toBeVisible(); + }, +); + +it("keeps a failed save off", async () => { + failSave = true; + renderWithProviders(); + fireEvent.click(await screen.findByRole("switch")); + await waitFor(() => expect(updates).toHaveLength(1)); + expect(screen.getByRole("switch")).not.toBeChecked(); +}); + +it("offers retry for a settings fetch failure instead of claiming Redis is missing", async () => { + failLoad = true; + renderWithProviders(); + await screen.findByRole("button", { name: "Retry" }); + expect(screen.queryByRole("link")).not.toBeInTheDocument(); + failLoad = false; + fireEvent.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByRole("switch")).not.toHaveAttribute("aria-disabled", "true"); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx new file mode 100644 index 00000000000..428edfff86d --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/BackgroundChatSettings.tsx @@ -0,0 +1,73 @@ +import { $api } from "@/lib/http/api"; +import { toast } from "@/lib/toast"; +import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Switch } from "@/components/ui/switch"; +import { uiHref } from "@/utils/uiHref"; + +export default function BackgroundChatSettings() { + const query = $api.useQuery("get", "/get/background_chat_settings"); + const update = $api.useMutation("patch", "/update/background_chat_settings", { + onSuccess: () => { + void query.refetch(); + toast.success("Background chat settings saved."); + }, + onError: (error) => toast.fromError(error), + }); + + if (query.isPending) return ; + if (query.isError || !query.data) + return ( +
+ Could not load background chat settings. + +
+ ); + + const settings = query.data; + const readOnly = settings.managed_by_config || !settings.can_save; + const needsRedis = !settings.redis_configured && !settings.enabled; + const disabled = update.isPending || readOnly || needsRedis; + + return ( +
+ update.mutate({ body: { enabled } })} + aria-label="Continue chats in the background" + aria-describedby="background-chat-description" + /> +
+

Continue chats in the background

+

+ Keep replies and tool runs going after users close the browser. Return to the same browser to see the result. +

+ {!settings.redis_configured && ( +

+ Requires Redis.{" "} + + Configure it in Caching → Coordination Redis + + , then restart the proxy. +

+ )} + {settings.managed_by_config && ( +

+ Managed by your config file. Update its background polling setting to change this. +

+ )} + {!settings.can_save && !settings.managed_by_config && ( +

Enable database-backed configuration to save this setting.

+ )} + {!readOnly && ( +

+ Changes apply automatically when the proxy refreshes its configuration. +

+ )} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx index 2aa5c738880..c2bfd6076d4 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.test.tsx @@ -5,6 +5,8 @@ import userEvent from "@testing-library/user-event"; import UISettings from "./UISettings"; import { toast } from "@/lib/toast"; +vi.mock("./BackgroundChatSettings", () => ({ default: () => null })); + const mockUseAuthorized = vi.hoisted(() => vi.fn()); const mockUseUISettings = vi.hoisted(() => vi.fn()); const mockUseUpdateUISettings = vi.hoisted(() => vi.fn()); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx index 3103851f280..08f345af928 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/UISettings/UISettings.tsx @@ -16,6 +16,7 @@ import { relativeTimeOptions } from "@/components/shared/date_range_presets"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Label } from "@/components/ui/label"; import PageVisibilitySettings from "./PageVisibilitySettings"; +import BackgroundChatSettings from "./BackgroundChatSettings"; interface SettingRowProps { ariaLabel: string; @@ -429,13 +430,15 @@ export default function UISettings() { disabled={isUpdating} onCheckedChange={handleToggleEnableChatUI} ariaLabel={enableChatUIProperty?.description ?? "Enable Chat page"} - label="[BETA] Enable Chat page (page will refresh)" + label="Enable Chat page (page will refresh)" description={ enableChatUIProperty?.description ?? "If enabled, shows the Chat page in the UI sidebar, letting users chat with an LLM and connect their own MCP server credentials via OAuth." } /> + +
diff --git a/ui/litellm-dashboard/src/components/chat/ChatShell.tsx b/ui/litellm-dashboard/src/components/chat/ChatShell.tsx index 280e2d10b8c..03d91bb13bc 100644 --- a/ui/litellm-dashboard/src/components/chat/ChatShell.tsx +++ b/ui/litellm-dashboard/src/components/chat/ChatShell.tsx @@ -192,9 +192,6 @@ const ChatShell: React.FC<{ children: React.ReactNode }> = ({ children }) => { ))} -

- Chat is a pre-v0 preview and may change unexpectedly. Do not use in production. -

diff --git a/ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts b/ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts new file mode 100644 index 00000000000..fd3d51883ff --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat/backgroundChat.test.ts @@ -0,0 +1,177 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { Response as ModelResponse } from "openai/resources/responses/responses"; +import { backgroundResponseUpdate, cancelBackgroundChat, watchBackgroundChat } from "./backgroundChat"; + +vi.mock("@/components/networking", () => ({ getGatewayBaseUrl: () => "http://localhost:4000/v1" })); + +afterEach(() => vi.unstubAllGlobals()); + +it("should restore text, reasoning, tool results and usage from a completed response", () => { + const response = { + id: "litellm_poll_example", + status: "completed", + output: [ + { type: "reasoning", summary: [{ type: "summary_text", text: "Check the inventory" }] }, + { + type: "mcp_call", + id: "call_lookup", + name: "lookup", + arguments: "{}", + output: "12 widgets", + server_label: "inventory", + }, + { type: "message", content: [{ type: "output_text", text: "There are 12 widgets." }] }, + ], + usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, output_tokens_details: { reasoning_tokens: 2 } }, + } as ModelResponse; + const update = backgroundResponseUpdate(response); + expect(update.content).toBe("There are 12 widgets."); + expect(update.reasoningContent).toBe("Check the inventory"); + expect(update.mcpEvents?.[0].item?.output).toBe("12 widgets"); + const usage = { promptTokens: 4, completionTokens: 9, totalTokens: 13, reasoningTokens: 2 }; + expect(update.usage).toMatchObject(usage); + expect(update.backgroundRun).toEqual({ id: response.id, status: "completed" }); +}); + +describe("background response lifecycle", () => { + it("should detach polling without cancelling the server run", async () => { + const controller = new AbortController(); + const fetch = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) => + Response.json({ id: "litellm_poll_example", status: "in_progress", output: [] }), + ); + vi.stubGlobal("fetch", fetch); + const onUpdate = vi.fn(() => controller.abort()); + const polling = { + accessToken: "test-token", + responseId: "litellm_poll_example", + signal: controller.signal, + onUpdate, + }; + await watchBackgroundChat(polling); + expect(onUpdate).toHaveBeenCalledOnce(); + expect(fetch).toHaveBeenCalledOnce(); + expect(fetch.mock.calls[0][1]?.method).toBe("GET"); + }); + + it("should mark an expired run unavailable without clearing its partial answer", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ error: { message: "Expired" } }, { status: 404 })), + ); + const onUpdate = vi.fn(); + const polling = { + accessToken: "test-token", + responseId: "litellm_poll_expired", + signal: new AbortController().signal, + onUpdate, + }; + await watchBackgroundChat(polling); + expect(onUpdate).toHaveBeenCalledWith({ + backgroundRun: { id: "litellm_poll_expired", status: "expired" }, + backgroundError: "This saved run is no longer available. Its result may have expired on the server.", + }); + }); + + it("should cancel explicitly through the server and preserve returned partial output", async () => { + const fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const request = input instanceof Request ? input : new Request(input, init); + expect(request.method).toBe("POST"); + expect(new URL(request.url).pathname).toBe("/v1/responses/litellm_poll_example/cancel"); + return Response.json({ + id: "litellm_poll_example", + status: "cancelled", + output: [{ type: "message", content: [{ type: "output_text", text: "Partial answer" }] }], + }); + }); + vi.stubGlobal("fetch", fetch); + expect(await cancelBackgroundChat("test-token", "litellm_poll_example")).toMatchObject({ + content: "Partial answer", + backgroundRun: { id: "litellm_poll_example", status: "cancelled" }, + }); + }); +}); + +it.each([ + [0.125, 0.125], + ["0.125", 0.125], + [0, 0], + ["invalid", undefined], +])("should preserve valid returned cost %s", (cost, expected) => { + const response = { + id: "litellm_poll_cost", + status: "completed", + output: [], + usage: { input_tokens: 4, output_tokens: 9, total_tokens: 13, cost }, + } as ModelResponse; + expect(backgroundResponseUpdate(response).usage?.cost).toBe(expected); +}); + +it("should keep polling through unfinished output items until the answer and tools arrive", async () => { + let polls = 0; + vi.stubGlobal( + "fetch", + vi.fn(async () => { + polls++; + return Response.json({ + id: "litellm_poll_partial", + status: polls === 1 ? "in_progress" : "completed", + output: + polls === 1 + ? [ + { type: "message", id: "message_1" }, + { type: "mcp_list_tools", id: "tools_1", server_label: "inventory" }, + ] + : [ + { type: "message", content: [{ type: "output_text", text: "12 widgets" }] }, + { type: "mcp_list_tools", server_label: "inventory", tools: [{ name: "lookup", input_schema: {} }] }, + ], + }); + }), + ); + const onUpdate = vi.fn(); + const polling = { + accessToken: "test-token", + responseId: "litellm_poll_partial", + signal: new AbortController().signal, + onUpdate, + }; + await watchBackgroundChat(polling); + expect(polls).toBe(2); + expect(onUpdate.mock.lastCall?.[0]).toMatchObject({ + content: "12 widgets", + backgroundRun: { status: "completed" }, + mcpEvents: [{ item: { tools: [{ name: "lookup" }] } }], + }); +}); + +it("should restore server timings from polling without counting time away from the chat", async () => { + const response = { + id: "litellm_poll_timing", + status: "completed", + created_at: 1000, + output: [], + litellm_response_timing: { time_to_first_token_ms: 250, total_latency_ms: 12500 }, + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json(response)), + ); + const onUpdate = vi.fn(); + const polling = { + accessToken: "test-token", + responseId: "litellm_poll_timing", + signal: new AbortController().signal, + onUpdate, + }; + await watchBackgroundChat(polling); + expect(onUpdate).toHaveBeenCalledWith(expect.objectContaining({ timeToFirstToken: 250, totalLatency: 12500 })); +}); + +it.each(["queued", "completed"] as const)( + "should not invent timings for a %s response without measurements", + (status) => { + const update = backgroundResponseUpdate({ id: "litellm_poll_legacy", status, output: [] } as ModelResponse); + expect(update).not.toHaveProperty("timeToFirstToken"); + expect(update).not.toHaveProperty("totalLatency"); + }, +); diff --git a/ui/litellm-dashboard/src/components/chat/backgroundChat.ts b/ui/litellm-dashboard/src/components/chat/backgroundChat.ts new file mode 100644 index 00000000000..177d25f2e16 --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat/backgroundChat.ts @@ -0,0 +1,138 @@ +import OpenAI from "openai"; +import type { Response } from "openai/resources/responses/responses"; +import { getGatewayBaseUrl } from "@/components/networking"; +import { parseUsageCost } from "@/components/llm_calls/usage_cost"; +import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; +import type { AssistantMessageUpdate, ChatBackgroundRun, MCPEvent } from "./types"; + +export const isBackgroundRunPending = (run?: ChatBackgroundRun): boolean => + run?.status === "queued" || run?.status === "in_progress"; + +type TimedResponse = Response & { + litellm_response_timing?: { + time_to_first_token_ms?: number; + total_latency_ms?: number; + }; +}; + +export function backgroundResponseUpdate(response: TimedResponse): AssistantMessageUpdate { + const output = response.output ?? []; + const content = output + .filter((item) => item.type === "message") + .flatMap((item) => item.content ?? []) + .map((part) => (part.type === "output_text" ? part.text : part.refusal)) + .join(""); + const reasoningContent = output + .filter((item) => item.type === "reasoning") + .flatMap((item) => item.summary ?? []) + .map((part) => part.text) + .join("\n\n"); + const mcpEvents: MCPEvent[] = output.flatMap((item) => { + if (item.type === "mcp_call") { + return [{ type: "response.output_item.done", item: { ...item, output: item.output ?? undefined } }]; + } + if (item.type === "mcp_list_tools") { + return [ + { + type: "response.output_item.done", + item: { + ...item, + tools: (item.tools ?? []).map((tool) => ({ + name: tool.name, + input_schema: tool.input_schema, + description: tool.description ?? "", + })), + }, + }, + ]; + } + return []; + }); + const status = response.status ?? "in_progress"; + const errors: Partial> = { + failed: response.error?.message || "The background run failed.", + incomplete: "The run ended before completing its answer.", + }; + const backgroundError = errors[status]; + const cost = response.usage && "cost" in response.usage ? parseUsageCost(response.usage.cost) : undefined; + const timing = response.litellm_response_timing; + return { + content, + reasoningContent, + mcpEvents, + backgroundRun: { id: response.id, status }, + backgroundError, + ...(timing?.time_to_first_token_ms !== undefined && { timeToFirstToken: timing.time_to_first_token_ms }), + ...(timing?.total_latency_ms !== undefined && { totalLatency: timing.total_latency_ms }), + ...(response.usage && { + usage: { + promptTokens: response.usage.input_tokens, + completionTokens: response.usage.output_tokens, + totalTokens: response.usage.total_tokens, + reasoningTokens: response.usage.output_tokens_details?.reasoning_tokens, + ...extractPromptCacheTokens(response.usage), + ...(cost !== undefined ? { cost } : {}), + }, + }), + }; +} + +const backgroundClient = (accessToken: string) => + new OpenAI({ + apiKey: accessToken, + baseURL: getGatewayBaseUrl(), + dangerouslyAllowBrowser: true, + }); + +function waitForPoll(signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const abort = () => { + clearTimeout(timer); + reject(signal.reason); + }; + const timer = setTimeout(() => { + signal.removeEventListener("abort", abort); + resolve(); + }, 1000); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + }); +} + +export async function watchBackgroundChat({ + accessToken, + responseId, + signal, + onUpdate, +}: { + accessToken: string; + responseId: string; + signal: AbortSignal; + onUpdate: (update: AssistantMessageUpdate) => void; +}): Promise { + const client = backgroundClient(accessToken); + while (!signal.aborted) { + try { + const response = await client.responses.retrieve(responseId, { stream: false }, { signal }); + if (signal.aborted) return; + onUpdate(backgroundResponseUpdate(response)); + if (!isBackgroundRunPending({ id: response.id, status: response.status ?? "in_progress" })) return; + await waitForPoll(signal); + } catch (error) { + if (signal.aborted) return; + if (error instanceof OpenAI.APIError && error.status === 404) { + onUpdate({ + backgroundRun: { id: responseId, status: "expired" }, + backgroundError: "This saved run is no longer available. Its result may have expired on the server.", + }); + return; + } + throw error; + } + } +} + +export async function cancelBackgroundChat(accessToken: string, responseId: string): Promise { + const response = await backgroundClient(accessToken).responses.cancel(responseId); + return backgroundResponseUpdate(response); +} diff --git a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts index 157363d2461..9e8016b608e 100644 --- a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts +++ b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts @@ -1,5 +1,6 @@ import OpenAI from "openai"; import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api"; +import { backgroundResponseUpdate } from "./backgroundChat"; import { generateChatImages } from "./imageHelpers"; import { toResponsesMessages } from "./attachmentHelpers"; import type { AssistantMessageUpdate, ChatMessage, MCPEvent } from "./types"; @@ -15,6 +16,7 @@ interface ChatRequestOptions { "role" | "content" | "attachments" | "generatedImages" | "imageGeneration" | "mcpEvents" >[]; previousResponseId?: string | null; + background?: boolean; mcpServers?: string[]; onResponseId: (id: string | null) => void; onUpdate: (update: AssistantMessageUpdate) => void; @@ -43,6 +45,7 @@ export async function runChatRequest({ signal, messages, previousResponseId, + background = false, mcpServers, onResponseId, onUpdate, @@ -94,6 +97,14 @@ export async function runChatRequest({ undefined, true, (totalLatency) => onUpdate({ totalLatency }), + undefined, + undefined, + background + ? (response) => { + onResponseId(null); + onUpdate(backgroundResponseUpdate(response)); + } + : undefined, ); signal.throwIfAborted(); } catch (error: unknown) { diff --git a/ui/litellm-dashboard/src/components/chat/types.ts b/ui/litellm-dashboard/src/components/chat/types.ts index 25b232e40c4..e98b61e2fe4 100644 --- a/ui/litellm-dashboard/src/components/chat/types.ts +++ b/ui/litellm-dashboard/src/components/chat/types.ts @@ -25,6 +25,11 @@ export interface ChatImageGeneration { prompt: string; } +export interface ChatBackgroundRun { + id: string; + status: "queued" | "in_progress" | "completed" | "failed" | "cancelled" | "incomplete" | "expired"; +} + export interface ChatMessage { id: string; role: "user" | "assistant" | "tool"; @@ -33,6 +38,8 @@ export interface ChatMessage { generatedImages?: ChatImageAttachment[]; imageGeneration?: ChatImageGeneration; reasoningContent?: string; + backgroundRun?: ChatBackgroundRun; + backgroundError?: string; /** MCP tool events that occurred during this assistant turn, in order. */ mcpEvents?: MCPEvent[]; toolName?: string; @@ -55,6 +62,8 @@ export type AssistantMessageUpdate = Partial< | "usage" | "generatedImages" | "imageGeneration" + | "backgroundRun" + | "backgroundError" > >; diff --git a/ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts b/ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts new file mode 100644 index 00000000000..c0d1c43bcde --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat/useBackgroundChat.ts @@ -0,0 +1,62 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { cancelBackgroundChat, isBackgroundRunPending, watchBackgroundChat } from "./backgroundChat"; +import type { AssistantMessageUpdate, ChatBackgroundRun } from "./types"; + +export function useBackgroundChat( + conversationId: string | null, + run: ChatBackgroundRun | undefined, + accessToken: string, + updateMessage: (conversationId: string, update: AssistantMessageUpdate) => void, +) { + const [failure, setFailure] = useState<{ responseId: string; retry: number; message: string } | null>(null); + const [retry, setRetry] = useState(0); + const [cancellingId, setCancellingId] = useState(null); + const controllerRef = useRef(null); + const pending = isBackgroundRunPending(run); + const responseId = run?.id; + const cancelling = Boolean(responseId && cancellingId === responseId); + const connectionError = + pending && failure?.responseId === responseId && failure?.retry === retry ? failure.message : null; + + useEffect(() => { + if (!conversationId || !responseId || !pending) return; + const controller = new AbortController(); + controllerRef.current = controller; + const polling = { + accessToken, + responseId, + signal: controller.signal, + onUpdate: (update: AssistantMessageUpdate) => updateMessage(conversationId, update), + }; + void watchBackgroundChat(polling).catch(() => { + if (!controller.signal.aborted) + setFailure({ responseId, retry, message: "Connection lost. Your run may still be running on the server." }); + }); + return () => { + controller.abort(); + if (controllerRef.current === controller) controllerRef.current = null; + }; + }, [conversationId, responseId, pending, accessToken, updateMessage, retry]); + + const reconnect = useCallback(() => setRetry((value) => value + 1), []); + + const cancel = useCallback(async () => { + if (!conversationId || !responseId || cancelling) return; + const controller = controllerRef.current; + controller?.abort(); + setCancellingId(responseId); + try { + const update = await cancelBackgroundChat(accessToken, responseId); + updateMessage(conversationId, update); + if (controllerRef.current === controller) reconnect(); + } catch { + if (controllerRef.current === controller) { + setFailure({ responseId, retry, message: "Could not stop the run. Reconnect and try Stop again." }); + } + } finally { + setCancellingId(null); + } + }, [conversationId, responseId, accessToken, cancelling, reconnect, retry, updateMessage]); + + return { pending, connectionError, cancelling, reconnect, cancel }; +} diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx index d38db631c6e..cac91cf92d4 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx @@ -9,6 +9,7 @@ export interface ModelGroup { mode?: string; supports_reasoning?: boolean; supports_fast_mode?: boolean; + supports_background_chat?: boolean; supported_reasoning_efforts?: string[] | null; } @@ -20,6 +21,7 @@ interface AvailableModel { mode?: string | null; supports_reasoning?: boolean | null; supports_fast_mode?: boolean | null; + supports_background_chat?: boolean | null; supported_reasoning_efforts?: string[] | null; } @@ -30,6 +32,7 @@ const toModelGroup = (item: AvailableModel): ModelGroup => { ...(item.providers && { providers: item.providers }), ...(item.mode && { mode: item.mode }), ...(item.supports_reasoning === true && { supports_reasoning: true }), + ...(item.supports_background_chat === true && { supports_background_chat: true }), ...(item.supports_fast_mode === true && { supports_fast_mode: true }), ...(item.supported_reasoning_efforts !== undefined && { supported_reasoning_efforts: item.supported_reasoning_efforts, diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index ce1a8aab0c5..ef737c22cdb 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -1,6 +1,10 @@ import { observeResponseHeaders, type ResponseHeadersListener } from "@/lib/http/observeResponseHeaders"; import openai from "openai"; -import type { ResponseFunctionToolCall, ResponseInputItem } from "openai/resources/responses/responses"; +import type { + Response as OpenAIResponse, + ResponseFunctionToolCall, + ResponseInputItem, +} from "openai/resources/responses/responses"; import { MessageType } from "../chat_ui/types"; import { TokenUsage } from "../chat_ui/ResponseMetrics"; import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking"; @@ -110,6 +114,7 @@ export async function makeOpenAIResponsesRequest( onTotalLatency?: (latency: number) => void, customHeaders?: CustomHeaders, onResponseHeaders?: ResponseHeadersListener, + onBackgroundResponse?: (response: OpenAIResponse) => void, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -239,6 +244,16 @@ export async function makeOpenAIResponsesRequest( ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), }; + if (onBackgroundResponse) { + const response = await client.post("/responses", { + body: { ...requestBody, background: true, stream: false }, + signal, + maxRetries: 0, + }); + onBackgroundResponse(response); + return response; + } + // Create request to OpenAI responses API // Use 'any' type to avoid TypeScript issues with the experimental API const response = streamingEnabled diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e80df24a818..0dce821f862 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5849,6 +5849,26 @@ export interface paths { patch?: never; trace?: never; }; + "/get/background_chat_settings": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Background Chat Settings + * @description Get saved background-response settings and the prerequisites for changing them. + */ + get: operations["get_background_chat_settings_get_background_chat_settings_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/get/config/callbacks": { parameters: { query?: never; @@ -18240,6 +18260,26 @@ export interface paths { patch: operations["typesafe_proxy_route_typesafe__endpoint__patch"]; trace?: never; }; + "/update/background_chat_settings": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + /** + * Update Background Chat Settings + * @description Save background-response polling settings for the proxy configuration refresh. + */ + patch: operations["update_background_chat_settings_update_background_chat_settings_patch"]; + trace?: never; + }; "/update/default_team_settings": { parameters: { query?: never; @@ -27157,6 +27197,37 @@ export interface components { } & { [key: string]: unknown; }; + /** BackgroundChatSettings */ + BackgroundChatSettings: { + /** + * Enabled + * @description Enable cache-backed background Responses for Chat and API requests. + */ + enabled: boolean; + }; + /** BackgroundChatSettingsResponse */ + BackgroundChatSettingsResponse: { + /** + * Can Save + * @description Whether database-backed configuration updates are available. + */ + can_save: boolean; + /** + * Enabled + * @description Enable cache-backed background Responses for Chat and API requests. + */ + enabled: boolean; + /** + * Managed By Config + * @description Whether the responses configuration is owned by the config file. + */ + managed_by_config: boolean; + /** + * Redis Configured + * @description Whether this proxy has coordination Redis configured. + */ + redis_configured: boolean; + }; /** BaseLitellmParams */ BaseLitellmParams: { /** @@ -38804,6 +38875,12 @@ export interface components { supported_openai_params: string[] | null; /** Supported Reasoning Efforts */ supported_reasoning_efforts?: string[] | null; + /** + * Supports Background Chat + * @description Whether cache-backed background Responses runs are enabled for this model group + * @default false + */ + supports_background_chat: boolean; /** * Supports Fast Mode * @default false @@ -60982,6 +61059,26 @@ export interface operations { }; }; }; + get_background_chat_settings_get_background_chat_settings_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BackgroundChatSettingsResponse"]; + }; + }; + }; + }; get_config_get_config_callbacks_get: { parameters: { query?: never; @@ -76302,6 +76399,39 @@ export interface operations { }; }; }; + update_background_chat_settings_update_background_chat_settings_patch: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BackgroundChatSettings"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BackgroundChatSettingsResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; update_default_team_settings_update_default_team_settings_patch: { parameters: { query?: never; From 6aa546303e09326372f42488ddc01c0883344aae Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:48:56 +0000 Subject: [PATCH 4/6] docs(pr-template): wrap the agent-drafted summary in make-me-human-readable tags (#49494) * docs(pr-template): wrap the agent-drafted summary in make-me-human-readable tags Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(pr-template): keep the make-me-human-readable tags visible in rendered descriptions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(pr-template): mark the agent-drafted summary with a visible Make me human-readable warning block Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Revert "docs(pr-template): mark the agent-drafted summary with a visible Make me human-readable warning block" This reverts commit 4eb7f666a451cc9a88c63ab50b84d0eb4998f660. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/PULL_REQUEST_TEMPLATE/general.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/general.md b/.github/PULL_REQUEST_TEMPLATE/general.md index f6ce687a552..aafd968ee0f 100644 --- a/.github/PULL_REQUEST_TEMPLATE/general.md +++ b/.github/PULL_REQUEST_TEMPLATE/general.md @@ -9,10 +9,16 @@ +`` + +`` +
Details From da5ea5966797499dc6aa22ad1d89df1a012c27ae Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 31 Oct 2026 15:09:15 -0700 Subject: [PATCH 5/6] refactor(ui): remove dead API calls from networking.tsx (#49507) Deletes 17 exported calls that nothing references anymore. The Old Usage page removal left 10 behind, the move of agent traces to the Lens service left 5, and modelInfoV1Call and getUISettings lost their last callers in earlier refactors Seven names that are only used inside the file stop being exported. buildModelGroupTestRequest stays exported because networking.test.ts calls it directly Also drops the two test mocks that still stubbed deleted calls and prunes the lower eslint suppression counts for the file --- ui/litellm-dashboard/eslint-suppressions.json | 4 +- .../components/SidebarProvider.test.tsx | 2 +- .../model_info_view.integration.test.tsx | 23 -- .../src/components/networking.tsx | 352 +----------------- 4 files changed, 10 insertions(+), 371 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index a4e0d59d73d..d4ac60e501b 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1732,10 +1732,10 @@ "count": 5 }, "no-restricted-syntax": { - "count": 139 + "count": 133 }, "prefer-const": { - "count": 28 + "count": 26 } }, "src/components/object_permissions_view.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx index a3d75df9606..fa4b48a5a9d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.test.tsx @@ -9,7 +9,7 @@ const { getSettings } = vi.hoisted(() => ({ getSettings: vi.fn().mockResolvedValue({ values: { enable_projects_ui: true } }), })); -vi.mock("@/components/networking", () => ({ getUiSettings: getSettings, getUISettings: getSettings })); +vi.mock("@/components/networking", () => ({ getUiSettings: getSettings })); vi.mock("@/components/leftnav", () => ({ default: ({ enableProjectsUI }: ComponentProps) => ( diff --git a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx index f93e01eb583..32e590017b0 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx @@ -19,7 +19,6 @@ vi.mock("../../utils/dataUtils", () => ({ })); vi.mock("./networking", () => ({ - modelInfoV1Call: vi.fn(), credentialGetCall: vi.fn(), credentialListCall: vi.fn(), getGuardrailsList: vi.fn(), @@ -56,7 +55,6 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () => })); const mockToast = vi.mocked(toast); -const mockModelInfoV1Call = vi.mocked(networking.modelInfoV1Call); const mockCredentialGetCall = vi.mocked(networking.credentialGetCall); const mockCredentialListCall = vi.mocked(networking.credentialListCall); const mockGetGuardrailsList = vi.mocked(networking.getGuardrailsList); @@ -134,10 +132,6 @@ describe("ModelInfoView", () => { error: null, }); - mockModelInfoV1Call.mockResolvedValue({ - data: [defaultModelData], - }); - mockCredentialGetCall.mockResolvedValue({ credential_name: "test-credential", credential_values: {}, @@ -394,7 +388,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, mode: "chat", key: "anthropic.claude-haiku-4-5" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [displayedModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [displayedModel] }); render(, { wrapper }); await waitFor(() => { @@ -869,7 +862,6 @@ describe("ModelInfoView", () => { const renderWithPtuModel = () => { mockUseModelsInfo.mockReturnValue({ data: { data: [ptuModelData] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [ptuModelData] }); return render(, { wrapper }); }; @@ -967,7 +959,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, team_id: "team-1", input_cost_per_token: 0.0000003 }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [seededModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [seededModel] }); const user = userEvent.setup(); render(, { wrapper }); @@ -1008,7 +999,6 @@ describe("ModelInfoView", () => { model_info: { ...ptuModelData.model_info, cost_per_ptu_per_hour: null, ptu_effective_from: null }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [halfSetPtuModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [halfSetPtuModel] }); const user = userEvent.setup(); render(, { wrapper }); @@ -1057,7 +1047,6 @@ describe("ModelInfoView", () => { model_info: { ...ptuModelData.model_info, ptu_effective_from: null, ptu_effective_to: null }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [undatedPtuModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [undatedPtuModel] }); const user = userEvent.setup(); render(, { wrapper }); @@ -1201,7 +1190,6 @@ describe("ModelInfoView", () => { isLoading: false, error: null, }); - mockModelInfoV1Call.mockResolvedValue({ data: [maskedModelData] }); const user = userEvent.setup(); render(, { wrapper }); @@ -1508,7 +1496,6 @@ describe("ModelInfoView", () => { beforeEach(() => { mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] }); }); const readRawJson = async (user: ReturnType) => { @@ -1555,7 +1542,6 @@ describe("ModelInfoView", () => { it("shows Not Set and no team_alias for a model without a team", async () => { mockUseModelsInfo.mockReturnValue({ data: { data: [defaultModelData] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [defaultModelData] }); mockUseTeams.mockReturnValue({ data: [{ team_id: "team-1", team_alias: "alpha" }], isLoading: false, @@ -1849,7 +1835,6 @@ describe("ModelInfoView", () => { beforeEach(() => { mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheWriteTiers] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [withCacheWriteTiers] }); }); it("shows both configured cache write rates per million tokens", async () => { @@ -1926,7 +1911,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, team_id: "team-1" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] }); const user = userEvent.setup(); await enterEditMode(user); @@ -1952,7 +1936,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, team_id: "team-1" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] }); const user = userEvent.setup(); await enterEditMode(user); @@ -1992,7 +1975,6 @@ describe("ModelInfoView", () => { model_info: { ...defaultModelData.model_info, team_id: "team-1" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] }); const user = userEvent.setup(); render(, { wrapper }); await user.click(await screen.findByRole("button", { name: /edit settings/i })); @@ -2069,7 +2051,6 @@ describe("ModelInfoView", () => { litellm_params: litellmParamsWithoutCredential, }; mockUseModelsInfo.mockReturnValue({ data: { data: [modelWithoutCredential] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [modelWithoutCredential] }); const user = userEvent.setup(); await enterEditMode(user); @@ -2150,7 +2131,6 @@ describe("ModelInfoView", () => { litellm_params: { ...defaultModelData.litellm_params, model: "openai/gpt-4*" }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [wildcard] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [wildcard] }); mockUseModelHub.mockReturnValue({ data: { data: [{ model_group: "openai/gpt-4o", providers: ["openai"] }] }, isLoading: false, @@ -2207,7 +2187,6 @@ describe("ModelInfoView", () => { litellm_params: { ...defaultModelData.litellm_params, cache_control_injection_points: points }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [data] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [data] }); }; it("omits the key when the deployment has none and the operator leaves the toggle alone", async () => { @@ -2330,7 +2309,6 @@ describe("ModelInfoView", () => { litellm_params: { ...defaultModelData.litellm_params, cache_read_input_token_cost: 0.000001 }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [withCacheRead] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [withCacheRead] }); const user = userEvent.setup(); await enterEditMode(user); setInputCost("2"); @@ -2365,7 +2343,6 @@ describe("ModelInfoView", () => { }, }; mockUseModelsInfo.mockReturnValue({ data: { data: [resolved] }, isLoading: false, error: null }); - mockModelInfoV1Call.mockResolvedValue({ data: [resolved] }); const user = userEvent.setup(); await enterEditMode(user); setInputCost("5"); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index c13052be245..994c2948346 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -48,7 +48,7 @@ export const getAutoRouterClassifierDefaultPromptCall = async ( } }; -export type AssembledPromptTierSource = +type AssembledPromptTierSource = | { tierDefinitions: { name: string; description?: string }[] } | { tierLabels?: Record; classificationRubric?: string }; @@ -127,16 +127,6 @@ import type { SubscriptionUsageResponse, } from "@/app/(dashboard)/usage/_components/components/SubscriptionCost/types"; import type { RoutingDecision } from "./logs/detail/RoutingDecisionCard"; -import type { - SpanDetail, - SpanErrorPage, - SpanErrorQuery, - SpanQuery, - Trace, - TraceDetailQuery, - TraceListQuery, - TracePage, -} from "@litellm/lens-ui"; import { createApiClient, deriveErrorMessage, @@ -541,7 +531,7 @@ export const apiClient = createApiClient({ onError: handleError, }); -export const gatewayApiClient = createApiClient({ +const gatewayApiClient = createApiClient({ getBaseUrl: getGatewayBaseUrl, getAuthHeaderName: getGlobalLitellmHeaderName, onError: handleError, @@ -1725,37 +1715,6 @@ export const modelInfoCall = async ( } }; -export const modelInfoV1Call = async (accessToken: string, modelId: string) => { - /** - * Get all models on proxy - */ - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/v1/model/info` : `/v1/model/info`; - url += `?litellm_model_id=${modelId}`; - - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - export const modelHubPublicModelsCall = async () => { const url = proxyBaseUrl ? `${proxyBaseUrl}/public/model_hub` : `/public/model_hub`; const response = await fetch(url, { @@ -1906,66 +1865,6 @@ export const modelAvailableCall = async ( } }; -export const teamSpendLogsCall = async (accessToken: string) => { - try { - const data = await apiClient.get(`/global/spend/teams`, { accessToken }); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const tagsSpendLogsCall = async ( - accessToken: string, - startTime: string | undefined, - endTime: string | undefined, - tags: string[] | undefined, -) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/tags` : `/global/spend/tags`; - - if (startTime && endTime) { - url = `${url}?start_date=${startTime}&end_date=${endTime}`; - } - - // if tags, convert the list to a comma separated string - if (tags) { - url += `&tags=${tags.join(",")}`; - } - - const response = await fetch(`${url}`, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const allTagNamesCall = async (accessToken: string) => { - try { - const data = await apiClient.get(`/global/spend/all_tag_names`, { accessToken }); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - export const userFilterUICall = async (accessToken: string, params: URLSearchParams) => { try { return await apiClient.get(`/user/filter/ui`, { @@ -2081,214 +1980,6 @@ export const uiSpendLogsCall = async ({ } }; -/** - * Agent tracing. All three respond 501 `{detail}` when `general_settings.tracing` is not - * configured; callers can detect that through the thrown `ApiError`'s `status`. - */ -export const agentTraceListCall = async ({ - accessToken, - startMs, - endMs, - cursor, -}: { - accessToken: string; - startMs: number; - endMs: number; - cursor?: string | null; -}): Promise => { - const query = { start_ms: startMs, end_ms: endMs, cursor: cursor ?? undefined } satisfies TraceListQuery; - return apiClient.get(`/v1/traces`, { accessToken, query }); -}; - -export const sendOtlpTraceCall = async (accessToken: string, exportRequest: object): Promise => - apiClient.post(`/v1/traces`, { accessToken, body: exportRequest }); - -export const agentTraceCall = async ( - accessToken: string, - traceId: string, - traceRef?: string, - cursor?: string | null, -): Promise => - apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}`, { - accessToken, - query: { trace_ref: traceRef || undefined, cursor: cursor ?? undefined, page_size: 200 } satisfies TraceDetailQuery, - }); - -export const agentTraceSpanCall = async ( - accessToken: string, - traceId: string, - spanId: string, - traceRef?: string, -): Promise => - apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, { - accessToken, - query: { trace_ref: traceRef || undefined } satisfies SpanQuery, - }); - -export const agentTraceSpanErrorCall = async ( - accessToken: string, - traceId: string, - spanId: string, - options: { traceRef?: string; cursor?: string | null }, -): Promise => - apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}/error`, { - accessToken, - query: { trace_ref: options.traceRef || undefined, cursor: options.cursor || undefined } satisfies SpanErrorQuery, - }); - -export const adminSpendLogsCall = async (accessToken: string) => { - try { - const data = await apiClient.get(`/global/spend/logs`, { accessToken }); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const adminTopKeysCall = async (accessToken: string) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/keys?limit=5` : `/global/spend/keys?limit=5`; - - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const adminTopEndUsersCall = async ( - accessToken: string, - keyToken: string | null, - startTime: string | undefined, - endTime: string | undefined, -) => { - try { - const body = keyToken - ? { api_key: keyToken, startTime: startTime, endTime: endTime } - : { startTime: startTime, endTime: endTime }; - - const data = await apiClient.post(`/global/spend/end_users`, { accessToken, body }); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - -export const adminspendByProvider = async ( - accessToken: string, - startTime: string | undefined, - endTime: string | undefined, -) => { - try { - const data = await apiClient.get(`/global/spend/provider`, { - accessToken, - query: { - ...(startTime && endTime ? { start_date: startTime, end_date: endTime } : {}), - }, - }); - return data; - } catch (error) { - console.error("Failed to fetch spend data:", error); - throw error; - } -}; - -export const adminGlobalActivity = async ( - accessToken: string, - startTime: string | undefined, - endTime: string | undefined, -) => { - try { - const data = await apiClient.get(`/global/activity`, { - accessToken, - query: startTime && endTime ? { start_date: startTime, end_date: endTime } : undefined, - }); - return data; - } catch (error) { - console.error("Failed to fetch spend data:", error); - throw error; - } -}; - -export const adminGlobalActivityPerModel = async ( - accessToken: string, - startTime: string | undefined, - endTime: string | undefined, -) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/global/activity/model` : `/global/activity/model`; - - if (startTime && endTime) { - url += `?start_date=${startTime}&end_date=${endTime}`; - } - - const requestOptions = { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - }, - }; - - const response = await fetch(url, requestOptions); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to fetch spend data:", error); - throw error; - } -}; - -export const adminTopModelsCall = async (accessToken: string) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/models?limit=5` : `/global/spend/models?limit=5`; - - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - handleError(errorMessage); - throw new Error(errorMessage); - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to create key:", error); - throw error; - } -}; - export const testConnectionRequest = async ( accessToken: string, litellm_params: Record, @@ -2421,7 +2112,7 @@ export const testAutoRouterRouting = async ( } }; -export interface ComplexityRouterConfigValidation { +interface ComplexityRouterConfigValidation { valid: boolean; error?: string | null; } @@ -2733,7 +2424,7 @@ export interface UserConnectionStartResponse { flow_handle: string; } -export type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected"; +type UserConnectionPollStatus = "pending" | "slow_down" | "expired" | "denied" | "no_copilot_seat" | "connected"; export interface UserConnectionPollResponse { status: UserConnectionPollStatus; @@ -3697,35 +3388,6 @@ export const getProxyUISettings = async (accessToken: string) => { } }; -export const getUISettings = async (accessToken: string) => { - /** - * Get UI-specific configuration flags from the database - */ - try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/get/ui_settings` : `/get/ui_settings`; - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - - if (!response.ok) { - const errorData = await response.json(); - const errorMessage = deriveErrorMessage(errorData); - console.error("Failed to get UI settings:", errorMessage); - return null; - } - - const data = await response.json(); - return data; - } catch (error) { - console.error("Failed to get UI settings:", error); - return null; - } -}; - export const getMCPSemanticFilterSettings = async (accessToken: string) => { /** * Get MCP semantic filter configuration @@ -4685,12 +4347,12 @@ export const createAgentCall = async (accessToken: string, agentData: any) => { export type DiscoveredAgentCard = Agent["agent_card_params"]; -export type DiscoverAgentCardResponse = Omit & { +type DiscoverAgentCardResponse = Omit & { agent_card: DiscoveredAgentCard; }; export type DiscoveryMode = components["schemas"]["DiscoveryMode"]; -export type DiscoverAgentCardOptions = Partial< +type DiscoverAgentCardOptions = Partial< Pick >; @@ -5613,7 +5275,7 @@ export const vectorStoreListCall = async ( } }; -export interface IndexesListResponse { +interface IndexesListResponse { object: string; data: VectorStoreIndex[]; } From e708343309981ec61d4d51b7e8b0c1f1d0156ffb Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 31 Oct 2026 15:14:33 -0700 Subject: [PATCH 6/6] test: deflake the json accumulator drain test and the armed routing prefetch test (#49505) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_request_redis_batch_pre_call.py | 6 +++- .../test_json_fragment_accumulator.py | 34 ++++--------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/tests/unit/caching/test_request_redis_batch_pre_call.py b/tests/unit/caching/test_request_redis_batch_pre_call.py index 37169e13a0b..94f19dbdc7c 100644 --- a/tests/unit/caching/test_request_redis_batch_pre_call.py +++ b/tests/unit/caching/test_request_redis_batch_pre_call.py @@ -290,7 +290,11 @@ def _router(redis_cache: FakeRedisCache, routing_strategy: str = "usage-based-ro @pytest.mark.asyncio -async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own(): +async def test_armed_routing_read_rides_the_admission_pipeline_and_routing_issues_no_read_of_its_own(monkeypatch): + monkeypatch.setattr( + "litellm.router_strategy.lowest_tpm_rpm_v2.get_utc_datetime", + lambda: datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc), + ) client = FakeClient(_lua_ok_replies) redis_cache = FakeRedisCache(client) router = _router(redis_cache) diff --git a/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py b/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py index f27a5fd2307..212dff79daf 100644 --- a/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py +++ b/tests/unit/litellm_core_utils/test_json_fragment_accumulator.py @@ -1,6 +1,5 @@ import json import time -import tracemalloc from collections.abc import Iterator from typing import Final from unittest.mock import patch @@ -193,34 +192,15 @@ def _streamed(fragments: tuple[str, ...]) -> JSONFragmentAccumulator: return accumulator -def _drain_peak_bytes(fragments: tuple[str, ...]) -> int: - accumulator: Final = _streamed(fragments) - first_found, _ = accumulator.pop_next_value() - assert first_found is True - already_tracing: Final = tracemalloc.is_tracing() - if not already_tracing: - tracemalloc.start() - tracemalloc.reset_peak() - baseline, _ = tracemalloc.get_traced_memory() - try: - drained: Final = sum(1 for _ in _popped_values(accumulator)) - _, peak = tracemalloc.get_traced_memory() - finally: - if not already_tracing: - tracemalloc.stop() - assert drained == len(fragments) - 1 - return peak - baseline - - -def test_draining_many_concatenated_values_never_copies_the_remainder(): +def test_draining_many_concatenated_values_yields_every_value_in_order(): value: Final = '{"text": "' + "x" * 20 + '"}' fragments: Final = tuple(value if index % 2 else f" {value}" for index in range(40_000)) - payload_bytes: Final = sum(len(fragment) for fragment in fragments) - peak_bytes: Final = _drain_peak_bytes(fragments) - assert peak_bytes < payload_bytes // 4, ( - f"draining {len(fragments)} values allocated a peak of {peak_bytes} bytes over a {payload_bytes} byte payload, " - "copying the remainder on every pop?" - ) + accumulator: Final = _streamed(fragments) + + drained: Final = tuple(_popped_values(accumulator)) + + assert drained == (json.loads(value),) * len(fragments) + assert accumulator.pop_next_value() == (False, None) def test_could_close_json_after_many_blank_fragments_is_not_quadratic():