From 902a93d10d2159f66b9f8b5cacbde527d50e88b0 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Tue, 4 Aug 2026 16:51:21 -0700 Subject: [PATCH 01/60] fix(router): honor ServiceUnavailableErrorRetries and InternalServerErrorRetries in retry policy --- litellm/router_utils/get_retry_from_policy.py | 8 ++ litellm/types/router.py | 1 + .../test_get_retry_from_policy.py | 102 ++++++++++++++++++ tests/test_litellm/test_router.py | 31 ++++++ .../components/ModelRetrySettingsTab.tsx | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 6 files changed, 145 insertions(+) create mode 100644 tests/test_litellm/router_utils/test_get_retry_from_policy.py diff --git a/litellm/router_utils/get_retry_from_policy.py b/litellm/router_utils/get_retry_from_policy.py index 1645e6776fc..fbcc3de83c0 100644 --- a/litellm/router_utils/get_retry_from_policy.py +++ b/litellm/router_utils/get_retry_from_policy.py @@ -8,7 +8,9 @@ from litellm.exceptions import ( AuthenticationError, BadRequestError, ContentPolicyViolationError, + InternalServerError, RateLimitError, + ServiceUnavailableError, Timeout, ) from litellm.types.router import RetryPolicy @@ -26,6 +28,8 @@ def get_num_retries_from_retry_policy( TimeoutErrorRetries: Optional[int] = None RateLimitErrorRetries: Optional[int] = None ContentPolicyViolationErrorRetries: Optional[int] = None + InternalServerErrorRetries: Optional[int] = None + ServiceUnavailableErrorRetries: Optional[int] = None """ # if we can find the exception then in the retry policy -> return the number of retries @@ -48,6 +52,10 @@ def get_num_retries_from_retry_policy( and retry_policy.ContentPolicyViolationErrorRetries is not None ): return retry_policy.ContentPolicyViolationErrorRetries + if isinstance(exception, ServiceUnavailableError) and retry_policy.ServiceUnavailableErrorRetries is not None: + return retry_policy.ServiceUnavailableErrorRetries + if isinstance(exception, InternalServerError) and retry_policy.InternalServerErrorRetries is not None: + return retry_policy.InternalServerErrorRetries if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None: return retry_policy.BadRequestErrorRetries diff --git a/litellm/types/router.py b/litellm/types/router.py index 21bed84a3a1..e0952d9dd02 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -95,6 +95,7 @@ class RetryPolicy(BaseModel): RateLimitErrorRetries: Optional[int] = None ContentPolicyViolationErrorRetries: Optional[int] = None InternalServerErrorRetries: Optional[int] = None + ServiceUnavailableErrorRetries: Optional[int] = None class UpdateRouterConfig(BaseModel): diff --git a/tests/test_litellm/router_utils/test_get_retry_from_policy.py b/tests/test_litellm/router_utils/test_get_retry_from_policy.py new file mode 100644 index 00000000000..a5e239b8595 --- /dev/null +++ b/tests/test_litellm/router_utils/test_get_retry_from_policy.py @@ -0,0 +1,102 @@ +import litellm +from litellm.router_utils.get_retry_from_policy import ( + get_num_retries_from_retry_policy, +) +from litellm.types.router import RetryPolicy + + +def _service_unavailable_error() -> litellm.ServiceUnavailableError: + return litellm.ServiceUnavailableError( + message="model is down", + llm_provider="openai", + model="gpt-5.6", + ) + + +def _internal_server_error() -> litellm.InternalServerError: + return litellm.InternalServerError( + message="upstream 500", + llm_provider="openai", + model="gpt-5.6", + ) + + +def test_service_unavailable_error_retries_honored(): + policy = RetryPolicy(ServiceUnavailableErrorRetries=0) + + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + retry_policy=policy, + ) + == 0 + ) + + +def test_service_unavailable_error_retries_nonzero(): + policy = RetryPolicy(ServiceUnavailableErrorRetries=4) + + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + retry_policy=policy, + ) + == 4 + ) + + +def test_internal_server_error_retries_honored(): + policy = RetryPolicy(InternalServerErrorRetries=0) + + assert ( + get_num_retries_from_retry_policy( + exception=_internal_server_error(), + retry_policy=policy, + ) + == 0 + ) + + +def test_service_unavailable_not_covered_by_internal_server_error_retries(): + policy = RetryPolicy(InternalServerErrorRetries=0) + + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + retry_policy=policy, + ) + is None + ) + + +def test_internal_server_error_not_covered_by_service_unavailable_retries(): + policy = RetryPolicy(ServiceUnavailableErrorRetries=0) + + assert ( + get_num_retries_from_retry_policy( + exception=_internal_server_error(), + retry_policy=policy, + ) + is None + ) + + +def test_service_unavailable_error_retries_from_dict_policy(): + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + retry_policy={"ServiceUnavailableErrorRetries": 0}, + ) + == 0 + ) + + +def test_service_unavailable_error_retries_from_model_group_policy(): + assert ( + get_num_retries_from_retry_policy( + exception=_service_unavailable_error(), + model_group="gpt-5.6", + model_group_retry_policy={"gpt-5.6": RetryPolicy(ServiceUnavailableErrorRetries=1)}, + ) + == 1 + ) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 46b5ce65c3f..c98be9c7d80 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -6574,3 +6574,34 @@ def test_model_info_is_active_for_environment_matrix(monkeypatch): monkeypatch.delenv("LITELLM_ENVIRONMENT") with pytest.raises(ValueError, match="LITELLM_ENVIRONMENT"): model_info_is_active_for_environment(model_info={"supported_environments": ["production"]}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy_retries,expected_calls", [(0, 1), (1, 2)]) +async def test_router_retry_policy_service_unavailable_retries(policy_retries, expected_calls): + from litellm.types.router import RetryPolicy + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, + } + ], + retry_policy=RetryPolicy(ServiceUnavailableErrorRetries=policy_retries), + disable_cooldowns=True, + ) + + error = litellm.ServiceUnavailableError( + message="model is down", + llm_provider="openai", + model="gpt-5.6", + ) + with patch.object(litellm, "acompletion", AsyncMock(side_effect=error)) as mock_acompletion: + with pytest.raises(litellm.ServiceUnavailableError): + await router.acompletion( + model="gpt-5.6", + messages=[{"role": "user", "content": "hi"}], + ) + + assert mock_acompletion.call_count == expected_calls diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx index a4e3c4b958c..a9e0b8eb051 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx @@ -30,6 +30,7 @@ const retryPolicyMap: Record = { "RateLimitError (429)": "RateLimitErrorRetries", "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", "InternalServerError (500)": "InternalServerErrorRetries", + "ServiceUnavailableError (503)": "ServiceUnavailableErrorRetries", }; const ModelRetrySettingsTab = ({ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9133bfb5cf4..cf20f86a28d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30947,6 +30947,8 @@ export interface components { InternalServerErrorRetries?: number | null; /** Ratelimiterrorretries */ RateLimitErrorRetries?: number | null; + /** Serviceunavailableerrorretries */ + ServiceUnavailableErrorRetries?: number | null; /** Timeouterrorretries */ TimeoutErrorRetries?: number | null; }; From 7b181ef1978cfc5d4169ff1397ebef65229bb595 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 21:53:25 +0000 Subject: [PATCH 02/60] fix(responses/mcp): make MCP follow-up calls stateless when store=false Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/main.py | 9 +- .../mcp/litellm_proxy_mcp_handler.py | 31 +++- .../responses/mcp/mcp_streaming_iterator.py | 7 + .../mcp/test_litellm_proxy_mcp_handler.py | 149 ++++++++++++++++++ .../mcp/test_mcp_streaming_iterator.py | 78 +++++++++ 5 files changed, 270 insertions(+), 4 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e0af363b1a5..0f6119ce94c 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -326,8 +326,13 @@ async def aresponses_api_with_mcp( ) if tool_results: + persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) + follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( - response=response, tool_results=tool_results, original_input=input + response=response, + tool_results=tool_results, + original_input=input, + preserve_reasoning=persistence_disabled, ) # Prepare parameters for follow-up call (restores original stream setting) @@ -346,7 +351,7 @@ async def aresponses_api_with_mcp( follow_up_input=follow_up_input, model=model, all_tools=all_tools, - response_id=response.id, + response_id=None if persistence_disabled else response.id, **follow_up_call_params, ) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index c6e17502e5d..6271c5888db 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -951,11 +951,30 @@ class LiteLLM_Proxy_MCP_Handler: return follow_up_messages + @staticmethod + def _is_persistence_disabled(call_params: Mapping[str, object]) -> bool: + """Whether the caller opted out of server-side response persistence (store=false). + + Zero data retention callers send store=false, so the provider never persisted the + first response and previous_response_id cannot be used to link the follow-up call. + """ + return call_params.get("store") is False + + @staticmethod + def _extract_reasoning_items(response: ResponsesAPIResponse) -> tuple[Mapping[str, object], ...]: + """Reasoning output items, kept whole so reasoning.encrypted_content survives replay.""" + normalized: Final = tuple( + output_item if isinstance(output_item, dict) else output_item.model_dump(exclude_none=True) + for output_item in response.output + ) + return tuple(item for item in normalized if item.get("type") == "reasoning") + @staticmethod def _create_follow_up_input( response: ResponsesAPIResponse, tool_results: Sequence[Mapping[str, object]], original_input: str | ResponseInputParam | None = None, + preserve_reasoning: bool = False, ) -> list[object]: """Create follow-up input with tool results in proper format.""" follow_up_input: Final[list[object]] = [] @@ -1013,6 +1032,10 @@ class LiteLLM_Proxy_MCP_Handler: } ) + # Reasoning items must precede the function calls they produced + if preserve_reasoning: + follow_up_input.extend(LiteLLM_Proxy_MCP_Handler._extract_reasoning_items(response)) + # Add function calls (these can come directly after user message for LLM) for function_call in function_calls: follow_up_input.append(function_call) @@ -1034,10 +1057,14 @@ class LiteLLM_Proxy_MCP_Handler: follow_up_input: list[Any], model: str, all_tools: Sequence[ResponsesToolParam] | None, - response_id: str, + response_id: str | None, **call_params: Any, ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: - """Make follow-up response API call with tool results.""" + """Make follow-up response API call with tool results. + + response_id is None for stateless (store=false) requests, where the whole prior + turn is replayed in follow_up_input instead of linked by previous_response_id. + """ return await aresponses( input=follow_up_input, model=model, diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 186852f91c2..f1560e7ec84 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -774,10 +774,15 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): try: # Create follow-up input if self.collected_response is not None: + persistence_disabled: Final = LiteLLM_Proxy_MCP_Handler._is_persistence_disabled( + self.original_request_params + ) + follow_up_input: Final = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( response=self.collected_response, tool_results=self.tool_results, original_input=self.original_request_params.get("input"), + preserve_reasoning=persistence_disabled, ) # Make follow-up call with streaming @@ -788,6 +793,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): "stream": True, } ) + if persistence_disabled: + follow_up_params.pop("previous_response_id", None) else: return # Remove tool_choice to avoid forcing more tool calls diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index d60fff66c44..f08666f7b0d 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -9,10 +9,13 @@ from fastapi import HTTPException import importlib from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing +from litellm.responses import main as responses_main +from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_module from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from typing import Any, cast +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse from litellm.types.responses.main import OutputFunctionToolCall @@ -648,3 +651,149 @@ def test_extract_tool_call_details_still_prefers_openai_arguments(): assert name == "get_weather" assert call_id == "call_123" assert arguments == '{"city": "Paris"}' + + +def _response_with_reasoning_and_tool_call() -> Any: + """A first-turn response as a reasoning model returns it: reasoning item, then a function call.""" + return ResponsesAPIResponse( + id="resp_first", + created_at=1234567890, + model="gpt-5", + object="response", + status="completed", + output=[ + { + "type": "reasoning", + "id": "rs_1", + "summary": [], + "encrypted_content": "gAAAAA-opaque-blob", + }, + { + "type": "function_call", + "id": "fc_1", + "call_id": "call-1", + "name": "foo", + "arguments": "{}", + "status": "completed", + }, + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + + +def test_create_follow_up_input_preserves_reasoning_when_stateless(): + """ + Regression test (LIT-5427): a store=false follow-up has to replay the reasoning + item, including reasoning.encrypted_content, since the provider kept no state. + """ + follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=_response_with_reasoning_and_tool_call(), + tool_results=[{"tool_call_id": "call-1", "name": "foo", "result": "done"}], + original_input="hi", + preserve_reasoning=True, + ) + + assert follow_up[1] == { + "type": "reasoning", + "id": "rs_1", + "summary": [], + "encrypted_content": "gAAAAA-opaque-blob", + } + # the reasoning item has to come before the function call it produced + assert follow_up[2] == { + "type": "function_call", + "call_id": "call-1", + "name": "foo", + "arguments": "{}", + } + assert follow_up[3] == { + "type": "function_call_output", + "call_id": "call-1", + "output": "done", + } + + +def test_create_follow_up_input_omits_reasoning_when_stateful(): + """With store=true the provider still holds the reasoning item, so don't resend it.""" + follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=_response_with_reasoning_and_tool_call(), + tool_results=[{"tool_call_id": "call-1", "name": "foo", "result": "done"}], + original_input="hi", + ) + + assert not [item for item in follow_up if isinstance(item, dict) and item.get("type") == "reasoning"] + + +@pytest.mark.parametrize( + "call_params, expected", + [ + ({"store": False}, True), + ({"store": True}, False), + ({"store": None}, False), + ({}, False), + ], +) +def test_is_persistence_disabled(call_params: dict[str, Any], expected: bool): + assert LiteLLM_Proxy_MCP_Handler._is_persistence_disabled(call_params) is expected + + +@pytest.mark.parametrize( + "store, expected_previous_response_id", + [(False, None), (True, "resp_first")], +) +@pytest.mark.asyncio +async def test_mcp_follow_up_call_is_stateless_when_store_is_false( + monkeypatch: pytest.MonkeyPatch, store: bool, expected_previous_response_id: str | None +): + """ + Regression test (LIT-5427): linking the MCP follow-up call with + previous_response_id fails for zero data retention callers, because store=false + means the first response was never persisted. + """ + captured_calls: list[dict[str, Any]] = [] + first_response = _response_with_reasoning_and_tool_call() + + async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse: + captured_calls.append(kwargs) + return first_response if len(captured_calls) == 1 else ResponsesAPIResponse( + id="resp_follow_up", + created_at=1234567891, + model="gpt-5", + object="response", + status="completed", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + + async def fake_process(**kwargs: Any) -> tuple[list[Any], dict[str, str]]: + return ([], {"foo": "litellm_proxy"}) + + async def fake_execute(**kwargs: Any) -> list[dict[str, Any]]: + return [{"tool_call_id": "call-1", "name": "foo", "result": "done"}] + + monkeypatch.setattr(responses_main, "aresponses", fake_aresponses) + monkeypatch.setattr(mcp_handler_module, "aresponses", fake_aresponses) + monkeypatch.setattr( + LiteLLM_Proxy_MCP_Handler, "_process_mcp_tools_without_openai_transform", staticmethod(fake_process) + ) + monkeypatch.setattr(LiteLLM_Proxy_MCP_Handler, "_execute_tool_calls", staticmethod(fake_execute)) + + await responses_main.aresponses_api_with_mcp( + input="hi", + model="gpt-5", + tools=[{"type": "mcp", "server_url": "litellm_proxy", "require_approval": "never"}], + store=store, + ) + + assert len(captured_calls) == 2 + follow_up_call = captured_calls[1] + assert follow_up_call["previous_response_id"] == expected_previous_response_id + + reasoning_items = [ + item for item in follow_up_call["input"] if isinstance(item, dict) and item.get("type") == "reasoning" + ] + assert bool(reasoning_items) is (store is False) diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index 24edf12fffe..040ee26d796 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -257,3 +257,81 @@ async def test_initial_call_failure_is_stashed_for_eager_reraise(monkeypatch): assert iterator._initial_creation_error is not None assert "initial boom" in str(iterator._initial_creation_error) + + +def _reasoning_item(encrypted_content: str): + return {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": encrypted_content} + + +@pytest.mark.asyncio +async def test_streaming_follow_up_is_stateless_when_store_is_false(monkeypatch): + """ + Regression test (LIT-5427): with store=false the provider persisted nothing, so the + streaming follow-up must drop previous_response_id and replay the reasoning item + (carrying reasoning.encrypted_content) instead of pointing at a response id. + """ + _mock_mcp_environment(monkeypatch) + + aresponses_mock = AsyncMock(side_effect=[_text_only_stream("done")]) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = MCPEnhancedStreamingIterator( + base_iterator=_FakeAsyncStream( + [ + _output_item_added_chunk(), + _completed_chunk([_reasoning_item("gAAAAA-opaque-blob"), _function_call("call_1", "read_wiki_contents")]), + ] + ), + mcp_events=[], + tool_server_map={"read_wiki_contents": "deepwiki"}, + mcp_tools_with_litellm_proxy=[{"require_approval": "never"}], + user_api_key_auth=None, + original_request_params={ + "model": "gpt-5", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + "store": False, + "previous_response_id": "resp_prev", + }, + ) + + _ = [chunk async for chunk in iterator] + + assert aresponses_mock.call_count == 1 + follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs + assert "previous_response_id" not in follow_up_kwargs + assert _reasoning_item("gAAAAA-opaque-blob") in follow_up_kwargs["input"] + + +@pytest.mark.asyncio +async def test_streaming_follow_up_keeps_previous_response_id_when_stored(monkeypatch): + """The stateful default is unchanged: previous_response_id still links the follow-up.""" + _mock_mcp_environment(monkeypatch) + + aresponses_mock = AsyncMock(side_effect=[_text_only_stream("done")]) + monkeypatch.setattr(responses_main_module, "aresponses", aresponses_mock) + + iterator = MCPEnhancedStreamingIterator( + base_iterator=_FakeAsyncStream( + [ + _output_item_added_chunk(), + _completed_chunk([_reasoning_item("gAAAAA-opaque-blob"), _function_call("call_1", "read_wiki_contents")]), + ] + ), + mcp_events=[], + tool_server_map={"read_wiki_contents": "deepwiki"}, + mcp_tools_with_litellm_proxy=[{"require_approval": "never"}], + user_api_key_auth=None, + original_request_params={ + "model": "gpt-5", + "input": "what is berriai/litellm?", + "tools": [{"type": "mcp"}], + "previous_response_id": "resp_prev", + }, + ) + + _ = [chunk async for chunk in iterator] + + follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs + assert follow_up_kwargs["previous_response_id"] == "resp_prev" + assert not [item for item in follow_up_kwargs["input"] if item.get("type") == "reasoning"] From fae91c2a1137c57f64106ceec02c1a6d8ad3938f Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 22:27:41 +0000 Subject: [PATCH 03/60] chore(responses/mcp): drop explanatory comments per repo policy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/mcp/litellm_proxy_mcp_handler.py | 1 - .../test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py | 1 - 2 files changed, 2 deletions(-) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 6271c5888db..33ca99c1a7e 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1032,7 +1032,6 @@ class LiteLLM_Proxy_MCP_Handler: } ) - # Reasoning items must precede the function calls they produced if preserve_reasoning: follow_up_input.extend(LiteLLM_Proxy_MCP_Handler._extract_reasoning_items(response)) diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index f08666f7b0d..dbb3ad9fd44 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -701,7 +701,6 @@ def test_create_follow_up_input_preserves_reasoning_when_stateless(): "summary": [], "encrypted_content": "gAAAAA-opaque-blob", } - # the reasoning item has to come before the function call it produced assert follow_up[2] == { "type": "function_call", "call_id": "call-1", From e5582b65c9b69adf636c8a9c739a9fa1b96e01a7 Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 28 Aug 2026 20:33:53 +0000 Subject: [PATCH 04/60] fix(team_endpoints): let member_delete clear a team left on the user row Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/team_endpoints.py | 38 +++--- .../test_team_endpoints.py | 118 ++++++++++++++++++ 2 files changed, 138 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c6d7975b75e..88c58b8887a 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3303,9 +3303,6 @@ async def team_member_delete( data=data, ) - if not removed_team_members: - raise HTTPException(status_code=400, detail={"error": "User not found in team"}) - existing_team_row.members_with_roles = new_team_members _db_new_team_members: Final[list[dict]] = [m.model_dump() for m in new_team_members] @@ -3313,17 +3310,22 @@ async def team_member_delete( ## DELETE TEAM ID from USER ROW, IF EXISTS ## # get user row removed_user_ids: Final = frozenset(m.user_id for m in removed_team_members if m.user_id is not None) + addressed_user_ids: Final = removed_user_ids.union((data.user_id,) if data.user_id is not None else ()) key_val: Final[Mapping[str, object]] = ( - {"user_id": {"in": sorted(removed_user_ids)}} if removed_user_ids else {"user_email": data.user_email} + {"user_id": {"in": sorted(addressed_user_ids)}} if addressed_user_ids else {"user_email": data.user_email} ) member_tx: Final[_MemberDeleteTx] = tx existing_user_rows: Final = await member_tx.litellm_usertable.find_many(where=key_val) # Also clean up any existing team membership rows for this user and team - user_ids_to_delete: Final = removed_user_ids.union( - (data.user_id,) if data.user_id is not None else (), - (user.user_id for user in existing_user_rows if user.user_id), - ) + user_ids_to_delete: Final = addressed_user_ids.union(user.user_id for user in existing_user_rows if user.user_id) + + # A user row can outlive its roster entry, and until the team is off user.teams the user + # still sees it and still fails key creation against it, so removal has to clear it too + stale_user_rows: Final = tuple(user for user in existing_user_rows if data.team_id in user.teams) + + if not removed_team_members and not stale_user_rows: + raise HTTPException(status_code=400, detail={"error": "User not found in team"}) ## DELETE KEYS CREATED BY USER FOR THIS TEAM # Fetch keys before deletion so their audit records can be persisted alongside the delete. @@ -3335,17 +3337,17 @@ async def team_member_delete( } ) - await _team_tx_db(tx).update( - where={"team_id": data.team_id}, - data={"members_with_roles": json.dumps(_db_new_team_members)}, - ) + if removed_team_members: + await _team_tx_db(tx).update( + where={"team_id": data.team_id}, + data={"members_with_roles": json.dumps(_db_new_team_members)}, + ) - for existing_user in existing_user_rows: - if data.team_id in existing_user.teams: - await tx.litellm_usertable.update( - where={"user_id": existing_user.user_id}, - data={"teams": {"set": [team for team in existing_user.teams if team != data.team_id]}}, - ) + for existing_user in stale_user_rows: + await tx.litellm_usertable.update( + where={"user_id": existing_user.user_id}, + data={"teams": {"set": [team for team in existing_user.teams if team != data.team_id]}}, + ) for _uid in sorted(user_ids_to_delete): await tx.litellm_teammembership.delete_many(where={"team_id": data.team_id, "user_id": _uid}) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index ffa6bc601e9..e5b58f4b38e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -4775,6 +4775,124 @@ async def test_team_member_delete_by_email_the_user_row_does_not_carry( ) +@pytest.mark.asyncio +async def test_team_member_delete_clears_team_left_on_the_user_row_without_a_roster_entry( + mock_db_client, mock_admin_auth +): + """ + A user row can keep a team (several times over, from older duplicate-prone adds) after the + roster entry is gone, which leaves the team listed on the user, offered in the key creation + dropdown, and rejected by key creation itself. Reporting "User not found in team" left that + residue unremovable, so the delete now cleans every copy of the team off the user row. + """ + from litellm.proxy._types import TeamMemberDeleteRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-orphan-123" + test_user_id = "user-del-orphan-123" + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_team_row + ) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + mock_user_row = MagicMock() + mock_user_row.user_id = test_user_id + mock_user_row.user_email = None + mock_user_row.teams = [test_team_id, "other-team", test_team_id] + mock_db_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[mock_user_row] + ) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock( + return_value=MagicMock() + ) + + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock( + return_value=MagicMock() + ) + + _wire_member_delete_tx(mock_db_client) + + await team_member_delete( + data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), + user_api_key_dict=mock_admin_auth, + ) + + mock_db_client.db.litellm_usertable.update.assert_awaited_once_with( + where={"user_id": test_user_id}, + data={"teams": {"set": ["other-team"]}}, + ) + mock_db_client.db.litellm_teammembership.delete_many.assert_awaited_once_with( + where={"team_id": test_team_id, "user_id": test_user_id} + ) + + +@pytest.mark.asyncio +async def test_team_member_delete_still_rejects_a_user_the_team_has_no_trace_of( + mock_db_client, mock_admin_auth +): + from litellm.proxy._types import TeamMemberDeleteRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-absent-123" + test_user_id = "user-del-absent-123" + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_team_row + ) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + mock_user_row = MagicMock() + mock_user_row.user_id = test_user_id + mock_user_row.user_email = None + mock_user_row.teams = ["other-team"] + mock_db_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[mock_user_row] + ) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock( + return_value=MagicMock() + ) + + _wire_member_delete_tx(mock_db_client) + + with pytest.raises(HTTPException) as exc_info: + await team_member_delete( + data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), + user_api_key_dict=mock_admin_auth, + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.detail == {"error": "User not found in team"} + mock_db_client.db.litellm_usertable.update.assert_not_awaited() + mock_db_client.db.litellm_teammembership.delete_many.assert_not_awaited() + + class _InjectedMemberDeleteFailure(Exception): pass From df9b9f9ffb486561cd4cbbacffd2402b9b43ae16 Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 28 Aug 2026 21:12:05 +0000 Subject: [PATCH 05/60] style(team_endpoints): apply ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/management_endpoints/team_endpoints.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 88c58b8887a..fce109c0c8b 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3318,7 +3318,9 @@ async def team_member_delete( existing_user_rows: Final = await member_tx.litellm_usertable.find_many(where=key_val) # Also clean up any existing team membership rows for this user and team - user_ids_to_delete: Final = addressed_user_ids.union(user.user_id for user in existing_user_rows if user.user_id) + user_ids_to_delete: Final = addressed_user_ids.union( + user.user_id for user in existing_user_rows if user.user_id + ) # A user row can outlive its roster entry, and until the team is off user.teams the user # still sees it and still fails key creation against it, so removal has to clear it too From f43a93eaad188543e1f9124b8699ef5d21ea6499 Mon Sep 17 00:00:00 2001 From: milan Date: Mon, 31 Aug 2026 22:22:42 +0000 Subject: [PATCH 06/60] fix(team_endpoints): only widen cleanup to the requested user when the roster is empty Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/team_endpoints.py | 4 +- .../test_team_endpoints.py | 102 ++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index fce109c0c8b..7112c6ec40f 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3310,7 +3310,9 @@ async def team_member_delete( ## DELETE TEAM ID from USER ROW, IF EXISTS ## # get user row removed_user_ids: Final = frozenset(m.user_id for m in removed_team_members if m.user_id is not None) - addressed_user_ids: Final = removed_user_ids.union((data.user_id,) if data.user_id is not None else ()) + addressed_user_ids: Final = ( + removed_user_ids if removed_team_members else frozenset((data.user_id,) if data.user_id is not None else ()) + ) key_val: Final[Mapping[str, object]] = ( {"user_id": {"in": sorted(addressed_user_ids)}} if addressed_user_ids else {"user_email": data.user_email} ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index e5b58f4b38e..973a37ccc06 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -4893,6 +4893,108 @@ async def test_team_member_delete_still_rejects_a_user_the_team_has_no_trace_of( mock_db_client.db.litellm_teammembership.delete_many.assert_not_awaited() +@pytest.mark.asyncio +async def test_team_member_delete_leaves_a_bystander_named_by_a_conflicting_user_id_alone( + mock_db_client, mock_admin_auth +): + """ + A request can carry a user_id and a user_email that point at two different people, and only the + email matches a roster entry. Cleaning up both ids would strip the team, the membership row and + the keys off the bystander the roster never listed, so the user_id only widens the cleanup when + the roster came back empty. + """ + from litellm.proxy._types import TeamMemberDeleteRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-conflict-123" + roster_user_id = "user-del-conflict-roster" + bystander_user_id = "user-del-conflict-bystander" + roster_email = "roster@example.com" + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [ + {"user_id": roster_user_id, "user_email": roster_email, "role": "user"} + ], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_team_row + ) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + roster_user_row = MagicMock() + roster_user_row.user_id = roster_user_id + roster_user_row.user_email = roster_email + roster_user_row.teams = [test_team_id] + + bystander_user_row = MagicMock() + bystander_user_row.user_id = bystander_user_id + bystander_user_row.user_email = "bystander@example.com" + bystander_user_row.teams = [test_team_id] + + rows_by_user_id = { + roster_user_id: roster_user_row, + bystander_user_id: bystander_user_row, + } + + async def find_user_rows(where): + user_id_filter = where.get("user_id") + if isinstance(user_id_filter, dict): + return [ + rows_by_user_id[uid] + for uid in user_id_filter.get("in", []) + if uid in rows_by_user_id + ] + return [ + row + for row in rows_by_user_id.values() + if row.user_email == where.get("user_email") + ] + + mock_db_client.db.litellm_usertable.find_many = AsyncMock( + side_effect=find_user_rows + ) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock( + return_value=MagicMock() + ) + + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock( + return_value=MagicMock() + ) + + _wire_member_delete_tx(mock_db_client) + + await team_member_delete( + data=TeamMemberDeleteRequest( + team_id=test_team_id, + user_id=bystander_user_id, + user_email=roster_email, + ), + user_api_key_dict=mock_admin_auth, + ) + + mock_db_client.db.litellm_usertable.update.assert_awaited_once_with( + where={"user_id": roster_user_id}, + data={"teams": {"set": []}}, + ) + mock_db_client.db.litellm_teammembership.delete_many.assert_awaited_once_with( + where={"team_id": test_team_id, "user_id": roster_user_id} + ) + mock_db_client.db.litellm_verificationtoken.delete_many.assert_awaited_once_with( + where={"user_id": {"in": [roster_user_id]}, "team_id": test_team_id} + ) + + class _InjectedMemberDeleteFailure(Exception): pass From 84fad12f8546851c68c8471f14b221274a04038a Mon Sep 17 00:00:00 2001 From: milan Date: Mon, 31 Aug 2026 22:37:30 +0000 Subject: [PATCH 07/60] fix(team_endpoints): keep an email delete off the namesakes that never had the team Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/team_endpoints.py | 11 +-- .../test_team_endpoints.py | 76 +++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 7112c6ec40f..bbd9ae1fccd 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3319,15 +3319,16 @@ async def team_member_delete( member_tx: Final[_MemberDeleteTx] = tx existing_user_rows: Final = await member_tx.litellm_usertable.find_many(where=key_val) - # Also clean up any existing team membership rows for this user and team - user_ids_to_delete: Final = addressed_user_ids.union( - user.user_id for user in existing_user_rows if user.user_id - ) - # A user row can outlive its roster entry, and until the team is off user.teams the user # still sees it and still fails key creation against it, so removal has to clear it too stale_user_rows: Final = tuple(user for user in existing_user_rows if data.team_id in user.teams) + # Also clean up any existing team membership rows for this user and team. An email can + # match several user rows, so with no roster entry to name the member, only the rows + # actually carrying the team are the ones this request is allowed to touch + cleanup_user_rows: Final = existing_user_rows if removed_team_members else stale_user_rows + user_ids_to_delete: Final = addressed_user_ids.union(user.user_id for user in cleanup_user_rows if user.user_id) + if not removed_team_members and not stale_user_rows: raise HTTPException(status_code=400, detail={"error": "User not found in team"}) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 973a37ccc06..e54326d1afd 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -4995,6 +4995,82 @@ async def test_team_member_delete_leaves_a_bystander_named_by_a_conflicting_user ) +@pytest.mark.asyncio +async def test_team_member_delete_by_email_only_touches_the_row_carrying_the_stale_team( + mock_db_client, mock_admin_auth +): + """ + user_email is not unique, so an email delete against an empty roster can match several user + rows. Only the row that actually carries the team is stale; the namesake keeps its team, its + membership row and its keys. + """ + from litellm.proxy._types import TeamMemberDeleteRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-shared-email-123" + stale_user_id = "user-del-shared-email-stale" + namesake_user_id = "user-del-shared-email-namesake" + shared_email = "shared@example.com" + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_team_row + ) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + stale_user_row = MagicMock() + stale_user_row.user_id = stale_user_id + stale_user_row.user_email = shared_email + stale_user_row.teams = [test_team_id] + + namesake_user_row = MagicMock() + namesake_user_row.user_id = namesake_user_id + namesake_user_row.user_email = shared_email + namesake_user_row.teams = ["other-team"] + + mock_db_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[stale_user_row, namesake_user_row] + ) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock( + return_value=MagicMock() + ) + + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock( + return_value=MagicMock() + ) + + _wire_member_delete_tx(mock_db_client) + + await team_member_delete( + data=TeamMemberDeleteRequest(team_id=test_team_id, user_email=shared_email), + user_api_key_dict=mock_admin_auth, + ) + + mock_db_client.db.litellm_usertable.update.assert_awaited_once_with( + where={"user_id": stale_user_id}, + data={"teams": {"set": []}}, + ) + mock_db_client.db.litellm_teammembership.delete_many.assert_awaited_once_with( + where={"team_id": test_team_id, "user_id": stale_user_id} + ) + mock_db_client.db.litellm_verificationtoken.delete_many.assert_awaited_once_with( + where={"user_id": {"in": [stale_user_id]}, "team_id": test_team_id} + ) + + class _InjectedMemberDeleteFailure(Exception): pass From a616b8aaed87fcde79b0be3c4ab7776b35b5e42e Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 09:02:36 -0700 Subject: [PATCH 08/60] feat(vector_stores): add MongoDB Atlas vector store provider Atlas Vector Search has no HTTP query API, since the Data API and HTTPS Endpoints are end-of-life, so this provider extends BaseDirectVectorStoreConfig and runs the $vectorSearch aggregation through pymongo rather than shaping an httpx request. That is the same seam Valkey uses for RESP. vector_store_id names the Atlas Search index, matching Valkey, with the database and collection supplied through litellm_params. pymongo lives in a new optional `mongodb` extra and is imported lazily, so the base install still pulls no MongoDB driver. The floor is 4.17 because that is where dnspython became a core dependency instead of the `srv` extra, and Atlas issues mongodb+srv:// URIs that will not resolve without it. Clients are cached per connection rather than opened per search. Measured against Atlas, a fresh client costs ~890ms versus ~80ms warm, so copying the Valkey open-and-close-per-call pattern would have added ~810ms to every query. --- litellm/llms/mongodb/__init__.py | 0 litellm/llms/mongodb/common_utils.py | 163 +++++++++ .../llms/mongodb/vector_stores/__init__.py | 0 .../mongodb/vector_stores/transformation.py | 337 ++++++++++++++++++ litellm/types/utils.py | 1 + litellm/utils.py | 6 + pyproject.toml | 6 + uv.lock | 79 +++- 8 files changed, 590 insertions(+), 2 deletions(-) create mode 100644 litellm/llms/mongodb/__init__.py create mode 100644 litellm/llms/mongodb/common_utils.py create mode 100644 litellm/llms/mongodb/vector_stores/__init__.py create mode 100644 litellm/llms/mongodb/vector_stores/transformation.py diff --git a/litellm/llms/mongodb/__init__.py b/litellm/llms/mongodb/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py new file mode 100644 index 00000000000..7f26eadb08f --- /dev/null +++ b/litellm/llms/mongodb/common_utils.py @@ -0,0 +1,163 @@ +"""Shared helpers for MongoDB Atlas integrations. + +pymongo ships in the optional ``mongodb`` extra, so every import of it is +deferred to call time and raises an actionable error when it is absent. + +Clients are cached per connection because building one costs an SRV lookup, a +TLS handshake and topology discovery: measured at ~890ms against Atlas versus +~80ms on a warm client, so a client per search would dominate query latency. +""" + +import asyncio +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from pymongo import AsyncMongoClient, MongoClient + +PYMONGO_INSTALL_HINT: Final = ( + "The MongoDB vector store requires the 'pymongo' package. " + "Run 'pip install litellm[mongodb]' (or 'pip install pymongo') to install it." +) + +DEFAULT_CONNECT_TIMEOUT_MS: Final = 10_000 +DEFAULT_SOCKET_TIMEOUT_MS: Final = 30_000 +DEFAULT_SERVER_SELECTION_TIMEOUT_MS: Final = 10_000 + +_MAX_CACHED_CLIENTS: Final = 32 + +_APP_NAME: Final = "litellm" + + +@dataclass(frozen=True, slots=True) +class MongoClientKey: + connection_string: str + connect_timeout_ms: int + socket_timeout_ms: int + server_selection_timeout_ms: int + + +_sync_clients: dict[MongoClientKey, "MongoClient"] = {} # mutable-ok: process-level connection cache, see module docstring +_async_clients: dict[tuple[MongoClientKey, int], "AsyncMongoClient"] = {} # mutable-ok: same cache, keyed per event loop + + +def import_sync_mongo_client() -> "type[MongoClient]": + try: + from pymongo import MongoClient as SyncMongoClient + except ImportError as e: + raise ValueError(PYMONGO_INSTALL_HINT) from e + return SyncMongoClient + + +def import_async_mongo_client() -> "type[AsyncMongoClient]": + try: + from pymongo import AsyncMongoClient as AsyncMongoClientClass + except ImportError as e: + raise ValueError(PYMONGO_INSTALL_HINT) from e + return AsyncMongoClientClass + + +def _client_kwargs(key: MongoClientKey) -> dict[str, object]: + return { # mutable-ok: pymongo's client constructor takes keyword arguments + "connectTimeoutMS": key.connect_timeout_ms, + "socketTimeoutMS": key.socket_timeout_ms, + "serverSelectionTimeoutMS": key.server_selection_timeout_ms, + "appname": _APP_NAME, + } + + +def get_sync_client(key: MongoClientKey) -> "MongoClient": + cached: Final = _sync_clients.get(key) + if cached is not None: + return cached + client: Final = import_sync_mongo_client()(key.connection_string, **_client_kwargs(key)) + if len(_sync_clients) < _MAX_CACHED_CLIENTS: + _sync_clients[key] = client + return client + + +def get_async_client(key: MongoClientKey) -> "AsyncMongoClient": + """Async clients bind to the loop that created them, so the cache is keyed per loop.""" + loop_key: Final = (key, id(asyncio.get_running_loop())) + cached: Final = _async_clients.get(loop_key) + if cached is not None: + return cached + client: Final = import_async_mongo_client()(key.connection_string, **_client_kwargs(key)) + if len(_async_clients) < _MAX_CACHED_CLIENTS: + _async_clients[loop_key] = client + return client + + +def reset_client_cache() -> None: + _sync_clients.clear() + _async_clients.clear() + + +_AUTHENTICATION_FAILED_CODE: Final = 18 +_UNAUTHORIZED_CODE: Final = 13 + + +def _index_hint(index_name: str, database: str, collection: str) -> str: + return ( + f"No queryable Atlas Vector Search index named '{index_name}' was found on " + f"'{database}.{collection}'. Confirm the index exists on that exact collection, that its " + "status is READY rather than still building, and that the vector store id matches the index name." + ) + + +def translate_mongo_error(error: Exception, index_name: str, database: str, collection: str) -> Exception: + """Turn a driver failure into a message that names the misconfiguration, never a silent empty result. + + Returns the exception to raise so callers keep the original as ``__cause__``. + """ + try: + from pymongo.errors import ( + ConfigurationError, + ExecutionTimeout, + InvalidOperation, + NetworkTimeout, + OperationFailure, + ServerSelectionTimeoutError, + ) + except ImportError: + return error + + if isinstance(error, ServerSelectionTimeoutError): + return ValueError( + "Could not reach the MongoDB deployment before the timeout. On Atlas this is usually the " + "project's IP access list not containing this host, or a paused cluster; it can also be an " + f"unresolvable hostname. Driver detail: {error}" + ) + if isinstance(error, OperationFailure): + code: Final = error.code + if code in (_AUTHENTICATION_FAILED_CODE, _UNAUTHORIZED_CODE): + return ValueError( + "MongoDB rejected the credentials in mongodb_connection_string, or the database user " + f"lacks read access to '{database}.{collection}'. Driver detail: {error.details}" + ) + detail: Final = str(error).lower() + if "index" in detail and ("not found" in detail or "does not exist" in detail or "unknown" in detail): + return ValueError(f"{_index_hint(index_name, database, collection)} Driver detail: {error}") + if "dimension" in detail or "numdimensions" in detail or "queryvector" in detail: + return ValueError( + "The query embedding does not match the vector dimensions the Atlas index was built for. " + "litellm_embedding_model must be the same model that produced the stored vectors. " + f"Driver detail: {error}" + ) + return ValueError( + f"MongoDB rejected the vector search against '{database}.{collection}' using index " + f"'{index_name}'. Driver detail: {error}" + ) + if isinstance(error, (NetworkTimeout, ExecutionTimeout)): + return ValueError( + f"The MongoDB vector search against '{database}.{collection}' timed out before returning. " + f"Driver detail: {error}" + ) + if isinstance(error, ConfigurationError): + return ValueError( + "mongodb_connection_string is not a usable MongoDB connection string. " + f"Driver detail: {error}" + ) + if isinstance(error, InvalidOperation): + return ValueError(f"The MongoDB client was already closed or is unusable. Driver detail: {error}") + return error diff --git a/litellm/llms/mongodb/vector_stores/__init__.py b/litellm/llms/mongodb/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py new file mode 100644 index 00000000000..efeff16ae5a --- /dev/null +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -0,0 +1,337 @@ +"""MongoDB Atlas vector store provider. + +Atlas Vector Search has no HTTP query API (the Data API and HTTPS Endpoints are +end-of-life), so this config extends BaseDirectVectorStoreConfig and runs the +``$vectorSearch`` aggregation itself through pymongo instead of shaping an httpx +request. + +``vector_store_id`` is the Atlas Search index name, matching the Valkey provider +where the id names the index; the database and collection it covers come from +litellm_params. +""" + +from collections.abc import Awaitable, Callable, Mapping, Sequence +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, NoReturn + +import httpx +from pydantic import BaseModel, ConfigDict + +import litellm +from litellm.llms.base_llm.vector_store.transformation import BaseDirectVectorStoreConfig +from litellm.llms.mongodb.common_utils import ( + DEFAULT_CONNECT_TIMEOUT_MS, + DEFAULT_SERVER_SELECTION_TIMEOUT_MS, + DEFAULT_SOCKET_TIMEOUT_MS, + MongoClientKey, + get_async_client, + get_sync_client, + translate_mongo_error, +) +from litellm.types.utils import EmbeddingResponse +from litellm.types.vector_stores import ( + VectorStoreCreateOptionalRequestParams, + VectorStoreResultContent, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, + VectorStoreSearchResult, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +DEFAULT_EMBEDDING_FIELD_NAME: Final = "embedding" +DEFAULT_TEXT_FIELD_NAME: Final = "text" +SCORE_FIELD_NAME: Final = "score" + +DEFAULT_MAX_NUM_RESULTS: Final = 10 +MIN_MAX_NUM_RESULTS: Final = 1 +MAX_MAX_NUM_RESULTS: Final = 50 + +NUM_CANDIDATES_MULTIPLIER: Final = 10 +MIN_NUM_CANDIDATES: Final = 100 +MAX_NUM_CANDIDATES: Final = 10_000 + +MAX_QUERY_CHARACTERS: Final = 32_000 + +_EMPTY_EMBEDDING_CONFIG: Final = MappingProxyType({}) + +_SEARCH_ONLY_MESSAGE: Final = ( + "MongoDB vector store is search-only. Create the collection and its Atlas Vector Search " + "index in MongoDB directly, then register it here by index name." +) + + +class _MongoDBSearchParams(BaseModel): + """Typed view over the vector store's litellm_params; unrelated keys are ignored.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + litellm_embedding_model: str | None = None + litellm_embedding_config: Mapping[str, object] | None = None + mongodb_connection_string: str | None = None + mongodb_database: str | None = None + mongodb_collection: str | None = None + mongodb_text_field: str | None = None + mongodb_embedding_field: str | None = None + mongodb_num_candidates: int | None = None + + @property + def text_field(self) -> str: + return self.mongodb_text_field or DEFAULT_TEXT_FIELD_NAME + + @property + def embedding_field(self) -> str: + return self.mongodb_embedding_field or DEFAULT_EMBEDDING_FIELD_NAME + + def require_embedding_model(self) -> str: + if not self.litellm_embedding_model: + raise ValueError( + "litellm_embedding_model is required in litellm_params for the MongoDB vector store. " + "It must be the same model that produced the vectors stored in " + f"'{self.mongodb_collection or ''}.{self.embedding_field}', or search results " + "will be meaningless. Example: litellm_embedding_model: openai/text-embedding-3-small" + ) + return self.litellm_embedding_model + + def require_connection_string(self) -> str: + if not self.mongodb_connection_string: + raise ValueError( + "mongodb_connection_string is required in litellm_params for the MongoDB vector store. " + "Example: mongodb+srv://:@.mongodb.net" + ) + scheme: Final = self.mongodb_connection_string.split("://", 1)[0].lower() + if scheme not in ("mongodb", "mongodb+srv"): + raise ValueError( + "mongodb_connection_string must start with 'mongodb://' or 'mongodb+srv://', " + f"got '{self.mongodb_connection_string.split('://', 1)[0]}://'" + ) + return self.mongodb_connection_string + + def require_database(self) -> str: + if not self.mongodb_database: + raise ValueError( + "mongodb_database is required in litellm_params for the MongoDB vector store. " + "Example: mongodb_database: sample_mflix" + ) + return self.mongodb_database + + def require_collection(self) -> str: + if not self.mongodb_collection: + raise ValueError( + "mongodb_collection is required in litellm_params for the MongoDB vector store. " + "Example: mongodb_collection: embedded_movies" + ) + return self.mongodb_collection + + +class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): + def __init__( + self, + embedding_fn: Callable[..., EmbeddingResponse] | None = None, + aembedding_fn: Callable[..., Awaitable[EmbeddingResponse]] | None = None, + sync_client_factory: Callable[[MongoClientKey], object] | None = None, + async_client_factory: Callable[[MongoClientKey], object] | None = None, + ) -> None: + super().__init__() + self.embedding_fn = embedding_fn if embedding_fn is not None else litellm.embedding + self.aembedding_fn = aembedding_fn if aembedding_fn is not None else litellm.aembedding + self.sync_client_factory = sync_client_factory if sync_client_factory is not None else get_sync_client + self.async_client_factory = async_client_factory if async_client_factory is not None else get_async_client + + @staticmethod + def _query_text(query: str | Sequence[str]) -> str: + text: Final = query if isinstance(query, str) else " ".join(query) + if not text.strip(): + raise ValueError("query must not be empty") + if len(text) > MAX_QUERY_CHARACTERS: + raise ValueError(f"query must be at most {MAX_QUERY_CHARACTERS} characters, got {len(text)}") + return text + + @staticmethod + def _limit(vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams) -> int: + requested: Final = vector_store_search_optional_params.get("max_num_results") + if requested is None: + return DEFAULT_MAX_NUM_RESULTS + if not MIN_MAX_NUM_RESULTS <= requested <= MAX_MAX_NUM_RESULTS: + raise ValueError( + f"max_num_results must be between {MIN_MAX_NUM_RESULTS} and {MAX_MAX_NUM_RESULTS}, got {requested}" + ) + return requested + + @staticmethod + def _num_candidates(limit: int, configured: int | None) -> int: + if configured is not None: + if not limit <= configured <= MAX_NUM_CANDIDATES: + raise ValueError( + f"mongodb_num_candidates must be between max_num_results ({limit}) and " + f"{MAX_NUM_CANDIDATES}, got {configured}" + ) + return configured + return min(max(limit * NUM_CANDIDATES_MULTIPLIER, MIN_NUM_CANDIDATES), MAX_NUM_CANDIDATES) + + @staticmethod + def _client_key(params: _MongoDBSearchParams, timeout: float | httpx.Timeout | None) -> MongoClientKey: + if isinstance(timeout, httpx.Timeout): + connect_ms: Final = int((timeout.connect or DEFAULT_CONNECT_TIMEOUT_MS / 1000) * 1000) + socket_ms: Final = int((timeout.read or DEFAULT_SOCKET_TIMEOUT_MS / 1000) * 1000) + elif timeout is not None: + connect_ms = min(int(float(timeout) * 1000), DEFAULT_CONNECT_TIMEOUT_MS) + socket_ms = int(float(timeout) * 1000) + else: + connect_ms = DEFAULT_CONNECT_TIMEOUT_MS + socket_ms = DEFAULT_SOCKET_TIMEOUT_MS + return MongoClientKey( + connection_string=params.require_connection_string(), + connect_timeout_ms=connect_ms, + socket_timeout_ms=socket_ms, + server_selection_timeout_ms=min(connect_ms, DEFAULT_SERVER_SELECTION_TIMEOUT_MS), + ) + + @classmethod + def _pipeline( + cls, + vector_store_id: str, + query_vector: Sequence[float], + params: _MongoDBSearchParams, + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + ) -> list[dict[str, object]]: + if vector_store_search_optional_params.get("filters") is not None: + raise ValueError( + "MongoDB vector store does not support the filters parameter yet. " + "Restrict the collection or the Atlas Vector Search index definition instead." + ) + limit: Final = cls._limit(vector_store_search_optional_params) + return [ # mutable-ok: pymongo's aggregate contract is a list of stage dicts + { + "$vectorSearch": { + "index": vector_store_id, + "path": params.embedding_field, + "queryVector": list(query_vector), + "numCandidates": cls._num_candidates(limit, params.mongodb_num_candidates), + "limit": limit, + } + }, + {"$project": {params.text_field: 1, SCORE_FIELD_NAME: {"$meta": "vectorSearchScore"}}}, + ] + + @staticmethod + def _field_value(document: Mapping[str, object], dotted_path: str) -> str: + current: object = document + for segment in dotted_path.split("."): + if not isinstance(current, Mapping): + return "" + current = current.get(segment) + return "" if current is None else str(current) + + @classmethod + def _to_result(cls, document: Mapping[str, object], text_field: str) -> VectorStoreSearchResult: + document_id: Final = document.get("_id") + identifier: Final = None if document_id is None else str(document_id) + content: Final = [ # mutable-ok: VectorStoreSearchResult declares a list of content parts + VectorStoreResultContent(text=cls._field_value(document, text_field), type="text") + ] + raw_score: Final = document.get(SCORE_FIELD_NAME) + return VectorStoreSearchResult( + score=float(raw_score) if isinstance(raw_score, (int, float)) else None, + content=content, + file_id=identifier, + filename=identifier, + ) + + @classmethod + def _to_response( + cls, documents: Sequence[Mapping[str, object]], query_text: str, text_field: str + ) -> VectorStoreSearchResponse: + return VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query=query_text, + data=[cls._to_result(document, text_field) for document in documents], + ) + + @staticmethod + def _embedding_vector(embedding_response: EmbeddingResponse) -> Sequence[float]: + data: Final = embedding_response.data + if not data: + raise ValueError( + "The embedding model returned no embedding for the search query, so there is nothing " + "to search MongoDB with. Check the embedding deployment named by litellm_embedding_model." + ) + return data[0]["embedding"] + + def execute_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + litellm_logging_obj: "LiteLLMLoggingObj", + litellm_params: Mapping[str, object], + timeout: float | httpx.Timeout | None = None, + ) -> VectorStoreSearchResponse: + params: Final = _MongoDBSearchParams.model_validate(litellm_params) + query_text: Final = self._query_text(query) + key: Final = self._client_key(params, timeout) + database: Final = params.require_database() + collection: Final = params.require_collection() + + embedding_response: Final = self.embedding_fn( + model=params.require_embedding_model(), + input=[query_text], # mutable-ok: litellm.embedding's input contract is a list + **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + ) + pipeline: Final = self._pipeline( + vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params + ) + + client: Final = self.sync_client_factory(key) + try: + documents: Final = list(client[database][collection].aggregate(pipeline)) # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted + except Exception as e: + raise translate_mongo_error( + e, index_name=vector_store_id, database=database, collection=collection + ) from e + return self._to_response(documents, query_text, params.text_field) + + async def aexecute_search_vector_store_request( + self, + vector_store_id: str, + query: str | Sequence[str], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + litellm_logging_obj: "LiteLLMLoggingObj", + litellm_params: Mapping[str, object], + timeout: float | httpx.Timeout | None = None, + ) -> VectorStoreSearchResponse: + params: Final = _MongoDBSearchParams.model_validate(litellm_params) + query_text: Final = self._query_text(query) + key: Final = self._client_key(params, timeout) + database: Final = params.require_database() + collection: Final = params.require_collection() + + embedding_response: Final = await self.aembedding_fn( + model=params.require_embedding_model(), + input=[query_text], # mutable-ok: litellm.embedding's input contract is a list + **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + ) + pipeline: Final = self._pipeline( + vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params + ) + + client: Final = self.async_client_factory(key) + try: + cursor: Final = await client[database][collection].aggregate(pipeline) # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted + documents: Final = [document async for document in cursor] + except Exception as e: + raise translate_mongo_error( + e, index_name=vector_store_id, database=database, collection=collection + ) from e + return self._to_response(documents, query_text, params.text_field) + + def transform_create_vector_store_request( + self, + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, + api_base: str, + ) -> NoReturn: + raise NotImplementedError(_SEARCH_ONLY_MESSAGE) + + def transform_create_vector_store_response(self, response: httpx.Response) -> NoReturn: + raise NotImplementedError(_SEARCH_ONLY_MESSAGE) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5783a39b30c..fd98bbf4896 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3867,6 +3867,7 @@ class LlmProviders(str, Enum): PG_VECTOR = "pg_vector" S3_VECTORS = "s3_vectors" VALKEY = "valkey" + MONGODB = "mongodb" HELICONE = "helicone" HYPERBOLIC = "hyperbolic" RECRAFT = "recraft" diff --git a/litellm/utils.py b/litellm/utils.py index 252b6756937..2dbfe096a59 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9022,6 +9022,12 @@ class ProviderConfigManager: ) return ValkeyVectorStoreConfig() + elif litellm.LlmProviders.MONGODB == provider: + from litellm.llms.mongodb.vector_stores.transformation import ( + MongoDBVectorStoreConfig, + ) + + return MongoDBVectorStoreConfig() return None @staticmethod diff --git a/pyproject.toml b/pyproject.toml index 2866e27e84c..65d8886bc26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,12 @@ utils = [ ] caching = ["diskcache>=5.6.3,<6.0"] mcp = ["mcp>=1.28.1,<2.0"] +# Driver for the MongoDB Atlas vector store. Atlas Vector Search has no HTTP query +# API, so that provider talks to the cluster over the wire protocol. Imported lazily +# and kept out of the base install, which never needs a MongoDB driver. The floor is +# 4.17 because that is where dnspython became a core dependency rather than the `srv` +# extra, and Atlas hands out mongodb+srv:// URIs that do not resolve without it. +mongodb = ["pymongo>=4.17,<5.0"] # SAML SSO for the admin UI. python3-saml pulls in xmlsec/lxml, whose wheels # bundle the native libxmlsec1/libxml2 libraries, so no system packages are # required. Kept out of the base `proxy` extra so it stays optional. diff --git a/uv.lock b/uv.lock index 27be919eea1..e4b23804b4a 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-29T17:58:57.633306Z" +exclude-newer = "2026-08-30T07:50:56.793842Z" exclude-newer-span = "P3D" [manifest] @@ -4323,6 +4323,9 @@ mcp = [ mlflow = [ { name = "mlflow" }, ] +mongodb = [ + { name = "pymongo" }, +] proxy = [ { name = "apscheduler" }, { name = "azure-identity" }, @@ -4551,6 +4554,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.10.0,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.14.1,<3.0" }, { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.13.0,<3.0" }, + { name = "pymongo", marker = "extra == 'mongodb'", specifier = ">=4.17,<5.0" }, { name = "pynacl", marker = "extra == 'proxy'", specifier = ">=1.6.2,<2.0" }, { name = "pypdf", marker = "extra == 'proxy-runtime'", specifier = ">=6.12.0,<7.0" }, { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.8.16,<1.0" }, @@ -4577,7 +4581,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.21.0,<1.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, ] -provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] +provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "mongodb", "saml", "semantic-router", "mlflow", "grpc", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] [package.metadata.requires-dev] ci = [ @@ -7518,6 +7522,77 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/6f/9ac2548e290764781f9e7e2aaf0685b086379dabfb29ca38536985471eaf/pylint-4.0.5-py3-none-any.whl", hash = "sha256:00f51c9b14a3b3ae08cff6b2cdd43f28165c78b165b628692e428fb1f8dc2cf2", size = 536694, upload-time = "2026-02-20T09:07:31.028Z" }, ] +[[package]] +name = "pymongo" +version = "4.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/64/50be6fbac9c79fe2e4c17401a467da2d8764d82833d83cec325afe5cab32/pymongo-4.17.0.tar.gz", hash = "sha256:70ffa08ba641468cc068cf46c06b34f01a8ce3489f6411309fcb5ceabe6b2fc0", size = 2523370, upload-time = "2026-04-20T16:39:53.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/77/28ebbf69772a4341d530831c7a006cdb06877ac23075cb53b0a227df4fe1/pymongo-4.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47b021363cd923ace5edc7a1d63c0ff8a6d9d43859b8a1ba23645f5afae63221", size = 819234, upload-time = "2026-04-20T16:37:20.888Z" }, + { url = "https://files.pythonhosted.org/packages/88/cf/5a70cee503ff9a2fea20607607f14d189f4d975960ac0945ec306ee7b695/pymongo-4.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:422fa50d7d7f5c22ea0953554396c9ef95684a2d775f860bd75a7b510538dfca", size = 819969, upload-time = "2026-04-20T16:37:24.187Z" }, + { url = "https://files.pythonhosted.org/packages/23/d5/07b7e27e662c58d872efd104a0e8055eb6569aa1b6d4da436f3fdee7f897/pymongo-4.17.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:addd0498ebbdc6354227f6ed457ed9fce442d48a3bb30d5b5bad33e104996561", size = 1244510, upload-time = "2026-04-20T16:37:26.069Z" }, + { url = "https://files.pythonhosted.org/packages/fb/be/7cac5b1e89bd5a8e395067648241390321593a7c29243e36f91343c02a90/pymongo-4.17.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5c8e180cb2cabe37300e1e36c60aa4f2ff956cc579f0142135a5d2cba252243", size = 1263245, upload-time = "2026-04-20T16:37:28.003Z" }, + { url = "https://files.pythonhosted.org/packages/2e/20/40e8e99824c1fda18261411e65ce3b0cd3d9a6ed3c056cdd0a569adc870b/pymongo-4.17.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd835cdb37a1adec359dd072c24f8bb14809e2644fde86fab4ee2fc9719b9483", size = 1304113, upload-time = "2026-04-20T16:37:30.048Z" }, + { url = "https://files.pythonhosted.org/packages/3a/94/fb7e25441dd66f2069a9b172380849b0eaa5881c18b3db217bf64a6d393c/pymongo-4.17.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4979e7e8887862bbb44d203f00cc8263a3f27237876fa691b6beba23e40e6d8", size = 1297046, upload-time = "2026-04-20T16:37:32.054Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c9/7352e0c20fe772541556e4d283c05e07ec48f8b0d2737ad930ac4a1b6655/pymongo-4.17.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:77aa4bc164b4de60d5db193b322f0f5b6ead716e831031bfdef8e8bd92205556", size = 1265708, upload-time = "2026-04-20T16:37:33.934Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e4/3df15494c2015ed297958517f0e4f6493e21b00990748068a973e66d45e0/pymongo-4.17.0-cp310-cp310-win32.whl", hash = "sha256:48bbc576677b50af043df870d84ded67cc3a9b4aa7553201beef4da5dc050a0a", size = 805533, upload-time = "2026-04-20T16:37:35.744Z" }, + { url = "https://files.pythonhosted.org/packages/22/fa/b4e71bb8cb82ad7d21bb4e8c476f2d573ba68b20368aac36ef06e4a196b4/pymongo-4.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46767f28dea610e02edf6c5d956ce615c3c7790ea396660b9b1efd5c5ead2e0", size = 815677, upload-time = "2026-04-20T16:37:37.808Z" }, + { url = "https://files.pythonhosted.org/packages/22/e2/0a4bba644f1cda3970ea1012149eeae3594ebfeed3f81fdaf32b61d90c95/pymongo-4.17.0-cp310-cp310-win_arm64.whl", hash = "sha256:757f2a4c0c2c46cab87df0333681ce69e86c9d5b45bc5203ceba5410b3489e59", size = 807293, upload-time = "2026-04-20T16:37:39.707Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e2/336d86f221cf1b56b2ed9330d4a3b98f9f38f0b37829ae9a9184617d5419/pymongo-4.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4141e6c6a339789b2974efa00ecd9409101672d77a0e3ee2cc3839eedf8ec4df", size = 874668, upload-time = "2026-04-20T16:37:41.39Z" }, + { url = "https://files.pythonhosted.org/packages/34/8e/75d3c6c935d187ab59c61e9c15d9aab3f274b563eaf1706e8cae5f508dec/pymongo-4.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e68c76b84e0c132d9dbf9307f12ff8185702328187a87b9aca8c941303873433", size = 875294, upload-time = "2026-04-20T16:37:43.432Z" }, + { url = "https://files.pythonhosted.org/packages/5f/ec/62e855744489dbcd54fd778aae4d80fa4c4819e8fb228ca0cf6f21a03997/pymongo-4.17.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ba2195d4f386f839a52a23ea1cfd60ffaaba78a3d7841db51b7e433001139918", size = 1496233, upload-time = "2026-04-20T16:37:45.518Z" }, + { url = "https://files.pythonhosted.org/packages/82/e8/93e4e5e5ce8fdf8929dabeefe24aafa5ce046028eed0dfa8eeb936e72c49/pymongo-4.17.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8446ff4bfcb6ec2a2e50998c860986a1e992136f998b7f53e7a717fb8aa5a0b9", size = 1522927, upload-time = "2026-04-20T16:37:47.492Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ca/425dc1d21e0f17bdea0072fc463f662f7fa06d2852af52975c9eced3c07c/pymongo-4.17.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2a0d5ac205728c86e0a02192f1aa5f865b0d7d51f8df6101c01a69a7fc620d72", size = 1583468, upload-time = "2026-04-20T16:37:49.221Z" }, + { url = "https://files.pythonhosted.org/packages/b3/9d/f08b07eeffda1a43c1759f0fa625e88ae12360996eb56d42aad832fa7dff/pymongo-4.17.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:485c8a8eaa4c739f00a331fc73757898ee7c092c214a79e63866ff76aaf282ff", size = 1572787, upload-time = "2026-04-20T16:37:51.061Z" }, + { url = "https://files.pythonhosted.org/packages/e9/c2/6855a07aafa7b894929af23675b6fb9634800ce43122b76a62f6eeb8da2a/pymongo-4.17.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b2dfcc795f5b9fedbe179a11fdf6051581479d196582a3fe819a92a00e9b9969", size = 1526184, upload-time = "2026-04-20T16:37:53.358Z" }, + { url = "https://files.pythonhosted.org/packages/4e/05/c952bac7db71c1942ea3559fcd308b49754cc5004b455935fb4000d1f37b/pymongo-4.17.0-cp311-cp311-win32.whl", hash = "sha256:c2292144505fb12156b981bd440f3dc994a883da06ac726c0c8692ccdbc1c510", size = 852621, upload-time = "2026-04-20T16:37:55.28Z" }, + { url = "https://files.pythonhosted.org/packages/11/c0/c04da9f4c0c6252404598f4e394b862a58a9e866822a70ae261c8a018fdf/pymongo-4.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:2e190827834fce70ecdf9d46796c6dbc0ce08ea87dc2ff5bc6f3f5579b605cb9", size = 867852, upload-time = "2026-04-20T16:37:57.233Z" }, + { url = "https://files.pythonhosted.org/packages/1d/b2/c7b4870fbeef471e947d3e014676f5910d02e0197074d692ebcf24ec049a/pymongo-4.17.0-cp311-cp311-win_arm64.whl", hash = "sha256:a8f9c40a09bb7d4b9fc8b1da65ecf6efa79bda5cb2756f39d9b6940fac1d19ae", size = 855019, upload-time = "2026-04-20T16:37:58.983Z" }, + { url = "https://files.pythonhosted.org/packages/98/90/60bcb508840135d5ee46b51b1a950f548338aa8145a8366dbe6639ae51ac/pymongo-4.17.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53ffa94b2340dbf6b055e09a0090618c60482c158ecfc9565642fc996bf0944", size = 930529, upload-time = "2026-04-20T16:38:00.936Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e9/313840f1e52c6dfac47f704428cbfbce59956ebe7633bffc92b03f74f0ad/pymongo-4.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6fe0de9d0f6791abce3471230b32b4817bf89d27b1182b6a550e1ec0fa72aa9a", size = 930665, upload-time = "2026-04-20T16:38:02.915Z" }, + { url = "https://files.pythonhosted.org/packages/78/35/9d3565ea45b1606f635c1e2cd2563c28d66caafdc50f7ad7d979fcd1b363/pymongo-4.17.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e537e95514dae1aaa718f481ec03151a0f0394bcd05f1322896d8fc1330cb729", size = 1762369, upload-time = "2026-04-20T16:38:05.375Z" }, + { url = "https://files.pythonhosted.org/packages/95/ee/149b0d4b1a11c38bff6f14c23d5814c9b0843fd6dc38ad40596bdb1a62d2/pymongo-4.17.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37a8385c29881b43eab31f584100fa0eaddedd5607adf010147ba1810118be90", size = 1798044, upload-time = "2026-04-20T16:38:07.195Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d4/4cee4a7b8d8f6f0550ef6cd2fea42455c5ed619a220cb6ba4fb40d6a5bc8/pymongo-4.17.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f3ee3d241ed77a4fc99ce3cff3b289c3ebce37f61fdd7349d3592c23b82c8784", size = 1878567, upload-time = "2026-04-20T16:38:09.121Z" }, + { url = "https://files.pythonhosted.org/packages/45/ef/7fe366c84952619ee2f69973566c214775e083dd4df465751912153e4b72/pymongo-4.17.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9eb5d63a3c518cb0804ed678f5e2b875af032d89a7cf57a57360322cf6a4d222", size = 1864881, upload-time = "2026-04-20T16:38:10.896Z" }, + { url = "https://files.pythonhosted.org/packages/2f/35/b577d82c6d1be7aee7ac7e249bc86f7847998345042e5f8360de238e177b/pymongo-4.17.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e97e03fa13327c87e3fdc5656acd01e71817f0c1dc3221cd8f30de136bf4ec3", size = 1800349, upload-time = "2026-04-20T16:38:13.589Z" }, + { url = "https://files.pythonhosted.org/packages/b8/69/dafcf04f66e130ddd91aeb92e7a692480eda46dcd04ec1dbe82c06619e10/pymongo-4.17.0-cp312-cp312-win32.whl", hash = "sha256:6877214bff5f06f6884a9fc8d9016a4a7a5f51f537f5c51ac3a576f93e7dfb32", size = 900518, upload-time = "2026-04-20T16:38:15.541Z" }, + { url = "https://files.pythonhosted.org/packages/11/35/5c9262a459f988b4eb2605f70815240b77a0d4131136c4326d18f1822b89/pymongo-4.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:9828485f72f63c7d802e0ec41f71906f633c2692621ab3af55ca990186b091b1", size = 920335, upload-time = "2026-04-20T16:38:17.665Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/e9c7265ee176faccf4e52c4797837e794d93569a1046f6b19a4acc36e5ad/pymongo-4.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:1195370a77baf003b59b10e91ecc4706297197f0dd9d29c840cc556dc08f7cee", size = 903289, upload-time = "2026-04-20T16:38:19.33Z" }, + { url = "https://files.pythonhosted.org/packages/2a/6b/c1206879708b94e82fcd8b9653440ec271f79a3674d122192df383047f5a/pymongo-4.17.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:809ec74de3b9148ae43fa8df9faf53470f511c8d384f13b99d6f671f2a379f15", size = 985829, upload-time = "2026-04-20T16:38:21.031Z" }, + { url = "https://files.pythonhosted.org/packages/cb/cf/bb044ed85160e5c40f568c7c4f4e8ea16f40764ff5d302e5befbe8f6f814/pymongo-4.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a431b737816bf4cddd4fa0fcef04e424ad36b7692734a64150f872fb8f3208be", size = 985899, upload-time = "2026-04-20T16:38:23.409Z" }, + { url = "https://files.pythonhosted.org/packages/74/0a/f6dfd5ea3901e5d6888da8de8ba728971a1d447debab681cfc56f90d1208/pymongo-4.17.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e4fab10f8403169ce92f3cea921609d9ee81107306caae06c08f592d4b8ad2b5", size = 2028569, upload-time = "2026-04-20T16:38:25.343Z" }, + { url = "https://files.pythonhosted.org/packages/4a/c5/081f59a1c02ae8c0dc73ae58e563838c44eec81aeafa7d0b93a637841c9b/pymongo-4.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20323b0b1c1d33770ad1fc68d429c757734ce9ad3594421c3d6618f10572b1b9", size = 2072916, upload-time = "2026-04-20T16:38:27.291Z" }, + { url = "https://files.pythonhosted.org/packages/31/42/6e41d434297ffe8b30d9c3717916591a4a7be9075a0dcc2fafdfaaaa62ed/pymongo-4.17.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5a5de048e6da5c18e27cc2437e8c15b3b0cdc8385c15b41178b0caa3322a09c2", size = 2173234, upload-time = "2026-04-20T16:38:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/3d/cf/1e4a7db352ef9485831c7268dfe8402f0117b32a9ad54b16e810699e3617/pymongo-4.17.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dff3de1294fbbc1db0ba6b511f77b8e540601d092538a31312e99c8a91a78b1e", size = 2156784, upload-time = "2026-04-20T16:38:32.134Z" }, + { url = "https://files.pythonhosted.org/packages/12/10/6195be29962a61ebb5f4bd9e4c7519890b172f7968a0a0d880398c6ddb02/pymongo-4.17.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faf03e4c2aafd6de626dbd30ba246d369ae33f47f10629d1bbe40f72115027a6", size = 2074446, upload-time = "2026-04-20T16:38:34.004Z" }, + { url = "https://files.pythonhosted.org/packages/37/48/33410b8819837ed370c738587306bdf060b59cef11823be212f4a07703c5/pymongo-4.17.0-cp313-cp313-win32.whl", hash = "sha256:c9786665926a09630c5d420c79762cfadbff35a9438bcbc4c81a9fb5ab9228b7", size = 948435, upload-time = "2026-04-20T16:38:35.922Z" }, + { url = "https://files.pythonhosted.org/packages/6f/77/c0ed522f798a286b99acaa7914ed8d9c80ab091f97f57c59ffed72906e5e/pymongo-4.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:5960519b4d7168f1ecdd3ea10c81b2aedeb9423651aca953cfbc8e76705d3b38", size = 972847, upload-time = "2026-04-20T16:38:37.888Z" }, + { url = "https://files.pythonhosted.org/packages/97/f0/c39480a2db385fde23861d0c8acda41cdaf1d43e46579db72c5c013a2e81/pymongo-4.17.0-cp313-cp313-win_arm64.whl", hash = "sha256:0ff6bd2f735ab5356541e3e57d5b7dbfbc3f2ee1ccb10b6b0f82d58af69d1d8e", size = 951575, upload-time = "2026-04-20T16:38:40.544Z" }, + { url = "https://files.pythonhosted.org/packages/da/49/2b0250762a89737ed6f9cea238331baca061b89a8ddd10dd17fee52c3970/pymongo-4.17.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff5aa3f1c7e3f08eb0e7a016c91ba468b1850ccfd63d9b1f12f56350f4974cef", size = 1040945, upload-time = "2026-04-20T16:38:42.783Z" }, + { url = "https://files.pythonhosted.org/packages/89/1c/7a9b5447a08be20e84b6e5b17330917e8d6d9507daa3cd099a9309f11ad7/pymongo-4.17.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e816db649ba5d7de0568cf3a9f287a9dc9aad21cf0ca667ab156a7ef47fca0b0", size = 1041187, upload-time = "2026-04-20T16:38:45.358Z" }, + { url = "https://files.pythonhosted.org/packages/78/a1/71704f61632dfc90407a5834fe5f6132854937c4a3648f6c05c351d85a45/pymongo-4.17.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:12c4fded3a9f1d6a687e36ebd384ac6d00b9b00de1969aa74048e7051ec2a713", size = 2294806, upload-time = "2026-04-20T16:38:47.734Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b9/aff42be75108b96c2469b1d9329b912c15108f3e7ef32fdc86da8423c330/pymongo-4.17.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2db66aa8dd253a0fc1fad3b0d23d5b3993f7ebde02fbbd7727128debf2853675", size = 2348231, upload-time = "2026-04-20T16:38:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/f2/30/44c115b8ba1479942c15fd9480eb29a7da0ba68acd56983423ba0deb4a94/pymongo-4.17.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3987e96e7c7be4083d42e8ac2cc6c0d5b78db9973c90fce42ae800b616ca6b20", size = 2467614, upload-time = "2026-04-20T16:38:52.665Z" }, + { url = "https://files.pythonhosted.org/packages/d2/84/21ee95c8bf0ca7acae7ec7eb365d740bf8fc0156c194baf2c3bdfcb85ec0/pymongo-4.17.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cee36b3c0d0354f880fa7a7fdcdaf2bb5e542c2281e25c1bfadf8cfe21eba7d2", size = 2445970, upload-time = "2026-04-20T16:38:55.175Z" }, + { url = "https://files.pythonhosted.org/packages/06/89/081d7f1809d5ca09d1e47e49f2111b245f5694de3a7af32cd3a353a6f43f/pymongo-4.17.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:320b34457b20bbcc79997801f95d25ce00472915ca5241167242b42c4359e027", size = 2348605, upload-time = "2026-04-20T16:38:57.557Z" }, + { url = "https://files.pythonhosted.org/packages/ea/c3/0d949f9d3f2a341c1f635c398c16615e96f89f51ff424ed81e914cf1a4de/pymongo-4.17.0-cp314-cp314-win32.whl", hash = "sha256:df4a644af9ae132d4bfdb2e9516ea51a615fd881caddfbfbd071cf1354844479", size = 1004119, upload-time = "2026-04-20T16:39:00.309Z" }, + { url = "https://files.pythonhosted.org/packages/f7/55/5c3a3db1048054c695c75c5964cc8bedc2247fdb5a75ef6fab4ec8bb013e/pymongo-4.17.0-cp314-cp314-win_amd64.whl", hash = "sha256:c797f8a80957134f6dd9690367a0f8f5906d672119af2c6aa55f0c527b656bed", size = 1032314, upload-time = "2026-04-20T16:39:02.665Z" }, + { url = "https://files.pythonhosted.org/packages/e0/19/e235f39906134cb0ffd5574c5a59c355ef5380f0499644ab94994afbb109/pymongo-4.17.0-cp314-cp314-win_arm64.whl", hash = "sha256:68fca71e05ee5da23a8d73cee8379dfb3d26e609a377cae731d742771ed96946", size = 1007627, upload-time = "2026-04-20T16:39:04.678Z" }, + { url = "https://files.pythonhosted.org/packages/1e/e0/c4c1a86791415b14c684fa0908f9da96de91594a3fd1fa1b8dc689fbb800/pymongo-4.17.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b4384700cffc3f1dd98e088bc0072dedf6d7d68a230bb4b972665cf69c071c1e", size = 1099151, upload-time = "2026-04-20T16:39:06.969Z" }, + { url = "https://files.pythonhosted.org/packages/81/4b/69c67f3e23fd9b23b9bedc7ebd23754881cc9d5c5d5b2a9811e96b07f475/pymongo-4.17.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:93641192644fa1ee0f34030e774fd31022a27ad11ba22cb1716142231524f8bd", size = 1099346, upload-time = "2026-04-20T16:39:08.996Z" }, + { url = "https://files.pythonhosted.org/packages/a2/19/a5208f62f9508a26d73acc69bd3821b8c8adae253679a3c26d2f9652f0d5/pymongo-4.17.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:75bc3aa5b94fdb7138d357ec6ca61cd97e0c79f4f7f0bd3efe9639b15cc50942", size = 2619034, upload-time = "2026-04-20T16:39:11.049Z" }, + { url = "https://files.pythonhosted.org/packages/77/27/426cba1ec5973082a56d4150798529bfdf4151c31391ed1fbbecb23ef2ac/pymongo-4.17.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e8f8e23c6df7c6d6929f5e734980b227706e73ee847517c9ba5af90f7fc466", size = 2689939, upload-time = "2026-04-20T16:39:13.617Z" }, + { url = "https://files.pythonhosted.org/packages/ef/2e/f70993d1255e33f6ee59a4ec4371cc65bff7a7e3fda7d55c3386f25287e8/pymongo-4.17.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:15d3f3d732aecac1f8d481bde4029755615639bd3076f258a2147210aec8515a", size = 2824994, upload-time = "2026-04-20T16:39:16.057Z" }, + { url = "https://files.pythonhosted.org/packages/b3/eb/87b0e988ba889e1fcc3430c2cfc166b251872c813e92b43174298bee17ff/pymongo-4.17.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c5f62862d0f87be481fa1fe8cb811994486773c94a2b61e509285e3f2890763", size = 2801745, upload-time = "2026-04-20T16:39:18.476Z" }, + { url = "https://files.pythonhosted.org/packages/67/4c/3f83412d086f682d4d468761d66ddc49cf161e786ea74073045eb4491c60/pymongo-4.17.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64837adbbd72073301af51bb0fc80e3d7707fe5527cea1033ba0320f0b2f881b", size = 2684636, upload-time = "2026-04-20T16:39:20.878Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d8/b75f6f4ab6c8beb50b0270a4f1e2530b5774f5e116563440e1677ca1820f/pymongo-4.17.0-cp314-cp314t-win32.whl", hash = "sha256:b93b22eedc62598cf5ee9d8c8007a8e9121c50fd88137012d8985500e9dc3151", size = 1056356, upload-time = "2026-04-20T16:39:22.996Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5e/648c8a238eef18a25ed8a169ea6542d4a860bbec3e95b3d9badac2935c71/pymongo-4.17.0-cp314-cp314t-win_amd64.whl", hash = "sha256:3689ea34f6b647c7d1e7bdc60fcfb214b2789ed1359a7fb96569c69f50e5f18f", size = 1090964, upload-time = "2026-04-20T16:39:24.989Z" }, + { url = "https://files.pythonhosted.org/packages/dc/cb/d9780b66939c4fc1f024bcc7be23a2abcfe06a9745ca8fa76dc73395482e/pymongo-4.17.0-cp314-cp314t-win_arm64.whl", hash = "sha256:9543d8f84c2e5608565c08ac679774811e6730770d8a645439b073422a4276fb", size = 1058526, upload-time = "2026-04-20T16:39:27.924Z" }, +] + [[package]] name = "pynacl" version = "1.6.2" From 800cd17d17369db618f0cdb5fa817d262a3d9ef0 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 09:04:54 -0700 Subject: [PATCH 09/60] test(vector_stores): cover the MongoDB Atlas vector store config 65 cases across pipeline construction, response mapping, parameter validation, client caching, and driver-error translation. The sad-path cases assert on the message the caller actually sees, since a vector search that fails quietly returns an empty result set rather than an error. --- .../test_mongodb_transformation.py | 644 ++++++++++++++++++ 1 file changed, 644 insertions(+) create mode 100644 tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py new file mode 100644 index 00000000000..bae0ad51b05 --- /dev/null +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -0,0 +1,644 @@ +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from litellm.llms.mongodb.common_utils import ( + MongoClientKey, + get_async_client, + get_sync_client, + reset_client_cache, + translate_mongo_error, +) +from litellm.llms.mongodb.vector_stores.transformation import ( + MongoDBVectorStoreConfig, + _MongoDBSearchParams, +) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +CONNECTION_STRING = "mongodb+srv://user:pw@cluster.example.mongodb.net" +INDEX = "movies_vector_index" + +BASE_PARAMS = { + "litellm_embedding_model": "openai/text-embedding-ada-002", + "mongodb_connection_string": CONNECTION_STRING, + "mongodb_database": "sample_mflix", + "mongodb_collection": "embedded_movies", +} + + +class FakeCollection: + def __init__(self, documents, error=None): + self.documents = documents + self.error = error + self.pipeline = None + + def aggregate(self, pipeline): + self.pipeline = pipeline + if self.error is not None: + raise self.error + return iter(self.documents) + + +class FakeAsyncCollection(FakeCollection): + async def aggregate(self, pipeline): + self.pipeline = pipeline + if self.error is not None: + raise self.error + + async def cursor(): + for document in self.documents: + yield document + + return cursor() + + +class FakeDatabase: + def __init__(self, collection): + self.collection = collection + self.requested_collection = None + + def __getitem__(self, name): + self.requested_collection = name + return self.collection + + +class FakeClient: + def __init__(self, collection): + self.database = FakeDatabase(collection) + self.requested_database = None + + def __getitem__(self, name): + self.requested_database = name + return self.database + + +class FakeEmbeddingFn: + def __init__(self, embedding): + self.embedding = embedding + self.captured_kwargs = None + + def __call__(self, **kwargs): + self.captured_kwargs = kwargs + return SimpleNamespace(data=[{"embedding": self.embedding}] if self.embedding is not None else []) + + +class FakeAsyncEmbeddingFn(FakeEmbeddingFn): + async def __call__(self, **kwargs): + self.captured_kwargs = kwargs + return SimpleNamespace(data=[{"embedding": self.embedding}] if self.embedding is not None else []) + + +def _config(documents=(), embedding=(0.1, 0.2, 0.3), error=None): + collection = FakeCollection(list(documents), error) + client = FakeClient(collection) + config = MongoDBVectorStoreConfig( + embedding_fn=FakeEmbeddingFn(list(embedding) if embedding is not None else None), + sync_client_factory=lambda key: client, + ) + return config, client, collection + + +def _async_config(documents=(), embedding=(0.1, 0.2, 0.3), error=None): + collection = FakeAsyncCollection(list(documents), error) + client = FakeClient(collection) + config = MongoDBVectorStoreConfig( + aembedding_fn=FakeAsyncEmbeddingFn(list(embedding) if embedding is not None else None), + async_client_factory=lambda key: client, + ) + return config, client, collection + + +def _search(config, query="a lone astronaut", optional_params=None, litellm_params=None, timeout=None): + return config.execute_search_vector_store_request( + vector_store_id=INDEX, + query=query, + vector_store_search_optional_params=optional_params or {}, + litellm_logging_obj=MagicMock(), + litellm_params={**BASE_PARAMS, **(litellm_params or {})}, + timeout=timeout, + ) + + +async def _asearch(config, query="a lone astronaut", optional_params=None, litellm_params=None): + return await config.aexecute_search_vector_store_request( + vector_store_id=INDEX, + query=query, + vector_store_search_optional_params=optional_params or {}, + litellm_logging_obj=MagicMock(), + litellm_params={**BASE_PARAMS, **(litellm_params or {})}, + ) + + +def _stage(collection, name): + return next(stage[name] for stage in collection.pipeline if name in stage) + + +def test_search_builds_vector_search_stage_against_the_named_index(): + config, client, collection = _config() + + _search(config, optional_params={"max_num_results": 5}) + + assert client.requested_database == "sample_mflix" + assert client.database.requested_collection == "embedded_movies" + assert _stage(collection, "$vectorSearch") == { + "index": INDEX, + "path": "embedding", + "queryVector": [0.1, 0.2, 0.3], + "numCandidates": 100, + "limit": 5, + } + + +def test_search_projects_the_text_field_and_the_similarity_score(): + config, _, collection = _config() + + _search(config) + + assert _stage(collection, "$project") == {"text": 1, "score": {"$meta": "vectorSearchScore"}} + + +def test_search_defaults_to_ten_results(): + config, _, collection = _config() + + _search(config) + + assert _stage(collection, "$vectorSearch")["limit"] == 10 + + +def test_search_honors_custom_field_names(): + config, _, collection = _config() + + _search( + config, + litellm_params={"mongodb_embedding_field": "plot_embedding", "mongodb_text_field": "plot"}, + ) + + assert _stage(collection, "$vectorSearch")["path"] == "plot_embedding" + assert _stage(collection, "$project") == {"plot": 1, "score": {"$meta": "vectorSearchScore"}} + + +def test_num_candidates_scales_with_the_requested_limit(): + config, _, collection = _config() + + _search(config, optional_params={"max_num_results": 40}) + + assert _stage(collection, "$vectorSearch")["numCandidates"] == 400 + + +def test_num_candidates_can_be_overridden(): + config, _, collection = _config() + + _search(config, optional_params={"max_num_results": 5}, litellm_params={"mongodb_num_candidates": 250}) + + assert _stage(collection, "$vectorSearch")["numCandidates"] == 250 + + +@pytest.mark.parametrize("configured", [4, 10_001]) +def test_num_candidates_below_the_limit_or_above_the_ceiling_is_rejected(configured): + config, _, _ = _config() + + with pytest.raises(ValueError, match="mongodb_num_candidates"): + _search(config, optional_params={"max_num_results": 5}, litellm_params={"mongodb_num_candidates": configured}) + + +def test_list_query_is_joined_into_one_embedding_input(): + config, _, _ = _config() + embedding_fn = config.embedding_fn + + _search(config, query=["deep", "space", "rescue"]) + + assert embedding_fn.captured_kwargs["input"] == ["deep space rescue"] + + +def test_embedding_config_is_expanded_into_the_embedding_call(): + config, _, _ = _config() + embedding_fn = config.embedding_fn + + _search(config, litellm_params={"litellm_embedding_config": {"api_base": "https://example.test", "timeout": 7}}) + + assert embedding_fn.captured_kwargs["api_base"] == "https://example.test" + assert embedding_fn.captured_kwargs["timeout"] == 7 + assert embedding_fn.captured_kwargs["model"] == "openai/text-embedding-ada-002" + + +def test_response_maps_documents_to_openai_shaped_results(): + documents = [ + {"_id": "abc123", "text": "an astronaut adrift", "score": 0.94}, + {"_id": "def456", "text": "a robot dog", "score": 0.81}, + ] + config, _, _ = _config(documents=documents) + + response = _search(config) + + assert response["object"] == "vector_store.search_results.page" + assert response["search_query"] == "a lone astronaut" + assert [result["score"] for result in response["data"]] == [0.94, 0.81] + assert [result["content"][0]["text"] for result in response["data"]] == ["an astronaut adrift", "a robot dog"] + assert [result["file_id"] for result in response["data"]] == ["abc123", "def456"] + assert [result["filename"] for result in response["data"]] == ["abc123", "def456"] + assert response["data"][0]["content"][0]["type"] == "text" + + +def test_response_reads_a_dotted_text_field_path(): + config, _, _ = _config(documents=[{"_id": 1, "metadata": {"body": "nested text"}, "score": 0.5}]) + + response = _search(config, litellm_params={"mongodb_text_field": "metadata.body"}) + + assert response["data"][0]["content"][0]["text"] == "nested text" + + +def test_response_tolerates_a_document_missing_the_text_field(): + config, _, _ = _config(documents=[{"_id": 1, "score": 0.5}]) + + response = _search(config) + + assert response["data"][0]["content"][0]["text"] == "" + + +def test_response_tolerates_a_document_missing_a_score(): + config, _, _ = _config(documents=[{"_id": 1, "text": "no score"}]) + + response = _search(config) + + assert response["data"][0]["score"] is None + + +def test_response_stringifies_a_non_string_document_id(): + config, _, _ = _config(documents=[{"_id": 12345, "text": "numeric id", "score": 0.5}]) + + response = _search(config) + + assert response["data"][0]["file_id"] == "12345" + + +def test_search_requires_an_embedding_model(): + config, _, _ = _config() + + with pytest.raises(ValueError, match="litellm_embedding_model is required"): + config.execute_search_vector_store_request( + vector_store_id=INDEX, + query="q", + vector_store_search_optional_params={}, + litellm_logging_obj=MagicMock(), + litellm_params={k: v for k, v in BASE_PARAMS.items() if k != "litellm_embedding_model"}, + ) + + +def test_missing_embedding_model_message_names_the_field_being_searched(): + config, _, _ = _config() + + with pytest.raises(ValueError, match=r"embedded_movies\.embedding"): + config.execute_search_vector_store_request( + vector_store_id=INDEX, + query="q", + vector_store_search_optional_params={}, + litellm_logging_obj=MagicMock(), + litellm_params={k: v for k, v in BASE_PARAMS.items() if k != "litellm_embedding_model"}, + ) + + +def test_search_requires_a_connection_string(): + config, _, _ = _config() + + with pytest.raises(ValueError, match="mongodb_connection_string is required"): + _search(config, litellm_params={"mongodb_connection_string": None}) + + +@pytest.mark.parametrize("connection_string", ["postgres://host/db", "https://cluster.mongodb.net", "redis://host"]) +def test_search_rejects_a_non_mongodb_connection_scheme(connection_string): + config, _, _ = _config() + + with pytest.raises(ValueError, match="must start with 'mongodb://' or 'mongodb\\+srv://'"): + _search(config, litellm_params={"mongodb_connection_string": connection_string}) + + +def test_search_accepts_the_plain_mongodb_scheme(): + config, _, collection = _config() + + _search(config, litellm_params={"mongodb_connection_string": "mongodb://localhost:27017"}) + + assert collection.pipeline is not None + + +def test_search_requires_a_database(): + config, _, _ = _config() + + with pytest.raises(ValueError, match="mongodb_database is required"): + _search(config, litellm_params={"mongodb_database": None}) + + +def test_search_requires_a_collection(): + config, _, _ = _config() + + with pytest.raises(ValueError, match="mongodb_collection is required"): + _search(config, litellm_params={"mongodb_collection": None}) + + +def test_search_rejects_filters_rather_than_silently_ignoring_them(): + config, _, _ = _config() + + with pytest.raises(ValueError, match="does not support the filters parameter"): + _search(config, optional_params={"filters": {"genre": "sci-fi"}}) + + +@pytest.mark.asyncio +async def test_async_search_rejects_filters_rather_than_silently_ignoring_them(): + config, _, _ = _async_config() + + with pytest.raises(ValueError, match="does not support the filters parameter"): + await _asearch(config, optional_params={"filters": {"genre": "sci-fi"}}) + + +@pytest.mark.parametrize("query", ["", " ", "\n\t", []]) +def test_search_rejects_an_empty_query(query): + config, _, _ = _config() + + with pytest.raises(ValueError, match="query must not be empty"): + _search(config, query=query) + + +def test_search_rejects_an_oversized_query(): + config, _, _ = _config() + + with pytest.raises(ValueError, match="at most 32000 characters"): + _search(config, query="x" * 32_001) + + +def test_search_accepts_a_query_at_the_size_ceiling(): + config, _, collection = _config() + + _search(config, query="x" * 32_000) + + assert collection.pipeline is not None + + +@pytest.mark.parametrize("max_num_results", [0, -1, 51, 1000]) +def test_search_rejects_out_of_range_max_num_results(max_num_results): + config, _, _ = _config() + + with pytest.raises(ValueError, match="max_num_results must be between 1 and 50"): + _search(config, optional_params={"max_num_results": max_num_results}) + + +@pytest.mark.parametrize("max_num_results", [1, 50]) +def test_search_allows_max_num_results_at_the_bounds(max_num_results): + config, _, collection = _config() + + _search(config, optional_params={"max_num_results": max_num_results}) + + assert _stage(collection, "$vectorSearch")["limit"] == max_num_results + + +def test_search_treats_an_explicit_null_max_num_results_as_the_default(): + config, _, collection = _config() + + _search(config, optional_params={"max_num_results": None}) + + assert _stage(collection, "$vectorSearch")["limit"] == 10 + + +def test_search_fails_when_the_embedding_model_returns_nothing(): + config, _, _ = _config(embedding=None) + + with pytest.raises(ValueError, match="returned no embedding"): + _search(config) + + +def test_validation_runs_before_any_connection_is_opened(): + opened = [] + config = MongoDBVectorStoreConfig( + embedding_fn=FakeEmbeddingFn([0.1]), + sync_client_factory=lambda key: opened.append(key) or FakeClient(FakeCollection([])), + ) + + with pytest.raises(ValueError, match="query must not be empty"): + _search(config, query="") + + assert opened == [] + + +def test_create_vector_store_is_not_supported_and_says_why(): + config = MongoDBVectorStoreConfig() + + with pytest.raises(NotImplementedError, match="search-only"): + config.transform_create_vector_store_request({}, "https://example.test") + + with pytest.raises(NotImplementedError, match="search-only"): + config.transform_create_vector_store_response(httpx.Response(200)) + + +def test_provider_config_manager_returns_the_mongodb_config(): + config = ProviderConfigManager.get_provider_vector_stores_config(LlmProviders.MONGODB) + + assert isinstance(config, MongoDBVectorStoreConfig) + + +@pytest.mark.asyncio +async def test_async_search_builds_the_same_pipeline_and_maps_the_response(): + documents = [{"_id": "abc123", "text": "an astronaut adrift", "score": 0.94}] + config, client, collection = _async_config(documents=documents) + + response = await _asearch(config, optional_params={"max_num_results": 3}) + + assert client.requested_database == "sample_mflix" + assert client.database.requested_collection == "embedded_movies" + assert _stage(collection, "$vectorSearch")["limit"] == 3 + assert _stage(collection, "$vectorSearch")["queryVector"] == [0.1, 0.2, 0.3] + assert response["data"][0]["content"][0]["text"] == "an astronaut adrift" + assert response["data"][0]["score"] == 0.94 + + +@pytest.mark.asyncio +async def test_async_search_requires_an_embedding_model(): + config, _, _ = _async_config() + + with pytest.raises(ValueError, match="litellm_embedding_model is required"): + await config.aexecute_search_vector_store_request( + vector_store_id=INDEX, + query="q", + vector_store_search_optional_params={}, + litellm_logging_obj=MagicMock(), + litellm_params={k: v for k, v in BASE_PARAMS.items() if k != "litellm_embedding_model"}, + ) + + +class TestClientCache: + def setup_method(self): + reset_client_cache() + + def teardown_method(self): + reset_client_cache() + + def _key(self, connection_string=CONNECTION_STRING, socket_timeout_ms=30_000): + return MongoClientKey( + connection_string=connection_string, + connect_timeout_ms=10_000, + socket_timeout_ms=socket_timeout_ms, + server_selection_timeout_ms=10_000, + ) + + def test_the_same_connection_reuses_one_client(self): + with patch("litellm.llms.mongodb.common_utils.import_sync_mongo_client") as importer: + importer.return_value = lambda *args, **kwargs: MagicMock() + + first = get_sync_client(self._key()) + second = get_sync_client(self._key()) + + assert first is second + assert importer.return_value + + def test_a_different_connection_gets_its_own_client(self): + with patch("litellm.llms.mongodb.common_utils.import_sync_mongo_client") as importer: + importer.return_value = lambda *args, **kwargs: MagicMock() + + first = get_sync_client(self._key()) + second = get_sync_client(self._key(connection_string="mongodb://other.example.test")) + + assert first is not second + + def test_a_different_timeout_gets_its_own_client(self): + with patch("litellm.llms.mongodb.common_utils.import_sync_mongo_client") as importer: + importer.return_value = lambda *args, **kwargs: MagicMock() + + first = get_sync_client(self._key()) + second = get_sync_client(self._key(socket_timeout_ms=5_000)) + + assert first is not second + + @pytest.mark.asyncio + async def test_async_clients_are_cached_per_event_loop(self): + with patch("litellm.llms.mongodb.common_utils.import_async_mongo_client") as importer: + importer.return_value = lambda *args, **kwargs: MagicMock() + + first = get_async_client(self._key()) + second = get_async_client(self._key()) + + assert first is second + + +class TestClientKeyDerivation: + def test_no_timeout_uses_the_bounded_defaults(self): + key = MongoDBVectorStoreConfig._client_key(_MongoDBSearchParams.model_validate(BASE_PARAMS), None) + + assert key.connect_timeout_ms == 10_000 + assert key.socket_timeout_ms == 30_000 + assert key.server_selection_timeout_ms == 10_000 + + def test_a_numeric_timeout_bounds_the_connect_phase(self): + key = MongoDBVectorStoreConfig._client_key(_MongoDBSearchParams.model_validate(BASE_PARAMS), 3.0) + + assert key.socket_timeout_ms == 3_000 + assert key.connect_timeout_ms == 3_000 + + def test_an_httpx_timeout_maps_connect_and_read_separately(self): + key = MongoDBVectorStoreConfig._client_key( + _MongoDBSearchParams.model_validate(BASE_PARAMS), httpx.Timeout(connect=2.0, read=45.0, write=5.0, pool=5.0) + ) + + assert key.connect_timeout_ms == 2_000 + assert key.socket_timeout_ms == 45_000 + + +class TestErrorTranslation: + def _translate(self, error): + return translate_mongo_error(error, index_name=INDEX, database="sample_mflix", collection="embedded_movies") + + def test_server_selection_timeout_points_at_the_atlas_access_list(self): + from pymongo.errors import ServerSelectionTimeoutError + + translated = self._translate(ServerSelectionTimeoutError("no servers")) + + assert "IP access list" in str(translated) + assert "paused cluster" in str(translated) + + def test_authentication_failure_points_at_the_connection_string_credentials(self): + from pymongo.errors import OperationFailure + + translated = self._translate(OperationFailure("auth failed", code=18)) + + assert "rejected the credentials" in str(translated) + + def test_unauthorized_points_at_the_database_user_permissions(self): + from pymongo.errors import OperationFailure + + translated = self._translate(OperationFailure("not authorized", code=13)) + + assert "sample_mflix.embedded_movies" in str(translated) + + def test_a_missing_index_names_the_index_and_the_collection(self): + from pymongo.errors import OperationFailure + + translated = self._translate(OperationFailure("Index not found for name movies_vector_index", code=27)) + + assert INDEX in str(translated) + assert "READY" in str(translated) + + def test_a_dimension_mismatch_points_at_the_embedding_model(self): + from pymongo.errors import OperationFailure + + translated = self._translate(OperationFailure("queryVector has 1536 dimensions, index expects 2048")) + + assert "litellm_embedding_model must be the same model" in str(translated) + + def test_an_unrecognised_operation_failure_still_names_the_target(self): + from pymongo.errors import OperationFailure + + translated = self._translate(OperationFailure("something else entirely")) + + assert "sample_mflix.embedded_movies" in str(translated) + assert INDEX in str(translated) + + def test_a_configuration_error_points_at_the_connection_string(self): + from pymongo.errors import ConfigurationError + + translated = self._translate(ConfigurationError("bad uri")) + + assert "not a usable MongoDB connection string" in str(translated) + + def test_a_non_driver_error_is_returned_unchanged(self): + original = RuntimeError("unrelated") + + assert self._translate(original) is original + + def test_search_surfaces_a_translated_driver_error(self): + from pymongo.errors import ServerSelectionTimeoutError + + config, _, _ = _config(error=ServerSelectionTimeoutError("no servers")) + + with pytest.raises(ValueError, match="IP access list"): + _search(config) + + @pytest.mark.asyncio + async def test_async_search_surfaces_a_translated_driver_error(self): + from pymongo.errors import OperationFailure + + config, _, _ = _async_config(error=OperationFailure("auth failed", code=18)) + + with pytest.raises(ValueError, match="rejected the credentials"): + await _asearch(config) + + +class TestMissingDriver: + def test_the_sync_import_names_the_extra_to_install(self): + from litellm.llms.mongodb.common_utils import import_sync_mongo_client + + with patch.dict(sys.modules, {"pymongo": None}): + with pytest.raises(ValueError, match=r"pip install litellm\[mongodb\]"): + import_sync_mongo_client() + + def test_the_async_import_names_the_extra_to_install(self): + from litellm.llms.mongodb.common_utils import import_async_mongo_client + + with patch.dict(sys.modules, {"pymongo": None}): + with pytest.raises(ValueError, match=r"pip install litellm\[mongodb\]"): + import_async_mongo_client() + + def test_error_translation_degrades_gracefully_without_the_driver(self): + original = RuntimeError("boom") + + with patch.dict(sys.modules, {"pymongo.errors": None}): + assert translate_mongo_error(original, INDEX, "db", "col") is original From 85bda43d632a6521f59aacafbc2bebd198cfdf20 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 09:10:51 -0700 Subject: [PATCH 10/60] fix(vector_stores): turn MongoDB's silent misconfiguration failures into errors Driving the sad path against a live Atlas cluster showed four cases returning an empty result set instead of failing: a missing index, a missing database, a missing collection, and the async path for all three. $vectorSearch reports none of these as errors, so a misconfigured store looked exactly like a query that matched nothing, which is the worst shape for this to fail in. An empty result set is now checked against the index catalogue, which does report all three correctly, and a store that cannot work says so. The check costs one extra round trip and only on the empty path, so a search that returned hits is unaffected. Atlas also reports a wrong vector path and a dimension mismatch under the same error code. Both previously surfaced as "index not found", which sent the reader looking in the wrong place; they are now told apart and each names the setting that is actually wrong. --- litellm/llms/mongodb/common_utils.py | 29 +++- .../mongodb/vector_stores/transformation.py | 38 ++++- .../test_mongodb_transformation.py | 153 +++++++++++++++++- 3 files changed, 210 insertions(+), 10 deletions(-) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 7f26eadb08f..2571821381a 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -105,6 +105,24 @@ def _index_hint(index_name: str, database: str, collection: str) -> str: ) +def missing_index_error(index_name: str, database: str, collection: str) -> ValueError: + """$vectorSearch against a missing index, database or collection returns zero documents + instead of failing, so an empty result set is checked against the index catalogue and + turned into this rather than being reported as 'no matches'.""" + return ValueError( + f"{_index_hint(index_name, database, collection)} A vector search against a database, " + "collection or index that does not exist returns no results rather than an error, so this " + "was reported as an empty result set by MongoDB." + ) + + +def index_not_ready_error(index_name: str, database: str, collection: str, status: str) -> ValueError: + return ValueError( + f"The Atlas Vector Search index '{index_name}' on '{database}.{collection}' is not queryable " + f"yet; its status is {status}. Searches against it return no results until the build finishes." + ) + + def translate_mongo_error(error: Exception, index_name: str, database: str, collection: str) -> Exception: """Turn a driver failure into a message that names the misconfiguration, never a silent empty result. @@ -136,14 +154,19 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll f"lacks read access to '{database}.{collection}'. Driver detail: {error.details}" ) detail: Final = str(error).lower() - if "index" in detail and ("not found" in detail or "does not exist" in detail or "unknown" in detail): - return ValueError(f"{_index_hint(index_name, database, collection)} Driver detail: {error}") - if "dimension" in detail or "numdimensions" in detail or "queryvector" in detail: + if "dimension" in detail: return ValueError( "The query embedding does not match the vector dimensions the Atlas index was built for. " "litellm_embedding_model must be the same model that produced the stored vectors. " f"Driver detail: {error}" ) + if "is not indexed as vector" in detail: + return ValueError( + "mongodb_embedding_field names a field the Atlas Vector Search index does not cover. " + f"It must match the 'path' the index '{index_name}' was created on. Driver detail: {error}" + ) + if "index" in detail and ("not found" in detail or "does not exist" in detail or "unknown" in detail): + return ValueError(f"{_index_hint(index_name, database, collection)} Driver detail: {error}") return ValueError( f"MongoDB rejected the vector search against '{database}.{collection}' using index " f"'{index_name}'. Driver detail: {error}" diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index efeff16ae5a..20dcf62dcc3 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -26,6 +26,8 @@ from litellm.llms.mongodb.common_utils import ( MongoClientKey, get_async_client, get_sync_client, + index_not_ready_error, + missing_index_error, translate_mongo_error, ) from litellm.types.utils import EmbeddingResponse @@ -249,6 +251,19 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): data=[cls._to_result(document, text_field) for document in documents], ) + @staticmethod + def _raise_for_unusable_index( + catalogue: Sequence[Mapping[str, object]], index_name: str, database: str, collection: str + ) -> None: + """An empty result set is ambiguous: Atlas returns zero documents both for a query that + genuinely matched nothing and for a missing database, collection or index. Only the second + is a misconfiguration, so the index catalogue decides which one happened.""" + if not catalogue: + raise missing_index_error(index_name, database, collection) + entry: Final = catalogue[0] + if not entry.get("queryable"): + raise index_not_ready_error(index_name, database, collection, str(entry.get("status") or "unknown")) + @staticmethod def _embedding_vector(embedding_response: EmbeddingResponse) -> Sequence[float]: data: Final = embedding_response.data @@ -284,12 +299,21 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): ) client: Final = self.sync_client_factory(key) + target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted try: - documents: Final = list(client[database][collection].aggregate(pipeline)) # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted + documents: Final = list(target.aggregate(pipeline)) except Exception as e: raise translate_mongo_error( e, index_name=vector_store_id, database=database, collection=collection ) from e + if not documents: + try: + catalogue: Final = list(target.list_search_indexes(vector_store_id)) + except Exception as e: + raise translate_mongo_error( + e, index_name=vector_store_id, database=database, collection=collection + ) from e + self._raise_for_unusable_index(catalogue, vector_store_id, database, collection) return self._to_response(documents, query_text, params.text_field) async def aexecute_search_vector_store_request( @@ -317,13 +341,23 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): ) client: Final = self.async_client_factory(key) + target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted try: - cursor: Final = await client[database][collection].aggregate(pipeline) # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted + cursor: Final = await target.aggregate(pipeline) documents: Final = [document async for document in cursor] except Exception as e: raise translate_mongo_error( e, index_name=vector_store_id, database=database, collection=collection ) from e + if not documents: + try: + index_cursor: Final = await target.list_search_indexes(vector_store_id) + catalogue: Final = [entry async for entry in index_cursor] + except Exception as e: + raise translate_mongo_error( + e, index_name=vector_store_id, database=database, collection=collection + ) from e + self._raise_for_unusable_index(catalogue, vector_store_id, database, collection) return self._to_response(documents, query_text, params.text_field) def transform_create_vector_store_request( diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index bae0ad51b05..c9378015f5a 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -30,11 +30,16 @@ BASE_PARAMS = { } +READY_INDEX = [{"name": INDEX, "status": "READY", "queryable": True}] + + class FakeCollection: - def __init__(self, documents, error=None): + def __init__(self, documents, error=None, search_indexes=None): self.documents = documents self.error = error + self.search_indexes = READY_INDEX if search_indexes is None else search_indexes self.pipeline = None + self.listed_indexes = [] def aggregate(self, pipeline): self.pipeline = pipeline @@ -42,6 +47,10 @@ class FakeCollection: raise self.error return iter(self.documents) + def list_search_indexes(self, name): + self.listed_indexes.append(name) + return iter(self.search_indexes) + class FakeAsyncCollection(FakeCollection): async def aggregate(self, pipeline): @@ -55,6 +64,15 @@ class FakeAsyncCollection(FakeCollection): return cursor() + async def list_search_indexes(self, name): + self.listed_indexes.append(name) + + async def cursor(): + for entry in self.search_indexes: + yield entry + + return cursor() + class FakeDatabase: def __init__(self, collection): @@ -92,8 +110,8 @@ class FakeAsyncEmbeddingFn(FakeEmbeddingFn): return SimpleNamespace(data=[{"embedding": self.embedding}] if self.embedding is not None else []) -def _config(documents=(), embedding=(0.1, 0.2, 0.3), error=None): - collection = FakeCollection(list(documents), error) +def _config(documents=(), embedding=(0.1, 0.2, 0.3), error=None, search_indexes=None): + collection = FakeCollection(list(documents), error, search_indexes) client = FakeClient(collection) config = MongoDBVectorStoreConfig( embedding_fn=FakeEmbeddingFn(list(embedding) if embedding is not None else None), @@ -102,8 +120,8 @@ def _config(documents=(), embedding=(0.1, 0.2, 0.3), error=None): return config, client, collection -def _async_config(documents=(), embedding=(0.1, 0.2, 0.3), error=None): - collection = FakeAsyncCollection(list(documents), error) +def _async_config(documents=(), embedding=(0.1, 0.2, 0.3), error=None, search_indexes=None): + collection = FakeAsyncCollection(list(documents), error, search_indexes) client = FakeClient(collection) config = MongoDBVectorStoreConfig( aembedding_fn=FakeAsyncEmbeddingFn(list(embedding) if embedding is not None else None), @@ -642,3 +660,128 @@ class TestMissingDriver: with patch.dict(sys.modules, {"pymongo.errors": None}): assert translate_mongo_error(original, INDEX, "db", "col") is original + + +class TestEmptyResultsAreDisambiguated: + """$vectorSearch returns zero documents for a missing database, collection or index just as it + does for a query that matched nothing, so an empty result set is checked against the index + catalogue before it is reported as 'no matches'.""" + + def test_a_missing_index_becomes_an_error_rather_than_an_empty_page(self): + config, _, collection = _config(documents=[], search_indexes=[]) + + with pytest.raises(ValueError, match="No queryable Atlas Vector Search index"): + _search(config) + + assert collection.listed_indexes == [INDEX] + + def test_the_missing_index_error_explains_why_mongodb_reported_no_results(self): + config, _, _ = _config(documents=[], search_indexes=[]) + + with pytest.raises(ValueError, match="returns no results rather than an error"): + _search(config) + + def test_an_index_still_building_becomes_an_error_naming_its_status(self): + config, _, _ = _config( + documents=[], search_indexes=[{"name": INDEX, "status": "PENDING", "queryable": False}] + ) + + with pytest.raises(ValueError, match="not queryable yet; its status is PENDING"): + _search(config) + + def test_a_genuine_no_match_against_a_ready_index_returns_an_empty_page(self): + config, _, collection = _config(documents=[]) + + response = _search(config) + + assert response["data"] == [] + assert response["object"] == "vector_store.search_results.page" + assert collection.listed_indexes == [INDEX] + + def test_the_catalogue_is_not_consulted_when_the_search_returned_hits(self): + config, _, collection = _config(documents=[{"_id": 1, "text": "hit", "score": 0.9}]) + + _search(config) + + assert collection.listed_indexes == [] + + @pytest.mark.asyncio + async def test_async_missing_index_becomes_an_error_rather_than_an_empty_page(self): + config, _, collection = _async_config(documents=[], search_indexes=[]) + + with pytest.raises(ValueError, match="No queryable Atlas Vector Search index"): + await _asearch(config) + + assert collection.listed_indexes == [INDEX] + + @pytest.mark.asyncio + async def test_async_index_still_building_becomes_an_error_naming_its_status(self): + config, _, _ = _async_config( + documents=[], search_indexes=[{"name": INDEX, "status": "PENDING", "queryable": False}] + ) + + with pytest.raises(ValueError, match="not queryable yet; its status is PENDING"): + await _asearch(config) + + @pytest.mark.asyncio + async def test_async_genuine_no_match_returns_an_empty_page(self): + config, _, _ = _async_config(documents=[]) + + response = await _asearch(config) + + assert response["data"] == [] + + @pytest.mark.asyncio + async def test_async_catalogue_is_not_consulted_when_the_search_returned_hits(self): + config, _, collection = _async_config(documents=[{"_id": 1, "text": "hit", "score": 0.9}]) + + await _asearch(config) + + assert collection.listed_indexes == [] + + def test_a_failure_while_checking_the_catalogue_is_translated_too(self): + from pymongo.errors import OperationFailure + + class ExplodingCollection(FakeCollection): + def list_search_indexes(self, name): + raise OperationFailure("not authorized", code=13) + + collection = ExplodingCollection([], None, []) + config = MongoDBVectorStoreConfig( + embedding_fn=FakeEmbeddingFn([0.1]), + sync_client_factory=lambda key: FakeClient(collection), + ) + + with pytest.raises(ValueError, match="lacks read access"): + _search(config) + + +class TestAtlasPlanExecutorErrors: + """Atlas reports a wrong vector path and a dimension mismatch through the same error code, so + each one has to be told apart by its message or both come back as a generic index failure.""" + + def _translate(self, message): + from pymongo.errors import OperationFailure + + return translate_mongo_error( + OperationFailure(message, code=8), + index_name=INDEX, + database="sample_mflix", + collection="embedded_movies", + ) + + def test_a_wrong_vector_path_points_at_the_embedding_field_setting(self): + translated = self._translate( + "PlanExecutor error during aggregation :: caused by :: nope is not indexed as vector" + ) + + assert "mongodb_embedding_field names a field" in str(translated) + + def test_a_dimension_mismatch_is_not_reported_as_a_wrong_path(self): + translated = self._translate( + "PlanExecutor error during aggregation :: caused by :: vector field is indexed with " + "1536 dimensions but queried with 3072" + ) + + assert "does not match the vector dimensions" in str(translated) + assert "mongodb_embedding_field" not in str(translated) From 22d34960e5471e7640a4b99e72387401dcf85218 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 09:56:17 -0700 Subject: [PATCH 11/60] fix(vector_stores): redact wire-protocol connection strings in management responses A MongoDB vector store's whole credential is its connection string, and mongodb+srv://:@ embeds the database password. None of the masker's default patterns (api_key, secret, token, credential) match a key named mongodb_connection_string, so /vector_store/list and /vector_store/info returned it verbatim to every caller that can read a vector store. SensitiveDataMasker gains extra_sensitive_patterns, which unions onto the defaults instead of replacing them, and the vector-store redactor adds "connection" so the URI is masked while mongodb_database, mongodb_collection and the field names stay readable. --- .../sensitive_data_masker.py | 42 +++++++++++-------- .../management_endpoints.py | 5 ++- .../test_sensitive_data_masker.py | 19 +++++++++ .../test_vector_store_endpoints.py | 36 ++++++++++++++++ 4 files changed, 84 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 3d60c1bda12..3b0806ab069 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -6,33 +6,41 @@ from pydantic import BaseModel from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER +_DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( + { + "password", + "secret", + "key", + "token", + "auth", + "authorization", + "credential", + # Plural form: Vertex uses ``vertex_credentials``; segment-exact + # matching otherwise misses it because "credential" != "credentials". + "credentials", + "access", + "private", + "certificate", + "fingerprint", + "tenancy", + } +) + + class SensitiveDataMasker: def __init__( self, sensitive_patterns: set[str] | None = None, + extra_sensitive_patterns: set[str] | None = None, non_sensitive_overrides: set[str] | None = None, visible_prefix: int = 4, visible_suffix: int = 4, mask_char: str = "*", mask_short_values: bool = True, ): - self.sensitive_patterns = sensitive_patterns or { - "password", - "secret", - "key", - "token", - "auth", - "authorization", - "credential", - # Plural form: Vertex uses ``vertex_credentials``; segment-exact - # matching otherwise misses it because "credential" != "credentials". - "credentials", - "access", - "private", - "certificate", - "fingerprint", - "tenancy", - } + self.sensitive_patterns = (sensitive_patterns or _DEFAULT_SENSITIVE_PATTERNS) | ( + extra_sensitive_patterns or frozenset() + ) # If any key segment matches one of these, the key is not considered sensitive # even if it also matches a sensitive pattern. For example, "input_cost_per_token" # contains "token" but "cost" overrides that — it's a pricing field, not a secret. diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 183a03cc13c..a62c0f711cb 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -59,7 +59,10 @@ def _row_to_vector_store(row: "_VectorStoreRow") -> LiteLLM_ManagedVectorStore: return LiteLLM_ManagedVectorStore(**row.model_dump()) -_LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker() +# "connection" covers wire-protocol providers whose whole credential is a URI +# (mongodb_connection_string embeds the username and password), which the +# default api_key/secret/token patterns do not match. +_LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker(extra_sensitive_patterns={"connection"}) _REDACT_LITELLM_PARAMS_MAX_DEPTH: Final = 10 diff --git a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py index f6b8a93c472..27a83223864 100644 --- a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py @@ -312,3 +312,22 @@ def test_mask_credentials_in_payload_masks_only_sensitive_string_leaves(): assert masked != plaintext assert masked.startswith(plaintext[:4]) assert masked.endswith(plaintext[-4:]) + + +def test_extra_sensitive_patterns_add_to_the_defaults(): + from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker + + masker = SensitiveDataMasker(extra_sensitive_patterns={"connection"}) + + assert masker.is_sensitive_key("mongodb_connection_string") is True + assert masker.is_sensitive_key("api_key") is True + assert masker.is_sensitive_key("aws_secret_access_key") is True + assert masker.is_sensitive_key("mongodb_database") is False + + +def test_extra_sensitive_patterns_do_not_leak_into_other_maskers(): + from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker + + SensitiveDataMasker(extra_sensitive_patterns={"connection"}) + + assert SensitiveDataMasker().is_sensitive_key("mongodb_connection_string") is False diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index eae6f90863a..16ec6e9796c 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -1,3 +1,4 @@ +import json from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -2700,6 +2701,41 @@ class TestRedactSensitiveLitellmParams: for k, v in params.items(): assert out[k] == v, f"{k} should be preserved verbatim" + def test_redacts_wire_protocol_connection_strings(self): + """ + A MongoDB vector store's whole credential is its connection string: + ``mongodb+srv://:@`` embeds the database + password, and none of the default api_key/secret/token patterns match + the key name, so an unextended masker returns it verbatim to every + caller of /vector_store/list and /vector_store/info. + """ + from litellm.constants import REDACTED_BY_LITELM_STRING + from litellm.proxy.vector_store_endpoints.management_endpoints import ( + _redact_sensitive_litellm_params, + ) + + password = "hunter2-not-for-callers" + params = { + "mongodb_connection_string": f"mongodb+srv://dbuser:{password}@cluster0.mongodb.net", + "mongodb_database": "sample_mflix", + "mongodb_collection": "embedded_movies", + "mongodb_embedding_field": "plot_embedding", + "mongodb_text_field": "plot", + "litellm_embedding_model": "openai/text-embedding-ada-002", + } + out = _redact_sensitive_litellm_params(params) + + assert out["mongodb_connection_string"] == REDACTED_BY_LITELM_STRING + assert password not in json.dumps(out) + for k in ( + "mongodb_database", + "mongodb_collection", + "mongodb_embedding_field", + "mongodb_text_field", + "litellm_embedding_model", + ): + assert out[k] == params[k], f"{k} is not a credential and must survive redaction" + def test_handles_none_and_empty(self): from litellm.proxy.vector_store_endpoints.management_endpoints import ( _redact_sensitive_litellm_params, From 8374b34b8168a9fd643b0d7fc5404ea7ff3c2edf Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 10:01:31 -0700 Subject: [PATCH 12/60] fix(vector_stores): return 400 for MongoDB misconfiguration instead of 500 litellm.exception_type passes a litellm exception through untouched and wraps anything else into APIConnectionError, so every bare ValueError this provider raised reached the caller as HTTP 500 with a Python traceback in the response body. "max_num_results must be between 1 and 50" is the caller's to fix, not a connection failure. Configuration and validation failures now raise BadRequestError (400) and the two timeout cases raise Timeout (408). ExecutionTimeout subclasses OperationFailure, so it is matched before it; previously an Atlas query that ran out of time was reported as "MongoDB rejected the vector search". --- litellm/llms/mongodb/common_utils.py | 54 +++++--- .../mongodb/vector_stores/transformation.py | 23 ++-- .../test_mongodb_transformation.py | 125 ++++++++++++++---- 3 files changed, 147 insertions(+), 55 deletions(-) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 2571821381a..299a9772817 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -12,6 +12,8 @@ import asyncio from dataclasses import dataclass from typing import TYPE_CHECKING, Final +from litellm.exceptions import BadRequestError, Timeout + if TYPE_CHECKING: from pymongo import AsyncMongoClient, MongoClient @@ -20,6 +22,19 @@ PYMONGO_INSTALL_HINT: Final = ( "Run 'pip install litellm[mongodb]' (or 'pip install pymongo') to install it." ) +MONGODB_PROVIDER: Final = "mongodb" + + +def config_error(message: str) -> BadRequestError: + """Misconfiguration is the caller's to fix, so it maps to 400 rather than the 500 + a bare ValueError would become once litellm.exception_type wraps it.""" + return BadRequestError(message=message, model=None, llm_provider=MONGODB_PROVIDER) + + +def timeout_error(message: str) -> Timeout: + return Timeout(message=message, model=None, llm_provider=MONGODB_PROVIDER) + + DEFAULT_CONNECT_TIMEOUT_MS: Final = 10_000 DEFAULT_SOCKET_TIMEOUT_MS: Final = 30_000 DEFAULT_SERVER_SELECTION_TIMEOUT_MS: Final = 10_000 @@ -45,7 +60,7 @@ def import_sync_mongo_client() -> "type[MongoClient]": try: from pymongo import MongoClient as SyncMongoClient except ImportError as e: - raise ValueError(PYMONGO_INSTALL_HINT) from e + raise config_error(PYMONGO_INSTALL_HINT) from e return SyncMongoClient @@ -53,7 +68,7 @@ def import_async_mongo_client() -> "type[AsyncMongoClient]": try: from pymongo import AsyncMongoClient as AsyncMongoClientClass except ImportError as e: - raise ValueError(PYMONGO_INSTALL_HINT) from e + raise config_error(PYMONGO_INSTALL_HINT) from e return AsyncMongoClientClass @@ -105,19 +120,19 @@ def _index_hint(index_name: str, database: str, collection: str) -> str: ) -def missing_index_error(index_name: str, database: str, collection: str) -> ValueError: +def missing_index_error(index_name: str, database: str, collection: str) -> BadRequestError: """$vectorSearch against a missing index, database or collection returns zero documents instead of failing, so an empty result set is checked against the index catalogue and turned into this rather than being reported as 'no matches'.""" - return ValueError( + return config_error( f"{_index_hint(index_name, database, collection)} A vector search against a database, " "collection or index that does not exist returns no results rather than an error, so this " "was reported as an empty result set by MongoDB." ) -def index_not_ready_error(index_name: str, database: str, collection: str, status: str) -> ValueError: - return ValueError( +def index_not_ready_error(index_name: str, database: str, collection: str, status: str) -> BadRequestError: + return config_error( f"The Atlas Vector Search index '{index_name}' on '{database}.{collection}' is not queryable " f"yet; its status is {status}. Searches against it return no results until the build finishes." ) @@ -141,46 +156,47 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll return error if isinstance(error, ServerSelectionTimeoutError): - return ValueError( + return timeout_error( "Could not reach the MongoDB deployment before the timeout. On Atlas this is usually the " "project's IP access list not containing this host, or a paused cluster; it can also be an " f"unresolvable hostname. Driver detail: {error}" ) + # ExecutionTimeout subclasses OperationFailure, so it has to be matched before it + if isinstance(error, (NetworkTimeout, ExecutionTimeout)): + return timeout_error( + f"The MongoDB vector search against '{database}.{collection}' timed out before returning. " + f"Driver detail: {error}" + ) if isinstance(error, OperationFailure): code: Final = error.code if code in (_AUTHENTICATION_FAILED_CODE, _UNAUTHORIZED_CODE): - return ValueError( + return config_error( "MongoDB rejected the credentials in mongodb_connection_string, or the database user " f"lacks read access to '{database}.{collection}'. Driver detail: {error.details}" ) detail: Final = str(error).lower() if "dimension" in detail: - return ValueError( + return config_error( "The query embedding does not match the vector dimensions the Atlas index was built for. " "litellm_embedding_model must be the same model that produced the stored vectors. " f"Driver detail: {error}" ) if "is not indexed as vector" in detail: - return ValueError( + return config_error( "mongodb_embedding_field names a field the Atlas Vector Search index does not cover. " f"It must match the 'path' the index '{index_name}' was created on. Driver detail: {error}" ) if "index" in detail and ("not found" in detail or "does not exist" in detail or "unknown" in detail): - return ValueError(f"{_index_hint(index_name, database, collection)} Driver detail: {error}") - return ValueError( + return config_error(f"{_index_hint(index_name, database, collection)} Driver detail: {error}") + return config_error( f"MongoDB rejected the vector search against '{database}.{collection}' using index " f"'{index_name}'. Driver detail: {error}" ) - if isinstance(error, (NetworkTimeout, ExecutionTimeout)): - return ValueError( - f"The MongoDB vector search against '{database}.{collection}' timed out before returning. " - f"Driver detail: {error}" - ) if isinstance(error, ConfigurationError): - return ValueError( + return config_error( "mongodb_connection_string is not a usable MongoDB connection string. " f"Driver detail: {error}" ) if isinstance(error, InvalidOperation): - return ValueError(f"The MongoDB client was already closed or is unusable. Driver detail: {error}") + return config_error(f"The MongoDB client was already closed or is unusable. Driver detail: {error}") return error diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index 20dcf62dcc3..2570e368990 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -24,6 +24,7 @@ from litellm.llms.mongodb.common_utils import ( DEFAULT_SERVER_SELECTION_TIMEOUT_MS, DEFAULT_SOCKET_TIMEOUT_MS, MongoClientKey, + config_error, get_async_client, get_sync_client, index_not_ready_error, @@ -88,7 +89,7 @@ class _MongoDBSearchParams(BaseModel): def require_embedding_model(self) -> str: if not self.litellm_embedding_model: - raise ValueError( + raise config_error( "litellm_embedding_model is required in litellm_params for the MongoDB vector store. " "It must be the same model that produced the vectors stored in " f"'{self.mongodb_collection or ''}.{self.embedding_field}', or search results " @@ -98,13 +99,13 @@ class _MongoDBSearchParams(BaseModel): def require_connection_string(self) -> str: if not self.mongodb_connection_string: - raise ValueError( + raise config_error( "mongodb_connection_string is required in litellm_params for the MongoDB vector store. " "Example: mongodb+srv://:@.mongodb.net" ) scheme: Final = self.mongodb_connection_string.split("://", 1)[0].lower() if scheme not in ("mongodb", "mongodb+srv"): - raise ValueError( + raise config_error( "mongodb_connection_string must start with 'mongodb://' or 'mongodb+srv://', " f"got '{self.mongodb_connection_string.split('://', 1)[0]}://'" ) @@ -112,7 +113,7 @@ class _MongoDBSearchParams(BaseModel): def require_database(self) -> str: if not self.mongodb_database: - raise ValueError( + raise config_error( "mongodb_database is required in litellm_params for the MongoDB vector store. " "Example: mongodb_database: sample_mflix" ) @@ -120,7 +121,7 @@ class _MongoDBSearchParams(BaseModel): def require_collection(self) -> str: if not self.mongodb_collection: - raise ValueError( + raise config_error( "mongodb_collection is required in litellm_params for the MongoDB vector store. " "Example: mongodb_collection: embedded_movies" ) @@ -145,9 +146,9 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): def _query_text(query: str | Sequence[str]) -> str: text: Final = query if isinstance(query, str) else " ".join(query) if not text.strip(): - raise ValueError("query must not be empty") + raise config_error("query must not be empty") if len(text) > MAX_QUERY_CHARACTERS: - raise ValueError(f"query must be at most {MAX_QUERY_CHARACTERS} characters, got {len(text)}") + raise config_error(f"query must be at most {MAX_QUERY_CHARACTERS} characters, got {len(text)}") return text @staticmethod @@ -156,7 +157,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): if requested is None: return DEFAULT_MAX_NUM_RESULTS if not MIN_MAX_NUM_RESULTS <= requested <= MAX_MAX_NUM_RESULTS: - raise ValueError( + raise config_error( f"max_num_results must be between {MIN_MAX_NUM_RESULTS} and {MAX_MAX_NUM_RESULTS}, got {requested}" ) return requested @@ -165,7 +166,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): def _num_candidates(limit: int, configured: int | None) -> int: if configured is not None: if not limit <= configured <= MAX_NUM_CANDIDATES: - raise ValueError( + raise config_error( f"mongodb_num_candidates must be between max_num_results ({limit}) and " f"{MAX_NUM_CANDIDATES}, got {configured}" ) @@ -199,7 +200,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, ) -> list[dict[str, object]]: if vector_store_search_optional_params.get("filters") is not None: - raise ValueError( + raise config_error( "MongoDB vector store does not support the filters parameter yet. " "Restrict the collection or the Atlas Vector Search index definition instead." ) @@ -268,7 +269,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): def _embedding_vector(embedding_response: EmbeddingResponse) -> Sequence[float]: data: Final = embedding_response.data if not data: - raise ValueError( + raise config_error( "The embedding model returned no embedding for the search query, so there is nothing " "to search MongoDB with. Check the embedding deployment named by litellm_embedding_model." ) diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index c9378015f5a..1e5bce3f0ee 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -5,8 +5,11 @@ from unittest.mock import MagicMock, patch import httpx import pytest +from litellm.exceptions import BadRequestError, Timeout from litellm.llms.mongodb.common_utils import ( MongoClientKey, + index_not_ready_error, + missing_index_error, get_async_client, get_sync_client, reset_client_cache, @@ -219,7 +222,7 @@ def test_num_candidates_can_be_overridden(): def test_num_candidates_below_the_limit_or_above_the_ceiling_is_rejected(configured): config, _, _ = _config() - with pytest.raises(ValueError, match="mongodb_num_candidates"): + with pytest.raises(BadRequestError, match="mongodb_num_candidates"): _search(config, optional_params={"max_num_results": 5}, litellm_params={"mongodb_num_candidates": configured}) @@ -296,7 +299,7 @@ def test_response_stringifies_a_non_string_document_id(): def test_search_requires_an_embedding_model(): config, _, _ = _config() - with pytest.raises(ValueError, match="litellm_embedding_model is required"): + with pytest.raises(BadRequestError, match="litellm_embedding_model is required"): config.execute_search_vector_store_request( vector_store_id=INDEX, query="q", @@ -309,7 +312,7 @@ def test_search_requires_an_embedding_model(): def test_missing_embedding_model_message_names_the_field_being_searched(): config, _, _ = _config() - with pytest.raises(ValueError, match=r"embedded_movies\.embedding"): + with pytest.raises(BadRequestError, match=r"embedded_movies\.embedding"): config.execute_search_vector_store_request( vector_store_id=INDEX, query="q", @@ -322,7 +325,7 @@ def test_missing_embedding_model_message_names_the_field_being_searched(): def test_search_requires_a_connection_string(): config, _, _ = _config() - with pytest.raises(ValueError, match="mongodb_connection_string is required"): + with pytest.raises(BadRequestError, match="mongodb_connection_string is required"): _search(config, litellm_params={"mongodb_connection_string": None}) @@ -330,7 +333,7 @@ def test_search_requires_a_connection_string(): def test_search_rejects_a_non_mongodb_connection_scheme(connection_string): config, _, _ = _config() - with pytest.raises(ValueError, match="must start with 'mongodb://' or 'mongodb\\+srv://'"): + with pytest.raises(BadRequestError, match="must start with 'mongodb://' or 'mongodb\\+srv://'"): _search(config, litellm_params={"mongodb_connection_string": connection_string}) @@ -345,21 +348,21 @@ def test_search_accepts_the_plain_mongodb_scheme(): def test_search_requires_a_database(): config, _, _ = _config() - with pytest.raises(ValueError, match="mongodb_database is required"): + with pytest.raises(BadRequestError, match="mongodb_database is required"): _search(config, litellm_params={"mongodb_database": None}) def test_search_requires_a_collection(): config, _, _ = _config() - with pytest.raises(ValueError, match="mongodb_collection is required"): + with pytest.raises(BadRequestError, match="mongodb_collection is required"): _search(config, litellm_params={"mongodb_collection": None}) def test_search_rejects_filters_rather_than_silently_ignoring_them(): config, _, _ = _config() - with pytest.raises(ValueError, match="does not support the filters parameter"): + with pytest.raises(BadRequestError, match="does not support the filters parameter"): _search(config, optional_params={"filters": {"genre": "sci-fi"}}) @@ -367,7 +370,7 @@ def test_search_rejects_filters_rather_than_silently_ignoring_them(): async def test_async_search_rejects_filters_rather_than_silently_ignoring_them(): config, _, _ = _async_config() - with pytest.raises(ValueError, match="does not support the filters parameter"): + with pytest.raises(BadRequestError, match="does not support the filters parameter"): await _asearch(config, optional_params={"filters": {"genre": "sci-fi"}}) @@ -375,14 +378,14 @@ async def test_async_search_rejects_filters_rather_than_silently_ignoring_them() def test_search_rejects_an_empty_query(query): config, _, _ = _config() - with pytest.raises(ValueError, match="query must not be empty"): + with pytest.raises(BadRequestError, match="query must not be empty"): _search(config, query=query) def test_search_rejects_an_oversized_query(): config, _, _ = _config() - with pytest.raises(ValueError, match="at most 32000 characters"): + with pytest.raises(BadRequestError, match="at most 32000 characters"): _search(config, query="x" * 32_001) @@ -398,7 +401,7 @@ def test_search_accepts_a_query_at_the_size_ceiling(): def test_search_rejects_out_of_range_max_num_results(max_num_results): config, _, _ = _config() - with pytest.raises(ValueError, match="max_num_results must be between 1 and 50"): + with pytest.raises(BadRequestError, match="max_num_results must be between 1 and 50"): _search(config, optional_params={"max_num_results": max_num_results}) @@ -422,7 +425,7 @@ def test_search_treats_an_explicit_null_max_num_results_as_the_default(): def test_search_fails_when_the_embedding_model_returns_nothing(): config, _, _ = _config(embedding=None) - with pytest.raises(ValueError, match="returned no embedding"): + with pytest.raises(BadRequestError, match="returned no embedding"): _search(config) @@ -433,7 +436,7 @@ def test_validation_runs_before_any_connection_is_opened(): sync_client_factory=lambda key: opened.append(key) or FakeClient(FakeCollection([])), ) - with pytest.raises(ValueError, match="query must not be empty"): + with pytest.raises(BadRequestError, match="query must not be empty"): _search(config, query="") assert opened == [] @@ -474,7 +477,7 @@ async def test_async_search_builds_the_same_pipeline_and_maps_the_response(): async def test_async_search_requires_an_embedding_model(): config, _, _ = _async_config() - with pytest.raises(ValueError, match="litellm_embedding_model is required"): + with pytest.raises(BadRequestError, match="litellm_embedding_model is required"): await config.aexecute_search_vector_store_request( vector_store_id=INDEX, query="q", @@ -627,7 +630,7 @@ class TestErrorTranslation: config, _, _ = _config(error=ServerSelectionTimeoutError("no servers")) - with pytest.raises(ValueError, match="IP access list"): + with pytest.raises(Timeout, match="IP access list"): _search(config) @pytest.mark.asyncio @@ -636,7 +639,7 @@ class TestErrorTranslation: config, _, _ = _async_config(error=OperationFailure("auth failed", code=18)) - with pytest.raises(ValueError, match="rejected the credentials"): + with pytest.raises(BadRequestError, match="rejected the credentials"): await _asearch(config) @@ -645,14 +648,14 @@ class TestMissingDriver: from litellm.llms.mongodb.common_utils import import_sync_mongo_client with patch.dict(sys.modules, {"pymongo": None}): - with pytest.raises(ValueError, match=r"pip install litellm\[mongodb\]"): + with pytest.raises(BadRequestError, match=r"pip install litellm\[mongodb\]"): import_sync_mongo_client() def test_the_async_import_names_the_extra_to_install(self): from litellm.llms.mongodb.common_utils import import_async_mongo_client with patch.dict(sys.modules, {"pymongo": None}): - with pytest.raises(ValueError, match=r"pip install litellm\[mongodb\]"): + with pytest.raises(BadRequestError, match=r"pip install litellm\[mongodb\]"): import_async_mongo_client() def test_error_translation_degrades_gracefully_without_the_driver(self): @@ -670,7 +673,7 @@ class TestEmptyResultsAreDisambiguated: def test_a_missing_index_becomes_an_error_rather_than_an_empty_page(self): config, _, collection = _config(documents=[], search_indexes=[]) - with pytest.raises(ValueError, match="No queryable Atlas Vector Search index"): + with pytest.raises(BadRequestError, match="No queryable Atlas Vector Search index"): _search(config) assert collection.listed_indexes == [INDEX] @@ -678,7 +681,7 @@ class TestEmptyResultsAreDisambiguated: def test_the_missing_index_error_explains_why_mongodb_reported_no_results(self): config, _, _ = _config(documents=[], search_indexes=[]) - with pytest.raises(ValueError, match="returns no results rather than an error"): + with pytest.raises(BadRequestError, match="returns no results rather than an error"): _search(config) def test_an_index_still_building_becomes_an_error_naming_its_status(self): @@ -686,7 +689,7 @@ class TestEmptyResultsAreDisambiguated: documents=[], search_indexes=[{"name": INDEX, "status": "PENDING", "queryable": False}] ) - with pytest.raises(ValueError, match="not queryable yet; its status is PENDING"): + with pytest.raises(BadRequestError, match="not queryable yet; its status is PENDING"): _search(config) def test_a_genuine_no_match_against_a_ready_index_returns_an_empty_page(self): @@ -709,7 +712,7 @@ class TestEmptyResultsAreDisambiguated: async def test_async_missing_index_becomes_an_error_rather_than_an_empty_page(self): config, _, collection = _async_config(documents=[], search_indexes=[]) - with pytest.raises(ValueError, match="No queryable Atlas Vector Search index"): + with pytest.raises(BadRequestError, match="No queryable Atlas Vector Search index"): await _asearch(config) assert collection.listed_indexes == [INDEX] @@ -720,7 +723,7 @@ class TestEmptyResultsAreDisambiguated: documents=[], search_indexes=[{"name": INDEX, "status": "PENDING", "queryable": False}] ) - with pytest.raises(ValueError, match="not queryable yet; its status is PENDING"): + with pytest.raises(BadRequestError, match="not queryable yet; its status is PENDING"): await _asearch(config) @pytest.mark.asyncio @@ -752,7 +755,7 @@ class TestEmptyResultsAreDisambiguated: sync_client_factory=lambda key: FakeClient(collection), ) - with pytest.raises(ValueError, match="lacks read access"): + with pytest.raises(BadRequestError, match="lacks read access"): _search(config) @@ -785,3 +788,75 @@ class TestAtlasPlanExecutorErrors: assert "does not match the vector dimensions" in str(translated) assert "mongodb_embedding_field" not in str(translated) + + +class TestErrorsCarryTheRightHttpStatus: + """litellm.exception_type passes a litellm exception through untouched but wraps anything + else into APIConnectionError, which the proxy serves as a 500 with a Python traceback in the + body. A misconfigured connection string is the caller's to fix, so it has to arrive as a 400. + """ + + @pytest.mark.parametrize( + "invoke", + [ + pytest.param(lambda: _search(_config()[0], query=" "), id="empty-query"), + pytest.param( + lambda: _search(_config()[0], optional_params={"max_num_results": 999}), + id="max-num-results-out-of-range", + ), + pytest.param( + lambda: _search(_config()[0], optional_params={"filters": {"genre": "Action"}}), + id="unsupported-filters", + ), + pytest.param( + lambda: _search(_config()[0], litellm_params={"mongodb_connection_string": "postgres://host/db"}), + id="wrong-uri-scheme", + ), + pytest.param( + lambda: _search(_config()[0], litellm_params={"mongodb_database": None}), id="missing-database" + ), + pytest.param( + lambda: _search(_config()[0], litellm_params={"litellm_embedding_model": None}), + id="missing-embedding-model", + ), + ], + ) + def test_configuration_failures_are_400(self, invoke): + with pytest.raises(BadRequestError) as excinfo: + invoke() + assert excinfo.value.status_code == 400 + assert excinfo.value.llm_provider == "mongodb" + + def test_missing_index_is_400(self): + error = missing_index_error("idx", "db", "coll") + assert error.status_code == 400 + assert error.llm_provider == "mongodb" + + def test_index_still_building_is_400(self): + error = index_not_ready_error("idx", "db", "coll", "PENDING") + assert error.status_code == 400 + + def test_unreachable_deployment_is_a_timeout_not_a_bad_request(self): + from pymongo.errors import ServerSelectionTimeoutError + + translated = translate_mongo_error( + ServerSelectionTimeoutError("no servers"), index_name="idx", database="db", collection="coll" + ) + assert isinstance(translated, Timeout) + assert translated.status_code == 408 + + def test_query_execution_timeout_is_a_timeout(self): + from pymongo.errors import ExecutionTimeout + + translated = translate_mongo_error( + ExecutionTimeout("too slow"), index_name="idx", database="db", collection="coll" + ) + assert isinstance(translated, Timeout) + assert translated.status_code == 408 + + def test_unrecognised_errors_are_not_relabelled_as_bad_requests(self): + original = RuntimeError("something else entirely") + assert ( + translate_mongo_error(original, index_name="idx", database="db", collection="coll") + is original + ) From 85431297b91f8b0dec48648e2244c837de6c743e Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 10:03:23 -0700 Subject: [PATCH 13/60] fix(vector_stores): name the connection string when Atlas rejects MongoDB credentials Atlas answers a wrong password with code 8000 "AtlasError" rather than the 18 a self-hosted deployment returns, so the code-only check never fired and a bad password came back as a generic "MongoDB rejected the vector search", pointing the reader at the index instead of at their credentials. Verified live against Atlas with a tampered password. --- litellm/llms/mongodb/common_utils.py | 9 +++++-- .../test_mongodb_transformation.py | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 299a9772817..12391b80959 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -110,6 +110,9 @@ def reset_client_cache() -> None: _AUTHENTICATION_FAILED_CODE: Final = 18 _UNAUTHORIZED_CODE: Final = 13 +# Atlas reports a rejected user as code 8000 "AtlasError" rather than 18, so the +# message is the only reliable signal for a serverless or shared-tier deployment. +_AUTHENTICATION_MESSAGE_MARKERS: Final = ("bad auth", "authentication failed", "not authorized") def _index_hint(index_name: str, database: str, collection: str) -> str: @@ -169,12 +172,14 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll ) if isinstance(error, OperationFailure): code: Final = error.code - if code in (_AUTHENTICATION_FAILED_CODE, _UNAUTHORIZED_CODE): + detail: Final = str(error).lower() + if code in (_AUTHENTICATION_FAILED_CODE, _UNAUTHORIZED_CODE) or any( + marker in detail for marker in _AUTHENTICATION_MESSAGE_MARKERS + ): return config_error( "MongoDB rejected the credentials in mongodb_connection_string, or the database user " f"lacks read access to '{database}.{collection}'. Driver detail: {error.details}" ) - detail: Final = str(error).lower() if "dimension" in detail: return config_error( "The query embedding does not match the vector dimensions the Atlas index was built for. " diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 1e5bce3f0ee..15e1b1efab5 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -860,3 +860,30 @@ class TestErrorsCarryTheRightHttpStatus: translate_mongo_error(original, index_name="idx", database="db", collection="coll") is original ) + + +def test_atlas_rejected_credentials_are_named_even_though_the_code_is_8000(): + """Atlas answers a wrong password with code 8000 "AtlasError", not the 18 that a + self-hosted deployment returns, so a code-only check reports it as a generic + rejected search and never tells the caller to look at their connection string.""" + from pymongo.errors import OperationFailure + + error = OperationFailure( + "bad auth : authentication failed", + code=8000, + details={"ok": 0, "errmsg": "bad auth : authentication failed", "code": 8000, "codeName": "AtlasError"}, + ) + translated = translate_mongo_error(error, index_name="idx", database="sample_mflix", collection="embedded_movies") + + assert isinstance(translated, BadRequestError) + assert "mongodb_connection_string" in str(translated) + assert "sample_mflix.embedded_movies" in str(translated) + + +def test_a_rejected_search_that_is_not_an_auth_failure_keeps_the_generic_message(): + from pymongo.errors import OperationFailure + + error = OperationFailure("PlanExecutor error", code=8, details={"errmsg": "PlanExecutor error"}) + translated = translate_mongo_error(error, index_name="idx", database="db", collection="coll") + + assert "mongodb_connection_string" not in str(translated) From 1f8cbee8aa306c5f169d11b49dbf1fe28010ffd4 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 10:22:01 -0700 Subject: [PATCH 14/60] feat(ui): add MongoDB Atlas to the vector store provider dropdown The create form now offers MongoDB Atlas with its connection string, database, collection, embedding model, vector field, text field and candidate count. The connection string renders as a password input because it carries the database user's password, and the embedding model is picked from the proxy's own models, matching how Milvus and Valkey do it. The vector store id doubles as the Atlas Vector Search index name, so the placeholder says so. --- .../public/assets/logos/mongodb.svg | 6 ++ .../_components/VectorStoreForm.test.tsx | 51 ++++++++++++++ .../_components/VectorStoreForm.tsx | 20 +++++- .../vector_store_providers.test.tsx | 41 +++++++++++ .../src/components/vector_store_providers.tsx | 69 +++++++++++++++++++ 5 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 ui/litellm-dashboard/public/assets/logos/mongodb.svg diff --git a/ui/litellm-dashboard/public/assets/logos/mongodb.svg b/ui/litellm-dashboard/public/assets/logos/mongodb.svg new file mode 100644 index 00000000000..fb0d3cbdfab --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/mongodb.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx index 71e2a7224ae..94aa6cc99a0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx @@ -110,6 +110,57 @@ describe("buildVectorStoreLitellmParams", () => { }); }); + it("renames embedding_model to litellm_embedding_model for mongodb", () => { + const params = buildVectorStoreLitellmParams("mongodb", { + mongodb_connection_string: "mongodb+srv://user:pass@cluster0.mongodb.net", + mongodb_database: "sample_mflix", + mongodb_collection: "embedded_movies", + mongodb_embedding_field: "plot_embedding", + mongodb_text_field: "plot", + mongodb_num_candidates: "200", + embedding_model: "text-embedding-ada-002", + }); + + expect(params).toEqual({ + mongodb_connection_string: "mongodb+srv://user:pass@cluster0.mongodb.net", + mongodb_database: "sample_mflix", + mongodb_collection: "embedded_movies", + mongodb_embedding_field: "plot_embedding", + mongodb_text_field: "plot", + mongodb_num_candidates: "200", + litellm_embedding_model: "text-embedding-ada-002", + }); + }); + + it("sends only mongodb fields when an earlier provider left values in the form", () => { + const params = buildVectorStoreLitellmParams("mongodb", { + mongodb_connection_string: "mongodb+srv://user:pass@cluster0.mongodb.net", + mongodb_database: "sample_mflix", + mongodb_collection: "embedded_movies", + embedding_model: "text-embedding-ada-002", + valkey_host: "left-over-from-valkey.example.com", + valkey_port: "6379", + aws_region_name: "us-west-2", + }); + + expect(params).not.toHaveProperty("valkey_host"); + expect(params).not.toHaveProperty("valkey_port"); + expect(params).not.toHaveProperty("aws_region_name"); + expect(params.mongodb_connection_string).toBe("mongodb+srv://user:pass@cluster0.mongodb.net"); + }); + + it("omits a blank mongodb_num_candidates so litellm picks its own candidate count", () => { + const params = buildVectorStoreLitellmParams("mongodb", { + mongodb_connection_string: "mongodb+srv://user:pass@cluster0.mongodb.net", + mongodb_database: "sample_mflix", + mongodb_collection: "embedded_movies", + embedding_model: "text-embedding-ada-002", + }); + + expect(params.mongodb_num_candidates).toBeUndefined(); + expect(JSON.parse(JSON.stringify(params))).not.toHaveProperty("mongodb_num_candidates"); + }); + it("keeps embedding_model as-is for providers outside the rename set", () => { const params = buildVectorStoreLitellmParams("s3_vectors", { vector_bucket_name: "my-vector-bucket", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx index 9d78b727768..e25dbe30005 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx @@ -34,7 +34,7 @@ import { Textarea } from "@/components/ui/textarea"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { useZodForm } from "@/lib/forms/useZodForm"; -const EMBEDDING_MODEL_RENAME_PROVIDERS = new Set(["milvus", "valkey"]); +const EMBEDDING_MODEL_RENAME_PROVIDERS = new Set(["milvus", "valkey", "mongodb"]); export const buildVectorStoreLitellmParams = ( provider: string, @@ -70,6 +70,12 @@ const PROVIDER_FIELD_NAMES = [ "vector_bucket_name", "index_name", "aws_region_name", + "mongodb_connection_string", + "mongodb_database", + "mongodb_collection", + "mongodb_embedding_field", + "mongodb_text_field", + "mongodb_num_candidates", "valkey_host", "valkey_port", "valkey_password", @@ -101,6 +107,12 @@ const vectorStoreShape = { vector_bucket_name: optionalText, index_name: optionalText, aws_region_name: optionalText, + mongodb_connection_string: optionalText, + mongodb_database: optionalText, + mongodb_collection: optionalText, + mongodb_embedding_field: optionalText, + mongodb_text_field: optionalText, + mongodb_num_candidates: optionalText, valkey_host: optionalText, valkey_port: optionalText, valkey_password: optionalText, @@ -130,6 +142,8 @@ const EMPTY_VALUES: VectorStoreFormValues = { custom_llm_provider: "bedrock", vector_store_id: "", vertex_location: "global", + mongodb_embedding_field: "embedding", + mongodb_text_field: "text", valkey_port: "6379", valkey_ssl: "false", valkey_text_field: "text", @@ -262,7 +276,9 @@ const VectorStoreForm: React.FC = ({ : 'my-datastore_1234567890 (data store ID from Vertex AI / "Agent Search" console)' : selectedProvider === "valkey" ? "my-search-index (FT index name in Valkey)" - : "Enter vector store ID from your provider"; + : selectedProvider === "mongodb" + ? "my-vector-index (Atlas Vector Search index name)" + : "Enter vector store ID from your provider"; return ( !open && handleCancel()}> diff --git a/ui/litellm-dashboard/src/components/vector_store_providers.test.tsx b/ui/litellm-dashboard/src/components/vector_store_providers.test.tsx index eaf2a52853f..8e3a3aa3402 100644 --- a/ui/litellm-dashboard/src/components/vector_store_providers.test.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_providers.test.tsx @@ -28,6 +28,47 @@ describe("getVectorStoreProviderLogoAndName", () => { }); }); + it("registers mongodb in the provider, logo, and field maps", () => { + expect(getVectorStoreProviderLogoAndName("mongodb")).toEqual({ + logo: expect.stringContaining("mongodb"), + displayName: VectorStoreProviders.MongoDB, + }); + expect(vectorStoreProviderMap.MongoDB).toBe("mongodb"); + expect(getProviderSpecificFields("mongodb").map((field) => field.name)).toEqual([ + "mongodb_connection_string", + "mongodb_database", + "mongodb_collection", + "embedding_model", + "mongodb_embedding_field", + "mongodb_text_field", + "mongodb_num_candidates", + ]); + }); + + it("hides the mongodb connection string, which carries the database password", () => { + const connectionString = getProviderSpecificFields("mongodb").find( + (field) => field.name === "mongodb_connection_string", + ); + + expect(connectionString).toMatchObject({ type: "password", required: true }); + }); + + it("picks the mongodb embedding model from the proxy's models rather than a fixed list", () => { + const embeddingField = getProviderSpecificFields("mongodb").find((field) => field.name === "embedding_model"); + + expect(embeddingField).toMatchObject({ type: "select", required: true }); + expect(embeddingField).not.toHaveProperty("options"); + }); + + it("defaults the mongodb field names so a standard collection needs no extra input", () => { + const fields = getProviderSpecificFields("mongodb"); + const byName = (name: string) => fields.find((field) => field.name === name); + + expect(byName("mongodb_embedding_field")).toMatchObject({ required: false, initialValue: "embedding" }); + expect(byName("mongodb_text_field")).toMatchObject({ required: false, initialValue: "text" }); + expect(byName("mongodb_num_candidates")).toMatchObject({ required: false }); + }); + it("registers valkey in the provider, logo, and field maps", () => { expect(vectorStoreProviderMap.Valkey).toBe("valkey"); expect(vectorStoreProviderLogoMap[VectorStoreProviders.Valkey]).toContain("valkey"); diff --git a/ui/litellm-dashboard/src/components/vector_store_providers.tsx b/ui/litellm-dashboard/src/components/vector_store_providers.tsx index 35cd5c383f7..a75f10771a8 100644 --- a/ui/litellm-dashboard/src/components/vector_store_providers.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_providers.tsx @@ -1,5 +1,6 @@ import { getProviderLogoAndName, Providers, providerLogoMap } from "@/components/provider_info_helpers"; import milvusLogo from "../../public/assets/logos/milvus.svg"; +import mongodbLogo from "../../public/assets/logos/mongodb.svg"; import postgresqlLogo from "../../public/assets/logos/postgresql.svg"; import s3VectorLogo from "../../public/assets/logos/s3_vector.png"; import valkeyLogo from "../../public/assets/logos/valkey.svg"; @@ -13,6 +14,7 @@ export enum VectorStoreProviders { OpenAI = "OpenAI", Azure = "Azure OpenAI", Milvus = "Milvus", + MongoDB = "MongoDB Atlas", Valkey = "Valkey", } @@ -24,6 +26,7 @@ export const vectorStoreProviderMap: Record = { OpenAI: "openai", Azure: "azure", Milvus: "milvus", + MongoDB: "mongodb", S3Vectors: "s3_vectors", Valkey: "valkey", }; @@ -36,6 +39,7 @@ export const vectorStoreProviderLogoMap: Record = { [VectorStoreProviders.OpenAI]: providerLogoMap[Providers.OpenAI] ?? "", [VectorStoreProviders.Azure]: providerLogoMap[Providers.Azure] ?? "", [VectorStoreProviders.Milvus]: milvusLogo.src, + [VectorStoreProviders.MongoDB]: mongodbLogo.src, [VectorStoreProviders.S3Vectors]: s3VectorLogo.src, [VectorStoreProviders.Valkey]: valkeyLogo.src, }; @@ -169,6 +173,71 @@ export const vectorStoreProviderFields: Record type: "select", }, ], + mongodb: [ + { + name: "mongodb_connection_string", + label: "Connection String", + tooltip: + "The full MongoDB connection string for your Atlas cluster, including the database user and password. Copy it from Atlas under Connect, Drivers (e.g. mongodb+srv://user:password@cluster.mongodb.net)", + placeholder: "mongodb+srv://user:password@cluster.mongodb.net", + required: true, + type: "password", + }, + { + name: "mongodb_database", + label: "Database", + tooltip: "The Atlas database holding the collection you want to search", + placeholder: "sample_mflix", + required: true, + type: "text", + }, + { + name: "mongodb_collection", + label: "Collection", + tooltip: "The collection your Atlas Vector Search index was built on", + placeholder: "embedded_movies", + required: true, + type: "text", + }, + { + name: "embedding_model", + label: "Embedding Model", + tooltip: + "The embedding model on this proxy that created the vectors already stored in your collection. LiteLLM embeds every search query with it, so it must be the same model. A different model of the same size will not error, it will just return wrong results. Add it under Models first if it is not listed", + placeholder: "text-embedding-3-small", + required: true, + type: "select", + }, + { + name: "mongodb_embedding_field", + label: "Vector Field Name", + tooltip: + "The field in each document that holds its embedding. It must match the path your Atlas Vector Search index was created on (default: embedding)", + placeholder: "embedding", + required: false, + type: "text", + initialValue: "embedding", + }, + { + name: "mongodb_text_field", + label: "Text Field", + tooltip: + "The field in each document that holds its readable text. LiteLLM returns this text in search results, and it accepts a dotted path such as metadata.body (default: text)", + placeholder: "text", + required: false, + type: "text", + initialValue: "text", + }, + { + name: "mongodb_num_candidates", + label: "Candidates Considered", + tooltip: + "How many nearest neighbours Atlas examines before returning the top results. Higher is more accurate and slower. Leave blank to let LiteLLM scale it with the requested result count", + placeholder: "100", + required: false, + type: "text", + }, + ], valkey: [ { name: "valkey_host", From a472484291fcc5b5a386c92855130c58e423a7e1 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 10:36:36 -0700 Subject: [PATCH 15/60] fix(vector_stores): stop MongoDB handing a new event loop a closed loop's client The async client cache was keyed on id(loop). CPython recycles those ids so aggressively that a fresh event loop nearly always lands on the id of one already collected, measured at 37 of 40 rounds, so the cache handed the new loop an AsyncMongoClient bound to a closed loop and every operation on it raised "Event loop is closed". The entry now carries a weak reference to the loop it was built on and a hit only counts when that reference still points at the running loop, so a recycled id misses and builds a fresh client. A stale entry can also be replaced once the cache is full, which the old size check prevented. pymongo's own client keeps its loop alive, which is why the sync proxy path never saw this; a script calling asyncio.run() per search, or a test suite with a loop per test, does. --- litellm/llms/mongodb/common_utils.py | 20 ++++++--- .../test_mongodb_transformation.py | 43 +++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 12391b80959..4aafaf86a5e 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -9,6 +9,8 @@ TLS handshake and topology discovery: measured at ~890ms against Atlas versus """ import asyncio +import weakref +from asyncio import AbstractEventLoop from dataclasses import dataclass from typing import TYPE_CHECKING, Final @@ -53,7 +55,12 @@ class MongoClientKey: _sync_clients: dict[MongoClientKey, "MongoClient"] = {} # mutable-ok: process-level connection cache, see module docstring -_async_clients: dict[tuple[MongoClientKey, int], "AsyncMongoClient"] = {} # mutable-ok: same cache, keyed per event loop +# The value carries a weak reference to the loop the client was built on: CPython recycles +# id() aggressively (measured: 200 of 200 fresh loops landed on an id already in this cache), +# so the id alone would hand a new loop a client bound to a closed one. +_async_clients: dict[ # mutable-ok: same cache, keyed per event loop + tuple[MongoClientKey, int], tuple["weakref.ref[AbstractEventLoop]", "AsyncMongoClient"] +] = {} def import_sync_mongo_client() -> "type[MongoClient]": @@ -93,13 +100,14 @@ def get_sync_client(key: MongoClientKey) -> "MongoClient": def get_async_client(key: MongoClientKey) -> "AsyncMongoClient": """Async clients bind to the loop that created them, so the cache is keyed per loop.""" - loop_key: Final = (key, id(asyncio.get_running_loop())) + loop: Final = asyncio.get_running_loop() + loop_key: Final = (key, id(loop)) cached: Final = _async_clients.get(loop_key) - if cached is not None: - return cached + if cached is not None and cached[0]() is loop: + return cached[1] client: Final = import_async_mongo_client()(key.connection_string, **_client_kwargs(key)) - if len(_async_clients) < _MAX_CACHED_CLIENTS: - _async_clients[loop_key] = client + if len(_async_clients) < _MAX_CACHED_CLIENTS or loop_key in _async_clients: + _async_clients[loop_key] = (weakref.ref(loop), client) return client diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 15e1b1efab5..9e2bccc3f26 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -1,4 +1,7 @@ +import asyncio +import gc import sys +import weakref from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -541,6 +544,46 @@ class TestClientCache: assert first is second + def test_a_new_loop_never_inherits_a_closed_loop_client(self): + """CPython recycles id() so aggressively that a fresh event loop almost always lands on + the id of one already collected: measured at 37 of 40 rounds. Keying the cache on the id + alone therefore hands the new loop an AsyncMongoClient bound to a closed loop, and every + operation on it raises "Event loop is closed".""" + + class LoopAgnosticClient: + """Holds no reference to the loop, unlike pymongo's, whose own reference happens to + keep ids from being recycled and hides the bug until the cache fills.""" + + def __init__(self, *args, **kwargs): + self.built_on = None + + key = self._key() + clients_handed_out = [] + + async def fetch(): + return get_async_client(key) + + with patch("litellm.llms.mongodb.common_utils.import_async_mongo_client") as importer: + importer.return_value = LoopAgnosticClient + + for _ in range(20): + loop = asyncio.new_event_loop() + client = loop.run_until_complete(fetch()) + clients_handed_out.append((client, client.built_on, loop.is_closed())) + client.built_on = weakref.ref(loop) + loop.close() + del loop + gc.collect() + + stale = [ + handed_out + for client, built_on, _ in clients_handed_out + if built_on is not None and (built_on() is None or built_on().is_closed()) + for handed_out in (client,) + ] + assert stale == [], f"{len(stale)} of 20 loops were handed a client built on a closed loop" + + class TestClientKeyDerivation: def test_no_timeout_uses_the_bounded_defaults(self): key = MongoDBVectorStoreConfig._client_key(_MongoDBSearchParams.model_validate(BASE_PARAMS), None) From 1fed1029e01c84f447f52119ef96757a62ae69b0 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 11:12:51 -0700 Subject: [PATCH 16/60] fix(vector_stores): report a MongoDB text field that no matched document has Atlas matches on the vector alone, so a mistyped mongodb_text_field still returns confidently scored results whose content is empty, and the model is handed an empty context with nothing to explain it. When every matched document lacks the field the search now says which setting to fix; a sparse document among others that do have it, and a document whose text is genuinely the empty string, both still come back normally. Unrecognised mongodb_* parameters are named too. The params model has to ignore unrelated keys because litellm_params carries plenty of them, which turned a mistyped mongodb_collection into "mongodb_collection is required" pointing the reader at a key they can see they have set. --- .../mongodb/vector_stores/transformation.py | 54 ++++++++++++++++-- .../test_mongodb_transformation.py | 57 ++++++++++++++++++- 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index 2570e368990..8b8ca5188cf 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -128,6 +128,12 @@ class _MongoDBSearchParams(BaseModel): return self.mongodb_collection +_MONGODB_PARAM_PREFIX: Final = "mongodb_" +_KNOWN_MONGODB_PARAMS: Final = frozenset( + name for name in _MongoDBSearchParams.model_fields if name.startswith(_MONGODB_PARAM_PREFIX) +) + + class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): def __init__( self, @@ -142,6 +148,22 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): self.sync_client_factory = sync_client_factory if sync_client_factory is not None else get_sync_client self.async_client_factory = async_client_factory if async_client_factory is not None else get_async_client + @staticmethod + def _reject_unknown_params(litellm_params: Mapping[str, object]) -> None: + """The params model ignores unrelated keys because litellm_params carries plenty of them, + which would otherwise turn a mistyped mongodb_collection into 'mongodb_collection is + required' pointing at a key the reader can see they have set.""" + unknown: Final = sorted( + key + for key in litellm_params + if key.startswith(_MONGODB_PARAM_PREFIX) and key not in _KNOWN_MONGODB_PARAMS + ) + if unknown: + raise config_error( + f"Unrecognised MongoDB vector store parameter(s): {', '.join(unknown)}. " + f"Supported: {', '.join(sorted(_KNOWN_MONGODB_PARAMS))}." + ) + @staticmethod def _query_text(query: str | Sequence[str]) -> str: text: Final = query if isinstance(query, str) else " ".join(query) @@ -219,20 +241,22 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): ] @staticmethod - def _field_value(document: Mapping[str, object], dotted_path: str) -> str: + def _field_value(document: Mapping[str, object], dotted_path: str) -> str | None: + """None means the path is absent from the document, which is what separates a + mistyped mongodb_text_field from a document whose text is genuinely empty.""" current: object = document for segment in dotted_path.split("."): - if not isinstance(current, Mapping): - return "" - current = current.get(segment) - return "" if current is None else str(current) + if not isinstance(current, Mapping) or segment not in current: + return None + current = current[segment] + return None if current is None else str(current) @classmethod def _to_result(cls, document: Mapping[str, object], text_field: str) -> VectorStoreSearchResult: document_id: Final = document.get("_id") identifier: Final = None if document_id is None else str(document_id) content: Final = [ # mutable-ok: VectorStoreSearchResult declares a list of content parts - VectorStoreResultContent(text=cls._field_value(document, text_field), type="text") + VectorStoreResultContent(text=cls._field_value(document, text_field) or "", type="text") ] raw_score: Final = document.get(SCORE_FIELD_NAME) return VectorStoreSearchResult( @@ -242,6 +266,20 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): filename=identifier, ) + @classmethod + def _raise_for_missing_text_field( + cls, documents: Sequence[Mapping[str, object]], text_field: str, database: str, collection: str + ) -> None: + """Atlas happily matches vectors in documents that carry no text at all, so a mistyped + mongodb_text_field returns well-scored results whose content is empty and feeds an empty + context to the model. Every matched document lacking the field is the misconfiguration.""" + if documents and all(cls._field_value(document, text_field) is None for document in documents): + raise config_error( + f"None of the {len(documents)} matched documents in '{database}.{collection}' has a " + f"'{text_field}' field, so every result would carry empty text. Set mongodb_text_field " + "to the field holding the readable text; it accepts a dotted path such as metadata.body." + ) + @classmethod def _to_response( cls, documents: Sequence[Mapping[str, object]], query_text: str, text_field: str @@ -284,6 +322,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): litellm_params: Mapping[str, object], timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: + self._reject_unknown_params(litellm_params) params: Final = _MongoDBSearchParams.model_validate(litellm_params) query_text: Final = self._query_text(query) key: Final = self._client_key(params, timeout) @@ -315,6 +354,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): e, index_name=vector_store_id, database=database, collection=collection ) from e self._raise_for_unusable_index(catalogue, vector_store_id, database, collection) + self._raise_for_missing_text_field(documents, params.text_field, database, collection) return self._to_response(documents, query_text, params.text_field) async def aexecute_search_vector_store_request( @@ -326,6 +366,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): litellm_params: Mapping[str, object], timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: + self._reject_unknown_params(litellm_params) params: Final = _MongoDBSearchParams.model_validate(litellm_params) query_text: Final = self._query_text(query) key: Final = self._client_key(params, timeout) @@ -359,6 +400,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): e, index_name=vector_store_id, database=database, collection=collection ) from e self._raise_for_unusable_index(catalogue, vector_store_id, database, collection) + self._raise_for_missing_text_field(documents, params.text_field, database, collection) return self._to_response(documents, query_text, params.text_field) def transform_create_vector_store_request( diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 9e2bccc3f26..68437487176 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -275,12 +275,30 @@ def test_response_reads_a_dotted_text_field_path(): assert response["data"][0]["content"][0]["text"] == "nested text" -def test_response_tolerates_a_document_missing_the_text_field(): - config, _, _ = _config(documents=[{"_id": 1, "score": 0.5}]) +def test_response_tolerates_a_sparse_document_missing_the_text_field(): + config, _, _ = _config(documents=[{"_id": 1, "score": 0.5}, {"_id": 2, "text": "has text", "score": 0.4}]) response = _search(config) assert response["data"][0]["content"][0]["text"] == "" + assert response["data"][1]["content"][0]["text"] == "has text" + + +def test_a_present_but_empty_text_field_is_not_treated_as_a_misconfiguration(): + config, _, _ = _config(documents=[{"_id": 1, "text": "", "score": 0.5}]) + + response = _search(config) + + assert response["data"][0]["content"][0]["text"] == "" + + +def test_matches_that_all_lack_the_text_field_name_the_setting_to_fix(): + """Atlas matches on the vector, so a mistyped mongodb_text_field returns confidently + scored results whose content is empty and hands the model an empty context.""" + config, _, _ = _config(documents=[{"_id": 1, "score": 0.9}, {"_id": 2, "score": 0.8}]) + + with pytest.raises(BadRequestError, match="mongodb_text_field"): + _search(config) def test_response_tolerates_a_document_missing_a_score(): @@ -930,3 +948,38 @@ def test_a_rejected_search_that_is_not_an_auth_failure_keeps_the_generic_message translated = translate_mongo_error(error, index_name="idx", database="db", collection="coll") assert "mongodb_connection_string" not in str(translated) + + +class TestUnrecognisedParameters: + """litellm_params carries plenty of keys this provider does not own, so the params model has + to ignore extras. That turns a mistyped mongodb_collection into 'mongodb_collection is + required', pointing the reader at a key they can see they have set.""" + + def test_a_mistyped_parameter_is_named(self): + config, _, _ = _config() + + with pytest.raises(BadRequestError, match="mongodb_collectoin"): + _search(config, litellm_params={"mongodb_collectoin": "embedded_movies"}) + + def test_the_supported_names_are_listed(self): + config, _, _ = _config() + + with pytest.raises(BadRequestError, match="mongodb_connection_string"): + _search(config, litellm_params={"mongodb_databse": "sample_mflix"}) + + def test_unrelated_litellm_params_are_still_ignored(self): + config, _, _ = _config(documents=[{"_id": 1, "text": "hit", "score": 0.9}]) + + response = _search( + config, + litellm_params={"use_litellm_proxy": False, "use_in_pass_through": False, "vector_store_id": "x"}, + ) + + assert len(response["data"]) == 1 + + @pytest.mark.asyncio + async def test_the_async_path_rejects_them_too(self): + config, _, _ = _async_config() + + with pytest.raises(BadRequestError, match="mongodb_collectoin"): + await _asearch(config, litellm_params={"mongodb_collectoin": "embedded_movies"}) From 9434e563f33bc06165da3e7dce20187bda29e76b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 11:28:33 -0700 Subject: [PATCH 17/60] fix(vector_stores): translate MongoDB client construction failures too Building the client parses the URI and, for mongodb+srv://, performs a DNS SRV lookup, so it fails on exactly the inputs a user is most likely to get wrong. It sat outside the try that translates driver errors, so a malformed URI or an unresolvable cluster escaped as a raw pymongo exception and reached the caller as a 500 with a traceback in the body. The three DNS-shaped failures are also told apart now: a lookup that ran out of time is a Timeout, a cluster name that is not in DNS says so and points at the URI Atlas shows under Connect Drivers, and anything else keeps the generic "not a usable MongoDB connection string". Verified live: a tampered scheme, a nonexistent cluster and a 1ms timeout each come back as their own message instead of a traceback. --- litellm/llms/mongodb/common_utils.py | 14 +++++ .../mongodb/vector_stores/transformation.py | 8 +-- .../test_mongodb_transformation.py | 58 +++++++++++++++++++ 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 4aafaf86a5e..48496eee170 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -121,6 +121,8 @@ _UNAUTHORIZED_CODE: Final = 13 # Atlas reports a rejected user as code 8000 "AtlasError" rather than 18, so the # message is the only reliable signal for a serverless or shared-tier deployment. _AUTHENTICATION_MESSAGE_MARKERS: Final = ("bad auth", "authentication failed", "not authorized") +_RESOLUTION_TIMEOUT_MARKERS: Final = ("resolution lifetime expired", "dns operation timed out") +_UNKNOWN_HOSTNAME_MARKERS: Final = ("dns query name does not exist", "name or service not known") def _index_hint(index_name: str, database: str, collection: str) -> str: @@ -206,6 +208,18 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll f"'{index_name}'. Driver detail: {error}" ) if isinstance(error, ConfigurationError): + configuration_detail: Final = str(error).lower() + if any(marker in configuration_detail for marker in _RESOLUTION_TIMEOUT_MARKERS): + return timeout_error( + "The DNS lookup for the cluster in mongodb_connection_string did not finish in time. " + "A mongodb+srv:// URI needs an SRV lookup before any connection is attempted, so this " + f"is DNS or the configured timeout, not MongoDB. Driver detail: {error}" + ) + if any(marker in configuration_detail for marker in _UNKNOWN_HOSTNAME_MARKERS): + return config_error( + "The cluster hostname in mongodb_connection_string does not exist in DNS. Check the " + f"cluster name against the URI Atlas shows under Connect, Drivers. Driver detail: {error}" + ) return config_error( "mongodb_connection_string is not a usable MongoDB connection string. " f"Driver detail: {error}" diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index 8b8ca5188cf..4b792accc9f 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -338,9 +338,9 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params ) - client: Final = self.sync_client_factory(key) - target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted try: + client: Final = self.sync_client_factory(key) + target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted documents: Final = list(target.aggregate(pipeline)) except Exception as e: raise translate_mongo_error( @@ -382,9 +382,9 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params ) - client: Final = self.async_client_factory(key) - target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted try: + client: Final = self.async_client_factory(key) + target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted cursor: Final = await target.aggregate(pipeline) documents: Final = [document async for document in cursor] except Exception as e: diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 68437487176..84de42d2126 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -983,3 +983,61 @@ class TestUnrecognisedParameters: with pytest.raises(BadRequestError, match="mongodb_collectoin"): await _asearch(config, litellm_params={"mongodb_collectoin": "embedded_movies"}) + + +class TestClientConstructionFailures: + """Building the client parses the URI and, for mongodb+srv://, performs a DNS SRV lookup, so it + fails on exactly the inputs a user is most likely to get wrong. Constructing it outside the + translation boundary let those escape as raw pymongo errors, which litellm.exception_type then + wrapped into a 500 with a traceback in the body.""" + + def _config_that_fails_to_connect(self, error): + def factory(_key): + raise error + + return MongoDBVectorStoreConfig( + embedding_fn=FakeEmbeddingFn([0.1, 0.2, 0.3]), sync_client_factory=factory + ) + + def _async_config_that_fails_to_connect(self, error): + def factory(_key): + raise error + + return MongoDBVectorStoreConfig( + aembedding_fn=FakeAsyncEmbeddingFn([0.1, 0.2, 0.3]), async_client_factory=factory + ) + + def test_a_malformed_uri_is_a_bad_request_not_a_500(self): + from pymongo.errors import InvalidURI + + config = self._config_that_fails_to_connect(InvalidURI("Invalid URI scheme")) + + with pytest.raises(BadRequestError, match="not a usable MongoDB connection string"): + _search(config) + + def test_an_unresolvable_cluster_name_says_so(self): + from pymongo.errors import ConfigurationError + + config = self._config_that_fails_to_connect(ConfigurationError("The DNS query name does not exist")) + + with pytest.raises(BadRequestError, match="does not exist in DNS"): + _search(config) + + def test_a_dns_lookup_that_ran_out_of_time_is_a_timeout(self): + from pymongo.errors import ConfigurationError + + config = self._config_that_fails_to_connect( + ConfigurationError("The resolution lifetime expired after 0.291 seconds") + ) + + with pytest.raises(Timeout, match="did not finish in time"): + _search(config) + + @pytest.mark.asyncio + async def test_the_async_path_translates_them_too(self): + from pymongo.errors import InvalidURI + + config = self._async_config_that_fails_to_connect(InvalidURI("Invalid URI scheme")) + + with pytest.raises(BadRequestError, match="not a usable MongoDB connection string"): + await _asearch(config) From 5d7bf187a41386536fa7f5db989738c4abfebfe5 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 11:30:36 -0700 Subject: [PATCH 18/60] refactor(ui): pick the vector store id placeholder from a map The chain had grown to four nested ternaries with a fifth level inside the Vertex Search branch, which no-nested-ternary had two suppressions for. A lookup keyed by provider drops both suppressions and leaves one condition, the Vertex Search case that depends on whether an engine id has been entered. Also hoists the MongoDB form fixtures in the tests, which the inline-object budget counts. --- ui/litellm-dashboard/eslint-suppressions.json | 5 -- .../_components/VectorStoreForm.test.tsx | 47 ++++++++++--------- .../_components/VectorStoreForm.tsx | 25 +++++----- 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 7de7373b20b..c2bc823ea1f 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1208,11 +1208,6 @@ "count": 1 } }, - "src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx": { - "no-nested-ternary": { - "count": 2 - } - }, "src/app/(dashboard)/vector-stores/_components/index.tsx": { "local/filename-pascal-case": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx index 94aa6cc99a0..84a9314ecce 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx @@ -69,6 +69,15 @@ describe("VectorStoreForm", () => { }); }); +const MONGODB_URI = "mongodb+srv://user:pass@cluster0.mongodb.net"; + +const MONGODB_REQUIRED_FORM_VALUES = { + mongodb_connection_string: MONGODB_URI, + mongodb_database: "sample_mflix", + mongodb_collection: "embedded_movies", + embedding_model: "text-embedding-ada-002", +}; + describe("buildVectorStoreLitellmParams", () => { it("renames embedding_model to litellm_embedding_model for valkey", () => { const valkeyFormValues = { @@ -111,51 +120,43 @@ describe("buildVectorStoreLitellmParams", () => { }); it("renames embedding_model to litellm_embedding_model for mongodb", () => { - const params = buildVectorStoreLitellmParams("mongodb", { - mongodb_connection_string: "mongodb+srv://user:pass@cluster0.mongodb.net", - mongodb_database: "sample_mflix", - mongodb_collection: "embedded_movies", + const formValues = { + ...MONGODB_REQUIRED_FORM_VALUES, mongodb_embedding_field: "plot_embedding", mongodb_text_field: "plot", mongodb_num_candidates: "200", - embedding_model: "text-embedding-ada-002", - }); - - expect(params).toEqual({ - mongodb_connection_string: "mongodb+srv://user:pass@cluster0.mongodb.net", + }; + const expected = { + mongodb_connection_string: MONGODB_URI, mongodb_database: "sample_mflix", mongodb_collection: "embedded_movies", mongodb_embedding_field: "plot_embedding", mongodb_text_field: "plot", mongodb_num_candidates: "200", litellm_embedding_model: "text-embedding-ada-002", - }); + }; + + expect(buildVectorStoreLitellmParams("mongodb", formValues)).toEqual(expected); }); it("sends only mongodb fields when an earlier provider left values in the form", () => { - const params = buildVectorStoreLitellmParams("mongodb", { - mongodb_connection_string: "mongodb+srv://user:pass@cluster0.mongodb.net", - mongodb_database: "sample_mflix", - mongodb_collection: "embedded_movies", - embedding_model: "text-embedding-ada-002", + const formValues = { + ...MONGODB_REQUIRED_FORM_VALUES, valkey_host: "left-over-from-valkey.example.com", valkey_port: "6379", aws_region_name: "us-west-2", - }); + }; + + const params = buildVectorStoreLitellmParams("mongodb", formValues); expect(params).not.toHaveProperty("valkey_host"); expect(params).not.toHaveProperty("valkey_port"); expect(params).not.toHaveProperty("aws_region_name"); - expect(params.mongodb_connection_string).toBe("mongodb+srv://user:pass@cluster0.mongodb.net"); + expect(params.mongodb_connection_string).toBe(MONGODB_URI); }); it("omits a blank mongodb_num_candidates so litellm picks its own candidate count", () => { - const params = buildVectorStoreLitellmParams("mongodb", { - mongodb_connection_string: "mongodb+srv://user:pass@cluster0.mongodb.net", - mongodb_database: "sample_mflix", - mongodb_collection: "embedded_movies", - embedding_model: "text-embedding-ada-002", - }); + const params = buildVectorStoreLitellmParams("mongodb", MONGODB_REQUIRED_FORM_VALUES); expect(params.mongodb_num_candidates).toBeUndefined(); expect(JSON.parse(JSON.stringify(params))).not.toHaveProperty("mongodb_num_candidates"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx index e25dbe30005..61da25874a5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx @@ -138,6 +138,17 @@ const vectorStoreSchema = z.object(vectorStoreShape).superRefine((values, ctx) = type VectorStoreFormValues = z.output; +const VECTOR_STORE_ID_PLACEHOLDERS: Record = { + vertex_rag_engine: '6917529027641081856 (corpus ID from Vertex AI / "RAG Engine" console)', + "vertex_ai/search_api": 'my-datastore_1234567890 (data store ID from Vertex AI / "Agent Search" console)', + valkey: "my-search-index (FT index name in Valkey)", + mongodb: "my-vector-index (Atlas Vector Search index name)", +}; + +const VERTEX_SEARCH_API_WITH_ENGINE_PLACEHOLDER = "Any identifier you'll use to reference this in LiteLLM"; + +const DEFAULT_VECTOR_STORE_ID_PLACEHOLDER = "Enter vector store ID from your provider"; + const EMPTY_VALUES: VectorStoreFormValues = { custom_llm_provider: "bedrock", vector_store_id: "", @@ -268,17 +279,9 @@ const VectorStoreForm: React.FC = ({ }; const vectorStoreIdPlaceholder = - selectedProvider === "vertex_rag_engine" - ? '6917529027641081856 (corpus ID from Vertex AI / "RAG Engine" console)' - : selectedProvider === "vertex_ai/search_api" - ? vertexEngineId - ? "Any identifier you'll use to reference this in LiteLLM" - : 'my-datastore_1234567890 (data store ID from Vertex AI / "Agent Search" console)' - : selectedProvider === "valkey" - ? "my-search-index (FT index name in Valkey)" - : selectedProvider === "mongodb" - ? "my-vector-index (Atlas Vector Search index name)" - : "Enter vector store ID from your provider"; + selectedProvider === "vertex_ai/search_api" && vertexEngineId + ? VERTEX_SEARCH_API_WITH_ENGINE_PLACEHOLDER + : VECTOR_STORE_ID_PLACEHOLDERS[selectedProvider] ?? DEFAULT_VECTOR_STORE_ID_PLACEHOLDER; return ( !open && handleCancel()}> From fdbee3af2527c5fc1869b536deeafb4529a550ad Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 12:09:51 -0700 Subject: [PATCH 19/60] refactor(vector_stores): build the MongoDB pipeline immutably and inject the client class The type-discipline and test-quality gates blamed the branch for 4 LIT001, 12 LIT002 and 5 TQ008 violations. Rather than suppress them: - the $vectorSearch and $project stages are MappingProxyType and the query vector a tuple, verified against live Atlas to encode identically. The outer pipeline stays a list because pymongo's common.validate_list raises "pipeline must be a list, not ", which a unit test now pins. - the client caches are Final[dict[...]] and _client_kwargs returns a MappingProxyType. - _field_value recurses over the dotted path instead of rebinding a local. - _client_key declared Final locals in one branch and reassigned them in the others, so it is split into an early-returning _timeout_ms. - the injected callables carry explicit Final[Callable[...]] annotations, which stops pyright resolving self.embedding_fn against litellm.embedding's overloads. - get_sync_client and get_async_client take an optional client_class, so the cache tests inject a recording double instead of patching the importer, and can assert the connection string and timeouts the client was built with. SensitiveDataMasker is public SDK surface, so extra_sensitive_patterns moves to the end of the signature: in slot two it silently reinterpreted an existing caller's positional override set as extra sensitive patterns. --- .../sensitive_data_masker.py | 14 +-- litellm/llms/mongodb/common_utils.py | 53 +++++---- .../mongodb/vector_stores/transformation.py | 109 ++++++++++------- .../management_endpoints.py | 2 +- .../test_sensitive_data_masker.py | 12 ++ .../test_mongodb_transformation.py | 112 ++++++++++++------ 6 files changed, 192 insertions(+), 110 deletions(-) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 3b0806ab069..fcce63e016b 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -1,13 +1,13 @@ from collections.abc import Mapping +from collections.abc import Set as AbstractSet from typing import Any, Final from pydantic import BaseModel from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER - _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( - { + ( "password", "secret", "key", @@ -23,20 +23,20 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( "certificate", "fingerprint", "tenancy", - } + ) ) class SensitiveDataMasker: def __init__( self, - sensitive_patterns: set[str] | None = None, - extra_sensitive_patterns: set[str] | None = None, - non_sensitive_overrides: set[str] | None = None, + sensitive_patterns: AbstractSet[str] | None = None, + non_sensitive_overrides: AbstractSet[str] | None = None, visible_prefix: int = 4, visible_suffix: int = 4, mask_char: str = "*", mask_short_values: bool = True, + extra_sensitive_patterns: AbstractSet[str] | None = None, ): self.sensitive_patterns = (sensitive_patterns or _DEFAULT_SENSITIVE_PATTERNS) | ( extra_sensitive_patterns or frozenset() @@ -44,7 +44,7 @@ class SensitiveDataMasker: # If any key segment matches one of these, the key is not considered sensitive # even if it also matches a sensitive pattern. For example, "input_cost_per_token" # contains "token" but "cost" overrides that — it's a pricing field, not a secret. - self.non_sensitive_overrides = non_sensitive_overrides or {"cost"} + self.non_sensitive_overrides = non_sensitive_overrides or frozenset(("cost",)) self.visible_prefix = visible_prefix self.visible_suffix = visible_suffix diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 48496eee170..8ac02552ecb 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -11,8 +11,10 @@ TLS handshake and topology discovery: measured at ~890ms against Atlas versus import asyncio import weakref from asyncio import AbstractEventLoop +from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Final +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias from litellm.exceptions import BadRequestError, Timeout @@ -54,13 +56,17 @@ class MongoClientKey: server_selection_timeout_ms: int -_sync_clients: dict[MongoClientKey, "MongoClient"] = {} # mutable-ok: process-level connection cache, see module docstring -# The value carries a weak reference to the loop the client was built on: CPython recycles -# id() aggressively (measured: 200 of 200 fresh loops landed on an id already in this cache), -# so the id alone would hand a new loop a client bound to a closed one. -_async_clients: dict[ # mutable-ok: same cache, keyed per event loop - tuple[MongoClientKey, int], tuple["weakref.ref[AbstractEventLoop]", "AsyncMongoClient"] -] = {} +SyncClientFactory: TypeAlias = Callable[..., "MongoClient"] +AsyncClientFactory: TypeAlias = Callable[..., "AsyncMongoClient"] + +_AsyncClientCacheKey: TypeAlias = tuple[MongoClientKey, int] +# The entry carries a weak reference to the loop the client was built on: CPython recycles id() +# aggressively (measured: 200 of 200 fresh loops landed on an id already in this cache), so the +# id alone would hand a new loop a client bound to a closed one. +_AsyncClientEntry: TypeAlias = tuple["weakref.ref[AbstractEventLoop]", "AsyncMongoClient"] + +_sync_clients: Final[dict[MongoClientKey, "MongoClient"]] = {} # mutable-ok: process-level client cache +_async_clients: Final[dict[_AsyncClientCacheKey, _AsyncClientEntry]] = {} # mutable-ok: same cache, per loop def import_sync_mongo_client() -> "type[MongoClient]": @@ -79,33 +85,39 @@ def import_async_mongo_client() -> "type[AsyncMongoClient]": return AsyncMongoClientClass -def _client_kwargs(key: MongoClientKey) -> dict[str, object]: - return { # mutable-ok: pymongo's client constructor takes keyword arguments - "connectTimeoutMS": key.connect_timeout_ms, - "socketTimeoutMS": key.socket_timeout_ms, - "serverSelectionTimeoutMS": key.server_selection_timeout_ms, - "appname": _APP_NAME, - } +def _client_kwargs(key: MongoClientKey) -> Mapping[str, object]: + return MappingProxyType( + { + "connectTimeoutMS": key.connect_timeout_ms, + "socketTimeoutMS": key.socket_timeout_ms, + "serverSelectionTimeoutMS": key.server_selection_timeout_ms, + "appname": _APP_NAME, + } + ) -def get_sync_client(key: MongoClientKey) -> "MongoClient": +def get_sync_client(key: MongoClientKey, client_class: SyncClientFactory | None = None) -> "MongoClient": + """``client_class`` is the injection seam the tests build fake clients through; left unset the + real pymongo class is imported at call time, keeping pymongo out of import-time dependencies.""" cached: Final = _sync_clients.get(key) if cached is not None: return cached - client: Final = import_sync_mongo_client()(key.connection_string, **_client_kwargs(key)) + build: Final = client_class if client_class is not None else import_sync_mongo_client() + client: Final = build(key.connection_string, **_client_kwargs(key)) if len(_sync_clients) < _MAX_CACHED_CLIENTS: _sync_clients[key] = client return client -def get_async_client(key: MongoClientKey) -> "AsyncMongoClient": +def get_async_client(key: MongoClientKey, client_class: AsyncClientFactory | None = None) -> "AsyncMongoClient": """Async clients bind to the loop that created them, so the cache is keyed per loop.""" loop: Final = asyncio.get_running_loop() loop_key: Final = (key, id(loop)) cached: Final = _async_clients.get(loop_key) if cached is not None and cached[0]() is loop: return cached[1] - client: Final = import_async_mongo_client()(key.connection_string, **_client_kwargs(key)) + build: Final = client_class if client_class is not None else import_async_mongo_client() + client: Final = build(key.connection_string, **_client_kwargs(key)) if len(_async_clients) < _MAX_CACHED_CLIENTS or loop_key in _async_clients: _async_clients[loop_key] = (weakref.ref(loop), client) return client @@ -221,8 +233,7 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll f"cluster name against the URI Atlas shows under Connect, Drivers. Driver detail: {error}" ) return config_error( - "mongodb_connection_string is not a usable MongoDB connection string. " - f"Driver detail: {error}" + f"mongodb_connection_string is not a usable MongoDB connection string. Driver detail: {error}" ) if isinstance(error, InvalidOperation): return config_error(f"The MongoDB client was already closed or is unusable. Driver detail: {error}") diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index 4b792accc9f..d0a0f51cd77 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -143,10 +143,18 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): async_client_factory: Callable[[MongoClientKey], object] | None = None, ) -> None: super().__init__() - self.embedding_fn = embedding_fn if embedding_fn is not None else litellm.embedding - self.aembedding_fn = aembedding_fn if aembedding_fn is not None else litellm.aembedding - self.sync_client_factory = sync_client_factory if sync_client_factory is not None else get_sync_client - self.async_client_factory = async_client_factory if async_client_factory is not None else get_async_client + self.embedding_fn: Final[Callable[..., EmbeddingResponse]] = ( + embedding_fn if embedding_fn is not None else litellm.embedding + ) + self.aembedding_fn: Final[Callable[..., Awaitable[EmbeddingResponse]]] = ( + aembedding_fn if aembedding_fn is not None else litellm.aembedding + ) + self.sync_client_factory: Final[Callable[[MongoClientKey], object]] = ( + sync_client_factory if sync_client_factory is not None else get_sync_client + ) + self.async_client_factory: Final[Callable[[MongoClientKey], object]] = ( + async_client_factory if async_client_factory is not None else get_async_client + ) @staticmethod def _reject_unknown_params(litellm_params: Mapping[str, object]) -> None: @@ -154,9 +162,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): which would otherwise turn a mistyped mongodb_collection into 'mongodb_collection is required' pointing at a key the reader can see they have set.""" unknown: Final = sorted( - key - for key in litellm_params - if key.startswith(_MONGODB_PARAM_PREFIX) and key not in _KNOWN_MONGODB_PARAMS + key for key in litellm_params if key.startswith(_MONGODB_PARAM_PREFIX) and key not in _KNOWN_MONGODB_PARAMS ) if unknown: raise config_error( @@ -196,16 +202,20 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): return min(max(limit * NUM_CANDIDATES_MULTIPLIER, MIN_NUM_CANDIDATES), MAX_NUM_CANDIDATES) @staticmethod - def _client_key(params: _MongoDBSearchParams, timeout: float | httpx.Timeout | None) -> MongoClientKey: + def _timeout_ms(timeout: float | httpx.Timeout | None) -> tuple[int, int]: + """The connect and socket budgets pymongo is built with, in that order.""" if isinstance(timeout, httpx.Timeout): - connect_ms: Final = int((timeout.connect or DEFAULT_CONNECT_TIMEOUT_MS / 1000) * 1000) - socket_ms: Final = int((timeout.read or DEFAULT_SOCKET_TIMEOUT_MS / 1000) * 1000) - elif timeout is not None: - connect_ms = min(int(float(timeout) * 1000), DEFAULT_CONNECT_TIMEOUT_MS) - socket_ms = int(float(timeout) * 1000) - else: - connect_ms = DEFAULT_CONNECT_TIMEOUT_MS - socket_ms = DEFAULT_SOCKET_TIMEOUT_MS + return ( + int((timeout.connect or DEFAULT_CONNECT_TIMEOUT_MS / 1000) * 1000), + int((timeout.read or DEFAULT_SOCKET_TIMEOUT_MS / 1000) * 1000), + ) + if timeout is None: + return DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_SOCKET_TIMEOUT_MS + return min(int(float(timeout) * 1000), DEFAULT_CONNECT_TIMEOUT_MS), int(float(timeout) * 1000) + + @classmethod + def _client_key(cls, params: _MongoDBSearchParams, timeout: float | httpx.Timeout | None) -> MongoClientKey: + connect_ms, socket_ms = cls._timeout_ms(timeout) return MongoClientKey( connection_string=params.require_connection_string(), connect_timeout_ms=connect_ms, @@ -220,36 +230,41 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): query_vector: Sequence[float], params: _MongoDBSearchParams, vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, - ) -> list[dict[str, object]]: + ) -> Sequence[Mapping[str, object]]: if vector_store_search_optional_params.get("filters") is not None: raise config_error( "MongoDB vector store does not support the filters parameter yet. " "Restrict the collection or the Atlas Vector Search index definition instead." ) limit: Final = cls._limit(vector_store_search_optional_params) - return [ # mutable-ok: pymongo's aggregate contract is a list of stage dicts + search: Final = MappingProxyType( { - "$vectorSearch": { - "index": vector_store_id, - "path": params.embedding_field, - "queryVector": list(query_vector), - "numCandidates": cls._num_candidates(limit, params.mongodb_num_candidates), - "limit": limit, - } - }, - {"$project": {params.text_field: 1, SCORE_FIELD_NAME: {"$meta": "vectorSearchScore"}}}, + "index": vector_store_id, + "path": params.embedding_field, + "queryVector": tuple(query_vector), + "numCandidates": cls._num_candidates(limit, params.mongodb_num_candidates), + "limit": limit, + } + ) + projection: Final = MappingProxyType( + {params.text_field: 1, SCORE_FIELD_NAME: MappingProxyType({"$meta": "vectorSearchScore"})} + ) + return [ # mutable-ok: pymongo rejects any non-list pipeline in common.validate_list + MappingProxyType({"$vectorSearch": search}), + MappingProxyType({"$project": projection}), ] - @staticmethod - def _field_value(document: Mapping[str, object], dotted_path: str) -> str | None: + @classmethod + def _field_value(cls, document: Mapping[str, object], dotted_path: str) -> str | None: """None means the path is absent from the document, which is what separates a mistyped mongodb_text_field from a document whose text is genuinely empty.""" - current: object = document - for segment in dotted_path.split("."): - if not isinstance(current, Mapping) or segment not in current: - return None - current = current[segment] - return None if current is None else str(current) + head, _, rest = dotted_path.partition(".") + if head not in document: + return None + value: Final = document[head] + if not rest: + return None if value is None else str(value) + return cls._field_value(value, rest) if isinstance(value, Mapping) else None @classmethod def _to_result(cls, document: Mapping[str, object], text_field: str) -> VectorStoreSearchResult: @@ -287,7 +302,9 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): return VectorStoreSearchResponse( object="vector_store.search_results.page", search_query=query_text, - data=[cls._to_result(document, text_field) for document in documents], + data=[ # mutable-ok: VectorStoreSearchResponse declares data as a list + cls._to_result(document, text_field) for document in documents + ], ) @staticmethod @@ -341,14 +358,12 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): try: client: Final = self.sync_client_factory(key) target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted - documents: Final = list(target.aggregate(pipeline)) + documents: Final = tuple(target.aggregate(pipeline)) except Exception as e: - raise translate_mongo_error( - e, index_name=vector_store_id, database=database, collection=collection - ) from e + raise translate_mongo_error(e, index_name=vector_store_id, database=database, collection=collection) from e if not documents: try: - catalogue: Final = list(target.list_search_indexes(vector_store_id)) + catalogue: Final = tuple(target.list_search_indexes(vector_store_id)) except Exception as e: raise translate_mongo_error( e, index_name=vector_store_id, database=database, collection=collection @@ -386,15 +401,17 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): client: Final = self.async_client_factory(key) target: Final = client[database][collection] # pyright: ignore[reportIndexIssue] # factory is typed as returning object so injected doubles are accepted cursor: Final = await target.aggregate(pipeline) - documents: Final = [document async for document in cursor] + documents: Final = [ # mutable-ok: an async comprehension cannot build a tuple directly + document async for document in cursor + ] except Exception as e: - raise translate_mongo_error( - e, index_name=vector_store_id, database=database, collection=collection - ) from e + raise translate_mongo_error(e, index_name=vector_store_id, database=database, collection=collection) from e if not documents: try: index_cursor: Final = await target.list_search_indexes(vector_store_id) - catalogue: Final = [entry async for entry in index_cursor] + catalogue: Final = [ # mutable-ok: an async comprehension cannot build a tuple directly + entry async for entry in index_cursor + ] except Exception as e: raise translate_mongo_error( e, index_name=vector_store_id, database=database, collection=collection diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index a62c0f711cb..9ca0753f354 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -62,7 +62,7 @@ def _row_to_vector_store(row: "_VectorStoreRow") -> LiteLLM_ManagedVectorStore: # "connection" covers wire-protocol providers whose whole credential is a URI # (mongodb_connection_string embeds the username and password), which the # default api_key/secret/token patterns do not match. -_LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker(extra_sensitive_patterns={"connection"}) +_LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker(extra_sensitive_patterns=frozenset(("connection",))) _REDACT_LITELLM_PARAMS_MAX_DEPTH: Final = 10 diff --git a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py index 27a83223864..c2b4042bdba 100644 --- a/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/test_litellm/litellm_core_utils/test_sensitive_data_masker.py @@ -331,3 +331,15 @@ def test_extra_sensitive_patterns_do_not_leak_into_other_maskers(): SensitiveDataMasker(extra_sensitive_patterns={"connection"}) assert SensitiveDataMasker().is_sensitive_key("mongodb_connection_string") is False + + +def test_the_second_positional_argument_is_still_the_override_set(): + """SensitiveDataMasker is public SDK surface, so adding a keyword must not shift what an + existing positional call means. Putting extra_sensitive_patterns second would silently turn + an override set into an extra sensitive set and start masking the caller's pricing fields.""" + from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker + + masker = SensitiveDataMasker({"token"}, {"session"}) + + assert masker.is_sensitive_key("session_token") is False + assert masker.is_sensitive_key("auth_token") is True diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 84de42d2126..e7ed3d77407 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -39,6 +39,15 @@ BASE_PARAMS = { READY_INDEX = [{"name": INDEX, "status": "READY", "queryable": True}] +class RecordingClient: + """Stands in for pymongo's client class so the cache tests inject a fake rather than + patching the importer, and so they can assert what the client was actually built with.""" + + def __init__(self, connection_string, **kwargs): + self.connection_string = connection_string + self.kwargs = kwargs + + class FakeCollection: def __init__(self, documents, error=None, search_indexes=None): self.documents = documents @@ -171,12 +180,22 @@ def test_search_builds_vector_search_stage_against_the_named_index(): assert _stage(collection, "$vectorSearch") == { "index": INDEX, "path": "embedding", - "queryVector": [0.1, 0.2, 0.3], + "queryVector": (0.1, 0.2, 0.3), "numCandidates": 100, "limit": 5, } +def test_the_pipeline_reaches_pymongo_as_a_list(): + """pymongo's common.validate_list rejects any other sequence with + 'pipeline must be a list, not ', so the outer container is part of the contract.""" + config, _, collection = _config() + + _search(config) + + assert isinstance(collection.pipeline, list) + + def test_search_projects_the_text_field_and_the_similarity_score(): config, _, collection = _config() @@ -275,6 +294,38 @@ def test_response_reads_a_dotted_text_field_path(): assert response["data"][0]["content"][0]["text"] == "nested text" +def test_a_dotted_path_resolves_three_levels_deep(): + config, _, _ = _config(documents=[{"_id": 1, "a": {"b": {"c": "deep text"}}, "score": 0.5}]) + + response = _search(config, litellm_params={"mongodb_text_field": "a.b.c"}) + + assert response["data"][0]["content"][0]["text"] == "deep text" + + +def test_a_dotted_path_that_runs_through_a_scalar_counts_as_absent(): + """Walking 'plot.nope' when plot is a string must report the misconfiguration, not + stringify the scalar and hand the model text from the wrong field.""" + config, _, _ = _config(documents=[{"_id": 1, "plot": "a plain string", "score": 0.5}]) + + with pytest.raises(BadRequestError, match=r"has a 'plot\.nope' field"): + _search(config, litellm_params={"mongodb_text_field": "plot.nope"}) + + +def test_a_non_string_text_field_is_stringified(): + config, _, _ = _config(documents=[{"_id": 1, "year": 1979, "score": 0.5}]) + + response = _search(config, litellm_params={"mongodb_text_field": "year"}) + + assert response["data"][0]["content"][0]["text"] == "1979" + + +def test_a_null_text_field_counts_as_absent(): + config, _, _ = _config(documents=[{"_id": 1, "text": None, "score": 0.5}]) + + with pytest.raises(BadRequestError, match="has a 'text' field"): + _search(config) + + def test_response_tolerates_a_sparse_document_missing_the_text_field(): config, _, _ = _config(documents=[{"_id": 1, "score": 0.5}, {"_id": 2, "text": "has text", "score": 0.4}]) @@ -489,7 +540,7 @@ async def test_async_search_builds_the_same_pipeline_and_maps_the_response(): assert client.requested_database == "sample_mflix" assert client.database.requested_collection == "embedded_movies" assert _stage(collection, "$vectorSearch")["limit"] == 3 - assert _stage(collection, "$vectorSearch")["queryVector"] == [0.1, 0.2, 0.3] + assert _stage(collection, "$vectorSearch")["queryVector"] == (0.1, 0.2, 0.3) assert response["data"][0]["content"][0]["text"] == "an astronaut adrift" assert response["data"][0]["score"] == 0.94 @@ -524,42 +575,36 @@ class TestClientCache: ) def test_the_same_connection_reuses_one_client(self): - with patch("litellm.llms.mongodb.common_utils.import_sync_mongo_client") as importer: - importer.return_value = lambda *args, **kwargs: MagicMock() - - first = get_sync_client(self._key()) - second = get_sync_client(self._key()) + first = get_sync_client(self._key(), RecordingClient) + second = get_sync_client(self._key(), RecordingClient) assert first is second - assert importer.return_value + assert first.connection_string == CONNECTION_STRING + assert first.kwargs["socketTimeoutMS"] == 30_000 + assert first.kwargs["connectTimeoutMS"] == 10_000 + assert first.kwargs["appname"] == "litellm" def test_a_different_connection_gets_its_own_client(self): - with patch("litellm.llms.mongodb.common_utils.import_sync_mongo_client") as importer: - importer.return_value = lambda *args, **kwargs: MagicMock() - - first = get_sync_client(self._key()) - second = get_sync_client(self._key(connection_string="mongodb://other.example.test")) + first = get_sync_client(self._key(), RecordingClient) + second = get_sync_client(self._key(connection_string="mongodb://other.example.test"), RecordingClient) assert first is not second + assert second.connection_string == "mongodb://other.example.test" def test_a_different_timeout_gets_its_own_client(self): - with patch("litellm.llms.mongodb.common_utils.import_sync_mongo_client") as importer: - importer.return_value = lambda *args, **kwargs: MagicMock() - - first = get_sync_client(self._key()) - second = get_sync_client(self._key(socket_timeout_ms=5_000)) + first = get_sync_client(self._key(), RecordingClient) + second = get_sync_client(self._key(socket_timeout_ms=5_000), RecordingClient) assert first is not second + assert second.kwargs["socketTimeoutMS"] == 5_000 @pytest.mark.asyncio async def test_async_clients_are_cached_per_event_loop(self): - with patch("litellm.llms.mongodb.common_utils.import_async_mongo_client") as importer: - importer.return_value = lambda *args, **kwargs: MagicMock() - - first = get_async_client(self._key()) - second = get_async_client(self._key()) + first = get_async_client(self._key(), RecordingClient) + second = get_async_client(self._key(), RecordingClient) assert first is second + assert first.connection_string == CONNECTION_STRING def test_a_new_loop_never_inherits_a_closed_loop_client(self): @@ -579,19 +624,16 @@ class TestClientCache: clients_handed_out = [] async def fetch(): - return get_async_client(key) + return get_async_client(key, LoopAgnosticClient) - with patch("litellm.llms.mongodb.common_utils.import_async_mongo_client") as importer: - importer.return_value = LoopAgnosticClient - - for _ in range(20): - loop = asyncio.new_event_loop() - client = loop.run_until_complete(fetch()) - clients_handed_out.append((client, client.built_on, loop.is_closed())) - client.built_on = weakref.ref(loop) - loop.close() - del loop - gc.collect() + for _ in range(20): + loop = asyncio.new_event_loop() + client = loop.run_until_complete(fetch()) + clients_handed_out.append((client, client.built_on, loop.is_closed())) + client.built_on = weakref.ref(loop) + loop.close() + del loop + gc.collect() stale = [ handed_out From 3d0223b661227a122b5aaa42a79fd9b2d66f3420 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 12:51:19 -0700 Subject: [PATCH 20/60] ci: install the mongodb extra for the unit test shard that runs the provider tests/test_litellm/llms/mongodb imports pymongo's exception classes to check the error translation against the real hierarchy, and the shard that runs it (tests/test_litellm/llms, per test-unit.yml) synced --extra google, proxy, semantic-router and saml but not mongodb, so 24 of 109 tests would have errored with ModuleNotFoundError on the first CI run. CircleCI hid this because it syncs --all-groups --all-extras. uv export --frozen ... --extra saml -> no pymongo uv export --frozen ... --extra saml --extra mongodb -> pymongo==4.17.0 Also close the two gaps a mutation run found in the suite: nothing asserted that a short request timeout shortens server selection as well as connect, and the existing code 13 case carried "not authorized", which the message markers match too, so it could not tell whether the code was still being checked. 28 of 28 mutants now die. --- .github/workflows/_test-unit-base.yml | 2 +- .../test_mongodb_transformation.py | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index c4045a08ffb..80d743c5ad7 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -112,7 +112,7 @@ jobs: if: steps.changes.outputs.decision != 'skip' timeout-minutes: 8 run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra mongodb - name: Cache Prisma binaries if: steps.changes.outputs.decision != 'skip' diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index e7ed3d77407..0e62e7c11d6 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -658,6 +658,19 @@ class TestClientKeyDerivation: assert key.socket_timeout_ms == 3_000 assert key.connect_timeout_ms == 3_000 + def test_a_short_timeout_also_shortens_server_selection(self): + """Server selection runs before the connect attempt, so leaving it at the 10s default + would let a caller asking for a 3s budget block for 10s before anything is tried.""" + key = MongoDBVectorStoreConfig._client_key(_MongoDBSearchParams.model_validate(BASE_PARAMS), 3.0) + + assert key.server_selection_timeout_ms == 3_000 + + def test_a_generous_timeout_does_not_raise_server_selection_above_the_default(self): + key = MongoDBVectorStoreConfig._client_key(_MongoDBSearchParams.model_validate(BASE_PARAMS), 120.0) + + assert key.socket_timeout_ms == 120_000 + assert key.server_selection_timeout_ms == 10_000 + def test_an_httpx_timeout_maps_connect_and_read_separately(self): key = MongoDBVectorStoreConfig._client_key( _MongoDBSearchParams.model_validate(BASE_PARAMS), httpx.Timeout(connect=2.0, read=45.0, write=5.0, pool=5.0) @@ -693,6 +706,16 @@ class TestErrorTranslation: assert "sample_mflix.embedded_movies" in str(translated) + def test_code_13_alone_is_enough_without_a_recognisable_message(self): + """The other unauthorized case carries "not authorized", which the message markers also + match, so it cannot tell whether the code is still being checked at all.""" + from pymongo.errors import OperationFailure + + translated = self._translate(OperationFailure("user lacks privileges on this namespace", code=13)) + + assert "rejected the credentials" in str(translated) + assert "sample_mflix.embedded_movies" in str(translated) + def test_a_missing_index_names_the_index_and_the_collection(self): from pymongo.errors import OperationFailure From 32b501bf74abade544d79a349e200b0b757443c4 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 13:14:56 -0700 Subject: [PATCH 21/60] docs(vector_stores): register mongodb in the provider endpoint support matrix --- provider_endpoints_support.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index ebc220b3496..41ed8e1d975 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2880,6 +2880,13 @@ "vector_stores_search": true } }, + "mongodb": { + "display_name": "MongoDB Atlas (`mongodb`)", + "url": "https://docs.litellm.ai/docs/providers/mongodb_vector_stores", + "endpoints": { + "vector_stores_search": true + } + }, "valkey": { "display_name": "Valkey (`valkey`)", "url": "https://docs.litellm.ai/docs/providers/valkey_vector_stores", From ed8203757a7af4d7867dc7afce042454cf9b53b4 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 13:21:55 -0700 Subject: [PATCH 22/60] fix(vector_stores): refuse MongoDB vector store create with a 400, not a 500 litellm.exception_type passes only litellm's own exception types through untouched, so the NotImplementedError the search-only refusal raised reached the caller as APIConnectionError. The proxy served that as a 500 with a traceback in the body for what is a plain client mistake. Raising BadRequestError gives the caller a 400 and the message on its own. --- .../llms/mongodb/vector_stores/transformation.py | 4 ++-- .../vector_stores/test_mongodb_transformation.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index d0a0f51cd77..9f5e40f69ef 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -425,7 +425,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, api_base: str, ) -> NoReturn: - raise NotImplementedError(_SEARCH_ONLY_MESSAGE) + raise config_error(_SEARCH_ONLY_MESSAGE) def transform_create_vector_store_response(self, response: httpx.Response) -> NoReturn: - raise NotImplementedError(_SEARCH_ONLY_MESSAGE) + raise config_error(_SEARCH_ONLY_MESSAGE) diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 0e62e7c11d6..140efd53449 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -515,15 +515,27 @@ def test_validation_runs_before_any_connection_is_opened(): def test_create_vector_store_is_not_supported_and_says_why(): + """litellm.exception_type only passes its own exception types through untouched, so a + NotImplementedError here reaches the caller as APIConnectionError, which the proxy serves + as a 500 with a traceback. Refusing an unsupported operation is a client error.""" config = MongoDBVectorStoreConfig() - with pytest.raises(NotImplementedError, match="search-only"): + with pytest.raises(BadRequestError, match="search-only"): config.transform_create_vector_store_request({}, "https://example.test") - with pytest.raises(NotImplementedError, match="search-only"): + with pytest.raises(BadRequestError, match="search-only"): config.transform_create_vector_store_response(httpx.Response(200)) +def test_the_create_refusal_survives_the_public_sdk_error_wrapper(): + import litellm + + with pytest.raises(BadRequestError) as raised: + litellm.vector_stores.create(custom_llm_provider="mongodb", name="anything") + + assert "search-only" in str(raised.value) + + def test_provider_config_manager_returns_the_mongodb_config(): config = ProviderConfigManager.get_provider_vector_stores_config(LlmProviders.MONGODB) From d4b02661925adf261a49ba4a45ee20702aa94e69 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 13:39:14 -0700 Subject: [PATCH 23/60] fix(vector_stores): release MongoDB clients built on closed event loops The async client cache is keyed per event loop, and pymongo's AsyncMongoClient holds a reference to the loop it was built on, so an entry for a closed loop kept that client and its sockets alive for the life of the process. A script that calls asyncio.run once per search fills the cache to its cap this way and then stops caching entirely. Measured live against Atlas over 40 loops: 32 pinned clients and 212 open descriptors before, 1 cached client and no monotonic descriptor growth after. --- litellm/llms/mongodb/common_utils.py | 13 ++++++++++ .../test_mongodb_transformation.py | 24 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 8ac02552ecb..460a2903c60 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -109,6 +109,18 @@ def get_sync_client(key: MongoClientKey, client_class: SyncClientFactory | None return client +def _purge_dead_loops() -> None: + """The cached client holds its loop object alive, so a closed loop's entry would otherwise pin + that client and its sockets for the life of the process. Callers that run one loop per search + (``asyncio.run`` in a script) reach the cap this way and never release what is behind it.""" + for stale in tuple( + cache_key + for cache_key, (loop_ref, _) in _async_clients.items() + if (cached_loop := loop_ref()) is None or cached_loop.is_closed() + ): + del _async_clients[stale] + + def get_async_client(key: MongoClientKey, client_class: AsyncClientFactory | None = None) -> "AsyncMongoClient": """Async clients bind to the loop that created them, so the cache is keyed per loop.""" loop: Final = asyncio.get_running_loop() @@ -116,6 +128,7 @@ def get_async_client(key: MongoClientKey, client_class: AsyncClientFactory | Non cached: Final = _async_clients.get(loop_key) if cached is not None and cached[0]() is loop: return cached[1] + _purge_dead_loops() build: Final = client_class if client_class is not None else import_async_mongo_client() client: Final = build(key.connection_string, **_client_kwargs(key)) if len(_async_clients) < _MAX_CACHED_CLIENTS or loop_key in _async_clients: diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 140efd53449..6d932c05065 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -10,6 +10,8 @@ import pytest from litellm.exceptions import BadRequestError, Timeout from litellm.llms.mongodb.common_utils import ( + _MAX_CACHED_CLIENTS, + _async_clients, MongoClientKey, index_not_ready_error, missing_index_error, @@ -655,6 +657,28 @@ class TestClientCache: ] assert stale == [], f"{len(stale)} of 20 loops were handed a client built on a closed loop" + def test_the_cache_releases_clients_built_on_closed_loops(self): + """pymongo's AsyncMongoClient keeps a reference to the loop it was built on, so an entry + for a closed loop holds that client, and its sockets, for the life of the process. A + script calling asyncio.run per search fills the cache to its cap that way: measured live + against Atlas at 32 pinned clients and 212 open descriptors after 40 loops.""" + + class LoopHoldingClient: + def __init__(self, *args, **kwargs): + self.loop = asyncio.get_running_loop() + + key = self._key() + + async def fetch(): + return get_async_client(key, LoopHoldingClient) + + for _ in range(_MAX_CACHED_CLIENTS + 8): + loop = asyncio.new_event_loop() + loop.run_until_complete(fetch()) + loop.close() + + assert len(_async_clients) == 1, f"{len(_async_clients)} closed-loop clients are still cached" + class TestClientKeyDerivation: def test_no_timeout_uses_the_bounded_defaults(self): From 1b47486724d16798f5bf416067d5d68c63959d8e Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 13:50:45 -0700 Subject: [PATCH 24/60] style(vector_stores): cut the explanatory comments down to one line each The repo's rule allows a comment only where the logic stays confusing after the code has been made as clear as it can be, and then only one concise line about why. Three multi-line blocks did not meet that: the reason "connection" joins the sensitive patterns belongs in the commit that added it, and the weakref and Atlas error-code notes each say what they need to in a single line. --- litellm/llms/mongodb/common_utils.py | 7 ++----- .../proxy/vector_store_endpoints/management_endpoints.py | 3 --- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 460a2903c60..c2d081b8d8d 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -60,9 +60,7 @@ SyncClientFactory: TypeAlias = Callable[..., "MongoClient"] AsyncClientFactory: TypeAlias = Callable[..., "AsyncMongoClient"] _AsyncClientCacheKey: TypeAlias = tuple[MongoClientKey, int] -# The entry carries a weak reference to the loop the client was built on: CPython recycles id() -# aggressively (measured: 200 of 200 fresh loops landed on an id already in this cache), so the -# id alone would hand a new loop a client bound to a closed one. +# CPython recycles id() aggressively, so the id alone would hand a new loop a closed loop's client _AsyncClientEntry: TypeAlias = tuple["weakref.ref[AbstractEventLoop]", "AsyncMongoClient"] _sync_clients: Final[dict[MongoClientKey, "MongoClient"]] = {} # mutable-ok: process-level client cache @@ -143,8 +141,7 @@ def reset_client_cache() -> None: _AUTHENTICATION_FAILED_CODE: Final = 18 _UNAUTHORIZED_CODE: Final = 13 -# Atlas reports a rejected user as code 8000 "AtlasError" rather than 18, so the -# message is the only reliable signal for a serverless or shared-tier deployment. +# Atlas reports a rejected user as code 8000 "AtlasError", not 18, so only the message is reliable _AUTHENTICATION_MESSAGE_MARKERS: Final = ("bad auth", "authentication failed", "not authorized") _RESOLUTION_TIMEOUT_MARKERS: Final = ("resolution lifetime expired", "dns operation timed out") _UNKNOWN_HOSTNAME_MARKERS: Final = ("dns query name does not exist", "name or service not known") diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 6af2a8b7a6b..8b951556a14 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -59,9 +59,6 @@ def _row_to_vector_store(row: "_VectorStoreRow") -> LiteLLM_ManagedVectorStore: return LiteLLM_ManagedVectorStore(**row.model_dump()) -# "connection" covers wire-protocol providers whose whole credential is a URI -# (mongodb_connection_string embeds the username and password), which the -# default api_key/secret/token patterns do not match. _LITELLM_PARAMS_MASKER: Final = SensitiveDataMasker(extra_sensitive_patterns=frozenset(("connection",))) From 52de1bb1d3380fbbbde5cb4725dae81eb883d457 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 14:19:28 -0700 Subject: [PATCH 25/60] fix(vector_stores): reject MongoDB search params the provider cannot honour filters was already refused, but ranking_options and rewrite_query were accepted and then dropped. A caller asking for score_threshold 0.9 got results scoring 0.5 with a 200 and no indication the threshold never ran, which is the silent-wrong-answer case the filters check exists to prevent. Both now raise the same 400 naming the parameter and what to do instead. --- .../mongodb/vector_stores/transformation.py | 11 +++++++++ .../test_mongodb_transformation.py | 24 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index 9f5e40f69ef..5e59fd30f1b 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -236,6 +236,17 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): "MongoDB vector store does not support the filters parameter yet. " "Restrict the collection or the Atlas Vector Search index definition instead." ) + if vector_store_search_optional_params.get("ranking_options") is not None: + raise config_error( + "MongoDB vector store does not support the ranking_options parameter yet. " + "Every result already carries the Atlas vectorSearchScore, so filter or re-rank " + "on that rather than having the threshold silently ignored." + ) + if vector_store_search_optional_params.get("rewrite_query") is not None: + raise config_error( + "MongoDB vector store does not support the rewrite_query parameter. The query is " + "embedded exactly as sent; rewrite it before calling if you need that." + ) limit: Final = cls._limit(vector_store_search_optional_params) search: Final = MappingProxyType( { diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 6d932c05065..668fa676692 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -448,6 +448,30 @@ async def test_async_search_rejects_filters_rather_than_silently_ignoring_them() await _asearch(config, optional_params={"filters": {"genre": "sci-fi"}}) +def test_search_rejects_ranking_options_rather_than_silently_ignoring_them(): + """A score_threshold that is quietly dropped is worse than an error: the caller asked for + results above 0.9, gets results scoring 0.5, and nothing says the threshold never ran.""" + config, _, _ = _config() + + with pytest.raises(BadRequestError, match="does not support the ranking_options parameter"): + _search(config, optional_params={"ranking_options": {"score_threshold": 0.9}}) + + +def test_search_rejects_rewrite_query_rather_than_silently_ignoring_it(): + config, _, _ = _config() + + with pytest.raises(BadRequestError, match="does not support the rewrite_query parameter"): + _search(config, optional_params={"rewrite_query": True}) + + +@pytest.mark.asyncio +async def test_async_search_rejects_ranking_options_rather_than_silently_ignoring_them(): + config, _, _ = _async_config() + + with pytest.raises(BadRequestError, match="does not support the ranking_options parameter"): + await _asearch(config, optional_params={"ranking_options": {"score_threshold": 0.9}}) + + @pytest.mark.parametrize("query", ["", " ", "\n\t", []]) def test_search_rejects_an_empty_query(query): config, _, _ = _config() From cfe247ebfe23d41adc2d43b14bf58f278f9ff609 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 14:55:13 -0700 Subject: [PATCH 26/60] fix(vector_stores): translate the two MongoDB driver errors that still reached callers as 500s A connection string whose password holds an unescaped '/' makes pymongo's URI parser raise a plain ValueError, not a PyMongoError, and a URI with no credentials at all makes Atlas close the connection, which surfaces as AutoReconnect. Neither was handled, so both fell through to litellm's generic wrapper and were served as 500s with a traceback for what are routine typos. Both now return a 400 naming the cause. The ConnectionFailure branch sits after the ServerSelectionTimeoutError and NetworkTimeout branches, which subclass it, and two ordering tests pin that. --- litellm/llms/mongodb/common_utils.py | 16 +++++++++ .../test_mongodb_transformation.py | 36 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index c2d081b8d8d..bf3bf953772 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -181,6 +181,7 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll try: from pymongo.errors import ( ConfigurationError, + ConnectionFailure, ExecutionTimeout, InvalidOperation, NetworkTimeout, @@ -202,6 +203,14 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll f"The MongoDB vector search against '{database}.{collection}' timed out before returning. " f"Driver detail: {error}" ) + # ServerSelectionTimeoutError and NetworkTimeout both sit under ConnectionFailure, so this + # only sees what those two branches left: a dropped or refused connection + if isinstance(error, ConnectionFailure): + return config_error( + f"The connection to '{database}.{collection}' was refused or dropped. On Atlas this is " + "usually a connection string with no username and password, or a TLS failure. Confirm " + f"the URI is the one Atlas shows under Connect, Drivers. Driver detail: {error}" + ) if isinstance(error, OperationFailure): code: Final = error.code detail: Final = str(error).lower() @@ -247,4 +256,11 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll ) if isinstance(error, InvalidOperation): return config_error(f"The MongoDB client was already closed or is unusable. Driver detail: {error}") + # pymongo's URI parser raises a plain ValueError, not a PyMongoError, for a password holding an + # unescaped '/', which would otherwise reach the caller as a 500 + if isinstance(error, ValueError): + return config_error( + "mongodb_connection_string could not be parsed. A username or password containing " + f"'@', '/', ':' or '%' has to be percent-encoded per RFC 3986. Driver detail: {error}" + ) return error diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 668fa676692..d60504c31e5 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -759,6 +759,42 @@ class TestErrorTranslation: assert "rejected the credentials" in str(translated) + def test_a_dropped_connection_is_a_400_not_an_unhandled_driver_error(self): + """AutoReconnect sits under ConnectionFailure alongside the two timeout classes, and Atlas + answers a URI with no credentials by closing the connection rather than failing auth. Left + untranslated it is not a litellm exception type, so it reaches the caller as a 500.""" + from pymongo.errors import AutoReconnect + + translated = self._translate(AutoReconnect("connection closed")) + + assert isinstance(translated, BadRequestError) + assert "refused or dropped" in str(translated) + assert "no username and password" in str(translated) + + def test_server_selection_timeout_still_wins_over_the_connection_branch(self): + from pymongo.errors import ServerSelectionTimeoutError + + translated = self._translate(ServerSelectionTimeoutError("no servers")) + + assert isinstance(translated, Timeout) + assert "refused or dropped" not in str(translated) + + def test_network_timeout_still_wins_over_the_connection_branch(self): + from pymongo.errors import NetworkTimeout + + translated = self._translate(NetworkTimeout("socket timed out")) + + assert isinstance(translated, Timeout) + assert "refused or dropped" not in str(translated) + + def test_an_unescaped_password_character_is_a_400_not_a_500(self): + """pymongo's URI parser raises a plain ValueError, not a PyMongoError, when a password + holds an unescaped '/'. That is a routine mistake and it must not be a 500.""" + translated = self._translate(ValueError("Port contains non-digit characters")) + + assert isinstance(translated, BadRequestError) + assert "percent-encoded" in str(translated) + def test_unauthorized_points_at_the_database_user_permissions(self): from pymongo.errors import OperationFailure From 63482cfdbd4d6e623b984c9b65ab98d1f224d879 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 18:25:51 -0700 Subject: [PATCH 27/60] chore(deps): lower the pymongo floor for the mongodb extra to 4.9 4.17 was picked on the belief that dnspython only became a core pymongo dependency there, which is wrong: pymongo has declared dnspython>=1.16.0,<3.0.0 as a core requirement since well before that, so mongodb+srv:// URIs resolve at 4.9 too. The real floor is 4.9, the release AsyncMongoClient landed in, and 4.8 has no AsyncMongoClient at all. Verified against live Atlas on 4.9: sync and async search, list_search_indexes, same top hit and score as 4.17. Resolution is unchanged, pymongo 4.17.0 either way, so this only widens what an existing environment is allowed to bring. --- pyproject.toml | 9 +++------ uv.lock | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c2f0dc6ced7..e3e103e6d49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,12 +112,9 @@ utils = [ ] caching = ["diskcache>=5.6.3,<6.0"] mcp = ["mcp>=1.28.1,<2.0"] -# Driver for the MongoDB Atlas vector store. Atlas Vector Search has no HTTP query -# API, so that provider talks to the cluster over the wire protocol. Imported lazily -# and kept out of the base install, which never needs a MongoDB driver. The floor is -# 4.17 because that is where dnspython became a core dependency rather than the `srv` -# extra, and Atlas hands out mongodb+srv:// URIs that do not resolve without it. -mongodb = ["pymongo>=4.17,<5.0"] +# Driver for the MongoDB Atlas vector store; Atlas Vector Search has no HTTP query API. +# The floor is 4.9 because that is the release AsyncMongoClient landed in. +mongodb = ["pymongo>=4.9,<5.0"] # SAML SSO for the admin UI. python3-saml pulls in xmlsec/lxml, whose wheels # bundle the native libxmlsec1/libxml2 libraries, so no system packages are # required. Kept out of the base `proxy` extra so it stays optional. diff --git a/uv.lock b/uv.lock index 362bb490a2a..bb1927ce093 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-30T17:51:25.171404Z" +exclude-newer = "2026-08-31T00:55:41.895302Z" exclude-newer-span = "P3D" [manifest] @@ -4554,7 +4554,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.10.0,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.14.1,<3.0" }, { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.13.0,<3.0" }, - { name = "pymongo", marker = "extra == 'mongodb'", specifier = ">=4.17,<5.0" }, + { name = "pymongo", marker = "extra == 'mongodb'", specifier = ">=4.9,<5.0" }, { name = "pynacl", marker = "extra == 'proxy'", specifier = ">=1.6.2,<2.0" }, { name = "pypdf", marker = "extra == 'proxy-runtime'", specifier = ">=6.16.1,<7.0" }, { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.8.16,<1.0" }, From 35d3478818c3b26aeac18fbea63ed7da512d0514 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:27:48 -0700 Subject: [PATCH 28/60] fix(responses/mcp): keep reasoning order and caller previous_response_id on stateless follow-ups --- litellm/responses/main.py | 2 +- .../mcp/litellm_proxy_mcp_handler.py | 36 ++------- .../responses/mcp/mcp_streaming_iterator.py | 2 - .../mcp/test_litellm_proxy_mcp_handler.py | 74 +++++++++++++++++-- .../mcp/test_mcp_streaming_iterator.py | 8 +- 5 files changed, 81 insertions(+), 41 deletions(-) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index abf8fefe78a..ed2d6a216fd 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -352,7 +352,7 @@ async def aresponses_api_with_mcp( follow_up_input=follow_up_input, model=model, all_tools=all_tools, - response_id=None if persistence_disabled else response.id, + response_id=previous_response_id if persistence_disabled else response.id, **follow_up_call_params, ) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 7656dbd38df..15434bedbb7 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -965,22 +965,9 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def _is_persistence_disabled(call_params: Mapping[str, object]) -> bool: - """Whether the caller opted out of server-side response persistence (store=false). - - Zero data retention callers send store=false, so the provider never persisted the - first response and previous_response_id cannot be used to link the follow-up call. - """ + """store=false means the provider kept nothing, so the follow-up call cannot chain on a response id.""" return call_params.get("store") is False - @staticmethod - def _extract_reasoning_items(response: ResponsesAPIResponse) -> tuple[Mapping[str, object], ...]: - """Reasoning output items, kept whole so reasoning.encrypted_content survives replay.""" - normalized: Final = tuple( - output_item if isinstance(output_item, dict) else output_item.model_dump(exclude_none=True) - for output_item in response.output - ) - return tuple(item for item in normalized if item.get("type") == "reasoning") - @staticmethod def _create_follow_up_input( response: ResponsesAPIResponse, @@ -1002,11 +989,11 @@ class LiteLLM_Proxy_MCP_Handler: # Add the assistant message with function calls assistant_message_content: Final[list[object]] = [] - function_calls: Final[list[dict[str, object]]] = [] + turn_items: Final[list[Mapping[str, object]]] = [] for output_item in response.output: if not isinstance(output_item, dict) and hasattr(output_item, "model_dump"): - output_item = output_item.model_dump() + output_item = output_item.model_dump(exclude_none=True) if isinstance(output_item, dict): if output_item.get("type") == "function_call": @@ -1016,7 +1003,7 @@ class LiteLLM_Proxy_MCP_Handler: # Only add if we have required fields if call_id and name: - function_calls.append( + turn_items.append( { "type": "function_call", "call_id": call_id, @@ -1024,6 +1011,8 @@ class LiteLLM_Proxy_MCP_Handler: "arguments": arguments, } ) + elif output_item.get("type") == "reasoning" and preserve_reasoning: + turn_items.append(output_item) elif output_item.get("type") == "message": # Extract content from message content = output_item.get("content", []) @@ -1044,12 +1033,7 @@ class LiteLLM_Proxy_MCP_Handler: } ) - if preserve_reasoning: - follow_up_input.extend(LiteLLM_Proxy_MCP_Handler._extract_reasoning_items(response)) - - # Add function calls (these can come directly after user message for LLM) - for function_call in function_calls: - follow_up_input.append(function_call) + follow_up_input.extend(turn_items) # Add tool results (function call outputs) for tool_result in tool_results: @@ -1071,11 +1055,7 @@ class LiteLLM_Proxy_MCP_Handler: response_id: str | None, **call_params: Any, ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: - """Make follow-up response API call with tool results. - - response_id is None for stateless (store=false) requests, where the whole prior - turn is replayed in follow_up_input instead of linked by previous_response_id. - """ + """Make follow-up response API call with tool results.""" return await aresponses( input=follow_up_input, model=model, diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 83322b8d837..ca12b3e7cc3 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -800,8 +800,6 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): "stream": True, } ) - if persistence_disabled: - follow_up_params.pop("previous_response_id", None) else: return # Remove tool_choice to avoid forcing more tool calls diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index 8ebea685d5a..80151d0cba8 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -785,6 +785,59 @@ def test_create_follow_up_input_preserves_reasoning_when_stateless(): } +def _response_with_interleaved_reasoning_and_tool_calls() -> Any: + """A first-turn response that reasons before each of two function calls.""" + return ResponsesAPIResponse( + id="resp_first", + created_at=1234567890, + model="gpt-5", + object="response", + status="completed", + output=[ + {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "blob-1"}, + {"type": "function_call", "id": "fc_1", "call_id": "call-1", "name": "foo", "arguments": "{}"}, + {"type": "reasoning", "id": "rs_2", "summary": [], "encrypted_content": "blob-2"}, + {"type": "function_call", "id": "fc_2", "call_id": "call-2", "name": "bar", "arguments": "{}"}, + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + ) + + +def test_create_follow_up_input_keeps_each_reasoning_item_before_its_function_call(): + """ + Regression test (LIT-5427): the provider pairs a replayed reasoning item with the + item that follows it, so the replay has to keep the response's output order instead + of grouping every reasoning item ahead of every function call. + """ + follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( + response=_response_with_interleaved_reasoning_and_tool_calls(), + tool_results=[ + {"tool_call_id": "call-1", "name": "foo", "result": "one"}, + {"tool_call_id": "call-2", "name": "bar", "result": "two"}, + ], + original_input="hi", + preserve_reasoning=True, + ) + + assert [cast(dict[str, Any], item)["type"] for item in follow_up] == [ + "message", + "reasoning", + "function_call", + "reasoning", + "function_call", + "function_call_output", + "function_call_output", + ] + assert [cast(dict[str, Any], item).get("id") or cast(dict[str, Any], item).get("call_id") for item in follow_up[1:5]] == [ + "rs_1", + "call-1", + "rs_2", + "call-2", + ] + + def test_create_follow_up_input_omits_reasoning_when_stateful(): """With store=true the provider still holds the reasoning item, so don't resend it.""" follow_up = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( @@ -810,17 +863,25 @@ def test_is_persistence_disabled(call_params: dict[str, Any], expected: bool): @pytest.mark.parametrize( - "store, expected_previous_response_id", - [(False, None), (True, "resp_first")], + "store, caller_previous_response_id, expected_previous_response_id", + [ + (False, None, None), + (False, "resp_caller", "resp_caller"), + (True, None, "resp_first"), + (True, "resp_caller", "resp_first"), + ], ) @pytest.mark.asyncio async def test_mcp_follow_up_call_is_stateless_when_store_is_false( - monkeypatch: pytest.MonkeyPatch, store: bool, expected_previous_response_id: str | None + monkeypatch: pytest.MonkeyPatch, + store: bool, + caller_previous_response_id: str | None, + expected_previous_response_id: str | None, ): """ - Regression test (LIT-5427): linking the MCP follow-up call with - previous_response_id fails for zero data retention callers, because store=false - means the first response was never persisted. + Regression test (LIT-5427): linking the MCP follow-up call to the first response's id + fails for zero data retention callers, because store=false means it was never persisted. + The caller's own previous_response_id was valid for the first call, so it stays. """ captured_calls: list[dict[str, Any]] = [] first_response = _response_with_reasoning_and_tool_call() @@ -857,6 +918,7 @@ async def test_mcp_follow_up_call_is_stateless_when_store_is_false( model="gpt-5", tools=[{"type": "mcp", "server_url": "litellm_proxy", "require_approval": "never"}], store=store, + previous_response_id=caller_previous_response_id, ) assert len(captured_calls) == 2 diff --git a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py index ac0c5ef6392..5001589ce54 100644 --- a/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py +++ b/tests/test_litellm/responses/mcp/test_mcp_streaming_iterator.py @@ -265,11 +265,11 @@ def _reasoning_item(encrypted_content: str): @pytest.mark.asyncio -async def test_streaming_follow_up_is_stateless_when_store_is_false(monkeypatch): +async def test_streaming_follow_up_replays_reasoning_when_store_is_false(monkeypatch): """ Regression test (LIT-5427): with store=false the provider persisted nothing, so the - streaming follow-up must drop previous_response_id and replay the reasoning item - (carrying reasoning.encrypted_content) instead of pointing at a response id. + streaming follow-up must replay the reasoning item (carrying reasoning.encrypted_content). + The caller's own previous_response_id was valid for the first call and stays on the follow-up. """ _mock_mcp_environment(monkeypatch) @@ -300,7 +300,7 @@ async def test_streaming_follow_up_is_stateless_when_store_is_false(monkeypatch) assert aresponses_mock.call_count == 1 follow_up_kwargs = aresponses_mock.call_args_list[0].kwargs - assert "previous_response_id" not in follow_up_kwargs + assert follow_up_kwargs["previous_response_id"] == "resp_prev" assert _reasoning_item("gAAAAA-opaque-blob") in follow_up_kwargs["input"] From daced81f20a6b98c01beecf66fa997310d90bfb4 Mon Sep 17 00:00:00 2001 From: amasen02 Date: Fri, 4 Sep 2026 15:37:53 +0530 Subject: [PATCH 29/60] fix(proxy): invalidate end-user spend counter and cache on budget reset (#39726) Signed-off-by: amasen02 --- .../proxy/common_utils/reset_budget_job.py | 24 ++++++++++++++++- .../common_utils/test_reset_budget_job.py | 26 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 47f69732e95..12ba75aea24 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -38,6 +38,7 @@ from litellm.proxy.common_utils.timezone_utils import ( get_budget_reset_settings, ) from litellm.proxy.common_utils.user_api_key_cache import ( + end_user_cache_key, model_access_group_cache_key, model_access_group_spend_counter_key, tag_cache_key, @@ -177,6 +178,21 @@ def _model_access_group_cache_keys(row: _ModelAccessGroupRow) -> tuple[str, ...] return (model_access_group_cache_key(row.access_group_name),) +def _enduser_counter_key(row: _EndUserRow) -> str: + return f"spend:end_user:{row.user_id}" + + +def _enduser_cache_keys(row: _EndUserRow) -> tuple[str, ...]: + return (end_user_cache_key(row.user_id),) + + +def _enduser_carried_spend(row: _EndUserRow, caps: Mapping[str, float]) -> float: + if not caps: + return 0.0 + effective_budget_id = row.budget_id or litellm.max_end_user_budget_id + return _carried_spend(row.spend, caps.get(effective_budget_id) if effective_budget_id is not None else None) + + def _budget_link_where( budget_ids: Sequence[str], extra: Mapping[str, object] = MappingProxyType({}), @@ -650,6 +666,7 @@ class ResetBudgetJob: if _rollover_enabled() else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType ) + endusers: Final[tuple[_EndUserRow, ...]] = await self._collect_endusers_to_reset(budget_ids) return _BudgetCascade( budgets=tuple(budgets_to_reset), budget_ids=budget_ids, @@ -661,7 +678,7 @@ class ResetBudgetJob: for b in budgets_to_reset if b.budget_id is not None and b.budget_duration is not None ), - endusers=await self._collect_endusers_to_reset(budget_ids), + endusers=endusers, counter_resets=( *( (_team_membership_counter_key(row), _row_carried_spend(row, rollover_caps)) @@ -674,6 +691,10 @@ class ResetBudgetJob: (_model_access_group_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in model_access_groups ), + *( + (_enduser_counter_key(row), _enduser_carried_spend(row, rollover_caps)) + for row in endusers + ), ), rollover_caps=rollover_caps, cache_keys=( @@ -682,6 +703,7 @@ class ResetBudgetJob: *(key for row in orgs for key in _org_cache_keys(row)), *(key for row in tags for key in _tag_cache_keys(row)), *(key for row in model_access_groups for key in _model_access_group_cache_keys(row)), + *(key for row in endusers for key in _enduser_cache_keys(row)), ), ) diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 03b05bd9d87..e6bbfd8c7d4 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -1495,6 +1495,32 @@ def test_budget_table_reset_invalidates_every_tag_not_just_the_first(reset_budge assert deleted == {"tag:tenant-a", "tag:tenant-b", "tag:tenant-c"} +def test_budget_table_reset_invalidates_enduser_counter_and_cache(reset_budget_job, mock_prisma_client, monkeypatch): + """When an end user's budget resets, its Redis spend counter is zeroed and its management cache is evicted.""" + counter_cache = _make_counter_invalidation_job(monkeypatch) + budget = _budget_row(budget_id="budget-1") + mock_prisma_client.data["budget"] = [budget] + test_enduser = type( + "LiteLLM_EndUserTable", + (), + { + "spend": 20.0, + "litellm_budget_table": budget, + "budget_id": "budget-1", + "user_id": "customer-42", + }, + ) + mock_prisma_client.data["enduser"] = [test_enduser] + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:end_user:customer-42", value=0.0, ttl=60) + counter_cache.redis_cache.async_set_cache.assert_any_await(key="spend:end_user:customer-42", value=0.0, ttl=60) + deleted = {call.kwargs.get("key") for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list} + assert "end_user_id:customer-42" in deleted + + + def test_budget_table_reset_commits_even_when_cache_eviction_fails(reset_budget_job, mock_prisma_client, monkeypatch): """Eviction runs after the commit, so a broken cache cannot undo the write.""" counter_cache = _make_counter_invalidation_job(monkeypatch) From 3623aecc6419ed5442bea4efd435cdca3246101a Mon Sep 17 00:00:00 2001 From: amasen02 Date: Fri, 4 Sep 2026 16:33:57 +0530 Subject: [PATCH 30/60] style(proxy): add Final type annotations to enduser budget reset variables --- litellm/proxy/common_utils/reset_budget_job.py | 2 +- .../proxy/common_utils/test_reset_budget_job.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 12ba75aea24..4fb544cbb15 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -189,7 +189,7 @@ def _enduser_cache_keys(row: _EndUserRow) -> tuple[str, ...]: def _enduser_carried_spend(row: _EndUserRow, caps: Mapping[str, float]) -> float: if not caps: return 0.0 - effective_budget_id = row.budget_id or litellm.max_end_user_budget_id + effective_budget_id: Final[str | None] = row.budget_id or litellm.max_end_user_budget_id return _carried_spend(row.spend, caps.get(effective_budget_id) if effective_budget_id is not None else None) diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index e6bbfd8c7d4..bc9926a314f 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -4,7 +4,7 @@ import sys import types from datetime import datetime, timedelta, timezone from datetime import time as dt_time -from typing import Any, Dict, List +from typing import Any, Dict, Final, List from unittest.mock import AsyncMock, MagicMock import httpx @@ -1497,10 +1497,10 @@ def test_budget_table_reset_invalidates_every_tag_not_just_the_first(reset_budge def test_budget_table_reset_invalidates_enduser_counter_and_cache(reset_budget_job, mock_prisma_client, monkeypatch): """When an end user's budget resets, its Redis spend counter is zeroed and its management cache is evicted.""" - counter_cache = _make_counter_invalidation_job(monkeypatch) - budget = _budget_row(budget_id="budget-1") + counter_cache: Final = _make_counter_invalidation_job(monkeypatch) + budget: Final = _budget_row(budget_id="budget-1") mock_prisma_client.data["budget"] = [budget] - test_enduser = type( + test_enduser: Final = type( "LiteLLM_EndUserTable", (), { @@ -1516,7 +1516,7 @@ def test_budget_table_reset_invalidates_enduser_counter_and_cache(reset_budget_j counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:end_user:customer-42", value=0.0, ttl=60) counter_cache.redis_cache.async_set_cache.assert_any_await(key="spend:end_user:customer-42", value=0.0, ttl=60) - deleted = {call.kwargs.get("key") for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list} + deleted: Final = {call.kwargs.get("key") for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list} assert "end_user_id:customer-42" in deleted From 9c8594c7b8c4c948abb85507e7611762b0b0f70f Mon Sep 17 00:00:00 2001 From: amasen02 Date: Fri, 4 Sep 2026 16:37:58 +0530 Subject: [PATCH 31/60] style(proxy): format reset_budget_job with ruff --- litellm/proxy/common_utils/reset_budget_job.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 4fb544cbb15..f2648c8466e 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -691,10 +691,7 @@ class ResetBudgetJob: (_model_access_group_counter_key(row), _row_carried_spend(row, rollover_caps)) for row in model_access_groups ), - *( - (_enduser_counter_key(row), _enduser_carried_spend(row, rollover_caps)) - for row in endusers - ), + *((_enduser_counter_key(row), _enduser_carried_spend(row, rollover_caps)) for row in endusers), ), rollover_caps=rollover_caps, cache_keys=( From 788efea7b3f137de91848feb67a2cad473a874fb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 4 Sep 2026 16:25:15 +0000 Subject: [PATCH 32/60] fix(fireworks_ai): resolve tool_choice/reasoning support for short model names Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/fireworks_ai/chat/transformation.py | 5 ++--- .../chat/test_fireworks_ai_chat_transformation.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index b6a5ee40672..b9c81a93730 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -32,7 +32,6 @@ from litellm.utils import ( get_model_cost_mutation_generation, supports_function_calling, supports_reasoning, - supports_tool_choice, ) from ...openai.chat.gpt_transformation import ( @@ -272,11 +271,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): ) # Only add tool_choice for models that explicitly support it - if supports_tool_choice(model=model, custom_llm_provider="fireworks_ai"): + if self._get_model_cost_capability_exact(model=model, capability="supports_tool_choice"): supported_params.append("tool_choice") # Only add reasoning params for models that support it - if supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): + if self._get_model_cost_capability_exact(model=model, capability="supports_reasoning"): supported_params.append("reasoning_effort") supported_params.append("reasoning_history") supported_params.append("thinking") diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index ec8725db5f7..672d47a2c03 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -363,6 +363,17 @@ def test_get_supported_openai_params_parallel_tool_calls(): assert "parallel_tool_calls" not in unsupported_params +def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_entry(): + config = FireworksAIConfig() + + supported_params = config.get_supported_openai_params( + "fireworks_ai/deepseek-v4-pro-0813" + ) + + assert "tool_choice" in supported_params + assert "reasoning_effort" in supported_params + + def test_get_supported_openai_params_parallel_tool_calls_without_tool_choice( monkeypatch, ): From 2f1da035ae7fa578b4ed76933fc43ec0248f1a0b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 4 Sep 2026 17:06:01 +0000 Subject: [PATCH 33/60] fix(fireworks_ai): keep generic capability fallback for reasoning and tool_choice Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/fireworks_ai/chat/transformation.py | 9 +++++++-- .../chat/test_fireworks_ai_chat_transformation.py | 14 +++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index b9c81a93730..6aa6a3600f0 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -32,6 +32,7 @@ from litellm.utils import ( get_model_cost_mutation_generation, supports_function_calling, supports_reasoning, + supports_tool_choice, ) from ...openai.chat.gpt_transformation import ( @@ -271,11 +272,15 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): ) # Only add tool_choice for models that explicitly support it - if self._get_model_cost_capability_exact(model=model, capability="supports_tool_choice"): + if self._get_model_cost_capability_exact( + model=model, capability="supports_tool_choice" + ) or supports_tool_choice(model=model, custom_llm_provider="fireworks_ai"): supported_params.append("tool_choice") # Only add reasoning params for models that support it - if self._get_model_cost_capability_exact(model=model, capability="supports_reasoning"): + if self._get_model_cost_capability_exact( + model=model, capability="supports_reasoning" + ) or supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): supported_params.append("reasoning_effort") supported_params.append("reasoning_history") supported_params.append("thinking") diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 672d47a2c03..e6fe01be4ba 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -4,11 +4,9 @@ from unittest.mock import MagicMock, patch import pytest import litellm - - from litellm import get_model_info, supports_reasoning, supports_vision -from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY +from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -374,6 +372,16 @@ def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_ assert "reasoning_effort" in supported_params +def test_get_supported_openai_params_preserves_generic_reasoning_fallback(): + config = FireworksAIConfig() + + supported_params = config.get_supported_openai_params( + "fireworks_ai/accounts/fireworks/models/glm-5p3-flash" + ) + + assert "reasoning_effort" in supported_params + + def test_get_supported_openai_params_parallel_tool_calls_without_tool_choice( monkeypatch, ): From dad1b132258edf500597b47dcfb0ddecf9b76fa7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 4 Sep 2026 17:07:59 +0000 Subject: [PATCH 34/60] style(fireworks_ai): format capability fallback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/fireworks_ai/chat/transformation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 6aa6a3600f0..26ad9a02a79 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -278,9 +278,9 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): supported_params.append("tool_choice") # Only add reasoning params for models that support it - if self._get_model_cost_capability_exact( - model=model, capability="supports_reasoning" - ) or supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): + if self._get_model_cost_capability_exact(model=model, capability="supports_reasoning") or supports_reasoning( + model=model, custom_llm_provider="fireworks_ai" + ): supported_params.append("reasoning_effort") supported_params.append("reasoning_history") supported_params.append("thinking") From 2042364fc2976ea735ab3d8c77dd4f4b27df3b84 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 11:50:28 -0700 Subject: [PATCH 35/60] fix(proxy): strip every TypedDict qualifier before numeric form-field detection _numeric_form_type only peeled a single ReadOnly layer, so a field still wrapped in Required/NotRequired was read as non-numeric and dropped from the mapping. Which qualifiers survive get_type_hints varies by interpreter version and by include_extras, so on Python 3.10 NotRequired[ReadOnly[int]] reached the check intact and the field was silently skipped, which is what turns the mapped test red on the 3.10 leg only. Peel Required/NotRequired/ReadOnly/Annotated in any order and nesting instead. The one production caller feeds a schema with no qualifiers, so the resulting mapping is unchanged on every interpreter in the matrix, but a field written the house-convention way stops being dropped. --- .../proxy/common_utils/http_parsing_utils.py | 16 +++++++++++++--- .../common_utils/test_http_parsing_utils.py | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 552d1ea434f..a396e543e94 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -2,11 +2,11 @@ import json import re from collections.abc import Collection, Mapping from types import MappingProxyType, UnionType -from typing import Any, Final, Union, get_args, get_origin +from typing import Annotated, Any, Final, Union, get_args, get_origin import orjson from fastapi import Request, UploadFile, status -from typing_extensions import ReadOnly +from typing_extensions import NotRequired, ReadOnly, Required from litellm._logging import verbose_proxy_logger from litellm.constants import MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB @@ -18,6 +18,8 @@ from litellm.types.router import Deployment _FORM_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-www-form-urlencoded", "multipart/form-data"}) +_ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequired, ReadOnly, Required}) + def _normalize_media_type(content_type: str) -> str: """Return the bare media type per RFC 7231: strip params, trim, lowercase.""" @@ -42,9 +44,17 @@ def _is_json_content_type(content_type: str) -> bool: return _normalize_media_type(content_type) == "application/json" +def _unqualified(annotation: object) -> object: + """Which qualifiers ``get_type_hints`` already stripped varies by interpreter version, so peel them all.""" + if get_origin(annotation) not in _ANNOTATION_QUALIFIERS: + return annotation + qualified: Final[tuple[object, ...]] = get_args(annotation) + return _unqualified(qualified[0]) + + def _numeric_form_type(annotation: object) -> type[int] | type[float] | None: """The scalar to parse an ``int``/``float``-typed field as, else ``None``.""" - unwrapped: Final = get_args(annotation)[0] if get_origin(annotation) is ReadOnly else annotation + unwrapped: Final = _unqualified(annotation) candidates: Final = ( tuple(arg for arg in get_args(unwrapped) if arg is not type(None)) if get_origin(unwrapped) in (Union, UnionType) diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index fcfb9342176..011571a37e0 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -1053,6 +1053,8 @@ class TestNumericFormFields: read_only: ReadOnly[int | None] not_required: NotRequired[ReadOnly[int]] required: Required[ReadOnly[Annotated[float, "meta"]]] + read_only_not_required: ReadOnly[NotRequired[int]] + read_only_required: ReadOnly[Required[float]] assert dict(numeric_form_fields(get_type_hints(Schema))) == { "plain": int, @@ -1061,6 +1063,22 @@ class TestNumericFormFields: "read_only": int, "not_required": int, "required": float, + "read_only_not_required": int, + "read_only_required": float, + } + + def test_qualifiers_are_unwrapped_when_get_type_hints_keeps_extras(self): + from typing_extensions import Annotated, NotRequired, ReadOnly, Required, TypedDict + + class Schema(TypedDict, total=False): + annotated: ReadOnly[Annotated[int, "meta"]] + not_required: NotRequired[ReadOnly[int]] + required: Required[ReadOnly[Annotated[float, "meta"]]] + + assert dict(numeric_form_fields(get_type_hints(Schema, include_extras=True))) == { + "annotated": int, + "not_required": int, + "required": float, } def test_non_scalar_and_bool_fields_are_skipped(self): From 50fb35e17eecef15260eb4c1cd3610afef8e08cc Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 13:39:54 -0700 Subject: [PATCH 36/60] fix(vector_stores): make MongoDB errors actionable on self-managed deployments mongod serves $vectorSearch identically whether mongot runs under Atlas or beside a self-managed deployment, so the provider already worked against on-prem. The guidance did not: a refused connection told the operator to check their project's IP access list and whether the cluster was paused, neither of which exists outside Atlas, and the index errors claimed an "Atlas Vector Search index" they do not have. Every message now names a remedy for both, keeping the Atlas-specific hint labelled as such. Also diagnoses unescaped credentials, which self-managed deployments hit more often because the password is usually generated. pymongo reports those three different ways and none of them mentions the password: '@', ':' and '%' raise an RFC 3986 complaint, '/' is read as the database separator and surfaces as Bad database name, and an unescaped ':' looks like a bad port and comes back as a plain ValueError. All three now point at the credentials. The ValueError branch's comment claimed it fired on an unescaped '/', which pymongo actually reports as InvalidURI; corrected to the port parse it really catches. Verified against a self-managed mongod 8.0 with mongot, reached over plain mongodb:// with no SRV and no TLS: 13 cases with live OpenAI embeddings, and 4 credential cases against an auth-enabled instance whose password holds % @ / and :. list_search_indexes returns the same queryable and status fields there as on Atlas, so the index-readiness check needed no change. --- litellm/llms/mongodb/common_utils.py | 46 +++-- .../mongodb/vector_stores/transformation.py | 22 +-- .../test_mongodb_transformation.py | 168 +++++++++++++++++- 3 files changed, 205 insertions(+), 31 deletions(-) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index bf3bf953772..0978368e874 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -1,10 +1,10 @@ -"""Shared helpers for MongoDB Atlas integrations. +"""Shared helpers for MongoDB integrations. pymongo ships in the optional ``mongodb`` extra, so every import of it is deferred to call time and raises an actionable error when it is absent. Clients are cached per connection because building one costs an SRV lookup, a -TLS handshake and topology discovery: measured at ~890ms against Atlas versus +TLS handshake and topology discovery: measured at ~890ms against a remote deployment versus ~80ms on a warm client, so a client per search would dominate query latency. """ @@ -141,15 +141,16 @@ def reset_client_cache() -> None: _AUTHENTICATION_FAILED_CODE: Final = 18 _UNAUTHORIZED_CODE: Final = 13 -# Atlas reports a rejected user as code 8000 "AtlasError", not 18, so only the message is reliable +# Atlas reports a rejected user as code 8000 "AtlasError" where a self-managed mongod reports 18 _AUTHENTICATION_MESSAGE_MARKERS: Final = ("bad auth", "authentication failed", "not authorized") _RESOLUTION_TIMEOUT_MARKERS: Final = ("resolution lifetime expired", "dns operation timed out") _UNKNOWN_HOSTNAME_MARKERS: Final = ("dns query name does not exist", "name or service not known") +_CREDENTIAL_ESCAPING_MARKERS: Final = ("must be escaped according to rfc 3986", "bad database name") def _index_hint(index_name: str, database: str, collection: str) -> str: return ( - f"No queryable Atlas Vector Search index named '{index_name}' was found on " + f"No queryable MongoDB Vector Search index named '{index_name}' was found on " f"'{database}.{collection}'. Confirm the index exists on that exact collection, that its " "status is READY rather than still building, and that the vector store id matches the index name." ) @@ -168,7 +169,7 @@ def missing_index_error(index_name: str, database: str, collection: str) -> BadR def index_not_ready_error(index_name: str, database: str, collection: str, status: str) -> BadRequestError: return config_error( - f"The Atlas Vector Search index '{index_name}' on '{database}.{collection}' is not queryable " + f"The MongoDB Vector Search index '{index_name}' on '{database}.{collection}' is not queryable " f"yet; its status is {status}. Searches against it return no results until the build finishes." ) @@ -194,8 +195,9 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll if isinstance(error, ServerSelectionTimeoutError): return timeout_error( "Could not reach the MongoDB deployment before the timeout. On Atlas this is usually the " - "project's IP access list not containing this host, or a paused cluster; it can also be an " - f"unresolvable hostname. Driver detail: {error}" + "project's IP access list not containing this host, or a paused cluster. On a self-managed " + "deployment it is usually the host or port in the URI, or a firewall between this process " + f"and mongod. Either way it can also be an unresolvable hostname. Driver detail: {error}" ) # ExecutionTimeout subclasses OperationFailure, so it has to be matched before it if isinstance(error, (NetworkTimeout, ExecutionTimeout)): @@ -208,8 +210,9 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll if isinstance(error, ConnectionFailure): return config_error( f"The connection to '{database}.{collection}' was refused or dropped. On Atlas this is " - "usually a connection string with no username and password, or a TLS failure. Confirm " - f"the URI is the one Atlas shows under Connect, Drivers. Driver detail: {error}" + "usually a connection string with no username and password, or a TLS failure, so confirm " + "the URI is the one Atlas shows under Connect, Drivers. On a self-managed deployment, check " + f"that mongod is listening on the host and port in the URI. Driver detail: {error}" ) if isinstance(error, OperationFailure): code: Final = error.code @@ -223,13 +226,13 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll ) if "dimension" in detail: return config_error( - "The query embedding does not match the vector dimensions the Atlas index was built for. " + "The query embedding does not match the vector dimensions the index was built for. " "litellm_embedding_model must be the same model that produced the stored vectors. " f"Driver detail: {error}" ) if "is not indexed as vector" in detail: return config_error( - "mongodb_embedding_field names a field the Atlas Vector Search index does not cover. " + "mongodb_embedding_field names a field the MongoDB Vector Search index does not cover. " f"It must match the 'path' the index '{index_name}' was created on. Driver detail: {error}" ) if "index" in detail and ("not found" in detail or "does not exist" in detail or "unknown" in detail): @@ -248,19 +251,28 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll ) if any(marker in configuration_detail for marker in _UNKNOWN_HOSTNAME_MARKERS): return config_error( - "The cluster hostname in mongodb_connection_string does not exist in DNS. Check the " - f"cluster name against the URI Atlas shows under Connect, Drivers. Driver detail: {error}" + "The hostname in mongodb_connection_string does not exist in DNS. On Atlas, check the " + "cluster name against the URI shown under Connect, Drivers. On a self-managed deployment, " + f"check that the hostname resolves from this process. Driver detail: {error}" + ) + if any(marker in configuration_detail for marker in _CREDENTIAL_ESCAPING_MARKERS): + return config_error( + "mongodb_connection_string could not be parsed. A username or password containing " + "'@', '/', ':' or '%' has to be percent-encoded per RFC 3986, so 'p@ss/word' becomes " + "'p%40ss%2Fword'. If the credentials are already encoded, check the database name in " + f"the URI path instead. Driver detail: {error}" ) return config_error( f"mongodb_connection_string is not a usable MongoDB connection string. Driver detail: {error}" ) if isinstance(error, InvalidOperation): return config_error(f"The MongoDB client was already closed or is unusable. Driver detail: {error}") - # pymongo's URI parser raises a plain ValueError, not a PyMongoError, for a password holding an - # unescaped '/', which would otherwise reach the caller as a 500 + # pymongo raises a plain ValueError, not a PyMongoError, for an unusable port, which an unescaped + # ':' in a password also produces, and which would otherwise reach the caller as a 500 if isinstance(error, ValueError): return config_error( - "mongodb_connection_string could not be parsed. A username or password containing " - f"'@', '/', ':' or '%' has to be percent-encoded per RFC 3986. Driver detail: {error}" + "The host and port in mongodb_connection_string could not be parsed. If the port is a " + "number between 0 and 65535, the cause is usually an unescaped ':' in the password, which " + f"has to be percent-encoded per RFC 3986 as '%3A'. Driver detail: {error}" ) return error diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index 5e59fd30f1b..571061d39a2 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -1,11 +1,12 @@ -"""MongoDB Atlas vector store provider. +"""MongoDB vector store provider, for Atlas and self-managed deployments alike. -Atlas Vector Search has no HTTP query API (the Data API and HTTPS Endpoints are +MongoDB Vector Search has no HTTP query API (the Data API and HTTPS Endpoints are end-of-life), so this config extends BaseDirectVectorStoreConfig and runs the ``$vectorSearch`` aggregation itself through pymongo instead of shaping an httpx -request. +request. mongod serves that stage identically whether mongot runs under Atlas or +beside a self-managed deployment, so one code path covers both. -``vector_store_id`` is the Atlas Search index name, matching the Valkey provider +``vector_store_id`` is the search index name, matching the Valkey provider where the id names the index; the database and collection it covers come from litellm_params. """ @@ -60,7 +61,7 @@ MAX_QUERY_CHARACTERS: Final = 32_000 _EMPTY_EMBEDDING_CONFIG: Final = MappingProxyType({}) _SEARCH_ONLY_MESSAGE: Final = ( - "MongoDB vector store is search-only. Create the collection and its Atlas Vector Search " + "MongoDB vector store is search-only. Create the collection and its MongoDB Vector Search " "index in MongoDB directly, then register it here by index name." ) @@ -101,7 +102,8 @@ class _MongoDBSearchParams(BaseModel): if not self.mongodb_connection_string: raise config_error( "mongodb_connection_string is required in litellm_params for the MongoDB vector store. " - "Example: mongodb+srv://:@.mongodb.net" + "Example: mongodb+srv://:@.mongodb.net for Atlas, or " + "mongodb://:@:27017 for a self-managed deployment" ) scheme: Final = self.mongodb_connection_string.split("://", 1)[0].lower() if scheme not in ("mongodb", "mongodb+srv"): @@ -234,12 +236,12 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): if vector_store_search_optional_params.get("filters") is not None: raise config_error( "MongoDB vector store does not support the filters parameter yet. " - "Restrict the collection or the Atlas Vector Search index definition instead." + "Restrict the collection or the MongoDB Vector Search index definition instead." ) if vector_store_search_optional_params.get("ranking_options") is not None: raise config_error( "MongoDB vector store does not support the ranking_options parameter yet. " - "Every result already carries the Atlas vectorSearchScore, so filter or re-rank " + "Every result already carries the vectorSearchScore, so filter or re-rank " "on that rather than having the threshold silently ignored." ) if vector_store_search_optional_params.get("rewrite_query") is not None: @@ -296,7 +298,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): def _raise_for_missing_text_field( cls, documents: Sequence[Mapping[str, object]], text_field: str, database: str, collection: str ) -> None: - """Atlas happily matches vectors in documents that carry no text at all, so a mistyped + """$vectorSearch happily matches documents that carry no text at all, so a mistyped mongodb_text_field returns well-scored results whose content is empty and feeds an empty context to the model. Every matched document lacking the field is the misconfiguration.""" if documents and all(cls._field_value(document, text_field) is None for document in documents): @@ -322,7 +324,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): def _raise_for_unusable_index( catalogue: Sequence[Mapping[str, object]], index_name: str, database: str, collection: str ) -> None: - """An empty result set is ambiguous: Atlas returns zero documents both for a query that + """An empty result set is ambiguous: mongod returns zero documents both for a query that genuinely matched nothing and for a missing database, collection or index. Only the second is a misconfiguration, so the index catalogue decides which one happened.""" if not catalogue: diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index d60504c31e5..4e57755076f 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -788,8 +788,8 @@ class TestErrorTranslation: assert "refused or dropped" not in str(translated) def test_an_unescaped_password_character_is_a_400_not_a_500(self): - """pymongo's URI parser raises a plain ValueError, not a PyMongoError, when a password - holds an unescaped '/'. That is a routine mistake and it must not be a 500.""" + """pymongo's URI parser raises a plain ValueError, not a PyMongoError, for an unusable port, + which is also what an unescaped ':' in a password produces. It must not be a 500.""" translated = self._translate(ValueError("Port contains non-digit characters")) assert isinstance(translated, BadRequestError) @@ -895,7 +895,7 @@ class TestEmptyResultsAreDisambiguated: def test_a_missing_index_becomes_an_error_rather_than_an_empty_page(self): config, _, collection = _config(documents=[], search_indexes=[]) - with pytest.raises(BadRequestError, match="No queryable Atlas Vector Search index"): + with pytest.raises(BadRequestError, match="No queryable MongoDB Vector Search index"): _search(config) assert collection.listed_indexes == [INDEX] @@ -934,7 +934,7 @@ class TestEmptyResultsAreDisambiguated: async def test_async_missing_index_becomes_an_error_rather_than_an_empty_page(self): config, _, collection = _async_config(documents=[], search_indexes=[]) - with pytest.raises(BadRequestError, match="No queryable Atlas Vector Search index"): + with pytest.raises(BadRequestError, match="No queryable MongoDB Vector Search index"): await _asearch(config) assert collection.listed_indexes == [INDEX] @@ -1202,3 +1202,163 @@ class TestClientConstructionFailures: with pytest.raises(BadRequestError, match="not a usable MongoDB connection string"): await _asearch(config) + + +class TestSelfManagedDeploymentsAreFirstClass: + """mongod serves $vectorSearch identically whether mongot runs under Atlas or beside a + self-managed deployment, so an operator without an Atlas account has to be able to act on + every message. Guidance that only names Atlas remedies sends them looking for an IP access + list and a paused cluster that do not exist in their deployment.""" + + def _config_that_fails_to_connect(self, error): + def factory(_key): + raise error + + return MongoDBVectorStoreConfig( + embedding_fn=FakeEmbeddingFn([0.1, 0.2, 0.3]), sync_client_factory=factory + ) + + def test_a_plain_mongodb_uri_without_srv_or_credentials_is_accepted(self): + params = _MongoDBSearchParams.model_validate( + {**BASE_PARAMS, "mongodb_connection_string": "mongodb://mongod.internal:27017"} + ) + + assert params.require_connection_string() == "mongodb://mongod.internal:27017" + + def test_an_unreachable_deployment_names_a_self_managed_remedy(self): + from pymongo.errors import ServerSelectionTimeoutError + + config = self._config_that_fails_to_connect(ServerSelectionTimeoutError("connection refused")) + + with pytest.raises(Timeout) as excinfo: + _search(config) + + assert "self-managed" in str(excinfo.value) + assert "host or port" in str(excinfo.value) + + def test_a_refused_connection_names_a_self_managed_remedy(self): + from pymongo.errors import ConnectionFailure + + config = self._config_that_fails_to_connect(ConnectionFailure("connection closed")) + + with pytest.raises(BadRequestError) as excinfo: + _search(config) + + assert "self-managed" in str(excinfo.value) + assert "mongod is listening" in str(excinfo.value) + + def test_an_unresolvable_hostname_names_a_self_managed_remedy(self): + from pymongo.errors import ConfigurationError + + config = self._config_that_fails_to_connect(ConfigurationError("The DNS query name does not exist")) + + with pytest.raises(BadRequestError) as excinfo: + _search(config) + + assert "self-managed" in str(excinfo.value) + + def test_the_missing_index_message_does_not_claim_atlas(self): + message = str(missing_index_error(INDEX, "sample_mflix", "embedded_movies")) + + assert "MongoDB Vector Search index" in message + assert "Atlas" not in message + + def test_the_not_ready_message_does_not_claim_atlas(self): + message = str(index_not_ready_error(INDEX, "sample_mflix", "embedded_movies", "PENDING")) + + assert "MongoDB Vector Search index" in message + assert "Atlas" not in message + + def test_the_search_only_refusal_does_not_claim_atlas(self): + config = MongoDBVectorStoreConfig() + + with pytest.raises(BadRequestError) as excinfo: + config.transform_create_vector_store_request({}, api_base="") + + assert "Atlas" not in str(excinfo.value) + + def test_a_dimension_mismatch_does_not_claim_atlas(self): + from pymongo.errors import OperationFailure + + error = OperationFailure("vector field is indexed with 128 dimensions but queried with 256") + translated = translate_mongo_error(error, index_name=INDEX, database="db", collection="c") + + assert "Atlas" not in str(translated) + assert "dimensions the index was built for" in str(translated) + + def test_an_uncovered_embedding_field_does_not_claim_atlas(self): + from pymongo.errors import OperationFailure + + error = OperationFailure("embedding is not indexed as vector") + translated = translate_mongo_error(error, index_name=INDEX, database="db", collection="c") + + assert "MongoDB Vector Search index does not cover" in str(translated) + assert "Atlas" not in str(translated) + + def test_a_self_managed_auth_failure_is_still_recognised_by_code_18(self): + from pymongo.errors import OperationFailure + + error = OperationFailure("Authentication failed.", code=18, details={"code": 18}) + translated = translate_mongo_error(error, index_name=INDEX, database="db", collection="c") + + assert isinstance(translated, BadRequestError) + assert "rejected the credentials" in str(translated) + + +class TestUnescapedCredentialsAreDiagnosed: + """Self-managed deployments usually carry a generated password, so '@', '/', ':' and '%' in one + are routine. pymongo reports those as a port, a database name or an RFC 3986 complaint, none of + which points the operator at their password, so each has to be named for what it is. The errors + here come from pymongo's real parser rather than a synthetic stand-in.""" + + @staticmethod + def _real_parse_error(uri): + from pymongo import MongoClient + + try: + MongoClient(uri, serverSelectionTimeoutMS=1) + except Exception as e: + return e + raise AssertionError(f"expected {uri!r} to fail parsing") + + def _translated(self, uri): + return translate_mongo_error( + self._real_parse_error(uri), index_name=INDEX, database="db", collection="c" + ) + + @pytest.mark.parametrize( + "uri", + [ + "mongodb://user:pa@ss@host:27017/", + "mongodb://user:pa:ss@host:27017/", + "mongodb://user:pa%ss@host:27017/", + "mongodb://user@x:pw@host:27017/", + ], + ) + def test_rfc_3986_complaints_tell_the_operator_to_encode_the_password(self, uri): + translated = self._translated(uri) + + assert isinstance(translated, BadRequestError) + assert "percent-encoded per RFC 3986" in str(translated) + + @pytest.mark.parametrize( + "uri", + ["mongodb://user:pa/ss@host:27017/", "mongodb://user/x:pw@host:27017/"], + ) + def test_a_slash_in_the_credentials_is_not_reported_as_a_database_name(self, uri): + translated = self._translated(uri) + + assert isinstance(translated, BadRequestError) + assert "percent-encoded per RFC 3986" in str(translated) + + def test_an_unusable_port_names_the_host_and_port_not_the_database(self): + translated = self._translated("mongodb://host:99999/") + + assert isinstance(translated, BadRequestError) + assert "host and port" in str(translated) + + def test_a_genuinely_bad_database_name_still_mentions_the_uri_path(self): + translated = self._translated("mongodb://host:27017/has space") + + assert isinstance(translated, BadRequestError) + assert "database name in the URI path" in str(translated) From 323f51269d3d781e19a68aa658b9158fd4d9edcb Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 13:59:07 -0700 Subject: [PATCH 37/60] fix(vector_stores): return 400 when a MongoDB TLS file cannot be read tlsCAFile and tlsCertificateKeyFile are how a self-managed deployment presents a private CA, so they are the options on-prem operators actually set. pymongo opens those files itself during TLS setup and lets OSError out, which is neither a PyMongoError nor a ValueError, so it missed every branch of the translator and litellm.exception_type turned it into a 500 with a traceback in the body. A mistyped path, or one that exists on the host but not inside the container, is a routine mistake and has to read as a 400 naming the file. Matched on the exception carrying a filename so a socket-level OSError still falls through to the branches that handle it. Verified against a self-managed mongod with a missing CA file, a CA path that is a directory, and a missing client certificate. --- litellm/llms/mongodb/common_utils.py | 8 ++++ .../test_mongodb_transformation.py | 43 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 0978368e874..4e37e21948b 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -267,6 +267,14 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll ) if isinstance(error, InvalidOperation): return config_error(f"The MongoDB client was already closed or is unusable. Driver detail: {error}") + # A tlsCAFile or tlsCertificateKeyFile the process cannot open raises OSError from the TLS setup + # rather than a PyMongoError, and those options are how self-managed deployments present a private CA + if isinstance(error, OSError) and error.filename: + return config_error( + f"'{error.filename}', named by a TLS option in mongodb_connection_string, could not be read. " + "Check that tlsCAFile and tlsCertificateKeyFile point at files this process can open; inside " + f"a container that is the path in the container, not on the host. Driver detail: {error}" + ) # pymongo raises a plain ValueError, not a PyMongoError, for an unusable port, which an unescaped # ':' in a password also produces, and which would otherwise reach the caller as a 500 if isinstance(error, ValueError): diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 4e57755076f..7d71ff5c213 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -1362,3 +1362,46 @@ class TestUnescapedCredentialsAreDiagnosed: assert isinstance(translated, BadRequestError) assert "database name in the URI path" in str(translated) + + +class TestUnreadableTlsFilesAreDiagnosed: + """A private CA is how self-managed deployments present TLS, so tlsCAFile and + tlsCertificateKeyFile are on-prem options in practice. pymongo opens those files itself and + lets OSError out, which is not a PyMongoError, so before this they reached the caller as a 500 + with a traceback. The errors here come from pymongo's real TLS setup.""" + + @staticmethod + def _real_tls_error(uri): + from pymongo import MongoClient + + try: + MongoClient(uri, serverSelectionTimeoutMS=1500).admin.command("ping") + except Exception as e: + return e + raise AssertionError(f"expected {uri!r} to fail") + + def _translated(self, uri): + return translate_mongo_error(self._real_tls_error(uri), index_name=INDEX, database="db", collection="c") + + @pytest.mark.parametrize( + "path", + ["/nonexistent-directory-for-tests/ca.pem", "/tmp"], + ) + def test_an_unreadable_ca_file_is_a_400_naming_the_path(self, path): + translated = self._translated(f"mongodb://localhost:27717/?tls=true&tlsCAFile={path}") + + assert isinstance(translated, BadRequestError) + assert path in str(translated) + assert "tlsCAFile" in str(translated) + + def test_an_unreadable_client_certificate_is_a_400_naming_the_path(self): + path = "/nonexistent-directory-for-tests/client.pem" + translated = self._translated(f"mongodb://localhost:27717/?tls=true&tlsCertificateKeyFile={path}") + + assert isinstance(translated, BadRequestError) + assert path in str(translated) + + def test_an_oserror_carrying_no_filename_is_left_for_the_other_branches(self): + translated = translate_mongo_error(OSError("socket hung up"), index_name=INDEX, database="db", collection="c") + + assert not isinstance(translated, BadRequestError) From 7c6638e5c34c2deed2dda7268eeeafdedb341308 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 4 Sep 2026 14:15:39 -0700 Subject: [PATCH 38/60] feat(router): auto-escalate stalled complexity-router tasks Adds stall_escalation_enabled to the complexity router: when the assistant's own recent tool calls look stuck (identical repeats, or repeated tool errors on a surface that reports one), the request is bumped one configured tier higher, the automatic counterpart to escalation_keywords. Detection is stateless: it rereads the last stall_escalation_window tool calls from that request's own message list on every classified turn, so the bump lasts only as long as the recent calls still look stuck and lifts on its own once they don't, and evidence survives a plain follow-up like "try again" instead of resetting on the newest human ask. Off by default. Rejected together with session_affinity and classification_mode='user_turn', which both replay a held routing decision instead of classifying most turns, and with tier_definitions, for the same reason escalation_keywords is: both rely on the built-in tier severity order a custom tier set does not define. Dashboard controls for this are not included; config.yaml and the management API accept it today through ComplexityRouterConfig. --- .../complexity_router/README.md | 46 ++++++ .../complexity_router/complexity_router.py | 12 ++ .../complexity_router/config.py | 56 +++++++ .../complexity_router/stall_detector.py | 126 +++++++++++++++ .../router_strategy/test_complexity_router.py | 151 +++++++++++++++--- .../router_strategy/test_stall_detector.py | 121 ++++++++++++++ 6 files changed, 492 insertions(+), 20 deletions(-) create mode 100644 litellm/router_strategy/complexity_router/stall_detector.py create mode 100644 tests/test_litellm/router_strategy/test_stall_detector.py diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index afa27719064..a7ed9e9dc21 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -247,6 +247,52 @@ unless `modality_routing` is also on. `session_affinity_ttl_seconds` is the idle window for both the model pin selected by session affinity and the deployment pin. Every request that reuses a pin refreshes its TTL, so a session actively sending requests stays pinned. After the window passes with no pin reuse, the next request classifies again and creates a fresh pin. Omit the setting to track the default of 3600 seconds. +### Mid-task stall escalation + +A weak model working an agentic task can get stuck: it keeps calling the same tool with the +same arguments, or the same call keeps erroring, when a stronger model would have broken the +loop. `stall_escalation_enabled: true` catches this and bumps the request one tier higher, the +automatic counterpart to a user typing an escalation keyword: + +```yaml +model_list: + - model_name: smart-router + litellm_params: + model: auto_router/complexity_router + complexity_router_config: + stall_escalation_enabled: true + stall_escalation_window: 6 + stall_escalation_repeat_threshold: 3 + tiers: + SIMPLE: gpt-4o-mini + MEDIUM: gpt-4o + COMPLEX: claude-sonnet-4 + REASONING: o1-preview +``` + +Detection looks at the assistant's own tool calls, not the human's messages: of the last +`stall_escalation_window` tool calls, if `stall_escalation_repeat_threshold` or more are +identical (same tool, same arguments) or came back as errors, the task counts as stalled and the +classified tier is bumped one step by the same `_escalate_tier` ladder `escalation_keywords` +uses, capped at the highest configured tier. It reads both tool-call shapes: Anthropic Messages +`tool_use`/`tool_result` blocks (including `is_error`) and chat-completions `tool_calls`/`tool` +messages (which carry no standard error flag, so those calls are judged on repetition alone). + +There is no state to expire or leak: detection reruns on every classified turn from that +request's own message list, so the bump lasts only as long as the recent tool calls still look +stuck and lifts on its own the moment they don't. This also means it reads the whole +conversation rather than only the turns since the newest human ask, so a plain follow-up like +"try again" does not discard evidence from before it. Escalation records `stall_escalation` in +`routing_decision.signals`; unlike `escalation_keywords`, it does not set the +`escalated`/`escalation_keyword` pair, which is reserved for the keyword mechanism specifically. + +`stall_escalation_enabled` cannot be combined with `session_affinity` or +`classification_mode: user_turn`: both replay a held routing decision on most turns instead of +classifying, so detection would never see the tool calls it needs to look at. It is also +rejected together with `tier_definitions`, for the same reason `escalation_keywords` is: both +rely on the built-in tier severity order, which a custom tier set does not define. Off by +default. + ### Heuristic-first chaining `classifier_type: heuristic_first` runs the local scorer on every request and only calls the LLM diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 17e3d1256d0..01b4665a7d9 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -70,6 +70,7 @@ from .config import ( ComplexityTier, TierDefinition, ) +from .stall_detector import detect_stalled_task if TYPE_CHECKING: from semantic_router.routers import SemanticRouter @@ -3135,6 +3136,17 @@ class ComplexityRouter(CustomLogger): escalated: Final = tier != classified_tier if escalated: signals = (*signals, "escalation") + # Recomputed from this request's own tool calls, not remembered from a prior turn: the + # bump lasts only as long as the recent tool calls still look stuck, and lifts itself + # the moment they don't, with nothing to expire or leak past the task that earned it. + stalled: Final = self.config.stall_escalation_enabled and detect_stalled_task( + resolved_messages, + window=self.config.stall_escalation_window, + repeat_threshold=self.config.stall_escalation_repeat_threshold, + ) + if stalled: + tier = self._escalate_tier(tier) + signals = (*signals, "stall_escalation") pre_floor_tier: Final = tier if plan_floor is not None: tier = self._apply_plan_mode_floor(tier) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 70c1b281e31..508c4ec8c91 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -809,6 +809,42 @@ class ComplexityRouterConfig(BaseModel): description="Rules that force a specific tier when their keywords match the prompt", ) + stall_escalation_enabled: bool = Field( + default=False, + description=( + "Escalate mid-task to the next-higher configured tier when the assistant's own recent " + "tool calls look stuck: stall_escalation_repeat_threshold or more of the last " + "stall_escalation_window tool calls are identical repeats (same tool, same arguments) " + "or came back as errors. One tier at most, on the same ladder escalation_keywords bumps " + "along, and never above the highest configured tier. Detection re-runs on every " + "classified turn from the tool calls visible in that request, so it needs no state and " + "nothing survives past the task: once the recent tool calls stop looking stuck, the " + "next classified turn routes normally again. Mutually exclusive with session_affinity " + "and classification_mode='user_turn', which both replay a held routing decision instead " + "of classifying most turns, so this would never see the tool calls to look at. Off by " + "default." + ), + ) + stall_escalation_window: int = Field( + default=6, + gt=0, + description=( + "How many of the assistant's most recent tool calls stall detection looks at, oldest " + "ones dropped as new calls happen. Counted across the whole visible conversation " + "rather than reset at the newest human ask, so evidence from before a plain follow-up " + "message like 'try again' is still visible on the turn after it." + ), + ) + stall_escalation_repeat_threshold: int = Field( + default=3, + ge=2, + description=( + "How many of the last stall_escalation_window tool calls must be identical repeats, or " + "error results, before the task counts as stalled. Must not exceed " + "stall_escalation_window, or the condition could never be reached." + ), + ) + plan_mode_min_tier: str | None = Field( default=None, description=( @@ -1246,6 +1282,7 @@ class ComplexityRouterConfig(BaseModel): ("adaptive", self.adaptive), ("session_affinity", self.session_affinity), ("escalation_keywords", bool(self.escalation_keywords)), + ("stall_escalation_enabled", self.stall_escalation_enabled), ("plugins", bool(self.plugins)), ) if enabled @@ -1422,6 +1459,25 @@ class ComplexityRouterConfig(BaseModel): ) return self + @model_validator(mode="after") + def _validate_stall_escalation(self) -> "ComplexityRouterConfig": + if not self.stall_escalation_enabled: + return self + if self.session_affinity or self.classification_mode == "user_turn": + raise ValueError( + "stall_escalation_enabled cannot be combined with session_affinity or " + "classification_mode='user_turn': both replay a held routing decision on most " + "turns instead of classifying, so stall detection would never see the tool calls " + "of the turns it needs to look at. Disable one or the other." + ) + if self.stall_escalation_repeat_threshold > self.stall_escalation_window: + raise ValueError( + "stall_escalation_repeat_threshold " + f"({self.stall_escalation_repeat_threshold}) cannot exceed stall_escalation_window " + f"({self.stall_escalation_window}); the condition could never be reached." + ) + return self + @model_validator(mode="after") def _validate_tier_param_placement(self) -> "ComplexityRouterConfig": """Reject a router setting written into a tier entry's request params. diff --git a/litellm/router_strategy/complexity_router/stall_detector.py b/litellm/router_strategy/complexity_router/stall_detector.py new file mode 100644 index 00000000000..450f8b6a653 --- /dev/null +++ b/litellm/router_strategy/complexity_router/stall_detector.py @@ -0,0 +1,126 @@ +""" +Mid-task stall detection for the Complexity Router. + +Looks at the assistant's own recent tool calls -- visible on every request an agentic +client resends, since each turn carries the whole conversation so far -- for a tight loop +of identical calls or repeated tool errors. No LLM call, no state: the same fixed-size +window is rescanned on every classified turn, so a stall reads the same way whether it +started one turn ago or ten, and stops reading as a stall the moment the recent calls +change. + +Assistant tool calls appear in two shapes depending on the API surface, and this module +reads both without translating one into the other: +- Anthropic Messages: assistant `content` blocks of type "tool_use" (id, name, input), + answered by a later user-turn `content` block of type "tool_result" (tool_use_id, + is_error). +- Chat completions: assistant `tool_calls` entries (id, function.name, function.arguments + as a JSON string), answered by a later `role: "tool"` message. Chat completions has no + standard error flag on that message, so those calls are judged on repetition alone. +""" + +from __future__ import annotations + +import json +from collections import Counter +from collections.abc import Iterator, Mapping, Sequence +from itertools import islice +from typing import Final, NamedTuple + +_ARGUMENTS_PARSE_FAILED: Final = object() + + +class _ToolCallEvent(NamedTuple): + signature: tuple[str, str] + is_error: bool | None + """None when the surface carries no structured error signal for this call. Never + treated as an error: a call this module cannot judge must not count toward the tally.""" + + +def _json_arguments(raw: str) -> object: + try: + return json.loads(raw) + except (TypeError, ValueError): + return _ARGUMENTS_PARSE_FAILED + + +def _tool_call_signature(name: str, raw_arguments: object) -> tuple[str, str]: + """A (name, canonical-arguments) pair that compares equal across both surfaces' + argument shapes: a dict (Anthropic `input`) and a JSON-encoded string (chat + completions `function.arguments`) representing the same call must match.""" + parsed: Final = _json_arguments(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments + arguments: Final = raw_arguments if parsed is _ARGUMENTS_PARSE_FAILED else parsed + try: + return name, json.dumps(arguments, sort_keys=True, default=str) + except (TypeError, ValueError): + return name, str(arguments) + + +def _iter_tool_result_error_pairs(messages: Sequence[Mapping[str, object]]) -> Iterator[tuple[str, bool]]: + """(call id, whether that call's result was an error), read only where the surface + reports one: an Anthropic Messages `tool_result` content block's `is_error`.""" + for msg in messages: + content = msg.get("content") + if msg.get("role") != "user" or not isinstance(content, list): + continue + for part in content: + if isinstance(part, Mapping) and part.get("type") == "tool_result": + call_id = part.get("tool_use_id") + if isinstance(call_id, str): + yield call_id, bool(part.get("is_error", False)) + + +def _iter_tool_call_events_newest_first(messages: Sequence[Mapping[str, object]]) -> Iterator[_ToolCallEvent]: + """Every tool call the assistant made, newest first, paired with its result's error + status where the surface reports one.""" + error_by_call_id: Final = dict(_iter_tool_result_error_pairs(messages)) + for msg in reversed(messages): + if msg.get("role") != "assistant": + continue + content = msg.get("content") + if isinstance(content, list): + for part in reversed(content): + if not (isinstance(part, Mapping) and part.get("type") == "tool_use"): + continue + name = part.get("name") + if isinstance(name, str): + call_id = part.get("id") + yield _ToolCallEvent( + signature=_tool_call_signature(name, part.get("input")), + is_error=error_by_call_id.get(call_id) if isinstance(call_id, str) else None, + ) + tool_calls = msg.get("tool_calls") + if not isinstance(tool_calls, list): + continue + for call in reversed(tool_calls): + function = call.get("function") if isinstance(call, Mapping) else None + name = function.get("name") if isinstance(function, Mapping) else None + if isinstance(name, str): + yield _ToolCallEvent( + signature=_tool_call_signature(name, function.get("arguments") if function else None), + is_error=None, + ) + + +def detect_stalled_task( + messages: Sequence[Mapping[str, object]] | None, + *, + window: int, + repeat_threshold: int, +) -> bool: + """Whether the assistant's recent tool-call activity looks stuck: repeat_threshold or + more of the last `window` tool calls share an identical signature, or resolved to an + error on a surface that reports one. + + Reads the whole message list rather than only the turns since the newest human ask, + so a follow-up like "try again" does not discard the evidence that came before it. + """ + if not messages or repeat_threshold <= 0: + return False + recent: Final = tuple(islice(_iter_tool_call_events_newest_first(messages), window)) + if len(recent) < repeat_threshold: + return False + _, most_common_count = Counter(event.signature for event in recent).most_common(1)[0] + if most_common_count >= repeat_threshold: + return True + error_count: Final = sum(1 for event in recent if event.is_error) + return error_count >= repeat_threshold diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index c74360875f7..3ae3165bf62 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2690,9 +2690,7 @@ class TestRouterPreRoutingAliasOverrides: import time monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) - (tmp_path / "api-key.json").write_text( - json.dumps({"token": "tid=test", "expires_at": int(time.time()) + 3600}) - ) + (tmp_path / "api-key.json").write_text(json.dumps({"token": "tid=test", "expires_at": int(time.time()) + 3600})) router = Router( model_list=[ { @@ -2717,7 +2715,9 @@ class TestRouterPreRoutingAliasOverrides: copilot_resolutions: List = [] def _guarded(*args, **kwargs): - target = str(kwargs.get("model") or (args[0] if args else "")) + str(kwargs.get("custom_llm_provider") or "") + target = str(kwargs.get("model") or (args[0] if args else "")) + str( + kwargs.get("custom_llm_provider") or "" + ) if "github_copilot" in target: copilot_resolutions.append(target) raise RuntimeError("routing must not resolve an authenticating provider") @@ -5887,6 +5887,123 @@ class TestEscalationKeywords: assert result.model == "o1-b" # unchanged: no random hop to o1-a / o1-c +def _stalled_tool_history(repeats: int = 3) -> List[Dict]: + """`repeats` identical bash tool calls in a row, the automatic counterpart to a user + typing an escalation keyword: the assistant, not the human, is the one stuck.""" + return [ + turn + for i in range(repeats) + for turn in ( + { + "role": "assistant", + "content": [{"type": "tool_use", "id": f"call-{i}", "name": "bash", "input": {"cmd": "pytest"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": f"call-{i}", "is_error": True, "content": "fail"}], + }, + ) + ] + + +class TestStallEscalation: + """Mid-task auto-escalation when the assistant's own recent tool calls look stuck: the + automatic counterpart to escalation_keywords, gated by stall_escalation_enabled and off + by default.""" + + @pytest.mark.asyncio + async def test_repeated_tool_calls_escalate_the_classified_tier(self, mock_router_instance, basic_config): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + messages = [*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}] + result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) + assert result.model == "gpt-4o" # SIMPLE bumped to MEDIUM + + @pytest.mark.asyncio + async def test_varied_tool_calls_do_not_escalate(self, mock_router_instance, basic_config): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + messages = [ + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "c1", "name": "bash", "input": {"cmd": "ls"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "c1", "is_error": False, "content": "ok"}], + }, + {"role": "user", "content": "Hello there!"}, + ] + result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) + assert result.model == "gpt-4o-mini" # not escalated + + @pytest.mark.asyncio + async def test_disabled_by_default_ignores_repeated_tool_calls(self, mock_router_instance, basic_config): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=basic_config, + ) + messages = [*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}] + result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) + assert result.model == "gpt-4o-mini" # stall_escalation_enabled defaults False + + @pytest.mark.asyncio + async def test_signals_record_stall_escalation(self, mock_router_instance, basic_config): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + messages = [*_stalled_tool_history(), {"role": "user", "content": "Hello there!"}] + result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) + assert "stall_escalation" in result.routing_decision["signals"] + + @pytest.mark.asyncio + async def test_stall_escalation_caps_at_highest_tier(self, mock_router_instance, basic_config): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + messages = [ + *_stalled_tool_history(), + {"role": "user", "content": "Let's think step by step and reason through this carefully."}, + ] + result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) + assert result.model == "o1-preview" # already REASONING, stays there + + @pytest.mark.asyncio + async def test_stall_escalation_stacks_with_keyword_escalation(self, mock_router_instance, basic_config): + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + messages = [*_stalled_tool_history(), {"role": "user", "content": "LITELLM ESCALATE Hello there!"}] + result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) + assert result.model == "claude-sonnet-4-20250514" # SIMPLE -> MEDIUM (keyword) -> COMPLEX (stall) + + @pytest.mark.asyncio + async def test_evidence_survives_a_new_human_ask(self, mock_router_instance, basic_config): + """A plain follow-up like 'try again' must not erase the stall evidence that came + before it: escalation still fires on the turn carrying that follow-up.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "stall_escalation_enabled": True}, + ) + messages = [*_stalled_tool_history(), {"role": "user", "content": "try again"}] + result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) + assert result.model == "gpt-4o" # SIMPLE ("try again" carries no signal) bumped to MEDIUM + + class TestRoutingDecisionContents: """Every routing path must return a PreRoutingHookResponse carrying a routing_decision that names the mechanism that actually decided, with the facts of that path only.""" @@ -7781,7 +7898,6 @@ class TestClientHousekeepingCalls: assert result is not None assert result.model == "claude-sonnet-4-20250514" - @pytest.mark.asyncio async def test_a_classifier_plugin_still_decides_its_own_routers(self, mock_router_instance): """A plugin is where an operator encodes policy the tier ladder cannot express. @@ -7816,9 +7932,7 @@ class TestClientHousekeepingCalls: assert result.model == "o1-preview" assert result.routing_decision["cause"] == "classifier_plugin" - def _adaptive_router( - self, tier_distance_penalty: float, plan_mode_min_tier: str | None = None - ) -> ComplexityRouter: + def _adaptive_router(self, tier_distance_penalty: float, plan_mode_min_tier: str | None = None) -> ComplexityRouter: adaptive_instance = MagicMock() adaptive_instance.model_list = [ { @@ -7855,9 +7969,7 @@ class TestClientHousekeepingCalls: return router @pytest.mark.asyncio - async def test_the_bandit_cannot_route_a_housekeeping_call_above_the_cheapest_tier( - self, mock_router_instance - ): + async def test_the_bandit_cannot_route_a_housekeeping_call_above_the_cheapest_tier(self, mock_router_instance): """The tier here is what the request IS, not how hard it is, so the bandit has nothing to win. Without a ceiling the tier distance penalty is the only thing holding the tier, so a @@ -7890,7 +8002,6 @@ class TestClientHousekeepingCalls: assert result is not None assert result.model == "premium" - @pytest.mark.asyncio async def test_a_housekeeping_call_never_becomes_the_session_pin(self, mock_router_instance): """Pinning this is the most expensive mistake of the transient causes. @@ -7932,9 +8043,7 @@ class TestClientHousekeepingCalls: assert work_turn.routing_decision["cause"] == "llm_classifier" @pytest.mark.asyncio - async def test_the_decision_records_which_sentinel_matched( - self, mock_router_instance, llm_classifier_config - ): + async def test_the_decision_records_which_sentinel_matched(self, mock_router_instance, llm_classifier_config): """The cause's contract says the sentinel rides in matched_keyword, so it has to be there. Without it an operator reading the logs can see that a call was treated as housekeeping but @@ -7955,7 +8064,6 @@ class TestClientHousekeepingCalls: "Write the title in the predominant language of the session" ) - @pytest.mark.asyncio async def test_the_plan_mode_floor_raises_a_housekeeping_call_under_adaptive(self, mock_router_instance): """Floor and ceiling must not contradict each other on the same request. @@ -9054,6 +9162,7 @@ class TestTierDefinitions: ({"adaptive": True}, "severity order"), ({"session_affinity": True}, "severity order"), ({"escalation_keywords": ["GO UP"]}, "severity order"), + ({"stall_escalation_enabled": True}, "severity order"), ( {"classifier_llm_config": {"model": "haiku-classifier", "system_prompt": "grade it"}}, "system_prompt", @@ -10340,9 +10449,7 @@ class TestHeuristicFirst: # Scores 0.175 with one signal, so it sits 0.025 from simple_medium: the pair of tiers either side of # that boundary are different model pools, and a hair's difference in score picks the other one. -NEAR_BOUNDARY_PROMPT = ( - "design a distributed cache with consistent hashing, then explain the failure modes step by step" -) +NEAR_BOUNDARY_PROMPT = "design a distributed cache with consistent hashing, then explain the failure modes step by step" # Scores 0.075 with signals, the far side of any margin under 0.075: the scorer is decided here. CLEAR_OF_BOUNDARY_PROMPT = "explain step by step how consistent hashing rebalances keys" @@ -10784,6 +10891,7 @@ class TestContextWindowEscalation: litellm_router_instance=_windowed_router(_SMALL, _BIG), complexity_router_config=_tier_config(session_affinity=True), ) + def session_kwargs() -> dict[str, object]: return {"metadata": {"session_id": "s-1", "user_api_key_hash": "k-1"}} @@ -10808,6 +10916,7 @@ class TestContextWindowEscalation: litellm_router_instance=_windowed_router(_SMALL, _BIG), complexity_router_config=_tier_config(session_affinity=True), ) + def session_kwargs() -> dict[str, object]: return {"metadata": {"session_id": "s-2", "user_api_key_hash": "k-2"}} @@ -10885,7 +10994,9 @@ class TestContextWindowEscalation: copilot_resolutions: List = [] def _guarded(*args, **kwargs): - target = str(kwargs.get("model") or (args[0] if args else "")) + str(kwargs.get("custom_llm_provider") or "") + target = str(kwargs.get("model") or (args[0] if args else "")) + str( + kwargs.get("custom_llm_provider") or "" + ) if "github_copilot" in target: copilot_resolutions.append(target) raise RuntimeError("the gate must not resolve an authenticating provider") diff --git a/tests/test_litellm/router_strategy/test_stall_detector.py b/tests/test_litellm/router_strategy/test_stall_detector.py new file mode 100644 index 00000000000..8f39969a8ec --- /dev/null +++ b/tests/test_litellm/router_strategy/test_stall_detector.py @@ -0,0 +1,121 @@ +""" +Tests for mid-task stall detection: repeated identical tool calls or repeated tool +errors, read from both Anthropic Messages and chat-completions tool-call shapes. +""" + +from litellm.router_strategy.complexity_router.stall_detector import detect_stalled_task + + +def _anthropic_call(call_id: str, name: str, arguments: dict, *, is_error: bool) -> list[dict]: + return [ + {"role": "assistant", "content": [{"type": "tool_use", "id": call_id, "name": name, "input": arguments}]}, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": call_id, "is_error": is_error, "content": "result"}], + }, + ] + + +def _chat_completions_call(call_id: str, name: str, arguments_json: str) -> list[dict]: + return [ + { + "role": "assistant", + "tool_calls": [ + {"id": call_id, "type": "function", "function": {"name": name, "arguments": arguments_json}} + ], + }, + {"role": "tool", "tool_call_id": call_id, "content": "result"}, + ] + + +class TestDetectStalledTask: + def test_repeated_identical_anthropic_calls_are_stalled(self): + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=False), + *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False), + *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + + def test_repeated_errors_are_stalled_even_with_varied_arguments(self): + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest tests/a.py"}, is_error=True), + *_anthropic_call("t2", "bash", {"cmd": "pytest tests/b.py"}, is_error=True), + *_anthropic_call("t3", "bash", {"cmd": "pytest tests/c.py"}, is_error=True), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + + def test_varied_successful_calls_are_not_stalled(self): + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "ls"}, is_error=False), + *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False), + *_anthropic_call("t3", "grep", {"pattern": "x"}, is_error=False), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + + def test_chat_completions_repeats_are_stalled(self): + messages = [ + *_chat_completions_call("c1", "bash", '{"cmd": "pytest"}'), + *_chat_completions_call("c2", "bash", '{"cmd": "pytest"}'), + *_chat_completions_call("c3", "bash", '{"cmd": "pytest"}'), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + + def test_chat_completions_has_no_structured_error_signal(self): + """A chat-completions tool message carries no standard error flag, so varied calls + whose content happens to read like failures still aren't flagged on error alone.""" + messages = [ + *_chat_completions_call("c1", "bash", '{"cmd": "a"}'), + *_chat_completions_call("c2", "bash", '{"cmd": "b"}'), + *_chat_completions_call("c3", "bash", '{"cmd": "c"}'), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + + def test_dict_and_json_string_arguments_compare_equal_across_surfaces(self): + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=False), + *_chat_completions_call("c2", "bash", '{"cmd": "pytest"}'), + *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + + def test_below_repeat_threshold_is_not_stalled(self): + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=False), + *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + + def test_evidence_older_than_the_window_does_not_count(self): + """Only the most recent `window` tool calls are considered, so a stall the model + already recovered from does not keep re-triggering forever.""" + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=False), + *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False), + *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False), + *_anthropic_call("t4", "grep", {"pattern": "a"}, is_error=False), + *_anthropic_call("t5", "grep", {"pattern": "b"}, is_error=False), + ] + assert detect_stalled_task(messages, window=2, repeat_threshold=2) is False + + def test_evidence_survives_a_new_human_ask(self): + """A follow-up like 'try again' must not erase evidence from before it: detection + reads the whole message list, not just the turns since the newest human ask.""" + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=False), + *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=False), + *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=False), + {"role": "user", "content": [{"type": "text", "text": "try again"}]}, + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + + def test_no_messages_is_not_stalled(self): + assert detect_stalled_task(None, window=6, repeat_threshold=3) is False + assert detect_stalled_task([], window=6, repeat_threshold=3) is False + + def test_zero_threshold_never_flags_stalled(self): + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=True), + *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=True), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=0) is False From 4774a426c5b4dd9bb4e5122941661bf36c0c9fbb Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 14:20:28 -0700 Subject: [PATCH 39/60] refactor(vector_stores): route MongoDB query embeddings through the shared executor The base vector store interface grew an embedding_executor argument, and litellm.vector_stores.search now always passes one. MongoDB still carried its own embedding_fn/aembedding_fn constructor seam, so every search through the public entry point failed with an unexpected keyword argument. Drop the local seam in favour of the shared executor: one path instead of two, and the unit tests now drive the same seam production uses. --- .../mongodb/vector_stores/transformation.py | 37 ++++----- .../test_mongodb_transformation.py | 78 ++++++++++++++----- 2 files changed, 76 insertions(+), 39 deletions(-) diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index 571061d39a2..2e69e35edcf 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -11,15 +11,18 @@ where the id names the index; the database and collection it covers come from litellm_params. """ -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, NoReturn import httpx from pydantic import BaseModel, ConfigDict -import litellm -from litellm.llms.base_llm.vector_store.transformation import BaseDirectVectorStoreConfig +from litellm.llms.base_llm.vector_store.transformation import ( + BaseDirectVectorStoreConfig, + LiteLLMVectorStoreEmbeddingExecutor, + VectorStoreEmbeddingExecutor, +) from litellm.llms.mongodb.common_utils import ( DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_SERVER_SELECTION_TIMEOUT_MS, @@ -139,17 +142,13 @@ _KNOWN_MONGODB_PARAMS: Final = frozenset( class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): def __init__( self, - embedding_fn: Callable[..., EmbeddingResponse] | None = None, - aembedding_fn: Callable[..., Awaitable[EmbeddingResponse]] | None = None, + embedding_executor: VectorStoreEmbeddingExecutor | None = None, sync_client_factory: Callable[[MongoClientKey], object] | None = None, async_client_factory: Callable[[MongoClientKey], object] | None = None, ) -> None: super().__init__() - self.embedding_fn: Final[Callable[..., EmbeddingResponse]] = ( - embedding_fn if embedding_fn is not None else litellm.embedding - ) - self.aembedding_fn: Final[Callable[..., Awaitable[EmbeddingResponse]]] = ( - aembedding_fn if aembedding_fn is not None else litellm.aembedding + self.embedding_executor: Final[VectorStoreEmbeddingExecutor] = ( + embedding_executor if embedding_executor is not None else LiteLLMVectorStoreEmbeddingExecutor() ) self.sync_client_factory: Final[Callable[[MongoClientKey], object]] = ( sync_client_factory if sync_client_factory is not None else get_sync_client @@ -350,6 +349,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: "LiteLLMLoggingObj", litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: self._reject_unknown_params(litellm_params) @@ -359,10 +359,10 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): database: Final = params.require_database() collection: Final = params.require_collection() - embedding_response: Final = self.embedding_fn( - model=params.require_embedding_model(), - input=[query_text], # mutable-ok: litellm.embedding's input contract is a list - **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + embedding_response: Final = (embedding_executor or self.embedding_executor).embed( + params.require_embedding_model(), + query_text, + params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG, ) pipeline: Final = self._pipeline( vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params @@ -392,6 +392,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, litellm_logging_obj: "LiteLLMLoggingObj", litellm_params: Mapping[str, object], + embedding_executor: VectorStoreEmbeddingExecutor | None = None, timeout: float | httpx.Timeout | None = None, ) -> VectorStoreSearchResponse: self._reject_unknown_params(litellm_params) @@ -401,10 +402,10 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): database: Final = params.require_database() collection: Final = params.require_collection() - embedding_response: Final = await self.aembedding_fn( - model=params.require_embedding_model(), - input=[query_text], # mutable-ok: litellm.embedding's input contract is a list - **(params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG), + embedding_response: Final = await (embedding_executor or self.embedding_executor).aembed( + params.require_embedding_model(), + query_text, + params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG, ) pipeline: Final = self._pipeline( vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 7d71ff5c213..7a2df28cc04 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -111,27 +111,27 @@ class FakeClient: return self.database -class FakeEmbeddingFn: +class FakeEmbeddingExecutor: def __init__(self, embedding): self.embedding = embedding - self.captured_kwargs = None + self.captured = None - def __call__(self, **kwargs): - self.captured_kwargs = kwargs + def _respond(self, model, query, configuration): + self.captured = SimpleNamespace(model=model, query=query, configuration=configuration) return SimpleNamespace(data=[{"embedding": self.embedding}] if self.embedding is not None else []) + def embed(self, model, query, configuration): + return self._respond(model, query, configuration) -class FakeAsyncEmbeddingFn(FakeEmbeddingFn): - async def __call__(self, **kwargs): - self.captured_kwargs = kwargs - return SimpleNamespace(data=[{"embedding": self.embedding}] if self.embedding is not None else []) + async def aembed(self, model, query, configuration): + return self._respond(model, query, configuration) def _config(documents=(), embedding=(0.1, 0.2, 0.3), error=None, search_indexes=None): collection = FakeCollection(list(documents), error, search_indexes) client = FakeClient(collection) config = MongoDBVectorStoreConfig( - embedding_fn=FakeEmbeddingFn(list(embedding) if embedding is not None else None), + embedding_executor=FakeEmbeddingExecutor(list(embedding) if embedding is not None else None), sync_client_factory=lambda key: client, ) return config, client, collection @@ -141,7 +141,7 @@ def _async_config(documents=(), embedding=(0.1, 0.2, 0.3), error=None, search_in collection = FakeAsyncCollection(list(documents), error, search_indexes) client = FakeClient(collection) config = MongoDBVectorStoreConfig( - aembedding_fn=FakeAsyncEmbeddingFn(list(embedding) if embedding is not None else None), + embedding_executor=FakeEmbeddingExecutor(list(embedding) if embedding is not None else None), async_client_factory=lambda key: client, ) return config, client, collection @@ -252,22 +252,20 @@ def test_num_candidates_below_the_limit_or_above_the_ceiling_is_rejected(configu def test_list_query_is_joined_into_one_embedding_input(): config, _, _ = _config() - embedding_fn = config.embedding_fn _search(config, query=["deep", "space", "rescue"]) - assert embedding_fn.captured_kwargs["input"] == ["deep space rescue"] + assert config.embedding_executor.captured.query == "deep space rescue" def test_embedding_config_is_expanded_into_the_embedding_call(): config, _, _ = _config() - embedding_fn = config.embedding_fn _search(config, litellm_params={"litellm_embedding_config": {"api_base": "https://example.test", "timeout": 7}}) - assert embedding_fn.captured_kwargs["api_base"] == "https://example.test" - assert embedding_fn.captured_kwargs["timeout"] == 7 - assert embedding_fn.captured_kwargs["model"] == "openai/text-embedding-ada-002" + captured = config.embedding_executor.captured + assert captured.configuration == {"api_base": "https://example.test", "timeout": 7} + assert captured.model == "openai/text-embedding-ada-002" def test_response_maps_documents_to_openai_shaped_results(): @@ -530,7 +528,7 @@ def test_search_fails_when_the_embedding_model_returns_nothing(): def test_validation_runs_before_any_connection_is_opened(): opened = [] config = MongoDBVectorStoreConfig( - embedding_fn=FakeEmbeddingFn([0.1]), + embedding_executor=FakeEmbeddingExecutor([0.1]), sync_client_factory=lambda key: opened.append(key) or FakeClient(FakeCollection([])), ) @@ -973,7 +971,7 @@ class TestEmptyResultsAreDisambiguated: collection = ExplodingCollection([], None, []) config = MongoDBVectorStoreConfig( - embedding_fn=FakeEmbeddingFn([0.1]), + embedding_executor=FakeEmbeddingExecutor([0.1]), sync_client_factory=lambda key: FakeClient(collection), ) @@ -1157,7 +1155,7 @@ class TestClientConstructionFailures: raise error return MongoDBVectorStoreConfig( - embedding_fn=FakeEmbeddingFn([0.1, 0.2, 0.3]), sync_client_factory=factory + embedding_executor=FakeEmbeddingExecutor([0.1, 0.2, 0.3]), sync_client_factory=factory ) def _async_config_that_fails_to_connect(self, error): @@ -1165,7 +1163,7 @@ class TestClientConstructionFailures: raise error return MongoDBVectorStoreConfig( - aembedding_fn=FakeAsyncEmbeddingFn([0.1, 0.2, 0.3]), async_client_factory=factory + embedding_executor=FakeEmbeddingExecutor([0.1, 0.2, 0.3]), async_client_factory=factory ) def test_a_malformed_uri_is_a_bad_request_not_a_500(self): @@ -1215,7 +1213,7 @@ class TestSelfManagedDeploymentsAreFirstClass: raise error return MongoDBVectorStoreConfig( - embedding_fn=FakeEmbeddingFn([0.1, 0.2, 0.3]), sync_client_factory=factory + embedding_executor=FakeEmbeddingExecutor([0.1, 0.2, 0.3]), sync_client_factory=factory ) def test_a_plain_mongodb_uri_without_srv_or_credentials_is_accepted(self): @@ -1405,3 +1403,41 @@ class TestUnreadableTlsFilesAreDiagnosed: translated = translate_mongo_error(OSError("socket hung up"), index_name=INDEX, database="db", collection="c") assert not isinstance(translated, BadRequestError) + + +class TestTheCallerSuppliedEmbeddingExecutorIsUsed: + """litellm.vector_stores.search always hands a direct provider an embedding_executor, so the + provider has to accept it and route the query through it rather than its own default.""" + + def test_the_supplied_executor_produces_the_query_vector(self): + config, _, collection = _config(embedding=(0.9, 0.9, 0.9), search_indexes=READY_INDEX) + caller = FakeEmbeddingExecutor([0.4, 0.5, 0.6]) + + config.execute_search_vector_store_request( + vector_store_id=INDEX, + query="a lone astronaut", + vector_store_search_optional_params={}, + litellm_logging_obj=MagicMock(), + litellm_params=BASE_PARAMS, + embedding_executor=caller, + ) + + assert caller.captured.query == "a lone astronaut" + assert _stage(collection, "$vectorSearch")["queryVector"] == (0.4, 0.5, 0.6) + + @pytest.mark.asyncio + async def test_the_supplied_executor_produces_the_query_vector_on_the_async_path(self): + config, _, collection = _async_config(embedding=(0.9, 0.9, 0.9), search_indexes=READY_INDEX) + caller = FakeEmbeddingExecutor([0.4, 0.5, 0.6]) + + await config.aexecute_search_vector_store_request( + vector_store_id=INDEX, + query="a lone astronaut", + vector_store_search_optional_params={}, + litellm_logging_obj=MagicMock(), + litellm_params=BASE_PARAMS, + embedding_executor=caller, + ) + + assert caller.captured.query == "a lone astronaut" + assert _stage(collection, "$vectorSearch")["queryVector"] == (0.4, 0.5, 0.6) From da58c0c6d5ecd34ff2af2398271034e14eb3fe06 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 15:08:37 -0700 Subject: [PATCH 40/60] fix(vector_stores): keep a lost MongoDB connection retryable and bound the client cache by use litellm only retries 408, 409, 429 and 5xx, so classifying a dropped connection as a 400 turned one replica set failover into a permanently failed search. It is a 503 now, with the message still naming the misconfigurations that also close a connection. The client cache skipped insertion once it held 32 entries, so any store added after that rebuilt its client on every search, paying an SRV lookup, a TLS handshake and topology discovery each time. It evicts the least recently used entry instead, which only drops the cache's own reference. Also trims the explanatory comments to the one-line form the repo asks for. --- litellm/llms/mongodb/common_utils.py | 89 ++++++++++--------- .../mongodb/vector_stores/transformation.py | 32 ++----- .../test_mongodb_transformation.py | 87 +++++++++++++++--- 3 files changed, 133 insertions(+), 75 deletions(-) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 4e37e21948b..27a2a96bd1f 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -1,22 +1,16 @@ -"""Shared helpers for MongoDB integrations. - -pymongo ships in the optional ``mongodb`` extra, so every import of it is -deferred to call time and raises an actionable error when it is absent. - -Clients are cached per connection because building one costs an SRV lookup, a -TLS handshake and topology discovery: measured at ~890ms against a remote deployment versus -~80ms on a warm client, so a client per search would dominate query latency. -""" +"""Shared helpers for the MongoDB integrations. pymongo lives in the optional ``mongodb`` extra, +so every import of it is deferred to call time.""" import asyncio import weakref from asyncio import AbstractEventLoop +from collections import OrderedDict from collections.abc import Callable, Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import TYPE_CHECKING, Final, TypeAlias, TypeVar -from litellm.exceptions import BadRequestError, Timeout +from litellm.exceptions import BadRequestError, ServiceUnavailableError, Timeout if TYPE_CHECKING: from pymongo import AsyncMongoClient, MongoClient @@ -30,8 +24,7 @@ MONGODB_PROVIDER: Final = "mongodb" def config_error(message: str) -> BadRequestError: - """Misconfiguration is the caller's to fix, so it maps to 400 rather than the 500 - a bare ValueError would become once litellm.exception_type wraps it.""" + """400 rather than the 500 a bare ValueError becomes once litellm.exception_type wraps it.""" return BadRequestError(message=message, model=None, llm_provider=MONGODB_PROVIDER) @@ -39,6 +32,11 @@ def timeout_error(message: str) -> Timeout: return Timeout(message=message, model=None, llm_provider=MONGODB_PROVIDER) +def unavailable_error(message: str) -> ServiceUnavailableError: + """litellm only retries 408, 409, 429 and 5xx, so a 400 here would make a failover permanent.""" + return ServiceUnavailableError(message=message, model=None, llm_provider=MONGODB_PROVIDER) + + DEFAULT_CONNECT_TIMEOUT_MS: Final = 10_000 DEFAULT_SOCKET_TIMEOUT_MS: Final = 30_000 DEFAULT_SERVER_SELECTION_TIMEOUT_MS: Final = 10_000 @@ -59,12 +57,26 @@ class MongoClientKey: SyncClientFactory: TypeAlias = Callable[..., "MongoClient"] AsyncClientFactory: TypeAlias = Callable[..., "AsyncMongoClient"] +_K = TypeVar("_K") +_V = TypeVar("_V") + _AsyncClientCacheKey: TypeAlias = tuple[MongoClientKey, int] # CPython recycles id() aggressively, so the id alone would hand a new loop a closed loop's client _AsyncClientEntry: TypeAlias = tuple["weakref.ref[AbstractEventLoop]", "AsyncMongoClient"] -_sync_clients: Final[dict[MongoClientKey, "MongoClient"]] = {} # mutable-ok: process-level client cache -_async_clients: Final[dict[_AsyncClientCacheKey, _AsyncClientEntry]] = {} # mutable-ok: same cache, per loop +_SyncClientCache: TypeAlias = "OrderedDict[MongoClientKey, MongoClient]" +_AsyncClientCache: TypeAlias = "OrderedDict[_AsyncClientCacheKey, _AsyncClientEntry]" + +_sync_clients: Final[_SyncClientCache] = OrderedDict() # mutable-ok: process-level client cache +_async_clients: Final[_AsyncClientCache] = OrderedDict() # mutable-ok: same cache, per loop + + +def _store_bounded(cache: "OrderedDict[_K, _V]", cache_key: "_K", value: "_V") -> None: + """Eviction only drops this cache's reference; an in-flight search keeps its client alive.""" + cache[cache_key] = value # mutable-ok: an LRU cache is mutable state by definition + cache.move_to_end(cache_key) + while len(cache) > _MAX_CACHED_CLIENTS: + cache.popitem(last=False) def import_sync_mongo_client() -> "type[MongoClient]": @@ -95,22 +107,19 @@ def _client_kwargs(key: MongoClientKey) -> Mapping[str, object]: def get_sync_client(key: MongoClientKey, client_class: SyncClientFactory | None = None) -> "MongoClient": - """``client_class`` is the injection seam the tests build fake clients through; left unset the - real pymongo class is imported at call time, keeping pymongo out of import-time dependencies.""" cached: Final = _sync_clients.get(key) if cached is not None: + _sync_clients.move_to_end(key) return cached build: Final = client_class if client_class is not None else import_sync_mongo_client() client: Final = build(key.connection_string, **_client_kwargs(key)) - if len(_sync_clients) < _MAX_CACHED_CLIENTS: - _sync_clients[key] = client + _store_bounded(_sync_clients, key, client) return client def _purge_dead_loops() -> None: - """The cached client holds its loop object alive, so a closed loop's entry would otherwise pin - that client and its sockets for the life of the process. Callers that run one loop per search - (``asyncio.run`` in a script) reach the cap this way and never release what is behind it.""" + """A cached client holds its loop alive, so a closed loop's entry would pin that client and its + sockets for the life of the process.""" for stale in tuple( cache_key for cache_key, (loop_ref, _) in _async_clients.items() @@ -125,12 +134,12 @@ def get_async_client(key: MongoClientKey, client_class: AsyncClientFactory | Non loop_key: Final = (key, id(loop)) cached: Final = _async_clients.get(loop_key) if cached is not None and cached[0]() is loop: + _async_clients.move_to_end(loop_key) return cached[1] _purge_dead_loops() build: Final = client_class if client_class is not None else import_async_mongo_client() client: Final = build(key.connection_string, **_client_kwargs(key)) - if len(_async_clients) < _MAX_CACHED_CLIENTS or loop_key in _async_clients: - _async_clients[loop_key] = (weakref.ref(loop), client) + _store_bounded(_async_clients, loop_key, (weakref.ref(loop), client)) return client @@ -157,9 +166,8 @@ def _index_hint(index_name: str, database: str, collection: str) -> str: def missing_index_error(index_name: str, database: str, collection: str) -> BadRequestError: - """$vectorSearch against a missing index, database or collection returns zero documents - instead of failing, so an empty result set is checked against the index catalogue and - turned into this rather than being reported as 'no matches'.""" + """$vectorSearch against a missing index, database or collection returns zero documents rather + than failing, so an empty result set is checked against the catalogue and reported as this.""" return config_error( f"{_index_hint(index_name, database, collection)} A vector search against a database, " "collection or index that does not exist returns no results rather than an error, so this " @@ -175,10 +183,7 @@ def index_not_ready_error(index_name: str, database: str, collection: str, statu def translate_mongo_error(error: Exception, index_name: str, database: str, collection: str) -> Exception: - """Turn a driver failure into a message that names the misconfiguration, never a silent empty result. - - Returns the exception to raise so callers keep the original as ``__cause__``. - """ + """Returns the exception to raise, so callers keep the driver error as ``__cause__``.""" try: from pymongo.errors import ( ConfigurationError, @@ -205,14 +210,16 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll f"The MongoDB vector search against '{database}.{collection}' timed out before returning. " f"Driver detail: {error}" ) - # ServerSelectionTimeoutError and NetworkTimeout both sit under ConnectionFailure, so this - # only sees what those two branches left: a dropped or refused connection + # ServerSelectionTimeoutError and NetworkTimeout also subclass ConnectionFailure, so this only + # sees what those branches left if isinstance(error, ConnectionFailure): - return config_error( - f"The connection to '{database}.{collection}' was refused or dropped. On Atlas this is " - "usually a connection string with no username and password, or a TLS failure, so confirm " - "the URI is the one Atlas shows under Connect, Drivers. On a self-managed deployment, check " - f"that mongod is listening on the host and port in the URI. Driver detail: {error}" + return unavailable_error( + f"The connection to '{database}.{collection}' was dropped or refused. That is usually a " + "replica set failover or a restarted node, so the search is worth retrying. If it keeps " + "happening: on Atlas the usual cause is a connection string with no username and password, " + "or a TLS failure, so confirm the URI is the one Atlas shows under Connect, Drivers; on a " + "self-managed deployment, check that mongod is listening on the host and port in the URI. " + f"Driver detail: {error}" ) if isinstance(error, OperationFailure): code: Final = error.code @@ -267,16 +274,14 @@ def translate_mongo_error(error: Exception, index_name: str, database: str, coll ) if isinstance(error, InvalidOperation): return config_error(f"The MongoDB client was already closed or is unusable. Driver detail: {error}") - # A tlsCAFile or tlsCertificateKeyFile the process cannot open raises OSError from the TLS setup - # rather than a PyMongoError, and those options are how self-managed deployments present a private CA + # An unreadable tlsCAFile or tlsCertificateKeyFile raises OSError, not a PyMongoError if isinstance(error, OSError) and error.filename: return config_error( f"'{error.filename}', named by a TLS option in mongodb_connection_string, could not be read. " "Check that tlsCAFile and tlsCertificateKeyFile point at files this process can open; inside " f"a container that is the path in the container, not on the host. Driver detail: {error}" ) - # pymongo raises a plain ValueError, not a PyMongoError, for an unusable port, which an unescaped - # ':' in a password also produces, and which would otherwise reach the caller as a 500 + # pymongo raises a plain ValueError, not a PyMongoError, for an unusable port if isinstance(error, ValueError): return config_error( "The host and port in mongodb_connection_string could not be parsed. If the port is a " diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index 2e69e35edcf..3382c931c96 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -1,15 +1,5 @@ -"""MongoDB vector store provider, for Atlas and self-managed deployments alike. - -MongoDB Vector Search has no HTTP query API (the Data API and HTTPS Endpoints are -end-of-life), so this config extends BaseDirectVectorStoreConfig and runs the -``$vectorSearch`` aggregation itself through pymongo instead of shaping an httpx -request. mongod serves that stage identically whether mongot runs under Atlas or -beside a self-managed deployment, so one code path covers both. - -``vector_store_id`` is the search index name, matching the Valkey provider -where the id names the index; the database and collection it covers come from -litellm_params. -""" +"""MongoDB Vector Search has no HTTP query API, so this is a direct provider that runs the +``$vectorSearch`` aggregation through pymongo. ``vector_store_id`` is the search index name.""" from collections.abc import Callable, Mapping, Sequence from types import MappingProxyType @@ -159,9 +149,8 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): @staticmethod def _reject_unknown_params(litellm_params: Mapping[str, object]) -> None: - """The params model ignores unrelated keys because litellm_params carries plenty of them, - which would otherwise turn a mistyped mongodb_collection into 'mongodb_collection is - required' pointing at a key the reader can see they have set.""" + """Without this a mistyped mongodb_collection reads as 'mongodb_collection is required', + naming a key the reader can see they have set.""" unknown: Final = sorted( key for key in litellm_params if key.startswith(_MONGODB_PARAM_PREFIX) and key not in _KNOWN_MONGODB_PARAMS ) @@ -268,8 +257,7 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): @classmethod def _field_value(cls, document: Mapping[str, object], dotted_path: str) -> str | None: - """None means the path is absent from the document, which is what separates a - mistyped mongodb_text_field from a document whose text is genuinely empty.""" + """None means absent, which is what separates a mistyped field from genuinely empty text.""" head, _, rest = dotted_path.partition(".") if head not in document: return None @@ -297,9 +285,8 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): def _raise_for_missing_text_field( cls, documents: Sequence[Mapping[str, object]], text_field: str, database: str, collection: str ) -> None: - """$vectorSearch happily matches documents that carry no text at all, so a mistyped - mongodb_text_field returns well-scored results whose content is empty and feeds an empty - context to the model. Every matched document lacking the field is the misconfiguration.""" + """$vectorSearch matches documents carrying no text, so a mistyped mongodb_text_field + returns well-scored results with empty content instead of failing.""" if documents and all(cls._field_value(document, text_field) is None for document in documents): raise config_error( f"None of the {len(documents)} matched documents in '{database}.{collection}' has a " @@ -323,9 +310,8 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig): def _raise_for_unusable_index( catalogue: Sequence[Mapping[str, object]], index_name: str, database: str, collection: str ) -> None: - """An empty result set is ambiguous: mongod returns zero documents both for a query that - genuinely matched nothing and for a missing database, collection or index. Only the second - is a misconfiguration, so the index catalogue decides which one happened.""" + """mongod returns zero documents both for a query that matched nothing and for a missing + database, collection or index, so the catalogue decides which one happened.""" if not catalogue: raise missing_index_error(index_name, database, collection) entry: Final = catalogue[0] diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index 7a2df28cc04..faf20f87ae5 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -8,10 +8,12 @@ from unittest.mock import MagicMock, patch import httpx import pytest -from litellm.exceptions import BadRequestError, Timeout +import litellm +from litellm.exceptions import BadRequestError, ServiceUnavailableError, Timeout from litellm.llms.mongodb.common_utils import ( _MAX_CACHED_CLIENTS, _async_clients, + _sync_clients, MongoClientKey, index_not_ready_error, missing_index_error, @@ -643,6 +645,37 @@ class TestClientCache: assert first.connection_string == CONNECTION_STRING + def _fill_cache(self): + for slot in range(_MAX_CACHED_CLIENTS): + get_sync_client(self._key(f"mongodb://cold-{slot}:27017"), RecordingClient) + + def test_a_store_added_after_the_cache_filled_is_still_cached(self): + """Rebuilding a client costs an SRV lookup, a TLS handshake and topology discovery, so a + store that misses the cache on every single search pays that on every search.""" + self._fill_cache() + latecomer = self._key("mongodb://latecomer:27017") + + first = get_sync_client(latecomer, RecordingClient) + + assert get_sync_client(latecomer, RecordingClient) is first + + def test_the_cache_evicts_the_least_recently_used_client(self): + self._fill_cache() + oldest = self._key("mongodb://cold-0:27017") + newest = self._key(f"mongodb://cold-{_MAX_CACHED_CLIENTS - 1}:27017") + kept = get_sync_client(newest, RecordingClient) + + get_sync_client(self._key("mongodb://latecomer:27017"), RecordingClient) + + assert get_sync_client(newest, RecordingClient) is kept + assert oldest not in _sync_clients + + def test_the_cache_never_grows_past_its_cap(self): + for slot in range(_MAX_CACHED_CLIENTS * 3): + get_sync_client(self._key(f"mongodb://host-{slot}:27017"), RecordingClient) + + assert len(_sync_clients) == _MAX_CACHED_CLIENTS + def test_a_new_loop_never_inherits_a_closed_loop_client(self): """CPython recycles id() so aggressively that a fresh event loop almost always lands on the id of one already collected: measured at 37 of 40 rounds. Keying the cache on the id @@ -757,17 +790,51 @@ class TestErrorTranslation: assert "rejected the credentials" in str(translated) - def test_a_dropped_connection_is_a_400_not_an_unhandled_driver_error(self): - """AutoReconnect sits under ConnectionFailure alongside the two timeout classes, and Atlas - answers a URI with no credentials by closing the connection rather than failing auth. Left - untranslated it is not a litellm exception type, so it reaches the caller as a 500.""" + def test_a_dropped_connection_stays_retryable(self): + """A replica set failover reaches the driver as AutoReconnect. litellm only retries 408, + 409, 429 and 5xx, so classifying it as a client error would turn one failover into a + permanently failed search.""" + from pymongo.errors import AutoReconnect + + translated = self._translate(AutoReconnect("connection closed")) + + assert litellm._should_retry(translated.status_code) + assert "dropped or refused" in str(translated) + + def test_a_dropped_connection_still_names_the_misconfigurations_behind_it(self): + """Atlas answers a URI with no credentials by closing the connection rather than failing + auth, so the retryable message still has to name that.""" from pymongo.errors import AutoReconnect translated = self._translate(AutoReconnect("connection closed")) - assert isinstance(translated, BadRequestError) - assert "refused or dropped" in str(translated) assert "no username and password" in str(translated) + assert "mongod is listening" in str(translated) + + def test_the_retryable_classification_survives_the_public_sdk_error_wrapper(self): + """litellm.exception_type only passes its own exception types through; anything else becomes + an APIConnectionError and a 500, which would drop the retryable classification.""" + from pymongo.errors import AutoReconnect + + translated = self._translate(AutoReconnect("connection closed")) + + wrapped = litellm.exception_type( + model=None, + original_exception=translated, + custom_llm_provider="mongodb", + completion_kwargs={}, + extra_kwargs={}, + ) + + assert isinstance(wrapped, ServiceUnavailableError) + assert litellm._should_retry(wrapped.status_code) + + def test_a_pool_wait_queue_timeout_stays_retryable(self): + from pymongo.errors import WaitQueueTimeoutError + + translated = self._translate(WaitQueueTimeoutError("timed out waiting for a connection")) + + assert litellm._should_retry(translated.status_code) def test_server_selection_timeout_still_wins_over_the_connection_branch(self): from pymongo.errors import ServerSelectionTimeoutError @@ -775,7 +842,7 @@ class TestErrorTranslation: translated = self._translate(ServerSelectionTimeoutError("no servers")) assert isinstance(translated, Timeout) - assert "refused or dropped" not in str(translated) + assert "dropped or refused" not in str(translated) def test_network_timeout_still_wins_over_the_connection_branch(self): from pymongo.errors import NetworkTimeout @@ -783,7 +850,7 @@ class TestErrorTranslation: translated = self._translate(NetworkTimeout("socket timed out")) assert isinstance(translated, Timeout) - assert "refused or dropped" not in str(translated) + assert "dropped or refused" not in str(translated) def test_an_unescaped_password_character_is_a_400_not_a_500(self): """pymongo's URI parser raises a plain ValueError, not a PyMongoError, for an unusable port, @@ -1239,7 +1306,7 @@ class TestSelfManagedDeploymentsAreFirstClass: config = self._config_that_fails_to_connect(ConnectionFailure("connection closed")) - with pytest.raises(BadRequestError) as excinfo: + with pytest.raises(ServiceUnavailableError) as excinfo: _search(config) assert "self-managed" in str(excinfo.value) From 2a11c2747f58f24a1c9f1babc30027afa9ec2a8a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 4 Sep 2026 15:25:38 -0700 Subject: [PATCH 41/60] fix(vector_stores): serialize the MongoDB client cache so concurrent searches cannot trip over an eviction Async searches reach the sync client through executor threads, so the LRU cache is shared state. A key could be evicted between the lookup and the reordering that followed it, and the reordering then raised KeyError and became a 500. Reproduced at 15 failures per run with 16 threads over 34 keys and a 1ns switch interval; the regression test is that workload. --- litellm/llms/mongodb/common_utils.py | 40 ++++++++++++------- .../test_mongodb_transformation.py | 27 +++++++++++++ 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py index 27a2a96bd1f..02c0b359407 100644 --- a/litellm/llms/mongodb/common_utils.py +++ b/litellm/llms/mongodb/common_utils.py @@ -2,6 +2,7 @@ so every import of it is deferred to call time.""" import asyncio +import threading import weakref from asyncio import AbstractEventLoop from collections import OrderedDict @@ -69,14 +70,23 @@ _AsyncClientCache: TypeAlias = "OrderedDict[_AsyncClientCacheKey, _AsyncClientEn _sync_clients: Final[_SyncClientCache] = OrderedDict() # mutable-ok: process-level client cache _async_clients: Final[_AsyncClientCache] = OrderedDict() # mutable-ok: same cache, per loop +# async searches reach the sync client through executor threads, so both caches are shared state +_cache_lock: Final = threading.Lock() def _store_bounded(cache: "OrderedDict[_K, _V]", cache_key: "_K", value: "_V") -> None: """Eviction only drops this cache's reference; an in-flight search keeps its client alive.""" - cache[cache_key] = value # mutable-ok: an LRU cache is mutable state by definition - cache.move_to_end(cache_key) - while len(cache) > _MAX_CACHED_CLIENTS: - cache.popitem(last=False) + with _cache_lock: + cache[cache_key] = value # mutable-ok: an LRU cache is mutable state by definition + cache.move_to_end(cache_key) + while len(cache) > _MAX_CACHED_CLIENTS: + cache.popitem(last=False) + + +def _mark_used(cache: "OrderedDict[_K, _V]", cache_key: "_K") -> None: + with _cache_lock: + if cache_key in cache: + cache.move_to_end(cache_key) def import_sync_mongo_client() -> "type[MongoClient]": @@ -109,7 +119,7 @@ def _client_kwargs(key: MongoClientKey) -> Mapping[str, object]: def get_sync_client(key: MongoClientKey, client_class: SyncClientFactory | None = None) -> "MongoClient": cached: Final = _sync_clients.get(key) if cached is not None: - _sync_clients.move_to_end(key) + _mark_used(_sync_clients, key) return cached build: Final = client_class if client_class is not None else import_sync_mongo_client() client: Final = build(key.connection_string, **_client_kwargs(key)) @@ -120,12 +130,13 @@ def get_sync_client(key: MongoClientKey, client_class: SyncClientFactory | None def _purge_dead_loops() -> None: """A cached client holds its loop alive, so a closed loop's entry would pin that client and its sockets for the life of the process.""" - for stale in tuple( - cache_key - for cache_key, (loop_ref, _) in _async_clients.items() - if (cached_loop := loop_ref()) is None or cached_loop.is_closed() - ): - del _async_clients[stale] + with _cache_lock: + for stale in tuple( + cache_key + for cache_key, (loop_ref, _) in _async_clients.items() + if (cached_loop := loop_ref()) is None or cached_loop.is_closed() + ): + del _async_clients[stale] def get_async_client(key: MongoClientKey, client_class: AsyncClientFactory | None = None) -> "AsyncMongoClient": @@ -134,7 +145,7 @@ def get_async_client(key: MongoClientKey, client_class: AsyncClientFactory | Non loop_key: Final = (key, id(loop)) cached: Final = _async_clients.get(loop_key) if cached is not None and cached[0]() is loop: - _async_clients.move_to_end(loop_key) + _mark_used(_async_clients, loop_key) return cached[1] _purge_dead_loops() build: Final = client_class if client_class is not None else import_async_mongo_client() @@ -144,8 +155,9 @@ def get_async_client(key: MongoClientKey, client_class: AsyncClientFactory | Non def reset_client_cache() -> None: - _sync_clients.clear() - _async_clients.clear() + with _cache_lock: + _sync_clients.clear() + _async_clients.clear() _AUTHENTICATION_FAILED_CODE: Final = 18 diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py index faf20f87ae5..f5d31c0da54 100644 --- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -1,6 +1,7 @@ import asyncio import gc import sys +import threading import weakref from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -670,6 +671,32 @@ class TestClientCache: assert get_sync_client(newest, RecordingClient) is kept assert oldest not in _sync_clients + def test_concurrent_searches_never_trip_over_an_eviction(self): + """Async searches run the sync client through executor threads, so a key can be evicted + between the lookup and the reordering that follows it.""" + errors = [] + churn = _MAX_CACHED_CLIENTS + 2 + + def hammer(offset): + try: + for step in range(3_000): + get_sync_client(self._key(f"mongodb://h-{(step + offset) % churn}:27017"), RecordingClient) + except Exception as e: + errors.append(repr(e)) + + previous = sys.getswitchinterval() + sys.setswitchinterval(1e-9) + try: + threads = [threading.Thread(target=hammer, args=(offset,)) for offset in range(16)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + finally: + sys.setswitchinterval(previous) + + assert errors == [] + def test_the_cache_never_grows_past_its_cap(self): for slot in range(_MAX_CACHED_CLIENTS * 3): get_sync_client(self._key(f"mongodb://host-{slot}:27017"), RecordingClient) From 01b55daee971d3ea3865d223ea28daf70c0bb3ec Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 4 Sep 2026 15:31:54 -0700 Subject: [PATCH 42/60] feat(ui): add stalled task escalation controls to the auto-router form Adds an "Advanced: Stalled Task Escalation" section to the complexity router config: a toggle plus the repeat threshold and the window of recent tool calls to examine. Both knobs are seeded on enable and cleared on disable, so an off router sends none of the three keys, which is what the backend requires next to session pinning and a custom tier set. The toggle locks out with an explanation when "How often to classify" is set to once-per-session or new-user-message, since both replay a held routing decision instead of classifying and a stall would never reach the classifier. The keys join the custom-tier restriction registry, which both strips them from a custom-tier save and marks the section restricted. ResponseFormatControls moves into its own file to keep ComplexityRouterConfig.tsx under the 800-line lint ceiling, matching the one-file-per-control layout its siblings already use. --- .../add_model/ComplexityRouterConfig.tsx | 38 +++--- .../add_model/ResponseFormatControls.tsx | 24 ++++ .../add_model/StallEscalationConfig.test.tsx | 108 ++++++++++++++++ .../add_model/StallEscalationConfig.tsx | 116 ++++++++++++++++++ .../add_model/add_auto_router_tab.tsx | 3 + .../build_complexity_router_config.test.ts | 31 +++++ .../build_complexity_router_config.ts | 18 +++ .../src/components/add_model/tier_rows.ts | 4 + ...d_updated_complexity_router_config.test.ts | 38 ++++++ .../edit_auto_router_modal.tsx | 16 +++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 18 +++ 11 files changed, 395 insertions(+), 19 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.test.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 06363830d64..a6d5932dea0 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -33,6 +33,8 @@ import { ModelGroup } from "@/components/llm_calls/fetch_models"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig"; +import ResponseFormatControls from "./ResponseFormatControls"; +import StallEscalationConfig from "./StallEscalationConfig"; import { Restricted, restrictedBy } from "./TierRestrictions"; import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions"; import { @@ -418,6 +420,14 @@ export interface ComplexityRouterConfigValue { deployment_affinity?: boolean; /** Plan-mode floor as a tier ROW ID, unset meaning off. The wire carries the row's name. */ plan_mode_min_tier?: string; + /** + * Mid-task stall escalation. Undefined means off, which keeps all three keys out of the payload: + * the backend rejects them alongside session pinning, user-turn classification and a custom tier + * set, so an off router must stay silent about them rather than send an explicit false. + */ + stall_escalation_enabled?: boolean; + stall_escalation_window?: number; + stall_escalation_repeat_threshold?: number; adaptive?: boolean; adaptive_weights?: AdaptiveRouterWeights; tier_distance_penalty?: number; @@ -571,25 +581,6 @@ const PlanModeOverrideControls: React.FC<{ ); -const ResponseFormatControls: React.FC<{ - value: ComplexityRouterConfigValue; - onChange: (value: ComplexityRouterConfigValue) => void; -}> = ({ value, onChange }) => ( - <> -
- onChange({ ...value, return_raw_model_name: returnRawModelName })} - aria-label="Return raw model name" - /> - Return raw model name -
- - Return the resolved underlying model name in responses instead of the autorouter alias. - - -); - const ComplexityRouterConfig: React.FC = ({ modelInfo, value, @@ -855,6 +846,15 @@ const ComplexityRouterConfig: React.FC = ({ label: Advanced: Context Window Escalation, children: , }, + { + key: "stall-escalation", + label: Advanced: Stalled Task Escalation, + children: ( + + + + ), + }, { key: "response", label: Advanced: Response Format, diff --git a/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx new file mode 100644 index 00000000000..68dd880a684 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx @@ -0,0 +1,24 @@ +import { Switch } from "@/components/ui/switch"; +import React from "react"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const ResponseFormatControls: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => ( + <> +
+ onChange({ ...value, return_raw_model_name: returnRawModelName })} + aria-label="Return raw model name" + /> + Return raw model name +
+ + Return the resolved underlying model name in responses instead of the autorouter alias. + + +); + +export default ResponseFormatControls; diff --git a/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.test.tsx new file mode 100644 index 00000000000..2c346306307 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.test.tsx @@ -0,0 +1,108 @@ +import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; +import { vi } from "vitest"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import StallEscalationConfig, { stallEscalationBlockedReason } from "./StallEscalationConfig"; + +const tiers = { SIMPLE: "gpt-4o-mini", MEDIUM: "gpt-4o", COMPLEX: "claude-sonnet-4", REASONING: "o1-preview" }; + +const baseValue: ComplexityRouterConfigValue = { + tiers, + classifier_type: "heuristic", +}; + +const renderConfig = (value: Partial = {}) => { + const onChange = vi.fn(); + renderWithProviders(); + return onChange; +}; + +const toggle = () => screen.getByRole("switch", { name: "Escalate a stalled task to a stronger model" }); + +describe("stallEscalationBlockedReason", () => { + it("blocks on session pinning, which replays a model instead of classifying", () => { + expect(stallEscalationBlockedReason({ ...baseValue, session_affinity: true })).toContain("Classification Method"); + }); + + it("blocks on user-turn classification, which skips the agent-loop turns a stall shows up in", () => { + expect(stallEscalationBlockedReason({ ...baseValue, classification_mode: "user_turn" })).toContain("every request"); + }); + + it("allows the default every-request router", () => { + expect(stallEscalationBlockedReason(baseValue)).toBeNull(); + }); +}); + +describe("StallEscalationConfig", () => { + it("hides the knobs until the feature is turned on", () => { + renderConfig(); + expect(toggle()).not.toBeChecked(); + expect(screen.queryByLabelText("Repeats before escalating")).not.toBeInTheDocument(); + }); + + it("turning it on seeds both knobs so the saved config is explicit rather than half-set", () => { + const onChange = renderConfig(); + fireEvent.click(toggle()); + expect(onChange).toHaveBeenCalledWith( + expect.objectContaining({ + stall_escalation_enabled: true, + stall_escalation_window: 6, + stall_escalation_repeat_threshold: 3, + }), + ); + }); + + it("turning it off clears all three keys, since the backend rejects them next to session pinning", () => { + const onChange = renderConfig({ + stall_escalation_enabled: true, + stall_escalation_window: 6, + stall_escalation_repeat_threshold: 3, + }); + fireEvent.click(toggle()); + expect(onChange).toHaveBeenCalledWith( + expect.objectContaining({ + stall_escalation_enabled: undefined, + stall_escalation_window: undefined, + stall_escalation_repeat_threshold: undefined, + }), + ); + }); + + it("raises the window to match a larger threshold, which could otherwise never be reached", () => { + const onChange = renderConfig({ + stall_escalation_enabled: true, + stall_escalation_window: 4, + stall_escalation_repeat_threshold: 3, + }); + fireEvent.change(screen.getByLabelText("Repeats before escalating"), { target: { value: "9" } }); + expect(onChange).toHaveBeenCalledWith( + expect.objectContaining({ stall_escalation_repeat_threshold: 9, stall_escalation_window: 9 }), + ); + }); + + it("holds the window at the threshold when someone types a smaller one", () => { + const onChange = renderConfig({ + stall_escalation_enabled: true, + stall_escalation_window: 6, + stall_escalation_repeat_threshold: 3, + }); + fireEvent.change(screen.getByLabelText("Recent calls examined"), { target: { value: "1" } }); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ stall_escalation_window: 3 })); + }); + + it("floors the threshold at 2, below which a single ordinary retry would escalate", () => { + const onChange = renderConfig({ stall_escalation_enabled: true }); + fireEvent.change(screen.getByLabelText("Repeats before escalating"), { target: { value: "1" } }); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ stall_escalation_repeat_threshold: 2 })); + }); + + it("disables the toggle and says why when session pinning is on", () => { + renderConfig({ session_affinity: true }); + expect(toggle()).toHaveAttribute("aria-disabled", "true"); + expect(screen.getByText(/How often to classify/)).toBeInTheDocument(); + }); + + it("hides the knobs when a blocker is switched on under an already-enabled router", () => { + renderConfig({ stall_escalation_enabled: true, session_affinity: true }); + expect(screen.queryByLabelText("Repeats before escalating")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.tsx b/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.tsx new file mode 100644 index 00000000000..fdb8c30f3b8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.tsx @@ -0,0 +1,116 @@ +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import React from "react"; +import { type ComplexityRouterConfigValue, classificationFrequency } from "./ComplexityRouterConfig"; + +export const DEFAULT_STALL_ESCALATION_WINDOW = 6; +export const DEFAULT_STALL_ESCALATION_REPEAT_THRESHOLD = 3; + +/** + * Why the toggle is unavailable, or null when it can be turned on. Both blockers replay a held + * routing decision instead of classifying most turns, so detection would never see the tool + * calls it reads. + */ +export const stallEscalationBlockedReason = (value: ComplexityRouterConfigValue): string | null => { + const frequency = classificationFrequency(value); + if (frequency === "session") + return 'Set "How often to classify" to every request under Advanced: Classification Method to use this. Scoring once per session replays that model instead of classifying, so a stall never reaches the classifier.'; + if (frequency === "user_turn") + return 'Set "How often to classify" to every request under Advanced: Classification Method to use this. Scoring only new user messages skips the tool-call turns a stall shows up in.'; + return null; +}; + +const clampedInt = (raw: string, min: number, fallback: number): number => { + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return fallback; + return Math.max(min, Math.trunc(parsed)); +}; + +const StallEscalationConfig: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => { + const enabled = value.stall_escalation_enabled ?? false; + const blockedReason = stallEscalationBlockedReason(value); + const window = value.stall_escalation_window ?? DEFAULT_STALL_ESCALATION_WINDOW; + const threshold = value.stall_escalation_repeat_threshold ?? DEFAULT_STALL_ESCALATION_REPEAT_THRESHOLD; + // A threshold above the window can never be reached, and the backend rejects the pair, so the + // window rises with the threshold rather than letting the form save something inert. + const commitThreshold = (raw: string) => { + const nextThreshold = clampedInt(raw, 2, DEFAULT_STALL_ESCALATION_REPEAT_THRESHOLD); + onChange({ + ...value, + stall_escalation_repeat_threshold: nextThreshold, + stall_escalation_window: Math.max(window, nextThreshold), + }); + }; + const commitWindow = (raw: string) => { + const nextWindow = clampedInt(raw, 1, DEFAULT_STALL_ESCALATION_WINDOW); + onChange({ + ...value, + stall_escalation_window: Math.max(nextWindow, threshold), + }); + }; + const toggle = (next: boolean) => { + const enabledValue: ComplexityRouterConfigValue = { + ...value, + stall_escalation_enabled: next || undefined, + stall_escalation_window: next ? window : undefined, + stall_escalation_repeat_threshold: next ? threshold : undefined, + }; + onChange(enabledValue); + }; + return ( + <> +
+ + Escalate a stalled task to a stronger model +
+ + When the model keeps repeating the same tool call, or the same call keeps erroring, bump the request one tier + higher for as long as it looks stuck. The automatic counterpart to an escalation keyword: nobody has to notice + the loop and ask. Off means a stuck task keeps the model it was classified onto. + {blockedReason !== null && ` ${blockedReason}`} + + {enabled && blockedReason === null && ( +
+
+ + commitThreshold(event.target.value)} + /> + + How many identical or failing calls count as stuck. At least 2; lower reacts sooner and misfires more. + +
+
+ + commitWindow(event.target.value)} + /> + + How far back to look, in tool calls. Never below the repeat count, since that could never be reached. + +
+
+ )} + + ); +}; + +export default StallEscalationConfig; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index a548c2c6533..ac6e18614bb 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -394,6 +394,9 @@ const AddAutoRouterTab: React.FC = ({ embeddingModel, matchThreshold, escalationKeywords, + stallEscalationEnabled: complexityRouterConfig.stall_escalation_enabled, + stallEscalationWindow: complexityRouterConfig.stall_escalation_window, + stallEscalationRepeatThreshold: complexityRouterConfig.stall_escalation_repeat_threshold, adaptive: complexityRouterConfig.adaptive ?? false, adaptiveWeights: complexityRouterConfig.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS, tierDistancePenalty: complexityRouterConfig.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 9ee555f5dd2..b4f0affd3df 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -1020,6 +1020,9 @@ describe("buildComplexityRouterConfig with an edited tier set", () => { heuristicFirstMaxTier: "SIMPLE", hybridBoundaryMargin: 0.03, customTechnicalKeywords: ["kubernetes"], + stallEscalationEnabled: true, + stallEscalationWindow: 6, + stallEscalationRepeatThreshold: 3, }; const emittingType = key === "heuristic_first_max_tier" ? "heuristic_first" : "llm"; const typeForKey = key === "hybrid_boundary_margin" ? "hybrid" : emittingType; @@ -1114,6 +1117,34 @@ describe("hydrateCustomTierSet", () => { }); }); +describe("buildComplexityRouterConfig stall escalation", () => { + it("omits all three keys when the toggle is off, since the backend rejects them next to session pinning", () => { + const config = buildComplexityRouterConfig({ ...baseParams, stallEscalationEnabled: false }); + expect(config).not.toHaveProperty("stall_escalation_enabled"); + expect(config).not.toHaveProperty("stall_escalation_window"); + expect(config).not.toHaveProperty("stall_escalation_repeat_threshold"); + }); + + it("emits the toggle and both knobs when it is on", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + stallEscalationEnabled: true, + stallEscalationWindow: 8, + stallEscalationRepeatThreshold: 4, + }); + expect(config.stall_escalation_enabled).toBe(true); + expect(config.stall_escalation_window).toBe(8); + expect(config.stall_escalation_repeat_threshold).toBe(4); + }); + + it("emits the toggle alone when neither knob was touched, so both track the backend defaults", () => { + const config = buildComplexityRouterConfig({ ...baseParams, stallEscalationEnabled: true }); + expect(config.stall_escalation_enabled).toBe(true); + expect(config).not.toHaveProperty("stall_escalation_window"); + expect(config).not.toHaveProperty("stall_escalation_repeat_threshold"); + }); +}); + describe("dryRunRejection", () => { it("blocks the save on a rejection whose message is missing, which the write would return as a raw 400", () => { expect(dryRunRejection({ valid: false })).toBe("The proxy rejected this auto-router configuration"); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 956e593a234..3d9cf1c747e 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -126,6 +126,9 @@ export interface BuildComplexityRouterConfigParams { embeddingModel: string | undefined; matchThreshold: number; escalationKeywords: string[]; + stallEscalationEnabled?: boolean; + stallEscalationWindow?: number; + stallEscalationRepeatThreshold?: number; adaptive: boolean; adaptiveWeights: AdaptiveRouterWeights; tierDistancePenalty: number; @@ -186,6 +189,9 @@ export interface ComplexityRouterConfigPayload { embedding_model?: string; match_threshold?: number; escalation_keywords?: string[]; + stall_escalation_enabled?: boolean; + stall_escalation_window?: number; + stall_escalation_repeat_threshold?: number; adaptive?: boolean; adaptive_weights?: AdaptiveRouterWeights; tier_distance_penalty?: number; @@ -446,6 +452,9 @@ export const buildComplexityRouterConfig = ({ embeddingModel, matchThreshold, escalationKeywords, + stallEscalationEnabled, + stallEscalationWindow, + stallEscalationRepeatThreshold, adaptive, adaptiveWeights, tierDistancePenalty, @@ -507,6 +516,15 @@ export const buildComplexityRouterConfig = ({ ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), ...(cleanedKeywordTierRules.length > 0 && { keyword_tier_rules: cleanedKeywordTierRules }), escalation_keywords: cleanedEscalationKeywords, + // Only written when on: the backend rejects it alongside session_affinity, user_turn mode and + // a custom tier set, so an off router must not carry the key into any of those saves. + ...(stallEscalationEnabled && { + stall_escalation_enabled: true, + ...(stallEscalationWindow !== undefined && { stall_escalation_window: stallEscalationWindow }), + ...(stallEscalationRepeatThreshold !== undefined && { + stall_escalation_repeat_threshold: stallEscalationRepeatThreshold, + }), + }), ...(semanticMatchingEnabled && { semantic_keyword_matching: true, embedding_model: embeddingModel, diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts index 3c5b149f4da..5e2a32addee 100644 --- a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts @@ -113,6 +113,10 @@ export const CUSTOM_TIER_RESTRICTIONS = { omit: ["escalation_keywords"], reason: "Escalation bumps a request along the built-in tier ladder, which your tier set replaces", }, + stallEscalation: { + omit: ["stall_escalation_enabled", "stall_escalation_window", "stall_escalation_repeat_threshold"], + reason: "Stall escalation bumps a request along the built-in tier ladder, which your tier set replaces", + }, adaptive: { omit: ["adaptive", "adaptive_weights", "tier_distance_penalty", "adaptive_eligible"], reason: "Adaptive routing scores models along the built-in tier ladder, which your tier set replaces", diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 877b35199a4..44152b08ef9 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -593,16 +593,54 @@ describe("managed keys survive an untouched open-and-save", () => { "hybrid_boundary_margin", ]); + // The stall keys are rejected beside the session pinning and user-turn classification this + // fixture sets, so they get their own round trip below rather than widening this one. + const KEYS_ANOTHER_CLASSIFICATION_FREQUENCY_OWNS = new Set([ + "stall_escalation_enabled", + "stall_escalation_window", + "stall_escalation_repeat_threshold", + ]); + it("carries every managed key a built-in router can hold through hydrate then save", () => { const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined); const saved = buildUpdatedComplexityRouterConfig(STORED_ALL_MANAGED, hydrated); const dropped = [...MANAGED_COMPLEXITY_ROUTER_KEYS] .filter((key) => !KEYS_ANOTHER_CLASSIFIER_TYPE_OWNS.has(key)) + .filter((key) => !KEYS_ANOTHER_CLASSIFICATION_FREQUENCY_OWNS.has(key)) .filter((key) => saved[key] === undefined); expect(dropped).toEqual([]); }); + it("carries the stall-escalation keys through their own round trip", () => { + const stored: Record = { + ...STORED_ALL_MANAGED, + session_affinity: false, + classification_mode: "every_request", + stall_escalation_enabled: true, + stall_escalation_window: 8, + stall_escalation_repeat_threshold: 4, + }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + const saved = buildUpdatedComplexityRouterConfig(stored, hydrated); + + expect(saved.stall_escalation_enabled).toBe(true); + expect(saved.stall_escalation_window).toBe(8); + expect(saved.stall_escalation_repeat_threshold).toBe(4); + }); + + it("leaves the stall keys out of a saved config that never had them on", () => { + const stored: Record = { + ...STORED_ALL_MANAGED, + session_affinity: false, + classification_mode: "every_request", + }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + const saved = buildUpdatedComplexityRouterConfig(stored, hydrated); + + expect(saved).not.toHaveProperty("stall_escalation_enabled"); + }); + it("drops a stored local-scorer threshold when the operator converts the router to custom tiers", () => { const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined); const converted = { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index ea1e5cba6a3..1b65c1c35a4 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -115,6 +115,9 @@ export interface StoredComplexityRouterConfig { return_raw_model_name?: boolean; enable_context_window_escalation?: unknown; context_window_escalation_buffer?: unknown; + stall_escalation_enabled?: unknown; + stall_escalation_window?: unknown; + stall_escalation_repeat_threshold?: unknown; } /** @@ -208,6 +211,13 @@ export const hydrateComplexityRouterConfig = ( typeof parsedConfig.context_window_escalation_buffer === "number" ? parsedConfig.context_window_escalation_buffer : undefined, + stall_escalation_enabled: parsedConfig.stall_escalation_enabled === true || undefined, + stall_escalation_window: + typeof parsedConfig.stall_escalation_window === "number" ? parsedConfig.stall_escalation_window : undefined, + stall_escalation_repeat_threshold: + typeof parsedConfig.stall_escalation_repeat_threshold === "number" + ? parsedConfig.stall_escalation_repeat_threshold + : undefined, }; }; @@ -245,6 +255,9 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "reasoning_override_min_score", "enable_context_window_escalation", "context_window_escalation_buffer", + "stall_escalation_enabled", + "stall_escalation_window", + "stall_escalation_repeat_threshold", ]); // Managed only when the caller passes the corresponding state. A caller that does not render @@ -351,6 +364,9 @@ export const buildUpdatedComplexityRouterConfig = ( tierModelParams: value.tier_model_params, enableContextWindowEscalation: value.enable_context_window_escalation, contextWindowEscalationBuffer: value.context_window_escalation_buffer, + stallEscalationEnabled: value.stall_escalation_enabled, + stallEscalationWindow: value.stall_escalation_window, + stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold, }; const built = buildComplexityRouterConfig(builderParams); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f4cb88bbae1..09dfbd841c8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34896,6 +34896,24 @@ export interface components { * @description Keywords indicating simple/basic queries */ simple_keywords?: string[] | null; + /** + * Stall Escalation Enabled + * @description Escalate mid-task to the next-higher configured tier when the assistant's own recent tool calls look stuck: stall_escalation_repeat_threshold or more of the last stall_escalation_window tool calls are identical repeats (same tool, same arguments) or came back as errors. One tier at most, on the same ladder escalation_keywords bumps along, and never above the highest configured tier. Detection re-runs on every classified turn from the tool calls visible in that request, so it needs no state and nothing survives past the task: once the recent tool calls stop looking stuck, the next classified turn routes normally again. Mutually exclusive with session_affinity and classification_mode='user_turn', which both replay a held routing decision instead of classifying most turns, so this would never see the tool calls to look at. Off by default. + * @default false + */ + stall_escalation_enabled: boolean; + /** + * Stall Escalation Repeat Threshold + * @description How many of the last stall_escalation_window tool calls must be identical repeats, or error results, before the task counts as stalled. Must not exceed stall_escalation_window, or the condition could never be reached. + * @default 3 + */ + stall_escalation_repeat_threshold: number; + /** + * Stall Escalation Window + * @description How many of the assistant's most recent tool calls stall detection looks at, oldest ones dropped as new calls happen. Counted across the whole visible conversation rather than reset at the newest human ask, so evidence from before a plain follow-up message like 'try again' is still visible on the turn after it. + * @default 6 + */ + stall_escalation_window: number; /** * Technical Keywords * @description Keywords indicating technical content From 939039f4927b219b92efbacc83b94a5a51839cd6 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 4 Sep 2026 16:03:20 -0700 Subject: [PATCH 43/60] fix(router): anchor stall detection on the newest tool call Counting whichever pattern was most common across the window escalated a task that had already recovered: three identical failures stay in the window for a few turns after the model breaks out of them, and on their own they met the threshold. Both tests now anchor on the newest call. The repeat test counts calls matching the newest one, and the error test only runs while the newest call is itself an error, so a window whose recent calls are healthy no longer escalates. The matches still do not have to be adjacent, so a retry loop broken up by an unrelated lookup keeps counting. Found by Greptile on #39809. --- .../complexity_router/README.md | 16 +++-- .../complexity_router/config.py | 27 ++++----- .../complexity_router/stall_detector.py | 58 ++++++++----------- .../router_strategy/test_stall_detector.py | 33 +++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 5 files changed, 85 insertions(+), 53 deletions(-) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index a7ed9e9dc21..ad84d4499d2 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -270,14 +270,20 @@ model_list: REASONING: o1-preview ``` -Detection looks at the assistant's own tool calls, not the human's messages: of the last -`stall_escalation_window` tool calls, if `stall_escalation_repeat_threshold` or more are -identical (same tool, same arguments) or came back as errors, the task counts as stalled and the -classified tier is bumped one step by the same `_escalate_tier` ladder `escalation_keywords` -uses, capped at the highest configured tier. It reads both tool-call shapes: Anthropic Messages +Detection looks at the assistant's own tool calls, not the human's messages. The task counts as +stalled when the NEWEST tool call is still part of a stuck pattern: it repeats, or it errored, at +least `stall_escalation_repeat_threshold` times across the last `stall_escalation_window` calls. +The tier is then bumped one step by the same `_escalate_tier` ladder `escalation_keywords` uses, +capped at the highest configured tier. It reads both tool-call shapes: Anthropic Messages `tool_use`/`tool_result` blocks (including `is_error`) and chat-completions `tool_calls`/`tool` messages (which carry no standard error flag, so those calls are judged on repetition alone). +Anchoring on the newest call is what keeps a recovered task from being escalated on stale +evidence. A model that tried the same command three times and then moved on still has those +three calls sitting in the window for a few turns, and counting whichever pattern is most common +in the window would escalate a request that is already making progress again. Anchoring still +leaves room between the matches, so a retry loop broken up by an unrelated lookup counts. + There is no state to expire or leak: detection reruns on every classified turn from that request's own message list, so the bump lasts only as long as the recent tool calls still look stuck and lifts on its own the moment they don't. This also means it reads the whole diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 508c4ec8c91..46b5dd4a0f0 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -813,16 +813,17 @@ class ComplexityRouterConfig(BaseModel): default=False, description=( "Escalate mid-task to the next-higher configured tier when the assistant's own recent " - "tool calls look stuck: stall_escalation_repeat_threshold or more of the last " - "stall_escalation_window tool calls are identical repeats (same tool, same arguments) " - "or came back as errors. One tier at most, on the same ladder escalation_keywords bumps " - "along, and never above the highest configured tier. Detection re-runs on every " - "classified turn from the tool calls visible in that request, so it needs no state and " - "nothing survives past the task: once the recent tool calls stop looking stuck, the " - "next classified turn routes normally again. Mutually exclusive with session_affinity " - "and classification_mode='user_turn', which both replay a held routing decision instead " - "of classifying most turns, so this would never see the tool calls to look at. Off by " - "default." + "tool calls look stuck: the newest tool call repeats, or errors, at least " + "stall_escalation_repeat_threshold times across the last stall_escalation_window " + "calls. Both tests are anchored on the newest call, so a task that tried the same " + "thing a few times and then moved on is not escalated on the strength of those older " + "calls alone, while a retry loop broken up by an unrelated lookup still counts. One " + "tier at most, on the same ladder escalation_keywords bumps along, and never above " + "the highest configured tier. Detection re-runs on every classified turn from the " + "tool calls visible in that request, so it needs no state and nothing survives past " + "the task. Mutually exclusive with session_affinity and classification_mode=" + "'user_turn', which both replay a held routing decision instead of classifying most " + "turns, so this would never see the tool calls to look at. Off by default." ), ) stall_escalation_window: int = Field( @@ -839,9 +840,9 @@ class ComplexityRouterConfig(BaseModel): default=3, ge=2, description=( - "How many of the last stall_escalation_window tool calls must be identical repeats, or " - "error results, before the task counts as stalled. Must not exceed " - "stall_escalation_window, or the condition could never be reached." + "How many of the last stall_escalation_window tool calls must repeat the newest call, " + "or must have errored alongside it, before the task counts as stalled. Must not " + "exceed stall_escalation_window, or the condition could never be reached." ), ) diff --git a/litellm/router_strategy/complexity_router/stall_detector.py b/litellm/router_strategy/complexity_router/stall_detector.py index 450f8b6a653..690603f05d5 100644 --- a/litellm/router_strategy/complexity_router/stall_detector.py +++ b/litellm/router_strategy/complexity_router/stall_detector.py @@ -1,27 +1,21 @@ """ Mid-task stall detection for the Complexity Router. -Looks at the assistant's own recent tool calls -- visible on every request an agentic -client resends, since each turn carries the whole conversation so far -- for a tight loop -of identical calls or repeated tool errors. No LLM call, no state: the same fixed-size -window is rescanned on every classified turn, so a stall reads the same way whether it -started one turn ago or ten, and stops reading as a stall the moment the recent calls -change. +Reads the assistant's own recent tool calls, which every agentic client resends on each +turn, and reports whether the task currently looks stuck. No LLM call and no stored state: +the same window is rescanned per classified turn, so the verdict follows the conversation +rather than latching. -Assistant tool calls appear in two shapes depending on the API surface, and this module -reads both without translating one into the other: -- Anthropic Messages: assistant `content` blocks of type "tool_use" (id, name, input), - answered by a later user-turn `content` block of type "tool_result" (tool_use_id, - is_error). -- Chat completions: assistant `tool_calls` entries (id, function.name, function.arguments - as a JSON string), answered by a later `role: "tool"` message. Chat completions has no - standard error flag on that message, so those calls are judged on repetition alone. +Tool calls arrive in two shapes and are read in place rather than translated: +- Anthropic Messages: assistant `tool_use` content blocks, answered by a user-turn + `tool_result` block carrying `is_error` +- Chat completions: assistant `tool_calls` entries, answered by a `role: "tool"` message, + which has no standard error flag, so those calls are judged on repetition alone """ from __future__ import annotations import json -from collections import Counter from collections.abc import Iterator, Mapping, Sequence from itertools import islice from typing import Final, NamedTuple @@ -32,8 +26,7 @@ _ARGUMENTS_PARSE_FAILED: Final = object() class _ToolCallEvent(NamedTuple): signature: tuple[str, str] is_error: bool | None - """None when the surface carries no structured error signal for this call. Never - treated as an error: a call this module cannot judge must not count toward the tally.""" + """None where the surface reports no error status, and never counted as an error.""" def _json_arguments(raw: str) -> object: @@ -44,9 +37,8 @@ def _json_arguments(raw: str) -> object: def _tool_call_signature(name: str, raw_arguments: object) -> tuple[str, str]: - """A (name, canonical-arguments) pair that compares equal across both surfaces' - argument shapes: a dict (Anthropic `input`) and a JSON-encoded string (chat - completions `function.arguments`) representing the same call must match.""" + """Canonicalized so the same call compares equal across both surfaces, which carry + arguments as a dict and as a JSON string respectively.""" parsed: Final = _json_arguments(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments arguments: Final = raw_arguments if parsed is _ARGUMENTS_PARSE_FAILED else parsed try: @@ -56,8 +48,6 @@ def _tool_call_signature(name: str, raw_arguments: object) -> tuple[str, str]: def _iter_tool_result_error_pairs(messages: Sequence[Mapping[str, object]]) -> Iterator[tuple[str, bool]]: - """(call id, whether that call's result was an error), read only where the surface - reports one: an Anthropic Messages `tool_result` content block's `is_error`.""" for msg in messages: content = msg.get("content") if msg.get("role") != "user" or not isinstance(content, list): @@ -70,8 +60,6 @@ def _iter_tool_result_error_pairs(messages: Sequence[Mapping[str, object]]) -> I def _iter_tool_call_events_newest_first(messages: Sequence[Mapping[str, object]]) -> Iterator[_ToolCallEvent]: - """Every tool call the assistant made, newest first, paired with its result's error - status where the surface reports one.""" error_by_call_id: Final = dict(_iter_tool_result_error_pairs(messages)) for msg in reversed(messages): if msg.get("role") != "assistant": @@ -107,20 +95,24 @@ def detect_stalled_task( window: int, repeat_threshold: int, ) -> bool: - """Whether the assistant's recent tool-call activity looks stuck: repeat_threshold or - more of the last `window` tool calls share an identical signature, or resolved to an - error on a surface that reports one. + """Whether the newest tool call is still part of a stuck pattern: it repeats, or it + errored, at least repeat_threshold times across the last `window` calls. - Reads the whole message list rather than only the turns since the newest human ask, - so a follow-up like "try again" does not discard the evidence that came before it. + Both tests are anchored on the newest call rather than counting whichever pattern is + most common in the window. A task that tried the same thing three times and then moved + on has those three calls in the window for a while yet, and counting them alone would + escalate a request that already recovered. Anchoring also leaves room between the + matches, so a retry loop broken up by an unrelated lookup still reads as stuck. """ if not messages or repeat_threshold <= 0: return False recent: Final = tuple(islice(_iter_tool_call_events_newest_first(messages), window)) if len(recent) < repeat_threshold: return False - _, most_common_count = Counter(event.signature for event in recent).most_common(1)[0] - if most_common_count >= repeat_threshold: + newest: Final = recent[0] + repeats: Final = sum(1 for event in recent if event.signature == newest.signature) + if repeats >= repeat_threshold: return True - error_count: Final = sum(1 for event in recent if event.is_error) - return error_count >= repeat_threshold + if not newest.is_error: + return False + return sum(1 for event in recent if event.is_error) >= repeat_threshold diff --git a/tests/test_litellm/router_strategy/test_stall_detector.py b/tests/test_litellm/router_strategy/test_stall_detector.py index 8f39969a8ec..34067cc3626 100644 --- a/tests/test_litellm/router_strategy/test_stall_detector.py +++ b/tests/test_litellm/router_strategy/test_stall_detector.py @@ -109,6 +109,39 @@ class TestDetectStalledTask: ] assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + def test_a_recovered_task_is_not_stalled_while_its_old_failures_sit_in_the_window(self): + """The three identical failures stay in the window for a few turns after the model + breaks out of them, and counting them on their own would escalate a request that is + already making progress again.""" + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=True), + *_anthropic_call("t2", "bash", {"cmd": "pytest"}, is_error=True), + *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=True), + *_anthropic_call("t4", "read_file", {"path": "conftest.py"}, is_error=False), + *_anthropic_call("t5", "edit_file", {"path": "conftest.py"}, is_error=False), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + + def test_a_retry_loop_broken_up_by_an_unrelated_call_still_counts(self): + """Anchoring on the newest call must not require the repeats to be adjacent: a model + re-running the same failing command around a lookup in between is still stuck.""" + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest"}, is_error=True), + *_anthropic_call("t2", "read_file", {"path": "conftest.py"}, is_error=False), + *_anthropic_call("t3", "bash", {"cmd": "pytest"}, is_error=True), + *_anthropic_call("t4", "bash", {"cmd": "pytest"}, is_error=True), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is True + + def test_errors_only_count_while_the_newest_call_is_still_failing(self): + messages = [ + *_anthropic_call("t1", "bash", {"cmd": "pytest a"}, is_error=True), + *_anthropic_call("t2", "bash", {"cmd": "pytest b"}, is_error=True), + *_anthropic_call("t3", "bash", {"cmd": "pytest c"}, is_error=True), + *_anthropic_call("t4", "bash", {"cmd": "pytest d"}, is_error=False), + ] + assert detect_stalled_task(messages, window=6, repeat_threshold=3) is False + def test_no_messages_is_not_stalled(self): assert detect_stalled_task(None, window=6, repeat_threshold=3) is False assert detect_stalled_task([], window=6, repeat_threshold=3) is False diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 09dfbd841c8..ce19330a511 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -34898,13 +34898,13 @@ export interface components { simple_keywords?: string[] | null; /** * Stall Escalation Enabled - * @description Escalate mid-task to the next-higher configured tier when the assistant's own recent tool calls look stuck: stall_escalation_repeat_threshold or more of the last stall_escalation_window tool calls are identical repeats (same tool, same arguments) or came back as errors. One tier at most, on the same ladder escalation_keywords bumps along, and never above the highest configured tier. Detection re-runs on every classified turn from the tool calls visible in that request, so it needs no state and nothing survives past the task: once the recent tool calls stop looking stuck, the next classified turn routes normally again. Mutually exclusive with session_affinity and classification_mode='user_turn', which both replay a held routing decision instead of classifying most turns, so this would never see the tool calls to look at. Off by default. + * @description Escalate mid-task to the next-higher configured tier when the assistant's own recent tool calls look stuck: the newest tool call repeats, or errors, at least stall_escalation_repeat_threshold times across the last stall_escalation_window calls. Both tests are anchored on the newest call, so a task that tried the same thing a few times and then moved on is not escalated on the strength of those older calls alone, while a retry loop broken up by an unrelated lookup still counts. One tier at most, on the same ladder escalation_keywords bumps along, and never above the highest configured tier. Detection re-runs on every classified turn from the tool calls visible in that request, so it needs no state and nothing survives past the task. Mutually exclusive with session_affinity and classification_mode='user_turn', which both replay a held routing decision instead of classifying most turns, so this would never see the tool calls to look at. Off by default. * @default false */ stall_escalation_enabled: boolean; /** * Stall Escalation Repeat Threshold - * @description How many of the last stall_escalation_window tool calls must be identical repeats, or error results, before the task counts as stalled. Must not exceed stall_escalation_window, or the condition could never be reached. + * @description How many of the last stall_escalation_window tool calls must repeat the newest call, or must have errored alongside it, before the task counts as stalled. Must not exceed stall_escalation_window, or the condition could never be reached. * @default 3 */ stall_escalation_repeat_threshold: number; From c5c10bc91fe1b6e7aba457fae4a081059f588a5d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 16:08:50 -0700 Subject: [PATCH 44/60] feat(access-groups): resolve resource names on access group responses The access group detail page rendered MCP servers, agents, attached teams and keys as bare ids, so an admin had to look each one up elsewhere to audit a group Every access group response now also carries access_mcp_servers, access_agents, assigned_teams and assigned_keys as {id, name} pairs. Names come from the DB rows first and fall back to config-declared MCP servers and agents (including legacy agent ids), resolved with one query per table across all groups in a list call. The existing *_ids columns are unchanged The UI renders the name with the id in a tooltip, links teams and keys to their detail pages, and shows the raw id only when nothing resolves --- litellm/proxy/_lazy_openapi_snapshot.json | 58 +++ .../access_group_endpoints.py | 120 ++++-- .../resource_display_names.py | 61 +++ litellm/types/access_group.py | 11 + .../test_access_group_endpoints.py | 190 +++++++++- .../test_resource_display_names.py | 130 +++++++ .../AccessGroupsDetailsPage.test.tsx | 349 +++++++++++------- .../_components/AccessGroupsDetailsPage.tsx | 93 +++-- .../AccessGroupEditModal.integration.test.tsx | 4 + .../_components/AccessGroupsPage.test.tsx | 8 + .../accessGroups/useAccessGroups.test.ts | 4 + .../hooks/accessGroups/useAccessGroups.ts | 16 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 18 + 13 files changed, 850 insertions(+), 212 deletions(-) create mode 100644 litellm/proxy/management_helpers/resource_display_names.py create mode 100644 tests/test_litellm/proxy/management_helpers/test_resource_display_names.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 316bfb8cf92..c24eea968f8 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -741,6 +741,32 @@ "title": "AccessGroupInfo", "type": "object" }, + "AccessGroupResource": { + "description": "A resource referenced by an access group. `name` is null when the id no longer resolves or has no alias.", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + } + }, + "required": [ + "id", + "name" + ], + "title": "AccessGroupResource", + "type": "object" + }, "AccessGroupResponse": { "properties": { "access_agent_ids": { @@ -750,6 +776,13 @@ "title": "Access Agent Ids", "type": "array" }, + "access_agents": { + "items": { + "$ref": "#/components/schemas/AccessGroupResource" + }, + "title": "Access Agents", + "type": "array" + }, "access_group_id": { "title": "Access Group Id", "type": "string" @@ -765,6 +798,13 @@ "title": "Access Mcp Server Ids", "type": "array" }, + "access_mcp_servers": { + "items": { + "$ref": "#/components/schemas/AccessGroupResource" + }, + "title": "Access Mcp Servers", + "type": "array" + }, "access_model_names": { "items": { "type": "string" @@ -779,6 +819,13 @@ "title": "Assigned Key Ids", "type": "array" }, + "assigned_keys": { + "items": { + "$ref": "#/components/schemas/AccessGroupResource" + }, + "title": "Assigned Keys", + "type": "array" + }, "assigned_team_ids": { "items": { "type": "string" @@ -786,6 +833,13 @@ "title": "Assigned Team Ids", "type": "array" }, + "assigned_teams": { + "items": { + "$ref": "#/components/schemas/AccessGroupResource" + }, + "title": "Assigned Teams", + "type": "array" + }, "created_at": { "format": "date-time", "title": "Created At", @@ -838,6 +892,10 @@ "access_agent_ids", "assigned_team_ids", "assigned_key_ids", + "access_mcp_servers", + "access_agents", + "assigned_teams", + "assigned_keys", "created_at", "updated_at" ], diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 1f91eeedf64..a6cc5140b15 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -1,16 +1,20 @@ -from collections.abc import Mapping, Sequence +import asyncio +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType from typing import Final, Protocol from fastapi import APIRouter, Depends, HTTPException, status from litellm._logging import verbose_proxy_logger +from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import ( CommonProxyErrors, LiteLLM_AccessGroupTable, LitellmUserRoles, UserAPIKeyAuth, ) +from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry from litellm.proxy.auth.auth_checks import ( _cache_access_object, _cache_key_object, @@ -20,10 +24,16 @@ from litellm.proxy.auth.auth_checks import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.management_helpers.access_group_team_sync import invalidate_access_group_cache -from litellm.proxy.utils import get_prisma_client_or_throw +from litellm.proxy.management_helpers.resource_display_names import ( + agent_display_names, + key_display_names, + mcp_server_display_names, +) +from litellm.proxy.utils import PrismaClient, get_prisma_client_or_throw from litellm.repositories.table_repositories import AccessGroupRepository, TeamRepository from litellm.types.access_group import ( AccessGroupCreateRequest, + AccessGroupResource, AccessGroupResponse, AccessGroupUpdateRequest, ) @@ -37,6 +47,12 @@ class _AccessGroupRecord(Protocol): @property def access_group_id(self) -> str: ... + @property + def access_mcp_server_ids(self) -> Sequence[str] | None: ... + + @property + def access_agent_ids(self) -> Sequence[str] | None: ... + @property def assigned_team_ids(self) -> Sequence[str] | None: ... @@ -50,6 +66,9 @@ class _TeamRecord(Protocol): @property def team_id(self) -> str: ... + @property + def team_alias(self) -> str | None: ... + @property def access_group_ids(self) -> Sequence[str] | None: ... @@ -120,16 +139,75 @@ def _require_admin_view(user_api_key_dict: UserAPIKeyAuth) -> None: ) +@dataclass(frozen=True, slots=True) +class _ResourceNames: + mcp_servers: Mapping[str, str] + agents: Mapping[str, str] + teams: Mapping[str, str | None] + keys: Mapping[str, str] + + +def _label(ids: Sequence[str], names: Mapping[str, str | None]) -> tuple[AccessGroupResource, ...]: + return tuple(AccessGroupResource(id=resource_id, name=names.get(resource_id)) for resource_id in ids) + + def _record_to_response( - record: _AccessGroupRecord, *, assigned_team_ids: Sequence[str] | None = None + record: _AccessGroupRecord, *, assigned_team_ids: Sequence[str], names: _ResourceNames ) -> AccessGroupResponse: - stored: Final = record.dict() - payload: Final = ( - stored if assigned_team_ids is None else MappingProxyType({**stored, "assigned_team_ids": assigned_team_ids}) + payload: Final = MappingProxyType( + { + **record.dict(), + "assigned_team_ids": assigned_team_ids, + "access_mcp_servers": _label(record.access_mcp_server_ids or (), names.mcp_servers), + "access_agents": _label(record.access_agent_ids or (), names.agents), + "assigned_teams": _label(assigned_team_ids, names.teams), + "assigned_keys": _label(record.assigned_key_ids or (), names.keys), + } ) return AccessGroupResponse.model_validate(payload) +def _ids_across( + records: Sequence[_AccessGroupRecord], pick: Callable[[_AccessGroupRecord], Sequence[str] | None] +) -> tuple[str, ...]: + return tuple(dict.fromkeys(resource_id for record in records for resource_id in (pick(record) or ()))) + + +async def _responses_for( + prisma_client: PrismaClient, records: Sequence[_AccessGroupRecord] +) -> tuple[AccessGroupResponse, ...]: + if not records: + return () + teams: Final = await _teams_touching(TeamRepository(prisma_client).table, records) + mcp_servers, agents, keys = await asyncio.gather( + mcp_server_display_names( + prisma_client, + _ids_across(records, lambda record: record.access_mcp_server_ids), + global_mcp_server_manager.config_mcp_servers, + ), + agent_display_names( + prisma_client, _ids_across(records, lambda record: record.access_agent_ids), global_agent_registry + ), + key_display_names(prisma_client, _ids_across(records, lambda record: record.assigned_key_ids)), + ) + names: Final = _ResourceNames( + mcp_servers=mcp_servers, + agents=agents, + teams=MappingProxyType({team.team_id: team.team_alias for team in teams}), + keys=keys, + ) + attached: Final = _attached_team_ids_by_group(records, teams) + return tuple( + _record_to_response(record, assigned_team_ids=attached[record.access_group_id], names=names) + for record in records + ) + + +async def _response_for(prisma_client: PrismaClient, record: _AccessGroupRecord) -> AccessGroupResponse: + (response,) = await _responses_for(prisma_client, (record,)) + return response + + def _attached_team_ids_by_group( records: Sequence[_AccessGroupRecord], teams: Sequence[_TeamRecord] ) -> Mapping[str, tuple[str, ...]]: @@ -144,19 +222,21 @@ def _attached_team_ids_by_group( return MappingProxyType({record.access_group_id: attached(record) for record in records}) +async def _teams_touching(team_table: _TeamTable, records: Sequence[_AccessGroupRecord]) -> Sequence[_TeamRecord]: + """Team rows listed on any of the groups or carrying any of them in access_group_ids.""" + group_ids: Final = tuple(record.access_group_id for record in records) + stored_team_ids: Final = _ids_across(records, lambda record: record.assigned_team_ids) + carrying: Final = {"access_group_ids": {"hasSome": group_ids}} # mutable-ok: prisma where is a dict + listed: Final = {"team_id": {"in": stored_team_ids}} # mutable-ok: prisma where is a dict + return await team_table.find_many(where={"OR": (carrying, listed)}) # mutable-ok: prisma where is a dict + + async def _attached_team_ids_for( team_table: _TeamTable, records: Sequence[_AccessGroupRecord] ) -> Mapping[str, tuple[str, ...]]: if not records: return MappingProxyType({}) - group_ids: Final = tuple(record.access_group_id for record in records) - stored_team_ids: Final = tuple( - dict.fromkeys(team_id for record in records for team_id in (record.assigned_team_ids or ())) - ) - carrying: Final = {"access_group_ids": {"hasSome": group_ids}} # mutable-ok: prisma where is a dict - listed: Final = {"team_id": {"in": stored_team_ids}} # mutable-ok: prisma where is a dict - teams: Final = await team_table.find_many(where={"OR": (carrying, listed)}) # mutable-ok: prisma where is a dict - return _attached_team_ids_by_group(records, teams) + return _attached_team_ids_by_group(records, await _teams_touching(team_table, records)) async def _require_teams_exist(tx: _AccessGroupTx, team_ids: Sequence[str]) -> None: @@ -425,7 +505,7 @@ async def create_access_group( proxy_logging_obj, ) - return _record_to_response(record) + return await _response_for(prisma_client, record) @router.get( @@ -434,14 +514,13 @@ async def create_access_group( ) async def list_access_groups( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -) -> list[AccessGroupResponse]: +) -> Sequence[AccessGroupResponse]: _require_admin_view(user_api_key_dict) prisma_client: Final = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) table: Final = AccessGroupRepository(prisma_client).table records: Final = await table.find_many(order={"created_at": "desc"}) - attached: Final = await _attached_team_ids_for(TeamRepository(prisma_client).table, records) - return [_record_to_response(r, assigned_team_ids=attached[r.access_group_id]) for r in records] + return await _responses_for(prisma_client, records) @router.get( @@ -462,8 +541,7 @@ async def get_access_group( status_code=status.HTTP_404_NOT_FOUND, detail=f"Access group '{access_group_id}' not found", ) - attached: Final = await _attached_team_ids_for(TeamRepository(prisma_client).table, (record,)) - return _record_to_response(record, assigned_team_ids=attached[record.access_group_id]) + return await _response_for(prisma_client, record) @router.put( @@ -560,7 +638,7 @@ async def update_access_group( await _patch_key_caches_add_access_group(keys_to_add, access_group_id, user_api_key_cache, proxy_logging_obj) await _patch_key_caches_remove_access_group(keys_to_remove, access_group_id, user_api_key_cache, proxy_logging_obj) - return _record_to_response(record) + return await _response_for(prisma_client, record) @router.delete( diff --git a/litellm/proxy/management_helpers/resource_display_names.py b/litellm/proxy/management_helpers/resource_display_names.py new file mode 100644 index 00000000000..31b7b68d233 --- /dev/null +++ b/litellm/proxy/management_helpers/resource_display_names.py @@ -0,0 +1,61 @@ +"""Display names for ids stored on management objects. DB rows win; config-declared servers and agents fill the gaps.""" + +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry +from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import AgentsRepository, MCPServerRepository +from litellm.repositories.verification_token_repository import VerificationTokenRepository +from litellm.types.mcp_server.mcp_server_manager import MCPServer + + +async def mcp_server_display_names( + prisma_client: PrismaClient, + server_ids: Sequence[str], + config_servers: Mapping[str, MCPServer], +) -> Mapping[str, str]: + """server_id -> alias, falling back to server_name; config-only servers also fall back to their registry name.""" + if not server_ids: + return MappingProxyType({}) + wanted: Final = frozenset(server_ids) + where: Final = {"server_id": {"in": tuple(wanted)}} # mutable-ok: prisma where is a dict + rows: Final = await MCPServerRepository(prisma_client).table.find_many(where=where) + from_config: Final = { + server_id: server.alias or server.server_name or server.name + for server_id, server in config_servers.items() + if server_id in wanted + } + from_db: Final = {row.server_id: name for row in rows if (name := row.alias or row.server_name)} + return MappingProxyType({**from_config, **from_db}) + + +async def agent_display_names( + prisma_client: PrismaClient, + agent_ids: Sequence[str], + registry: AgentRegistry, +) -> Mapping[str, str]: + """agent_id -> agent_name. The registry covers config-declared agents and their legacy ids.""" + if not agent_ids: + return MappingProxyType({}) + wanted: Final = frozenset(agent_ids) + where: Final = {"agent_id": {"in": tuple(wanted)}} # mutable-ok: prisma where is a dict + rows: Final = await AgentsRepository(prisma_client).table.find_many(where=where) + from_registry: Final = { + alias_id: agent.agent_name + for agent in registry.get_agent_list() + for alias_id in registry.ids_for_agent(agent.agent_id) + if alias_id in wanted + } + from_db: Final = {row.agent_id: row.agent_name for row in rows} + return MappingProxyType({**from_registry, **from_db}) + + +async def key_display_names(prisma_client: PrismaClient, tokens: Sequence[str]) -> Mapping[str, str]: + """token hash -> key_alias for the keys that have one.""" + if not tokens: + return MappingProxyType({}) + where: Final = {"token": {"in": tuple(frozenset(tokens))}} # mutable-ok: prisma where is a dict + rows: Final = await VerificationTokenRepository(prisma_client).table.find_many(where=where) + return MappingProxyType({row.token: row.key_alias for row in rows if row.key_alias}) diff --git a/litellm/types/access_group.py b/litellm/types/access_group.py index b477ce309b7..951e5a414b4 100644 --- a/litellm/types/access_group.py +++ b/litellm/types/access_group.py @@ -23,6 +23,13 @@ class AccessGroupUpdateRequest(BaseModel): assigned_key_ids: list[str] | None = None +class AccessGroupResource(BaseModel): + """A resource referenced by an access group. `name` is null when the id no longer resolves or has no alias.""" + + id: str + name: str | None + + class AccessGroupResponse(BaseModel): access_group_id: str access_group_name: str @@ -32,6 +39,10 @@ class AccessGroupResponse(BaseModel): access_agent_ids: list[str] assigned_team_ids: list[str] assigned_key_ids: list[str] + access_mcp_servers: tuple[AccessGroupResource, ...] + access_agents: tuple[AccessGroupResource, ...] + assigned_teams: tuple[AccessGroupResource, ...] + assigned_keys: tuple[AccessGroupResource, ...] created_at: datetime created_by: str | None = None updated_at: datetime diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index 81816e21c10..d687f8d1c8d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -57,8 +57,20 @@ def _make_access_group_record( return record -def _make_team_record(team_id: str, access_group_ids: list[str] | None = None): - return types.SimpleNamespace(team_id=team_id, access_group_ids=access_group_ids or []) +def _make_team_record(team_id: str, access_group_ids: list[str] | None = None, team_alias: str | None = None): + return types.SimpleNamespace(team_id=team_id, access_group_ids=access_group_ids or [], team_alias=team_alias) + + +def _make_mcp_server_record(server_id: str, alias: str | None = None, server_name: str | None = None): + return types.SimpleNamespace(server_id=server_id, alias=alias, server_name=server_name) + + +def _make_agent_record(agent_id: str, agent_name: str): + return types.SimpleNamespace(agent_id=agent_id, agent_name=agent_name) + + +def _make_key_record(token: str, key_alias: str | None = None): + return types.SimpleNamespace(token=token, key_alias=key_alias) @pytest.fixture @@ -109,6 +121,12 @@ def client_and_mocks(monkeypatch): mock_key_table.find_unique = AsyncMock(return_value=None) mock_key_table.update = AsyncMock(return_value=None) + mock_mcp_server_table = MagicMock() + mock_mcp_server_table.find_many = AsyncMock(return_value=[]) + + mock_agents_table = MagicMock() + mock_agents_table.find_many = AsyncMock(return_value=[]) + @asynccontextmanager async def mock_tx(): tx = types.SimpleNamespace( @@ -122,6 +140,8 @@ def client_and_mocks(monkeypatch): litellm_accessgrouptable=mock_access_group_table, litellm_teamtable=mock_team_table, litellm_verificationtoken=mock_key_table, + litellm_mcpservertable=mock_mcp_server_table, + litellm_agentstable=mock_agents_table, tx=mock_tx, ) mock_prisma.db = mock_db @@ -1447,3 +1467,169 @@ def test_update_access_group_null_assigned_ids_treated_as_empty(client_and_mocks update_call_kwargs = mock_table.update.call_args.kwargs assert update_call_kwargs["data"]["assigned_team_ids"] == [] assert update_call_kwargs["data"]["assigned_key_ids"] == [] + + +# --------------------------------------------------------------------------- +# Resolved resource names (LIT-6594) +# --------------------------------------------------------------------------- + + +def _mock_resource_tables(mock_prisma, *, mcp_servers=(), agents=(), teams=(), keys=()): + mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=list(mcp_servers)) + mock_prisma.db.litellm_agentstable.find_many = AsyncMock(return_value=list(agents)) + mock_prisma.db.litellm_teamtable.find_many = AsyncMock(return_value=list(teams)) + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=list(keys)) + + +@pytest.mark.parametrize("base_path", ACCESS_GROUP_PATHS) +def test_get_access_group_resolves_resource_names(client_and_mocks, base_path): + """Every id list gets a sibling list of {id, name}; name is null when the id has no alias or no longer resolves.""" + client, mock_prisma, mock_table, *_ = client_and_mocks + mock_table.find_unique = AsyncMock( + return_value=_make_access_group_record( + access_group_id="ag-123", + access_mcp_server_ids=["mcp-a", "mcp-b", "mcp-ghost"], + access_agent_ids=["agent-a", "agent-ghost"], + assigned_team_ids=["team-a", "team-b"], + assigned_key_ids=["key-a", "key-b"], + ) + ) + _mock_resource_tables( + mock_prisma, + mcp_servers=[ + _make_mcp_server_record("mcp-a", alias="GitHub"), + _make_mcp_server_record("mcp-b", server_name="jira_tools"), + ], + agents=[_make_agent_record("agent-a", "support-bot")], + teams=[ + _make_team_record("team-a", ["ag-123"], team_alias="Platform"), + _make_team_record("team-b", ["ag-123"]), + ], + keys=[_make_key_record("key-a", key_alias="ci-key"), _make_key_record("key-b")], + ) + + resp = client.get(f"{base_path}/ag-123") + assert resp.status_code == 200 + body = resp.json() + assert body["access_mcp_servers"] == [ + {"id": "mcp-a", "name": "GitHub"}, + {"id": "mcp-b", "name": "jira_tools"}, + {"id": "mcp-ghost", "name": None}, + ] + assert body["access_agents"] == [{"id": "agent-a", "name": "support-bot"}, {"id": "agent-ghost", "name": None}] + assert body["assigned_teams"] == [{"id": "team-a", "name": "Platform"}, {"id": "team-b", "name": None}] + assert body["assigned_keys"] == [{"id": "key-a", "name": "ci-key"}, {"id": "key-b", "name": None}] + assert body["access_mcp_server_ids"] == ["mcp-a", "mcp-b", "mcp-ghost"] + assert body["assigned_team_ids"] == ["team-a", "team-b"] + + mcp_where = mock_prisma.db.litellm_mcpservertable.find_many.call_args.kwargs["where"] + assert sorted(mcp_where["server_id"]["in"]) == ["mcp-a", "mcp-b", "mcp-ghost"] + agent_where = mock_prisma.db.litellm_agentstable.find_many.call_args.kwargs["where"] + assert sorted(agent_where["agent_id"]["in"]) == ["agent-a", "agent-ghost"] + key_where = mock_prisma.db.litellm_verificationtoken.find_many.call_args.kwargs["where"] + assert sorted(key_where["token"]["in"]) == ["key-a", "key-b"] + + +def test_list_access_groups_resolves_names_with_one_query_per_table(client_and_mocks): + """List batches every group's ids into one lookup per table and attributes names back to the right group.""" + client, mock_prisma, mock_table, *_ = client_and_mocks + mock_table.find_many = AsyncMock( + return_value=[ + _make_access_group_record( + access_group_id="ag-1", access_mcp_server_ids=["mcp-a"], access_agent_ids=["agent-a"], assigned_key_ids=["key-a"] + ), + _make_access_group_record( + access_group_id="ag-2", access_mcp_server_ids=["mcp-b"], access_agent_ids=["agent-b"], assigned_key_ids=["key-b"] + ), + ] + ) + _mock_resource_tables( + mock_prisma, + mcp_servers=[_make_mcp_server_record("mcp-a", alias="A"), _make_mcp_server_record("mcp-b", alias="B")], + agents=[_make_agent_record("agent-a", "Agent A"), _make_agent_record("agent-b", "Agent B")], + keys=[_make_key_record("key-a", key_alias="Key A"), _make_key_record("key-b", key_alias="Key B")], + ) + + resp = client.get("/v1/access_group") + assert resp.status_code == 200 + first, second = resp.json() + assert first["access_mcp_servers"] == [{"id": "mcp-a", "name": "A"}] + assert first["access_agents"] == [{"id": "agent-a", "name": "Agent A"}] + assert first["assigned_keys"] == [{"id": "key-a", "name": "Key A"}] + assert second["access_mcp_servers"] == [{"id": "mcp-b", "name": "B"}] + assert second["access_agents"] == [{"id": "agent-b", "name": "Agent B"}] + assert second["assigned_keys"] == [{"id": "key-b", "name": "Key B"}] + + for table, column in ( + (mock_prisma.db.litellm_mcpservertable, "server_id"), + (mock_prisma.db.litellm_agentstable, "agent_id"), + (mock_prisma.db.litellm_verificationtoken, "token"), + ): + table.find_many.assert_awaited_once() + assert len(table.find_many.call_args.kwargs["where"][column]["in"]) == 2 + + +def test_list_access_groups_skips_lookups_when_nothing_to_resolve(client_and_mocks): + """Groups with no MCP servers, agents or keys must not trigger an empty IN () query per table.""" + client, mock_prisma, mock_table, *_ = client_and_mocks + mock_table.find_many = AsyncMock( + return_value=[_make_access_group_record(access_group_id="ag-1"), _make_access_group_record(access_group_id="ag-2")] + ) + + resp = client.get("/v1/access_group") + assert resp.status_code == 200 + assert all(group["access_mcp_servers"] == [] and group["assigned_keys"] == [] for group in resp.json()) + + mock_prisma.db.litellm_mcpservertable.find_many.assert_not_awaited() + mock_prisma.db.litellm_agentstable.find_many.assert_not_awaited() + mock_prisma.db.litellm_verificationtoken.find_many.assert_not_awaited() + + +def test_create_access_group_response_carries_resolved_names(client_and_mocks): + """The create response already shows names so the UI never has to refetch to label what it just saved.""" + client, mock_prisma, *_ = client_and_mocks + team_record = _make_team_record("team-1", team_alias="Platform") + mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_record) + _mock_resource_tables( + mock_prisma, + mcp_servers=[_make_mcp_server_record("mcp-a", alias="GitHub")], + agents=[_make_agent_record("agent-a", "support-bot")], + teams=[team_record], + ) + + resp = client.post( + "/v1/access_group", + json={ + "access_group_name": "new-group", + "access_mcp_server_ids": ["mcp-a"], + "access_agent_ids": ["agent-a"], + "assigned_team_ids": ["team-1"], + }, + ) + assert resp.status_code == 201 + body = resp.json() + assert body["access_mcp_servers"] == [{"id": "mcp-a", "name": "GitHub"}] + assert body["access_agents"] == [{"id": "agent-a", "name": "support-bot"}] + assert body["assigned_teams"] == [{"id": "team-1", "name": "Platform"}] + + +def test_update_access_group_response_carries_resolved_names(client_and_mocks): + """The update response reflects the new ids with their names, not the pre-update state.""" + client, mock_prisma, mock_table, *_ = client_and_mocks + mock_table.find_unique = AsyncMock( + return_value=_make_access_group_record(access_group_id="ag-update", access_mcp_server_ids=["mcp-old"]) + ) + _mock_resource_tables( + mock_prisma, + mcp_servers=[_make_mcp_server_record("mcp-new", alias="Linear")], + agents=[_make_agent_record("agent-a", "support-bot")], + ) + + resp = client.put( + "/v1/access_group/ag-update", json={"access_mcp_server_ids": ["mcp-new"], "access_agent_ids": ["agent-a"]} + ) + assert resp.status_code == 200 + body = resp.json() + assert body["access_mcp_servers"] == [{"id": "mcp-new", "name": "Linear"}] + assert body["access_agents"] == [{"id": "agent-a", "name": "support-bot"}] + assert body["access_mcp_server_ids"] == ["mcp-new"] diff --git a/tests/test_litellm/proxy/management_helpers/test_resource_display_names.py b/tests/test_litellm/proxy/management_helpers/test_resource_display_names.py new file mode 100644 index 00000000000..b530bc15c25 --- /dev/null +++ b/tests/test_litellm/proxy/management_helpers/test_resource_display_names.py @@ -0,0 +1,130 @@ +import types +from types import MappingProxyType +from unittest.mock import AsyncMock + +import pytest + +from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry +from litellm.proxy.management_helpers.resource_display_names import ( + agent_display_names, + key_display_names, + mcp_server_display_names, +) +from litellm.types.agents import AgentResponse +from litellm.types.mcp_server.mcp_server_manager import MCPServer + + +def _table(rows=()): + return types.SimpleNamespace(find_many=AsyncMock(return_value=list(rows))) + + +def _prisma(**tables): + return types.SimpleNamespace(db=types.SimpleNamespace(**tables)) + + +def _config_server(server_id: str, name: str, alias: str | None = None, server_name: str | None = None) -> MCPServer: + return MCPServer(server_id=server_id, name=name, alias=alias, server_name=server_name, transport="http") + + +def _registry_with(*agents: AgentResponse, legacy_ids: dict[str, str] | None = None) -> AgentRegistry: + registry = AgentRegistry() + for agent in agents: + registry.register_agent(agent) + registry.config_agent_legacy_ids = MappingProxyType(legacy_ids or {}) + return registry + + +def _agent(agent_id: str, agent_name: str) -> AgentResponse: + return AgentResponse(agent_id=agent_id, agent_name=agent_name, agent_card_params={}) + + +@pytest.mark.asyncio +async def test_mcp_db_row_beats_config_entry_for_the_same_server(): + """The DB is authoritative when both sources know a server; the registry may lag behind a rename on another pod.""" + prisma = _prisma( + litellm_mcpservertable=_table([types.SimpleNamespace(server_id="s1", alias="db-alias", server_name=None)]) + ) + names = await mcp_server_display_names(prisma, ("s1",), {"s1": _config_server("s1", "config-name")}) + assert dict(names) == {"s1": "db-alias"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("alias", "server_name", "expected"), + [("Alias", "server_name", "Alias"), (None, "server_name", "server_name"), (None, None, "config-name")], +) +async def test_mcp_config_only_server_falls_back_alias_then_server_name_then_name(alias, server_name, expected): + """Config-declared servers have no DB row, so their registry entry supplies the label.""" + prisma = _prisma(litellm_mcpservertable=_table()) + config = {"s1": _config_server("s1", "config-name", alias=alias, server_name=server_name)} + names = await mcp_server_display_names(prisma, ("s1",), config) + assert dict(names) == {"s1": expected} + + +@pytest.mark.asyncio +async def test_mcp_db_row_without_alias_or_server_name_yields_no_label(): + """A bare DB row must not produce an empty string label; the caller falls back to the id.""" + prisma = _prisma( + litellm_mcpservertable=_table([types.SimpleNamespace(server_id="s1", alias=None, server_name=None)]) + ) + assert dict(await mcp_server_display_names(prisma, ("s1",), {})) == {} + + +@pytest.mark.asyncio +async def test_mcp_only_requested_ids_are_returned_and_the_query_is_deduped(): + """Unrequested config servers stay out of the result and repeated ids collapse to one IN filter entry.""" + table = _table([types.SimpleNamespace(server_id="s1", alias="A", server_name=None)]) + prisma = _prisma(litellm_mcpservertable=table) + config = {"other": _config_server("other", "not-requested")} + names = await mcp_server_display_names(prisma, ("s1", "s1", "missing"), config) + assert dict(names) == {"s1": "A"} + assert sorted(table.find_many.call_args.kwargs["where"]["server_id"]["in"]) == ["missing", "s1"] + + +@pytest.mark.asyncio +async def test_mcp_empty_ids_skip_the_db(): + table = _table() + names = await mcp_server_display_names(_prisma(litellm_mcpservertable=table), (), {}) + assert dict(names) == {} + table.find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_agent_db_name_beats_registry_name(): + prisma = _prisma(litellm_agentstable=_table([types.SimpleNamespace(agent_id="a1", agent_name="from-db")])) + registry = _registry_with(_agent("a1", "from-registry")) + assert dict(await agent_display_names(prisma, ("a1",), registry)) == {"a1": "from-db"} + + +@pytest.mark.asyncio +async def test_agent_legacy_config_id_resolves_to_the_stable_agent_name(): + """Access groups saved before agent ids were stabilised still carry the legacy hash; it must still get a name.""" + prisma = _prisma(litellm_agentstable=_table()) + registry = _registry_with(_agent("stable-id", "config-agent"), legacy_ids={"legacy-id": "stable-id"}) + names = await agent_display_names(prisma, ("legacy-id", "stable-id", "unknown"), registry) + assert dict(names) == {"legacy-id": "config-agent", "stable-id": "config-agent"} + + +@pytest.mark.asyncio +async def test_agent_empty_ids_skip_the_db(): + table = _table() + names = await agent_display_names(_prisma(litellm_agentstable=table), (), _registry_with()) + assert dict(names) == {} + table.find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_key_alias_only_for_keys_that_have_one(): + table = _table( + [types.SimpleNamespace(token="k1", key_alias="ci-key"), types.SimpleNamespace(token="k2", key_alias=None)] + ) + names = await key_display_names(_prisma(litellm_verificationtoken=table), ("k1", "k2", "k1")) + assert dict(names) == {"k1": "ci-key"} + assert sorted(table.find_many.call_args.kwargs["where"]["token"]["in"]) == ["k1", "k2"] + + +@pytest.mark.asyncio +async def test_key_empty_ids_skip_the_db(): + table = _table() + assert dict(await key_display_names(_prisma(litellm_verificationtoken=table), ())) == {} + table.find_many.assert_not_awaited() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx index cf41f623fd6..aad63e979ac 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx @@ -7,6 +7,7 @@ import { renderWithProviders } from "../../../../../tests/test-utils"; import { AccessGroupDetail } from "./AccessGroupsDetailsPage"; vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroupDetails"); +vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) })); vi.mock("./AccessGroupsModal/AccessGroupEditModal", () => ({ AccessGroupEditModal: ({ visible, onCancel }: { visible: boolean; onCancel: () => void }) => visible ? ( @@ -44,6 +45,8 @@ const baseMockReturnValue = { refetch: vi.fn(), } as unknown as ReturnType; +const unnamed = (ids: readonly string[]) => ids.map((id) => ({ id, name: null })); + const createMockAccessGroup = (overrides: Partial = {}): AccessGroupResponse => ({ access_group_id: "ag-1", access_group_name: "Test Group", @@ -53,6 +56,13 @@ const createMockAccessGroup = (overrides: Partial = {}): Ac access_agent_ids: ["agent-1"], assigned_team_ids: ["team-1"], assigned_key_ids: ["key-1", "key-2"], + access_mcp_servers: [{ id: "mcp-1", name: "GitHub MCP" }], + access_agents: [{ id: "agent-1", name: "Support Agent" }], + assigned_teams: [{ id: "team-1", name: "Platform Team" }], + assigned_keys: [ + { id: "key-1", name: "ci-key" }, + { id: "key-2", name: null }, + ], created_at: "2025-01-01T00:00:00Z", created_by: null, updated_at: "2025-01-02T00:00:00Z", @@ -60,6 +70,14 @@ const createMockAccessGroup = (overrides: Partial = {}): Ac ...overrides, }); +const renderWith = (overrides: Partial = {}) => { + mockUseAccessGroupDetails.mockReturnValue({ + ...baseMockReturnValue, + data: createMockAccessGroup(overrides), + } as ReturnType); + return renderWithProviders(); +}; + describe("AccessGroupDetail", () => { const mockOnBack = vi.fn(); const accessGroupId = "ag-1"; @@ -106,9 +124,7 @@ describe("AccessGroupDetail", () => { const user = userEvent.setup(); renderWithProviders(); - const buttons = screen.getAllByRole("button"); - const backButton = buttons.find((btn) => !btn.textContent?.includes("Edit")); - await user.click(backButton!); + await user.click(screen.getByRole("button", { name: "Back" })); expect(mockOnBack).toHaveBeenCalledTimes(1); }); @@ -128,12 +144,7 @@ describe("AccessGroupDetail", () => { }); it("should display em dash when description is empty", () => { - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ description: null }), - } as ReturnType); - - renderWithProviders(); + renderWith({ description: null }); expect(screen.getByText("—")).toBeInTheDocument(); }); @@ -144,8 +155,7 @@ describe("AccessGroupDetail", () => { expect(screen.queryByRole("dialog", { name: "Edit Access Group" })).not.toBeInTheDocument(); - const editButton = screen.getByRole("button", { name: /Edit Access Group/i }); - await user.click(editButton); + await user.click(screen.getByRole("button", { name: /Edit Access Group/i })); expect(screen.getByRole("dialog", { name: "Edit Access Group" })).toBeInTheDocument(); }); @@ -161,88 +171,126 @@ describe("AccessGroupDetail", () => { expect(screen.queryByRole("dialog", { name: "Edit Access Group" })).not.toBeInTheDocument(); }); - it("should display attached keys", () => { - renderWithProviders(); + describe("attached keys", () => { + it("should show the key alias and hide the token when the key has an alias", () => { + renderWithProviders(); - expect(screen.getByText("Attached Keys")).toBeInTheDocument(); - expect(screen.getByText("key-1")).toBeInTheDocument(); - expect(screen.getByText("key-2")).toBeInTheDocument(); + expect(screen.getByText("Attached Keys")).toBeInTheDocument(); + expect(screen.getByText("ci-key")).toBeInTheDocument(); + expect(screen.queryByText("key-1")).not.toBeInTheDocument(); + }); + + it("should fall back to the token when the key has no alias", () => { + renderWithProviders(); + + expect(screen.getByText("key-2")).toBeInTheDocument(); + }); + + it("should link each key to its detail page", () => { + renderWithProviders(); + + expect(screen.getByRole("link", { name: "ci-key" })).toHaveAttribute( + "href", + expect.stringContaining("key=key-1"), + ); + expect(screen.getByRole("link", { name: "key-2" })).toHaveAttribute("href", expect.stringContaining("key=key-2")); + }); + + it("should reveal the token in a tooltip when hovering an aliased key", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.hover(screen.getByText("ci-key")); + + expect(await screen.findByText("key-1")).toBeInTheDocument(); + }); + + it("should show View All button for keys when more than 5", () => { + renderWith({ assigned_keys: unnamed(["k1", "k2", "k3", "k4", "k5", "k6"]) }); + + expect(screen.getByRole("button", { name: "View All (6)" })).toBeInTheDocument(); + expect(screen.queryByText("k6")).not.toBeInTheDocument(); + }); + + it("should toggle between View All and Show Less for keys", async () => { + const user = userEvent.setup(); + renderWith({ assigned_keys: unnamed(["k1", "k2", "k3", "k4", "k5", "k6"]) }); + + await user.click(screen.getByRole("button", { name: "View All (6)" })); + expect(screen.getByRole("button", { name: "Show Less" })).toBeInTheDocument(); + expect(screen.getByText("k6")).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Show Less" })); + expect(screen.getByRole("button", { name: "View All (6)" })).toBeInTheDocument(); + }); + + it("should show empty state when no keys attached", () => { + renderWith({ assigned_keys: [] }); + + expect(screen.getByText("No keys attached")).toBeInTheDocument(); + }); + + it("should truncate long unaliased tokens with ellipsis", () => { + renderWith({ assigned_keys: unnamed(["a".repeat(25)]) }); + + expect(screen.getByText(/^a{10}\.\.\.a{6}$/)).toBeInTheDocument(); + }); + + it("should not truncate a long alias", () => { + const alias = "b".repeat(25); + renderWith({ assigned_keys: [{ id: "a".repeat(25), name: alias }] }); + + expect(screen.getByText(alias)).toBeInTheDocument(); + }); }); - it("should display attached teams", () => { - renderWithProviders(); + describe("attached teams", () => { + it("should show the team alias and hide the id when the team has an alias", () => { + renderWithProviders(); - expect(screen.getByText("Attached Teams")).toBeInTheDocument(); - expect(screen.getByText("team-1")).toBeInTheDocument(); + expect(screen.getByText("Attached Teams")).toBeInTheDocument(); + expect(screen.getByText("Platform Team")).toBeInTheDocument(); + expect(screen.queryByText("team-1")).not.toBeInTheDocument(); + }); + + it("should link each team to its detail page", () => { + renderWithProviders(); + + expect(screen.getByRole("link", { name: "Platform Team" })).toHaveAttribute( + "href", + expect.stringContaining("team=team-1"), + ); + }); + + it("should reveal the team id in a tooltip when hovering an aliased team", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.hover(screen.getByText("Platform Team")); + + expect(await screen.findByText("team-1")).toBeInTheDocument(); + }); + + it("should fall back to the team id when the team has no alias", () => { + renderWith({ assigned_teams: unnamed(["team-ghost"]) }); + + expect(screen.getByText("team-ghost")).toBeInTheDocument(); + }); + + it("should show View All button for teams when more than 5", () => { + renderWith({ assigned_teams: unnamed(["t1", "t2", "t3", "t4", "t5", "t6"]) }); + + expect(screen.getByRole("button", { name: "View All (6)" })).toBeInTheDocument(); + }); + + it("should show empty state when no teams attached", () => { + renderWith({ assigned_teams: [] }); + + expect(screen.getByText("No teams attached")).toBeInTheDocument(); + }); }); - it("should show View All button for keys when more than 5", () => { - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ - assigned_key_ids: ["k1", "k2", "k3", "k4", "k5", "k6"], - }), - } as ReturnType); - - renderWithProviders(); - - expect(screen.getByRole("button", { name: "View All (6)" })).toBeInTheDocument(); - }); - - it("should toggle between View All and Show Less for keys", async () => { - const user = userEvent.setup(); - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ - assigned_key_ids: ["k1", "k2", "k3", "k4", "k5", "k6"], - }), - } as ReturnType); - - renderWithProviders(); - - await user.click(screen.getByRole("button", { name: "View All (6)" })); - expect(screen.getByRole("button", { name: "Show Less" })).toBeInTheDocument(); - - await user.click(screen.getByRole("button", { name: "Show Less" })); - expect(screen.getByRole("button", { name: "View All (6)" })).toBeInTheDocument(); - }); - - it("should show View All button for teams when more than 5", () => { - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ - assigned_team_ids: ["t1", "t2", "t3", "t4", "t5", "t6"], - }), - } as ReturnType); - - renderWithProviders(); - - expect(screen.getByRole("button", { name: "View All (6)" })).toBeInTheDocument(); - }); - - it("should show empty state when no keys attached", () => { - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ assigned_key_ids: [] }), - } as ReturnType); - - renderWithProviders(); - - expect(screen.getByText("No keys attached")).toBeInTheDocument(); - }); - - it("should show empty state when no teams attached", () => { - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ assigned_team_ids: [] }), - } as ReturnType); - - renderWithProviders(); - - expect(screen.getByText("No teams attached")).toBeInTheDocument(); - }); - - it("should display Models tab with model IDs", () => { + it("should display Models tab with model names", () => { renderWithProviders(); expect(screen.getByRole("tab", { name: /Models/i })).toBeInTheDocument(); @@ -250,73 +298,90 @@ describe("AccessGroupDetail", () => { expect(screen.getByText("model-2")).toBeInTheDocument(); }); - it("should display MCP Servers tab with server IDs", async () => { - const user = userEvent.setup(); - renderWithProviders(); + describe("MCP Servers tab", () => { + it("should show server names instead of ids", async () => { + const user = userEvent.setup(); + renderWithProviders(); - const mcpTab = screen.getByRole("tab", { name: /MCP Servers/i }); - expect(mcpTab).toBeInTheDocument(); - await user.click(mcpTab); - expect(screen.getByText("mcp-1")).toBeInTheDocument(); + await user.click(screen.getByRole("tab", { name: /MCP Servers/i })); + + expect(screen.getByText("GitHub MCP")).toBeInTheDocument(); + expect(screen.queryByText("mcp-1")).not.toBeInTheDocument(); + }); + + it("should reveal the server id in a tooltip when hovering the name", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("tab", { name: /MCP Servers/i })); + await user.hover(screen.getByText("GitHub MCP")); + + expect(await screen.findByText("mcp-1")).toBeInTheDocument(); + }); + + it("should fall back to the id when the server has no name", async () => { + const user = userEvent.setup(); + renderWith({ access_mcp_servers: unnamed(["mcp-deleted"]) }); + + await user.click(screen.getByRole("tab", { name: /MCP Servers/i })); + + expect(screen.getByText("mcp-deleted")).toBeInTheDocument(); + }); + + it("should show empty state when none assigned", async () => { + const user = userEvent.setup(); + renderWith({ access_mcp_servers: [] }); + + await user.click(screen.getByRole("tab", { name: /MCP Servers/i })); + + expect(screen.getByText("No MCP servers assigned to this group")).toBeInTheDocument(); + }); }); - it("should display Agents tab with agent IDs", async () => { - const user = userEvent.setup(); - renderWithProviders(); + describe("Agents tab", () => { + it("should show agent names instead of ids", async () => { + const user = userEvent.setup(); + renderWithProviders(); - const agentsTab = screen.getByRole("tab", { name: /Agents/i }); - expect(agentsTab).toBeInTheDocument(); - await user.click(agentsTab); - expect(screen.getByText("agent-1")).toBeInTheDocument(); + await user.click(screen.getByRole("tab", { name: /Agents/i })); + + expect(screen.getByText("Support Agent")).toBeInTheDocument(); + expect(screen.queryByText("agent-1")).not.toBeInTheDocument(); + }); + + it("should fall back to the id when the agent has no name", async () => { + const user = userEvent.setup(); + renderWith({ access_agents: unnamed(["agent-deleted"]) }); + + await user.click(screen.getByRole("tab", { name: /Agents/i })); + + expect(screen.getByText("agent-deleted")).toBeInTheDocument(); + }); + + it("should show empty state when none assigned", async () => { + const user = userEvent.setup(); + renderWith({ access_agents: [] }); + + await user.click(screen.getByRole("tab", { name: /Agents/i })); + + expect(screen.getByText("No agents assigned to this group")).toBeInTheDocument(); + }); }); it("should show empty state in Models tab when no models assigned", () => { - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ access_model_names: [] }), - } as ReturnType); - - renderWithProviders(); + renderWith({ access_model_names: [] }); expect(screen.getByText("No models assigned to this group")).toBeInTheDocument(); }); - it("should show empty state in MCP Servers tab when none assigned", async () => { - const user = userEvent.setup(); - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ access_mcp_server_ids: [] }), - } as ReturnType); + it("should count resources from the resolved lists in the tab badges", () => { + renderWith({ + access_mcp_servers: unnamed(["m1", "m2", "m3"]), + access_agents: unnamed(["a1", "a2"]), + }); - renderWithProviders(); - - await user.click(screen.getByRole("tab", { name: /MCP Servers/i })); - expect(screen.getByText("No MCP servers assigned to this group")).toBeInTheDocument(); - }); - - it("should show empty state in Agents tab when none assigned", async () => { - const user = userEvent.setup(); - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ access_agent_ids: [] }), - } as ReturnType); - - renderWithProviders(); - - await user.click(screen.getByRole("tab", { name: /Agents/i })); - expect(screen.getByText("No agents assigned to this group")).toBeInTheDocument(); - }); - - it("should truncate long key IDs with ellipsis", () => { - const longKeyId = "a".repeat(25); - mockUseAccessGroupDetails.mockReturnValue({ - ...baseMockReturnValue, - data: createMockAccessGroup({ assigned_key_ids: [longKeyId] }), - } as ReturnType); - - renderWithProviders(); - - expect(screen.getByText(/a{10}\.\.\.a{6}/)).toBeInTheDocument(); + expect(screen.getByRole("tab", { name: /MCP Servers/i })).toHaveTextContent("3"); + expect(screen.getByRole("tab", { name: /Agents/i })).toHaveTextContent("2"); }); it("should display created and last updated timestamps", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx index 9476a8d98af..1eeebe4ebba 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx @@ -2,14 +2,20 @@ import { useAccessGroupDetails } from "@/app/(dashboard)/hooks/accessGroups/useA import { ArrowLeftIcon, BotIcon, EditIcon, KeyIcon, LayersIcon, ServerIcon, UsersIcon } from "lucide-react"; import { useState } from "react"; import DefaultProxyAdminTag from "@/components/common_components/DefaultProxyAdminTag"; +import { BadgeLink } from "@/components/shared/BadgeLink"; import CopyButton from "@/components/shared/CopyButton"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardAction, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { SimpleTooltip } from "@/components/ui/tooltip"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; +import type { components } from "@/lib/http/schema"; +import { keyDetailHref, teamDetailHref } from "@/utils/entityLinks"; import { AccessGroupEditModal } from "./AccessGroupsModal/AccessGroupEditModal"; +type AccessGroupResource = components["schemas"]["AccessGroupResource"]; + interface AccessGroupDetailProps { accessGroupId: string; onBack: () => void; @@ -17,16 +23,24 @@ interface AccessGroupDetailProps { const MAX_PREVIEW = 5; -function ResourceList({ ids, emptyMessage }: { ids: string[]; emptyMessage: string }) { - if (ids.length === 0) { +const shortId = (id: string) => (id.length > 20 ? `${id.slice(0, 10)}...${id.slice(-6)}` : id); + +function ResourceList({ items, emptyMessage }: { items: readonly AccessGroupResource[]; emptyMessage: string }) { + if (items.length === 0) { return

{emptyMessage}

; } return (
- {ids.map((id) => ( + {items.map(({ id, name }) => ( - {id} + {name ? ( + + {name} + + ) : ( + {id} + )} ))} @@ -34,6 +48,23 @@ function ResourceList({ ids, emptyMessage }: { ids: string[]; emptyMessage: stri ); } +function ResourceBadge({ + resource: { id, name }, + href, + fallback, +}: { + resource: AccessGroupResource; + href: string; + fallback: (id: string) => string; +}) { + const badge = ( + + {name ?? fallback(id)} + + ); + return name ? {badge} : badge; +} + export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailProps) { const { data: accessGroup, isLoading } = useAccessGroupDetails(accessGroupId); const [isEditModalVisible, setIsEditModalVisible] = useState(false); @@ -61,14 +92,14 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr ); } - const modelIds = accessGroup.access_model_names ?? []; - const mcpServerIds = accessGroup.access_mcp_server_ids ?? []; - const agentIds = accessGroup.access_agent_ids ?? []; - const keyIds = accessGroup.assigned_key_ids ?? []; - const teamIds = accessGroup.assigned_team_ids ?? []; + const models = accessGroup.access_model_names.map((id) => ({ id, name: null })); + const mcpServers = accessGroup.access_mcp_servers; + const agents = accessGroup.access_agents; + const keys = accessGroup.assigned_keys; + const teams = accessGroup.assigned_teams; - const displayedKeys = showAllKeys ? keyIds : keyIds.slice(0, MAX_PREVIEW); - const displayedTeams = showAllTeams ? teamIds : teamIds.slice(0, MAX_PREVIEW); + const displayedKeys = showAllKeys ? keys : keys.slice(0, MAX_PREVIEW); + const displayedTeams = showAllTeams ? teams : teams.slice(0, MAX_PREVIEW); return (
@@ -129,23 +160,21 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr Attached Keys - {keyIds.length} + {keys.length} - {keyIds.length > MAX_PREVIEW && ( + {keys.length > MAX_PREVIEW && ( )} - {keyIds.length > 0 ? ( + {keys.length > 0 ? (
- {displayedKeys.map((id) => ( - - {id.length > 20 ? `${id.slice(0, 10)}...${id.slice(-6)}` : id} - + {displayedKeys.map((key) => ( + ))}
) : ( @@ -159,23 +188,21 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr Attached Teams - {teamIds.length} + {teams.length} - {teamIds.length > MAX_PREVIEW && ( + {teams.length > MAX_PREVIEW && ( )} - {teamIds.length > 0 ? ( + {teams.length > 0 ? (
- {displayedTeams.map((id) => ( - - {id} - + {displayedTeams.map((team) => ( + id} /> ))}
) : ( @@ -192,27 +219,27 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr Models - {modelIds.length} + {models.length} MCP Servers - {mcpServerIds.length} + {mcpServers.length} Agents - {agentIds.length} + {agents.length} - + - + - + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.integration.test.tsx index 2e65be36796..bd77ad8e897 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.integration.test.tsx @@ -42,6 +42,10 @@ const accessGroup: AccessGroupResponse = { access_agent_ids: ["agent-1"], assigned_team_ids: [], assigned_key_ids: [], + access_mcp_servers: [{ id: "srv-1", name: "Server One" }], + access_agents: [{ id: "agent-1", name: "Agent One" }], + assigned_teams: [], + assigned_keys: [], created_at: "2024-01-01T00:00:00Z", created_by: "user-1", updated_at: "2024-01-02T00:00:00Z", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx index 63ff0f4100f..12d3d773c1f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx @@ -15,6 +15,10 @@ const mockAccessGroups: AccessGroupResponse[] = [ access_agent_ids: ["a1"], assigned_team_ids: [], assigned_key_ids: [], + access_mcp_servers: [{ id: "s1", name: "Server One" }], + access_agents: [{ id: "a1", name: "Agent One" }], + assigned_teams: [], + assigned_keys: [], created_at: "2024-01-15T10:00:00Z", created_by: "user-1", updated_at: "2024-01-20T12:00:00Z", @@ -29,6 +33,10 @@ const mockAccessGroups: AccessGroupResponse[] = [ access_agent_ids: [], assigned_team_ids: [], assigned_key_ids: [], + access_mcp_servers: [], + access_agents: [], + assigned_teams: [], + assigned_keys: [], created_at: "2024-01-10T09:00:00Z", created_by: null, updated_at: "2024-01-12T11:00:00Z", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.test.ts index b15ea4491e9..14cae5b1c1f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.test.ts @@ -46,6 +46,10 @@ const mockAccessGroups: AccessGroupResponse[] = [ access_agent_ids: [], assigned_team_ids: [], assigned_key_ids: [], + access_mcp_servers: [], + access_agents: [], + assigned_teams: [], + assigned_keys: [], created_at: "2025-01-01T00:00:00Z", created_by: "user-1", updated_at: "2025-01-01T00:00:00Z", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.ts index 9f306c21459..b251c019187 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/accessGroups/useAccessGroups.ts @@ -3,23 +3,11 @@ import { createQueryKeys } from "../common/queryKeysFactory"; import { getProxyBaseUrl, getGlobalLitellmHeaderName, deriveErrorMessage, handleError } from "@/components/networking"; import { all_admin_roles } from "@/utils/roles"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import type { components } from "@/lib/http/schema"; // ── Types ──────────────────────────────────────────────────────────────────── -export interface AccessGroupResponse { - access_group_id: string; - access_group_name: string; - description: string | null; - access_model_names: string[]; - access_mcp_server_ids: string[]; - access_agent_ids: string[]; - assigned_team_ids: string[]; - assigned_key_ids: string[]; - created_at: string; - created_by: string | null; - updated_at: string; - updated_by: string | null; -} +export type AccessGroupResponse = components["schemas"]["AccessGroupResponse"]; // ── Query keys (shared across access-group hooks) ──────────────────────────── diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d94b5425ba9..d6459c06b90 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -22764,22 +22764,40 @@ export interface components { /** Spend */ spend?: number | null; }; + /** + * AccessGroupResource + * @description A resource referenced by an access group. `name` is null when the id no longer resolves or has no alias. + */ + AccessGroupResource: { + /** Id */ + id: string; + /** Name */ + name: string | null; + }; /** AccessGroupResponse */ AccessGroupResponse: { /** Access Agent Ids */ access_agent_ids: string[]; + /** Access Agents */ + access_agents: components["schemas"]["AccessGroupResource"][]; /** Access Group Id */ access_group_id: string; /** Access Group Name */ access_group_name: string; /** Access Mcp Server Ids */ access_mcp_server_ids: string[]; + /** Access Mcp Servers */ + access_mcp_servers: components["schemas"]["AccessGroupResource"][]; /** Access Model Names */ access_model_names: string[]; /** Assigned Key Ids */ assigned_key_ids: string[]; + /** Assigned Keys */ + assigned_keys: components["schemas"]["AccessGroupResource"][]; /** Assigned Team Ids */ assigned_team_ids: string[]; + /** Assigned Teams */ + assigned_teams: components["schemas"]["AccessGroupResource"][]; /** * Created At * Format: date-time From b29f9a94bccd10406fb3a78610041fc397a141c1 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 16:09:01 -0700 Subject: [PATCH 45/60] refactor(router): resolve retry policy by exception MRO and add DefaultRetries Replace the hand-ordered isinstance ladder in get_num_retries_from_retry_policy with a class-to-field mapping walked along the exception's MRO, most specific class first. A RetryPolicy field can no longer go silently dead the way InternalServerErrorRetries did, and subclasses such as ContentPolicyViolationError or MidStreamFallbackError pick up their parent's field when they have none of their own. Add a DefaultRetries catch-all so errors without a dedicated field (BadGatewayError, APIConnectionError, NotFoundError, ...) can be governed by the policy too. Specific fields still win over DefaultRetries. Wiring the previously dead InternalServerErrorRetries changes one test expectation: a policy of 2 now overrides a per-deployment num_retries of 5, so the amplification test sees 3 upstream requests instead of 6. Expose DefaultRetries as "All other errors" in the Admin UI retry settings tab and ratchet the lint budgets down by the violations this branch fixed. --- basedpyright-code-budget.json | 8 +- litellm/router_utils/get_retry_from_policy.py | 83 +++++---- litellm/types/router.py | 1 + ruff-strict-budget.json | 2 +- .../test_get_retry_from_policy.py | 169 +++++++++++------- tests/test_litellm/test_router.py | 31 ++-- .../test_router_per_deployment_num_retries.py | 7 +- type-discipline-budget.json | 4 +- .../components/ModelRetrySettingsTab.tsx | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 10 files changed, 179 insertions(+), 129 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 9b59480a0dc..669107bb5b1 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15285 + "limit": 15284 }, "reportMissingTypeStubs": { "limit": 40 @@ -99,7 +99,7 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 44360 + "limit": 44358 }, "reportUnknownLambdaType": { "limit": 109 @@ -108,10 +108,10 @@ "limit": 38309 }, "reportUnknownParameterType": { - "limit": 19622 + "limit": 19621 }, "reportUnknownVariableType": { - "limit": 29846 + "limit": 29844 }, "reportUnnecessaryCast": { "limit": 111 diff --git a/litellm/router_utils/get_retry_from_policy.py b/litellm/router_utils/get_retry_from_policy.py index 051cde127bf..ad4a6b0be99 100644 --- a/litellm/router_utils/get_retry_from_policy.py +++ b/litellm/router_utils/get_retry_from_policy.py @@ -1,8 +1,8 @@ -""" -Get num retries for an exception. +"""Resolve how many retries a RetryPolicy grants for a given exception.""" -- Account for retry policy by exception type. -""" +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import Final from litellm.exceptions import ( AuthenticationError, @@ -15,49 +15,48 @@ from litellm.exceptions import ( ) from litellm.types.router import RetryPolicy +_RETRIES_BY_EXCEPTION_TYPE: Final[Mapping[type, Callable[[RetryPolicy], int | None]]] = MappingProxyType( + { + AuthenticationError: lambda policy: policy.AuthenticationErrorRetries, + Timeout: lambda policy: policy.TimeoutErrorRetries, + RateLimitError: lambda policy: policy.RateLimitErrorRetries, + ContentPolicyViolationError: lambda policy: policy.ContentPolicyViolationErrorRetries, + BadRequestError: lambda policy: policy.BadRequestErrorRetries, + ServiceUnavailableError: lambda policy: policy.ServiceUnavailableErrorRetries, + InternalServerError: lambda policy: policy.InternalServerErrorRetries, + } +) + + +def _resolve_policy( + retry_policy: RetryPolicy | Mapping[str, int | None] | None, + model_group: str | None, + model_group_retry_policy: Mapping[str, RetryPolicy | Mapping[str, int | None]] | None, +) -> RetryPolicy | None: + selected: Final = ( + model_group_retry_policy[model_group] + if model_group_retry_policy is not None and model_group is not None and model_group in model_group_retry_policy + else retry_policy + ) + if isinstance(selected, Mapping): + return RetryPolicy(**selected) + return selected + def get_num_retries_from_retry_policy( exception: Exception, - retry_policy: RetryPolicy | dict | None = None, + retry_policy: RetryPolicy | Mapping[str, int | None] | None = None, model_group: str | None = None, - model_group_retry_policy: dict[str, RetryPolicy] | None = None, -): - """ - BadRequestErrorRetries: Optional[int] = None - AuthenticationErrorRetries: Optional[int] = None - TimeoutErrorRetries: Optional[int] = None - RateLimitErrorRetries: Optional[int] = None - ContentPolicyViolationErrorRetries: Optional[int] = None - InternalServerErrorRetries: Optional[int] = None - ServiceUnavailableErrorRetries: Optional[int] = None - """ - # if we can find the exception then in the retry policy -> return the number of retries - - if model_group_retry_policy is not None and model_group is not None and model_group in model_group_retry_policy: - retry_policy = model_group_retry_policy.get(model_group, None) - - if retry_policy is None: + model_group_retry_policy: Mapping[str, RetryPolicy | Mapping[str, int | None]] | None = None, +) -> int | None: + """Walk the exception's MRO, most specific class first, and return the first configured retry count.""" + policy: Final = _resolve_policy(retry_policy, model_group, model_group_retry_policy) + if policy is None: return None - if isinstance(retry_policy, dict): - retry_policy = RetryPolicy(**retry_policy) - - if isinstance(exception, AuthenticationError) and retry_policy.AuthenticationErrorRetries is not None: - return retry_policy.AuthenticationErrorRetries - if isinstance(exception, Timeout) and retry_policy.TimeoutErrorRetries is not None: - return retry_policy.TimeoutErrorRetries - if isinstance(exception, RateLimitError) and retry_policy.RateLimitErrorRetries is not None: - return retry_policy.RateLimitErrorRetries - if ( - isinstance(exception, ContentPolicyViolationError) - and retry_policy.ContentPolicyViolationErrorRetries is not None - ): - return retry_policy.ContentPolicyViolationErrorRetries - if isinstance(exception, ServiceUnavailableError) and retry_policy.ServiceUnavailableErrorRetries is not None: - return retry_policy.ServiceUnavailableErrorRetries - if isinstance(exception, InternalServerError) and retry_policy.InternalServerErrorRetries is not None: - return retry_policy.InternalServerErrorRetries - if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None: - return retry_policy.BadRequestErrorRetries + configured: Final = ( + _RETRIES_BY_EXCEPTION_TYPE[cls](policy) for cls in type(exception).__mro__ if cls in _RETRIES_BY_EXCEPTION_TYPE + ) + return next((retries for retries in configured if retries is not None), policy.DefaultRetries) def reset_retry_policy() -> RetryPolicy: diff --git a/litellm/types/router.py b/litellm/types/router.py index 6ed9b3efd03..267e8853db1 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -105,6 +105,7 @@ class RetryPolicy(BaseModel): ContentPolicyViolationErrorRetries: int | None = None InternalServerErrorRetries: int | None = None ServiceUnavailableErrorRetries: int | None = None + DefaultRetries: int | None = None OptionalPreCallChecks = list[ diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 4aac1756af4..70408ea022b 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -9,7 +9,7 @@ "limit": 809 }, "ANN201": { - "limit": 1999 + "limit": 1998 }, "ANN202": { "limit": 835 diff --git a/tests/test_litellm/router_utils/test_get_retry_from_policy.py b/tests/test_litellm/router_utils/test_get_retry_from_policy.py index a5e239b8595..df157ea5ff7 100644 --- a/tests/test_litellm/router_utils/test_get_retry_from_policy.py +++ b/tests/test_litellm/router_utils/test_get_retry_from_policy.py @@ -1,102 +1,147 @@ +from types import MappingProxyType +from typing import Final + +import pytest + import litellm -from litellm.router_utils.get_retry_from_policy import ( - get_num_retries_from_retry_policy, -) +from litellm.router_utils.get_retry_from_policy import get_num_retries_from_retry_policy from litellm.types.router import RetryPolicy +_EXCEPTION_FOR_FIELD: Final = MappingProxyType( + { + "BadRequestErrorRetries": litellm.BadRequestError, + "AuthenticationErrorRetries": litellm.AuthenticationError, + "TimeoutErrorRetries": litellm.Timeout, + "RateLimitErrorRetries": litellm.RateLimitError, + "ContentPolicyViolationErrorRetries": litellm.ContentPolicyViolationError, + "InternalServerErrorRetries": litellm.InternalServerError, + "ServiceUnavailableErrorRetries": litellm.ServiceUnavailableError, + } +) -def _service_unavailable_error() -> litellm.ServiceUnavailableError: - return litellm.ServiceUnavailableError( - message="model is down", - llm_provider="openai", - model="gpt-5.6", +_SPECIFIC_FIELDS: Final = tuple(name for name in RetryPolicy.model_fields if name != "DefaultRetries") + + +def _error(exception_type: type[Exception]) -> Exception: + return exception_type(message="boom", llm_provider="openai", model="gpt-5.6") + + +@pytest.mark.parametrize("field", _SPECIFIC_FIELDS) +def test_every_specific_field_controls_retries_for_its_exception(field: str): + exception: Final = _error(_EXCEPTION_FOR_FIELD[field]) + + assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(**{field: 0})) == 0 + assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(**{field: 4})) == 4 + + +@pytest.mark.parametrize("field", _SPECIFIC_FIELDS) +def test_specific_field_does_not_apply_to_unrelated_exceptions(field: str): + policy: Final = RetryPolicy(**{field: 0}) + unrelated: Final = tuple( + exception_type + for name, exception_type in _EXCEPTION_FOR_FIELD.items() + if name != field and not issubclass(exception_type, _EXCEPTION_FOR_FIELD[field]) ) - -def _internal_server_error() -> litellm.InternalServerError: - return litellm.InternalServerError( - message="upstream 500", - llm_provider="openai", - model="gpt-5.6", - ) + for exception_type in unrelated: + assert get_num_retries_from_retry_policy(exception=_error(exception_type), retry_policy=policy) is None -def test_service_unavailable_error_retries_honored(): - policy = RetryPolicy(ServiceUnavailableErrorRetries=0) +def test_subclass_prefers_its_own_field_over_the_parent_field(): + policy: Final = RetryPolicy(BadRequestErrorRetries=5, ContentPolicyViolationErrorRetries=1) assert ( - get_num_retries_from_retry_policy( - exception=_service_unavailable_error(), - retry_policy=policy, - ) - == 0 + get_num_retries_from_retry_policy(exception=_error(litellm.ContentPolicyViolationError), retry_policy=policy) + == 1 + ) + assert get_num_retries_from_retry_policy(exception=_error(litellm.BadRequestError), retry_policy=policy) == 5 + + +def test_subclass_falls_back_to_the_parent_field(): + policy: Final = RetryPolicy(BadRequestErrorRetries=5) + + assert ( + get_num_retries_from_retry_policy(exception=_error(litellm.ContentPolicyViolationError), retry_policy=policy) + == 5 ) -def test_service_unavailable_error_retries_nonzero(): - policy = RetryPolicy(ServiceUnavailableErrorRetries=4) +@pytest.mark.parametrize("exception_type", (litellm.BadGatewayError, litellm.NotFoundError)) +def test_default_retries_covers_exceptions_without_a_specific_field(exception_type: type[Exception]): + exception: Final = _error(exception_type) + assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(DefaultRetries=0)) == 0 assert ( get_num_retries_from_retry_policy( - exception=_service_unavailable_error(), - retry_policy=policy, - ) - == 4 - ) - - -def test_internal_server_error_retries_honored(): - policy = RetryPolicy(InternalServerErrorRetries=0) - - assert ( - get_num_retries_from_retry_policy( - exception=_internal_server_error(), - retry_policy=policy, - ) - == 0 - ) - - -def test_service_unavailable_not_covered_by_internal_server_error_retries(): - policy = RetryPolicy(InternalServerErrorRetries=0) - - assert ( - get_num_retries_from_retry_policy( - exception=_service_unavailable_error(), - retry_policy=policy, + exception=exception, retry_policy=RetryPolicy(ServiceUnavailableErrorRetries=0) ) is None ) -def test_internal_server_error_not_covered_by_service_unavailable_retries(): - policy = RetryPolicy(ServiceUnavailableErrorRetries=0) +def test_specific_field_wins_over_default_retries(): + policy: Final = RetryPolicy(DefaultRetries=0, RateLimitErrorRetries=3) + + assert get_num_retries_from_retry_policy(exception=_error(litellm.RateLimitError), retry_policy=policy) == 3 + assert get_num_retries_from_retry_policy(exception=_error(litellm.BadGatewayError), retry_policy=policy) == 0 + + +def test_default_retries_applies_when_the_specific_field_is_unset(): + policy: Final = RetryPolicy(DefaultRetries=2) assert ( - get_num_retries_from_retry_policy( - exception=_internal_server_error(), - retry_policy=policy, - ) - is None + get_num_retries_from_retry_policy(exception=_error(litellm.ServiceUnavailableError), retry_policy=policy) == 2 ) -def test_service_unavailable_error_retries_from_dict_policy(): +def test_empty_policy_matches_nothing(): + assert ( + get_num_retries_from_retry_policy(exception=_error(litellm.ServiceUnavailableError), retry_policy=RetryPolicy()) + is None + ) + assert ( + get_num_retries_from_retry_policy(exception=_error(litellm.ServiceUnavailableError), retry_policy=None) is None + ) + + +def test_dict_policy_is_accepted(): assert ( get_num_retries_from_retry_policy( - exception=_service_unavailable_error(), + exception=_error(litellm.ServiceUnavailableError), retry_policy={"ServiceUnavailableErrorRetries": 0}, ) == 0 ) -def test_service_unavailable_error_retries_from_model_group_policy(): +def test_model_group_policy_replaces_the_global_policy(): + exception: Final = _error(litellm.ServiceUnavailableError) + global_policy: Final = RetryPolicy(ServiceUnavailableErrorRetries=5) + assert ( get_num_retries_from_retry_policy( - exception=_service_unavailable_error(), + exception=exception, + retry_policy=global_policy, model_group="gpt-5.6", - model_group_retry_policy={"gpt-5.6": RetryPolicy(ServiceUnavailableErrorRetries=1)}, + model_group_retry_policy={"gpt-5.6": {"ServiceUnavailableErrorRetries": 1}}, ) == 1 ) + assert ( + get_num_retries_from_retry_policy( + exception=exception, + retry_policy=global_policy, + model_group="gpt-5.6", + model_group_retry_policy={"gpt-5.6": RetryPolicy(RateLimitErrorRetries=1)}, + ) + is None + ) + assert ( + get_num_retries_from_retry_policy( + exception=exception, + retry_policy=global_policy, + model_group="other-group", + model_group_retry_policy={"gpt-5.6": RetryPolicy(ServiceUnavailableErrorRetries=1)}, + ) + == 5 + ) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index cb3baf042dd..5d83d0f8877 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -12896,10 +12896,17 @@ async def test_prompt_management_factory_marks_injection_for_every_deployment(mo @pytest.mark.asyncio -@pytest.mark.parametrize("policy_retries,expected_calls", [(0, 1), (1, 2)]) -async def test_router_retry_policy_service_unavailable_retries(policy_retries, expected_calls): - from litellm.types.router import RetryPolicy - +@pytest.mark.parametrize( + "retry_policy,error_type,expected_calls", + [ + ({"ServiceUnavailableErrorRetries": 0}, litellm.ServiceUnavailableError, 1), + ({"ServiceUnavailableErrorRetries": 1}, litellm.ServiceUnavailableError, 2), + ({"InternalServerErrorRetries": 0}, litellm.InternalServerError, 1), + ({"DefaultRetries": 0}, litellm.BadGatewayError, 1), + ({"DefaultRetries": 0, "ServiceUnavailableErrorRetries": 1}, litellm.ServiceUnavailableError, 2), + ], +) +async def test_router_retry_policy_controls_attempt_count(retry_policy, error_type, expected_calls): router = litellm.Router( model_list=[ { @@ -12907,20 +12914,14 @@ async def test_router_retry_policy_service_unavailable_retries(policy_retries, e "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, } ], - retry_policy=RetryPolicy(ServiceUnavailableErrorRetries=policy_retries), + num_retries=2, + retry_policy=retry_policy, disable_cooldowns=True, ) + error = error_type(message="model is down", llm_provider="openai", model="gpt-5.6") - error = litellm.ServiceUnavailableError( - message="model is down", - llm_provider="openai", - model="gpt-5.6", - ) with patch.object(litellm, "acompletion", AsyncMock(side_effect=error)) as mock_acompletion: - with pytest.raises(litellm.ServiceUnavailableError): - await router.acompletion( - model="gpt-5.6", - messages=[{"role": "user", "content": "hi"}], - ) + with pytest.raises(error_type): + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) assert mock_acompletion.call_count == expected_calls diff --git a/tests/test_litellm/test_router_per_deployment_num_retries.py b/tests/test_litellm/test_router_per_deployment_num_retries.py index d75e32a1821..99ad7c224f8 100644 --- a/tests/test_litellm/test_router_per_deployment_num_retries.py +++ b/tests/test_litellm/test_router_per_deployment_num_retries.py @@ -415,8 +415,9 @@ class TestNoProviderRetryAmplification: @pytest.mark.asyncio async def test_retry_policy_configured_does_not_reintroduce_amplification(self): """ - With a retry policy configured alongside a per-deployment ``num_retries=5``, the - provider SDK still must not retry: exactly ``6`` upstream requests, not 36. + ``InternalServerErrorRetries=2`` overrides the per-deployment ``num_retries=5`` for the + 500s this upstream returns, and the provider SDK still must not retry on top: exactly + ``3`` upstream requests, not 18. """ router = self._router( "https://policy.local/v1", @@ -424,7 +425,7 @@ class TestNoProviderRetryAmplification: num_retries=1, retry_policy=RetryPolicy(InternalServerErrorRetries=2), ) - assert await self._call_and_count(router) == 6 + assert await self._call_and_count(router) == 3 @pytest.mark.asyncio async def test_global_num_retries_not_amplified(self): diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 8589a9451cf..3d01c08e8eb 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 22328 + "limit": 22326 }, "LIT002": { "limit": 26748 @@ -30,7 +30,7 @@ "limit": 16468 }, "LIT011": { - "limit": 5514 + "limit": 5512 }, "LIT012": { "limit": 4487 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx index 9d6501c97ba..069a3f27beb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx @@ -35,6 +35,7 @@ const retryPolicyMap: Record = { "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", "InternalServerError (500)": "InternalServerErrorRetries", "ServiceUnavailableError (503)": "ServiceUnavailableErrorRetries", + "All other errors": "DefaultRetries", }; const isValidRetryCount = (value: number) => Number.isFinite(value) && Number.isInteger(value) && value >= 0; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 549b9c0d01d..7d7fa8d7fe6 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35014,6 +35014,8 @@ export interface components { BadRequestErrorRetries?: number | null; /** Contentpolicyviolationerrorretries */ ContentPolicyViolationErrorRetries?: number | null; + /** Defaultretries */ + DefaultRetries?: number | null; /** Internalservererrorretries */ InternalServerErrorRetries?: number | null; /** Ratelimiterrorretries */ From 544822b1a8afc96c3cc471a9de8e57035a951e65 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 4 Sep 2026 16:14:28 -0700 Subject: [PATCH 46/60] fix: escalate a stalled keyword-forced tier, and let a blocked toggle clear Two issues Bugbot found on #39809. A keyword_tier_rule forces its tier and returns before any classification runs, so stall escalation never reached that path even though keyword escalation did. That left the one path that can pin a weak model to a whole conversation as the one path a stall could not lift. Stall detection now resolves before the override branch and both paths bump. The dashboard switch disabled itself whenever session pinning or user-turn classification was on, including for a router that already had stall escalation enabled. The conflicting keys stayed set, the backend rejected the save, and the disabled switch was the only way to clear them. It now disables only the off-to-on direction. --- .../complexity_router/complexity_router.py | 22 ++++++++------- .../router_strategy/test_complexity_router.py | 27 +++++++++++++++++++ .../add_model/StallEscalationConfig.test.tsx | 7 +++++ .../add_model/StallEscalationConfig.tsx | 5 +++- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 01b4665a7d9..5815f7577b9 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3053,6 +3053,14 @@ class ComplexityRouter(CustomLogger): newest_ask: Final = _newest_turn_ask(resolved_messages, self._reminder_markers) escalation_keyword: Final = self._matched_escalation_keyword(newest_ask) if newest_ask is not None else None + # Resolved here rather than beside the classifier because the keyword-override path below + # returns before any classification runs, and a forced tier gets stuck for the same reason + # a classified one does. + stalled: Final = self.config.stall_escalation_enabled and detect_stalled_task( + resolved_messages, + window=self.config.stall_escalation_window, + repeat_threshold=self.config.stall_escalation_repeat_threshold, + ) plan_mode_sentinel: Final = self._matched_plan_mode_signal(request_kwargs, resolved_messages) plan_floor: Final = self._resolve_plan_mode_floor() if plan_mode_sentinel is not None else None @@ -3082,10 +3090,11 @@ class ComplexityRouter(CustomLogger): override: Final = await self._resolve_keyword_tier_override(user_message, request_kwargs) if override is not None: - escalated_tier: Final = ( + keyword_bumped_tier: Final = ( self._escalate_tier(override.tier) if escalation_keyword is not None else override.tier ) - keyword_escalated: Final = escalated_tier != override.tier + escalated_tier: Final = self._escalate_tier(keyword_bumped_tier) if stalled else keyword_bumped_tier + keyword_escalated: Final = keyword_bumped_tier != override.tier routed_tier: Final = ( self._apply_plan_mode_floor(escalated_tier) if plan_floor is not None else escalated_tier ) @@ -3113,6 +3122,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=conversation_continuing, cause=keyword_cause, tier=routed_tier, + signals=("stall_escalation",) if stalled else None, matched_keyword=plan_mode_sentinel if keyword_plan_floored else override.matched_keyword, escalation_keyword=escalation_keyword, escalated=keyword_escalated, @@ -3136,14 +3146,6 @@ class ComplexityRouter(CustomLogger): escalated: Final = tier != classified_tier if escalated: signals = (*signals, "escalation") - # Recomputed from this request's own tool calls, not remembered from a prior turn: the - # bump lasts only as long as the recent tool calls still look stuck, and lifts itself - # the moment they don't, with nothing to expire or leak past the task that earned it. - stalled: Final = self.config.stall_escalation_enabled and detect_stalled_task( - resolved_messages, - window=self.config.stall_escalation_window, - repeat_threshold=self.config.stall_escalation_repeat_threshold, - ) if stalled: tier = self._escalate_tier(tier) signals = (*signals, "stall_escalation") diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3ae3165bf62..f2736492d30 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5990,6 +5990,33 @@ class TestStallEscalation: result = await router.async_pre_routing_hook(model="test-model", request_kwargs={}, messages=messages) assert result.model == "claude-sonnet-4-20250514" # SIMPLE -> MEDIUM (keyword) -> COMPLEX (stall) + @pytest.mark.asyncio + async def test_a_keyword_forced_tier_still_escalates_when_stalled(self, mock_router_instance, basic_config): + """A keyword rule forces its tier and returns before any classification runs, so + without its own bump the one path that can pin a weak model to a whole conversation + would be the one path a stall could never lift.""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **basic_config, + "stall_escalation_enabled": True, + "keyword_tier_rules": [{"keywords": ["billing"], "tier": "SIMPLE"}], + }, + ) + healthy = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "a billing question"}] + ) + assert healthy.model == "gpt-4o-mini" # forced SIMPLE, nothing stuck + + stalled = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[*_stalled_tool_history(), {"role": "user", "content": "a billing question"}], + ) + assert stalled.model == "gpt-4o" # forced SIMPLE bumped to MEDIUM + assert "stall_escalation" in stalled.routing_decision["signals"] + @pytest.mark.asyncio async def test_evidence_survives_a_new_human_ask(self, mock_router_instance, basic_config): """A plain follow-up like 'try again' must not erase the stall evidence that came diff --git a/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.test.tsx index 2c346306307..f215849c1a3 100644 --- a/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.test.tsx @@ -105,4 +105,11 @@ describe("StallEscalationConfig", () => { renderConfig({ stall_escalation_enabled: true, session_affinity: true }); expect(screen.queryByLabelText("Repeats before escalating")).not.toBeInTheDocument(); }); + + it("still lets an already-on router turn it off once a blocker appears, which the save needs", () => { + const onChange = renderConfig({ stall_escalation_enabled: true, session_affinity: true }); + expect(toggle()).not.toHaveAttribute("aria-disabled", "true"); + fireEvent.click(toggle()); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ stall_escalation_enabled: undefined })); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.tsx b/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.tsx index fdb8c30f3b8..6f2a9cce365 100644 --- a/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/StallEscalationConfig.tsx @@ -65,7 +65,10 @@ const StallEscalationConfig: React.FC<{
From 541ab50c043be762fb73d73cf2ae648235e06112 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 16:23:08 -0700 Subject: [PATCH 47/60] test(router): fake the upstream with respx in the retry policy attempt test The test-quality gate rejects patching litellm.acompletion, and faking the HTTP boundary is the stronger test anyway: the 503, 500 and 502 responses now travel through the real OpenAI SDK and exception mapping before the router decides how many times to retry. Adds a case showing that a 503 key does not govern a 502. --- tests/test_litellm/test_router.py | 39 ++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 5d83d0f8877..31eb46f1458 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import openai import pytest +import respx @@ -567,7 +568,6 @@ async def test_async_router_acancel_batch_does_not_fall_back_across_model_groups model string, and the fallback provider is then asked to cancel a batch it never issued, which can only answer not-found. The router re-raises the owner's error after that wasted round trip, so the pin's observable is the foreign call never happening.""" - import respx monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) router = litellm.Router( @@ -716,7 +716,6 @@ async def test_async_router_acreate_file_litellm_proxy_sends_target_model_names_ from io import BytesIO import httpx - import respx jsonl_file = BytesIO( json.dumps({"body": {"model": "chained-batch", "messages": [{"role": "user", "content": "hi"}]}}).encode( @@ -12897,31 +12896,45 @@ async def test_prompt_management_factory_marks_injection_for_every_deployment(mo @pytest.mark.asyncio @pytest.mark.parametrize( - "retry_policy,error_type,expected_calls", + "retry_policy,upstream_status,error_type,expected_upstream_calls", [ - ({"ServiceUnavailableErrorRetries": 0}, litellm.ServiceUnavailableError, 1), - ({"ServiceUnavailableErrorRetries": 1}, litellm.ServiceUnavailableError, 2), - ({"InternalServerErrorRetries": 0}, litellm.InternalServerError, 1), - ({"DefaultRetries": 0}, litellm.BadGatewayError, 1), - ({"DefaultRetries": 0, "ServiceUnavailableErrorRetries": 1}, litellm.ServiceUnavailableError, 2), + ({"ServiceUnavailableErrorRetries": 0}, 503, litellm.ServiceUnavailableError, 1), + ({"ServiceUnavailableErrorRetries": 1}, 503, litellm.ServiceUnavailableError, 2), + ({"InternalServerErrorRetries": 0}, 500, litellm.InternalServerError, 1), + ({"DefaultRetries": 0}, 502, litellm.BadGatewayError, 1), + ({"DefaultRetries": 0, "ServiceUnavailableErrorRetries": 1}, 503, litellm.ServiceUnavailableError, 2), + ({"ServiceUnavailableErrorRetries": 0}, 502, litellm.BadGatewayError, 3), ], ) -async def test_router_retry_policy_controls_attempt_count(retry_policy, error_type, expected_calls): +async def test_router_retry_policy_controls_upstream_attempt_count( + monkeypatch: pytest.MonkeyPatch, retry_policy, upstream_status, error_type, expected_upstream_calls +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) router = litellm.Router( model_list=[ { "model_name": "gpt-5.6", - "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "api_base": "https://retry-policy.local/v1", + }, } ], num_retries=2, retry_policy=retry_policy, disable_cooldowns=True, ) - error = error_type(message="model is down", llm_provider="openai", model="gpt-5.6") - with patch.object(litellm, "acompletion", AsyncMock(side_effect=error)) as mock_acompletion: + with respx.mock(assert_all_called=True) as respx_mock: + upstream = respx_mock.post("https://retry-policy.local/v1/chat/completions").mock( + return_value=httpx.Response( + upstream_status, + headers={"retry-after": "0"}, + json={"error": {"message": "model is down", "type": "server_error"}}, + ) + ) with pytest.raises(error_type): await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) - assert mock_acompletion.call_count == expected_calls + assert upstream.call_count == expected_upstream_calls From 976f8625f34c9c0eb7ac5e5976493dde2c4cd997 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:42:29 -0700 Subject: [PATCH 48/60] test(proxy): cover default-tier end-user counter reset with rollover --- .../common_utils/test_reset_budget_job.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index bc9926a314f..56c0efb41d2 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -3054,6 +3054,38 @@ def test_budget_cascade_carries_enduser_overage_when_rollover_enabled( } in enduser_writes +def test_budget_cascade_carries_default_tier_enduser_counter_when_rollover_enabled( + rollover_enabled, reset_budget_job, mock_prisma_client, monkeypatch +): + """An end user on the default budget (no budget_id on its row) 5 over the cap + keeps a counter of 5 in the next window and loses its cached object.""" + import litellm + + counter_cache: Final = _make_counter_invalidation_job(monkeypatch) + monkeypatch.setattr(litellm, "max_end_user_budget_id", "default-enduser-budget") + mock_prisma_client.data["budget"] = [ + _budget_row(budget_id="default-enduser-budget", budget_duration="1d", max_budget=10.0) + ] + implicit_enduser: Final = type( + "EndUserRow", + (), + { + "spend": 15.0, + "user_id": "enduser-implicit", + "budget_id": None, + "model_dump": lambda self=None: {"spend": 15.0, "user_id": "enduser-implicit", "budget_id": None, "blocked": False}, + }, + ) + mock_prisma_client.db.litellm_endusertable.set_find_many_results([implicit_enduser]) + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:end_user:enduser-implicit", value=5.0, ttl=60) + counter_cache.redis_cache.async_set_cache.assert_any_await(key="spend:end_user:enduser-implicit", value=5.0, ttl=60) + deleted: Final = {call.kwargs.get("key") for call in counter_cache.user_api_key_cache.async_delete_cache.await_args_list} + assert "end_user_id:enduser-implicit" in deleted + + def _replay_spend_writes(writes, spend): """Apply the queued update_many statements in order, the way the DB transaction executes them, and return the row's final spend.""" From 3202963f25482bb3a2fe312f93f1f12291f7d60e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:57:40 -0700 Subject: [PATCH 49/60] feat(cost-map): add azure/gpt-6-astra and azure/us/gpt-6-astra Foundry pricing --- ...odel_prices_and_context_window_backup.json | 94 +++++++++++++++++++ model_prices_and_context_window.json | 94 +++++++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 43 +++++++++ .../test_reasoning_effort_capability.py | 16 ++++ 4 files changed, 247 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7f5038e3073..d8b7287eba7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -7157,6 +7157,53 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure/gpt-6-astra": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "litellm_provider": "azure", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "azure/us/gpt-5.6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, @@ -7376,6 +7423,53 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure/us/gpt-6-astra": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "litellm_provider": "azure", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "azure/eu/gpt-5.6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7f5038e3073..d8b7287eba7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7157,6 +7157,53 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure/gpt-6-astra": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, + "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "litellm_provider": "azure", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "azure/us/gpt-5.6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, @@ -7376,6 +7423,53 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure/us/gpt-6-astra": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, + "cache_read_input_token_cost": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, + "input_cost_per_token": 1.1e-05, + "input_cost_per_token_above_272k_tokens": 2.2e-05, + "litellm_provider": "azure", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "output_cost_per_token_above_272k_tokens": 8.25e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "azure/eu/gpt-5.6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0f8084643ea..df680b7cb0e 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2008,6 +2008,49 @@ def test_generic_cost_per_token_azure_gpt56(_local_model_cost_map, assert round(completion_cost, 10) == round(output_cost * completion_tokens, 10) +@pytest.mark.parametrize("model,zone_multiplier", [("azure/gpt-6-astra", 1.0), ("azure/us/gpt-6-astra", 1.1)]) +@pytest.mark.parametrize( + "prompt_tokens,input_side_multiplier,output_multiplier", + [(100000, 1.0, 1.0), (300000, 2.0, 1.5)], +) +def test_generic_cost_per_token_azure_gpt_6_astra_foundry_price_sheet( + _local_model_cost_map, + model, + zone_multiplier, + prompt_tokens, + input_side_multiplier, + output_multiplier, +): + """Microsoft Foundry sells gpt-6-astra at the OpenAI rates: $10 input, $1 cache read, $12.50 cache write, + $50 output per 1M tokens on Standard Global, with the input side doubling and output 1.5x above 272K + prompt tokens. Standard US Data Zone carries the usual 10% uplift on every rate. + """ + cached_tokens = 50000 + cache_write_tokens = 40000 + text_tokens = prompt_tokens - cached_tokens - cache_write_tokens + completion_tokens = 1000 + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=cached_tokens, cache_write_tokens=cache_write_tokens + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider="azure", + ) + + input_side = zone_multiplier * input_side_multiplier + assert prompt_cost == pytest.approx( + input_side * (text_tokens * 1e-5 + cached_tokens * 1e-6 + cache_write_tokens * 1.25e-5) + ) + assert completion_cost == pytest.approx(zone_multiplier * output_multiplier * completion_tokens * 5e-5) + + @pytest.mark.parametrize( "model,expected_none,expected_xhigh,expected_minimal", [ diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py index 7b2e45ab3ed..504e87fb231 100644 --- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py +++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py @@ -388,3 +388,19 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels: "xhigh", "max", ) + + @pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"]) + def test_a_foundry_deployment_advertises_the_same_levels(self, local_model_cost_map, model): + """Microsoft Foundry serves the same model, so an Azure deployment must offer low + through max and never none, exactly like the OpenAI entry.""" + from litellm.utils import _get_model_info_helper + + model_info = dict(_get_model_info_helper(model=model, custom_llm_provider="azure")) + + assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( + "low", + "medium", + "high", + "xhigh", + "max", + ) From 51514b9123a6569a0857bd7e941726c16def9bd2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:29:55 -0700 Subject: [PATCH 50/60] fix(cost-map): azure/gpt-6-astra accepts reasoning_effort none on Foundry --- ...odel_prices_and_context_window_backup.json | 4 +-- model_prices_and_context_window.json | 4 +-- .../chat/test_azure_gpt5_transformation.py | 28 +++++++++++++++++++ .../test_reasoning_effort_capability.py | 8 ++++-- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d8b7287eba7..2459ed940e0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -7192,7 +7192,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": false, + "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -7458,7 +7458,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": false, + "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d8b7287eba7..2459ed940e0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7192,7 +7192,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": false, + "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -7458,7 +7458,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": false, + "supports_none_reasoning_effort": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index bd0f16a695b..3b3ef1a9cd4 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -348,3 +348,31 @@ def test_azure_gpt_6_astra_takes_the_reasoning_series_request_shape(): assert params["max_completion_tokens"] == 100 assert "max_tokens" not in params assert params["reasoning_effort"] == "max" + + +@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"]) +def test_azure_gpt6_astra_reasoning_effort_none_unlocks_temperature(config: AzureOpenAIGPT5Config, model: str): + """Foundry's gpt-6-astra accepts reasoning_effort='none' and, only then, a non-default + temperature (verified live against a Foundry deployment), unlike OpenAI's gpt-6-astra.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.2, "reasoning_effort": "none"}, + optional_params={}, + model=model, + drop_params=False, + api_version="2025-04-01-preview", + ) + assert params["temperature"] == 0.2 + assert params["reasoning_effort"] == "none" + + +@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"]) +def test_azure_gpt6_astra_rejects_reasoning_effort_minimal(config: AzureOpenAIGPT5Config, model: str): + """Foundry's gpt-6-astra lists none, low, medium, high, xhigh and max but not minimal.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model=model, + drop_params=False, + api_version="2025-04-01-preview", + ) diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py index 504e87fb231..f181370455d 100644 --- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py +++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py @@ -390,14 +390,16 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels: ) @pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"]) - def test_a_foundry_deployment_advertises_the_same_levels(self, local_model_cost_map, model): - """Microsoft Foundry serves the same model, so an Azure deployment must offer low - through max and never none, exactly like the OpenAI entry.""" + def test_a_foundry_deployment_also_advertises_none(self, local_model_cost_map, model): + """Microsoft Foundry serves the same model but its API accepts reasoning_effort none + (verified live: 200 with zero reasoning tokens, and it unlocks temperature), which + OpenAI's rejects, so an Azure deployment offers none on top of low through max.""" from litellm.utils import _get_model_info_helper model_info = dict(_get_model_info_helper(model=model, custom_llm_provider="azure")) assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( + "none", "low", "medium", "high", From 59d42d36e6a0a9a81baef774ad900a8e9889722b Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Fri, 4 Sep 2026 17:34:25 -0700 Subject: [PATCH 51/60] fix(headroom): bound the /v1/compress and /v1/retrieve calls with a timeout (#39527) * fix(headroom): bound the /v1/compress and /v1/retrieve calls with a timeout The headroom guardrail builds its client with get_async_httpx_client(GuardrailCallback) and no params, and passes no timeout on either outbound call. That client's read, write and pool legs are 600s (litellm.request_timeout when set explicitly, default 6000s), so an unreachable or stalled compression service holds the caller's pre-call request open for the whole window before unreachable_fallback ever runs. Because the client is shared with every other no-params guardrail, each stalled call also pins a pooled connection for the same window, so a saturated pool makes unrelated requests block on the pool leg. Bound both calls at 60s by default, honoring litellm_params.timeout when set (the field already exists and documents itself as the per-guardrail API timeout; headroom accepted it and ignored it). The connect leg stays at the http_handler default, or the configured budget when that is shorter, so a dead host still fails fast. Live on a proxy against a stalled /v1/compress: 600.4s -> 60.2s before the 502, and 5.2s with timeout: 5 configured. * fix(headroom): reject non-finite timeouts and trim the timeout commentary `timeout: .inf` on a Headroom guardrail reached httpx and the aiohttp transport raised OverflowError, so every request came back as a raw 500 instead of going through unreachable_fallback. Reject non-finite values the same way as non-positive ones, and cut the comments and docstrings back to what the code does not already say. --- .../guardrail_hooks/headroom/__init__.py | 1 + .../guardrail_hooks/headroom/headroom.py | 33 ++++ .../guardrail_hooks/test_headroom.py | 156 +++++++++++++++++- 3 files changed, 187 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py index cffef84e966..d569802ce89 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/__init__.py @@ -35,6 +35,7 @@ def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) -> event_hook=_coerce_event_hook(litellm_params.mode), default_on=litellm_params.default_on or False, unreachable_fallback=litellm_params.unreachable_fallback, + timeout=litellm_params.timeout, ) litellm.logging_callback_manager.add_litellm_callback( # pyright: ignore[reportUnknownMemberType] _callback diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index 9d993384461..685b90f1754 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import math import re import time import uuid @@ -15,6 +16,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger from litellm.compression.compress import get_protected_indices +from litellm.constants import HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, @@ -47,12 +49,16 @@ from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.guardrails import LitellmParams from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel BYPASS_HEADER: Final = "x-headroom-bypass" _STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset( (CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses) ) +# The shared GuardrailCallback client carries no per-call bound, so without this a +# stalled service holds the caller's request and a pooled connection for 600s or more. +_COMPRESS_TIMEOUT_SECONDS: Final = 60.0 HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve" _HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})") _HASH_CACHE_TTL_SECONDS: Final = 15 * 60 @@ -472,6 +478,7 @@ class HeadroomGuardrail(CustomGuardrail): event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | Mode | None = None, default_on: bool = False, unreachable_fallback: str | None = None, + timeout: float | None = None, ): self.headroom_api_base = (api_base or get_secret_str("HEADROOM_API_BASE") or "").rstrip("/") if not self.headroom_api_base: @@ -484,6 +491,7 @@ class HeadroomGuardrail(CustomGuardrail): self.unreachable_fallback: Literal["fail_closed", "fail_open"] = ( "fail_open" if unreachable_fallback == "fail_open" else "fail_closed" ) + self.timeout: httpx.Timeout = self._resolve_timeout(timeout) self.async_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback, ) @@ -511,6 +519,29 @@ class HeadroomGuardrail(CustomGuardrail): headers["Authorization"] = f"Bearer {self.headroom_api_key}" return headers + @staticmethod + def _resolve_timeout(timeout: float | None) -> httpx.Timeout: + """Budget for one call to the compression service, unset meaning the default. + + Zero, negative and non-finite values are rejected instead of passed through: + httpx accepts them, and the transport then reads 0 and inf as no deadline at + all and a negative one as a deadline already past. + """ + rejected: Final = timeout is not None and not (math.isfinite(timeout) and timeout > 0) + if rejected: + verbose_proxy_logger.warning( + "Headroom: ignoring unusable timeout %s, using %s seconds", + timeout, + _COMPRESS_TIMEOUT_SECONDS, + ) + seconds: Final = _COMPRESS_TIMEOUT_SECONDS if timeout is None or rejected else timeout + return httpx.Timeout(timeout=seconds, connect=min(seconds, HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS)) + + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + """Re-resolve the timeout, which the base implementation would otherwise null out.""" + super().update_in_memory_litellm_params(litellm_params) + self.timeout = self._resolve_timeout(litellm_params.timeout) + def _prune_expired_hashes(self) -> None: now: Final = time.monotonic() self._issued_hashes_by_call_id = { @@ -548,6 +579,7 @@ class HeadroomGuardrail(CustomGuardrail): url=f"{self.headroom_api_base}/v1/compress", json=payload, headers=self._request_headers(), + timeout=self.timeout, ) except httpx.HTTPStatusError as e: return ( @@ -685,6 +717,7 @@ class HeadroomGuardrail(CustomGuardrail): url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}", params=params, headers=self._request_headers(), + timeout=self.timeout, ) except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError, litellm.Timeout) as e: verbose_proxy_logger.warning("Headroom: retrieve failed for hash=%s: %s", hash_value, e) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py index 400eaf8ab3f..a49d7723bcc 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -1694,8 +1694,6 @@ async def test_apply_guardrail_litellm_timeout_fail_open_forwards_uncompressed() assert result["structured_messages"] == ORIGINAL_MESSAGES - - # --------------------------------------------------------------------------- # Content-parts flattening (LIT-4795) # @@ -2669,7 +2667,9 @@ async def _plan_for(guardrail: HeadroomGuardrail, response, messages: list): return_value=_make_retrieve_response("ORIGINAL CONTENT"), ): return await guardrail.async_build_agentic_loop_plan( - tools={"tool_calls": [{"id": "call_1", "name": HEADROOM_RETRIEVE_TOOL_NAME, "arguments": {"hash": "h" * 24}}]}, + tools={ + "tool_calls": [{"id": "call_1", "name": HEADROOM_RETRIEVE_TOOL_NAME, "arguments": {"hash": "h" * 24}}] + }, model="claude-sonnet-4-5-20250929", messages=messages, response=response, @@ -2732,3 +2732,153 @@ async def test_chat_followup_echoes_only_the_retrieve_call(guardrail: HeadroomGu assert assistant["content"] == "Getting the original first." assert [tc["id"] for tc in assistant["tool_calls"]] == ["call_1"] assert [m["tool_call_id"] for m in messages[2:]] == ["call_1"] + + +# --- LIT-5881: the calls to the compression service must be time-bounded --- + + +def _timeout_of(mock_call) -> httpx.Timeout: + timeout = mock_call.kwargs["timeout"] + assert isinstance(timeout, httpx.Timeout), timeout + return timeout + + +@pytest.mark.asyncio +async def test_compress_call_passes_bounded_timeout(guardrail: HeadroomGuardrail): + """Without an explicit timeout the call inherits the shared client's 600s read leg.""" + inputs = GenericGuardrailAPIInputs(texts=["A" * 5000], structured_messages=ORIGINAL_MESSAGES) + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=_make_compress_response(COMPRESSED_MESSAGES), + ) as mock_post: + await guardrail.apply_guardrail(inputs=inputs, request_data={}, input_type="request") + + timeout = _timeout_of(mock_post.call_args) + assert timeout.read == 60.0 + assert timeout.write == 60.0 + assert timeout.pool == 60.0 + assert timeout.connect == 5.0 + + +@pytest.mark.asyncio +async def test_retrieve_call_passes_bounded_timeout(guardrail: HeadroomGuardrail): + """The retrieval leg runs on the same request and needs the same bound.""" + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=_make_retrieve_response("original"), + ) as mock_get: + result = await guardrail._call_retrieve("a" * 24) + + assert result == "original" + timeout = _timeout_of(mock_get.call_args) + assert timeout.read == 60.0 + assert timeout.connect == 5.0 + + +@pytest.mark.asyncio +async def test_configured_timeout_overrides_the_default(): + """Headroom accepted litellm_params.timeout and ignored it.""" + guardrail = _make_guardrail(timeout=3.5) + inputs = GenericGuardrailAPIInputs(texts=["A" * 5000], structured_messages=ORIGINAL_MESSAGES) + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + return_value=_make_compress_response(COMPRESSED_MESSAGES), + ) as mock_post: + await guardrail.apply_guardrail(inputs=inputs, request_data={}, input_type="request") + + timeout = _timeout_of(mock_post.call_args) + assert timeout.read == 3.5 + assert timeout.connect == 3.5 + + +@pytest.mark.asyncio +async def test_read_timeout_is_surfaced_as_unreachable_under_fail_closed(): + """A stalled service must reach the fail policy, not escape as a 500.""" + guardrail = _make_guardrail() + inputs = GenericGuardrailAPIInputs(texts=["hello"], structured_messages=ORIGINAL_MESSAGES) + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + side_effect=httpx.ReadTimeout("timed out"), + ): + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail(inputs=inputs, request_data={}, input_type="request") + + assert exc_info.value.status_code == 502 + assert "unreachable" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_read_timeout_forwards_uncompressed_under_fail_open(): + guardrail = _make_guardrail(unreachable_fallback="fail_open") + inputs = GenericGuardrailAPIInputs(texts=["hello"], structured_messages=ORIGINAL_MESSAGES) + + with patch.object( + guardrail.async_handler, + "post", + new_callable=AsyncMock, + side_effect=httpx.ReadTimeout("timed out"), + ): + result = await guardrail.apply_guardrail(inputs=inputs, request_data={}, input_type="request") + + assert result.get("structured_messages") == ORIGINAL_MESSAGES + + +def test_initializer_forwards_configured_timeout(monkeypatch: pytest.MonkeyPatch): + """Wiring it only in __init__ leaves `timeout:` in config.yaml silently ignored.""" + from litellm.proxy.guardrails.guardrail_hooks.headroom import initialize_guardrail + from litellm.types.guardrails import LitellmParams + + monkeypatch.setattr( + litellm.logging_callback_manager, + "add_litellm_callback", + lambda callback: None, + ) + params = LitellmParams( + guardrail="headroom", + mode="pre_call", + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + timeout=7.0, + ) + callback = initialize_guardrail(params, {"guardrail_name": "headroom"}) # type: ignore[arg-type] + + assert callback.timeout.read == 7.0 + + +def test_in_place_update_keeps_the_timeout_resolved(): + """The base implementation copies every attribute over, nulling an unset timeout.""" + from litellm.types.guardrails import LitellmParams + + guardrail = _make_guardrail(timeout=5.0) + assert guardrail.timeout.read == 5.0 + + guardrail.update_in_memory_litellm_params( + LitellmParams(guardrail="headroom", mode="pre_call", api_base=FAKE_API_BASE) + ) + assert isinstance(guardrail.timeout, httpx.Timeout) + assert guardrail.timeout.read == 60.0 + + guardrail.update_in_memory_litellm_params( + LitellmParams(guardrail="headroom", mode="pre_call", api_base=FAKE_API_BASE, timeout=7.0) + ) + assert guardrail.timeout.read == 7.0 + + +@pytest.mark.parametrize("configured", [0, 0.0, -1, -30.0, float("inf"), float("-inf"), float("nan")]) +def test_unusable_timeout_falls_back_to_the_default(configured: float): + """0 and inf read as no deadline at all, a negative one as a deadline already past.""" + guardrail = _make_guardrail(timeout=configured) + + assert guardrail.timeout.read == 60.0 + assert guardrail.timeout.connect == 5.0 From 89086db28233514c3cc07333fcffdcd974cc8573 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:39:46 -0700 Subject: [PATCH 52/60] fix(proxy): floor end-user budget checks on the DB row after a reset The reset job evicts the cached end-user object only from its own worker's in-memory cache (plus Redis), so every other uvicorn worker and replica keeps the pre-reset spend for up to user_api_key_cache_ttl (60s by default). Those workers pass that stale spend as fallback_spend, and since the authoritative floor read returned None for spend:end_user: keys, get_current_spend handed the stale value straight back and the end user kept getting 429 after the rollover on every worker but the one that ran the reset. The floor read now consults LiteLLM_EndUserTable.spend for end-user counters, the same way keys, teams, users, and orgs already read their rows. It runs only when the shared counter sits below the cached spend (a reset or a Redis restart) and stays behind the existing 5s in-process marker, so the normal request path still does no DB read. Cold end-user counters keep seeding from the cached object rather than the row, so from_db is unchanged for them. --- litellm/proxy/db/spend_counter_reseed.py | 25 +++++- litellm/proxy/proxy_server.py | 46 +++++++---- .../proxy/db/test_spend_counter_reseed.py | 69 +++++++++++++++- .../proxy/proxy_server/test_spend_counters.py | 79 +++++++++++++++++-- 4 files changed, 196 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index 7b3c261036e..a38b8a47dbd 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -26,6 +26,7 @@ from litellm.proxy._types import Litellm_EntityType from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.table_repositories import ( BudgetWindowSpendRepository, + EndUserRepository, SpendLogsRepository, TeamMembershipRepository, ) @@ -36,6 +37,8 @@ from litellm.repositories.verification_token_repository import ( ) if TYPE_CHECKING: + from prisma.types import LiteLLM_EndUserTableWhereUniqueInput + from litellm.caching.dual_cache import DualCache from litellm.proxy.utils import PrismaClient @@ -47,6 +50,8 @@ _WINDOW_SPEND_ENTITY_TYPES: Final[Mapping[str, str]] = MappingProxyType( } ) +END_USER_COUNTER_PREFIX: Final = "spend:end_user:" + _WINDOW_SPEND_LOG_FIELDS: Final[Mapping[str, str]] = MappingProxyType( { "Key": "api_key", @@ -74,6 +79,10 @@ class SpendCounterReseed: End-user and tag spend counters intentionally do not reseed here. Their auth paths already load the corresponding objects via get_end_user_object() and get_tag_objects_batch(); callers pass those values as fallback_spend. + end_user_from_db is the one end-user read, used only as the budget floor when + a counter sits below that cached spend: a worker that did not run the budget + reset still caches the pre-reset end-user object, and LiteLLM_EndUserTable + is the row the reset zeroed. """ _locks: ClassVar["OrderedDict[str, asyncio.Lock]"] = OrderedDict() @@ -129,7 +138,7 @@ class SpendCounterReseed: elif counter_key.startswith("spend:user:"): user_id = counter_key[len("spend:user:") :] row = await UserRepository(prisma_client).table.find_unique(where={"user_id": user_id}) - elif counter_key.startswith("spend:end_user:") or counter_key.startswith("spend:tag:"): + elif counter_key.startswith(END_USER_COUNTER_PREFIX) or counter_key.startswith("spend:tag:"): return None elif counter_key.startswith("spend:org:"): org_id: Final = counter_key[len("spend:org:") :] @@ -143,6 +152,20 @@ class SpendCounterReseed: return None return float(getattr(row, "spend", 0.0) or 0.0) + @staticmethod + async def end_user_from_db(prisma_client: Optional["PrismaClient"], counter_key: str) -> float | None: + if prisma_client is None or not counter_key.startswith(END_USER_COUNTER_PREFIX): + return None + where: Final[LiteLLM_EndUserTableWhereUniqueInput] = {"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]} + try: + row: Final = await EndUserRepository(prisma_client).table.find_unique(where=where) + except Exception: # noqa: BLE001 # a failed floor read falls back to the cached spend, like from_db + verbose_proxy_logger.exception("SpendCounterReseed.end_user_from_db: failed for %s", counter_key) + return None + if row is None: + return None + return float(row.spend or 0.0) + @staticmethod def _is_key_or_team_window_counter(counter_key: str) -> bool: for prefix in ("spend:key:", "spend:team:"): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1f39a78e12a..47c0811d903 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -423,7 +423,7 @@ from litellm.proxy.db.proxy_worker_heartbeat import ( PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS, ProxyWorkerHeartbeat, ) -from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed +from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config @@ -2580,6 +2580,29 @@ async def reseed_spend_counter_from_db(counter_key: str) -> None: await _repair_stale_spend_counter(counter_key=counter_key, db_spend=db_spend) +async def _floor_spend_from_db( + counter_key: str, + window_entity_type: str | None, + window_entity_id: str | None, + window_duration: str | None, + window_start: datetime | None, +) -> float | None: + if counter_key.startswith(END_USER_COUNTER_PREFIX): + return await SpendCounterReseed.end_user_from_db(prisma_client=prisma_client, counter_key=counter_key) + entity_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key) + if entity_spend is not None: + return entity_spend + if window_entity_type is None or window_entity_id is None or window_start is None: + return None + return await SpendCounterReseed.window_from_db( + prisma_client=prisma_client, + entity_type=window_entity_type, + entity_id=window_entity_id, + window_duration=window_duration, + window_start=window_start, + ) + + async def _authoritative_floor_spend( counter_key: str, window_entity_type: str | None = None, @@ -2592,20 +2615,13 @@ async def _authoritative_floor_spend( if cached is not None: return float(cached) - db_spend = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key) - if ( - db_spend is None - and window_entity_type is not None - and window_entity_id is not None - and window_start is not None - ): - db_spend = await SpendCounterReseed.window_from_db( - prisma_client=prisma_client, - entity_type=window_entity_type, - entity_id=window_entity_id, - window_duration=window_duration, - window_start=window_start, - ) + db_spend: Final = await _floor_spend_from_db( + counter_key=counter_key, + window_entity_type=window_entity_type, + window_entity_id=window_entity_id, + window_duration=window_duration, + window_start=window_start, + ) if db_spend is None: return None diff --git a/tests/test_litellm/proxy/db/test_spend_counter_reseed.py b/tests/test_litellm/proxy/db/test_spend_counter_reseed.py index 816f9ae72f4..3bd6d93328d 100644 --- a/tests/test_litellm/proxy/db/test_spend_counter_reseed.py +++ b/tests/test_litellm/proxy/db/test_spend_counter_reseed.py @@ -18,7 +18,7 @@ from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed WINDOW_START = datetime(2026, 8, 1, tzinfo=timezone.utc) -class _FakeWindowSpendTable: +class _FakeFindUniqueTable: def __init__(self, row: SimpleNamespace | None, error: Exception | None = None) -> None: self._row = row self._error = error @@ -47,10 +47,13 @@ class _FakePrismaClient: row: SimpleNamespace | None = None, spend_logs_total: float = 0.0, error: Exception | None = None, + end_user_row: SimpleNamespace | None = None, + end_user_error: Exception | None = None, ) -> None: self.db = SimpleNamespace( - litellm_budgetwindowspend=_FakeWindowSpendTable(row=row, error=error), + litellm_budgetwindowspend=_FakeFindUniqueTable(row=row, error=error), litellm_spendlogs=_FakeSpendLogsTable(total=spend_logs_total), + litellm_endusertable=_FakeFindUniqueTable(row=end_user_row, error=end_user_error), ) @@ -248,3 +251,65 @@ async def test_coalesced_window_seeds_a_cold_counter_from_the_row(): assert result == 4.5 assert cache.in_memory_cache.get_cache(key=counter_key) == 4.5 assert prisma.db.litellm_spendlogs.call_count == 0 + + +@pytest.mark.asyncio +async def test_end_user_from_db_reads_the_end_user_row_by_user_id(): + prisma = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=0.0)) + + result = await SpendCounterReseed.end_user_from_db( + prisma_client=prisma, counter_key="spend:end_user:customer-42" + ) + + assert result == 0.0 + assert prisma.db.litellm_endusertable.where_clauses == [{"user_id": "customer-42"}] + + +@pytest.mark.asyncio +async def test_end_user_from_db_returns_the_recorded_spend(): + prisma = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5)) + + assert ( + await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") + == 12.5 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("counter_key", ["spend:key:hashed", "spend:team:t1", "spend:tag:t1"]) +async def test_end_user_from_db_ignores_other_counter_kinds_without_touching_the_db(counter_key): + prisma = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="x", spend=5.0)) + + assert await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key=counter_key) is None + assert prisma.db.litellm_endusertable.where_clauses == [] + + +@pytest.mark.asyncio +async def test_end_user_from_db_returns_none_without_a_row_a_client_or_on_db_error(): + assert ( + await SpendCounterReseed.end_user_from_db(prisma_client=None, counter_key="spend:end_user:customer-42") + is None + ) + assert ( + await SpendCounterReseed.end_user_from_db( + prisma_client=_FakePrismaClient(end_user_row=None), counter_key="spend:end_user:customer-42" + ) + is None + ) + assert ( + await SpendCounterReseed.end_user_from_db( + prisma_client=_FakePrismaClient(end_user_error=RuntimeError("db down")), + counter_key="spend:end_user:customer-42", + ) + is None + ) + + +@pytest.mark.asyncio +async def test_from_db_still_never_reads_the_end_user_row(): + """A cold end-user counter keeps seeding from the cached end-user object the auth + path already loaded; the row is read only as the budget floor.""" + prisma = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=5.0)) + + assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") is None + assert prisma.db.litellm_endusertable.where_clauses == [] diff --git a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py index fb3de990deb..7cc1390fcbd 100644 --- a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py +++ b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py @@ -223,21 +223,90 @@ async def test_get_current_spend_floor_caches_db_read(monkeypatch): @pytest.mark.asyncio async def test_get_current_spend_floors_end_user_tag_against_fallback(monkeypatch): - """End-user and tag counters have no DB row (from_db returns None). When the - counter is stale-low, enforcement falls back to the caller's recorded spend - (loaded fresh in auth) instead of trusting the stale counter.""" + """Tag counters have no DB row (from_db returns None), and an end-user counter has + none to read without a DB client. When such a counter is stale-low, enforcement + falls back to the caller's recorded spend (loaded fresh in auth) instead of + trusting the stale counter.""" fake_cache = _make_spend_counter_cache(redis_get_value=2.0) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) + monkeypatch.setattr(ps, "prisma_client", None) monkeypatch.setattr(ps.SpendCounterReseed, "from_db", AsyncMock(return_value=None)) + for counter_key in ("spend:end_user:e1", "spend:tag:t1"): + result = await ps.get_current_spend( + counter_key=counter_key, + fallback_spend=20.0, + max_budget=10.0, + ) + + assert result == 20.0 + # no DB row to repair against, so the shared counter is left untouched + fake_cache.redis_cache.async_set_max.assert_not_called() + + +def _make_prisma_with_end_user_row(spend: float | None): + prisma = MagicMock() + prisma.db.litellm_endusertable.find_unique = AsyncMock( + return_value=None if spend is None else MagicMock(spend=spend) + ) + return prisma + + +@pytest.mark.asyncio +async def test_get_current_spend_end_user_floor_admits_after_a_reset_on_a_stale_worker(monkeypatch): + """The reset job zeroes LiteLLM_EndUserTable.spend and the shared counter, but it + evicts the cached end-user object only on the worker that ran the reset. Every + other worker still passes the pre-reset spend as fallback_spend, and that stale + copy must not out-vote the reset row.""" + fake_cache = _make_spend_counter_cache(redis_get_value=0.0) + monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) + prisma = _make_prisma_with_end_user_row(spend=0.0) + monkeypatch.setattr(ps, "prisma_client", prisma) + result = await ps.get_current_spend( - counter_key="spend:end_user:e1", + counter_key="spend:end_user:customer-42", + fallback_spend=0.000032, + max_budget=0.00003, + fallback_authoritative=True, + ) + + assert result == 0.0 + prisma.db.litellm_endusertable.find_unique.assert_awaited_once_with(where={"user_id": "customer-42"}) + fake_cache.redis_cache.async_set_max.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_current_spend_end_user_floor_repairs_a_stale_low_counter(monkeypatch): + """After a Redis restart the end-user counter can sit below the recorded spend; + the row wins and the shared counter is raised so other workers stop admitting on + the stale value.""" + fake_cache = _make_spend_counter_cache(redis_get_value=2.0) + monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) + monkeypatch.setattr(ps, "prisma_client", _make_prisma_with_end_user_row(spend=12.0)) + + result = await ps.get_current_spend( + counter_key="spend:end_user:customer-42", + fallback_spend=12.0, + max_budget=10.0, + ) + + assert result == 12.0 + fake_cache.redis_cache.async_set_max.assert_awaited_once_with(key="spend:end_user:customer-42", value=12.0) + + +@pytest.mark.asyncio +async def test_get_current_spend_end_user_without_a_row_keeps_the_cached_spend(monkeypatch): + fake_cache = _make_spend_counter_cache(redis_get_value=0.0) + monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) + monkeypatch.setattr(ps, "prisma_client", _make_prisma_with_end_user_row(spend=None)) + + result = await ps.get_current_spend( + counter_key="spend:end_user:customer-42", fallback_spend=20.0, max_budget=10.0, ) assert result == 20.0 - # no DB row to repair against, so the shared counter is left untouched fake_cache.redis_cache.async_set_max.assert_not_called() From c02f198a4ac4ca58d53c6c9b625a02c610e4d367 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:45:06 -0700 Subject: [PATCH 53/60] fix(azure): responses none-effort temperature gate reads the azure/ cost-map entry --- .../llms/azure/responses/transformation.py | 9 ++++ .../response/test_azure_transformation.py | 43 +++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 0dd5e87e4ba..cfd0e96639f 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -6,6 +6,7 @@ from openai.types.responses import ResponseReasoningItem from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import * @@ -29,6 +30,14 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.AZURE + @staticmethod + def _supports_reasoning_effort_none(model: str) -> bool: + return AzureOpenAIGPT5Config._supports_reasoning_effort_level(model, "none") + + @staticmethod + def _effort_resolves_to_none(model: str, effort: str | None) -> bool: + return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort) + def get_supported_openai_params(self, model: str) -> list: """ Azure Responses API does not support context_management (compaction). diff --git a/tests/test_litellm/llms/azure/response/test_azure_transformation.py b/tests/test_litellm/llms/azure/response/test_azure_transformation.py index f6bbf685f26..0cac2705ab0 100644 --- a/tests/test_litellm/llms/azure/response/test_azure_transformation.py +++ b/tests/test_litellm/llms/azure/response/test_azure_transformation.py @@ -1,11 +1,10 @@ from copy import deepcopy -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest - -from unittest.mock import MagicMock - +import litellm +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.azure.responses.o_series_transformation import ( AzureOpenAIOSeriesResponsesAPIConfig, ) @@ -613,3 +612,39 @@ class TestAzureResponsesAPIConfig: assert result["tools"][0] is tool assert "anyOf" in result["tools"][0]["parameters"] + + +@pytest.fixture() +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + """Pin the bundled cost map: the published map lags a key added in this repo.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) + litellm.add_known_models(model_cost_map=litellm.model_cost) + + +def test_azure_responses_gpt6_astra_reasoning_effort_none_unlocks_temperature(local_model_cost_map: None): + """Foundry's gpt-6-astra accepts reasoning.effort='none' with a non-default temperature + while OpenAI's gpt-6-astra does not, so the gate must read the azure/ cost-map entry + for the bare deployment name rather than OpenAI's.""" + params = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=0.2, + reasoning={"effort": "none"}, + ), + model="gpt-6-astra", + drop_params=False, + ) + assert params["temperature"] == 0.2 + assert params["reasoning"] == {"effort": "none"} + + +def test_azure_responses_gpt6_astra_rejects_temperature_while_reasoning(local_model_cost_map: None): + with pytest.raises(litellm.UnsupportedParamsError): + AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + temperature=0.2, + reasoning={"effort": "low"}, + ), + model="gpt-6-astra", + drop_params=False, + ) From b4fd63f621cd8dd98755944aeca0e9dfaa78daa1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:53:45 -0700 Subject: [PATCH 54/60] chore(proxy): annotate the new spend-counter test locals and correct the floor comments --- litellm/proxy/proxy_server.py | 7 ++++--- .../proxy/db/test_spend_counter_reseed.py | 11 ++++++----- .../proxy/proxy_server/test_spend_counters.py | 17 +++++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 47c0811d903..9ad3c910f58 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2477,7 +2477,8 @@ async def get_current_spend( authoritative source depends on the counter: primary key/team/user/org counters read the DB row; per-window counters (``window_start`` supplied) read the maintained window-spend row and only aggregate spend logs when - that row is missing or stale; end-user/tag counters have no DB row, so the caller's + that row is missing or stale; end-user counters read ``LiteLLM_EndUserTable``, the + row the budget reset zeroes; tag counters have no DB row, so the caller's ``fallback_spend`` (loaded fresh in auth) is authoritative. The DB read is skipped for healthy primary counters (counter at or above recorded spend) and cached in-process for a few seconds, so a persistently stale counter @@ -2511,8 +2512,8 @@ async def get_current_spend( await _repair_stale_spend_counter(counter_key=counter_key, db_spend=authoritative) return authoritative elif fallback_spend > current: - # end-user / tag counters have no DB row; fallback_spend is the - # authoritative recorded value loaded in auth. + # nothing to read (tag counters, an end user without a row or a DB client, a + # failed read); fallback_spend is the authoritative recorded value loaded in auth. return fallback_spend # Opt-in hard guarantee: when the spend backing this admit decision came diff --git a/tests/test_litellm/proxy/db/test_spend_counter_reseed.py b/tests/test_litellm/proxy/db/test_spend_counter_reseed.py index 3bd6d93328d..8cb3fc665eb 100644 --- a/tests/test_litellm/proxy/db/test_spend_counter_reseed.py +++ b/tests/test_litellm/proxy/db/test_spend_counter_reseed.py @@ -9,6 +9,7 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone from types import SimpleNamespace +from typing import Final import pytest @@ -255,9 +256,9 @@ async def test_coalesced_window_seeds_a_cold_counter_from_the_row(): @pytest.mark.asyncio async def test_end_user_from_db_reads_the_end_user_row_by_user_id(): - prisma = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=0.0)) + prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=0.0)) - result = await SpendCounterReseed.end_user_from_db( + result: Final = await SpendCounterReseed.end_user_from_db( prisma_client=prisma, counter_key="spend:end_user:customer-42" ) @@ -267,7 +268,7 @@ async def test_end_user_from_db_reads_the_end_user_row_by_user_id(): @pytest.mark.asyncio async def test_end_user_from_db_returns_the_recorded_spend(): - prisma = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5)) + prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5)) assert ( await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") @@ -278,7 +279,7 @@ async def test_end_user_from_db_returns_the_recorded_spend(): @pytest.mark.asyncio @pytest.mark.parametrize("counter_key", ["spend:key:hashed", "spend:team:t1", "spend:tag:t1"]) async def test_end_user_from_db_ignores_other_counter_kinds_without_touching_the_db(counter_key): - prisma = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="x", spend=5.0)) + prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="x", spend=5.0)) assert await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key=counter_key) is None assert prisma.db.litellm_endusertable.where_clauses == [] @@ -309,7 +310,7 @@ async def test_end_user_from_db_returns_none_without_a_row_a_client_or_on_db_err async def test_from_db_still_never_reads_the_end_user_row(): """A cold end-user counter keeps seeding from the cached end-user object the auth path already loaded; the row is read only as the budget floor.""" - prisma = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=5.0)) + prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=5.0)) assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") is None assert prisma.db.litellm_endusertable.where_clauses == [] diff --git a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py index 7cc1390fcbd..ef6f8120c82 100644 --- a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py +++ b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py @@ -22,6 +22,7 @@ from __future__ import annotations import asyncio from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -233,7 +234,7 @@ async def test_get_current_spend_floors_end_user_tag_against_fallback(monkeypatc monkeypatch.setattr(ps.SpendCounterReseed, "from_db", AsyncMock(return_value=None)) for counter_key in ("spend:end_user:e1", "spend:tag:t1"): - result = await ps.get_current_spend( + result: Final = await ps.get_current_spend( counter_key=counter_key, fallback_spend=20.0, max_budget=10.0, @@ -245,7 +246,7 @@ async def test_get_current_spend_floors_end_user_tag_against_fallback(monkeypatc def _make_prisma_with_end_user_row(spend: float | None): - prisma = MagicMock() + prisma: Final = MagicMock() prisma.db.litellm_endusertable.find_unique = AsyncMock( return_value=None if spend is None else MagicMock(spend=spend) ) @@ -258,9 +259,9 @@ async def test_get_current_spend_end_user_floor_admits_after_a_reset_on_a_stale_ evicts the cached end-user object only on the worker that ran the reset. Every other worker still passes the pre-reset spend as fallback_spend, and that stale copy must not out-vote the reset row.""" - fake_cache = _make_spend_counter_cache(redis_get_value=0.0) + fake_cache: Final = _make_spend_counter_cache(redis_get_value=0.0) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) - prisma = _make_prisma_with_end_user_row(spend=0.0) + prisma: Final = _make_prisma_with_end_user_row(spend=0.0) monkeypatch.setattr(ps, "prisma_client", prisma) result = await ps.get_current_spend( @@ -280,11 +281,11 @@ async def test_get_current_spend_end_user_floor_repairs_a_stale_low_counter(monk """After a Redis restart the end-user counter can sit below the recorded spend; the row wins and the shared counter is raised so other workers stop admitting on the stale value.""" - fake_cache = _make_spend_counter_cache(redis_get_value=2.0) + fake_cache: Final = _make_spend_counter_cache(redis_get_value=2.0) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "prisma_client", _make_prisma_with_end_user_row(spend=12.0)) - result = await ps.get_current_spend( + result: Final = await ps.get_current_spend( counter_key="spend:end_user:customer-42", fallback_spend=12.0, max_budget=10.0, @@ -296,11 +297,11 @@ async def test_get_current_spend_end_user_floor_repairs_a_stale_low_counter(monk @pytest.mark.asyncio async def test_get_current_spend_end_user_without_a_row_keeps_the_cached_spend(monkeypatch): - fake_cache = _make_spend_counter_cache(redis_get_value=0.0) + fake_cache: Final = _make_spend_counter_cache(redis_get_value=0.0) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "prisma_client", _make_prisma_with_end_user_row(spend=None)) - result = await ps.get_current_spend( + result: Final = await ps.get_current_spend( counter_key="spend:end_user:customer-42", fallback_spend=20.0, max_budget=10.0, From 4ffd2ffb25017c861a350f24a27b6441ceeb2fd1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:00:40 -0700 Subject: [PATCH 55/60] test(proxy): parametrize the stale end-user counter case so no Final local sits in a loop --- .../proxy/proxy_server/test_spend_counters.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py index ef6f8120c82..86e97a334df 100644 --- a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py +++ b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py @@ -223,24 +223,24 @@ async def test_get_current_spend_floor_caches_db_read(monkeypatch): @pytest.mark.asyncio -async def test_get_current_spend_floors_end_user_tag_against_fallback(monkeypatch): +@pytest.mark.parametrize("counter_key", ("spend:end_user:e1", "spend:tag:t1")) +async def test_get_current_spend_floors_end_user_tag_against_fallback(monkeypatch, counter_key): """Tag counters have no DB row (from_db returns None), and an end-user counter has none to read without a DB client. When such a counter is stale-low, enforcement falls back to the caller's recorded spend (loaded fresh in auth) instead of trusting the stale counter.""" - fake_cache = _make_spend_counter_cache(redis_get_value=2.0) + fake_cache: Final = _make_spend_counter_cache(redis_get_value=2.0) monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) monkeypatch.setattr(ps, "prisma_client", None) monkeypatch.setattr(ps.SpendCounterReseed, "from_db", AsyncMock(return_value=None)) - for counter_key in ("spend:end_user:e1", "spend:tag:t1"): - result: Final = await ps.get_current_spend( - counter_key=counter_key, - fallback_spend=20.0, - max_budget=10.0, - ) + result: Final = await ps.get_current_spend( + counter_key=counter_key, + fallback_spend=20.0, + max_budget=10.0, + ) - assert result == 20.0 + assert result == 20.0 # no DB row to repair against, so the shared counter is left untouched fake_cache.redis_cache.async_set_max.assert_not_called() From cd113c3a2e46bd8d468ea48219d861c107e06260 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:13:51 -0700 Subject: [PATCH 56/60] ci: allowlist the bounded _unqualified qualifier peel in the recursion detector --- tests/code_coverage_tests/recursive_detector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 0578dc60119..e9f87ba6cae 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -66,6 +66,7 @@ IGNORE_FUNCTIONS = [ "_json_safe", # max depth set (_MAX_DEPTH) plus a seen-ids cycle guard for self-referential input. "_redact_agent_params_tree", # max depth set (default 10), same shape as _redact_sensitive_litellm_params. "_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side. + "_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible). ] From e2741b564376f8b30ec93848cf2d9c3e94aa32af Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Fri, 4 Sep 2026 18:24:16 -0700 Subject: [PATCH 57/60] fix(datadog_llm_obs): keep the guardrail audit record under message redaction (#39702) * fix(datadog_llm_obs): keep the guardrail audit record under message redaction Redaction nulled `guardrail_information` on the span whole, so an operator running `turn_off_message_logging` (or a caller sending `x-litellm-enable-message-redaction`) lost the record of which guardrails ran, what they returned, and what they masked. Four of the record's fields can quote the prompt; the rest report what the guardrail decided without reproducing it. Replace only those four, the way `_sanitize_guardrail_information_for_spend_logs` already does for spend logs, and declare the field list once in `litellm/types/utils.py` so both readers share it. * fix(datadog_llm_obs): keep a lone guardrail record, and test through the span Review round 1. A guardrail that writes the metadata key itself leaves a single record where the type says list, which Prometheus already normalizes at `_guardrail_overhead_seconds`. Redaction dropped that shape and the latency extraction raised on it, so the span was lost outright. Normalize once and use it in both places. The new tests now drive `create_llm_obs_payload` instead of reading the module's private helpers and the record's declared field names. --- .../integrations/datadog/datadog_llm_obs.py | 64 +++++++-- .../spend_tracking/spend_tracking_utils.py | 10 +- litellm/types/utils.py | 44 ++++++ .../datadog/test_datadog_llm_obs.py | 129 +++++++++++++++++- 4 files changed, 228 insertions(+), 19 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index ec86c0ae1d9..728bf41856f 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -19,7 +19,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid -from litellm.constants import REDACTED_BY_LITELLM +from litellm.constants import REDACTED_BY_LITELLM, REDACTED_BY_LITELM_STRING from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.integrations.datadog.datadog_handler import ( get_datadog_base_url_from_env, @@ -46,9 +46,10 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens from litellm.types.integrations.datadog_llm_obs import * from litellm.types.utils import ( + AUDIT_GUARDRAIL_FIELDS, + PROMPT_CARRYING_GUARDRAIL_FIELDS, PROMPT_QUOTING_ROUTING_DECISION_FIELDS, CallTypes, - StandardLoggingGuardrailInformation, StandardLoggingPayload, StandardLoggingPayloadErrorInformation, ) @@ -60,6 +61,8 @@ _SAFE_REDACTED_MESSAGE_ROLES: Final = frozenset( {"agent", "assistant", "developer", "function", "model", "system", "tool", "user"} ) +_CLASSIFIED_GUARDRAIL_FIELDS: Final = AUDIT_GUARDRAIL_FIELDS | PROMPT_CARRYING_GUARDRAIL_FIELDS + _PROMPT_CARRYING_METADATA_FIELDS: Final = frozenset( { "routing_decision", @@ -108,6 +111,49 @@ def _router_span_fields( ) +def _guardrail_entries(guardrail_information: object) -> tuple[Mapping[str, object], ...]: + """The guardrail records as a sequence, whatever shape the payload carries. + + `guardrail_information` is typed as a list, but a guardrail that writes the metadata key itself + can leave a single record there; Prometheus normalizes the same shape at + `_guardrail_overhead_seconds`. + """ + if isinstance(guardrail_information, Mapping): + return (guardrail_information,) + if isinstance(guardrail_information, (list, tuple)): + return tuple(entry for entry in guardrail_information if isinstance(entry, Mapping)) + return () + + +def _guardrail_entry_without_prompt_carriers(entry: Mapping[str, object]) -> Mapping[str, object]: + """One guardrail record kept as its audit fields, with the prompt-quoting ones marked redacted. + + Built as an allow-list rather than a deny-list: a key neither set classifies is dropped, so a + guardrail that records its own extra detail cannot put the caller's prompt on a redacted span. + """ + return { # mutable-ok: a fresh record built per entry, handed straight to the span serializer + field: REDACTED_BY_LITELM_STRING if field in PROMPT_CARRYING_GUARDRAIL_FIELDS else value + for field, value in entry.items() + if field in _CLASSIFIED_GUARDRAIL_FIELDS + } + + +def _guardrail_information_without_prompt_carriers( + guardrail_information: object, +) -> tuple[Mapping[str, object], ...] | None: + """The guardrail records reduced to what a redacted span may carry. + + Redaction removes the prompt, not the record that a guardrail ran: the name, mode, status, + timings and masked-entity counts are what an operator reads to answer whether a guardrail + caught anything on a request, and none of them reproduce the prompt. Field-level rather than + dropping the list, which is what `_sanitize_guardrail_information_for_spend_logs` already does + for spend logs. + """ + if guardrail_information is None: + return None + return tuple(_guardrail_entry_without_prompt_carriers(entry) for entry in _guardrail_entries(guardrail_information)) + + def _metadata_without_prompt_carriers(standard_logging_metadata: Mapping[str, Any]) -> Mapping[str, Any]: """The metadata minus the records that quote prompts, tool arguments, tool results, or retrieved text.""" return MappingProxyType( @@ -872,7 +918,9 @@ class DataDogLLMObsLogger(CustomBatchLogger): "cache_key": standard_logging_payload.get("cache_key", "unknown"), "saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0), "guardrail_information": ( - None if redact_prompt_text else standard_logging_payload.get("guardrail_information", None) + _guardrail_information_without_prompt_carriers(standard_logging_payload.get("guardrail_information")) + if redact_prompt_text + else standard_logging_payload.get("guardrail_information", None) ), "is_streamed_request": self._get_stream_value_from_payload(standard_logging_payload), "latency_metrics": dict(self._get_latency_metrics(standard_logging_payload)), @@ -904,14 +952,12 @@ class DataDogLLMObsLogger(CustomBatchLogger): latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms # Guardrail overhead latency - guardrail_info: Final[list[StandardLoggingGuardrailInformation] | None] = standard_logging_payload.get( - "guardrail_information" - ) - if guardrail_info is not None: + guardrail_info: Final = _guardrail_entries(standard_logging_payload.get("guardrail_information")) + if guardrail_info: total_duration = 0.0 for info in guardrail_info: - _guardrail_duration_seconds: float | None = info.get("duration") - if _guardrail_duration_seconds is not None: + _guardrail_duration_seconds = info.get("duration") + if isinstance(_guardrail_duration_seconds, (int, float, str)): total_duration += float(_guardrail_duration_seconds) if total_duration > 0: diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 9ea2170d6ab..8a06bf68b81 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -37,6 +37,7 @@ from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload, SpendLogsR from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.proxy.utils import PrismaClient, hash_token from litellm.types.utils import ( + PROMPT_CARRYING_GUARDRAIL_FIELDS, CallTypes, CostBreakdown, StandardLoggingGuardrailInformation, @@ -1073,13 +1074,6 @@ def _sanitize_guardrail_information_for_spend_logs( return [_redact_prompt_fields_in_guardrail_entry(entry) for entry in entries if isinstance(entry, dict)] -_PROMPT_CARRYING_GUARDRAIL_FIELDS: Final = ( - "guardrail_request", - "guardrail_response", - "match_details", - "classification", -) - _NUMERIC_COMPRESSION_STAT_KEYS: Final = ( "tokens_before", "tokens_after", @@ -1114,7 +1108,7 @@ def _redact_prompt_fields_in_guardrail_entry( preserved_stats: Final = _numeric_compression_stats_from_guardrail_response(entry.get("guardrail_response")) redacted: Final[StandardLoggingGuardrailInformation] = { **entry, - **{key: REDACTED_BY_LITELM_STRING for key in _PROMPT_CARRYING_GUARDRAIL_FIELDS if key in entry}, + **{key: REDACTED_BY_LITELM_STRING for key in PROMPT_CARRYING_GUARDRAIL_FIELDS if key in entry}, } if preserved_stats is None: return redacted diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6745e65f81e..238986f86a6 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3080,6 +3080,50 @@ class GuardrailMode(TypedDict, total=False): GuardrailStatus = Literal["success", "guardrail_intervened", "guardrail_failed_to_respond", "not_run"] +# Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the +# guardrail, the provider response that echoes it back, and the two first-party hooks that inline +# prompt substrings (``block_code_execution`` and ``litellm_content_filter``). Every other field +# reports what the guardrail decided without reproducing the prompt, so redaction replaces these +# four and keeps the rest of the record. +PROMPT_CARRYING_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset( + { + "guardrail_request", + "guardrail_response", + "match_details", + "classification", + } +) + +# The rest of the record: what the guardrail is, what it decided, how long it took and what it cost. +# None of these reproduce the prompt, so a redacted record keeps them and stays explainable. +# `test_every_guardrail_field_is_classified` fails if a field is added to the record without being +# placed in one set or the other, so a new field is dropped from redacted records rather than +# shipped unexamined. +AUDIT_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset( + { + "guardrail_name", + "guardrail_provider", + "guardrail_mode", + "guardrail_status", + "start_time", + "end_time", + "duration", + "masked_entity_count", + "guardrail_id", + "policy_template", + "detection_method", + "confidence_score", + "patterns_checked", + "alert_recipients", + "risk_score", + "violation_categories", + "guardrail_action", + "guardrail_usage", + "guardrail_cost", + "guardrail_cost_in_spend", + } +) + class StandardLoggingGuardrailInformation(TypedDict, total=False): guardrail_name: str | None diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py index 34c62864c4e..0555447e34f 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py @@ -12,7 +12,7 @@ spelling of prompt-cache counts (`prompt_tokens_details.cached_tokens`). import json import os from datetime import datetime, timedelta -from typing import Any +from typing import Any, Final from unittest.mock import patch import pytest @@ -20,6 +20,8 @@ import pytest import litellm from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import StandardLoggingGuardrailInformation TOOL_DEFINITION: dict[str, Any] = { "type": "function", @@ -631,10 +633,133 @@ def test_redaction_drops_every_prompt_carrying_metadata_record(logger: DataDogLL for record in sensitive_metadata: assert record not in redacted["meta"]["metadata"] assert record in unredacted["meta"]["metadata"] - assert redacted["meta"]["metadata"]["guardrail_information"] is None + assert redacted["meta"]["metadata"]["guardrail_information"] == [ + {"guardrail_name": "g", "guardrail_request": "REDACTED_BY_LITELM"} + ] # the record survives; only the field quoting the prompt is replaced assert unredacted["meta"]["metadata"]["guardrail_information"] is not None +_AUDIT_RECORD: Final[StandardLoggingGuardrailInformation] = StandardLoggingGuardrailInformation( + guardrail_name="bedrock-pii", + guardrail_provider="bedrock", + guardrail_mode=GuardrailEventHooks.pre_call, + guardrail_status="guardrail_intervened", + guardrail_response={"action": "MASK", "match": "alice@acme.com"}, + match_details=[{"pattern": "email", "match": "alice@acme.com"}], + classification="the user asked for alice@acme.com", + masked_entity_count={"EMAIL": 2}, + violation_categories=["pii"], + duration=0.01, +) + + +def _payload_with_guardrail_record(guardrail_information: object) -> dict[str, Any]: + payload = build_payload() + payload["standard_logging_object"]["guardrail_information"] = guardrail_information + return payload + + +def test_redaction_keeps_the_guardrail_audit_record(logger: DataDogLLMObsLogger) -> None: + """Redaction removes the prompt, not the operator's record that a guardrail intervened.""" + redacted = _span_json( + _redacting_logger(turn_off_message_logging=True), + _payload_with_guardrail_record([dict(_AUDIT_RECORD)]), + ) + record = redacted["meta"]["metadata"]["guardrail_information"][0] + + for field in ("guardrail_request", "guardrail_response", "match_details", "classification"): + assert record.get(field, "REDACTED_BY_LITELM") == "REDACTED_BY_LITELM" + assert record["guardrail_name"] == "bedrock-pii" + assert record["guardrail_provider"] == "bedrock" + assert record["guardrail_mode"] == "pre_call" + assert record["guardrail_status"] == "guardrail_intervened" + assert record["masked_entity_count"] == {"EMAIL": 2} + assert record["violation_categories"] == ["pii"] + assert record["duration"] == 0.01 + assert "alice@acme.com" not in safe_dumps(redacted["meta"]["metadata"]) + + +def test_a_caller_supplied_redaction_header_cannot_blank_the_guardrail_record( + logger: DataDogLLMObsLogger, +) -> None: + """Any key may redact its own prompts with the header; none may erase what a guardrail caught.""" + payload = _payload_with_guardrail_record([dict(_AUDIT_RECORD)]) + payload["litellm_params"] = {"metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}} + + span = _span_json(logger, payload) + record = span["meta"]["metadata"]["guardrail_information"][0] + + assert span["meta"]["input"]["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert record["guardrail_status"] == "guardrail_intervened" + assert record["masked_entity_count"] == {"EMAIL": 2} + assert "alice@acme.com" not in safe_dumps(span["meta"]["metadata"]) + + +def test_a_guardrails_own_extra_field_never_reaches_a_redacted_span(logger: DataDogLLMObsLogger) -> None: + """A guardrail may record whatever it likes; only classified fields survive redaction.""" + span = _span_json( + _redacting_logger(turn_off_message_logging=True), + _payload_with_guardrail_record([{**_AUDIT_RECORD, "matched_text": "the caller asked about alice@acme.com"}]), + ) + record = span["meta"]["metadata"]["guardrail_information"][0] + + assert "matched_text" not in record + assert record["guardrail_status"] == "guardrail_intervened" + assert "alice@acme.com" not in safe_dumps(span["meta"]["metadata"]) + + +def test_a_lone_guardrail_record_survives_redaction(logger: DataDogLLMObsLogger) -> None: + """A guardrail that writes the metadata key itself leaves one record, not a list of them.""" + span = _span_json( + _redacting_logger(turn_off_message_logging=True), + _payload_with_guardrail_record(dict(_AUDIT_RECORD)), + ) + metadata = span["meta"]["metadata"] + + assert metadata["guardrail_information"] == [ + { + "guardrail_name": "bedrock-pii", + "guardrail_provider": "bedrock", + "guardrail_mode": "pre_call", + "guardrail_status": "guardrail_intervened", + "guardrail_response": "REDACTED_BY_LITELM", + "match_details": "REDACTED_BY_LITELM", + "classification": "REDACTED_BY_LITELM", + "masked_entity_count": {"EMAIL": 2}, + "violation_categories": ["pii"], + "duration": 0.01, + } + ] + assert metadata["latency_metrics"]["guardrail_overhead_time_ms"] == 10.0 + + +@pytest.mark.parametrize("guardrail_information", [None, [], 5, "abc", [None, "x"], {}]) +def test_odd_guardrail_shapes_still_produce_a_span( + guardrail_information: object, +) -> None: + """The redacted branch replaced an expression that could not fail, so it must not start failing.""" + span = _span_json( + _redacting_logger(turn_off_message_logging=True), + _payload_with_guardrail_record(guardrail_information), + ) + + assert span["meta"]["input"]["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert span["meta"]["metadata"]["guardrail_information"] in (None, [], [{}]) + + +def test_a_redacted_span_carries_every_declared_guardrail_field() -> None: + """A field added to the record without a redaction decision would be dropped, so it fails here.""" + declared = dict.fromkeys(StandardLoggingGuardrailInformation.__annotations__, "alice@acme.com") + payload = _payload_with_guardrail_record([{**declared, "duration": 0.01}]) + + span = _span_json(_redacting_logger(turn_off_message_logging=True), payload) + record = span["meta"]["metadata"]["guardrail_information"][0] + + assert set(record) == set(declared) + for field in ("guardrail_request", "guardrail_response", "match_details", "classification"): + assert record[field] == "REDACTED_BY_LITELM" + + def test_tool_definitions_accept_the_bare_anthropic_shape(logger: DataDogLLMObsLogger) -> None: """The Anthropic surface declares tools unwrapped, with input_schema instead of parameters.""" payload = build( From 853fed824e841ce3e3952f137b6a3d577ced07d4 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Fri, 4 Sep 2026 18:40:35 -0700 Subject: [PATCH 58/60] fix(bedrock): stop sending toolConfig tool definitions to guardrails on passthrough converse (#39281) Bedrock passthrough Converse routes flattened every non-empty string under toolConfig.tools into the guardrail INPUT texts, so tool names, tool descriptions and JSON-schema strings (object, property names, titles, type names, enum values) each arrived as a separate guardrail item. A request whose only prompt was one benign user message could be blocked outright because a denied term appeared in an app-authored tool definition. Tool definitions are now excluded from the extracted texts, matching every other guardrail translation handler, which carries tool definitions in the structured tools input rather than in texts. Caller content stays scanned: message text, toolUse.input, toolResult content and json, and additionalModelRequestFields are unchanged. Resolves LIT-5797 --- .../guardrail_translation/handler.py | 27 ++++-- .../guardrail_translation/test_handler.py | 97 ++++++++++++++----- 2 files changed, 90 insertions(+), 34 deletions(-) diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index 9d35a87855e..b87f6196e51 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -89,11 +89,24 @@ def _extract_converse_texts( top-level ``text`` blocks this scans the arbitrary-JSON fields a caller can hide prompt content in -- ``toolUse.input`` and ``toolResult.content[].json`` (alongside ``toolResult.content[].text``) -- - as well as the request-level fields still forwarded to Bedrock that a caller - can route blocked content through: ``toolConfig.tools`` (tool names, - descriptions and input schemas) and ``additionalModelRequestFields``. Tool - message blocks are skipped when tool messages are excluded, but tool - definitions are always scanned to match the chat-completions guardrail path. + as well as ``additionalModelRequestFields``, a free-form model-parameter bag + with no schema that a caller can route blocked content through. + + ``toolConfig.tools`` is deliberately NOT scanned. Tool definitions are + app-authored config, so their names, descriptions and JSON-schema strings + ("object", property names, titles, type names, enum values) would each reach + the guardrail as a separate INPUT item, producing false positives and + inflating guardrail usage for a request whose only prompt is one user + message. No other guardrail translation handler puts tool definitions in + ``texts``; the chat and messages handlers carry them in the structured + ``tools`` input instead, which this handler does not populate because a + Bedrock ``toolSpec`` is not the OpenAI tool shape those consumers expect. + + ``additionalModelRequestFields`` is treated differently on purpose. Bedrock + gives ``toolConfig.tools`` a fixed schema whose contents are tool metadata by + contract, while ``additionalModelRequestFields`` is free-form and defined by + the target model, so what it carries cannot be classified without knowing + that model. Scanning it stays the fail-closed default. """ holders: Final[list[_StringHolder]] = [] @@ -121,10 +134,6 @@ def _extract_converse_texts( _collect_block_text(inner, holders) _collect_strings(inner.get("json"), holders) - tool_config: Final = body.get("toolConfig") - if isinstance(tool_config, dict): - _collect_strings(tool_config.get("tools"), holders) - _collect_strings(body.get("additionalModelRequestFields"), holders) texts: Final = [container[key] for container, key in holders] diff --git a/tests/test_litellm/llms/bedrock/passthrough/guardrail_translation/test_handler.py b/tests/test_litellm/llms/bedrock/passthrough/guardrail_translation/test_handler.py index 744ed50dbcb..dee8366ce2d 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/guardrail_translation/test_handler.py +++ b/tests/test_litellm/llms/bedrock/passthrough/guardrail_translation/test_handler.py @@ -170,22 +170,27 @@ class TestExtractConverseTexts: texts, _ = _extract_converse_texts(body, skip_system=False, skip_tool=False) assert texts == [] - def test_extracts_tool_config_description_and_schema(self): + def test_tool_config_definitions_not_extracted(self): + """Tool definitions are app-authored config, so nothing under + toolConfig.tools reaches the guardrail as input content.""" body = { - "messages": [{"role": "user", "content": [{"text": "hi"}]}], + "messages": [ + {"role": "user", "content": [{"text": "How much lag is there in my data?"}]} + ], "toolConfig": { "tools": [ { "toolSpec": { "name": "lookup", - "description": "blocked tool description", + "description": "tool description", "inputSchema": { "json": { "type": "object", "properties": { - "q": { + "agent_name": { "type": "string", - "description": "blocked schema description", + "title": "Agent Name", + "enum": ["alpha", "beta", "gamma"], } }, } @@ -196,20 +201,56 @@ class TestExtractConverseTexts: }, } texts, _ = _extract_converse_texts(body, skip_system=False, skip_tool=False) - assert "blocked tool description" in texts - assert "blocked schema description" in texts + assert texts == ["How much lag is there in my data?"] - def test_tool_config_scanned_even_when_tool_messages_skipped(self): + def test_every_tool_definition_excluded_not_just_the_first(self): + """A per-tool scan that only skipped tools[0] would still leak the rest.""" body = { "messages": [{"role": "user", "content": [{"text": "hi"}]}], "toolConfig": { "tools": [ - {"toolSpec": {"name": "fn", "description": "blocked description"}} + {"toolSpec": {"name": "first", "description": "first description"}}, + {"toolSpec": {"name": "second", "description": "second description"}}, + {"toolSpec": {"name": "third", "description": "third description"}}, + ] + }, + } + texts, _ = _extract_converse_texts(body, skip_system=False, skip_tool=False) + assert texts == ["hi"] + + def test_tool_config_definitions_not_extracted_when_tool_messages_skipped(self): + body = { + "messages": [{"role": "user", "content": [{"text": "hi"}]}], + "toolConfig": { + "tools": [ + {"toolSpec": {"name": "fn", "description": "tool description"}} ] }, } texts, _ = _extract_converse_texts(body, skip_system=False, skip_tool=True) - assert "blocked description" in texts + assert texts == ["hi"] + + def test_tool_use_input_still_extracted_alongside_tool_config(self): + """Only tool DEFINITIONS are excluded; caller content inside a toolUse + block is still scanned.""" + body = { + "messages": [ + { + "role": "user", + "content": [ + {"text": "hi"}, + {"toolUse": {"toolUseId": "t1", "name": "fn", "input": {"q": "user secret"}}}, + ], + } + ], + "toolConfig": { + "tools": [ + {"toolSpec": {"name": "fn", "description": "tool description"}} + ] + }, + } + texts, _ = _extract_converse_texts(body, skip_system=False, skip_tool=False) + assert texts == ["hi", "user secret"] def test_extracts_additional_model_request_fields(self): body = { @@ -437,9 +478,9 @@ class TestBedrockPassthroughGuardrailHandlerInput: assert "blocked content" in sent_texts @pytest.mark.asyncio - async def test_tool_config_description_scanned_and_masked(self): - """Blocked text hidden in toolConfig.tools[].toolSpec.description is still - forwarded to Bedrock, so the guardrail must see it and mask it in place.""" + async def test_tool_config_definitions_not_sent_and_left_untouched(self): + """Tool definitions never reach the guardrail, and the body forwarded to + Bedrock keeps them byte for byte.""" handler = BedrockPassthroughGuardrailHandler() data = _converse_data() data["data"]["toolConfig"] = { @@ -453,36 +494,42 @@ class TestBedrockPassthroughGuardrailHandlerInput: } ] } - guardrail = _make_guardrail( - {"texts": ["You are helpful.", "Hello world", "lookup", "[REDACTED]", "object"]} - ) + original_tool_config = copy.deepcopy(data["data"]["toolConfig"]) + guardrail = _make_guardrail({"texts": ["[REDACTED]", "[REDACTED]"]}) result = await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) sent_texts = guardrail.apply_guardrail.call_args.kwargs["inputs"]["texts"] - assert "email john@example.com" in sent_texts - tool_spec = result["data"]["toolConfig"]["tools"][0]["toolSpec"] - assert tool_spec["description"] == "[REDACTED]" + assert sent_texts == ["You are helpful.", "Hello world"] + assert result["data"]["toolConfig"] == original_tool_config @pytest.mark.asyncio - async def test_tool_config_description_blocking_propagates(self): - """A blocking guardrail must reject content hidden in a tool description.""" + async def test_blocking_guardrail_not_triggered_by_tool_description(self): + """LIT-5797: a request whose only prompt is a benign user message must not + be blocked because a denied term appears in a tool definition.""" handler = BedrockPassthroughGuardrailHandler() data = _converse_data() data["data"]["toolConfig"] = { "tools": [{"toolSpec": {"name": "fn", "description": "blocked content"}}] } + + async def _block_on_denied_term(**kwargs): + texts = kwargs["inputs"]["texts"] + if any("blocked content" in text for text in texts): + raise GuardrailBlocked("Blocked") + return {"texts": texts} + guardrail = MagicMock() guardrail.guardrail_name = "block-guard" guardrail.skip_system_message_in_guardrail = False guardrail.skip_tool_message_in_guardrail = False - guardrail.apply_guardrail = AsyncMock(side_effect=GuardrailBlocked("Blocked")) + guardrail.apply_guardrail = AsyncMock(side_effect=_block_on_denied_term) - with pytest.raises(GuardrailBlocked): - await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + result = await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) sent_texts = guardrail.apply_guardrail.call_args.kwargs["inputs"]["texts"] - assert "blocked content" in sent_texts + assert "blocked content" not in sent_texts + assert result["data"]["toolConfig"]["tools"][0]["toolSpec"]["description"] == "blocked content" @pytest.mark.asyncio async def test_additional_model_request_fields_scanned_and_masked(self): From 2c3c7dd1a60f80595350671c7f654dfb2c16933d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 4 Sep 2026 18:41:47 -0700 Subject: [PATCH 59/60] feat(shadow_eval): judge tool-call turns instead of dropping or erroring on them (#39818) * fix(shadow_eval): tell a tool-call shadow reply apart from an empty one Both arrive at the attempt row as the same 'shadow router returned an empty response', because _chat_final_text returns empty for a tool-final turn by design and for a reply that genuinely carried no text. Those are different things: an arm that chose a tool where the real model wrote prose is a divergence a text judge cannot score, and the sampling side already drops the real arm's tool-final turns for exactly that reason, so the shadow side reads as a fault where the real side reads as a filter. A job that is almost all 'empty response' gives no way to tell a tool-happy arm from a broken one. The error now names which of the two happened, and carries the finish_reason and the routed model so the row says what the arm was doing. Every varying part sits behind the first semicolon: operators read these by grouping on the error text, and interpolating the model into the leading sentence would make each row its own group. The outcome stays 'error'. Whether a tool-call reply should instead be its own non-judged outcome, excluded from the loss rate the way the real arm's tool-final turns already are, needs the four aggregation predicates that spell judged as outcome != 'error' rewritten, and a decision on how to surface the new bucket. That is a separate change. * fix(shadow_eval): read the tool name of a custom tool call A custom tool call carries its name under custom.name with no function key, so every one of them reported as tool=unnamed. * feat(shadow_eval): judge tool calls instead of dropping the turn A turn where either arm called a tool was discarded before it could be compared: the real arm's at sampling, the shadow arm's as an error row. On agentic traffic that is most of the traffic, so a job set to sample 10% was sampling 10% of the prose-only slice. Tool calls now serialize to text on every surface and are judged like any other response, and the judge is told a tool call is not a defect so it scores the choice rather than the shape. * feat(shadow_eval): show the judge what tools were available Both arms were offered the same tools, but the judge only ever saw the chosen call in isolation, with no way to tell whether a better tool existed or the arguments matched what the tool expects. Threads the request's tool definitions (name and description only) into the judge prompt, capped and omitted entirely on turns that offered none. * fix(shadow_eval): read a custom tool definition's name from custom, not function A chat-completions custom tool definition nests name and description under custom, mirroring how a custom tool call nests them (openai.types.chat. ChatCompletionCustomToolParam). Reading only function rendered every one as unnamed, telling the judge nothing about what it was. --- litellm/integrations/shadow_eval_logger.py | 163 ++++++++-- .../integrations/test_shadow_eval_logger.py | 287 +++++++++++++++++- 2 files changed, 416 insertions(+), 34 deletions(-) diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 27da785331a..b28d21ba1ca 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -165,28 +165,91 @@ def _chat_request_from_responses( ) -def _chat_final_text(response_obj: object) -> str: - """The assistant's text, or empty when the turn carries tool calls: only text-final - turns produce a judgeable A/B comparison.""" +def _chat_choice(response_obj: object) -> object | None: + """The response's first choice, from a payload mapping or a duck-typed ModelResponse.""" try: - message: Final = ( - response_obj["choices"][0]["message"] - if isinstance(response_obj, Mapping) - else response_obj.choices[0].message # pyright: ignore[reportAttributeAccessIssue] # duck-typed ModelResponse - ) + if isinstance(response_obj, Mapping): + return response_obj["choices"][0] + return response_obj.choices[0] # pyright: ignore[reportAttributeAccessIssue] # duck-typed ModelResponse except (AttributeError, KeyError, IndexError, TypeError): + return None + + +def _field_reader(obj: object) -> Callable[[str], object]: + return obj.get if isinstance(obj, Mapping) else lambda key: getattr(obj, key, None) + + +def _chat_message_reader(response_obj: object) -> Callable[[str], object] | None: + """Field access over the assistant message of a chat response, or None for a payload + with no readable message.""" + choice: Final = _chat_choice(response_obj) + if choice is None: + return None + message: Final = _field_reader(choice)("message") + return _field_reader(message) if message is not None else None + + +def _chat_final_text(response_obj: object) -> str: + """The turn's judgeable text: prose, or every tool call serialized alongside it as + `[tool call] name(arguments)` when the assistant chose to act instead of, or as well + as, answering directly. A tool call is a real turn, not a gap, so this is what both + the real arm's sampling decision and the shadow arm's reply compare against.""" + read: Final = _chat_message_reader(response_obj) + if read is None: return "" - read: Final = message.get if isinstance(message, Mapping) else lambda key: getattr(message, key, None) - if read("tool_calls") or read("function_call"): - return "" - return extract_text_from_content(read("content")) + prose: Final = extract_text_from_content(read("content")) + if not (read("tool_calls") or read("function_call")): + return prose + serialized: Final = _serialize_tool_calls(read) + return f"{prose} {serialized}".strip() if prose else serialized + + +def _chat_finish_reason(response_obj: object) -> str: + choice: Final = _chat_choice(response_obj) + raw: Final = _field_reader(choice)("finish_reason") if choice is not None else None + return str(raw) if raw else "unknown" + + +_RESPONSES_TOOL_CALL_TYPES: Final = frozenset(("function_call", "custom_tool_call")) + + +def _tool_calls_list(read: Callable[[str], object]) -> tuple[object, ...]: + calls: Final = read("tool_calls") + listed: Final = tuple(calls) if isinstance(calls, Sequence) and not isinstance(calls, str) else () + single: Final = read("function_call") + return listed if listed else ((single,) if single is not None else ()) + + +def _tool_call_invocation(call: object) -> str: + """One tool call as `name(arguments)`. Custom tool calls name themselves and carry their + arguments under `custom` rather than `function`.""" + read_call: Final = _field_reader(call) + payload: Final = read_call("function") or read_call("custom") or call + read_payload: Final = _field_reader(payload) + name: Final = read_payload("name") + arguments: Final = read_payload("arguments") or read_payload("input") or "" + return f"{name or 'unnamed'}({arguments})" + + +def _serialize_tool_calls(read: Callable[[str], object]) -> str: + """Every tool call in a reply as text a judge built for prose can still read.""" + return ", ".join(f"[tool call] {_tool_call_invocation(call)}" for call in _tool_calls_list(read)) + + +def _shadow_empty_reply_error(response_obj: object, routed_model: str) -> str: + """Why a shadow reply yielded no judgeable text at all: no prose, and no tool call to + serialize either. The stable sentence comes first and every varying part after the + semicolon, so grouping rows by error still yields one row per cause.""" + detail: Final = f"finish_reason={_chat_finish_reason(response_obj)}, model={routed_model or 'unknown'}" + return f"shadow router returned an empty response; {detail}" def _responses_final_text(response_obj: object) -> str: - """The turn's aggregated output text, or empty when the turn carries tool calls. A - dict-shaped payload is validated into the owner type first, because ``output_text`` - is a derived property rather than a serialized field, so it never exists on a dict; - a dict the owner type rejects is unjudgeable and skipped.""" + """The turn's judgeable text: the aggregated output plus any tool call serialized + alongside it, the same way the chat surface renders one. A dict-shaped payload is + validated into the owner type first, because ``output_text`` is a derived property + rather than a serialized field, so it never exists on a dict; a dict the owner type + rejects is unjudgeable and skipped.""" from litellm.types.llms.openai import ResponsesAPIResponse try: @@ -199,11 +262,16 @@ def _responses_final_text(response_obj: object) -> str: if not isinstance(output, Sequence): return "" items: Final = tuple(item.model_dump() if isinstance(item, BaseModel) else item for item in output) - if any( - not isinstance(item, Mapping) or item.get("type") in ("function_call", "custom_tool_call") for item in items - ): + if any(not isinstance(item, Mapping) for item in items): return "" - return str(getattr(response, "output_text", "") or "") + calls: Final = tuple( + item for item in items if isinstance(item, Mapping) and item.get("type") in _RESPONSES_TOOL_CALL_TYPES + ) + prose: Final = str(getattr(response, "output_text", "") or "") + if not calls: + return prose + serialized: Final = ", ".join(f"[tool call] {_tool_call_invocation(call)}" for call in calls) + return f"{prose} {serialized}".strip() if prose else serialized class _SurfaceOps: @@ -273,8 +341,8 @@ def _judgeable_sample( response_obj: object, ) -> tuple[tuple[Mapping[str, object], ...], Mapping[str, object], str] | None: """The normalized chat conversation, the forwardable generation params, and the - judgeable final text; None when this request's shapes cannot be sampled (tool-final - turn, empty text, or a shape the owner transformations reject).""" + judgeable final text; None when this request's shapes cannot be sampled (no text and no + tool call to serialize, or a shape the owner transformations reject).""" try: request: Final = ops.chat_request(kwargs, model_parameters) items: Final = _MESSAGE_ITEMS_ADAPTER.validate_python(request.get("messages")) @@ -307,6 +375,11 @@ PAIRWISE_JUDGE_SYSTEM_PROMPT: Final = """You are an impartial quality judge comp The responses are labeled A and B in random order. You do not know which system produced which. +A response may be prose, or a tool call shown as `[tool call] name(arguments)` if the +assistant chose to act instead of answering directly. A tool call is not a defect: judge +whether calling that tool was the right response to the conversation, the same as you +would judge prose. + Criteria: correctness, completeness, clarity, conciseness. Return ONLY valid JSON in this exact format, no other text: @@ -376,14 +449,37 @@ def _unmask_preference(raw_preference: str, real_is_a: bool) -> str: return "tie" -def _judge_user_prompt(conversation: str, response_a: str, response_b: str) -> str: +_MAX_JUDGE_TOOL_DEFS_CHARS: Final = 2_000 + + +def _tool_definitions_text(tools: object) -> str: + """The tools available to both arms, name and description only: enough for the judge + to tell whether the chosen tool, and not some other one, was the right call, without + forwarding parameter schemas it does not need to score that.""" + if not isinstance(tools, Sequence) or isinstance(tools, str): + return "" + entries: Final = tuple( + _field_reader(t)("function") or _field_reader(t)("custom") or t for t in tools if not isinstance(t, str) + ) + lines: Final = tuple( + f"- {_field_reader(e)('name') or 'unnamed'}: {_field_reader(e)('description') or 'no description'}" + for e in entries + ) + if not lines: + return "" + return ("Tools available to both responses:\n" + "\n".join(lines))[:_MAX_JUDGE_TOOL_DEFS_CHARS] + + +def _judge_user_prompt(conversation: str, response_a: str, response_b: str, tool_definitions: str = "") -> str: """The judge prompt under one total character budget: each response is capped, and - the conversation tail gets whatever budget the responses left over.""" + the conversation tail gets whatever budget the responses and tool definitions left + over.""" a: Final = response_a[:_MAX_JUDGE_RESPONSE_CHARS] b: Final = response_b[:_MAX_JUDGE_RESPONSE_CHARS] - conversation_budget: Final = _MAX_JUDGE_PROMPT_CHARS - len(a) - len(b) + prefix: Final = f"{tool_definitions}\n\n" if tool_definitions else "" + conversation_budget: Final = _MAX_JUDGE_PROMPT_CHARS - len(a) - len(b) - len(prefix) return ( - f"Conversation:\n{conversation[-conversation_budget:]}\n\n" + f"{prefix}Conversation:\n{conversation[-conversation_budget:]}\n\n" f"Response A:\n{a}\n\n" f"Response B:\n{b}\n\n" "Which response is better?" @@ -942,6 +1038,7 @@ class ShadowEvalLogger(CustomLogger): messages=messages, real_text=real_text, shadow_text=shadow.text, + tools=shadow_params.get("tools"), parent_metadata=parent_metadata, ) if isinstance(verdict, _CallFailure): @@ -1080,15 +1177,18 @@ class ShadowEvalLogger(CustomLogger): classifier_cost=_decision_classifier_cost(shadow_metadata), ) text: Final = _chat_final_text(response) + routed_model: Final = str( + getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or "" + ) if not text: return _CallFailure( - "shadow router returned an empty response", + _shadow_empty_reply_error(response, routed_model), cost=_call_cost(response), classifier_cost=_decision_classifier_cost(shadow_metadata), ) return _ShadowResponse( text=text, - model=str(getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or ""), + model=routed_model, tier=_routed_tier(shadow_metadata), cost=_call_cost(response), classifier_cost=_decision_classifier_cost(shadow_metadata), @@ -1100,9 +1200,12 @@ class ShadowEvalLogger(CustomLogger): messages: Sequence[Mapping[str, object]], real_text: str, shadow_text: str, + tools: object, parent_metadata: Mapping[str, object], ) -> "_JudgeVerdict | _CallFailure": - """Blind pairwise judge with A/B labels randomized to cancel position bias.""" + """Blind pairwise judge with A/B labels randomized to cancel position bias. Both + arms were offered the same tools, so the judge is shown their definitions too: a + tool call is only assessable against what else was available to call instead.""" real_is_a: Final = random.random() < 0.5 response_a: Final = real_text if real_is_a else shadow_text response_b: Final = shadow_text if real_is_a else real_text @@ -1117,7 +1220,7 @@ class ShadowEvalLogger(CustomLogger): {"role": "system", "content": PAIRWISE_JUDGE_SYSTEM_PROMPT}, # mutable-ok: SDK message { "role": "user", - "content": _judge_user_prompt(conversation, response_a, response_b), + "content": _judge_user_prompt(conversation, response_a, response_b, _tool_definitions_text(tools)), }, # mutable-ok: SDK message ] try: diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index 5628d69de26..f273a285d49 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -24,7 +24,13 @@ from litellm.integrations.shadow_eval_logger import ( _unmask_preference, ) from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN, ModelResponse +from litellm.types.utils import ( + SHADOW_EVAL_JUDGE_CALL_ORIGIN, + SHADOW_EVAL_ROUTER_CALL_ORIGIN, + ChatCompletionCustomToolCallPayload, + ChatCompletionMessageCustomToolCall, + ModelResponse, +) def _job(**overrides) -> ActiveShadowEvalJob: @@ -120,6 +126,39 @@ def _router( return router +def _shadow_reply_router(message, finish_reason="stop", routed_model="cheap-model"): + """A router whose shadow arm answers with a caller-supplied message, so a reply that + yields no judgeable text can be posed as the two different things it can be: an arm + that chose a tool, or an arm that returned nothing.""" + router = MagicMock() + router.model_group_alias = {} + router.get_model_list = MagicMock(return_value=[{"litellm_params": {"model": "openai/gpt-4o-mini"}}]) + + async def acompletion(**kwargs): + if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN: + return {"choices": [{"message": {"content": '{"preference": "A", "confidence": 0.9}'}}]} + kwargs["metadata"]["routing_decision"] = {"tier_label": "SIMPLE", "routed_model": routed_model} + return {"choices": [{"message": message, "finish_reason": finish_reason}]} + + router.acompletion = MagicMock(side_effect=acompletion) + return router + + +TOOL_CALL_MESSAGE = { + "content": None, + "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}], +} + +CUSTOM_TOOL_CALL_MESSAGE = { + "content": None, + "tool_calls": [ + ChatCompletionMessageCustomToolCall( + id="c2", custom=ChatCompletionCustomToolCallPayload(name="exec_sql", input="select 1") + ) + ], +} + + def _spend_counter(store=None): """In-memory stand-in for the proxy's cross-pod spend counter: reads take the max of the counter and the caller's fallback, exactly like get_current_spend does for a key @@ -368,7 +407,13 @@ class TestSurfaceNormalization: ], ids=["tool-final-chat-turn", "tool-final-responses-turn"], ) - async def test_unjudgeable_turns_are_skipped_without_consuming_budget(self, response_mutation, kwargs_mutation): + async def test_a_tool_final_turn_is_sampled_and_serialized_for_the_judge( + self, response_mutation, kwargs_mutation + ): + """A turn where the real model called a tool used to be dropped before sampling, on + every surface. On agentic traffic that is most of the traffic, so a job set to + sample 10% was really sampling 10% of the prose-only slice and calling it 10% of + the key. The turn is sampled like any other and the call is serialized as text.""" from litellm.types.llms.openai import ResponsesAPIResponse hook_kwargs = _success_kwargs(**({"call_type": "acompletion"} | kwargs_mutation)) @@ -406,6 +451,38 @@ class TestSurfaceNormalization: prisma, router = await self._drive(hook_kwargs, response) + judge_prompt = next( + call.kwargs["messages"][-1]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + assert "[tool call] f({})" in judge_prompt + prisma.db.litellm_shadowevalattempt.create.assert_called_once() + + @pytest.mark.parametrize( + "response_mutation,kwargs_mutation", + [ + ("chat-no-content", {}), + ("responses-no-output", {"call_type": "aresponses"}), + ], + ids=["empty-chat-turn", "empty-responses-turn"], + ) + async def test_turns_with_nothing_to_compare_are_skipped_without_consuming_budget( + self, response_mutation, kwargs_mutation + ): + """No prose and no tool call leaves the judge nothing to score, so the turn is + still skipped rather than billed.""" + from litellm.types.llms.openai import ResponsesAPIResponse + + hook_kwargs = _success_kwargs(**({"call_type": "acompletion"} | kwargs_mutation)) + if response_mutation == "chat-no-content": + response = {"choices": [{"message": {"content": ""}}]} + else: + hook_kwargs["messages"] = "do the thing" + response = ResponsesAPIResponse.model_validate(RESPONSES_API_RESPONSE | {"output": []}) + + prisma, router = await self._drive(hook_kwargs, response) + router.acompletion.assert_not_called() prisma.db.litellm_shadowevalattempt.create.assert_not_called() @@ -1134,6 +1211,206 @@ class TestShadowPipeline: assert row["shadow_cost"] == 0.007 assert logger._test_counter["spend:shadow_eval:job-1"] == 0.007 + async def _no_text_error(self, router) -> str: + prisma = _prisma() + await _logger(router=router, prisma=prisma)._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"] + assert row["outcome"] == "error" + return row["error"] + + async def _judged_shadow_row(self, router: MagicMock, shadow_params: dict | None = None) -> dict: + prisma = _prisma() + await _logger(router=router, prisma=prisma)._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params=shadow_params or {}, + parent_metadata={}, + ) + return prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"] + + async def test_a_tool_call_shadow_reply_is_judged_rather_than_discarded(self): + """An arm that calls a tool where the real model wrote prose has answered, it just + answered by acting. Dropping that turn threw away the comparison the job exists to + make, and on agentic traffic it threw away most of them, so the tool call is + serialized into text and judged like any other response.""" + row = await self._judged_shadow_row(_shadow_reply_router(TOOL_CALL_MESSAGE, finish_reason="tool_calls")) + + assert row["outcome"] != "error" + assert row["error"] is None + assert row["confidence"] == 0.9 + + async def test_a_tool_call_reaches_the_judge_as_readable_text(self): + """The judge only ever sees strings, so a tool call has to arrive as its name and + arguments. A serialization that dropped either would ask the judge to score a + response it cannot tell apart from any other tool call.""" + router = _shadow_reply_router(TOOL_CALL_MESSAGE, finish_reason="tool_calls") + await self._judged_shadow_row(router) + + judge_prompt = next( + call.kwargs["messages"][-1]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + + assert "[tool call] Read({})" in judge_prompt + + async def test_the_judge_sees_what_tools_were_available(self): + """Scoring whether a tool call was the right response needs to know what else the + arm could have called instead. Without the tool list, the judge can score the + arguments but not whether Read, specifically, was the correct choice.""" + router = _shadow_reply_router(TOOL_CALL_MESSAGE, finish_reason="tool_calls") + tools = [ + {"type": "function", "function": {"name": "Read", "description": "read a file from disk"}}, + {"type": "function", "function": {"name": "Bash", "description": "run a shell command"}}, + ] + await self._judged_shadow_row(router, shadow_params={"tools": tools}) + + judge_prompt = next( + call.kwargs["messages"][-1]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + + assert "Read: read a file from disk" in judge_prompt + assert "Bash: run a shell command" in judge_prompt + + async def test_a_custom_tool_definition_is_named_for_the_judge(self): + """A custom tool definition nests name and description under `custom`, not + `function`, so reading only `function` renders every one of them as unnamed and + tells the judge nothing about what the arm could have called.""" + from openai.types.chat import ChatCompletionCustomToolParam + + router = _shadow_reply_router(TOOL_CALL_MESSAGE, finish_reason="tool_calls") + tools = [ + ChatCompletionCustomToolParam( + type="custom", + custom={"name": "exec_sql", "description": "run a read-only sql query"}, + ) + ] + await self._judged_shadow_row(router, shadow_params={"tools": tools}) + + judge_prompt = next( + call.kwargs["messages"][-1]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + + assert "exec_sql: run a read-only sql query" in judge_prompt + assert "unnamed" not in judge_prompt + + @pytest.mark.parametrize("shadow_params", [{}, {"tools": []}], ids=["omitted", "empty-list"]) + async def test_no_tool_definitions_section_when_the_turn_offered_no_tools(self, shadow_params): + """Padding every judge prompt with an empty tools section wastes budget on the + turns, still the majority, that never offered one, whether tools was left out of + the request entirely or sent as an empty list.""" + router = _shadow_reply_router({"content": "hello"}, finish_reason="stop") + await self._judged_shadow_row(router, shadow_params=shadow_params) + + judge_prompt = next( + call.kwargs["messages"][-1]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + + assert "Tools available" not in judge_prompt + + async def test_a_custom_tool_call_serializes_its_name_and_input(self): + """Custom tool calls carry no `function` key: name and arguments live under + `custom`, so reading only `function` serializes every one of them as unnamed.""" + router = _shadow_reply_router(CUSTOM_TOOL_CALL_MESSAGE, finish_reason="tool_calls") + await self._judged_shadow_row(router) + + judge_prompt = next( + call.kwargs["messages"][-1]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + + assert "[tool call] exec_sql(select 1)" in judge_prompt + + async def test_the_judge_is_told_a_tool_call_is_not_a_defect(self): + """The judge scores on completeness and clarity. Handed a tool call with no + instruction, it marks it down for not reading like an answer, which would bias + every verdict against a tool-calling arm on exactly the traffic that calls tools.""" + router = _shadow_reply_router(TOOL_CALL_MESSAGE, finish_reason="tool_calls") + await self._judged_shadow_row(router) + + system_prompt = next( + call.kwargs["messages"][0]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + + assert "tool call" in system_prompt + assert "not a defect" in system_prompt + + async def test_prose_written_alongside_a_tool_call_survives_into_the_verdict(self): + """Some providers write a sentence before acting. Serializing only the call would + hide half of what the arm actually said from the judge.""" + router = _shadow_reply_router( + {"content": "Let me look that up.", "tool_calls": TOOL_CALL_MESSAGE["tool_calls"]}, + finish_reason="tool_calls", + ) + await self._judged_shadow_row(router) + + judge_prompt = next( + call.kwargs["messages"][-1]["content"] + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_ROUTER_CALL_ORIGIN + ) + + assert "Let me look that up. [tool call] Read({})" in judge_prompt + + async def test_an_empty_shadow_reply_names_the_finish_reason_and_the_routed_model(self): + """A reply that really carried no text is diagnosable only if the row says what + the arm was doing when it produced none: a truncated turn and a model that answers + with nothing are different faults with different fixes.""" + error = await self._no_text_error( + _shadow_reply_router({"content": ""}, finish_reason="length", routed_model="some-model") + ) + + assert "empty response" in error + assert "finish_reason=length" in error + assert "model=some-model" in error + + async def test_no_text_errors_stay_groupable_across_models_and_finish_reasons(self): + """Operators read these rows by grouping on the error text, which is how a job's + failures collapse to a handful of causes. Every varying part therefore has to sit + behind the first semicolon, or each row becomes its own group and the count that + made the problem visible stops existing.""" + first = await self._no_text_error( + _shadow_reply_router({"content": None}, finish_reason="length", routed_model="model-a") + ) + second = await self._no_text_error( + _shadow_reply_router( + {"content": ""}, + finish_reason="stop", + routed_model="model-b", + ) + ) + + assert first != second + assert first.split(";")[0] == second.split(";")[0] + async def test_a_pipeline_error_after_the_shadow_call_keeps_its_billed_cost(self, monkeypatch: pytest.MonkeyPatch): """An unexpected error between the billed shadow call and the attempt write must still record the shadow cost, or the per-key dollar gate undercounts forever.""" @@ -1691,11 +1968,13 @@ class TestSamplingFunnel: prisma.db.litellm_shadowevalattempt.create.assert_not_awaited() async def test_an_unjudgeable_sampled_request_counts_unjudgeable(self): + """A tool call still serializes into judgeable text; a turn with neither prose nor + a tool call to serialize is the one case left with nothing to compare.""" prisma = _prisma() logger = _logger(router=_router(), prisma=prisma, jobs=(_job(),)) - tool_final = {"choices": [{"message": {"content": None, "tool_calls": [{"type": "function", "function": {}}]}}]} + empty = {"choices": [{"message": {"content": None}}]} - await logger.async_log_success_event(_success_kwargs(), tool_final, None, None) + await logger.async_log_success_event(_success_kwargs(), empty, None, None) await _drain(logger) assert logger._test_funnel == [("job-1", "unjudgeable")] From d8ca43a800d2c814655981e2c1a565a4a029c9e9 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 4 Sep 2026 18:59:50 -0700 Subject: [PATCH 60/60] feat(complexity_router): let the LLM classifier see request images (#39825) The classifier scores extracted text, so a turn whose complexity lives in its image is invisible to it: a screenshot of a stack trace classifies on its caption, and an image-only turn flattens to empty text and never reaches the classifier at all. classifier_llm_config.vision opts in, off by default, with max_images bounding what one turn can add. Images are still dropped when the classifier model is declared supports_vision false. Anthropic and Responses image parts are rewritten into chat-completions dialect before they reach the classifier call, since /v1/messages hands the pre-routing hook its own dialect untranslated. The local scorer no longer short-circuits heuristic_first or hybrid on a turn carrying forwarded images, because it reads text alone and its confidence describes a request it has only partly seen. --- .../prompt_templates/common_utils.py | 39 +++ .../adapters/transformation.py | 16 +- .../complexity_router/complexity_router.py | 144 +++++++-- .../complexity_router/config.py | 35 +++ .../router_strategy/test_complexity_router.py | 274 ++++++++++++++++++ .../build_complexity_router_config.test.ts | 6 + .../build_complexity_router_config.ts | 11 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 24 ++ 8 files changed, 507 insertions(+), 42 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 17ebde83eee..00b80839dde 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -229,6 +229,45 @@ def _content_parts_contain_image(parts: Sequence[object]) -> bool: return False +def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> str | None: + """Data or remote URL for an Anthropic ``source`` block, in the form chat completions expects.""" + source_type: Final = image_source.get("type") + if source_type == "base64": + media_type: Final = image_source.get("media_type") or "image/jpeg" + image_data: Final = image_source.get("data") or "" + return f"data:{media_type};base64,{image_data}" if image_data else None + if source_type == "url": + url: Final = image_source.get("url") + return url if isinstance(url, str) else "" + return None + + +def _image_part_url(part: Mapping[str, object]) -> str | None: + """The image URL carried by one content part, whichever of the three dialects wrote it.""" + part_type: Final = part.get("type") + if part_type == "image_url": + image_url: Final = part.get("image_url") + if isinstance(image_url, str): + return image_url + return image_url.get("url") if isinstance(image_url, Mapping) else None + if part_type == "input_image": + responses_url: Final = part.get("image_url") + return responses_url if isinstance(responses_url, str) else None + if part_type == "image": + source: Final = part.get("source") + return anthropic_image_source_to_openai_url(source) if isinstance(source, Mapping) else None + return None + + +def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObject | None: + """One image content part rewritten into chat-completions dialect, or None when it is not one. + + Rebuilt rather than forwarded so no caller-controlled key beyond the URL rides along. + """ + url: Final = _image_part_url(part) + return {"type": "image_url", "image_url": {"url": url}} if url else None + + def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool: """Whether any message carries an image content part, across the dialects that reach pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``, diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 64f10046109..594fac512e6 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -99,6 +99,7 @@ def create_tool_name_mapping( from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice from litellm.litellm_core_utils.prompt_templates.common_utils import ( + anthropic_image_source_to_openai_url, parse_tool_call_arguments, reasoning_content_from_thinking_blocks, with_prompt_cache_breakpoint, @@ -1225,20 +1226,7 @@ class LiteLLMAnthropicMessagesAdapter: """ if not isinstance(image_source, dict): return None - - source_type: Final = image_source.get("type") - - if source_type == "base64": - # Base64 image format - media_type: Final = image_source.get("media_type", "image/jpeg") - image_data: Final = image_source.get("data", "") - if image_data: - return f"data:{media_type};base64,{image_data}" - elif source_type == "url": - # URL-referenced image format - return image_source.get("url", "") - - return None + return anthropic_image_source_to_openai_url(image_source) def _tool_result_content(self, raw_content: object) -> ToolResultContent: if isinstance(raw_content, str): diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index ec11c16eea0..98a1eb7ac9e 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -40,7 +40,10 @@ from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata -from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + as_openai_image_part, + request_contains_image_content, +) from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.router_strategy.adaptive_router.classifier import classify_prompt @@ -48,7 +51,11 @@ from litellm.router_strategy.complexity_router.tier_predictor import ( TierSuccessPredictor, resolve_tier_artifact, ) -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionImageObject, + ChatCompletionTextObject, +) from litellm.types.utils import ( AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, @@ -435,6 +442,23 @@ def _strip_reminder_blocks(text: str, marker_pairs: tuple[tuple[str, str], ...] return " ".join(kept for a, b in zip(keep_from, keep_to) if (kept := text[a:b].strip())) +def _inline_image_part(part: Mapping[str, object]) -> ChatCompletionImageObject | None: + """One image content part safe to hand the classifier, or None. + + Inline data URIs only. A remote URL is caller-controlled and provider adapters do not uniformly + delegate fetching to the provider: gigachat's file handler downloads any non-data URL with + `client.get` from the proxy host, so forwarding one would let a key scoped to this router aim a + proxy-side request at an internal address, on a call the caller never asked for. The routed + model still receives the original URL exactly as before. + """ + converted: Final = as_openai_image_part(part) + if converted is None: + return None + image_url: Final = converted["image_url"] + url: Final = image_url if isinstance(image_url, str) else image_url.get("url", "") + return converted if url.startswith("data:") else None + + def _human_text(content: object, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS) -> str: """Message content as the text a human wrote, with complete reminder blocks removed. @@ -1592,6 +1616,10 @@ class ComplexityRouter(CustomLogger): threshold check alone would hand that traffic to the cheapest model without ever consulting the classifier. Scores also go negative when simple indicators fire, so a score threshold would reject exactly the trivial prompts this path exists to serve. + + A turn carrying images the classifier would see is never decided cheaply: the scorer reads + text alone, so its confidence describes a request it has only partly seen, and a trivial + caption beside a screenshot is exactly the misrouting vision classification exists to stop. """ tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) @@ -1599,6 +1627,7 @@ class ComplexityRouter(CustomLogger): decided_cheaply: Final = ( threshold is not None and bool(signals) + and not self._classifier_image_parts(messages) and self._active_tier_severity(tier) <= self._active_tier_severity(threshold) ) if decided_cheaply: @@ -1623,11 +1652,43 @@ class ComplexityRouter(CustomLogger): tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) margin: Final = self.config.hybrid_boundary_margin - decided: Final = margin is not None and bool(signals) and not self._is_near_tier_boundary(score, margin) + decided: Final = ( + margin is not None + and bool(signals) + and not self._classifier_image_parts(messages) + and not self._is_near_tier_boundary(score, margin) + ) if decided: return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="hybrid_short_circuit") return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored) + def _classifier_image_parts( + self, messages: Sequence[Mapping[str, object]] | None + ) -> tuple[ChatCompletionImageObject, ...]: + """Images from the newest user turn to hand the classifier, capped by max_images. + + Empty unless the operator opted in AND the classifier model is declared vision-capable, so + every other deployment keeps today's text-only payload byte for byte. Only the newest user + turn is read: earlier turns are context the classifier already gets as quoted text, and an + image nested in a tool_result is tool output rather than the ask being classified. + Remote-URL images are left out entirely; `_inline_image_part` carries why. + """ + llm_config: Final = self.config.classifier_llm_config + if llm_config is None or not llm_config.vision.enabled or not self.config.uses_llm_classifier or not messages: + return () + if not self._model_declares_vision_support(llm_config.model): + return () + newest_user_turn: Final = next((msg for msg in reversed(messages) if msg.get("role") == "user"), None) + content: Final = newest_user_turn.get("content") if newest_user_turn is not None else None + if not isinstance(content, list): + return () + return tuple( + islice( + (part for raw in content if isinstance(raw, Mapping) and (part := _inline_image_part(raw)) is not None), + llm_config.vision.max_images, + ) + ) + async def _llm_classifier_outcome( self, prompt: str, @@ -1865,9 +1926,18 @@ class ComplexityRouter(CustomLogger): } turn_off_message_logging: Final = _effective_turn_off_message_logging(request_kwargs) + image_parts: Final = self._classifier_image_parts(messages) + user_content: Final[str | Sequence[ChatCompletionTextObject | ChatCompletionImageObject]] = ( + [ # mutable-ok: SDK request payload content list is built once + {"type": "text", "text": user_payload}, + *image_parts, + ] + if image_parts + else user_payload + ) messages_for_call: Final[list[AllMessageValues]] = [ # mutable-ok: SDK request payload list is built once {"role": "system", "content": classifier_system_prompt}, - {"role": "user", "content": user_payload}, + {"role": "user", "content": user_content}, ] response_format: Final = classifier_response_format classifier_call_params: Mapping[str, str] = EMPTY_MAPPING @@ -2558,31 +2628,53 @@ class ComplexityRouter(CustomLogger): return pinned_model return self.get_model_for_tier(escalated_tier) - def _model_accepts_image_input(self, model_name: str) -> bool: - """Whether a routed model or pool entry can serve an image request. + def _vision_verdicts(self, model_name: str) -> tuple[bool | None, ...]: + """Declared vision support per deployment serving the name: True, False, or None when + nothing declares either way. Resolved through the deployments that would actually serve the name; a name with no deployment on the router is served by the SDK directly and is checked against the model - cost map itself. Only an explicit supports_vision false excludes, a deployment-level - model_info override first and the map otherwise, so unmapped custom names stay routable. + cost map itself. A deployment-level model_info override wins over the map. + + One verdict set, two readings, because the two callers fail in opposite directions. + Routing a user's image asks whether anything RULES IT OUT, so an undeclared model stays + eligible and unmapped custom names keep routing. Handing an image to the classifier asks + whether something RULES IT IN: an undeclared model that turns out to be text-only rejects + every image request, and that rejection is swallowed by the classifier's own fallback, so + the router quietly serves all image traffic from the fallback tier and pays for the failed + call each time. An undeclared model instead keeps today's text-only payload, which is a + visible no-op the operator fixes by declaring supports_vision on the deployment. + """ + from litellm.utils import is_vision_explicitly_disabled, supports_vision + + def model_verdict(model: str) -> bool | None: + if supports_vision(model): + return True + return False if is_vision_explicitly_disabled(model) else None + + def deployment_verdict(deployment: Mapping[str, Any]) -> bool | None: + declared: Final = (deployment.get("model_info") or EMPTY_MAPPING).get("supports_vision") + if declared is not None: + return declared is True + return model_verdict((deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name) + + deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name) + if not deployments: + return (model_verdict(model_name),) + return tuple(deployment_verdict(deployment) for deployment in deployments) + + def _model_accepts_image_input(self, model_name: str) -> bool: + """Whether a routed model or pool entry can serve an image request. A multi-deployment group must accept on EVERY deployment: the router picks a deployment inside the group after this gate runs, so a mixed group marked eligible could still hand the image to its text-only member and fail with the exact 400 the gate exists to prevent. """ - from litellm.utils import is_vision_explicitly_disabled + return all(verdict is not False for verdict in self._vision_verdicts(model_name)) - def deployment_accepts(deployment: Mapping[str, Any]) -> bool: - declared: Final = (deployment.get("model_info") or EMPTY_MAPPING).get("supports_vision") - if declared is not None: - return declared is True - litellm_model: Final = (deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name - return not is_vision_explicitly_disabled(litellm_model) - - deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name) - if not deployments: - return not is_vision_explicitly_disabled(model_name) - return all(deployment_accepts(deployment) for deployment in deployments) + def _model_declares_vision_support(self, model_name: str) -> bool: + """Whether every deployment serving the name is declared vision-capable.""" + return all(verdict is True for verdict in self._vision_verdicts(model_name)) def _modality_eligible_models(self) -> frozenset[str]: """Every configured pool entry, plus default_model, that can serve an image request.""" @@ -3374,8 +3466,9 @@ class ComplexityRouter(CustomLogger): has_original_messages: Final = messages is not None and len(messages) > 0 user_message, system_prompt = _extract_current_ask_and_system_prompt(resolved_messages, self._reminder_markers) + classifier_images: Final = self._classifier_image_parts(resolved_messages) - if user_message is None: + if user_message is None and not classifier_images: verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model") default_model_first: Final = not self.config.plugins and self.config.default_model if default_model_first: @@ -3402,6 +3495,7 @@ class ComplexityRouter(CustomLogger): ), ) + ask: Final = user_message or "" newest_ask: Final = _newest_turn_ask(resolved_messages, self._reminder_markers) escalation_keyword: Final = self._matched_escalation_keyword(newest_ask) if newest_ask is not None else None # Resolved here rather than beside the classifier because the keyword-override path below @@ -3439,7 +3533,7 @@ class ComplexityRouter(CustomLogger): ), ) - override: Final = await self._resolve_keyword_tier_override(user_message, request_kwargs) + override: Final = await self._resolve_keyword_tier_override(ask, request_kwargs) if override is not None: keyword_bumped_tier: Final = ( self._escalate_tier(override.tier) if escalation_keyword is not None else override.tier @@ -3486,9 +3580,7 @@ class ComplexityRouter(CustomLogger): outcome: Final = ( ClassificationOutcome(tier=housekeeping_tier, score=None, signals=("housekeeping",), cause="housekeeping") if housekeeping_tier is not None - else await self.aclassify( - user_message, system_prompt, request_kwargs, resolved_messages, raw_messages=messages - ) + else await self.aclassify(ask, system_prompt, request_kwargs, resolved_messages, raw_messages=messages) ) tier, score, signals = outcome.tier, outcome.score, outcome.signals classified_tier: Final = tier @@ -3558,7 +3650,7 @@ class ComplexityRouter(CustomLogger): # under is not a floor. routed_model = self._soft_floor_pick( tier, - user_message, + ask, request_kwargs, hard_floor=tier if context_original_tier is not None else plan_floor, hard_ceiling=housekeeping_ceiling, diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 5848973a9c7..c483a0b7073 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -442,12 +442,47 @@ DEFAULT_TIER_MODELS: Final[dict[str, str]] = { } +class ClassifierVisionConfig(BaseModel): + """Whether the LLM classifier sees the images on the request it is classifying. + + Off by default because images cost far more than the text ask they arrive with, and the + classifier runs on every request. A turn whose complexity lives in the image ("what is wrong in + this stack trace screenshot") is invisible to a text-only classifier, which is what this buys. + """ + + enabled: bool = Field( + default=False, + description=( + "Forward image content to the classifier. Requires a classifier model declared " + "supports_vision, on the deployment's model_info or in the model cost map; images stay " + "stripped otherwise, so a classifier that cannot read them is never sent one. Declare " + "model_info.supports_vision on the deployment to enable a model the cost map does not " + "describe. Only inline data: URIs are forwarded. A request whose images are http(s) " + "URLs still classifies on its text alone, because some providers fetch such a URL from " + "the proxy rather than the provider, which would let a caller aim a proxy-side request " + "at an address of their choosing." + ), + ) + max_images: int = Field( + default=1, + ge=1, + description=( + "How many images from the newest user turn to forward, in wire order. Bounds the added " + "cost of a turn that attaches many images. Images on earlier turns are never forwarded." + ), + ) + + class ClassifierLLMConfig(BaseModel): """Configuration for the LLM-based complexity classifier.""" model: str = Field( description="Model name (from the router's model_list) to call for classification", ) + vision: ClassifierVisionConfig = Field( + default_factory=ClassifierVisionConfig, + description="Whether the classifier sees images on the request, and how many", + ) reasoning_effort: REASONING_EFFORT | None = Field( default=None, description=( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 8de138688f1..52e58304476 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -12429,3 +12429,277 @@ class TestTierHealthFailover: for _ in range(20) ] assert {r.model for r in results} == {"live-c"} + + +ANTHROPIC_IMG_PART = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}} +RESPONSES_IMG_PART = {"type": "input_image", "image_url": "data:image/png;base64,aGk="} + + +class TestClassifierVision: + """classifier_llm_config.vision: what the LLM classifier is shown for an image-bearing turn.""" + + TIERS = {"SIMPLE": "t-simple", "MEDIUM": "t-medium", "COMPLEX": "t-complex", "REASONING": "t-reasoning"} + + @staticmethod + def _router(mock_router_instance, *, vision, classifier_declares_vision=True, classifier_type="llm", **extra): + def get_model_list(model_name=None): + if model_name != "clf": + return [{"model_name": model_name, "litellm_params": {"model": "openai/gpt-4o"}}] + declared = classifier_declares_vision + return [ + { + "model_name": "clf", + "litellm_params": {"model": "openai/unmapped-classifier"}, + "model_info": {} if declared is None else {"supports_vision": declared}, + } + ] + + mock_router_instance.get_model_list = get_model_list + classifier_llm_config = {"model": "clf", "circuit_breaker_enabled": False} + return ComplexityRouter( + model_name="vision-classifier-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "classifier_type": classifier_type, + "classifier_llm_config": ( + classifier_llm_config if vision is None else {**classifier_llm_config, "vision": vision} + ), + "tiers": dict(TestClassifierVision.TIERS), + **extra, + }, + ) + + @staticmethod + def _classifier_user_content(mock_router_instance): + return mock_router_instance.acompletion.call_args.kwargs["messages"][-1]["content"] + + @staticmethod + def _turn(*parts): + return [{"role": "user", "content": list(parts)}] + + @pytest.fixture(autouse=True) + def _classifier_answers_complex(self, mock_router_instance): + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}')) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "vision, classifier_declares_vision", + [ + (None, True), + ({"enabled": False}, True), + ({"enabled": True}, False), + ({"enabled": True}, None), + ], + ids=["vision_unset", "vision_disabled", "classifier_declared_text_only", "classifier_undeclared"], + ) + async def test_payload_stays_text_only(self, mock_router_instance, vision, classifier_declares_vision): + """Off, or a classifier not declared vision-capable, keeps the plain-string payload. + + The undeclared case is the polarity. A text-only classifier handed an image rejects the + call, the rejection is swallowed by the classifier's own fallback, and every image request + then serves from the fallback tier while still paying for the failed call. Staying text-only + is instead a visible no-op the operator fixes by declaring supports_vision. + """ + router = self._router( + mock_router_instance, vision=vision, classifier_declares_vision=classifier_declares_vision + ) + await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART) + ) + content = self._classifier_user_content(mock_router_instance) + assert isinstance(content, str) + assert "what is this" in content + + @pytest.mark.asyncio + async def test_deployment_model_info_enables_a_classifier_the_cost_map_does_not_describe( + self, mock_router_instance + ): + """The escape hatch for an unmapped classifier name, and the reason undeclared can stay off. + + `_router` gives every deployment an `openai/unmapped-*` litellm_params model, so nothing in + the cost map declares it and the verdict comes only from model_info. + """ + router = self._router(mock_router_instance, vision={"enabled": True}, classifier_declares_vision=True) + await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART) + ) + assert [b["type"] for b in self._classifier_user_content(mock_router_instance)] == ["text", "image_url"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "part", + [IMG_PART, ANTHROPIC_IMG_PART, RESPONSES_IMG_PART], + ids=["chat_completions", "anthropic_messages", "responses"], + ) + async def test_image_reaches_the_classifier_in_chat_completions_dialect(self, mock_router_instance, part): + """Every surface's dialect arrives as a chat-completions image_url on the classifier call. + + /v1/messages hands the hook an Anthropic image block untranslated, so forwarding verbatim + would send the classifier a content part its own request dialect has no meaning for. + """ + router = self._router(mock_router_instance, vision={"enabled": True}) + await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, part) + ) + content = self._classifier_user_content(mock_router_instance) + assert [block["type"] for block in content] == ["text", "image_url"] + assert content[1]["image_url"] == {"url": "data:image/png;base64,aGk="} + assert "what is this" in content[0]["text"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "part", + [ + {"type": "image_url", "image_url": {"url": "http://169.254.169.254/latest/meta-data/"}}, + {"type": "image_url", "image_url": {"url": "https://example.internal/secret.png"}}, + {"type": "input_image", "image_url": "https://example.internal/secret.png"}, + {"type": "image", "source": {"type": "url", "url": "https://example.internal/secret.png"}}, + ], + ids=["metadata_service", "chat_completions", "responses", "anthropic"], + ) + async def test_remote_url_images_are_never_forwarded(self, mock_router_instance, part): + """A caller-supplied URL must not reach an internal call the caller did not ask for. + + Provider adapters do not uniformly delegate fetching: gigachat downloads any non-data URL + from the proxy host, so forwarding one would turn a router-scoped key into a proxy-side GET + at an address of the caller's choosing. + """ + router = self._router(mock_router_instance, vision={"enabled": True}) + await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, part) + ) + assert isinstance(self._classifier_user_content(mock_router_instance), str) + + @pytest.mark.asyncio + async def test_remote_url_image_only_turn_does_not_reach_the_classifier(self, mock_router_instance): + """With nothing forwardable left, the turn stays unclassifiable rather than sending the URL.""" + router = self._router(mock_router_instance, vision={"enabled": True}) + response = await router.async_pre_routing_hook( + model="m", + request_kwargs={}, + messages=self._turn({"type": "image_url", "image_url": {"url": "https://example.internal/x.png"}}), + ) + assert response.routing_decision["cause"] == "default_fallback" + mock_router_instance.acompletion.assert_not_awaited() + + @pytest.mark.asyncio + async def test_image_only_turn_is_classified_instead_of_falling_back(self, mock_router_instance): + """A turn carrying only an image reaches the classifier rather than the default model. + + It flattens to empty text, so before this it never reached the classifier at all and was + routed as default_fallback on text the request never contained. + """ + router = self._router(mock_router_instance, vision={"enabled": True}) + response = await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self._turn(IMG_PART) + ) + assert response.routing_decision["cause"] == "llm_classifier" + assert response.model == "t-complex" + assert [block["type"] for block in self._classifier_user_content(mock_router_instance)] == [ + "text", + "image_url", + ] + + @pytest.mark.asyncio + async def test_image_only_turn_still_falls_back_when_vision_is_off(self, mock_router_instance): + router = self._router(mock_router_instance, vision={"enabled": False}) + response = await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self._turn(IMG_PART) + ) + assert response.routing_decision["cause"] == "default_fallback" + mock_router_instance.acompletion.assert_not_awaited() + + @pytest.mark.asyncio + @pytest.mark.parametrize("max_images, expected", [(1, 1), (2, 2), (5, 3)]) + async def test_max_images_caps_what_is_forwarded(self, mock_router_instance, max_images, expected): + router = self._router(mock_router_instance, vision={"enabled": True, "max_images": max_images}) + images = [dict(IMG_PART, image_url={"url": f"data:image/png;base64,{n}"}) for n in ("a", "b", "c")] + await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "look"}, *images) + ) + content = self._classifier_user_content(mock_router_instance) + forwarded = [block for block in content if block["type"] == "image_url"] + assert len(forwarded) == expected + assert [block["image_url"]["url"] for block in forwarded] == [ + f"data:image/png;base64,{n}" for n in ("a", "b", "c")[:expected] + ] + + @pytest.mark.asyncio + async def test_earlier_turn_images_are_not_forwarded(self, mock_router_instance): + """Only the newest user turn's images ride along, so history cannot inflate every call. + + The two turns carry different images on purpose: identical ones would pass this assertion + whichever turn the helper read. + """ + older = dict(IMG_PART, image_url={"url": "data:image/png;base64,OLDER"}) + newer = dict(IMG_PART, image_url={"url": "data:image/png;base64,NEWER"}) + router = self._router(mock_router_instance, vision={"enabled": True, "max_images": 5}) + await router.async_pre_routing_hook( + model="m", + request_kwargs={}, + messages=[ + {"role": "user", "content": [{"type": "text", "text": "first"}, older]}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": [{"type": "text", "text": "second"}, newer]}, + ], + ) + content = self._classifier_user_content(mock_router_instance) + forwarded = [block for block in content if block["type"] == "image_url"] + assert [block["image_url"]["url"] for block in forwarded] == ["data:image/png;base64,NEWER"] + + @pytest.mark.asyncio + async def test_logged_request_body_matches_what_was_sent(self, mock_router_instance): + """proxy_server_request is the logged copy of the classifier call and must not drift.""" + router = self._router(mock_router_instance, vision={"enabled": True}) + await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART) + ) + call_kwargs = mock_router_instance.acompletion.call_args.kwargs + assert call_kwargs["proxy_server_request"]["body"]["messages"] == call_kwargs["messages"] + + SHORT_CIRCUIT_ARMS = [ + ("heuristic_first", {"heuristic_first_max_tier": "SIMPLE"}, "heuristic_first_short_circuit"), + ("hybrid", {"hybrid_boundary_margin": 0.05}, "hybrid_short_circuit"), + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"] + ) + async def test_local_scorer_cannot_short_circuit_a_turn_it_cannot_see( + self, mock_router_instance, classifier_type, extra, short_circuit_cause + ): + """The scorer reads text alone, so its confidence is not a verdict on an image turn. + + Both arms are tuned so the scorer WOULD short-circuit on this exact text, which is what + makes the image the only variable; a margin loose enough to leave the score undecided + would pass whether or not the guard exists. + """ + router = self._router( + mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra + ) + response = await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART) + ) + assert response.routing_decision["cause"] == "llm_classifier" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"] + ) + async def test_local_scorer_still_short_circuits_without_images( + self, mock_router_instance, classifier_type, extra, short_circuit_cause + ): + """The negative class: same router, same text, no image, and the scorer still decides.""" + router = self._router( + mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra + ) + response = await router.async_pre_routing_hook( + model="m", request_kwargs={}, messages=[{"role": "user", "content": "what is this"}] + ) + assert response.routing_decision["cause"] == short_circuit_cause + mock_router_instance.acompletion.assert_not_awaited() + + def test_max_images_must_be_positive(self): + with pytest.raises(ValidationError): + ClassifierLLMConfig(model="clf", vision={"enabled": True, "max_images": 0}) diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index edb32c6717f..1b5bb9e72eb 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -592,6 +592,12 @@ describe("classifier prompt and fallback", () => { timeout_ms: 1, }); }); + + it.each([{}, { system_prompt: "x" }])("normalizeClassifierLlmConfig carries vision through %o", (extra) => { + const base = { model: "m", timeout_ms: 1, ...extra }; + const vision = { enabled: true, max_images: 2 }; + expect(normalizeClassifierLlmConfig({ ...base, vision })).toEqual({ ...base, vision }); + }); }); describe("tier labels", () => { diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 1a395c38779..d7974484970 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -1,4 +1,7 @@ import { KeywordTierRule } from "./KeywordTierRules"; + +type ClassifierLLMConfigWire = ClassifierLLMConfig & { vision?: { enabled?: boolean; max_images?: number } }; + import type { ModelGroup } from "../llm_calls/fetch_models"; import { type CustomTierSet, @@ -61,7 +64,8 @@ export const normalizeClassifierLlmConfig = ({ reasoning_effort, classification_rubric, system_prompt, -}: ClassifierLLMConfig): ClassifierLLMConfig => + vision, +}: ClassifierLLMConfigWire): ClassifierLLMConfigWire => system_prompt?.trim() ? { model, @@ -69,6 +73,7 @@ export const normalizeClassifierLlmConfig = ({ ...(circuit_breaker_enabled !== undefined && { circuit_breaker_enabled }), ...(circuit_breaker_cooldown_seconds !== undefined && { circuit_breaker_cooldown_seconds }), ...(reasoning_effort && { reasoning_effort }), + ...(vision && { vision }), system_prompt, } : { @@ -78,6 +83,7 @@ export const normalizeClassifierLlmConfig = ({ ...(circuit_breaker_cooldown_seconds !== undefined && { circuit_breaker_cooldown_seconds }), ...(reasoning_effort && { reasoning_effort }), ...(classification_rubric && { classification_rubric }), + ...(vision && { vision }), }; interface ScorerKnobInputs { @@ -324,7 +330,7 @@ export const getSemanticConfigError = ({ }; interface CustomTierWireFieldInputs { - classifierLlmConfig: ClassifierLLMConfig | undefined; + classifierLlmConfig: ClassifierLLMConfigWire | undefined; planModeMinTierId: string | undefined; classificationPrompt: string | undefined; classificationExamples: string | undefined; @@ -356,6 +362,7 @@ export const customTierWireFields = ( circuit_breaker_cooldown_seconds: classifierLlmConfig.circuit_breaker_cooldown_seconds, }), ...(classifierLlmConfig.reasoning_effort && { reasoning_effort: classifierLlmConfig.reasoning_effort }), + ...(classifierLlmConfig.vision && { vision: classifierLlmConfig.vision }), }, }), session_affinity: false, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6f60905d4c1..45e1b1cab0b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25305,6 +25305,30 @@ export interface components { * @default 3000 */ timeout_ms: number; + /** @description Whether the classifier sees images on the request, and how many */ + vision?: components["schemas"]["ClassifierVisionConfig"]; + }; + /** + * ClassifierVisionConfig + * @description Whether the LLM classifier sees the images on the request it is classifying. + * + * Off by default because images cost far more than the text ask they arrive with, and the + * classifier runs on every request. A turn whose complexity lives in the image ("what is wrong in + * this stack trace screenshot") is invisible to a text-only classifier, which is what this buys. + */ + ClassifierVisionConfig: { + /** + * Enabled + * @description Forward image content to the classifier. Requires a classifier model declared supports_vision, on the deployment's model_info or in the model cost map; images stay stripped otherwise, so a classifier that cannot read them is never sent one. Declare model_info.supports_vision on the deployment to enable a model the cost map does not describe. Only inline data: URIs are forwarded. A request whose images are http(s) URLs still classifies on its text alone, because some providers fetch such a URL from the proxy rather than the provider, which would let a caller aim a proxy-side request at an address of their choosing. + * @default false + */ + enabled: boolean; + /** + * Max Images + * @description How many images from the newest user turn to forward, in wire order. Bounds the added cost of a turn that attaches many images. Images on earlier turns are never forwarded. + * @default 1 + */ + max_images: number; }; /** * CloudZeroExportRequest