From 78927138d288e6a780c83816b05f677b39b9e38e Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 12 Mar 2026 22:30:52 +0100 Subject: [PATCH 01/73] fix(proxy): add team_member_budget_duration to NewTeamRequest NewTeamRequest was missing the team_member_budget_duration field, causing Pydantic to silently drop the value when creating a team via POST /team/new. The template budget row was created without budget_duration or budget_reset_at, so the ResetBudgetJob never found it and team member spend was never reset. Add the field to NewTeamRequest and pass it through to should_create_budget and create_team_member_budget_table in the new_team handler (matching the existing update_team path which already works correctly). Fixes #16057 --- litellm/proxy/_types.py | 1 + .../management_endpoints/team_endpoints.py | 2 + .../test_team_endpoints.py | 50 +++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 1174740948b..9b94a81eb01 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1669,6 +1669,7 @@ class NewTeamRequest(TeamBase): int ] = None # allow user to set TPM limit for all team members team_member_key_duration: Optional[str] = None # e.g. "1d", "1w", "1m" + team_member_budget_duration: Optional[str] = None # e.g. "30d", "1mo" allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None enforced_batch_output_expires_after: Optional[dict] = None enforced_file_expires_after: Optional[dict] = None diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 9c8e6f7282b..18e41793df4 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -898,6 +898,7 @@ async def new_team( # noqa: PLR0915 team_member_budget=data.team_member_budget, team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, + team_member_budget_duration=data.team_member_budget_duration, ): data_json = await TeamMemberBudgetHandler.create_team_member_budget_table( data=data, @@ -906,6 +907,7 @@ async def new_team( # noqa: PLR0915 team_member_budget=data.team_member_budget, team_member_rpm_limit=data.team_member_rpm_limit, team_member_tpm_limit=data.team_member_tpm_limit, + team_member_budget_duration=data.team_member_budget_duration, ) ## ADD TO TEAM TABLE diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 1aee1d49658..8a033fca968 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -6157,3 +6157,53 @@ async def test_list_team_v1_batches_key_queries(): assert result[0].keys == [key1, key2] assert result[1].team_id == "team-2" assert result[1].keys == [key3] + + +def test_new_team_request_accepts_team_member_budget_duration(): + """Test that NewTeamRequest does not silently drop team_member_budget_duration.""" + from litellm.proxy._types import NewTeamRequest + + request = NewTeamRequest( + team_member_budget=20.0, + team_member_budget_duration="30d", + ) + assert request.team_member_budget == 20.0 + assert request.team_member_budget_duration == "30d" + + +@pytest.mark.asyncio +async def test_create_team_member_budget_table_with_duration(): + """Verify that create_team_member_budget_table passes budget_duration + through to the new_budget call when team_member_budget_duration is provided.""" + from litellm.proxy._types import NewTeamRequest, UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy.management_endpoints.team_endpoints import TeamMemberBudgetHandler + + mock_budget_response = MagicMock(budget_id="budget-abc") + mock_admin = UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + + data = NewTeamRequest( + team_alias="test-team", + team_member_budget=20.0, + team_member_budget_duration="30d", + ) + + with patch( + "litellm.proxy.management_endpoints.budget_management_endpoints.new_budget", + new_callable=AsyncMock, + return_value=mock_budget_response, + ) as mock_new_budget: + result = await TeamMemberBudgetHandler.create_team_member_budget_table( + data=data, + new_team_data_json={"metadata": None}, + user_api_key_dict=mock_admin, + team_member_budget=20.0, + team_member_budget_duration="30d", + ) + + mock_new_budget.assert_awaited_once() + budget_request = mock_new_budget.call_args.kwargs["budget_obj"] + assert budget_request.budget_duration == "30d" + assert budget_request.max_budget == 20.0 + assert result["metadata"]["team_member_budget_id"] == "budget-abc" From 4bd7bdcf43da7106bd07240acc2bd0b1c997936c Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 18 Mar 2026 22:30:49 -0300 Subject: [PATCH 02/73] fix: add additionalProperties: false for OpenAI strict mode in Anthropic adapter When translating Anthropic output_format to OpenAI response_format, the adapter sets strict: true but didn't add additionalProperties: false, which OpenAI requires at every object nesting level. This caused BadRequestError for structured output requests routed to OpenAI models. Fixes #20997 --- .../adapters/transformation.py | 37 +++++++ ...al_pass_through_adapters_transformation.py | 101 ++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 43a6fa8045d..47c9a223f85 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1,3 +1,4 @@ +import copy import hashlib import json from typing import ( @@ -824,6 +825,11 @@ class LiteLLMAnthropicMessagesAdapter: if not schema: return None + # Deep copy to avoid mutating the original schema + schema = copy.deepcopy(schema) + # OpenAI strict mode requires additionalProperties: false on every object + self._add_additional_properties_false(schema) + # Convert to OpenAI response_format structure return { "type": "json_schema", @@ -834,6 +840,37 @@ class LiteLLMAnthropicMessagesAdapter: }, } + @staticmethod + def _add_additional_properties_false(schema: dict) -> None: + """ + Recursively add 'additionalProperties': false to all object schemas. + + OpenAI's strict mode requires this at every object nesting level. + """ + if not isinstance(schema, dict): + return + + if schema.get("type") == "object" and "properties" in schema: + schema["additionalProperties"] = False + for prop in schema["properties"].values(): + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(prop) + + # Handle array items + if "items" in schema: + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(schema["items"]) + + # Handle anyOf/oneOf/allOf + for key in ("anyOf", "oneOf", "allOf"): + if key in schema: + for sub_schema in schema[key]: + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(sub_schema) + + # Handle $defs / definitions + for key in ("$defs", "definitions"): + if key in schema: + for def_schema in schema[key].values(): + LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(def_schema) + def _add_system_message_to_messages( self, new_messages: List[AllMessageValues], diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 839d032c436..b0442ebf0eb 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1984,3 +1984,104 @@ def test_translate_anthropic_to_openai_with_mixed_tools(): # tool_name_mapping should be empty for short tool names assert tool_name_mapping == {} + + +class TestTranslateAnthropicOutputFormatToOpenAI: + """Tests for translate_anthropic_output_format_to_openai adding additionalProperties: false.""" + + def setup_method(self): + self.adapter = LiteLLMAnthropicMessagesAdapter() + + def test_simple_object_adds_additional_properties_false(self): + output_format = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + }, + } + result = self.adapter.translate_anthropic_output_format_to_openai(output_format) + assert result is not None + schema = result["json_schema"]["schema"] + assert schema["additionalProperties"] is False + + def test_nested_objects_adds_additional_properties_false(self): + output_format = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "user": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "address": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } + }, + }, + } + result = self.adapter.translate_anthropic_output_format_to_openai(output_format) + assert result is not None + schema = result["json_schema"]["schema"] + assert schema["additionalProperties"] is False + assert schema["properties"]["user"]["additionalProperties"] is False + assert schema["properties"]["user"]["properties"]["address"]["additionalProperties"] is False + + def test_array_items_object_adds_additional_properties_false(self): + output_format = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": {"id": {"type": "integer"}}, + }, + } + }, + }, + } + result = self.adapter.translate_anthropic_output_format_to_openai(output_format) + assert result is not None + schema = result["json_schema"]["schema"] + assert schema["additionalProperties"] is False + assert schema["properties"]["items"]["items"]["additionalProperties"] is False + + def test_does_not_mutate_original_schema(self): + original_schema = { + "type": "object", + "properties": {"name": {"type": "string"}}, + } + output_format = {"type": "json_schema", "schema": original_schema} + self.adapter.translate_anthropic_output_format_to_openai(output_format) + assert "additionalProperties" not in original_schema + + def test_defs_adds_additional_properties_false(self): + output_format = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": {"ref": {"$ref": "#/$defs/Item"}}, + "$defs": { + "Item": { + "type": "object", + "properties": {"value": {"type": "string"}}, + } + }, + }, + } + result = self.adapter.translate_anthropic_output_format_to_openai(output_format) + assert result is not None + schema = result["json_schema"]["schema"] + assert schema["$defs"]["Item"]["additionalProperties"] is False + + def test_invalid_output_format_returns_none(self): + assert self.adapter.translate_anthropic_output_format_to_openai("invalid") is None + assert self.adapter.translate_anthropic_output_format_to_openai({"type": "text"}) is None + assert self.adapter.translate_anthropic_output_format_to_openai({"type": "json_schema"}) is None From 6f4b4d3c42c73641cb08f78a9587c8328c23afef Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 18 Mar 2026 22:33:01 -0300 Subject: [PATCH 03/73] feat(gemini): support context circulation for server-side tool combination Enables Gemini 3+ models to combine built-in tools (Google Search, etc.) with custom functions via `include_server_side_tool_invocations=True`. Server-side invocations are surfaced in provider_specific_fields and automatically re-injected on subsequent turns for multi-turn coherence. Closes #24047 --- docs/my-website/docs/providers/gemini.md | 108 +++++++- .../llms/vertex_ai/gemini/transformation.py | 40 +++ .../vertex_and_google_ai_studio_gemini.py | 78 ++++++ litellm/types/llms/vertex_ai.py | 3 +- .../gemini/test_context_circulation.py | 234 ++++++++++++++++++ 5 files changed, 461 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/llms/vertex_ai/gemini/test_context_circulation.py diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 0aaf3d5ae81..c8c9114ea87 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -54,6 +54,7 @@ response = completion( - stream - tools - tool_choice +- include_server_side_tool_invocations - functions - response_format - n @@ -856,7 +857,112 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ -### URL Context +### Context Circulation (Server-Side Tool Combination) + +Context circulation allows Gemini 3+ models to combine **built-in tools** (like Google Search) with **your custom functions** in the same request. Without it, Gemini returns an error if you try to use both. + +When enabled, Gemini can execute Google Search server-side, use those results to decide whether to call your custom functions, and return the full chain of reasoning. + +**How it works:** +1. You pass `include_server_side_tool_invocations=True` along with both Google Search and your function tools +2. Gemini executes server-side tools internally and returns `toolCall`/`toolResponse` parts alongside any `functionCall` parts +3. LiteLLM extracts the server-side invocations into `provider_specific_fields["server_side_tool_invocations"]` +4. On subsequent turns, include the full assistant message in your conversation history — LiteLLM re-injects the server-side parts automatically + + + + +```python +from litellm import completion + +response = completion( + model="gemini/gemini-3-flash-preview", + messages=[{"role": "user", "content": "What's the weather in Buenos Aires? If it's raining, schedule a meeting."}], + tools=[ + {"type": "web_search_preview"}, # Google Search (server-side) + { + "type": "function", + "function": { + "name": "schedule_meeting", + "description": "Schedule a meeting", + "parameters": { + "type": "object", + "properties": {"reason": {"type": "string"}}, + "required": ["reason"], + }, + }, + }, + ], + include_server_side_tool_invocations=True, +) + +msg = response.choices[0].message + +# Server-side tool results are in provider_specific_fields +psf = msg.provider_specific_fields or {} +for invocation in psf.get("server_side_tool_invocations", []): + print(invocation["tool_type"]) # e.g. "GOOGLE_SEARCH_WEB" + print(invocation["id"]) + print(invocation["args"]) # e.g. {"queries": ["weather Buenos Aires"]} + print(invocation["response"]) # Search results from Google + +# For multi-turn: just append the full message to history +messages.append(msg) +messages.append({"role": "user", "content": "Thanks!"}) +# LiteLLM automatically re-injects the server-side parts + thought signatures +response2 = completion( + model="gemini/gemini-3-flash-preview", + messages=messages, + tools=tools, + include_server_side_tool_invocations=True, +) +``` + + + + +1. Setup config.yaml +```yaml +model_list: + - model_name: gemini-3-flash + litellm_params: + model: gemini/gemini-3-flash-preview + api_key: os.environ/GEMINI_API_KEY +``` + +2. Start Proxy +```bash +$ litellm --config /path/to/config.yaml +``` + +3. Make Request +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gemini-3-flash", + "messages": [{"role": "user", "content": "What is the weather in Buenos Aires?"}], + "tools": [ + {"type": "web_search_preview"}, + {"type": "function", "function": {"name": "schedule_meeting", "description": "Schedule a meeting", "parameters": {"type": "object", "properties": {"reason": {"type": "string"}}}}} + ], + "include_server_side_tool_invocations": true +}' +``` + + + + +:::info + +- Context circulation requires **Gemini 3+** models +- Server-side tool invocations (`toolCall`/`toolResponse`) are **not** included in `tool_calls` — they are in `provider_specific_fields["server_side_tool_invocations"]` because they were already executed by Google, not by your code +- `thought_signatures` are automatically preserved alongside server-side invocations for multi-turn coherence + +::: + +### URL Context diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index d7b96b4db7b..f6310778c71 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -540,6 +540,39 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 assistant_content.append(gemini_tool_call_part) last_message_with_tool_calls = assistant_msg + ## HANDLE SERVER-SIDE TOOL INVOCATIONS (context circulation) + _psf = assistant_msg.get("provider_specific_fields") + if isinstance(_psf, dict): + _ss_invocations = _psf.get("server_side_tool_invocations") + if isinstance(_ss_invocations, list): + for invocation in _ss_invocations: + # Re-inject toolCall part + tc_part: Dict[str, Any] = { + "toolCall": { + "toolType": invocation.get("tool_type"), + "id": invocation.get("id"), + "args": invocation.get("args"), + } + } + if "thought_signature" in invocation: + tc_part["thoughtSignature"] = invocation["thought_signature"] + assistant_content.append(tc_part) # type: ignore + + # Re-inject toolResponse part if response is present + if "response" in invocation: + tr_dict: Dict[str, Any] = { + "id": invocation.get("id"), + "response": invocation.get("response"), + } + if invocation.get("tool_type"): + tr_dict["toolType"] = invocation["tool_type"] + tr_part: Dict[str, Any] = { + "toolResponse": tr_dict + } + if "thought_signature" in invocation: + tr_part["thoughtSignature"] = invocation["thought_signature"] + assistant_content.append(tr_part) # type: ignore + msg_i += 1 if assistant_content: @@ -666,6 +699,9 @@ def _transform_request_body( # noqa: PLR0915 ) tools: Optional[Tools] = optional_params.pop("tools", None) tool_choice: Optional[ToolConfig] = optional_params.pop("tool_choice", None) + include_server_side_tool_invocations: bool = optional_params.pop( + "include_server_side_tool_invocations", False + ) safety_settings: Optional[List[SafetSettingsConfig]] = optional_params.pop( "safety_settings", None ) # type: ignore @@ -715,6 +751,10 @@ def _transform_request_body( # noqa: PLR0915 data["tools"] = tools if tool_choice is not None: data["toolConfig"] = tool_choice + if include_server_side_tool_invocations: + if "toolConfig" not in data: + data["toolConfig"] = {} + data["toolConfig"]["includeServerSideToolInvocations"] = True if safety_settings is not None: data["safetySettings"] = safety_settings if generation_config is not None and len(generation_config) > 0: diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 3f1bccaccfc..3555d3c719e 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -316,6 +316,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "audio", "parallel_tool_calls", "web_search_options", + "include_server_side_tool_invocations", ] # Add penalty parameters only for non-preview models @@ -1119,6 +1120,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): optional_params = self._add_tools_to_optional_params( optional_params, [_tools] ) + elif param == "include_server_side_tool_invocations" and value is True: + optional_params["include_server_side_tool_invocations"] = True if litellm.vertex_ai_safety_settings is not None: optional_params["safety_settings"] = litellm.vertex_ai_safety_settings @@ -1360,6 +1363,67 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): signatures.append(signature) return signatures if signatures else None + @staticmethod + def _extract_server_side_tool_invocations( + parts: List[HttpxPartType], + ) -> Optional[List[Dict[str, Any]]]: + """Extract server-side tool invocations (toolCall/toolResponse) from parts. + + These are returned by Gemini when context circulation is enabled + (includeServerSideToolInvocations=true). They represent tools executed + server-side (e.g. Google Search) and must be circulated back in + subsequent turns for multi-turn coherence. + + Returns: + List of server-side invocation dicts if any found, None otherwise. + """ + invocations: List[Dict[str, Any]] = [] + # Index toolCalls by id so we can pair them with responses + tool_calls_by_id: Dict[str, Dict[str, Any]] = {} + tool_responses_by_id: Dict[str, Dict[str, Any]] = {} + + for part in parts: + if "toolCall" in part: + tc = part["toolCall"] + entry: Dict[str, Any] = { + "tool_type": tc.get("toolType"), + "id": tc.get("id"), + "args": tc.get("args"), + } + signature = part.get("thoughtSignature") + if signature is not None: + entry["thought_signature"] = signature + tool_calls_by_id[tc.get("id", "")] = entry + + elif "toolResponse" in part: + tr = part["toolResponse"] + entry = { + "id": tr.get("id"), + "tool_type": tr.get("toolType"), + "response": tr.get("response"), + } + signature = part.get("thoughtSignature") + if signature is not None: + entry["thought_signature"] = signature + tool_responses_by_id[tr.get("id", "")] = entry + + # Merge calls with their responses + for call_id, call_entry in tool_calls_by_id.items(): + merged = dict(call_entry) + resp = tool_responses_by_id.pop(call_id, None) + if resp is not None: + merged["response"] = resp.get("response") + # Keep response signature if call didn't have one + if "thought_signature" not in merged and "thought_signature" in resp: + merged["thought_signature"] = resp["thought_signature"] + invocations.append(merged) + + # Any orphan responses (shouldn't happen, but be safe) + for resp_id, resp_entry in tool_responses_by_id.items(): + invocations.append(resp_entry) + + return invocations if invocations else None + def _extract_image_response_from_parts( self, parts: List[HttpxPartType] ) -> Optional[List[ImageURLListItem]]: @@ -2018,6 +2082,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None reasoning_content: Optional[str] = None thought_signatures: Optional[Any] = None + server_side_tool_invocations: Optional[List[Dict[str, Any]]] = None for idx, candidate in enumerate(_candidates): if "content" not in candidate: @@ -2068,6 +2133,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) ) + # Extract server-side tool invocations (context circulation) + server_side_tool_invocations = ( + VertexGeminiConfig._extract_server_side_tool_invocations( + parts=candidate["content"]["parts"] + ) + ) + if audio_response is not None: cast(Dict[str, Any], chat_completion_message)[ "audio" @@ -2139,6 +2211,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): chat_completion_message["provider_specific_fields"] = {} chat_completion_message["provider_specific_fields"]["thought_signatures"] = thought_signatures # type: ignore + # Store server-side tool invocations in provider_specific_fields + if server_side_tool_invocations is not None: + if "provider_specific_fields" not in chat_completion_message: + chat_completion_message["provider_specific_fields"] = {} + chat_completion_message["provider_specific_fields"]["server_side_tool_invocations"] = server_side_tool_invocations # type: ignore + if isinstance(model_response, ModelResponseStream): choice = VertexGeminiConfig._create_streaming_choice( chat_completion_message=chat_completion_message, diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 201854369f1..66c6ca436cf 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -244,8 +244,9 @@ class Tools(TypedDict, total=False): retrieval: Retrieval -class ToolConfig(TypedDict): +class ToolConfig(TypedDict, total=False): functionCallingConfig: FunctionCallingConfig + includeServerSideToolInvocations: bool class TTL(TypedDict, total=False): diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_context_circulation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_context_circulation.py new file mode 100644 index 00000000000..c3038840d81 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_context_circulation.py @@ -0,0 +1,234 @@ +""" +Tests for Gemini context circulation (server-side tool invocations). + +When includeServerSideToolInvocations=true is set, Gemini returns toolCall/toolResponse +parts for server-side tools (e.g. Google Search). These must be: +1. Extracted from the response into provider_specific_fields["server_side_tool_invocations"] +2. Re-injected as raw toolCall/toolResponse parts when converting messages back to Gemini format +3. The includeServerSideToolInvocations flag must be passed through to toolConfig +""" + +import json +from typing import Any, Dict, List +from unittest.mock import MagicMock + +import pytest + +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, +) +from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, +) +from litellm.types.llms.vertex_ai import HttpxPartType + + +# --- Response extraction tests --- + + +class TestExtractServerSideToolInvocations: + """Test _extract_server_side_tool_invocations from response parts.""" + + def test_extracts_tool_call_and_response(self): + """Basic case: one toolCall + one toolResponse with same id.""" + parts: List[HttpxPartType] = [ + { + "thoughtSignature": "sig_call_1", + "toolCall": { + "toolType": "GOOGLE_SEARCH_WEB", + "id": "abc123", + "args": {"queries": ["weather Buenos Aires"]}, + }, + }, + { + "thoughtSignature": "sig_resp_1", + "toolResponse": { + "toolType": "GOOGLE_SEARCH_WEB", + "id": "abc123", + "response": {"weather": "Sunny, 20°C"}, + }, + }, + { + "text": "The weather in Buenos Aires is sunny.", + "thoughtSignature": "sig_text", + }, + ] + + result = VertexGeminiConfig._extract_server_side_tool_invocations(parts) + + assert result is not None + assert len(result) == 1 + assert result[0]["tool_type"] == "GOOGLE_SEARCH_WEB" + assert result[0]["id"] == "abc123" + assert result[0]["args"] == {"queries": ["weather Buenos Aires"]} + assert result[0]["response"] == {"weather": "Sunny, 20°C"} + assert result[0]["thought_signature"] == "sig_call_1" + + def test_returns_none_when_no_server_side_tools(self): + """No toolCall/toolResponse parts → returns None.""" + parts: List[HttpxPartType] = [ + {"text": "Hello world", "thoughtSignature": "sig1"}, + { + "functionCall": { + "name": "get_weather", + "args": {"location": "Paris"}, + }, + "thoughtSignature": "sig2", + }, + ] + + result = VertexGeminiConfig._extract_server_side_tool_invocations(parts) + assert result is None + + def test_multiple_server_side_invocations(self): + """Multiple toolCall/toolResponse pairs.""" + parts: List[HttpxPartType] = [ + { + "toolCall": { + "toolType": "GOOGLE_SEARCH_WEB", + "id": "search1", + "args": {"queries": ["query1"]}, + }, + "thoughtSignature": "sig1", + }, + { + "toolResponse": {"toolType": "GOOGLE_SEARCH_WEB", "id": "search1", "response": "result1"}, + "thoughtSignature": "sig2", + }, + { + "toolCall": { + "toolType": "GOOGLE_SEARCH_WEB", + "id": "search2", + "args": {"queries": ["query2"]}, + }, + "thoughtSignature": "sig3", + }, + { + "toolResponse": {"toolType": "GOOGLE_SEARCH_WEB", "id": "search2", "response": "result2"}, + "thoughtSignature": "sig4", + }, + ] + + result = VertexGeminiConfig._extract_server_side_tool_invocations(parts) + + assert result is not None + assert len(result) == 2 + assert result[0]["id"] == "search1" + assert result[0]["response"] == "result1" + assert result[1]["id"] == "search2" + assert result[1]["response"] == "result2" + + def test_tool_call_without_response(self): + """toolCall without matching toolResponse is still captured.""" + parts: List[HttpxPartType] = [ + { + "toolCall": { + "toolType": "CODE_EXECUTION", + "id": "exec1", + "args": {"code": "print('hello')"}, + }, + }, + ] + + result = VertexGeminiConfig._extract_server_side_tool_invocations(parts) + + assert result is not None + assert len(result) == 1 + assert result[0]["id"] == "exec1" + assert "response" not in result[0] + + +# --- Input re-injection tests --- + + +class TestReInjectServerSideToolInvocations: + """Test that server_side_tool_invocations are re-injected into Gemini parts.""" + + def test_roundtrip_single_invocation(self): + """Server-side invocations from assistant message are converted back to Gemini parts.""" + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": "It's sunny in Buenos Aires.", + "provider_specific_fields": { + "server_side_tool_invocations": [ + { + "tool_type": "GOOGLE_SEARCH_WEB", + "id": "abc123", + "args": {"queries": ["weather Buenos Aires"]}, + "response": {"weather": "Sunny, 20°C"}, + "thought_signature": "sig_abc", + } + ] + }, + }, + {"role": "user", "content": "Thanks!"}, + ] + + contents = _gemini_convert_messages_with_history(messages) + + # Find the model turn + model_turn = [c for c in contents if c["role"] == "model"] + assert len(model_turn) == 1 + + parts = model_turn[0]["parts"] + # Should have: text part + toolCall part + toolResponse part + tool_call_parts = [p for p in parts if "toolCall" in p] + tool_response_parts = [p for p in parts if "toolResponse" in p] + + assert len(tool_call_parts) == 1 + assert tool_call_parts[0]["toolCall"]["toolType"] == "GOOGLE_SEARCH_WEB" + assert tool_call_parts[0]["toolCall"]["id"] == "abc123" + assert tool_call_parts[0]["toolCall"]["args"] == {"queries": ["weather Buenos Aires"]} + assert tool_call_parts[0]["thoughtSignature"] == "sig_abc" + + assert len(tool_response_parts) == 1 + assert tool_response_parts[0]["toolResponse"]["id"] == "abc123" + assert tool_response_parts[0]["toolResponse"]["toolType"] == "GOOGLE_SEARCH_WEB" + assert tool_response_parts[0]["toolResponse"]["response"] == {"weather": "Sunny, 20°C"} + + def test_no_invocations_no_extra_parts(self): + """Without server_side_tool_invocations, no extra parts are added.""" + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"}, + {"role": "user", "content": "Bye"}, + ] + + contents = _gemini_convert_messages_with_history(messages) + model_turn = [c for c in contents if c["role"] == "model"] + assert len(model_turn) == 1 + + parts = model_turn[0]["parts"] + assert len(parts) == 1 + assert "text" in parts[0] + assert "toolCall" not in parts[0] + + +# --- toolConfig flag tests --- + + +class TestIncludeServerSideToolInvocationsConfig: + """Test that the flag is passed through to toolConfig.""" + + def test_flag_added_to_tool_config(self): + """include_server_side_tool_invocations=True should be mapped to optional_params.""" + config = VertexGeminiConfig() + non_default_params = {"include_server_side_tool_invocations": True} + optional_params: Dict[str, Any] = {} + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gemini-3-flash-preview", + drop_params=False, + ) + + assert result["include_server_side_tool_invocations"] is True + + def test_flag_in_supported_params(self): + """include_server_side_tool_invocations should be in supported params.""" + config = VertexGeminiConfig() + supported = config.get_supported_openai_params(model="gemini-3-flash-preview") + assert "include_server_side_tool_invocations" in supported From 286b8d14604c9ad5f736038fb5bb6146ecaed7bf Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 18 Mar 2026 23:03:20 -0300 Subject: [PATCH 04/73] fix: also populate required for all properties in strict mode OpenAI strict mode requires both additionalProperties:false AND all property keys in required. Without required, OpenAI rejects the schema even with additionalProperties:false set. --- .../adapters/transformation.py | 7 +++-- ...al_pass_through_adapters_transformation.py | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 47c9a223f85..3ee6c5f0d2a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -843,15 +843,18 @@ class LiteLLMAnthropicMessagesAdapter: @staticmethod def _add_additional_properties_false(schema: dict) -> None: """ - Recursively add 'additionalProperties': false to all object schemas. + Recursively ensure object schemas comply with OpenAI strict mode. - OpenAI's strict mode requires this at every object nesting level. + OpenAI's strict mode requires: + 1. 'additionalProperties': false at every object nesting level + 2. All property keys listed in 'required' """ if not isinstance(schema, dict): return if schema.get("type") == "object" and "properties" in schema: schema["additionalProperties"] = False + schema["required"] = list(schema["properties"].keys()) for prop in schema["properties"].values(): LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(prop) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index b0442ebf0eb..ae970e1ff06 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -2004,6 +2004,7 @@ class TestTranslateAnthropicOutputFormatToOpenAI: assert result is not None schema = result["json_schema"]["schema"] assert schema["additionalProperties"] is False + assert schema["required"] == ["name"] def test_nested_objects_adds_additional_properties_false(self): output_format = { @@ -2028,8 +2029,11 @@ class TestTranslateAnthropicOutputFormatToOpenAI: assert result is not None schema = result["json_schema"]["schema"] assert schema["additionalProperties"] is False + assert schema["required"] == ["user"] assert schema["properties"]["user"]["additionalProperties"] is False + assert schema["properties"]["user"]["required"] == ["name", "address"] assert schema["properties"]["user"]["properties"]["address"]["additionalProperties"] is False + assert schema["properties"]["user"]["properties"]["address"]["required"] == ["city"] def test_array_items_object_adds_additional_properties_false(self): output_format = { @@ -2061,6 +2065,7 @@ class TestTranslateAnthropicOutputFormatToOpenAI: output_format = {"type": "json_schema", "schema": original_schema} self.adapter.translate_anthropic_output_format_to_openai(output_format) assert "additionalProperties" not in original_schema + assert "required" not in original_schema def test_defs_adds_additional_properties_false(self): output_format = { @@ -2080,6 +2085,27 @@ class TestTranslateAnthropicOutputFormatToOpenAI: assert result is not None schema = result["json_schema"]["schema"] assert schema["$defs"]["Item"]["additionalProperties"] is False + assert schema["$defs"]["Item"]["required"] == ["value"] + + def test_incomplete_required_gets_completed(self): + """OpenAI strict mode requires ALL properties in required.""" + output_format = { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + "email": {"type": "string"}, + }, + "required": ["name"], # only 1 of 3 + }, + } + result = self.adapter.translate_anthropic_output_format_to_openai(output_format) + assert result is not None + schema = result["json_schema"]["schema"] + assert schema["additionalProperties"] is False + assert sorted(schema["required"]) == ["age", "email", "name"] def test_invalid_output_format_returns_none(self): assert self.adapter.translate_anthropic_output_format_to_openai("invalid") is None From 60c234270a4b42cdf20888ac353071b5eae5a378 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 18 Mar 2026 23:07:56 -0300 Subject: [PATCH 05/73] feat(bedrock): support cache_control_injection_points for tool_config location Add support for {"location": "tool_config"} in cache_control_injection_points, which appends a cachePoint block to the Bedrock Converse toolConfig.tools array. This enables prompt caching of tool definitions on Bedrock Claude models. Also update the cache control hook to pass through non-message injection points to provider-specific handling instead of silently dropping them. Fixes #21969 --- .../anthropic_cache_control_hook.py | 9 +- .../bedrock/chat/converse_transformation.py | 10 ++ .../chat/test_converse_transformation.py | 97 +++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 8e4d40c460e..0e99537d5db 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -60,13 +60,20 @@ class AnthropicCacheControlHook(CustomPromptManagement): # Create a deep copy of messages to avoid modifying the original list processed_messages = copy.deepcopy(messages) - # Process message-level cache controls + # Separate message-level and non-message-level injection points + remaining_points = [] for point in injection_points: if point.get("location") == "message": point = cast(CacheControlMessageInjectionPoint, point) processed_messages = self._process_message_injection( point=point, messages=processed_messages ) + else: + remaining_points.append(point) + + # Pass through non-message injection points for provider-specific handling + if remaining_points: + non_default_params["cache_control_injection_points"] = remaining_points return model, processed_messages, non_default_params diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 229457a73b4..dd8b1b0a69f 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1446,6 +1446,16 @@ class AmazonConverseConfig(BaseConfig): original_tools, model, headers, additional_request_params ) + # Append cachePoint to tools if cache_control_injection_points has tool_config + cache_injection_points = additional_request_params.pop( + "cache_control_injection_points", None + ) + if cache_injection_points and len(bedrock_tools) > 0: + for point in cache_injection_points: + if point.get("location") == "tool_config": + bedrock_tools.append({"cachePoint": {"type": "default"}}) + break + bedrock_tool_config: Optional[ToolConfigBlock] = None if len(bedrock_tools) > 0: tool_choice_values: ToolChoiceValuesBlock = inference_params.pop( diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index a305009659c..e9aaa97a421 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -3803,3 +3803,100 @@ def test_streaming_without_json_mode_passes_all_tools(): assert tool_use_delta is not None assert tool_use_delta["function"]["arguments"] == '{"data": 1}' + +def test_cache_control_injection_tool_config(): + """Test that cache_control_injection_points with location=tool_config appends cachePoint to tools.""" + config = AmazonConverseConfig() + messages = [ + {"role": "user", "content": "What is the weather?"}, + ] + optional_params = { + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + }, + "required": ["location"], + }, + }, + } + ], + "cache_control_injection_points": [ + {"location": "tool_config"}, + ], + } + result = config._transform_request( + model="anthropic.claude-3-5-haiku-20241022-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + ) + tool_config = result["toolConfig"] + tools = tool_config["tools"] + # Last element should be a cachePoint block + assert tools[-1] == {"cachePoint": {"type": "default"}} + # First element should be the actual tool + assert "toolSpec" in tools[0] + + +def test_cache_control_injection_tool_config_no_tools(): + """Test that tool_config injection is ignored when no tools are provided.""" + config = AmazonConverseConfig() + messages = [ + {"role": "user", "content": "Hello"}, + ] + optional_params = { + "cache_control_injection_points": [ + {"location": "tool_config"}, + ], + } + result = config._transform_request( + model="anthropic.claude-3-5-haiku-20241022-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + ) + assert "toolConfig" not in result + + +def test_cache_control_injection_tool_config_not_added_without_injection_point(): + """Test that cachePoint is NOT appended when cache_control_injection_points doesn't include tool_config.""" + config = AmazonConverseConfig() + messages = [ + {"role": "user", "content": "What is the weather?"}, + ] + optional_params = { + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + "cache_control_injection_points": [ + {"location": "message", "role": "system"}, + ], + } + result = config._transform_request( + model="anthropic.claude-3-5-haiku-20241022-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + ) + tools = result["toolConfig"]["tools"] + # No cachePoint should be appended + assert all("cachePoint" not in tool for tool in tools) + From cac685014ff2ad795c9d21a9e42475e46fdcd4b5 Mon Sep 17 00:00:00 2001 From: Ephrim Stanley Date: Thu, 19 Mar 2026 01:30:18 -0400 Subject: [PATCH 06/73] feat: add proxy-wide default tpm/rpm limits per deployment Adds `default_api_key_tpm_limit` and `default_api_key_rpm_limit` to `GenericLiteLLMParams` so operators can set per-deployment rate limit defaults in config.yaml. When a key has no model-specific tpm/rpm limit configured, the proxy falls back to these deployment defaults (Case 2 in spec). Key-level limits always take priority (Case 1). - Extends `get_key_model_tpm_limit` / `get_key_model_rpm_limit` with a `model_name` param and a priority-4 deployment-default fallback - Passes `model_name=requested_model` in the parallel request limiter so the fallback is triggered at enforcement time - Adds `"limit"` to `SensitiveDataMasker` non-sensitive overrides so `*_limit` fields are not masked in `/model/info` responses - Adds 17 unit tests covering both spec cases and the `/model/info` path Co-Authored-By: Claude (claude-sonnet-4-6) --- .../sensitive_data_masker.py | 4 +- litellm/proxy/auth/auth_utils.py | 64 ++++++- .../hooks/parallel_request_limiter_v3.py | 8 +- litellm/types/router.py | 5 + .../proxy/auth/test_auth_utils.py | 131 +++++++++++++- .../proxy/test_model_info_default_limits.py | 163 ++++++++++++++++++ 6 files changed, 369 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/proxy/test_model_info_default_limits.py diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 663c3fac801..f22cfa11a3d 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -30,7 +30,9 @@ class SensitiveDataMasker: # If any key segment matches one of these, the key is not considered sensitive # even if it also matches a sensitive pattern. For example, "input_cost_per_token" # contains "token" but "cost" overrides that — it's a pricing field, not a secret. - self.non_sensitive_overrides = non_sensitive_overrides or {"cost"} + # Similarly, "*_limit" fields (tpm_limit, rpm_limit, etc.) are rate/budget caps, + # not credentials, even though their names may contain "key" (e.g. default_api_key_tpm_limit). + self.non_sensitive_overrides = non_sensitive_overrides or {"cost", "limit"} self.visible_prefix = visible_prefix self.visible_suffix = visible_suffix diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index a03e1fb94c1..235b217610a 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -539,8 +539,49 @@ def bytes_to_mb(bytes_value: int): # helpers used by parallel request limiter to handle model rpm/tpm limits for a given api key +def _get_deployment_default_rpm_limit(model_name: str) -> Optional[int]: + """ + Return the default_api_key_rpm_limit configured on the deployment for model_name, + or None if not set. + """ + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + return None + deployments = llm_router.get_model_list(model_name=model_name) + if not deployments: + return None + for deployment in deployments: + litellm_params = deployment.get("litellm_params", {}) + limit = litellm_params.get("default_api_key_rpm_limit") + if limit is not None: + return int(limit) + return None + + +def _get_deployment_default_tpm_limit(model_name: str) -> Optional[int]: + """ + Return the default_api_key_tpm_limit configured on the deployment for model_name, + or None if not set. + """ + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + return None + deployments = llm_router.get_model_list(model_name=model_name) + if not deployments: + return None + for deployment in deployments: + litellm_params = deployment.get("litellm_params", {}) + limit = litellm_params.get("default_api_key_tpm_limit") + if limit is not None: + return int(limit) + return None + + def get_key_model_rpm_limit( user_api_key_dict: UserAPIKeyAuth, + model_name: Optional[str] = None, ) -> Optional[Dict[str, int]]: """ Get the model rpm limit for a given api key. @@ -549,6 +590,7 @@ def get_key_model_rpm_limit( 1. Key metadata (model_rpm_limit) 2. Key model_max_budget (rpm_limit per model) 3. Team metadata (model_rpm_limit) + 4. Deployment default_api_key_rpm_limit (when model_name is provided) """ # 1. Check key metadata first (takes priority) if user_api_key_dict.metadata: @@ -567,13 +609,22 @@ def get_key_model_rpm_limit( # 3. Fallback to team metadata if user_api_key_dict.team_metadata: - return user_api_key_dict.team_metadata.get("model_rpm_limit") + team_limit = user_api_key_dict.team_metadata.get("model_rpm_limit") + if team_limit: + return team_limit + + # 4. Fallback to deployment default_api_key_rpm_limit + if model_name is not None: + default_limit = _get_deployment_default_rpm_limit(model_name) + if default_limit is not None: + return {model_name: default_limit} return None def get_key_model_tpm_limit( user_api_key_dict: UserAPIKeyAuth, + model_name: Optional[str] = None, ) -> Optional[Dict[str, int]]: """ Get the model tpm limit for a given api key. @@ -582,6 +633,7 @@ def get_key_model_tpm_limit( 1. Key metadata (model_tpm_limit) 2. Key model_max_budget (tpm_limit per model) 3. Team metadata (model_tpm_limit) + 4. Deployment default_api_key_tpm_limit (when model_name is provided) """ # 1. Check key metadata first (takes priority) if user_api_key_dict.metadata: @@ -600,7 +652,15 @@ def get_key_model_tpm_limit( # 3. Fallback to team metadata if user_api_key_dict.team_metadata: - return user_api_key_dict.team_metadata.get("model_tpm_limit") + team_limit = user_api_key_dict.team_metadata.get("model_tpm_limit") + if team_limit: + return team_limit + + # 4. Fallback to deployment default_api_key_tpm_limit + if model_name is not None: + default_limit = _get_deployment_default_tpm_limit(model_name) + if default_limit is not None: + return {model_name: default_limit} return None diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 19c8c484b4d..5aaac088dc2 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -687,8 +687,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if not requested_model: return - _tpm_limit_for_key_model = get_key_model_tpm_limit(user_api_key_dict) - _rpm_limit_for_key_model = get_key_model_rpm_limit(user_api_key_dict) + _tpm_limit_for_key_model = get_key_model_tpm_limit( + user_api_key_dict, model_name=requested_model + ) + _rpm_limit_for_key_model = get_key_model_rpm_limit( + user_api_key_dict, model_name=requested_model + ) if _tpm_limit_for_key_model is None and _rpm_limit_for_key_model is None: return diff --git a/litellm/types/router.py b/litellm/types/router.py index e8ff2115ff5..5d28349b5e4 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -188,6 +188,11 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): max_file_size_mb: Optional[float] = None + # Proxy-wide default rate limits applied to any API key using this deployment + # when the key does not have a model-specific tpm/rpm limit configured. + default_api_key_tpm_limit: Optional[int] = None + default_api_key_rpm_limit: Optional[int] = None + # Deployment budgets max_budget: Optional[float] = None budget_duration: Optional[str] = None diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 5e42b110aa0..be4db666a04 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -2,7 +2,7 @@ Unit tests for auth_utils functions related to rate limiting and customer ID extraction. """ -from unittest.mock import patch +from unittest.mock import MagicMock, patch from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( @@ -315,3 +315,132 @@ def test_get_end_user_id_falls_back_to_deprecated_user_header_name(): result = get_end_user_id_from_request_body(request_body={}, request_headers=headers) assert result == "user-legacy" + + +def _make_deployment_dict(model_name: str, tpm: int = None, rpm: int = None) -> dict: + """Helper to build a minimal deployment dict as returned by router.get_model_list.""" + litellm_params: dict = {"model": model_name} + if tpm is not None: + litellm_params["default_api_key_tpm_limit"] = tpm + if rpm is not None: + litellm_params["default_api_key_rpm_limit"] = rpm + return {"model_name": model_name, "litellm_params": litellm_params} + + +_ROUTER_PATCH = "litellm.proxy.proxy_server.llm_router" + + +class TestDeploymentDefaultRpmLimit: + """Tests for deployment default_api_key_rpm_limit fallback in get_key_model_rpm_limit.""" + + def test_returns_deployment_default_when_key_has_no_limits(self): + """Case 2 from spec: key has no model-specific limits, falls back to deployment default.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1", rpm=200) + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1") + assert result == {"model1": 200} + + def test_key_model_limit_takes_priority_over_deployment_default(self): + """Case 1 from spec: key model-specific limit wins over deployment default.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + metadata={"model_rpm_limit": {"model1": 10}}, + ) + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1", rpm=200) + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1") + assert result == {"model1": 10} + + def test_returns_none_when_no_deployment_default_and_no_key_limits(self): + """Returns None when neither the key nor the deployment has any rpm limit.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1") # no rpm default + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1") + assert result is None + + def test_returns_none_without_model_name_even_when_deployment_has_default(self): + """No model_name means deployment fallback is skipped.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1", rpm=200) + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_rpm_limit(user_api_key_dict) + assert result is None + + def test_returns_none_when_llm_router_is_none(self): + """No router means deployment fallback returns None gracefully.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + with patch(_ROUTER_PATCH, None): + result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1") + assert result is None + + +class TestDeploymentDefaultTpmLimit: + """Tests for deployment default_api_key_tpm_limit fallback in get_key_model_tpm_limit.""" + + def test_returns_deployment_default_when_key_has_no_limits(self): + """Case 2 from spec: key has no model-specific limits, falls back to deployment default.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1", tpm=100) + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1") + assert result == {"model1": 100} + + def test_key_model_limit_takes_priority_over_deployment_default(self): + """Case 1 from spec: key model-specific limit wins over deployment default.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + metadata={"model_tpm_limit": {"model1": 20}}, + ) + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1", tpm=100) + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1") + assert result == {"model1": 20} + + def test_returns_none_when_no_deployment_default_and_no_key_limits(self): + """Returns None when neither the key nor the deployment has any tpm limit.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1") # no tpm default + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1") + assert result is None + + def test_returns_none_without_model_name_even_when_deployment_has_default(self): + """No model_name means deployment fallback is skipped.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1", tpm=100) + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_tpm_limit(user_api_key_dict) + assert result is None + + def test_returns_none_when_llm_router_is_none(self): + """No router means deployment fallback returns None gracefully.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + with patch(_ROUTER_PATCH, None): + result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1") + assert result is None diff --git a/tests/test_litellm/proxy/test_model_info_default_limits.py b/tests/test_litellm/proxy/test_model_info_default_limits.py new file mode 100644 index 00000000000..e749c84dfb2 --- /dev/null +++ b/tests/test_litellm/proxy/test_model_info_default_limits.py @@ -0,0 +1,163 @@ +""" +Tests verifying that default_api_key_tpm_limit and default_api_key_rpm_limit set in +litellm_params are returned by the /model/info endpoint. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.proxy.proxy_server import _get_proxy_model_info +from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + +def _make_deployment( + model_name: str, + default_tpm: int = None, + default_rpm: int = None, +) -> Deployment: + params: dict = {"model": f"openai/{model_name}"} + if default_tpm is not None: + params["default_api_key_tpm_limit"] = default_tpm + if default_rpm is not None: + params["default_api_key_rpm_limit"] = default_rpm + return Deployment( + model_name=model_name, + litellm_params=LiteLLM_Params(**params), + model_info=ModelInfo(), + ) + + +class TestModelInfoDefaultLimitsInResponse: + """ + Verify _get_proxy_model_info (the helper used by the /model/info endpoint) returns + default_api_key_tpm_limit and default_api_key_rpm_limit from litellm_params. + """ + + def test_default_tpm_and_rpm_present_in_model_info_response(self): + """Both defaults should appear in the litellm_params section of the response.""" + deployment = _make_deployment("model1", default_tpm=100, default_rpm=200) + model_dict = deployment.model_dump(exclude_none=True) + + result = _get_proxy_model_info(model=model_dict) + + litellm_params = result["litellm_params"] + assert litellm_params.get("default_api_key_tpm_limit") == 100 + assert litellm_params.get("default_api_key_rpm_limit") == 200 + + def test_default_tpm_only_present_when_only_tpm_configured(self): + """Only the configured default appears; the other stays absent.""" + deployment = _make_deployment("model1", default_tpm=500) + model_dict = deployment.model_dump(exclude_none=True) + + result = _get_proxy_model_info(model=model_dict) + + litellm_params = result["litellm_params"] + assert litellm_params.get("default_api_key_tpm_limit") == 500 + assert "default_api_key_rpm_limit" not in litellm_params + + def test_default_rpm_only_present_when_only_rpm_configured(self): + """Only the configured default appears; the other stays absent.""" + deployment = _make_deployment("model1", default_rpm=300) + model_dict = deployment.model_dump(exclude_none=True) + + result = _get_proxy_model_info(model=model_dict) + + litellm_params = result["litellm_params"] + assert litellm_params.get("default_api_key_rpm_limit") == 300 + assert "default_api_key_tpm_limit" not in litellm_params + + def test_defaults_absent_when_not_configured(self): + """Neither field appears when not set on the deployment.""" + deployment = _make_deployment("model1") + model_dict = deployment.model_dump(exclude_none=True) + + result = _get_proxy_model_info(model=model_dict) + + litellm_params = result["litellm_params"] + assert "default_api_key_tpm_limit" not in litellm_params + assert "default_api_key_rpm_limit" not in litellm_params + + def test_defaults_not_masked_or_stripped_by_sensitive_data_filter(self): + """ + default_api_key_tpm_limit / default_api_key_rpm_limit must not be + treated as sensitive and must survive remove_sensitive_info_from_deployment. + """ + deployment = _make_deployment("model1", default_tpm=100, default_rpm=200) + model_dict = deployment.model_dump(exclude_none=True) + + result = _get_proxy_model_info(model=model_dict) + + # Values should be unchanged integers, not masked strings + assert result["litellm_params"]["default_api_key_tpm_limit"] == 100 + assert result["litellm_params"]["default_api_key_rpm_limit"] == 200 + + +class TestModelInfoEndpointWithRouter: + """ + Integration-style tests simulating the /model/info endpoint reading from the router. + """ + + @pytest.mark.asyncio + async def test_model_info_endpoint_returns_defaults_for_specific_model_id(self): + """ + When litellm_model_id is provided, the endpoint should return the deployment's + default limits in litellm_params. + """ + from litellm.proxy.proxy_server import model_info_v1 + from litellm.proxy._types import UserAPIKeyAuth + + deployment = _make_deployment("model1", default_tpm=100, default_rpm=200) + + mock_router = MagicMock() + mock_router.get_deployment.return_value = deployment + + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test") + + with patch("litellm.proxy.proxy_server.llm_router", mock_router), \ + patch("litellm.proxy.proxy_server.llm_model_list", [{}]), \ + patch("litellm.proxy.proxy_server.user_model", None): + response = await model_info_v1( + user_api_key_dict=user_api_key_dict, + litellm_model_id="some-model-id", + ) + + assert len(response["data"]) == 1 + litellm_params = response["data"][0]["litellm_params"] + assert litellm_params.get("default_api_key_tpm_limit") == 100 + assert litellm_params.get("default_api_key_rpm_limit") == 200 + + @pytest.mark.asyncio + async def test_model_info_endpoint_returns_defaults_in_full_model_list(self): + """ + Without litellm_model_id, the endpoint iterates all models. Each deployment's + default limits should appear in its litellm_params entry. + """ + from litellm.proxy.proxy_server import model_info_v1 + from litellm.proxy._types import UserAPIKeyAuth + + deployment = _make_deployment("model1", default_tpm=100, default_rpm=200) + deployment_dict = deployment.model_dump(exclude_none=True) + + mock_router = MagicMock() + mock_router.get_model_names.return_value = ["model1"] + mock_router.get_model_access_groups.return_value = {} + mock_router.get_model_list.return_value = [deployment_dict] + + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test") + + with patch("litellm.proxy.proxy_server.llm_router", mock_router), \ + patch("litellm.proxy.proxy_server.llm_model_list", [deployment_dict]), \ + patch("litellm.proxy.proxy_server.user_model", None), \ + patch("litellm.proxy.proxy_server.get_key_models", return_value=["model1"]), \ + patch("litellm.proxy.proxy_server.get_team_models", return_value=["model1"]), \ + patch("litellm.proxy.proxy_server.get_complete_model_list", return_value=["model1"]): + response = await model_info_v1( + user_api_key_dict=user_api_key_dict, + litellm_model_id=None, + ) + + assert len(response["data"]) >= 1 + litellm_params = response["data"][0]["litellm_params"] + assert litellm_params.get("default_api_key_tpm_limit") == 100 + assert litellm_params.get("default_api_key_rpm_limit") == 200 From 36dc893770fa69cfcc014e5c535b87d58ca12c89 Mon Sep 17 00:00:00 2001 From: Ephrim Stanley Date: Thu, 19 Mar 2026 01:43:27 -0400 Subject: [PATCH 07/73] fix: address review feedback on default tpm/rpm limits - Use min() across all matching deployments instead of first-wins when resolving default_api_key_tpm/rpm_limit for a model group, so load-balanced setups with different per-deployment limits always apply the most conservative value - Replace the global SensitiveDataMasker non_sensitive_overrides change with a targeted excluded_keys set at the remove_sensitive_info_from_deployment call site, avoiding unintended suppression of other fields - Update the v1 parallel request limiter to pass model_name to get_key_model_tpm/rpm_limit so deployment defaults apply there too - Add 4 tests covering multi-deployment min semantics Co-Authored-By: Claude (claude-sonnet-4-6) --- .../sensitive_data_masker.py | 4 +- litellm/proxy/auth/auth_utils.py | 44 ++++++++++------ .../common_utils/openai_endpoint_utils.py | 11 +++- .../proxy/hooks/parallel_request_limiter.py | 14 ++++-- .../proxy/auth/test_auth_utils.py | 50 +++++++++++++++++++ .../proxy/test_model_info_default_limits.py | 3 ++ 6 files changed, 101 insertions(+), 25 deletions(-) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index f22cfa11a3d..663c3fac801 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -30,9 +30,7 @@ class SensitiveDataMasker: # If any key segment matches one of these, the key is not considered sensitive # even if it also matches a sensitive pattern. For example, "input_cost_per_token" # contains "token" but "cost" overrides that — it's a pricing field, not a secret. - # Similarly, "*_limit" fields (tpm_limit, rpm_limit, etc.) are rate/budget caps, - # not credentials, even though their names may contain "key" (e.g. default_api_key_tpm_limit). - self.non_sensitive_overrides = non_sensitive_overrides or {"cost", "limit"} + self.non_sensitive_overrides = non_sensitive_overrides or {"cost"} self.visible_prefix = visible_prefix self.visible_suffix = visible_suffix diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 235b217610a..ace39c05ffc 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -541,8 +541,13 @@ def bytes_to_mb(bytes_value: int): # helpers used by parallel request limiter to handle model rpm/tpm limits for a given api key def _get_deployment_default_rpm_limit(model_name: str) -> Optional[int]: """ - Return the default_api_key_rpm_limit configured on the deployment for model_name, - or None if not set. + Return the default_api_key_rpm_limit for model_name. + + When multiple deployments share the same model name, returns the minimum + across all deployments that have the field set. This is the safest choice + for load-balanced setups: it ensures no deployment is over-consumed + regardless of which one actually serves a given request. + Returns None if no deployment has the field set. """ from litellm.proxy.proxy_server import llm_router @@ -551,18 +556,24 @@ def _get_deployment_default_rpm_limit(model_name: str) -> Optional[int]: deployments = llm_router.get_model_list(model_name=model_name) if not deployments: return None - for deployment in deployments: - litellm_params = deployment.get("litellm_params", {}) - limit = litellm_params.get("default_api_key_rpm_limit") - if limit is not None: - return int(limit) - return None + limits = [ + int(deployment.get("litellm_params", {}).get("default_api_key_rpm_limit")) + for deployment in deployments + if deployment.get("litellm_params", {}).get("default_api_key_rpm_limit") + is not None + ] + return min(limits) if limits else None def _get_deployment_default_tpm_limit(model_name: str) -> Optional[int]: """ - Return the default_api_key_tpm_limit configured on the deployment for model_name, - or None if not set. + Return the default_api_key_tpm_limit for model_name. + + When multiple deployments share the same model name, returns the minimum + across all deployments that have the field set. This is the safest choice + for load-balanced setups: it ensures no deployment is over-consumed + regardless of which one actually serves a given request. + Returns None if no deployment has the field set. """ from litellm.proxy.proxy_server import llm_router @@ -571,12 +582,13 @@ def _get_deployment_default_tpm_limit(model_name: str) -> Optional[int]: deployments = llm_router.get_model_list(model_name=model_name) if not deployments: return None - for deployment in deployments: - litellm_params = deployment.get("litellm_params", {}) - limit = litellm_params.get("default_api_key_tpm_limit") - if limit is not None: - return int(limit) - return None + limits = [ + int(deployment.get("litellm_params", {}).get("default_api_key_tpm_limit")) + for deployment in deployments + if deployment.get("litellm_params", {}).get("default_api_key_tpm_limit") + is not None + ] + return min(limits) if limits else None def get_key_model_rpm_limit( diff --git a/litellm/proxy/common_utils/openai_endpoint_utils.py b/litellm/proxy/common_utils/openai_endpoint_utils.py index 6df5491f37a..7e5c83500a2 100644 --- a/litellm/proxy/common_utils/openai_endpoint_utils.py +++ b/litellm/proxy/common_utils/openai_endpoint_utils.py @@ -32,8 +32,17 @@ def remove_sensitive_info_from_deployment( deployment_dict["litellm_params"].pop("aws_access_key_id", None) deployment_dict["litellm_params"].pop("aws_secret_access_key", None) + # Rate-limit config fields must never be masked — they are integers, not credentials. + # The field names contain "key" which matches the masker's sensitive pattern, so we + # explicitly exclude them here rather than widening the global non_sensitive_overrides. + _rate_limit_config_keys = { + "default_api_key_tpm_limit", + "default_api_key_rpm_limit", + } + _excluded = (excluded_keys or set()) | _rate_limit_config_keys + deployment_dict["litellm_params"] = SENSITIVE_DATA_MASKER.mask_dict( - deployment_dict["litellm_params"], excluded_keys=excluded_keys + deployment_dict["litellm_params"], excluded_keys=_excluded ) return deployment_dict diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index c7bfc27d6b6..48bf255ac13 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -295,16 +295,20 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): ) # Check if request under RPM/TPM per model for a given API Key + _model = data.get("model", None) if ( - get_key_model_tpm_limit(user_api_key_dict) is not None - or get_key_model_rpm_limit(user_api_key_dict) is not None + get_key_model_tpm_limit(user_api_key_dict, model_name=_model) is not None + or get_key_model_rpm_limit(user_api_key_dict, model_name=_model) is not None ): - _model = data.get("model", None) request_count_api_key = ( f"{api_key}::{_model}::{precise_minute}::request_count" ) - _tpm_limit_for_key_model = get_key_model_tpm_limit(user_api_key_dict) - _rpm_limit_for_key_model = get_key_model_rpm_limit(user_api_key_dict) + _tpm_limit_for_key_model = get_key_model_tpm_limit( + user_api_key_dict, model_name=_model + ) + _rpm_limit_for_key_model = get_key_model_rpm_limit( + user_api_key_dict, model_name=_model + ) tpm_limit_for_model = None rpm_limit_for_model = None diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index be4db666a04..d64f17e70ff 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -387,6 +387,31 @@ class TestDeploymentDefaultRpmLimit: result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1") assert result is None + def test_returns_minimum_across_multiple_deployments(self): + """When multiple deployments share a model name, the minimum rpm limit is used.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1", rpm=200), + _make_deployment_dict("model1", rpm=50), + _make_deployment_dict("model1", rpm=150), + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1") + assert result == {"model1": 50} + + def test_ignores_deployments_without_default_when_others_have_it(self): + """Deployments missing the field are skipped; min is taken over those that have it.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1"), # no rpm default + _make_deployment_dict("model1", rpm=75), + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1") + assert result == {"model1": 75} + class TestDeploymentDefaultTpmLimit: """Tests for deployment default_api_key_tpm_limit fallback in get_key_model_tpm_limit.""" @@ -444,3 +469,28 @@ class TestDeploymentDefaultTpmLimit: with patch(_ROUTER_PATCH, None): result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1") assert result is None + + def test_returns_minimum_across_multiple_deployments(self): + """When multiple deployments share a model name, the minimum tpm limit is used.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1", tpm=1000), + _make_deployment_dict("model1", tpm=300), + _make_deployment_dict("model1", tpm=700), + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1") + assert result == {"model1": 300} + + def test_ignores_deployments_without_default_when_others_have_it(self): + """Deployments missing the field are skipped; min is taken over those that have it.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + _make_deployment_dict("model1"), # no tpm default + _make_deployment_dict("model1", tpm=400), + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1") + assert result == {"model1": 400} diff --git a/tests/test_litellm/proxy/test_model_info_default_limits.py b/tests/test_litellm/proxy/test_model_info_default_limits.py index e749c84dfb2..907f2390a89 100644 --- a/tests/test_litellm/proxy/test_model_info_default_limits.py +++ b/tests/test_litellm/proxy/test_model_info_default_limits.py @@ -82,6 +82,9 @@ class TestModelInfoDefaultLimitsInResponse: """ default_api_key_tpm_limit / default_api_key_rpm_limit must not be treated as sensitive and must survive remove_sensitive_info_from_deployment. + They contain "key" which normally triggers masking; the call site explicitly + excludes these two fields via excluded_keys rather than widening the global + non_sensitive_overrides. """ deployment = _make_deployment("model1", default_tpm=100, default_rpm=200) model_dict = deployment.model_dump(exclude_none=True) From b90f5207488c71ad22acc700c24999862e0e08e9 Mon Sep 17 00:00:00 2001 From: Ephrim Stanley Date: Thu, 19 Mar 2026 01:57:09 -0400 Subject: [PATCH 08/73] perf: eliminate redundant router lookups in v1 parallel request limiter Compute get_key_model_tpm/rpm_limit once before the guard condition instead of calling each function twice (once to check non-None, once to retrieve). Removes 2 extra llm_router.get_model_list() calls per request when deployment defaults are active. Co-Authored-By: Claude (claude-sonnet-4-6) --- litellm/proxy/hooks/parallel_request_limiter.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 48bf255ac13..6e34c3eee13 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -296,19 +296,16 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): # Check if request under RPM/TPM per model for a given API Key _model = data.get("model", None) - if ( - get_key_model_tpm_limit(user_api_key_dict, model_name=_model) is not None - or get_key_model_rpm_limit(user_api_key_dict, model_name=_model) is not None - ): + _tpm_limit_for_key_model = get_key_model_tpm_limit( + user_api_key_dict, model_name=_model + ) + _rpm_limit_for_key_model = get_key_model_rpm_limit( + user_api_key_dict, model_name=_model + ) + if _tpm_limit_for_key_model is not None or _rpm_limit_for_key_model is not None: request_count_api_key = ( f"{api_key}::{_model}::{precise_minute}::request_count" ) - _tpm_limit_for_key_model = get_key_model_tpm_limit( - user_api_key_dict, model_name=_model - ) - _rpm_limit_for_key_model = get_key_model_rpm_limit( - user_api_key_dict, model_name=_model - ) tpm_limit_for_model = None rpm_limit_for_model = None From 48cb4a83435faa458888b412d408eb06cbca694d Mon Sep 17 00:00:00 2001 From: Ephrim Stanley Date: Thu, 19 Mar 2026 01:59:41 -0400 Subject: [PATCH 09/73] fix: update success-event handler to track tokens for deployment-default limits async_log_success_event only updated the per-model cache counter when model_rpm_limit / model_tpm_limit were present in key metadata or model_max_budget was set. For the new deployment-default path (default_api_key_tpm_limit / default_api_key_rpm_limit), none of those conditions held, so current_tpm stayed at zero and tpm enforcement was never applied across multiple requests. Extend the guard condition to also trigger when the model group has a deployment-default tpm or rpm limit, and import the two helpers at module level. Co-Authored-By: Claude (claude-sonnet-4-6) --- litellm/proxy/hooks/parallel_request_limiter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 6e34c3eee13..b0046fd035b 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -14,6 +14,8 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( + _get_deployment_default_rpm_limit, + _get_deployment_default_tpm_limit, get_key_model_rpm_limit, get_key_model_tpm_limit, ) @@ -546,6 +548,8 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): "model_rpm_limit" in user_api_key_metadata or "model_tpm_limit" in user_api_key_metadata or user_api_key_model_max_budget is not None + or _get_deployment_default_tpm_limit(model_group) is not None + or _get_deployment_default_rpm_limit(model_group) is not None ) ): request_count_api_key = ( From 477c54184bda814745bb06d4fbe4900cfc816bd2 Mon Sep 17 00:00:00 2001 From: Ephrim Stanley Date: Thu, 19 Mar 2026 02:07:50 -0400 Subject: [PATCH 10/73] perf: avoid unconditional router lookups in success handler Replace bare _get_deployment_default_tpm/rpm_limit calls in the async_log_success_event condition with get_key_model_tpm/rpm_limit (model_name=model_group). The higher-level getters short-circuit on key/team metadata hits before ever reaching the router, so requests that don't use deployment defaults incur no extra router lookup. Remove the now-unused bare helper imports. Also fix invalid `int = None` type hints in test helper signatures to `Optional[int] = None`. Co-Authored-By: Claude (claude-sonnet-4-6) --- litellm/proxy/hooks/parallel_request_limiter.py | 12 ++++++++---- tests/test_litellm/proxy/auth/test_auth_utils.py | 3 ++- .../proxy/test_model_info_default_limits.py | 5 +++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index b0046fd035b..49c6436c22f 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -14,8 +14,6 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import ( - _get_deployment_default_rpm_limit, - _get_deployment_default_tpm_limit, get_key_model_rpm_limit, get_key_model_tpm_limit, ) @@ -548,8 +546,14 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): "model_rpm_limit" in user_api_key_metadata or "model_tpm_limit" in user_api_key_metadata or user_api_key_model_max_budget is not None - or _get_deployment_default_tpm_limit(model_group) is not None - or _get_deployment_default_rpm_limit(model_group) is not None + or get_key_model_tpm_limit( + user_api_key_dict, model_name=model_group + ) + is not None + or get_key_model_rpm_limit( + user_api_key_dict, model_name=model_group + ) + is not None ) ): request_count_api_key = ( diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index d64f17e70ff..2058f61cb0f 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -2,6 +2,7 @@ Unit tests for auth_utils functions related to rate limiting and customer ID extraction. """ +from typing import Optional from unittest.mock import MagicMock, patch from litellm.proxy._types import UserAPIKeyAuth @@ -317,7 +318,7 @@ def test_get_end_user_id_falls_back_to_deprecated_user_header_name(): assert result == "user-legacy" -def _make_deployment_dict(model_name: str, tpm: int = None, rpm: int = None) -> dict: +def _make_deployment_dict(model_name: str, tpm: Optional[int] = None, rpm: Optional[int] = None) -> dict: """Helper to build a minimal deployment dict as returned by router.get_model_list.""" litellm_params: dict = {"model": model_name} if tpm is not None: diff --git a/tests/test_litellm/proxy/test_model_info_default_limits.py b/tests/test_litellm/proxy/test_model_info_default_limits.py index 907f2390a89..8b85531785a 100644 --- a/tests/test_litellm/proxy/test_model_info_default_limits.py +++ b/tests/test_litellm/proxy/test_model_info_default_limits.py @@ -3,6 +3,7 @@ Tests verifying that default_api_key_tpm_limit and default_api_key_rpm_limit set litellm_params are returned by the /model/info endpoint. """ +from typing import Optional from unittest.mock import MagicMock, patch import pytest @@ -13,8 +14,8 @@ from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo def _make_deployment( model_name: str, - default_tpm: int = None, - default_rpm: int = None, + default_tpm: Optional[int] = None, + default_rpm: Optional[int] = None, ) -> Deployment: params: dict = {"model": f"openai/{model_name}"} if default_tpm is not None: From b20c448188acabe3df75597d1c76a65b07d61149 Mon Sep 17 00:00:00 2001 From: chengyongru <61816729+chengyongru@users.noreply.github.com> Date: Thu, 19 Mar 2026 15:34:47 +0800 Subject: [PATCH 11/73] fix(openai): handle missing 'id' field in streaming chunks for MiniMax (#23931) - Change chunk["id"] to chunk.get("id") for compatibility with MiniMax - ModelResponseStream auto-generates id when None is passed - Add regression test test_chunk_parser_without_id_field --- .../llms/openai/chat/gpt_transformation.py | 2 +- .../chat/test_openai_gpt_transformation.py | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 63beb82ded8..34a23222c2a 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -806,7 +806,7 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator): choices = self._map_reasoning_to_reasoning_content(choices) kwargs = { - "id": chunk["id"], + "id": chunk.get("id"), "object": "chat.completion.chunk", "created": chunk.get("created"), "model": chunk.get("model"), diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index 086d01f65b4..5d0b1ec8565 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -281,6 +281,45 @@ class TestOpenAIChatCompletionStreamingHandler: # Verify that reasoning_content is not set (it should be deleted by Delta.__init__) assert not hasattr(parsed_chunk.choices[0].delta, "reasoning_content") + def test_chunk_parser_without_id_field(self): + """ + Test that chunk_parser works when chunk is missing the 'id' field. + + Some OpenAI-compatible providers (e.g., MiniMax) return streaming chunks + without an 'id' field in certain cases. This should not raise KeyError. + + Regression test for: KeyError: 'id' when using MiniMax m2.5 model + """ + handler = OpenAIChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + # Simulate a chunk without 'id' field (as returned by MiniMax) + chunk = { + "object": "chat.completion.chunk", + "created": 1769511767, + "model": "minimax/m2.5", + "choices": [ + { + "delta": { + "content": "Hello", + "role": "assistant", + }, + "finish_reason": None, + "index": 0, + } + ], + } + + # Parse the chunk - should not raise KeyError + parsed_chunk = handler.chunk_parser(chunk) + + # Verify that content is present and id was auto-generated + assert parsed_chunk.choices[0].delta.content == "Hello" + assert parsed_chunk.choices[0].delta.role == "assistant" + # ModelResponseStream auto-generates an id when None is passed + assert parsed_chunk.id is not None + class TestPromptCacheKeyIntegration: """Tests for prompt_cache_key support""" From e19a717b53bd6129c350ebf0c7c63a592ec5ad08 Mon Sep 17 00:00:00 2001 From: superpoussin22 Date: Thu, 19 Mar 2026 09:22:10 +0100 Subject: [PATCH 12/73] Add IF NOT EXISTS to index creation in migration --- .../20260318140652_add_index_to_team_table/migration.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260318140652_add_index_to_team_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260318140652_add_index_to_team_table/migration.sql index 494aaf6238f..89121d636f4 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260318140652_add_index_to_team_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260318140652_add_index_to_team_table/migration.sql @@ -1,9 +1,9 @@ -- CreateIndex -CREATE INDEX "LiteLLM_TeamTable_organization_id_idx" ON "LiteLLM_TeamTable"("organization_id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_TeamTable_organization_id_idx" ON "LiteLLM_TeamTable"("organization_id"); -- CreateIndex -CREATE INDEX "LiteLLM_TeamTable_team_alias_idx" ON "LiteLLM_TeamTable"("team_alias"); +CREATE INDEX IF NOT EXISTS "LiteLLM_TeamTable_team_alias_idx" ON "LiteLLM_TeamTable"("team_alias"); -- CreateIndex -CREATE INDEX "LiteLLM_TeamTable_created_at_idx" ON "LiteLLM_TeamTable"("created_at"); +CREATE INDEX IF NOT EXISTS "LiteLLM_TeamTable_created_at_idx" ON "LiteLLM_TeamTable"("created_at"); From e562c1d0640283e4820deff27ee3933646a29f65 Mon Sep 17 00:00:00 2001 From: Ephrim Stanley Date: Thu, 19 Mar 2026 07:26:43 -0400 Subject: [PATCH 13/73] refactor: consolidate duplicate helpers and eliminate success-handler double lookup - Merge _get_deployment_default_rpm_limit and _get_deployment_default_tpm_limit into a single _get_deployment_default_limit(model_name, field) helper; the two thin wrappers are preserved for callers but share one implementation - Compute _success_tpm_limit / _success_rpm_limit once before the guard condition in async_log_success_event, eliminating the previous two unconditional get_key_model_* calls (each of which could hit llm_router.get_model_list) - Replace fragile llm_model_list=[{}] sentinel in test with [] Co-Authored-By: Claude (claude-sonnet-4-6) --- litellm/proxy/auth/auth_utils.py | 46 ++++++------------- .../proxy/hooks/parallel_request_limiter.py | 20 ++++---- .../proxy/test_model_info_default_limits.py | 2 +- 3 files changed, 26 insertions(+), 42 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index ace39c05ffc..b4e0093b91a 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -539,15 +539,14 @@ def bytes_to_mb(bytes_value: int): # helpers used by parallel request limiter to handle model rpm/tpm limits for a given api key -def _get_deployment_default_rpm_limit(model_name: str) -> Optional[int]: +def _get_deployment_default_limit(model_name: str, field: str) -> Optional[int]: """ - Return the default_api_key_rpm_limit for model_name. + Return the minimum value of `field` across all deployments for model_name, + or None if no deployment has the field set. - When multiple deployments share the same model name, returns the minimum - across all deployments that have the field set. This is the safest choice - for load-balanced setups: it ensures no deployment is over-consumed - regardless of which one actually serves a given request. - Returns None if no deployment has the field set. + When multiple deployments share the same model name, taking the minimum is + the safest choice for load-balanced setups: it ensures no deployment is + over-consumed regardless of which one actually serves a given request. """ from litellm.proxy.proxy_server import llm_router @@ -557,38 +556,19 @@ def _get_deployment_default_rpm_limit(model_name: str) -> Optional[int]: if not deployments: return None limits = [ - int(deployment.get("litellm_params", {}).get("default_api_key_rpm_limit")) + int(deployment.get("litellm_params", {}).get(field)) for deployment in deployments - if deployment.get("litellm_params", {}).get("default_api_key_rpm_limit") - is not None + if deployment.get("litellm_params", {}).get(field) is not None ] return min(limits) if limits else None +def _get_deployment_default_rpm_limit(model_name: str) -> Optional[int]: + return _get_deployment_default_limit(model_name, "default_api_key_rpm_limit") + + def _get_deployment_default_tpm_limit(model_name: str) -> Optional[int]: - """ - Return the default_api_key_tpm_limit for model_name. - - When multiple deployments share the same model name, returns the minimum - across all deployments that have the field set. This is the safest choice - for load-balanced setups: it ensures no deployment is over-consumed - regardless of which one actually serves a given request. - Returns None if no deployment has the field set. - """ - from litellm.proxy.proxy_server import llm_router - - if llm_router is None: - return None - deployments = llm_router.get_model_list(model_name=model_name) - if not deployments: - return None - limits = [ - int(deployment.get("litellm_params", {}).get("default_api_key_tpm_limit")) - for deployment in deployments - if deployment.get("litellm_params", {}).get("default_api_key_tpm_limit") - is not None - ] - return min(limits) if limits else None + return _get_deployment_default_limit(model_name, "default_api_key_tpm_limit") def get_key_model_rpm_limit( diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 49c6436c22f..55e89e02d67 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -539,6 +539,16 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): # Update usage - model group + API Key # ------------ model_group = get_model_group_from_litellm_kwargs(kwargs) + _success_tpm_limit = ( + get_key_model_tpm_limit(user_api_key_dict, model_name=model_group) + if model_group is not None + else None + ) + _success_rpm_limit = ( + get_key_model_rpm_limit(user_api_key_dict, model_name=model_group) + if model_group is not None + else None + ) if ( user_api_key is not None and model_group is not None @@ -546,14 +556,8 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): "model_rpm_limit" in user_api_key_metadata or "model_tpm_limit" in user_api_key_metadata or user_api_key_model_max_budget is not None - or get_key_model_tpm_limit( - user_api_key_dict, model_name=model_group - ) - is not None - or get_key_model_rpm_limit( - user_api_key_dict, model_name=model_group - ) - is not None + or _success_tpm_limit is not None + or _success_rpm_limit is not None ) ): request_count_api_key = ( diff --git a/tests/test_litellm/proxy/test_model_info_default_limits.py b/tests/test_litellm/proxy/test_model_info_default_limits.py index 8b85531785a..d9ebd554edc 100644 --- a/tests/test_litellm/proxy/test_model_info_default_limits.py +++ b/tests/test_litellm/proxy/test_model_info_default_limits.py @@ -119,7 +119,7 @@ class TestModelInfoEndpointWithRouter: user_api_key_dict = UserAPIKeyAuth(api_key="sk-test") with patch("litellm.proxy.proxy_server.llm_router", mock_router), \ - patch("litellm.proxy.proxy_server.llm_model_list", [{}]), \ + patch("litellm.proxy.proxy_server.llm_model_list", []), \ patch("litellm.proxy.proxy_server.user_model", None): response = await model_info_v1( user_api_key_dict=user_api_key_dict, From ae0769b1dfb44b4f4ec9a676e04c2e778b426957 Mon Sep 17 00:00:00 2001 From: Ephrim Stanley Date: Thu, 19 Mar 2026 07:40:47 -0400 Subject: [PATCH 14/73] fix: guard empty-dict team limits and malformed int in deployment default limits - Change `if team_limit:` to `if team_limit is not None:` in both get_key_model_rpm_limit and get_key_model_tpm_limit so that an explicitly-empty team rate-limit map ({}) is returned as-is instead of silently falling through to deployment defaults (P1 fix). - Replace the bare `int()` list comprehension in _get_deployment_default_limit with a loop that catches ValueError/TypeError so malformed config strings do not raise an unhandled exception during request handling (P2 fix). - Add corresponding unit tests for both edge cases. Co-Authored-By: Claude (claude-sonnet-4-6) --- litellm/proxy/auth/auth_utils.py | 17 +++--- .../proxy/auth/test_auth_utils.py | 54 +++++++++++++++++++ 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index b4e0093b91a..7d3427ed4c1 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -555,11 +555,14 @@ def _get_deployment_default_limit(model_name: str, field: str) -> Optional[int]: deployments = llm_router.get_model_list(model_name=model_name) if not deployments: return None - limits = [ - int(deployment.get("litellm_params", {}).get(field)) - for deployment in deployments - if deployment.get("litellm_params", {}).get(field) is not None - ] + limits = [] + for deployment in deployments: + raw = deployment.get("litellm_params", {}).get(field) + if raw is not None: + try: + limits.append(int(raw)) + except (ValueError, TypeError): + pass return min(limits) if limits else None @@ -602,7 +605,7 @@ def get_key_model_rpm_limit( # 3. Fallback to team metadata if user_api_key_dict.team_metadata: team_limit = user_api_key_dict.team_metadata.get("model_rpm_limit") - if team_limit: + if team_limit is not None: return team_limit # 4. Fallback to deployment default_api_key_rpm_limit @@ -645,7 +648,7 @@ def get_key_model_tpm_limit( # 3. Fallback to team metadata if user_api_key_dict.team_metadata: team_limit = user_api_key_dict.team_metadata.get("model_tpm_limit") - if team_limit: + if team_limit is not None: return team_limit # 4. Fallback to deployment default_api_key_tpm_limit diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 2058f61cb0f..b66c081a943 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -71,6 +71,19 @@ class TestGetKeyModelRpmLimit: assert result is None + def test_team_metadata_empty_rpm_dict_falls_through_to_deployment_default(self): + """Explicitly empty team model_rpm_limit ({}) should be returned as-is, not fallen through.""" + # An empty dict is a valid team limit map (no per-model limits configured). + # It should be returned directly rather than falling through to deployment defaults, + # so a team with an empty map is treated as unconstrained at the team level. + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + team_metadata={"model_rpm_limit": {}}, + ) + result = get_key_model_rpm_limit(user_api_key_dict) + assert result == {} + + class TestGetKeyModelTpmLimit: """Tests for get_key_model_tpm_limit function.""" @@ -137,6 +150,33 @@ class TestGetKeyModelTpmLimit: assert result == {"gpt-4": 10000} + def test_team_metadata_empty_tpm_dict_falls_through_to_deployment_default(self): + """Explicitly empty team model_tpm_limit ({}) should be returned as-is, not fallen through.""" + # An empty dict is a valid team limit map (no per-model limits configured). + # It should be returned directly rather than falling through to deployment defaults, + # so a team with an empty map is treated as unconstrained at the team level. + user_api_key_dict = UserAPIKeyAuth( + api_key="sk-123", + team_metadata={"model_tpm_limit": {}}, + ) + result = get_key_model_tpm_limit(user_api_key_dict) + assert result == {} + + + def test_skips_deployments_with_malformed_limit_value(self): + """Deployments with non-integer-parseable limit values are skipped without raising.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + {"model_name": "model1", "litellm_params": {"default_api_key_tpm_limit": "not-a-number"}}, + _make_deployment_dict("model1", tpm=500), + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_tpm_limit(user_api_key_dict, model_name="model1") + # The malformed deployment is skipped; the valid one provides 500 + assert result == {"model1": 500} + + class TestGetCustomerIdFromStandardHeaders: """Tests for _get_customer_id_from_standard_headers helper function.""" @@ -414,6 +454,20 @@ class TestDeploymentDefaultRpmLimit: assert result == {"model1": 75} + def test_skips_deployments_with_malformed_limit_value(self): + """Deployments with non-integer-parseable limit values are skipped without raising.""" + user_api_key_dict = UserAPIKeyAuth(api_key="sk-123") + mock_router = MagicMock() + mock_router.get_model_list.return_value = [ + {"model_name": "model1", "litellm_params": {"default_api_key_rpm_limit": "not-a-number"}}, + _make_deployment_dict("model1", rpm=100), + ] + with patch(_ROUTER_PATCH, mock_router): + result = get_key_model_rpm_limit(user_api_key_dict, model_name="model1") + # The malformed deployment is skipped; the valid one provides 100 + assert result == {"model1": 100} + + class TestDeploymentDefaultTpmLimit: """Tests for deployment default_api_key_tpm_limit fallback in get_key_model_tpm_limit.""" From 001501fb31dd77e6e911fdd565eec6f24a7ae26f Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 19 Mar 2026 15:56:11 +0100 Subject: [PATCH 15/73] fix(proxy): defer logging until post-call guardrails complete guardrail_information is None in StandardLoggingPayload because logging fires before post-call guardrails write to metadata. Non-streaming: wrapper_async stores a closure instead of calling create_task immediately. The proxy fires it in a try/finally after post_call_success_hook so the SLP is built with guardrail info. Streaming: a closure on logging_obj is called by CSW.__anext__ at stream end. The closure runs only guardrail hooks (not all callbacks) on the assembled response, then fires both logging handlers. This avoids behavioral changes for non-guardrail callbacks on streaming. --- .../docs/proxy/guardrails/custom_guardrail.md | 12 +- .../litellm_core_utils/streaming_handler.py | 36 +- litellm/proxy/common_request_processing.py | 407 ++++++++--- litellm/utils.py | 39 +- .../test_deferred_guardrail_logging.py | 687 ++++++++++++++++++ 5 files changed, 1056 insertions(+), 125 deletions(-) create mode 100644 tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py diff --git a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md index c9115cf8265..638cae9c835 100644 --- a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md +++ b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md @@ -117,6 +117,14 @@ guardrails: ::: +:::note Streaming and post_call guardrails + +For **streaming responses**, `post_call` guardrails run on the fully assembled response **after** all chunks have been delivered to the client. This means `post_call` guardrails on streaming are **audit-only** — they can inspect and log the complete response, but cannot block content delivery. Guardrail results are recorded in `guardrail_information` within the logging payload for compliance and auditing. + +To filter or block streaming content in real-time, use `async_post_call_streaming_iterator_hook` instead, which processes chunks as they arrive. + +::: +
Advanced: Multiple modes with individual event hooks @@ -655,8 +663,8 @@ class myCustomGuardrail(CustomGuardrail): | `apply_guardrail` | Simple method to check and optionally modify text | ✅ | INPUT or OUTPUT | ✅ | ✅ | ✅ | | `async_pre_call_hook` | A hook that runs before the LLM API call | ✅ | INPUT | ✅ | ❌ | ✅ | | `async_moderation_hook` | A hook that runs during the LLM API call| ✅ | INPUT | ❌ | ❌ | ✅ | -| `async_post_call_success_hook` | A hook that runs after a successful LLM API call| ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ | -| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses | ✅ | OUTPUT | ❌ | ✅ | ✅ | +| `async_post_call_success_hook` | A hook that runs after a successful LLM API call. For streaming, runs on the assembled response after delivery (audit-only, cannot block). | ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ (non-streaming only) | +| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses in real-time (can filter/block chunks) | ✅ | OUTPUT | ❌ | ✅ | ✅ | ## Frequently Asked Questions diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6e991e6911b..ccba18088ec 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2136,22 +2136,36 @@ class CustomStreamWrapper: self.sent_stream_usage = True return response - asyncio.create_task( - self.logging_obj.async_success_handler( + _deferred_cb = getattr( + self.logging_obj, + "_on_deferred_stream_complete", + None, + ) + if _deferred_cb is not None: + # Proxy has post-call guardrails — let the closure + # run guardrails on the assembled response, then + # fire logging with guardrail_information populated. + self.logging_obj._on_deferred_stream_complete = None # type: ignore[attr-defined] + asyncio.create_task( + _deferred_cb(complete_streaming_response, cache_hit) + ) + else: + asyncio.create_task( + self.logging_obj.async_success_handler( + complete_streaming_response, + cache_hit=cache_hit, + start_time=None, + end_time=None, + ) + ) + + executor.submit( + self.logging_obj.success_handler, complete_streaming_response, cache_hit=cache_hit, start_time=None, end_time=None, ) - ) - - executor.submit( - self.logging_obj.success_handler, - complete_streaming_response, - cache_hit=cache_hit, - start_time=None, - end_time=None, - ) raise StopAsyncIteration # Re-raise StopIteration else: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 72765aab7da..eb2a376ef16 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -45,7 +45,9 @@ from litellm.proxy.common_utils.callback_utils import ( from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.route_llm_request import route_request from litellm.proxy.utils import ProxyLogging +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.router import Router +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ServerToolUse # Type alias for streaming chunk serializer (chunk after hooks + cost injection -> wire format) @@ -801,7 +803,7 @@ class ProxyBaseLLMRequestProcessing: json.dumps(self.data, indent=4, default=str), ) - async def base_process_llm_request( + async def base_process_llm_request( # noqa: PLR0915 self, request: Request, fastapi_response: Response, @@ -926,6 +928,26 @@ class ProxyBaseLLMRequestProcessing: llm_router=llm_router, ) + # Defer async logging when post-call guardrails are configured so the + # StandardLoggingPayload is built after guardrails write to metadata. + # Cache the result to avoid scanning litellm.callbacks twice. + _has_post_call_guardrails = self._has_post_call_guardrails() + + # Non-streaming: defer the create_task in wrapper_async so the + # SLP is built after guardrails write to metadata. Streaming + # uses a separate closure mechanism (see below). + # + # Edge case: if _is_streaming_request is False but the response + # turns out to be a CustomStreamWrapper (rare provider behavior), + # wrapper_async exits early before the _defer_async_logging block + # so _enqueue_deferred_logging is never stored — the finally + # block is a no-op. The CSW path handles this correctly via + # _on_deferred_stream_complete, which fires its own logging. + if _has_post_call_guardrails and not self._is_streaming_request( + data=self.data, is_streaming_request=is_streaming_request + ): + logging_obj._defer_async_logging = True # type: ignore + tasks = [] # Start the moderation check (during_call_hook) as early as possible # This gives it a head start to mask/validate input while the proxy handles routing @@ -962,124 +984,181 @@ class ProxyBaseLLMRequestProcessing: response = responses[1] - hidden_params = getattr(response, "_hidden_params", {}) or {} - model_id = self._get_model_id_from_response(hidden_params, self.data) + try: + hidden_params = getattr(response, "_hidden_params", {}) or {} + model_id = self._get_model_id_from_response(hidden_params, self.data) - cache_key, api_base, response_cost = ( - hidden_params.get("cache_key", None) or "", - hidden_params.get("api_base", None) or "", - hidden_params.get("response_cost", None) or "", - ) - fastest_response_batch_completion, additional_headers = ( - hidden_params.get("fastest_response_batch_completion", None), - hidden_params.get("additional_headers", {}) or {}, - ) - - # Post Call Processing - if llm_router is not None: - self.data["deployment"] = llm_router.get_deployment(model_id=model_id) - asyncio.create_task( - proxy_logging_obj.update_request_status( - litellm_call_id=self.data.get("litellm_call_id", ""), status="success" + cache_key, api_base, response_cost = ( + hidden_params.get("cache_key", None) or "", + hidden_params.get("api_base", None) or "", + hidden_params.get("response_cost", None) or "", ) - ) - if self._is_streaming_request( - data=self.data, is_streaming_request=is_streaming_request - ) or self._is_streaming_response( - response - ): # use generate_responses to stream responses - custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( - user_api_key_dict=user_api_key_dict, - call_id=logging_obj.litellm_call_id, - model_id=model_id, - cache_key=cache_key, - api_base=api_base, - version=version, - response_cost=response_cost, - model_region=getattr(user_api_key_dict, "allowed_model_region", ""), - fastest_response_batch_completion=fastest_response_batch_completion, - request_data=self.data, - hidden_params=hidden_params, - litellm_logging_obj=logging_obj, - **additional_headers, + fastest_response_batch_completion, additional_headers = ( + hidden_params.get("fastest_response_batch_completion", None), + hidden_params.get("additional_headers", {}) or {}, ) - # Call response headers hook for streaming success - callback_headers = await proxy_logging_obj.post_call_response_headers_hook( - data=self.data, - user_api_key_dict=user_api_key_dict, - response=response, - request_headers=dict(request.headers), + # Post Call Processing + if llm_router is not None: + self.data["deployment"] = llm_router.get_deployment(model_id=model_id) + asyncio.create_task( + proxy_logging_obj.update_request_status( + litellm_call_id=self.data.get("litellm_call_id", ""), status="success" + ) ) - if callback_headers: - custom_headers.update(callback_headers) + if self._is_streaming_request( + data=self.data, is_streaming_request=is_streaming_request + ) or self._is_streaming_response( + response + ): # use generate_responses to stream responses + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=logging_obj.litellm_call_id, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + response_cost=response_cost, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + fastest_response_batch_completion=fastest_response_batch_completion, + request_data=self.data, + hidden_params=hidden_params, + litellm_logging_obj=logging_obj, + **additional_headers, + ) - # Preserve the original client-requested model (pre-alias mapping) for downstream - # streaming generators. Pre-call processing can rewrite `self.data["model"]` for - # aliasing/routing, but the OpenAI-compatible response `model` field should reflect - # what the client sent. - if requested_model_from_client: - self.data[ - "_litellm_client_requested_model" - ] = requested_model_from_client - if route_type == "allm_passthrough_route": - # Check if response is an async generator - if self._is_streaming_response(response): - if asyncio.iscoroutine(response): - generator = await response - else: - generator = response + # Call response headers hook for streaming success + callback_headers = await proxy_logging_obj.post_call_response_headers_hook( + data=self.data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=dict(request.headers), + ) + if callback_headers: + custom_headers.update(callback_headers) - # For passthrough routes, stream directly without error parsing - # since we're dealing with raw binary data (e.g., AWS event streams) - return StreamingResponse( - content=generator, - status_code=status.HTTP_200_OK, - headers=custom_headers, - ) - else: - # Traditional HTTP response with aiter_bytes - return StreamingResponse( - content=response.aiter_bytes(), - status_code=response.status_code, - headers=custom_headers, - ) - elif route_type == "anthropic_messages": - # Check if response is actually a streaming response (async generator) - # Non-streaming responses (dict) should be returned directly - # This handles cases like websearch_interception agentic loop - # which returns a non-streaming dict even for streaming requests - if self._is_streaming_response(response): - selected_data_generator = ( - ProxyBaseLLMRequestProcessing.async_sse_data_generator( - response=response, - user_api_key_dict=user_api_key_dict, - request_data=self.data, - proxy_logging_obj=proxy_logging_obj, + # Preserve the original client-requested model (pre-alias mapping) for downstream + # streaming generators. Pre-call processing can rewrite `self.data["model"]` for + # aliasing/routing, but the OpenAI-compatible response `model` field should reflect + # what the client sent. + if requested_model_from_client: + self.data[ + "_litellm_client_requested_model" + ] = requested_model_from_client + + # Streaming: attach a closure that CSW.__anext__ will call + # at stream end instead of firing logging directly. The + # closure runs ONLY guardrail hooks (not all callbacks) on + # the assembled response so guardrail_information is + # populated, then fires both logging handlers. + # Only for CustomStreamWrapper — raw async generators from + # passthrough routes bypass CSW and would orphan the closure. + from litellm.litellm_core_utils.streaming_handler import ( + CustomStreamWrapper, + ) + + if _has_post_call_guardrails and isinstance( + response, CustomStreamWrapper + ): + # Intentionally a live reference (not a copy) — mirrors + # ProxyLogging.post_call_success_hook which also mutates + # data["guardrail_to_apply"] during iteration. + _captured_data = self.data + _captured_user_api_key_dict = user_api_key_dict + _captured_logging_obj = logging_obj + + async def _on_deferred_stream_complete( + assembled_response, cache_hit + ): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data=_captured_data, + captured_user_api_key_dict=_captured_user_api_key_dict, + captured_logging_obj=_captured_logging_obj, + assembled_response=assembled_response, + cache_hit=cache_hit, ) + + logging_obj._on_deferred_stream_complete = _on_deferred_stream_complete # type: ignore[attr-defined] + + if route_type == "allm_passthrough_route": + # Check if response is an async generator + if self._is_streaming_response(response): + if asyncio.iscoroutine(response): + generator = await response + else: + generator = response + + # For passthrough routes, stream directly without error parsing + # since we're dealing with raw binary data (e.g., AWS event streams) + return StreamingResponse( + content=generator, + status_code=status.HTTP_200_OK, + headers=custom_headers, + ) + else: + # Traditional HTTP response with aiter_bytes + return StreamingResponse( + content=response.aiter_bytes(), + status_code=response.status_code, + headers=custom_headers, + ) + elif route_type == "anthropic_messages": + # Check if response is actually a streaming response (async generator) + # Non-streaming responses (dict) should be returned directly + # This handles cases like websearch_interception agentic loop + # which returns a non-streaming dict even for streaming requests + if self._is_streaming_response(response): + selected_data_generator = ( + ProxyBaseLLMRequestProcessing.async_sse_data_generator( + response=response, + user_api_key_dict=user_api_key_dict, + request_data=self.data, + proxy_logging_obj=proxy_logging_obj, + ) + ) + return await create_response( + generator=selected_data_generator, + media_type="text/event-stream", + headers=custom_headers, + ) + # Non-streaming response - fall through to normal response handling + elif select_data_generator: + selected_data_generator = select_data_generator( + response=response, + user_api_key_dict=user_api_key_dict, + request_data=self.data, ) return await create_response( generator=selected_data_generator, media_type="text/event-stream", headers=custom_headers, ) - # Non-streaming response - fall through to normal response handling - elif select_data_generator: - selected_data_generator = select_data_generator( - response=response, - user_api_key_dict=user_api_key_dict, - request_data=self.data, - ) - return await create_response( - generator=selected_data_generator, - media_type="text/event-stream", - headers=custom_headers, - ) - ### CALL HOOKS ### - modify outgoing data - response = await proxy_logging_obj.post_call_success_hook( - data=self.data, user_api_key_dict=user_api_key_dict, response=response - ) + ### CALL HOOKS ### - modify outgoing data + # If we reach here with a streaming closure still set, it means + # no early-return route consumed the CSW (hypothetical fallthrough). + # Clear the closure so guardrails run inline as before — this + # preserves blocking behavior and avoids double invocation. + if getattr(logging_obj, "_on_deferred_stream_complete", None): + logging_obj._on_deferred_stream_complete = None # type: ignore[attr-defined] + response = await proxy_logging_obj.post_call_success_hook( + data=self.data, user_api_key_dict=user_api_key_dict, response=response + ) + finally: + # Enqueue deferred logging after post-call guardrails have written + # guardrail_information to metadata. The finally block ensures + # logging fires even if a guardrail raises. + # For streaming early-returns: no closure is stored (wrapper_async + # returns before the deferred block), so _enqueue_fn is None — no-op. + _enqueue_fn = getattr(logging_obj, "_enqueue_deferred_logging", None) + if _enqueue_fn is not None: + logging_obj._enqueue_deferred_logging = None # type: ignore[attr-defined] + try: + _enqueue_fn() + except Exception as e: + verbose_proxy_logger.exception( + "Error firing deferred logging: %s", e + ) # Always return the client-requested model name (not provider-prefixed internal identifiers) # for OpenAI-compatible responses. @@ -1217,6 +1296,126 @@ class ProxyBaseLLMRequestProcessing: return True return False + @staticmethod + def _has_post_call_guardrails() -> bool: + """ + Check if any registered callback is a post-call guardrail. + + Uses the global litellm.callbacks list rather than per-request + should_run_guardrail() — intentionally conservative so that the + check is simple and stateless. The deferral path produces + identical logging output, just fires it slightly later, so + false-positives are harmless. + """ + for cb in litellm.callbacks: + if isinstance(cb, CustomGuardrail) and cb._event_hook_is_event_type( + GuardrailEventHooks.post_call + ): + return True + return False + + @staticmethod + async def _run_deferred_stream_guardrails( + captured_data: dict, + captured_user_api_key_dict: "UserAPIKeyAuth", + captured_logging_obj: Any, + assembled_response: Any, + cache_hit: Any, + ) -> None: + """ + Run only post-call guardrail hooks on an assembled streaming response, + then fire both async and sync logging handlers. + + Called by CSW.__anext__ at stream end via a closure stored on + logging_obj._on_deferred_stream_complete. + + This is audit-only — content has already been delivered to the client. + Blocking guardrails that raise HTTPException cannot prevent content + delivery for streaming. Per-chunk filtering should use + async_post_call_streaming_hook instead. + + Extracted as a static method so tests can call the production + implementation directly rather than reimplementing the closure. + """ + from litellm.litellm_core_utils.thread_pool_executor import executor + from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( + UnifiedLLMGuardrails, + ) + from litellm.proxy.proxy_server import llm_router as _global_llm_router + from litellm.proxy.utils import _check_and_merge_model_level_guardrails + + _response = assembled_response + _unified_guardrail = UnifiedLLMGuardrails() + guardrail_data = _check_and_merge_model_level_guardrails( + data=captured_data, llm_router=_global_llm_router + ) + for cb in litellm.callbacks: + if not isinstance(cb, CustomGuardrail): + continue + if not cb.should_run_guardrail( + data=guardrail_data, + event_type=GuardrailEventHooks.post_call, + ): + continue + try: + guardrail_result = None + if "apply_guardrail" in type(cb).__dict__: + captured_data["guardrail_to_apply"] = cb + guardrail_result = ( + await _unified_guardrail.async_post_call_success_hook( + user_api_key_dict=captured_user_api_key_dict, + data=captured_data, + response=_response, + ) + ) + else: + guardrail_result = await cb.async_post_call_success_hook( + user_api_key_dict=captured_user_api_key_dict, + data=captured_data, + response=_response, + ) + if guardrail_result is not None: + _response = guardrail_result + except Exception as e: + verbose_proxy_logger.exception( + "Error running post-call guardrail %s on streaming response: %s", + getattr(cb, "guardrail_name", type(cb).__name__), + e, + ) + if isinstance(e, HTTPException) and hasattr( + captured_logging_obj, "model_call_details" + ): + captured_logging_obj.model_call_details.setdefault( + "metadata", {} + )["guardrail_blocked"] = True + + try: + asyncio.create_task( + captured_logging_obj.async_success_handler( + _response, + cache_hit=cache_hit, + start_time=None, + end_time=None, + ) + ) + except Exception as e: + verbose_proxy_logger.exception( + "Error in deferred streaming async logging: %s", e, + ) + + try: + executor.submit( + captured_logging_obj.success_handler, + _response, + cache_hit=cache_hit, + start_time=None, + end_time=None, + ) + except Exception as e: + verbose_proxy_logger.exception( + "Error in deferred streaming sync logging: %s", e, + ) + async def _handle_llm_api_exception( self, e: Exception, diff --git a/litellm/utils.py b/litellm/utils.py index 81d749ab821..0fda994aea1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1944,15 +1944,38 @@ def client(original_function): # noqa: PLR0915 ) # LOG SUCCESS - handle streaming success logging in the _next_ object - asyncio.create_task( - _client_async_logging_helper( - logging_obj=logging_obj, - result=result, - start_time=start_time, - end_time=end_time, - is_completion_with_fallbacks=is_completion_with_fallbacks, + # NOTE: streaming requests return early (before this point) via + # CustomStreamWrapper, so this block is non-streaming only. + if getattr(logging_obj, "_defer_async_logging", False): + # Proxy has post-call guardrails that must complete before the + # SLP is built. Store a closure the proxy will call after + # post_call_success_hook so guardrail_information is in metadata. + # Only create_task is deferred; sync callbacks fire immediately + # (below, outside the if/else) for billing/rate-limiting. + def _enqueue_deferred_logging() -> None: + asyncio.create_task( + _client_async_logging_helper( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + ) + ) + + logging_obj._enqueue_deferred_logging = _enqueue_deferred_logging # type: ignore + else: + asyncio.create_task( + _client_async_logging_helper( + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + is_completion_with_fallbacks=is_completion_with_fallbacks, + ) ) - ) + + # Sync callbacks always fire immediately regardless of deferral logging_obj.handle_sync_success_callbacks_for_async_calls( result=result, start_time=start_time, diff --git a/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py new file mode 100644 index 00000000000..82e389da651 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py @@ -0,0 +1,687 @@ +""" +Tests for deferred logging with post-call guardrails. + +When post-call guardrails are configured, the async logging task is deferred +until after guardrails complete. This ensures the StandardLoggingPayload +is built with guardrail_information populated. + +Non-streaming: create_task in wrapper_async is replaced by a closure that + the proxy fires in a try/finally after post_call_success_hook. + +Streaming: a closure on logging_obj is called by CSW.__anext__ at stream end. + The closure runs ONLY guardrail hooks (not all callbacks), then fires + both logging handlers. +""" + +import asyncio +import os +import sys +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from starlette.exceptions import HTTPException + +sys.path.insert(0, os.path.abspath("../../../..")) + +import litellm +from litellm.caching.caching import DualCache +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.utils import ProxyLogging +from litellm.types.guardrails import GuardrailEventHooks + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class PostCallGuardrail(CustomGuardrail): + """A post-call guardrail.""" + + def __init__(self): + super().__init__( + guardrail_name="post-call", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def async_post_call_success_hook( + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: Any + ) -> Any: + return response + + +class PreCallGuardrail(CustomGuardrail): + """A pre-call-only guardrail — should NOT trigger deferral.""" + + def __init__(self): + super().__init__( + guardrail_name="pre-call", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + ) + + +class AllEventsGuardrail(CustomGuardrail): + """A guardrail with event_hook=None (runs on all events).""" + + def __init__(self): + super().__init__( + guardrail_name="all-events", + default_on=True, + event_hook=None, + ) + + +# --------------------------------------------------------------------------- +# 1. _has_post_call_guardrails detection +# --------------------------------------------------------------------------- + + +class TestHasPostCallGuardrails: + def test_returns_true_for_post_call_guardrail(self): + with patch("litellm.callbacks", [PostCallGuardrail()]): + assert ProxyBaseLLMRequestProcessing._has_post_call_guardrails() is True + + def test_returns_true_for_event_hook_none(self): + """event_hook=None means 'all events', including post_call.""" + with patch("litellm.callbacks", [AllEventsGuardrail()]): + assert ProxyBaseLLMRequestProcessing._has_post_call_guardrails() is True + + def test_returns_false_for_pre_call_only(self): + with patch("litellm.callbacks", [PreCallGuardrail()]): + assert ProxyBaseLLMRequestProcessing._has_post_call_guardrails() is False + + def test_returns_false_for_no_callbacks(self): + with patch("litellm.callbacks", []): + assert ProxyBaseLLMRequestProcessing._has_post_call_guardrails() is False + + def test_ignores_non_guardrail_callbacks(self): + """String callbacks and CustomLogger instances are not guardrails.""" + with patch("litellm.callbacks", ["langfuse", CustomLogger()]): + assert ProxyBaseLLMRequestProcessing._has_post_call_guardrails() is False + + def test_returns_true_for_list_with_post_call(self): + """event_hook as a list containing post_call should trigger deferral.""" + + class ListGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="list-post", + default_on=True, + event_hook=[GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call], + ) + + with patch("litellm.callbacks", [ListGuardrail()]): + assert ProxyBaseLLMRequestProcessing._has_post_call_guardrails() is True + + def test_returns_false_for_list_without_post_call(self): + """event_hook as a list without post_call should not trigger deferral.""" + + class ListGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="list-pre", + default_on=True, + event_hook=[GuardrailEventHooks.pre_call], + ) + + with patch("litellm.callbacks", [ListGuardrail()]): + assert ProxyBaseLLMRequestProcessing._has_post_call_guardrails() is False + + +# --------------------------------------------------------------------------- +# 2. Non-streaming: deferral flag → closure stored, create_task skipped +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_deferred_flag_stores_and_executes_closure(): + """ + When _defer_async_logging is True on logging_obj: + 1. wrapper_async stores a callable closure instead of calling create_task + 2. Calling the closure fires create_task + 3. Sync callbacks fire immediately (not deferred) + """ + mock_logging_obj = MagicMock() + mock_logging_obj._defer_async_logging = True + mock_logging_obj._enqueue_deferred_logging = None + + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + litellm_logging_obj=mock_logging_obj, + ) + + # Closure was stored + enqueue_fn = mock_logging_obj._enqueue_deferred_logging + assert callable(enqueue_fn), "Closure should be stored on logging_obj" + + # Sync callbacks fired immediately + mock_logging_obj.handle_sync_success_callbacks_for_async_calls.assert_called_once() + + # Calling the closure fires create_task + created_tasks = [] + real_create_task = asyncio.create_task + + def tracking_create_task(coro): + task = real_create_task(coro) + created_tasks.append(task) + return task + + with patch("asyncio.create_task", side_effect=tracking_create_task): + enqueue_fn() + + assert len(created_tasks) >= 1, "Closure should fire asyncio.create_task" + + for task in created_tasks: + if not task.done(): + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): + pass + + +# --------------------------------------------------------------------------- +# 3. Non-streaming regression: without flag, create_task fires normally +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_no_flag_fires_create_task_normally(): + """Without _defer_async_logging, wrapper_async calls create_task as before.""" + created_tasks = [] + real_create_task = asyncio.create_task + + def tracking_create_task(coro): + task = real_create_task(coro) + created_tasks.append(task) + return task + + with patch("asyncio.create_task", side_effect=tracking_create_task): + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + ) + + assert len(created_tasks) >= 1 + + for task in created_tasks: + if not task.done(): + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): + pass + + +# --------------------------------------------------------------------------- +# 4. Non-streaming: deferred logging fires even if guardrail raises +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_deferred_logging_fires_on_guardrail_exception(): + """ + If post_call_success_hook raises (e.g., guardrail blocks content), + the deferred logging closure must still fire (via try/finally). + """ + enqueue_called = False + + def mock_enqueue(): + nonlocal enqueue_called + enqueue_called = True + + class BlockingGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="blocker", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def async_post_call_success_hook( + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: Any + ) -> Any: + raise HTTPException(status_code=400, detail="Content blocked") + + guardrail = BlockingGuardrail() + + logging_obj = MagicMock() + logging_obj._enqueue_deferred_logging = mock_enqueue + + with patch("litellm.callbacks", [guardrail]): + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + + with pytest.raises(HTTPException): + try: + await proxy_logging.post_call_success_hook( + data={"model": "gpt-4", "metadata": {}}, + response=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(api_key="test"), + ) + finally: + # Mirrors the proxy's finally block + _enqueue_fn = getattr(logging_obj, "_enqueue_deferred_logging", None) + if _enqueue_fn is not None: + logging_obj._enqueue_deferred_logging = None + _enqueue_fn() + + assert enqueue_called is True + assert logging_obj._enqueue_deferred_logging is None + + +# --------------------------------------------------------------------------- +# 5. Streaming: closure defers logging at stream end +# --------------------------------------------------------------------------- + + +class TestDeferredStreamingClosure: + @pytest.mark.asyncio + async def test_streaming_closure_defers_logging(self): + """When _on_deferred_stream_complete is set, CSW calls the closure + instead of firing async_success_handler directly.""" + mock_logging_obj = MagicMock() + callback_called = False + callback_args = {} + + async def mock_callback(assembled_response, cache_hit): + nonlocal callback_called, callback_args + callback_called = True + callback_args = {"response": assembled_response, "cache_hit": cache_hit} + + mock_logging_obj._on_deferred_stream_complete = mock_callback + + resp = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + stream=True, + litellm_logging_obj=mock_logging_obj, + ) + async for _ in resp: + pass + + await asyncio.sleep(0) + + assert callback_called is True, "Closure should be called at stream end" + assert callback_args["response"] is not None + assert mock_logging_obj._on_deferred_stream_complete is None + + @pytest.mark.asyncio + async def test_streaming_no_closure_fires_normally(self): + """Regression: without closure, CSW fires logging immediately.""" + created_tasks = [] + real_create_task = asyncio.create_task + + def tracking_create_task(coro): + task = real_create_task(coro) + created_tasks.append(task) + return task + + resp = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + stream=True, + ) + with patch("asyncio.create_task", side_effect=tracking_create_task): + async for _ in resp: + pass + + assert len(created_tasks) >= 1 + for task in created_tasks: + if not task.done(): + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): + pass + + @pytest.mark.asyncio + async def test_closure_runs_only_guardrail_hooks(self): + """The closure must call only CustomGuardrail hooks, not all callbacks. + This is the key v2 change — PR #23929 called post_call_success_hook + which ran ALL callbacks, causing behavioral changes for streaming.""" + guardrail_called = False + logger_called = False + + class TrackingGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="tracker", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def async_post_call_success_hook( + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: Any + ) -> Any: + nonlocal guardrail_called + guardrail_called = True + return response + + class TrackingLogger(CustomLogger): + async def async_post_call_success_hook( + self, user_api_key_dict, data, response + ): + nonlocal logger_called + logger_called = True + return response + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + + async def track_async_success(*args, **kwargs): + pass + + mock_logging_obj.async_success_handler = track_async_success + + tracking_guardrail = TrackingGuardrail() + tracking_logger = TrackingLogger() + + # Use the real production static method via a thin closure + _captured_data = {"model": "gpt-4", "metadata": {}} + _captured_user_api_key_dict = UserAPIKeyAuth(api_key="test") + + async def _on_deferred_stream_complete(assembled_response, cache_hit): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data=_captured_data, + captured_user_api_key_dict=_captured_user_api_key_dict, + captured_logging_obj=mock_logging_obj, + assembled_response=assembled_response, + cache_hit=cache_hit, + ) + + mock_logging_obj._on_deferred_stream_complete = _on_deferred_stream_complete + + with patch("litellm.callbacks", [tracking_guardrail, tracking_logger]): + resp = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + stream=True, + litellm_logging_obj=mock_logging_obj, + ) + async for _ in resp: + pass + + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert guardrail_called is True, "Guardrail hook should be called" + assert logger_called is False, "Non-guardrail logger should NOT be called by closure" + + @pytest.mark.asyncio + async def test_closure_passes_guardrail_modified_response_to_logging(self): + """The closure passes the guardrail-modified response to logging handlers.""" + mock_logging_obj = MagicMock() + modified_response = MagicMock() + logged_response = None + + async def mock_async_success(*args, **kwargs): + nonlocal logged_response + logged_response = args[0] if args else None + + mock_logging_obj.async_success_handler = mock_async_success + + async def closure(assembled_response, cache_hit): + # Simulate guardrail modifying the response + asyncio.create_task( + mock_logging_obj.async_success_handler( + modified_response, cache_hit=cache_hit, start_time=None, end_time=None + ) + ) + + mock_logging_obj._on_deferred_stream_complete = closure + + resp = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + stream=True, + litellm_logging_obj=mock_logging_obj, + ) + async for _ in resp: + pass + + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert logged_response is modified_response + + @pytest.mark.asyncio + async def test_closure_logs_even_on_guardrail_exception(self): + """If the guardrail raises HTTPException, logging still fires + and guardrail_blocked is set in metadata.""" + logging_called = False + + async def mock_async_success(*args, **kwargs): + nonlocal logging_called + logging_called = True + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + mock_logging_obj.async_success_handler = mock_async_success + + async def closure(assembled_response, cache_hit): + _response = assembled_response + try: + raise HTTPException(status_code=400, detail="Blocked") + except Exception as e: + if isinstance(e, HTTPException) and hasattr( + mock_logging_obj, "model_call_details" + ): + mock_logging_obj.model_call_details.setdefault( + "metadata", {} + )["guardrail_blocked"] = True + + asyncio.create_task( + mock_logging_obj.async_success_handler( + _response, cache_hit=cache_hit, start_time=None, end_time=None + ) + ) + + mock_logging_obj._on_deferred_stream_complete = closure + + resp = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + stream=True, + litellm_logging_obj=mock_logging_obj, + ) + async for _ in resp: + pass + + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert logging_called is True + assert mock_logging_obj.model_call_details["metadata"].get( + "guardrail_blocked" + ) is True + + @pytest.mark.asyncio + async def test_transient_error_does_not_set_guardrail_blocked(self): + """Transient errors (not HTTPException) should NOT set guardrail_blocked.""" + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + + async def mock_async_success(*args, **kwargs): + pass + + mock_logging_obj.async_success_handler = mock_async_success + + async def closure(assembled_response, cache_hit): + try: + raise ConnectionError("Network timeout") + except Exception as e: + if isinstance(e, HTTPException) and hasattr( + mock_logging_obj, "model_call_details" + ): + mock_logging_obj.model_call_details.setdefault( + "metadata", {} + )["guardrail_blocked"] = True + + asyncio.create_task( + mock_logging_obj.async_success_handler( + assembled_response, cache_hit=cache_hit, start_time=None, end_time=None + ) + ) + + mock_logging_obj._on_deferred_stream_complete = closure + + resp = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + stream=True, + litellm_logging_obj=mock_logging_obj, + ) + async for _ in resp: + pass + + await asyncio.sleep(0) + + assert mock_logging_obj.model_call_details["metadata"].get( + "guardrail_blocked" + ) is not True + + @pytest.mark.asyncio + async def test_production_closure_integration(self): + """Integration test: calls the real _run_deferred_stream_guardrails + static method and verifies it calls guardrail hooks and passes + the modified response to logging.""" + hook_called = False + logged_response = None + modified_response = MagicMock() + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + + async def track_async_success(*args, **kwargs): + nonlocal logged_response + logged_response = args[0] if args else None + + mock_logging_obj.async_success_handler = track_async_success + + class TestGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="test", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def async_post_call_success_hook( + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: Any + ) -> Any: + nonlocal hook_called + hook_called = True + return modified_response + + guardrail = TestGuardrail() + + async def _on_deferred_stream_complete(assembled_response, cache_hit): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={"model": "gpt-4", "metadata": {}}, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="test"), + captured_logging_obj=mock_logging_obj, + assembled_response=assembled_response, + cache_hit=cache_hit, + ) + + mock_logging_obj._on_deferred_stream_complete = _on_deferred_stream_complete + + with patch("litellm.callbacks", [guardrail]): + resp = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + stream=True, + litellm_logging_obj=mock_logging_obj, + ) + async for _ in resp: + pass + + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert hook_called is True, \ + "Production closure must call guardrail hook" + assert logged_response is modified_response, \ + "Production closure must pass guardrail-modified response to logging" + + @pytest.mark.asyncio + async def test_apply_guardrail_path_uses_unified_guardrail(self): + """Guardrails that define apply_guardrail should be dispatched through + UnifiedLLMGuardrails.async_post_call_success_hook via the real + _run_deferred_stream_guardrails static method.""" + from litellm.types.utils import GenericGuardrailAPIInputs + + unified_hook_called = False + + class ApplyGuardrailType(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="apply-type", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ) -> GenericGuardrailAPIInputs: + nonlocal unified_hook_called + unified_hook_called = True + return inputs + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + logged_response = None + + async def track_async_success(*args, **kwargs): + nonlocal logged_response + logged_response = args[0] if args else None + + mock_logging_obj.async_success_handler = track_async_success + + guardrail = ApplyGuardrailType() + + async def _on_deferred_stream_complete(assembled_response, cache_hit): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={"model": "gpt-4", "metadata": {}}, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="test"), + captured_logging_obj=mock_logging_obj, + assembled_response=assembled_response, + cache_hit=cache_hit, + ) + + mock_logging_obj._on_deferred_stream_complete = _on_deferred_stream_complete + + with patch("litellm.callbacks", [guardrail]): + resp = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="Hello!", + stream=True, + litellm_logging_obj=mock_logging_obj, + ) + async for _ in resp: + pass + + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert unified_hook_called is True, \ + "apply_guardrail guardrails must be dispatched through UnifiedLLMGuardrails" + assert logged_response is not None, \ + "Logging must fire after unified guardrail path" From 4b8c532ba8cbdb9b55fc006c9e218e3dd669d5e3 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 19 Mar 2026 17:39:06 +0100 Subject: [PATCH 16/73] fix(proxy): pass guardrail_data to hooks in streaming deferred path Use the merged guardrail_data dict (from _check_and_merge_model_level_guardrails) for hook invocations in _run_deferred_stream_guardrails, instead of the original captured_data. This ensures model-level non-default guardrails are visible to inner should_run_guardrail re-checks inside UnifiedLLMGuardrails. Rewrite three hand-crafted closure tests to exercise the production _run_deferred_stream_guardrails exception-handling path. Add three new tests that use deep-copy mocks to prove hooks receive the merged dict. --- litellm/proxy/common_request_processing.py | 6 +- .../test_deferred_guardrail_logging.py | 395 ++++++++++++++---- 2 files changed, 309 insertions(+), 92 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index eb2a376ef16..1db8327482d 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1360,18 +1360,18 @@ class ProxyBaseLLMRequestProcessing: try: guardrail_result = None if "apply_guardrail" in type(cb).__dict__: - captured_data["guardrail_to_apply"] = cb + guardrail_data["guardrail_to_apply"] = cb guardrail_result = ( await _unified_guardrail.async_post_call_success_hook( user_api_key_dict=captured_user_api_key_dict, - data=captured_data, + data=guardrail_data, response=_response, ) ) else: guardrail_result = await cb.async_post_call_success_hook( user_api_key_dict=captured_user_api_key_dict, - data=captured_data, + data=guardrail_data, response=_response, ) if guardrail_result is not None: diff --git a/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py index 82e389da651..f5c9eeba1c6 100644 --- a/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py +++ b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py @@ -20,7 +20,7 @@ from typing import Any from unittest.mock import MagicMock, patch import pytest -from starlette.exceptions import HTTPException +from fastapi import HTTPException sys.path.insert(0, os.path.abspath("../../../..")) @@ -421,139 +421,141 @@ class TestDeferredStreamingClosure: @pytest.mark.asyncio async def test_closure_passes_guardrail_modified_response_to_logging(self): - """The closure passes the guardrail-modified response to logging handlers.""" - mock_logging_obj = MagicMock() - modified_response = MagicMock() + """The production _run_deferred_stream_guardrails must pass the + guardrail-modified response to async_success_handler.""" logged_response = None + modified_response = MagicMock() - async def mock_async_success(*args, **kwargs): + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + + async def track_async_success(*args, **kwargs): nonlocal logged_response logged_response = args[0] if args else None - mock_logging_obj.async_success_handler = mock_async_success + mock_logging_obj.async_success_handler = track_async_success - async def closure(assembled_response, cache_hit): - # Simulate guardrail modifying the response - asyncio.create_task( - mock_logging_obj.async_success_handler( - modified_response, cache_hit=cache_hit, start_time=None, end_time=None + class ModifyingGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="modifier", + default_on=True, + event_hook=GuardrailEventHooks.post_call, ) + + async def async_post_call_success_hook( + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: Any + ) -> Any: + return modified_response + + guardrail = ModifyingGuardrail() + + with patch("litellm.callbacks", [guardrail]): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={"model": "gpt-4", "metadata": {}}, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="test"), + captured_logging_obj=mock_logging_obj, + assembled_response=MagicMock(), + cache_hit=False, ) - mock_logging_obj._on_deferred_stream_complete = closure - - resp = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "hi"}], - mock_response="Hello!", - stream=True, - litellm_logging_obj=mock_logging_obj, - ) - async for _ in resp: - pass - await asyncio.sleep(0) await asyncio.sleep(0) - assert logged_response is modified_response + assert logged_response is modified_response, \ + "Logging must receive the guardrail-modified response" @pytest.mark.asyncio async def test_closure_logs_even_on_guardrail_exception(self): - """If the guardrail raises HTTPException, logging still fires - and guardrail_blocked is set in metadata.""" + """If a guardrail raises HTTPException, the production + _run_deferred_stream_guardrails must still fire logging + and set guardrail_blocked in metadata.""" logging_called = False - async def mock_async_success(*args, **kwargs): + class BlockingGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="blocker", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def async_post_call_success_hook( + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: Any + ) -> Any: + raise HTTPException(status_code=400, detail="Blocked") + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + + async def track_async_success(*args, **kwargs): nonlocal logging_called logging_called = True - mock_logging_obj = MagicMock() - mock_logging_obj.model_call_details = {"metadata": {}} - mock_logging_obj.async_success_handler = mock_async_success + mock_logging_obj.async_success_handler = track_async_success - async def closure(assembled_response, cache_hit): - _response = assembled_response - try: - raise HTTPException(status_code=400, detail="Blocked") - except Exception as e: - if isinstance(e, HTTPException) and hasattr( - mock_logging_obj, "model_call_details" - ): - mock_logging_obj.model_call_details.setdefault( - "metadata", {} - )["guardrail_blocked"] = True + guardrail = BlockingGuardrail() - asyncio.create_task( - mock_logging_obj.async_success_handler( - _response, cache_hit=cache_hit, start_time=None, end_time=None - ) + with patch("litellm.callbacks", [guardrail]): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={"model": "gpt-4", "metadata": {}}, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="test"), + captured_logging_obj=mock_logging_obj, + assembled_response=MagicMock(), + cache_hit=False, ) - mock_logging_obj._on_deferred_stream_complete = closure - - resp = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "hi"}], - mock_response="Hello!", - stream=True, - litellm_logging_obj=mock_logging_obj, - ) - async for _ in resp: - pass - await asyncio.sleep(0) await asyncio.sleep(0) - assert logging_called is True + assert logging_called is True, \ + "Logging must fire even when guardrail raises HTTPException" assert mock_logging_obj.model_call_details["metadata"].get( "guardrail_blocked" - ) is True + ) is True, "guardrail_blocked must be set for HTTPException" @pytest.mark.asyncio async def test_transient_error_does_not_set_guardrail_blocked(self): - """Transient errors (not HTTPException) should NOT set guardrail_blocked.""" + """Transient errors (not HTTPException) should NOT set + guardrail_blocked. Uses the production _run_deferred_stream_guardrails.""" + + class TransientErrorGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="transient", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def async_post_call_success_hook( + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: Any + ) -> Any: + raise ConnectionError("Network timeout") + mock_logging_obj = MagicMock() mock_logging_obj.model_call_details = {"metadata": {}} - async def mock_async_success(*args, **kwargs): + async def track_async_success(*args, **kwargs): pass - mock_logging_obj.async_success_handler = mock_async_success + mock_logging_obj.async_success_handler = track_async_success - async def closure(assembled_response, cache_hit): - try: - raise ConnectionError("Network timeout") - except Exception as e: - if isinstance(e, HTTPException) and hasattr( - mock_logging_obj, "model_call_details" - ): - mock_logging_obj.model_call_details.setdefault( - "metadata", {} - )["guardrail_blocked"] = True + guardrail = TransientErrorGuardrail() - asyncio.create_task( - mock_logging_obj.async_success_handler( - assembled_response, cache_hit=cache_hit, start_time=None, end_time=None - ) + with patch("litellm.callbacks", [guardrail]): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={"model": "gpt-4", "metadata": {}}, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="test"), + captured_logging_obj=mock_logging_obj, + assembled_response=MagicMock(), + cache_hit=False, ) - mock_logging_obj._on_deferred_stream_complete = closure - - resp = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "hi"}], - mock_response="Hello!", - stream=True, - litellm_logging_obj=mock_logging_obj, - ) - async for _ in resp: - pass - await asyncio.sleep(0) assert mock_logging_obj.model_call_details["metadata"].get( "guardrail_blocked" - ) is not True + ) is not True, "guardrail_blocked must NOT be set for transient errors" @pytest.mark.asyncio async def test_production_closure_integration(self): @@ -685,3 +687,218 @@ class TestDeferredStreamingClosure: "apply_guardrail guardrails must be dispatched through UnifiedLLMGuardrails" assert logged_response is not None, \ "Logging must fire after unified guardrail path" + + @pytest.mark.asyncio + async def test_hooks_receive_merged_guardrail_data(self): + """Hooks must receive guardrail_data (the merged dict from + _check_and_merge_model_level_guardrails), not the original + captured_data. This ensures model-level non-default guardrails + are visible to any inner should_run_guardrail re-checks. + + Uses a deep-copy mock to break the shallow-copy side-effect that + would otherwise mask the bug — verifying the code is explicitly + correct, not correct-by-accident.""" + import copy + + hook_received_data = None + + class InspectingGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="inspector", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def async_post_call_success_hook( + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: Any + ) -> Any: + nonlocal hook_received_data + hook_received_data = data + return response + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + + async def track_async_success(*args, **kwargs): + pass + + mock_logging_obj.async_success_handler = track_async_success + + guardrail = InspectingGuardrail() + + captured_data = {"model": "gpt-4", "metadata": {"existing_key": "value"}} + + def mock_merge(data, llm_router): + """Return a fully independent dict (deep copy) so the original + captured_data is NOT mutated. This simulates a correct merge + implementation and proves _run_deferred_stream_guardrails uses + the return value, not the original data.""" + merged = copy.deepcopy(data) + merged["metadata"]["guardrails"] = ["model-guardrail"] + merged["_merged_marker"] = True + return merged + + with patch("litellm.callbacks", [guardrail]), \ + patch( + "litellm.proxy.utils._check_and_merge_model_level_guardrails", + side_effect=mock_merge, + ): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data=captured_data, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="test"), + captured_logging_obj=mock_logging_obj, + assembled_response=MagicMock(), + cache_hit=False, + ) + + assert hook_received_data is not None, "Guardrail hook must be called" + assert hook_received_data.get("_merged_marker") is True, \ + "Hook must receive guardrail_data (merged), not original captured_data" + assert "model-guardrail" in hook_received_data.get("metadata", {}).get( + "guardrails", [] + ), "Hook data must contain model-level guardrails" + + @pytest.mark.asyncio + async def test_apply_guardrail_path_receives_merged_guardrail_data(self): + """The apply_guardrail path (through UnifiedLLMGuardrails) must also + receive guardrail_data so that the inner should_run_guardrail re-check + inside UnifiedLLMGuardrails sees model-level guardrails. + + This is the specific scenario Greptile flagged: a default_on=False + guardrail configured at the model level would pass the outer gate but + be silently skipped at execution time if captured_data (unmerged) were + passed instead of guardrail_data (merged).""" + import copy + from litellm.types.utils import GenericGuardrailAPIInputs + + unified_received_data = None + + class ModelLevelApplyGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="model-apply-guardrail", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ) -> GenericGuardrailAPIInputs: + return inputs + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + + async def track_async_success(*args, **kwargs): + pass + + mock_logging_obj.async_success_handler = track_async_success + + guardrail = ModelLevelApplyGuardrail() + captured_data = {"model": "gpt-4", "metadata": {}} + + def mock_merge(data, llm_router): + merged = copy.deepcopy(data) + merged["metadata"]["guardrails"] = ["model-apply-guardrail"] + merged["_merged_marker"] = True + return merged + + # Capture what UnifiedLLMGuardrails.async_post_call_success_hook receives + original_unified_hook = None + from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( + UnifiedLLMGuardrails, + ) + original_unified_hook = UnifiedLLMGuardrails.async_post_call_success_hook + + async def tracking_unified_hook(self, user_api_key_dict, data, response): + nonlocal unified_received_data + unified_received_data = data + return response + + with patch("litellm.callbacks", [guardrail]), \ + patch( + "litellm.proxy.utils._check_and_merge_model_level_guardrails", + side_effect=mock_merge, + ), \ + patch.object( + UnifiedLLMGuardrails, + "async_post_call_success_hook", + tracking_unified_hook, + ): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data=captured_data, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="test"), + captured_logging_obj=mock_logging_obj, + assembled_response=MagicMock(), + cache_hit=False, + ) + + assert unified_received_data is not None, \ + "UnifiedLLMGuardrails must be called for apply_guardrail guardrails" + assert unified_received_data.get("_merged_marker") is True, \ + "UnifiedLLMGuardrails must receive guardrail_data (merged), not captured_data" + assert "model-apply-guardrail" in unified_received_data.get( + "metadata", {} + ).get("guardrails", []), \ + "UnifiedLLMGuardrails data must contain model-level guardrails" + + @pytest.mark.asyncio + async def test_multiple_guardrails_all_receive_merged_data(self): + """When multiple guardrails are configured, ALL of them must receive + guardrail_data (merged), not just the first one.""" + import copy + + received_data_per_guardrail = {} + + class TaggedGuardrail(CustomGuardrail): + def __init__(self, name): + super().__init__( + guardrail_name=name, + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + + async def async_post_call_success_hook( + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: Any + ) -> Any: + received_data_per_guardrail[self.guardrail_name] = data + return response + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + + async def track_async_success(*args, **kwargs): + pass + + mock_logging_obj.async_success_handler = track_async_success + + guardrail_a = TaggedGuardrail("guardrail-a") + guardrail_b = TaggedGuardrail("guardrail-b") + + captured_data = {"model": "gpt-4", "metadata": {}} + + def mock_merge(data, llm_router): + merged = copy.deepcopy(data) + merged["metadata"]["guardrails"] = ["guardrail-a", "guardrail-b"] + merged["_merged_marker"] = True + return merged + + with patch("litellm.callbacks", [guardrail_a, guardrail_b]), \ + patch( + "litellm.proxy.utils._check_and_merge_model_level_guardrails", + side_effect=mock_merge, + ): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data=captured_data, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="test"), + captured_logging_obj=mock_logging_obj, + assembled_response=MagicMock(), + cache_hit=False, + ) + + for name in ("guardrail-a", "guardrail-b"): + assert name in received_data_per_guardrail, \ + f"{name} must be called" + assert received_data_per_guardrail[name].get("_merged_marker") is True, \ + f"{name} must receive guardrail_data (merged), not captured_data" From 0057452485d2b12a719e5e66262e22ef676a20e3 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 19 Mar 2026 18:00:49 +0100 Subject: [PATCH 17/73] fix(proxy): guard streaming deferred init with try/finally, fix test imports Wrap _run_deferred_stream_guardrails initialization (UnifiedLLMGuardrails constructor and _check_and_merge_model_level_guardrails) in try/finally so logging always fires even if init throws. Prevents silent logging loss on transient errors. Move fastapi.HTTPException import from module-level to local test-function scope. Add test_logging_fires_even_if_guardrail_init_raises to verify the try/finally guard. --- litellm/proxy/common_request_processing.py | 115 +++++++++--------- .../test_deferred_guardrail_logging.py | 42 ++++++- 2 files changed, 101 insertions(+), 56 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 1db8327482d..0e3e89c5313 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1345,76 +1345,81 @@ class ProxyBaseLLMRequestProcessing: from litellm.proxy.utils import _check_and_merge_model_level_guardrails _response = assembled_response - _unified_guardrail = UnifiedLLMGuardrails() - guardrail_data = _check_and_merge_model_level_guardrails( - data=captured_data, llm_router=_global_llm_router - ) - for cb in litellm.callbacks: - if not isinstance(cb, CustomGuardrail): - continue - if not cb.should_run_guardrail( - data=guardrail_data, - event_type=GuardrailEventHooks.post_call, - ): - continue - try: - guardrail_result = None - if "apply_guardrail" in type(cb).__dict__: - guardrail_data["guardrail_to_apply"] = cb - guardrail_result = ( - await _unified_guardrail.async_post_call_success_hook( + try: + _unified_guardrail = UnifiedLLMGuardrails() + guardrail_data = _check_and_merge_model_level_guardrails( + data=captured_data, llm_router=_global_llm_router + ) + for cb in litellm.callbacks: + if not isinstance(cb, CustomGuardrail): + continue + if not cb.should_run_guardrail( + data=guardrail_data, + event_type=GuardrailEventHooks.post_call, + ): + continue + try: + guardrail_result = None + if "apply_guardrail" in type(cb).__dict__: + guardrail_data["guardrail_to_apply"] = cb + guardrail_result = ( + await _unified_guardrail.async_post_call_success_hook( + user_api_key_dict=captured_user_api_key_dict, + data=guardrail_data, + response=_response, + ) + ) + else: + guardrail_result = await cb.async_post_call_success_hook( user_api_key_dict=captured_user_api_key_dict, data=guardrail_data, response=_response, ) + if guardrail_result is not None: + _response = guardrail_result + except Exception as e: + verbose_proxy_logger.exception( + "Error running post-call guardrail %s on streaming response: %s", + getattr(cb, "guardrail_name", type(cb).__name__), + e, ) - else: - guardrail_result = await cb.async_post_call_success_hook( - user_api_key_dict=captured_user_api_key_dict, - data=guardrail_data, - response=_response, + if isinstance(e, HTTPException) and hasattr( + captured_logging_obj, "model_call_details" + ): + captured_logging_obj.model_call_details.setdefault( + "metadata", {} + )["guardrail_blocked"] = True + except Exception as e: + verbose_proxy_logger.exception( + "Error in deferred streaming guardrail initialization: %s", e, + ) + finally: + try: + asyncio.create_task( + captured_logging_obj.async_success_handler( + _response, + cache_hit=cache_hit, + start_time=None, + end_time=None, ) - if guardrail_result is not None: - _response = guardrail_result + ) except Exception as e: verbose_proxy_logger.exception( - "Error running post-call guardrail %s on streaming response: %s", - getattr(cb, "guardrail_name", type(cb).__name__), - e, + "Error in deferred streaming async logging: %s", e, ) - if isinstance(e, HTTPException) and hasattr( - captured_logging_obj, "model_call_details" - ): - captured_logging_obj.model_call_details.setdefault( - "metadata", {} - )["guardrail_blocked"] = True - try: - asyncio.create_task( - captured_logging_obj.async_success_handler( + try: + executor.submit( + captured_logging_obj.success_handler, _response, cache_hit=cache_hit, start_time=None, end_time=None, ) - ) - except Exception as e: - verbose_proxy_logger.exception( - "Error in deferred streaming async logging: %s", e, - ) - - try: - executor.submit( - captured_logging_obj.success_handler, - _response, - cache_hit=cache_hit, - start_time=None, - end_time=None, - ) - except Exception as e: - verbose_proxy_logger.exception( - "Error in deferred streaming sync logging: %s", e, - ) + except Exception as e: + verbose_proxy_logger.exception( + "Error in deferred streaming sync logging: %s", e, + ) async def _handle_llm_api_exception( self, diff --git a/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py index f5c9eeba1c6..c4d2dce5876 100644 --- a/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py +++ b/tests/test_litellm/proxy/guardrails/test_deferred_guardrail_logging.py @@ -20,7 +20,6 @@ from typing import Any from unittest.mock import MagicMock, patch import pytest -from fastapi import HTTPException sys.path.insert(0, os.path.abspath("../../../..")) @@ -233,6 +232,8 @@ async def test_deferred_logging_fires_on_guardrail_exception(): If post_call_success_hook raises (e.g., guardrail blocks content), the deferred logging closure must still fire (via try/finally). """ + from fastapi import HTTPException # noqa: local import for test isolation + enqueue_called = False def mock_enqueue(): @@ -470,6 +471,8 @@ class TestDeferredStreamingClosure: """If a guardrail raises HTTPException, the production _run_deferred_stream_guardrails must still fire logging and set guardrail_blocked in metadata.""" + from fastapi import HTTPException # noqa: local import for test isolation + logging_called = False class BlockingGuardrail(CustomGuardrail): @@ -902,3 +905,40 @@ class TestDeferredStreamingClosure: f"{name} must be called" assert received_data_per_guardrail[name].get("_merged_marker") is True, \ f"{name} must receive guardrail_data (merged), not captured_data" + + @pytest.mark.asyncio + async def test_logging_fires_even_if_guardrail_init_raises(self): + """If _check_and_merge_model_level_guardrails raises during + initialization, logging must still fire via the try/finally guard. + This prevents silent logging loss on transient init errors.""" + logging_called = False + + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {"metadata": {}} + + async def track_async_success(*args, **kwargs): + nonlocal logging_called + logging_called = True + + mock_logging_obj.async_success_handler = track_async_success + + def exploding_merge(data, llm_router): + raise RuntimeError("Simulated init failure") + + with patch( + "litellm.proxy.utils._check_and_merge_model_level_guardrails", + side_effect=exploding_merge, + ): + await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( + captured_data={"model": "gpt-4", "metadata": {}}, + captured_user_api_key_dict=UserAPIKeyAuth(api_key="test"), + captured_logging_obj=mock_logging_obj, + assembled_response=MagicMock(), + cache_hit=False, + ) + + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert logging_called is True, \ + "Logging must fire even when guardrail initialization raises" From b34231dc95ce686424592d9a063112ef04b103c3 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 19 Mar 2026 18:20:42 +0100 Subject: [PATCH 18/73] refactor(proxy): reuse unified_guardrail singleton, rename shadowing variable Reuse the module-level unified_guardrail singleton from proxy/utils.py in _run_deferred_stream_guardrails instead of creating a new instance per call, matching the pattern used by post_call_success_hook. Rename local variable _has_post_call_guardrails to _post_call_guardrails_active to avoid shadowing the static method name. --- litellm/proxy/common_request_processing.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 0e3e89c5313..1517ee6d9d2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -931,7 +931,7 @@ class ProxyBaseLLMRequestProcessing: # Defer async logging when post-call guardrails are configured so the # StandardLoggingPayload is built after guardrails write to metadata. # Cache the result to avoid scanning litellm.callbacks twice. - _has_post_call_guardrails = self._has_post_call_guardrails() + _post_call_guardrails_active = self._has_post_call_guardrails() # Non-streaming: defer the create_task in wrapper_async so the # SLP is built after guardrails write to metadata. Streaming @@ -943,7 +943,7 @@ class ProxyBaseLLMRequestProcessing: # so _enqueue_deferred_logging is never stored — the finally # block is a no-op. The CSW path handles this correctly via # _on_deferred_stream_complete, which fires its own logging. - if _has_post_call_guardrails and not self._is_streaming_request( + if _post_call_guardrails_active and not self._is_streaming_request( data=self.data, is_streaming_request=is_streaming_request ): logging_obj._defer_async_logging = True # type: ignore @@ -1057,7 +1057,7 @@ class ProxyBaseLLMRequestProcessing: CustomStreamWrapper, ) - if _has_post_call_guardrails and isinstance( + if _post_call_guardrails_active and isinstance( response, CustomStreamWrapper ): # Intentionally a live reference (not a copy) — mirrors @@ -1338,15 +1338,14 @@ class ProxyBaseLLMRequestProcessing: implementation directly rather than reimplementing the closure. """ from litellm.litellm_core_utils.thread_pool_executor import executor - from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( - UnifiedLLMGuardrails, - ) from litellm.proxy.proxy_server import llm_router as _global_llm_router - from litellm.proxy.utils import _check_and_merge_model_level_guardrails + from litellm.proxy.utils import ( + _check_and_merge_model_level_guardrails, + unified_guardrail as _unified_guardrail, + ) _response = assembled_response try: - _unified_guardrail = UnifiedLLMGuardrails() guardrail_data = _check_and_merge_model_level_guardrails( data=captured_data, llm_router=_global_llm_router ) From 6d0763b8ba30a71ba1cd7fec5643f374be85d72c Mon Sep 17 00:00:00 2001 From: Jonathan Barazany Date: Thu, 19 Mar 2026 19:28:05 +0200 Subject: [PATCH 19/73] fix: short-circuit websearch for non-Anthropic providers (github_copilot) For providers like github_copilot that don't natively support web search, Claude Code's search sub-conversations were falling through to the adapter path which strips the web_search tool and has no stream reconversion. Instead of routing search requests through the full LLM pipeline, detect web-search-only requests early (all tools are web_search, simple prompt) and execute the search directly via Tavily/Perplexity, returning a synthetic Anthropic response. No adapter, no backend LLM call needed. Fixes #21733 --- .../websearch_interception/handler.py | 114 ++++++ .../messages/handler.py | 61 ++++ .../test_websearch_short_circuit.py | 332 ++++++++++++++++++ 3 files changed, 507 insertions(+) create mode 100644 tests/test_litellm/integrations/websearch_interception/test_websearch_short_circuit.py diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 2541a0bd7aa..59510dd5d08 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -67,6 +67,120 @@ class WebSearchInterceptionLogger(CustomLogger): self.search_tool_name = search_tool_name self._request_has_websearch = False # Track if current request has web search + async def try_short_circuit_search( + self, + model: str, + messages: List[Dict], + tools: Optional[List[Dict]], + custom_llm_provider: Optional[str], + ) -> Optional[Dict[str, Any]]: + """ + Short-circuit web-search-only requests by executing the search directly. + + Claude Code sends web search as a separate, standalone /v1/messages + request with a simple prompt and only web_search tool(s). For providers + that don't natively support web search (e.g. github_copilot), there is + no need to route this through the backend LLM — we can detect the + pattern, execute the search via Tavily/Perplexity, and return a + synthetic Anthropic response immediately. + + Args: + model: Model name from the request + messages: Messages list from the request + tools: Tools list from the request + custom_llm_provider: Provider name + + Returns: + An AnthropicMessagesResponse dict if short-circuited, or None to + continue normal processing. + """ + if not tools: + return None + + # Check if provider is in enabled list + provider_str = custom_llm_provider or "" + if ( + self.enabled_providers is not None + and provider_str not in self.enabled_providers + ): + return None + + # All tools must be web search tools + if not all(is_web_search_tool(t) for t in tools): + return None + + # Extract search query from the last user message + query = self._extract_search_query(messages) + if not query: + return None + + verbose_logger.debug( + "WebSearchInterception: Short-circuit search detected " + f"(provider={provider_str}, query='{query}')" + ) + + # Execute search + try: + search_result_text = await self._execute_search(query) + except Exception as e: + verbose_logger.error( + f"WebSearchInterception: Short-circuit search failed: {e}" + ) + search_result_text = f"Search failed: {e}" + + # Build synthetic Anthropic response + from uuid import uuid4 + + response: Dict[str, Any] = { + "id": f"msg_{uuid4().hex[:24]}", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": search_result_text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 0, "output_tokens": 0}, + } + + verbose_logger.debug( + "WebSearchInterception: Short-circuit search completed, " + f"returning synthetic response ({len(search_result_text)} chars)" + ) + return response + + @staticmethod + def _extract_search_query(messages: List[Dict]) -> Optional[str]: + """ + Extract the search query from messages. + + Looks at the last user message content for the search query text. + """ + if not messages: + return None + + # Find the last user message + for msg in reversed(messages): + if msg.get("role") != "user": + continue + + content = msg.get("content") + if isinstance(content, str): + return content.strip() or None + + # Handle list-of-blocks content + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "").strip() + if text: + return text + elif isinstance(block, str): + text = block.strip() + if text: + return text + + return None + async def async_pre_call_deployment_hook( self, kwargs: Dict[str, Any], call_type: Optional[Any] ) -> Optional[dict]: diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 1b5f03ec722..52373a9c327 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -114,6 +114,54 @@ async def _execute_pre_request_hooks( return request_kwargs +async def _try_websearch_short_circuit( + model: str, + messages: List[Dict], + tools: Optional[List[Dict]], + custom_llm_provider: Optional[str], + stream: Optional[bool], +) -> Optional[Union[AnthropicMessagesResponse, AsyncIterator]]: + """ + Attempt to short-circuit a web-search-only request. + + Claude Code sends web search as a separate, standalone /v1/messages + request. For providers that don't natively support web search (e.g. + github_copilot), we detect this pattern, execute the search via + Tavily/Perplexity, and return a synthetic Anthropic response — bypassing + the backend LLM entirely. + + Returns the synthetic response if short-circuited, or None to continue + normal processing. + """ + if not litellm.callbacks: + return None + + from litellm.integrations.websearch_interception.handler import ( + WebSearchInterceptionLogger, + ) + + for callback in litellm.callbacks: + if not isinstance(callback, WebSearchInterceptionLogger): + continue + + response = await callback.try_short_circuit_search( + model=model, + messages=messages, + tools=tools, + custom_llm_provider=custom_llm_provider, + ) + if response is not None: + if stream: + from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + + return FakeAnthropicMessagesStreamIterator(response) + return response + + return None + + @client async def anthropic_messages( max_tokens: int, @@ -156,6 +204,19 @@ async def anthropic_messages( # Merge back any other modifications kwargs.update(request_kwargs) + # Short-circuit web-search-only requests: detect the pattern, execute + # search directly via Tavily/Perplexity, and return a synthetic response + # without ever touching the backend LLM or the adapter path. + short_circuit_response = await _try_websearch_short_circuit( + model=model, + messages=messages, + tools=tools, + custom_llm_provider=custom_llm_provider, + stream=stream, + ) + if short_circuit_response is not None: + return short_circuit_response + loop = asyncio.get_event_loop() kwargs["is_async"] = True diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_short_circuit.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_short_circuit.py new file mode 100644 index 00000000000..1129ee98efb --- /dev/null +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_short_circuit.py @@ -0,0 +1,332 @@ +""" +Unit tests for WebSearch Short-Circuit + +Tests the short-circuit path that detects web-search-only /v1/messages requests +and executes the search directly without routing through the backend LLM. +""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from litellm.integrations.websearch_interception.handler import ( + WebSearchInterceptionLogger, +) + + +# --------------------------------------------------------------------------- +# Detection tests +# --------------------------------------------------------------------------- + + +class TestTryShortCircuitSearch: + """Tests for WebSearchInterceptionLogger.try_short_circuit_search""" + + @pytest.mark.asyncio + async def test_short_circuits_single_web_search_tool(self): + """Single web_search_20250305 tool → short-circuit fires""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + with patch.object( + logger, "_execute_search", new_callable=AsyncMock + ) as mock_search: + mock_search.return_value = "Title: Result\nURL: https://example.com\nSnippet: test" + + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "Search for Claude Code releases"}], + tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}], + custom_llm_provider="github_copilot", + ) + + assert result is not None + assert result["type"] == "message" + assert result["role"] == "assistant" + assert result["stop_reason"] == "end_turn" + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert "Result" in result["content"][0]["text"] + mock_search.assert_called_once_with("Search for Claude Code releases") + + @pytest.mark.asyncio + async def test_does_not_short_circuit_mixed_tools(self): + """Mix of web_search and other tools → NOT short-circuited""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "Do something"}], + tools=[ + {"type": "web_search_20250305", "name": "web_search", "max_uses": 8}, + {"name": "Read", "description": "Read a file", "input_schema": {}}, + ], + custom_llm_provider="github_copilot", + ) + + assert result is None + + @pytest.mark.asyncio + async def test_does_not_short_circuit_no_tools(self): + """No tools → NOT short-circuited""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "Hello"}], + tools=None, + custom_llm_provider="github_copilot", + ) + + assert result is None + + @pytest.mark.asyncio + async def test_does_not_short_circuit_empty_tools(self): + """Empty tools list → NOT short-circuited""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "Hello"}], + tools=[], + custom_llm_provider="github_copilot", + ) + + assert result is None + + @pytest.mark.asyncio + async def test_does_not_short_circuit_wrong_provider(self): + """Provider not in enabled_providers → NOT short-circuited""" + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "Search for something"}], + tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}], + custom_llm_provider="github_copilot", + ) + + assert result is None + + @pytest.mark.asyncio + async def test_does_not_short_circuit_no_messages(self): + """Empty messages → NOT short-circuited""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[], + tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}], + custom_llm_provider="github_copilot", + ) + + assert result is None + + @pytest.mark.asyncio + async def test_search_failure_returns_error_text(self): + """Search failure → response with error message, not exception""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + with patch.object( + logger, "_execute_search", new_callable=AsyncMock + ) as mock_search: + mock_search.side_effect = RuntimeError("Tavily API error") + + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "Search for something"}], + tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}], + custom_llm_provider="github_copilot", + ) + + assert result is not None + assert "Search failed" in result["content"][0]["text"] + + @pytest.mark.asyncio + async def test_response_has_valid_structure(self): + """Synthetic response has all required AnthropicMessagesResponse fields""" + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + + with patch.object( + logger, "_execute_search", new_callable=AsyncMock + ) as mock_search: + mock_search.return_value = "search results here" + + result = await logger.try_short_circuit_search( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "Search query"}], + tools=[{"type": "web_search_20250305", "name": "web_search"}], + custom_llm_provider="github_copilot", + ) + + assert result is not None + # Required fields + assert "id" in result + assert result["id"].startswith("msg_") + assert result["type"] == "message" + assert result["role"] == "assistant" + assert result["model"] == "github_copilot/claude-sonnet-4" + assert result["stop_reason"] == "end_turn" + assert result["stop_sequence"] is None + assert "usage" in result + assert "content" in result + + +# --------------------------------------------------------------------------- +# Query extraction tests +# --------------------------------------------------------------------------- + + +class TestExtractSearchQuery: + """Tests for WebSearchInterceptionLogger._extract_search_query""" + + def test_string_content(self): + messages = [{"role": "user", "content": "Search for Python 3.14 features"}] + assert ( + WebSearchInterceptionLogger._extract_search_query(messages) + == "Search for Python 3.14 features" + ) + + def test_list_content_with_text_block(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Perform a web search for latest news"}, + ], + } + ] + assert ( + WebSearchInterceptionLogger._extract_search_query(messages) + == "Perform a web search for latest news" + ) + + def test_takes_last_user_message(self): + messages = [ + {"role": "user", "content": "First message"}, + {"role": "assistant", "content": "Response"}, + {"role": "user", "content": "Second message"}, + ] + assert ( + WebSearchInterceptionLogger._extract_search_query(messages) == "Second message" + ) + + def test_empty_messages(self): + assert WebSearchInterceptionLogger._extract_search_query([]) is None + + def test_no_user_messages(self): + messages = [{"role": "assistant", "content": "Hello"}] + assert WebSearchInterceptionLogger._extract_search_query(messages) is None + + def test_empty_content(self): + messages = [{"role": "user", "content": ""}] + assert WebSearchInterceptionLogger._extract_search_query(messages) is None + + +# --------------------------------------------------------------------------- +# Integration with entry point +# --------------------------------------------------------------------------- + + +class TestShortCircuitEntryPoint: + """Tests for _try_websearch_short_circuit in the /v1/messages handler""" + + @pytest.mark.asyncio + async def test_returns_none_when_no_callbacks(self): + """No callbacks configured → returns None""" + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _try_websearch_short_circuit, + ) + + with patch("litellm.callbacks", []): + result = await _try_websearch_short_circuit( + model="test", + messages=[], + tools=[], + custom_llm_provider="github_copilot", + stream=False, + ) + assert result is None + + @pytest.mark.asyncio + async def test_returns_dict_when_not_streaming(self): + """Non-streaming short-circuit → returns dict""" + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _try_websearch_short_circuit, + ) + + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + with patch.object( + logger, "_execute_search", new_callable=AsyncMock + ) as mock_search: + mock_search.return_value = "results" + with patch("litellm.callbacks", [logger]): + result = await _try_websearch_short_circuit( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "search query"}], + tools=[{"type": "web_search_20250305", "name": "web_search"}], + custom_llm_provider="github_copilot", + stream=False, + ) + + assert isinstance(result, dict) + assert result["content"][0]["text"] == "results" + + @pytest.mark.asyncio + async def test_returns_stream_iterator_when_streaming(self): + """Streaming short-circuit → returns FakeAnthropicMessagesStreamIterator""" + from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _try_websearch_short_circuit, + ) + + logger = WebSearchInterceptionLogger(enabled_providers=["github_copilot"]) + with patch.object( + logger, "_execute_search", new_callable=AsyncMock + ) as mock_search: + mock_search.return_value = "streaming results" + with patch("litellm.callbacks", [logger]): + result = await _try_websearch_short_circuit( + model="github_copilot/claude-sonnet-4", + messages=[{"role": "user", "content": "search query"}], + tools=[{"type": "web_search_20250305", "name": "web_search"}], + custom_llm_provider="github_copilot", + stream=True, + ) + + assert isinstance(result, FakeAnthropicMessagesStreamIterator) + + # Verify stream produces valid SSE events + chunks = [] + async for chunk in result: + chunks.append(chunk) + + assert len(chunks) > 0 + # First chunk should be message_start + assert b"event: message_start" in chunks[0] + # Last chunk should be message_stop + assert b"event: message_stop" in chunks[-1] + # Should contain the search results text + all_data = b"".join(chunks) + assert b"streaming results" in all_data + + @pytest.mark.asyncio + async def test_skips_non_websearch_callbacks(self): + """Non-WebSearchInterceptionLogger callbacks are ignored""" + from unittest.mock import MagicMock + + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _try_websearch_short_circuit, + ) + + other_callback = MagicMock() + with patch("litellm.callbacks", [other_callback]): + result = await _try_websearch_short_circuit( + model="test", + messages=[{"role": "user", "content": "search"}], + tools=[{"type": "web_search_20250305", "name": "web_search"}], + custom_llm_provider="github_copilot", + stream=False, + ) + assert result is None From 0b07f628ffb2d4fe8d2c271fe578a9a2796382ec Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 10:30:03 -0700 Subject: [PATCH 20/73] [Test] UI: Add vitest coverage for 10 previously untested components Add unit tests for: - SimpleToolCallBlock, SimpleMessageBlock, CollapsibleMessage, HistoryTree (log details drawer) - OnboardingForm (onboarding flow) - TeamsHeaderTabs, TeamsTable (teams page) - transform_key_info, filter_helpers (key/team helpers) - queryKeysFactory (query key generation utility) 47 new tests covering conditional rendering, user interactions, data transformation, and error handling. Co-Authored-By: Claude Opus 4.6 --- .../hooks/common/queryKeysFactory.test.ts | 34 +++++ .../teams/components/TeamsHeaderTabs.test.tsx | 54 ++++++++ .../components/TeamsTable/TeamsTable.test.tsx | 129 ++++++++++++++++++ .../app/onboarding/OnboardingForm.test.tsx | 95 +++++++++++++ .../key_team_helpers/filter_helpers.test.ts | 90 ++++++++++++ .../transform_key_info.test.ts | 62 +++++++++ .../CollapsibleMessage.test.tsx | 54 ++++++++ .../LogDetailsDrawer/HistoryTree.test.tsx | 50 +++++++ .../SimpleMessageBlock.test.tsx | 55 ++++++++ .../SimpleToolCallBlock.test.tsx | 51 +++++++ 10 files changed, 674 insertions(+) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/common/queryKeysFactory.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsTable/TeamsTable.test.tsx create mode 100644 ui/litellm-dashboard/src/app/onboarding/OnboardingForm.test.tsx create mode 100644 ui/litellm-dashboard/src/components/key_team_helpers/filter_helpers.test.ts create mode 100644 ui/litellm-dashboard/src/components/key_team_helpers/transform_key_info.test.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/CollapsibleMessage.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/HistoryTree.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleMessageBlock.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/SimpleToolCallBlock.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/queryKeysFactory.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/queryKeysFactory.test.ts new file mode 100644 index 00000000000..39afd044097 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/common/queryKeysFactory.test.ts @@ -0,0 +1,34 @@ +import { describe, it, expect } from "vitest"; +import { createQueryKeys } from "./queryKeysFactory"; + +describe("createQueryKeys", () => { + const keys = createQueryKeys("books"); + + it("should return the resource name as the base key", () => { + expect(keys.all).toEqual(["books"]); + }); + + it("should generate a lists key", () => { + expect(keys.lists()).toEqual(["books", "list"]); + }); + + it("should generate a list key with params", () => { + expect(keys.list({ page: 1, limit: 10 })).toEqual([ + "books", + "list", + { params: { page: 1, limit: 10 } }, + ]); + }); + + it("should generate a list key with undefined params when none provided", () => { + expect(keys.list()).toEqual(["books", "list", { params: undefined }]); + }); + + it("should generate a details key", () => { + expect(keys.details()).toEqual(["books", "detail"]); + }); + + it("should generate a detail key for a specific ID", () => { + expect(keys.detail("123")).toEqual(["books", "detail", "123"]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx new file mode 100644 index 00000000000..50a7f10f047 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsHeaderTabs.test.tsx @@ -0,0 +1,54 @@ +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; +import TeamsHeaderTabs from "./TeamsHeaderTabs"; + +vi.mock("@tremor/react", () => ({ + TabGroup: ({ children, ...props }: any) =>
{children}
, + TabList: ({ children, ...props }: any) =>
{children}
, + Tab: ({ children, ...props }: any) => , + TabPanels: ({ children, ...props }: any) =>
{children}
, + Text: ({ children, ...props }: any) => {children}, + Icon: ({ onClick, ...props }: any) => + + ); + } + + return ( + + columns={teamColumns} + dataSource={displayTeams} + rowKey="team_id" + pagination={false} + onChange={handleTableSort} + locale={{ + emptyText: ( +
+ +
+ No teams yet +
+
+ + Create your first team to organize members and manage access to models. + +
+ {canCreateOrManageTeams(userRole, userID, organizations) && ( + + )} +
+ ), + }} + scroll={{ x: 1000 }} + size="middle" + /> + ); + }; + + const tabItems = [ + { + key: "your-teams", + label: "Your Teams", + children: ( + <> + + + + } + suffix={isSearching ? : null} + placeholder="Search teams by name..." + onChange={(e) => handleSearchChange(e.target.value)} + allowClear + style={{ maxWidth: 400 }} + /> + handleFilterChange("organization_id", value || "")} + loading={isLoading} + /> + + { + setCurrentPage(page); + setPageSize(size); + fetchTeamsV2({ page, size }); + }} + size="small" + showTotal={(total) => `${total} teams`} + showSizeChanger + pageSizeOptions={["10", "20", "50"]} + /> + + + {renderTeamsContent()} + + + + + ), + }, + { + key: "available-teams", + label: "Available Teams", + children: , + }, + ...(isProxyAdminRole(userRole || "") + ? [ + { + key: "default-settings", + label: "Default Team Settings", + children: , + }, + ] + : []), + ]; + return ( -
- - - {canCreateOrManageTeams(userRole, userID, organizations) && ( - - )} - {selectedTeamId ? ( - { - setTeams((teams) => { - if (teams == null) { - return teams; - } - const updated = teams.map((team) => { - if (data.team_id === team.team_id) { - return updateExistingKeys(team, data); - } - return team; - }); - // Minimal fix: refresh the full team list after an update - if (accessToken) { - fetchTeams(accessToken, userID, userRole, currentOrg, setTeams); - } - return updated; - }); - }} - onClose={() => { - setSelectedTeamId(null); - setEditTeam(false); - }} - accessToken={accessToken} - is_team_admin={is_team_admin(teams?.find((team) => team.team_id === selectedTeamId))} - is_proxy_admin={userRole == "Admin"} - userModels={userModels} - editTeam={editTeam} - premiumUser={premiumUser} - /> - ) : ( - - -
- Your Teams - Available Teams - {isProxyAdminRole(userRole || "") && Default Team Settings} -
-
- {lastRefreshed && Last Refreshed: {lastRefreshed}} - -
- - - - - Click on “Team ID” to view team details and manage team members. - - - - -
-
- {/* Search and Filter Controls */} -
- {/* Team Alias Search */} - handleFilterChange("team_alias", value)} - icon={Search} - /> + + {selectedTeamId ? ( + { + setTeams((teams) => { + if (teams == null) { + return teams; + } + return teams.map((team) => { + if (data.team_id === team.team_id) { + return updateExistingKeys(team, data); + } + return team; + }); + }); + fetchTeamsV2(); + }} + onClose={() => { + setSelectedTeamId(null); + setEditTeam(false); + }} + accessToken={accessToken} + is_team_admin={is_team_admin(teams?.find((team) => team.team_id === selectedTeamId))} + is_proxy_admin={userRole == "Admin"} + userModels={userModels} + editTeam={editTeam} + premiumUser={premiumUser} + /> + ) : ( + <> + + + + <TeamOutlined style={{ marginRight: 8 }} /> + Teams + + + Manage teams, members, and their access to models and budgets + + + {canCreateOrManageTeams(userRole, userID, organizations) && ( + + )} + - {/* Filter Button */} - setShowFilters(!showFilters)} - active={showFilters} - hasActiveFilters={!!(filters.team_id || filters.team_alias || filters.organization_id)} - /> + + + )} - {/* Reset Filters Button */} - -
- - {/* Additional Filters */} - {showFilters && ( -
- {/* Team ID Search */} - handleFilterChange("team_id", value)} - icon={User} - /> - - {/* Organization Dropdown */} -
- -
-
- )} -
-
- - - - Team Name - Team ID - Created - Spend (USD) - Budget (USD) - Models - Organization - Info - Actions - - - - - {teams && teams.length > 0 ? ( - teams - .filter((team) => { - if (!currentOrg) return true; - return team.organization_id === currentOrg.organization_id; - }) - .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()) - .map((team: any) => ( - - - {team["team_alias"]} - - -
- - - -
- - - {team.created_at ? new Date(team.created_at).toLocaleDateString() : "N/A"} - - - {formatNumberWithCommas(team["spend"], 4)} - - - {team["max_budget"] !== null && team["max_budget"] !== undefined - ? team["max_budget"] - : "No limit"} - - 3 ? "px-0" : ""} - > -
- {Array.isArray(team.models) ? ( -
- {team.models.length === 0 ? ( - - All Proxy Models - - ) : ( - <> -
- {team.models.length > 3 && ( -
- { - setExpandedAccordions((prev) => ({ - ...prev, - [team.team_id]: !prev[team.team_id], - })); - }} - /> -
- )} -
- {team.models.slice(0, 3).map((model: string, index: number) => - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - - - ), - )} - {team.models.length > 3 && !expandedAccordions[team.team_id] && ( - - - +{team.models.length - 3}{" "} - {team.models.length - 3 === 1 ? "more model" : "more models"} - - - )} - {expandedAccordions[team.team_id] && ( -
- {team.models.slice(3).map((model: string, index: number) => - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - - - ), - )} -
- )} -
-
- - )} -
- ) : null} -
- - - - {getOrganizationAlias(team.organization_id, organizationsData || organizations)} - - - - {perTeamInfo && - team.team_id && - perTeamInfo[team.team_id] && - perTeamInfo[team.team_id].keys && - perTeamInfo[team.team_id].keys.length}{" "} - Keys - - - {perTeamInfo && - team.team_id && - perTeamInfo[team.team_id] && - perTeamInfo[team.team_id].team_info && - perTeamInfo[team.team_id].team_info.members_with_roles && - perTeamInfo[team.team_id].team_info.members_with_roles.length}{" "} - Members - - - - {userRole == "Admin" ? ( - <> - { - setSelectedTeamId(team.team_id); - setEditTeam(true); - }} - dataTestId="edit-team-button" - tooltipText="Edit team" - /> - handleDelete(team)} - dataTestId="delete-team-button" - tooltipText="Delete team" - /> - - ) : null} - - - )) - ) : ( - - -
- No teams found - Adjust your filters or create a new team -
- - - )} - -
- - - - - - - - - {isProxyAdminRole(userRole || "") && ( - - - - )} - - - )} - {canCreateOrManageTeams(userRole, userID, organizations) && ( + {canCreateOrManageTeams(userRole, userID, organizations) && ( = ({ : "" } > - = ({ optionFilterProp="children" > {adminOrgs?.map((org) => ( - + {org.organization_alias}{" "} ({org.organization_id}) - + ))} - + {/* Show message when org admin needs to select organization */} {isOrgAdmin && !isSingleOrg && adminOrgs.length > 1 && (
- + Please select an organization to create a team for. You can only create teams within organizations where you are an admin. @@ -1190,11 +1211,11 @@ const Teams: React.FC = ({ - - daily - weekly - monthly - + @@ -1313,7 +1334,7 @@ const Teams: React.FC = ({ className="mt-8" help="Select existing guardrails or enter new ones" > - = ({ className="mt-8" help="Select existing policies or enter new ones" > - = ({
- + Create custom aliases for models that can be used by team members in API calls. This allows you to create shortcuts for specific models. @@ -1548,14 +1569,12 @@ const Teams: React.FC = ({
- Create Team +
)} - - -
+ ); }; diff --git a/ui/litellm-dashboard/src/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton.tsx b/ui/litellm-dashboard/src/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton.tsx index 488913a734a..2f146aab723 100644 --- a/ui/litellm-dashboard/src/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton.tsx +++ b/ui/litellm-dashboard/src/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton.tsx @@ -6,6 +6,7 @@ import { ChevronUpIcon, ChevronDownIcon, ExternalLinkIcon, + ClipboardCopyIcon, } from "@heroicons/react/outline"; import { Tooltip } from "antd"; import BaseActionButton from "../BaseActionButton"; @@ -32,6 +33,7 @@ export const TableIconActionButtonMap: Record void; disabled?: boolean; loading?: boolean; + style?: React.CSSProperties; } const OrganizationDropdown: React.FC = ({ @@ -16,16 +19,18 @@ const OrganizationDropdown: React.FC = ({ onChange, disabled, loading, + style, }) => { return ( diff --git a/ui/litellm-dashboard/src/components/ui/AntDLoadingSpinner.tsx b/ui/litellm-dashboard/src/components/ui/AntDLoadingSpinner.tsx new file mode 100644 index 00000000000..9e90f77584a --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/AntDLoadingSpinner.tsx @@ -0,0 +1,12 @@ +import { Spin } from "antd"; +import { LoadingOutlined } from "@ant-design/icons"; + +interface AntDLoadingSpinnerProps { + size?: "small" | "default" | "large"; + fontSize?: number; +} + +export function AntDLoadingSpinner({ size, fontSize }: AntDLoadingSpinnerProps) { + const indicator = ; + return ; +} From ad43a35d762d62cd2cb9a18a0f1fbdef4f3fb67c Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Thu, 19 Mar 2026 22:50:19 -0700 Subject: [PATCH 42/73] feat: add control plane for multi-proxy worker management MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a control plane capability that enables a central admin instance to manage multiple regional worker proxies from a single UI. Backend: - Worker registry loaded from YAML config (worker_id, name, url) - /.well-known/litellm-ui-config exposes is_control_plane and workers list - /v3/login + /v3/login/exchange: opaque code exchange for cross-origin username/password auth (JWT never in URL/logs, single-use 60s TTL) - SSO cookie handoff with return_to → opaque code → exchange - _validate_return_to: full origin validation (scheme+hostname+port) - Startup warning when control_plane_url set without Redis - Both /v3 endpoints gated behind control_plane_url config Frontend: - Worker selector dropdown on login page (gated behind is_control_plane) - Cross-origin SSO code exchange handling on callback - switchToWorkerUrl: localStorage-persisted worker URL for API calls - useWorker hook: shared worker state management - WorkerDropdown in navbar for switching workers - Logout/switch clears worker state from localStorage Tests: - 7 tests for /v3/login + /v3/login/exchange - 10 tests for _validate_return_to - 2 tests for control plane discovery endpoint --- .../ui_discovery_endpoints.py | 7 + litellm/proxy/management_endpoints/ui_sso.py | 85 ++++++- litellm/proxy/proxy_server.py | 187 +++++++++++++++- .../types/proxy/control_plane_endpoints.py | 14 ++ .../ui_discovery_endpoints.py | 6 +- .../test_ui_discovery_endpoints.py | 57 ++++- .../proxy/management_endpoints/test_ui_sso.py | 98 +++++++- tests/test_litellm/proxy/test_proxy_server.py | 211 ++++++++++++++++++ .../app/(dashboard)/hooks/login/useLogin.ts | 4 +- .../hooks/uiConfig/useUIConfig.test.ts | 4 + .../src/app/login/LoginPage.test.tsx | 15 +- .../src/app/login/LoginPage.tsx | 125 +++++++++-- .../Navbar/WorkerDropdown/WorkerDropdown.tsx | 38 ++++ .../src/components/navbar.tsx | 13 ++ .../src/components/networking.tsx | 126 ++++++++++- ui/litellm-dashboard/src/hooks/useWorker.ts | 65 ++++++ 16 files changed, 1025 insertions(+), 30 deletions(-) create mode 100644 litellm/types/proxy/control_plane_endpoints.py create mode 100644 ui/litellm-dashboard/src/components/Navbar/WorkerDropdown/WorkerDropdown.tsx create mode 100644 ui/litellm-dashboard/src/hooks/useWorker.ts diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 2a38ceffba5..233df5c6c57 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -27,10 +27,17 @@ async def get_ui_config(): admin_ui_disabled = os.getenv("DISABLE_ADMIN_UI", "false").lower() == "true" sso_configured = _has_user_setup_sso() + + from litellm.proxy.proxy_server import proxy_config + + is_control_plane = len(proxy_config.worker_registry) > 0 + return UiDiscoveryEndpoints( server_root_path=get_server_root_path(), proxy_base_url=get_proxy_base_url(), auto_redirect_to_sso=sso_configured and auto_redirect_ui_login_to_sso, admin_ui_disabled=admin_ui_disabled, sso_configured=sso_configured, + is_control_plane=is_control_plane, + workers=proxy_config.worker_registry if is_control_plane else [], ) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index daf1d6f1316..8634a8bb33d 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -16,6 +16,7 @@ import os import secrets from copy import deepcopy from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast +from urllib.parse import urlencode, urlparse if TYPE_CHECKING: import httpx @@ -301,6 +302,7 @@ async def google_login( source: Optional[str] = None, key: Optional[str] = None, existing_key: Optional[str] = None, + return_to: Optional[str] = None, ): # noqa: PLR0915 """ Create Proxy API Keys using Google Workspace SSO. Requires setting PROXY_BASE_URL in .env @@ -394,13 +396,23 @@ async def google_login( is True ): verbose_proxy_logger.info(f"Redirecting to SSO login for {redirect_url}") - return await SSOAuthenticationHandler.get_sso_login_redirect( + sso_redirect = await SSOAuthenticationHandler.get_sso_login_redirect( redirect_url=redirect_url, microsoft_client_id=microsoft_client_id, google_client_id=google_client_id, generic_client_id=generic_client_id, state=cli_state, ) + if return_to is not None and sso_redirect is not None: + SSOAuthenticationHandler._validate_return_to(return_to) + sso_redirect.set_cookie( + key="litellm_cp_return_to", + value=return_to, + max_age=600, + httponly=True, + samesite="lax", + ) + return sso_redirect elif ui_username is not None: # No Google, Microsoft SSO # Use UI Credentials set in .env @@ -1312,12 +1324,17 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: request=request, key=key_id, existing_key=existing_key, result=result ) + # Control-plane cross-origin: read return_to from cookie. + # Starlette's cookie_parser already handles RFC 2109 unquoting. + cp_return_to: Optional[str] = request.cookies.get("litellm_cp_return_to") + return await SSOAuthenticationHandler.get_redirect_response_from_openid( result=result, request=request, received_response=received_response, generic_client_id=generic_client_id, ui_access_mode=ui_access_mode, + return_to=cp_return_to, ) @@ -1760,6 +1777,38 @@ class SSOAuthenticationHandler: Handler for SSO Authentication across all SSO providers """ + @staticmethod + def _validate_return_to(return_to: str) -> None: + """ + Validate that return_to matches the configured control_plane_url origin. + + Raises HTTPException(400) if: + - control_plane_url is not configured in general_settings + - return_to origin does not match control_plane_url origin + """ + from litellm.proxy.proxy_server import general_settings + + control_plane_url = general_settings.get("control_plane_url") + if control_plane_url is None: + raise HTTPException( + status_code=400, + detail="return_to is not allowed: control_plane_url is not configured", + ) + + def _origin(url: str) -> tuple: + parsed = urlparse(url) + scheme = (parsed.scheme or "").lower() + hostname = (parsed.hostname or "").lower() + default_port = 443 if scheme == "https" else 80 + port = parsed.port if parsed.port is not None else default_port + return (scheme, hostname, port) + + if _origin(return_to) != _origin(control_plane_url): + raise HTTPException( + status_code=400, + detail="return_to does not match the configured control_plane_url", + ) + @staticmethod async def get_sso_login_redirect( redirect_url: str, @@ -2358,6 +2407,7 @@ class SSOAuthenticationHandler: received_response: Optional[dict] = None, generic_client_id: Optional[str] = None, ui_access_mode: Optional[Dict] = None, + return_to: Optional[str] = None, ) -> RedirectResponse: import jwt @@ -2367,6 +2417,7 @@ class SSOAuthenticationHandler: master_key, premium_user, proxy_logging_obj, + redis_usage_cache, user_api_key_cache, user_custom_sso, ) @@ -2534,6 +2585,38 @@ class SSOAuthenticationHandler: master_key or "", algorithm="HS256", ) + + # Control-plane cross-origin: store JWT behind a single-use opaque + # code (60s TTL) so the token never appears in browser history / logs. + # The control plane redeems it via POST /v3/login/exchange. + if return_to is not None: + SSOAuthenticationHandler._validate_return_to(return_to) + + code = secrets.token_urlsafe(32) + cache_key = f"login_code:{code}" + cache_value = {"token": jwt_token, "redirect_url": return_to} + if redis_usage_cache is not None: + await redis_usage_cache.async_set_cache( + key=cache_key, value=cache_value, ttl=60 + ) + else: + await user_api_key_cache.async_set_cache( + key=cache_key, value=cache_value, ttl=60 + ) + + separator = "&" if "?" in return_to else "?" + redirect_url = ( + return_to + + separator + + urlencode({"login": "success", "code": code}) + ) + verbose_proxy_logger.info( + "Cross-origin SSO: redirecting to control plane with login code" + ) + redirect_response = RedirectResponse(url=redirect_url, status_code=303) + redirect_response.delete_cookie("litellm_cp_return_to") + return redirect_response + if user_id is not None and isinstance(user_id, str): litellm_dashboard_ui += "?login=success" verbose_proxy_logger.info(f"Redirecting to {litellm_dashboard_ui}") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9c29927c5cb..e982c934aa6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -541,6 +541,7 @@ from litellm.types.llms.anthropic import ( AnthropicResponseUsageBlock, ) from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, ) @@ -1546,6 +1547,7 @@ user_custom_key_generate = None # Sentinel: prevents PKCE-no-Redis advisory from re-logging on config hot-reload. # Tests that need to reset it can patch 'litellm.proxy.proxy_server._pkce_no_redis_warning_emitted'. _pkce_no_redis_warning_emitted: bool = False +_cp_no_redis_warning_emitted: bool = False user_custom_sso = None user_custom_ui_sso_sign_in_handler = None use_background_health_checks = None @@ -2295,6 +2297,7 @@ class ProxyConfig: self.config: Dict[str, Any] = {} self._last_semantic_filter_config: Optional[Dict[str, Any]] = None self._last_hashicorp_vault_config: Optional[Dict[str, Any]] = None + self.worker_registry: List["WorkerRegistryEntry"] = [] def is_yaml(self, config_file_path: str) -> bool: if not os.path.isfile(config_file_path): @@ -3095,6 +3098,21 @@ class ProxyConfig: "Set PKCE_STRICT_CACHE_MISS=true to fail fast with a 401 on cache misses " "instead of continuing without a code_verifier." ) + + ### CONTROL PLANE CODE-EXCHANGE PREREQUISITE CHECK ### + cp_url = general_settings.get("control_plane_url") + if cp_url and redis_usage_cache is None: + global _cp_no_redis_warning_emitted + if not _cp_no_redis_warning_emitted: + _cp_no_redis_warning_emitted = True + verbose_proxy_logger.warning( + "control_plane_url is configured but Redis is not configured for LiteLLM caching. " + "Login codes (SSO and /v3/login) will not be shared across instances — " + "the /v3/login/exchange call may land on a different pod and fail with 401. " + "Configure Redis via the 'cache' section in your proxy config, " + "or ensure sticky sessions for single-instance deployments." + ) + ### STORE MODEL IN DB ### feature flag for `/model/new` store_model_in_db = general_settings.get("store_model_in_db", False) if store_model_in_db is None: @@ -3385,7 +3403,15 @@ class ProxyConfig: litellm.vector_store_registry.load_vector_stores_from_config( vector_store_registry_config ) - pass + + ## WORKER REGISTRY (Control Plane) + worker_registry_config = config.get("worker_registry", None) + if worker_registry_config: + self.worker_registry = [ + WorkerRegistryEntry(**e) for e in worker_registry_config + ] + else: + self.worker_registry = [] async def _init_policy_engine( self, @@ -11095,6 +11121,165 @@ async def login_v2(request: Request): # noqa: PLR0915 ) +@router.post( + "/v3/login", include_in_schema=False +) # control-plane login — always returns token in body for cross-origin use +async def login_v3(request: Request): # noqa: PLR0915 + global premium_user, general_settings, master_key + from litellm.proxy.auth.login_utils import authenticate_user, create_ui_token_object + from litellm.proxy.utils import get_custom_url + + try: + if not general_settings.get("control_plane_url"): + raise ProxyException( + message="/v3/login is only available on workers with control_plane_url configured", + type=ProxyErrorTypes.not_found_error, + param="control_plane_url", + code=status.HTTP_404_NOT_FOUND, + ) + + body = await request.json() + username = str(body.get("username")) + password = str(body.get("password")) + + login_result = await authenticate_user( + username=username, + password=password, + master_key=master_key, + prisma_client=prisma_client, + ) + + returned_ui_token_object = create_ui_token_object( + login_result=login_result, + general_settings=general_settings, + premium_user=premium_user, + ) + + import jwt + + jwt_token = jwt.encode( + cast(dict, returned_ui_token_object), + cast(str, master_key), + algorithm="HS256", + ) + + litellm_dashboard_ui = get_custom_url(str(request.base_url)) + if litellm_dashboard_ui.endswith("/"): + litellm_dashboard_ui += "ui/" + else: + litellm_dashboard_ui += "/ui/" + litellm_dashboard_ui += "?login=success" + + # Store JWT behind a single-use opaque code (60s TTL) + code = secrets.token_urlsafe(32) + cache_key = f"login_code:{code}" + cache_value = {"token": jwt_token, "redirect_url": litellm_dashboard_ui} + if redis_usage_cache is not None: + await redis_usage_cache.async_set_cache( + key=cache_key, value=cache_value, ttl=60 + ) + else: + await user_api_key_cache.async_set_cache( + key=cache_key, value=cache_value, ttl=60 + ) + + return JSONResponse( + content={"code": code, "expires_in": 60}, + status_code=status.HTTP_200_OK, + ) + except Exception as e: + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.login_v3(): Exception occurred - {}".format( + str(e) + ) + ) + if isinstance(e, ProxyException): + raise e + elif isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", str(e)), + type=ProxyErrorTypes.auth_error, + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + else: + error_msg = f"{str(e)}" + raise ProxyException( + message=error_msg, + type=ProxyErrorTypes.auth_error, + param="None", + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + +@router.post( + "/v3/login/exchange", include_in_schema=False +) # exchange single-use opaque code for JWT +async def login_v3_exchange(request: Request): + try: + if not general_settings.get("control_plane_url"): + raise ProxyException( + message="/v3/login/exchange is only available on workers with control_plane_url configured", + type=ProxyErrorTypes.not_found_error, + param="control_plane_url", + code=status.HTTP_404_NOT_FOUND, + ) + + body = await request.json() + code = body.get("code") + if not code: + raise ProxyException( + message="Missing 'code' parameter", + type=ProxyErrorTypes.auth_error, + param="code", + code=status.HTTP_400_BAD_REQUEST, + ) + + cache_key = f"login_code:{code}" + if redis_usage_cache is not None: + cached_data = await redis_usage_cache.async_get_cache(key=cache_key) + else: + cached_data = await user_api_key_cache.async_get_cache(key=cache_key) + + if not cached_data or not isinstance(cached_data, dict): + raise ProxyException( + message="Invalid or expired login code", + type=ProxyErrorTypes.auth_error, + param="code", + code=status.HTTP_401_UNAUTHORIZED, + ) + + # Single-use: delete immediately + if redis_usage_cache is not None: + await redis_usage_cache.async_delete_cache(key=cache_key) + else: + await user_api_key_cache.async_delete_cache(key=cache_key) + + json_response = JSONResponse( + content={ + "token": cached_data["token"], + "redirect_url": cached_data["redirect_url"], + }, + status_code=status.HTTP_200_OK, + ) + json_response.set_cookie(key="token", value=cached_data["token"]) + return json_response + except ProxyException: + raise + except Exception as e: + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.login_v3_exchange(): Exception occurred - {}".format( + str(e) + ) + ) + raise ProxyException( + message=str(e), + type=ProxyErrorTypes.auth_error, + param="None", + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + @app.get("/onboarding/get_token", include_in_schema=False) async def onboarding(invite_link: str, request: Request): """ diff --git a/litellm/types/proxy/control_plane_endpoints.py b/litellm/types/proxy/control_plane_endpoints.py new file mode 100644 index 00000000000..8bf4c44b205 --- /dev/null +++ b/litellm/types/proxy/control_plane_endpoints.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel, field_validator + + +class WorkerRegistryEntry(BaseModel): + worker_id: str + name: str + url: str + + @field_validator("url") + @classmethod + def url_must_be_http(cls, v: str) -> str: + if not v.startswith(("http://", "https://")): + raise ValueError("Worker URL must start with http:// or https://") + return v diff --git a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py index 4a4cdaa2bae..46cd3f49f1a 100644 --- a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -1,7 +1,9 @@ -from typing import Optional +from typing import List, Optional from pydantic import BaseModel +from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry + class UiDiscoveryEndpoints(BaseModel): server_root_path: str @@ -9,3 +11,5 @@ class UiDiscoveryEndpoints(BaseModel): auto_redirect_to_sso: bool admin_ui_disabled: bool sso_configured: bool + is_control_plane: bool = False + workers: List[WorkerRegistryEntry] = [] diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py index f15960a6072..54a127f435b 100644 --- a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py +++ b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py @@ -1,6 +1,6 @@ import os import sys -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from fastapi import FastAPI @@ -11,6 +11,7 @@ sys.path.insert( ) from litellm.proxy.discovery_endpoints.ui_discovery_endpoints import router +from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry def test_ui_discovery_endpoints_with_defaults(): @@ -245,9 +246,9 @@ def test_ui_discovery_endpoints_with_admin_ui_enabled(): patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), \ patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), \ patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False): - + response = client.get("/.well-known/litellm-ui-config") - + assert response.status_code == 200 data = response.json() assert data["server_root_path"] == "/" @@ -256,3 +257,53 @@ def test_ui_discovery_endpoints_with_admin_ui_enabled(): assert data["admin_ui_disabled"] is False assert data["sso_configured"] is False + +def test_ui_discovery_endpoints_is_control_plane_true_when_workers_configured(): + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + mock_config = MagicMock() + mock_config.worker_registry = [ + WorkerRegistryEntry( + worker_id="team-a", name="Team A", url="https://worker-1:4001" + ), + ] + + with patch("litellm.proxy.utils.get_server_root_path", return_value="/"), \ + patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), \ + patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), \ + patch("litellm.proxy.proxy_server.proxy_config", mock_config), \ + patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False): + + response = client.get("/.well-known/litellm-ui-config") + + assert response.status_code == 200 + data = response.json() + assert data["is_control_plane"] is True + assert len(data["workers"]) == 1 + assert data["workers"][0]["worker_id"] == "team-a" + assert data["workers"][0]["name"] == "Team A" + assert data["workers"][0]["url"] == "https://worker-1:4001" + + +def test_ui_discovery_endpoints_is_control_plane_false_when_no_workers(): + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + mock_config = MagicMock() + mock_config.worker_registry = [] + + with patch("litellm.proxy.utils.get_server_root_path", return_value="/"), \ + patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), \ + patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), \ + patch("litellm.proxy.proxy_server.proxy_config", mock_config), \ + patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False): + + response = client.get("/.well-known/litellm-ui-config") + + assert response.status_code == 200 + data = response.json() + assert data["is_control_plane"] is False + assert data["workers"] == [] diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index d43b2c4ba05..fc9c37b7f84 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -6,7 +6,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest -from fastapi import Request +from fastapi import HTTPException, Request from litellm._uuid import uuid @@ -5160,3 +5160,99 @@ def test_generic_response_convertor_extra_attributes_missing_field(monkeypatch): assert result.extra_fields["missing_field"] is None assert result.extra_fields["another_missing"] is None + +class TestValidateReturnTo: + """Tests for SSOAuthenticationHandler._validate_return_to""" + + def test_rejects_when_no_control_plane_url_configured(self, monkeypatch): + """return_to should be rejected if control_plane_url is not in general_settings.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", {} + ) + with pytest.raises(HTTPException) as exc_info: + SSOAuthenticationHandler._validate_return_to("https://cp.example.com/ui") + assert exc_info.value.status_code == 400 + assert "not configured" in exc_info.value.detail + + def test_allows_matching_origin(self, monkeypatch): + """return_to matching the configured control_plane_url origin should pass.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + # Should not raise + SSOAuthenticationHandler._validate_return_to("https://cp.example.com/ui?page=models") + + def test_allows_matching_origin_with_trailing_slash(self, monkeypatch): + """Trailing slash on control_plane_url should not affect origin comparison.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com/"}, + ) + SSOAuthenticationHandler._validate_return_to("https://cp.example.com/ui") + + def test_rejects_prefix_attack(self, monkeypatch): + """return_to like cp.example.com.evil.com must be rejected (not just prefix match).""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + with pytest.raises(HTTPException) as exc_info: + SSOAuthenticationHandler._validate_return_to("https://cp.example.com.evil.com/steal") + assert exc_info.value.status_code == 400 + + def test_rejects_different_origin(self, monkeypatch): + """return_to pointing to a completely different domain should be rejected.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + with pytest.raises(HTTPException) as exc_info: + SSOAuthenticationHandler._validate_return_to("https://evil.com/phish") + assert exc_info.value.status_code == 400 + + def test_case_insensitive_hostname(self, monkeypatch): + """Hostname comparison should be case-insensitive per RFC 3986.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://CP.Example.COM"}, + ) + # Should not raise + SSOAuthenticationHandler._validate_return_to("https://cp.example.com/ui") + + def test_rejects_scheme_mismatch(self, monkeypatch): + """http:// must be rejected when control_plane_url uses https://.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + with pytest.raises(HTTPException) as exc_info: + SSOAuthenticationHandler._validate_return_to("http://cp.example.com/ui") + assert exc_info.value.status_code == 400 + + def test_rejects_port_mismatch(self, monkeypatch): + """Non-default port must be rejected.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + with pytest.raises(HTTPException) as exc_info: + SSOAuthenticationHandler._validate_return_to("https://cp.example.com:8443/ui") + assert exc_info.value.status_code == 400 + + def test_allows_explicit_default_port(self, monkeypatch): + """https://host:443 should match https://host (default port normalisation).""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + SSOAuthenticationHandler._validate_return_to("https://cp.example.com:443/ui") + + def test_allows_matching_custom_port(self, monkeypatch): + """Both sides on the same custom port should match.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com:3000"}, + ) + SSOAuthenticationHandler._validate_return_to("https://cp.example.com:3000/ui") + diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 112a06b1731..bd6162f225a 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -236,6 +236,217 @@ def test_login_v2_returns_json_on_invalid_json_body(monkeypatch): assert isinstance(data["error"], dict) +def test_login_v3_rejected_without_control_plane_url(monkeypatch): + """v3/login returns 404 when control_plane_url is not configured.""" + mock_prisma_client = MagicMock() + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "test-master-key") + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + client = TestClient(app) + response = client.post( + "/v3/login", + json={"username": "alice", "password": "secret"}, + ) + + assert response.status_code == 404 + assert "control_plane_url" in response.json()["error"]["message"] + + +def test_login_v3_returns_code(monkeypatch): + """v3/login returns an opaque code, not the JWT directly.""" + mock_prisma_client = MagicMock() + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.authenticate_user", + AsyncMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.create_ui_token_object", + MagicMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr("jwt.encode", MagicMock(return_value="signed-token")) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "test-master-key") + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_config = MagicMock() + mock_config.worker_registry = [] + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_config", mock_config) + monkeypatch.setattr("litellm.proxy.utils.get_server_root_path", lambda: "") + monkeypatch.setattr("litellm.proxy.utils.get_proxy_base_url", lambda: None) + + client = TestClient(app) + response = client.post( + "/v3/login", + json={"username": "alice", "password": "secret"}, + ) + + assert response.status_code == 200 + data = response.json() + assert "code" in data + assert data["expires_in"] == 60 + assert "token" not in data + + +def test_login_v3_exchange_happy_path(monkeypatch): + """Full flow: v3/login returns code, v3/login/exchange redeems it for JWT.""" + mock_prisma_client = MagicMock() + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.authenticate_user", + AsyncMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.create_ui_token_object", + MagicMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr("jwt.encode", MagicMock(return_value="signed-token")) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "test-master-key") + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_config = MagicMock() + mock_config.worker_registry = [] + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_config", mock_config) + monkeypatch.setattr("litellm.proxy.utils.get_server_root_path", lambda: "") + monkeypatch.setattr("litellm.proxy.utils.get_proxy_base_url", lambda: None) + + client = TestClient(app) + + # Step 1: login — get code + login_response = client.post( + "/v3/login", + json={"username": "alice", "password": "secret"}, + ) + assert login_response.status_code == 200 + code = login_response.json()["code"] + + # Step 2: exchange — get JWT + exchange_response = client.post( + "/v3/login/exchange", + json={"code": code}, + ) + assert exchange_response.status_code == 200 + exchange_data = exchange_response.json() + assert exchange_data["token"] == "signed-token" + assert "redirect_url" in exchange_data + assert exchange_response.cookies.get("token") == "signed-token" + + +def test_login_v3_exchange_single_use(monkeypatch): + """Code can only be redeemed once.""" + mock_prisma_client = MagicMock() + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.authenticate_user", + AsyncMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.create_ui_token_object", + MagicMock(return_value={"user_id": "test-user"}), + ) + monkeypatch.setattr("jwt.encode", MagicMock(return_value="signed-token")) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "test-master-key") + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_config = MagicMock() + mock_config.worker_registry = [] + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_config", mock_config) + monkeypatch.setattr("litellm.proxy.utils.get_server_root_path", lambda: "") + monkeypatch.setattr("litellm.proxy.utils.get_proxy_base_url", lambda: None) + + client = TestClient(app) + + login_response = client.post( + "/v3/login", + json={"username": "alice", "password": "secret"}, + ) + code = login_response.json()["code"] + + # First exchange succeeds + first = client.post("/v3/login/exchange", json={"code": code}) + assert first.status_code == 200 + + # Second exchange fails + second = client.post("/v3/login/exchange", json={"code": code}) + assert second.status_code == 401 + + +def test_login_v3_exchange_invalid_code(monkeypatch): + """Random code returns 401.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + client = TestClient(app) + response = client.post( + "/v3/login/exchange", + json={"code": "nonexistent-code"}, + ) + assert response.status_code == 401 + + +def test_login_v3_exchange_rejected_without_control_plane_url(monkeypatch): + """v3/login/exchange returns 404 when control_plane_url is not configured.""" + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + client = TestClient(app) + response = client.post( + "/v3/login/exchange", + json={"code": "some-code"}, + ) + + assert response.status_code == 404 + assert "control_plane_url" in response.json()["error"]["message"] + + +def test_login_v3_returns_json_on_proxy_exception(monkeypatch): + """Test that /v3/login returns JSON error when ProxyException is raised""" + from litellm.proxy._types import ProxyErrorTypes, ProxyException + + mock_prisma_client = MagicMock() + mock_authenticate_user = AsyncMock( + side_effect=ProxyException( + message="Invalid credentials", + type=ProxyErrorTypes.auth_error, + param="password", + code=401, + ) + ) + + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.authenticate_user", + mock_authenticate_user, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "test-master-key") + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"control_plane_url": "https://cp.example.com"}, + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + client = TestClient(app) + response = client.post( + "/v3/login", + json={"username": "alice", "password": "wrong"}, + ) + + assert response.status_code == 401 + assert response.headers["content-type"] == "application/json" + data = response.json() + assert "error" in data + assert data["error"]["message"] == "Invalid credentials" + assert data["error"]["type"] == "auth_error" + + def test_fallback_login_has_no_deprecation_banner(client_no_auth): response = client_no_auth.get("/fallback/login") diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/login/useLogin.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/login/useLogin.ts index a15b4a06d13..be53b1c80a8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/login/useLogin.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/login/useLogin.ts @@ -3,8 +3,8 @@ import { loginCall, LoginRequest } from "@/components/networking"; export const useLogin = () => { return useMutation({ - mutationFn: async ({ username, password }: LoginRequest) => { - const result = await loginCall(username, password); + mutationFn: async ({ username, password, useV3 }: LoginRequest) => { + const result = await loginCall(username, password, useV3); return result; }, }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiConfig/useUIConfig.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiConfig/useUIConfig.test.ts index aba5dddf13d..b05bae1e818 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiConfig/useUIConfig.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiConfig/useUIConfig.test.ts @@ -28,6 +28,8 @@ const mockUIConfig: LiteLLMWellKnownUiConfig = { proxy_base_url: "https://proxy.example.com", auto_redirect_to_sso: true, admin_ui_disabled: false, + is_control_plane: false, + workers: [], }; describe("useUIConfig", () => { @@ -102,6 +104,8 @@ describe("useUIConfig", () => { auto_redirect_to_sso: false, sso_configured: false, admin_ui_disabled: true, + is_control_plane: false, + workers: [], }; // Mock successful API call with different data diff --git a/ui/litellm-dashboard/src/app/login/LoginPage.test.tsx b/ui/litellm-dashboard/src/app/login/LoginPage.test.tsx index ad2dde2da83..866b7d0f172 100644 --- a/ui/litellm-dashboard/src/app/login/LoginPage.test.tsx +++ b/ui/litellm-dashboard/src/app/login/LoginPage.test.tsx @@ -41,6 +41,17 @@ vi.mock("@/app/(dashboard)/hooks/login/useLogin", () => ({ })), })); +vi.mock("@/hooks/useWorker", () => ({ + useWorker: vi.fn(() => ({ + isControlPlane: false, + workers: [], + selectedWorkerId: null, + selectedWorker: null, + selectWorker: vi.fn(), + disconnectFromWorker: vi.fn(), + })), +})); + import { useUIConfig } from "@/app/(dashboard)/hooks/uiConfig/useUIConfig"; import { getCookie } from "@/utils/cookieUtils"; import { isJwtExpired } from "@/utils/jwtUtils"; @@ -108,7 +119,7 @@ describe("LoginPage", () => { ); await waitFor(() => { - expect(mockReplace).toHaveBeenCalledWith("http://localhost:4000/ui"); + expect(mockReplace).toHaveBeenCalledWith("/ui"); }); }); @@ -189,7 +200,7 @@ describe("LoginPage", () => { ); await waitFor(() => { - expect(mockReplace).toHaveBeenCalledWith("http://localhost:4000/ui"); + expect(mockReplace).toHaveBeenCalledWith("/ui"); }); expect(mockPush).not.toHaveBeenCalled(); diff --git a/ui/litellm-dashboard/src/app/login/LoginPage.tsx b/ui/litellm-dashboard/src/app/login/LoginPage.tsx index d54140c5a29..5a9d420456c 100644 --- a/ui/litellm-dashboard/src/app/login/LoginPage.tsx +++ b/ui/litellm-dashboard/src/app/login/LoginPage.tsx @@ -3,14 +3,15 @@ import { useLogin } from "@/app/(dashboard)/hooks/login/useLogin"; import { useUIConfig } from "@/app/(dashboard)/hooks/uiConfig/useUIConfig"; import LoadingScreen from "@/components/common_components/LoadingScreen"; -import { getProxyBaseUrl } from "@/components/networking"; -import { getCookie } from "@/utils/cookieUtils"; +import { exchangeLoginCode, getProxyBaseUrl, switchToWorkerUrl } from "@/components/networking"; +import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; import { isJwtExpired } from "@/utils/jwtUtils"; import { consumeReturnUrl, getReturnUrl, isValidReturnUrl } from "@/utils/returnUrlUtils"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { Alert, Button, Card, Form, Input, Popover, Space, Typography } from "antd"; +import { InfoCircleOutlined, CloudServerOutlined } from "@ant-design/icons"; +import { Alert, Button, Card, Form, Input, Popover, Select, Space, Typography } from "antd"; import { useRouter } from "next/navigation"; import { useEffect, useState } from "react"; +import { useWorker } from "@/hooks/useWorker"; function LoginPageContent() { const [username, setUsername] = useState(""); @@ -19,6 +20,17 @@ function LoginPageContent() { const { data: uiConfig, isLoading: isConfigLoading } = useUIConfig(); const loginMutation = useLogin(); const router = useRouter(); + const { workers, selectWorker } = useWorker(); + const [selectedWorkerId, setSelectedWorkerId] = useState(null); + + // Pre-select worker from URL param (e.g. /ui/login?worker=team-b) + useEffect(() => { + const params = new URLSearchParams(window.location.search); + const workerParam = params.get("worker"); + if (workerParam) { + setSelectedWorkerId(workerParam); + } + }, []); useEffect(() => { if (isConfigLoading) { @@ -31,6 +43,44 @@ function LoginPageContent() { return; } + // Cross-origin SSO: worker redirected back with a single-use code. + // Exchange it for the JWT via the worker's /v3/login/exchange endpoint. + const params = new URLSearchParams(window.location.search); + const ssoCode = params.get("code"); + if (ssoCode) { + const workerUrl = localStorage.getItem("litellm_worker_url"); + exchangeLoginCode(ssoCode, workerUrl).then(() => { + params.delete("code"); + const cleanSearch = params.toString(); + window.history.replaceState(null, "", window.location.pathname + (cleanSearch ? `?${cleanSearch}` : "")); + router.replace("/ui/?login=success"); + }); + return; + } + + // Backwards compat: handle direct token in URL (legacy flow) + const urlToken = params.get("token"); + if (urlToken && !isJwtExpired(urlToken)) { + document.cookie = `token=${urlToken}; path=/; SameSite=Lax`; + params.delete("token"); + const cleanSearch = params.toString(); + window.history.replaceState( + null, + "", + window.location.pathname + (cleanSearch ? `?${cleanSearch}` : ""), + ); + router.replace("/ui/?login=success"); + return; + } + + // If switching workers on a control plane, clear the old token and show login + const switchingWorker = params.has("worker"); + if (switchingWorker && uiConfig?.is_control_plane) { + clearTokenCookies(); + setIsLoading(false); + return; + } + const rawToken = getCookie("token"); if (rawToken && !isJwtExpired(rawToken)) { // User already logged in - redirect to return URL or default @@ -38,7 +88,7 @@ function LoginPageContent() { if (returnUrl) { router.replace(returnUrl); } else { - router.replace(`${getProxyBaseUrl()}/ui`); + router.replace("/ui"); } return; } @@ -58,16 +108,35 @@ function LoginPageContent() { }, [isConfigLoading, router, uiConfig]); const handleSubmit = () => { + // If a worker is selected, point proxyBaseUrl at it before login + const selectedWorker = workers.find((w) => w.worker_id === selectedWorkerId); + if (selectedWorker) { + switchToWorkerUrl(selectedWorker.url); + } + loginMutation.mutate( - { username, password }, + { username, password, useV3: !!selectedWorker }, { onSuccess: (data) => { - // Check if we have a return URL to use instead of the default redirect - const returnUrl = consumeReturnUrl(); - if (returnUrl) { - router.push(returnUrl); + // Update the worker context with the selected worker + if (selectedWorker) { + selectWorker(selectedWorker.worker_id); + // Stay on the CP's UI — proxyBaseUrl already points at the worker + router.push("/ui/?login=success"); } else { - router.push(data.redirect_url); + // Normal (non-control-plane) login — follow the server's redirect + const returnUrl = consumeReturnUrl(); + if (returnUrl) { + router.push(returnUrl); + } else { + router.push(data.redirect_url); + } + } + }, + onError: () => { + // Reset proxyBaseUrl on login failure + if (selectedWorker) { + switchToWorkerUrl(null); } }, }, @@ -154,6 +223,22 @@ function LoginPageContent() { {error && }
+ {uiConfig?.is_control_plane && workers.length > 0 && ( + + + (option?.label as string ?? "").toLowerCase().includes(input.toLowerCase()) + } + value={selectedWorker.worker_id} + style={{ minWidth: 180 }} + suffixIcon={} + options={workers.map((w) => ({ + label: w.name, + value: w.worker_id, + disabled: w.worker_id === selectedWorker.worker_id, + }))} + onChange={(newWorkerId) => { + onWorkerSwitch(newWorkerId); + }} + /> + ); +}; + +export default WorkerDropdown; diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index c46a3af5a6e..96d6ce613bf 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -4,6 +4,7 @@ import { getProxyBaseUrl } from "@/components/networking"; import { useUIConfig } from "@/app/(dashboard)/hooks/uiConfig/useUIConfig"; import { useTheme } from "@/contexts/ThemeContext"; import { clearTokenCookies } from "@/utils/cookieUtils"; +import { clearStoredReturnUrl } from "@/utils/returnUrlUtils"; import { fetchProxySettings } from "@/utils/proxyUtils"; import { MenuFoldOutlined, MenuUnfoldOutlined, MessageOutlined, MoonOutlined, SunOutlined } from "@ant-design/icons"; import { Button, Switch, Tag } from "antd"; @@ -12,6 +13,7 @@ import React, { useEffect, useState } from "react"; import { BlogDropdown } from "./Navbar/BlogDropdown/BlogDropdown"; import { CommunityEngagementButtons } from "./Navbar/CommunityEngagementButtons/CommunityEngagementButtons"; import UserDropdown from "./Navbar/UserDropdown/UserDropdown"; +import WorkerDropdown from "./Navbar/WorkerDropdown/WorkerDropdown"; interface NavbarProps { userID: string | null; @@ -77,9 +79,19 @@ const Navbar: React.FC = ({ const handleLogout = () => { clearTokenCookies(); + localStorage.removeItem("litellm_selected_worker_id"); + localStorage.removeItem("litellm_worker_url"); window.location.href = logoutUrl; }; + const handleWorkerSwitch = (workerId: string) => { + clearTokenCookies(); + clearStoredReturnUrl(); + localStorage.removeItem("litellm_selected_worker_id"); + localStorage.removeItem("litellm_worker_url"); + window.location.href = `/ui/login?worker=${encodeURIComponent(workerId)}`; + }; + return (