From 7c47609f7af6f32cc5777e04bfe19763fa8dd1f2 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 5 Mar 2026 00:33:17 -0500 Subject: [PATCH 01/62] fix(provider): register bedrock_mantle in model_list and models_by_provider Adds bedrock_mantle_models to the model_list union and models_by_provider dict so models are discoverable via litellm.model_list and litellm.models_by_provider["bedrock_mantle"]. Co-Authored-By: Claude Sonnet 4.6 --- litellm/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 57e9cb25f43..ff7ef55c50c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -962,6 +962,7 @@ model_list = list( | ovhcloud_models | lemonade_models | docker_model_runner_models + | bedrock_mantle_models | set(clarifai_models) ) @@ -1065,6 +1066,7 @@ models_by_provider: dict = { "aws_polly": aws_polly_models, "gigachat": gigachat_models, "llamagate": llamagate_models, + "bedrock_mantle": bedrock_mantle_models } # mapping for those models which have larger equivalents From f1b86366d38d0c090f483db6cd34d98f4452c013 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 5 Mar 2026 00:33:44 -0500 Subject: [PATCH 02/62] Revert "fix(provider): register bedrock_mantle in model_list and models_by_provider" This reverts commit 7c47609f7af6f32cc5777e04bfe19763fa8dd1f2. --- litellm/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index ff7ef55c50c..57e9cb25f43 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -962,7 +962,6 @@ model_list = list( | ovhcloud_models | lemonade_models | docker_model_runner_models - | bedrock_mantle_models | set(clarifai_models) ) @@ -1066,7 +1065,6 @@ models_by_provider: dict = { "aws_polly": aws_polly_models, "gigachat": gigachat_models, "llamagate": llamagate_models, - "bedrock_mantle": bedrock_mantle_models } # mapping for those models which have larger equivalents From fffc069eb959078fd38176e1b2241cf7577f8b17 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 7 Mar 2026 16:11:56 -0800 Subject: [PATCH 03/62] [Fix] UI - MCP Servers: stop health checks triggering on deletion Removing an MCP server caused useMCPServerHealth to receive a new serverIds array (one fewer item), which changed the React Query key and triggered a fresh health check for every remaining server. Fix: remove serverIds from the hook's signature and query key entirely. The hook now uses a stable, constant key and always fetches health for all servers. The 30-second polling interval is unaffected, and the serversWithHealth merge already ignores health data for deleted servers. Co-Authored-By: Claude Sonnet 4.6 --- .../mcpServers/useMCPServerHealth.test.ts | 58 ++++--------- .../hooks/mcpServers/useMCPServerHealth.ts | 6 +- .../components/mcp_tools/mcp_servers.test.tsx | 87 ++++++++++++++++++- .../src/components/mcp_tools/mcp_servers.tsx | 3 +- 4 files changed, 107 insertions(+), 47 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.test.ts index be910acf7e4..567e1d23013 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.test.ts @@ -38,7 +38,7 @@ describe("useMCPServerHealth", () => { vi.clearAllMocks(); }); - it("should fetch health status for given server IDs", async () => { + it("should fetch health status for all servers", async () => { const mockHealthStatuses = [ { server_id: "server-1", status: "healthy" }, { server_id: "server-2", status: "unhealthy" }, @@ -46,27 +46,6 @@ describe("useMCPServerHealth", () => { vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue(mockHealthStatuses); - const { result } = renderHook(() => useMCPServerHealth(["server-1", "server-2"]), { - wrapper, - }); - - await waitFor(() => { - expect(result.current.isSuccess).toBe(true); - }); - - expect(networking.fetchMCPServerHealth).toHaveBeenCalledWith("test-token-123", ["server-1", "server-2"]); - expect(result.current.data).toEqual(mockHealthStatuses); - }); - - it("should fetch health status for all servers when no server IDs provided", async () => { - const mockHealthStatuses = [ - { server_id: "server-1", status: "healthy" }, - { server_id: "server-2", status: "healthy" }, - { server_id: "server-3", status: "unhealthy" }, - ]; - - vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue(mockHealthStatuses); - const { result } = renderHook(() => useMCPServerHealth(), { wrapper, }); @@ -75,30 +54,15 @@ describe("useMCPServerHealth", () => { expect(result.current.isSuccess).toBe(true); }); - expect(networking.fetchMCPServerHealth).toHaveBeenCalledWith("test-token-123", undefined); + expect(networking.fetchMCPServerHealth).toHaveBeenCalledWith("test-token-123"); expect(result.current.data).toEqual(mockHealthStatuses); }); - it("should handle empty server list", async () => { - vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue([]); - - const { result } = renderHook(() => useMCPServerHealth([]), { - wrapper, - }); - - await waitFor(() => { - expect(result.current.isSuccess).toBe(true); - }); - - expect(networking.fetchMCPServerHealth).toHaveBeenCalledWith("test-token-123", []); - expect(result.current.data).toEqual([]); - }); - it("should handle errors when fetching health status", async () => { const mockError = new Error("Failed to fetch health status"); vi.mocked(networking.fetchMCPServerHealth).mockRejectedValue(mockError); - const { result } = renderHook(() => useMCPServerHealth(["server-1"]), { + const { result } = renderHook(() => useMCPServerHealth(), { wrapper, }); @@ -116,7 +80,7 @@ describe("useMCPServerHealth", () => { accessToken: null, } as any); - const { result } = renderHook(() => useMCPServerHealth(["server-1"]), { + const { result } = renderHook(() => useMCPServerHealth(), { wrapper, }); @@ -124,4 +88,18 @@ describe("useMCPServerHealth", () => { expect(result.current.status).toBe("pending"); expect(networking.fetchMCPServerHealth).not.toHaveBeenCalled(); }); + + it("should use a stable query key that does not include server IDs", () => { + // Regression test: deleting a server used to pass a changing serverIds array into the + // hook, which was embedded in the query key. React Query would see a new key and fire + // a health check for every remaining server. + // + // The fix: the hook takes no serverIds parameter and uses a constant query key, so + // deleting (or adding) a server never causes an extra health check request. + // + // We verify the contract here by confirming the hook accepts no arguments. + // The stable-key behaviour is further exercised by mcp_servers.test.tsx. + const hookLength = useMCPServerHealth.length; + expect(hookLength).toBe(0); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.ts index 95d7f3bcee0..f81ade047e2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.ts @@ -10,11 +10,11 @@ interface MCPServerHealth { status: string; } -export const useMCPServerHealth = (serverIds?: string[]) => { +export const useMCPServerHealth = () => { const { accessToken } = useAuthorized(); return useQuery({ - queryKey: [...mcpServerHealthKeys.lists(), { serverIds }], - queryFn: async () => await fetchMCPServerHealth(accessToken!, serverIds), + queryKey: mcpServerHealthKeys.lists(), + queryFn: async () => await fetchMCPServerHealth(accessToken!), enabled: !!accessToken, // Refetch health status every 30 seconds to keep it up to date refetchInterval: 30000, diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.test.tsx index 8385fc7ac70..0ca3d698956 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.test.tsx @@ -185,9 +185,10 @@ describe("MCPServers", () => { expect(getByText("MCP Servers")).toBeInTheDocument(); }); - // Verify the health check API was called with server IDs + // Verify the health check API was called (without a server ID filter — the hook always + // fetches health for all servers so the query key stays stable) await waitFor(() => { - expect(networking.fetchMCPServerHealth).toHaveBeenCalledWith("123", ["server-1", "server-2"]); + expect(networking.fetchMCPServerHealth).toHaveBeenCalledWith("123"); }); }); @@ -348,4 +349,86 @@ describe("MCPServers", () => { // Team B server should not be visible expect(screen.queryByText("Team B Server")).not.toBeInTheDocument(); }); + + it("should not trigger an extra health check when the server list changes after deletion", async () => { + // Regression test: previously useMCPServerHealth received serverIds derived from the + // server list. Deleting a server changed serverIds, which changed the React Query key, + // which caused a new health check request for every remaining server. + // + // Fix: useMCPServerHealth uses a stable, argument-free query key. The component + // re-rendering with a shorter server list must NOT produce a second health fetch. + const twoServers = [ + { + server_id: "server-1", + server_name: "Test Server 1", + alias: "test-server-1", + url: "https://example.com/mcp", + transport: "http", + auth_type: "none", + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + teams: [], + mcp_access_groups: [], + }, + { + server_id: "server-2", + server_name: "Test Server 2", + alias: "test-server-2", + url: "https://example2.com/mcp", + transport: "sse", + auth_type: "api_key", + created_at: "2024-01-02T00:00:00Z", + created_by: "user-2", + updated_at: "2024-01-02T00:00:00Z", + updated_by: "user-2", + teams: [], + mcp_access_groups: [], + }, + ]; + const oneServer = twoServers.slice(0, 1); + + // First call returns two servers; second (after deletion) returns one + vi.mocked(networking.fetchMCPServers) + .mockResolvedValueOnce(twoServers) + .mockResolvedValueOnce(oneServer); + vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue([ + { server_id: "server-1", status: "healthy" }, + { server_id: "server-2", status: "healthy" }, + ]); + + // Use a shared queryClient with a non-zero gcTime so cached health data survives + // the re-render triggered by the server list refresh + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: 60_000 } }, + }); + + const { rerender } = render( + + + , + ); + + // Wait for the initial health fetch to complete + await waitFor(() => { + expect(networking.fetchMCPServerHealth).toHaveBeenCalledTimes(1); + }); + + // Simulate what happens after a server is deleted: the server list query is + // refetched (returns oneServer), causing the component to re-render with the + // shorter list. + await act(async () => { + await queryClient.invalidateQueries({ queryKey: ["mcpServers"] }); + }); + + rerender( + + + , + ); + + // The server list refresh must NOT trigger a second health check + expect(networking.fetchMCPServerHealth).toHaveBeenCalledTimes(1); + }); }); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx index 75e5bc8d453..a2f63ef73ad 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx @@ -27,8 +27,7 @@ const MCPServers: React.FC = ({ accessToken, userRole, userID }) const { data: mcpServers, isLoading: isLoadingServers, refetch } = useMCPServers(); // Fetch health status for all servers - const serverIds = useMemo(() => mcpServers?.map((server) => server.server_id), [mcpServers]); - const { data: healthStatuses, isLoading: isLoadingHealth } = useMCPServerHealth(serverIds); + const { data: healthStatuses, isLoading: isLoadingHealth } = useMCPServerHealth(); // Merge health status data into servers const serversWithHealth = useMemo(() => { From 78159212d92c4709b8f679c6ccd44c3611ce1300 Mon Sep 17 00:00:00 2001 From: netbrah <162479981+netbrah@users.noreply.github.com> Date: Sun, 8 Mar 2026 07:31:13 -0400 Subject: [PATCH 04/62] fix(anthropic): enforce type:'object' on tool input schemas Anthropic's API requires all tool input_schema to have type:'object' at the root level. When OpenAI-format tools have parameters with a missing or non-'object' type field (common with MCP tool servers), the schema was passed through unchanged, causing Anthropic to reject with: 'tools.N.custom.input_schema.type: Input should be object'. The existing default handles the case where parameters is entirely missing, but does not normalize schemas that ARE provided with a wrong or absent type field. Fix: After extracting _input_schema in _map_tool_helper(), ensure type is set to 'object' and properties exists. This matches the normalization already done implicitly by the Bedrock handler. Added 4 unit tests covering: missing type, wrong type, valid schema (no-op), and entirely missing parameters. Related issues: #12020, #64, #1671 --- litellm/llms/anthropic/chat/transformation.py | 8 ++ .../test_anthropic_chat_transformation.py | 112 ++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 04b27e87821..41fcd3e752a 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -395,6 +395,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): }, ) + # Anthropic requires input_schema.type to be "object". Normalize + # schemas from external sources (MCP servers, OpenAI callers) that + # may omit the type field or use a non-object type. + if _input_schema.get("type") != "object": + _input_schema["type"] = "object" + if "properties" not in _input_schema: + _input_schema["properties"] = {} + _allowed_properties = set(AnthropicInputSchema.__annotations__.keys()) input_schema_filtered = { k: v for k, v in _input_schema.items() if k in _allowed_properties diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index b540b0d952d..a5388d19f2b 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3175,3 +3175,115 @@ def test_map_openai_params_max_tokens_normalized_to_int(): assert "max_tokens" in result assert result["max_tokens"] == 1 + + +# ======================================================================== +# Tool schema normalization tests +# ======================================================================== + + +def test_map_tool_helper_enforces_object_type_when_missing(): + """ + Anthropic requires input_schema.type to be "object". When an OpenAI tool + has parameters without a 'type' field (common with MCP servers), LiteLLM + should inject type:"object" before forwarding to Anthropic. + + Without this fix, Anthropic rejects with: + tools.N.custom.input_schema.type: Input should be 'object' + """ + config = AnthropicConfig() + + # Tool with parameters that has properties but no 'type' field + tool = { + "type": "function", + "function": { + "name": "search_code", + "description": "Search for code patterns", + "parameters": { + "properties": { + "query": {"type": "string", "description": "Search query"} + }, + "required": ["query"], + }, + }, + } + + result, _ = config._map_tool_helper(tool) + assert result is not None + assert result["input_schema"]["type"] == "object" + assert "properties" in result["input_schema"] + assert "query" in result["input_schema"]["properties"] + + +def test_map_tool_helper_enforces_object_type_when_wrong_type(): + """ + If a tool schema has type:"string" or type:"array" at the root level, + LiteLLM should normalize it to type:"object" for Anthropic compatibility. + """ + config = AnthropicConfig() + + tool = { + "type": "function", + "function": { + "name": "echo", + "description": "Echo input", + "parameters": { + "type": "string", + "description": "The input to echo", + }, + }, + } + + result, _ = config._map_tool_helper(tool) + assert result is not None + assert result["input_schema"]["type"] == "object" + + +def test_map_tool_helper_preserves_valid_object_schema(): + """ + When a tool schema already has type:"object", it should be preserved + without modification. + """ + config = AnthropicConfig() + + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + }, + "required": ["city"], + }, + }, + } + + result, _ = config._map_tool_helper(tool) + assert result is not None + assert result["input_schema"]["type"] == "object" + assert "city" in result["input_schema"]["properties"] + assert result["input_schema"]["required"] == ["city"] + + +def test_map_tool_helper_empty_parameters_get_default(): + """ + When parameters is entirely missing, the existing default should still + produce a valid {type:"object", properties:{}} schema. + """ + config = AnthropicConfig() + + tool = { + "type": "function", + "function": { + "name": "no_params_tool", + "description": "Tool with no parameters", + }, + } + + result, _ = config._map_tool_helper(tool) + assert result is not None + assert result["input_schema"]["type"] == "object" + assert result["input_schema"].get("properties") == {} From 5d1106f018a4ea0f17c23cef882e78857d1bfd48 Mon Sep 17 00:00:00 2001 From: netbrah <162479981+netbrah@users.noreply.github.com> Date: Sun, 8 Mar 2026 07:31:26 -0400 Subject: [PATCH 05/62] fix(anthropic): deduplicate tool_result messages by tool_call_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic requires exactly one tool_result per tool_use. When conversation history (e.g. from session resume/checkpoint restore) contains duplicate tool result messages with the same tool_call_id, the API rejects with: 'each tool_use must have a single result. Found multiple tool_result blocks with id: '. This is already handled for Bedrock via _deduplicate_bedrock_tool_content() but was missing from the Anthropic direct and Vertex AI partner paths, which share sanitize_messages_for_tool_calling(). Fix: Add Case D to sanitize_messages_for_tool_calling() — after the existing orphan detection passes, scan for duplicate tool_call_ids and keep only the last occurrence (most complete result). Added 3 unit tests: dedup with duplicates, no-op with unique IDs, and behavior when modify_params=False. Related issues: #11804, #11029, #6836, #1782, #151 --- .../prompt_templates/factory.py | 48 +++ ...llm_core_utils_prompt_templates_factory.py | 288 +++++++++++++++++- 2 files changed, 335 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index a694cec7d66..5e905da2230 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2221,6 +2221,11 @@ def sanitize_messages_for_tool_calling( Case C: Empty text content - Replace empty or whitespace-only text content with a placeholder message. + Case D: Duplicate tool_result for same tool_use (duplicate results) + - If multiple tool messages reference the same tool_call_id, keep only the last + occurrence. Anthropic requires exactly one tool_result per tool_use and rejects + with: "each tool_use must have a single result". + This function operates on OpenAI format messages before they are converted to provider-specific formats. """ @@ -2256,6 +2261,49 @@ def sanitize_messages_for_tool_calling( sanitized_messages.append(current_message) i += 1 + # Case D: Deduplicate tool results with the same tool_call_id. + # Anthropic requires exactly one tool_result per tool_use. Session history + # (e.g. from conversation resume) can contain duplicate tool_result messages + # for the same tool_call_id. Keep only the last occurrence *within each + # contiguous block of tool results following an assistant message*. This + # avoids dropping results from earlier turns if a tool_call_id is reused. + # + # NOTE: This intentionally keeps the *last* occurrence (most complete for + # session-resume duplicates), unlike _deduplicate_bedrock_content_blocks + # which keeps the *first*. The Bedrock case handles provider-side content + # block duplication where the first is authoritative; here the duplicate + # arises from history replay where the last entry is the final state. + duplicates_to_remove: Set[int] = set() + seen_in_block: Dict[str, int] = {} # tool_call_id -> index (reset per block) + for idx, msg in enumerate(sanitized_messages): + role = msg.get("role") + tcid = msg.get("tool_call_id") if role in ["tool", "function"] else None + if tcid: + if tcid in seen_in_block: + # Mark the earlier occurrence for removal (keep latest) + duplicates_to_remove.add(seen_in_block[tcid]) + verbose_logger.warning( + "sanitize_messages_for_tool_calling: dropping duplicate " + "tool_result with tool_call_id=%s. This may indicate " + "duplicate tool messages in conversation history.", + tcid, + ) + seen_in_block[tcid] = idx + elif role not in ("tool", "function"): + # Non-tool message (user, assistant, system) marks a + # conversational-turn boundary — reset tracking. + # Tool/function messages with no tool_call_id are malformed; + # they should NOT reset the block because they don't represent + # a turn boundary and would mask real within-block duplicates. + seen_in_block = {} + + if duplicates_to_remove: + sanitized_messages = [ + msg + for idx, msg in enumerate(sanitized_messages) + if idx not in duplicates_to_remove + ] + return sanitized_messages diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 707b5bdc777..8d68539564c 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -10,6 +10,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( BedrockImageProcessor, _convert_to_bedrock_tool_call_invoke, ollama_pt, + sanitize_messages_for_tool_calling, ) @@ -1179,7 +1180,7 @@ def test_bedrock_tools_pt_does_not_handle_system_tool(): System tools (nova_grounding) should be added via web_search_options, not via the tools parameter directly. """ - + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt # Regular function tools should still work @@ -1741,3 +1742,288 @@ def test_bedrock_tool_call_invoke_multiple_normal_tools(): assert len(result) == 2 assert result[0]["toolUse"]["toolUseId"] == "call_1" assert result[1]["toolUse"]["toolUseId"] == "call_2" + + +# ======================================================================== +# Tool result deduplication tests (Case D in sanitize_messages_for_tool_calling) +# ======================================================================== + + +def test_sanitize_messages_deduplicates_tool_results(): + """ + Anthropic requires exactly one tool_result per tool_use. When conversation + history (e.g. from session resume) contains duplicate tool result messages + with the same tool_call_id, sanitize_messages_for_tool_calling should keep + only the last occurrence. + + Without this fix, Anthropic rejects with: + each tool_use must have a single result. Found multiple tool_result + blocks with id: + """ + original = litellm.modify_params + litellm.modify_params = True + try: + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "NYC"}', + }, + } + ], + }, + # First tool result (stale/duplicate) + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": "Partial result...", + }, + # Second tool result (final/complete — should be kept) + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": '{"temperature": 72, "condition": "sunny"}', + }, + ] + + result = sanitize_messages_for_tool_calling(messages) + + # Count tool messages with this ID — should be exactly 1 + tool_results = [ + m for m in result if m.get("role") == "tool" and m.get("tool_call_id") == "call_abc123" + ] + assert len(tool_results) == 1 + # Should keep the LAST occurrence (most complete) + assert tool_results[0]["content"] == '{"temperature": 72, "condition": "sunny"}' + finally: + litellm.modify_params = original + + +def test_sanitize_messages_preserves_unique_tool_results(): + """ + When each tool_call_id has exactly one tool_result, no deduplication should + occur. Messages should pass through unchanged. + """ + original = litellm.modify_params + litellm.modify_params = True + try: + messages = [ + {"role": "user", "content": "Get weather for two cities"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "NYC"}', + }, + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "LA"}', + }, + }, + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "72F"}, + {"role": "tool", "tool_call_id": "call_2", "content": "85F"}, + ] + + result = sanitize_messages_for_tool_calling(messages) + + tool_results = [m for m in result if m.get("role") == "tool"] + assert len(tool_results) == 2 + assert tool_results[0]["tool_call_id"] == "call_1" + assert tool_results[0]["content"] == "72F" + assert tool_results[1]["tool_call_id"] == "call_2" + assert tool_results[1]["content"] == "85F" + finally: + litellm.modify_params = original + + +def test_sanitize_messages_dedup_disabled_when_modify_params_false(): + """ + When litellm.modify_params is False, messages should be returned as-is + even if they contain duplicate tool results. + """ + original = litellm.modify_params + litellm.modify_params = False + try: + messages = [ + {"role": "user", "content": "Test"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_dup", + "type": "function", + "function": {"name": "test", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_dup", "content": "first"}, + {"role": "tool", "tool_call_id": "call_dup", "content": "second"}, + ] + + result = sanitize_messages_for_tool_calling(messages) + + # Should be unchanged — no sanitization when modify_params=False + assert result == messages + finally: + litellm.modify_params = original + + +def test_sanitize_messages_dedup_scoped_per_turn_preserves_cross_turn(): + """ + When the same tool_call_id appears in two different assistant turns + (separated by a user message), both tool results must be preserved. + Deduplication should only apply within a single contiguous tool-result + block, not globally across the conversation. + + Without per-turn scoping this would incorrectly drop the first tool result, + leaving the first assistant message without its required result (which + Anthropic would reject). + """ + original = litellm.modify_params + litellm.modify_params = True + try: + messages = [ + {"role": "user", "content": "First question"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_X", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q": "a"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_X", "content": "result_turn_1"}, + {"role": "user", "content": "Second question"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_X", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q": "b"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_X", "content": "result_turn_2"}, + ] + + result = sanitize_messages_for_tool_calling(messages) + + # Both tool results must survive — one per turn + tool_results = [ + m for m in result + if m.get("role") == "tool" and m.get("tool_call_id") == "call_X" + ] + assert len(tool_results) == 2, ( + f"Expected 2 tool results (one per turn), got {len(tool_results)}. " + "Dedup may be global instead of per-turn scoped." + ) + assert tool_results[0]["content"] == "result_turn_1" + assert tool_results[1]["content"] == "result_turn_2" + finally: + litellm.modify_params = original + + +def test_sanitize_messages_combined_case_a_and_case_d(): + """ + Combined Case A + Case D: an assistant message has two tool_calls — + one with a missing result (Case A should inject a dummy) and one with + duplicate results (Case D should deduplicate to keep only the last). + + This validates that both sanitization passes compose correctly without + interfering with each other. + """ + original = litellm.modify_params + litellm.modify_params = True + try: + messages = [ + {"role": "user", "content": "Do two things"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_missing", + "type": "function", + "function": {"name": "tool_a", "arguments": "{}"}, + }, + { + "id": "call_duped", + "type": "function", + "function": {"name": "tool_b", "arguments": '{"q": "x"}'}, + }, + ], + }, + # No result for call_missing — Case A should inject a dummy + # Duplicate results for call_duped — Case D should keep last + {"role": "tool", "tool_call_id": "call_duped", "content": "stale_result"}, + {"role": "tool", "tool_call_id": "call_duped", "content": "fresh_result"}, + {"role": "user", "content": "Now summarize"}, + ] + + result = sanitize_messages_for_tool_calling(messages) + + # Collect tool results from the output + tool_results = [m for m in result if m.get("role") in ("tool", "function")] + + # Case A: call_missing should have a dummy result injected + missing_results = [ + m for m in tool_results if m.get("tool_call_id") == "call_missing" + ] + assert len(missing_results) == 1, ( + f"Expected 1 dummy result for call_missing (Case A), got {len(missing_results)}" + ) + + # Case D: call_duped should have exactly 1 result (the fresh one) + duped_results = [ + m for m in tool_results if m.get("tool_call_id") == "call_duped" + ] + assert len(duped_results) == 1, ( + f"Expected 1 result for call_duped after dedup (Case D), got {len(duped_results)}" + ) + assert duped_results[0]["content"] == "fresh_result", ( + f"Expected last-wins 'fresh_result', got '{duped_results[0]['content']}'" + ) + + # Verify tool results immediately follow the assistant message + asst_idx = next( + i for i, m in enumerate(result) if m.get("role") == "assistant" + ) + tool_msgs_after_asst = [ + m + for m in result[asst_idx + 1 :] + if m.get("role") in ("tool", "function") + ] + assert len(tool_msgs_after_asst) == 2, ( + f"Expected 2 tool results after assistant, got {len(tool_msgs_after_asst)}" + ) + # Both tool_call_ids should be present (order may vary) + tool_ids = {m["tool_call_id"] for m in tool_msgs_after_asst} + assert tool_ids == {"call_missing", "call_duped"}, ( + f"Expected tool_call_ids {{call_missing, call_duped}}, got {tool_ids}" + ) + finally: + litellm.modify_params = original From ffc6d84f2762939d11cec4b0ee478e925134f441 Mon Sep 17 00:00:00 2001 From: netbrah <162479981+netbrah@users.noreply.github.com> Date: Sun, 8 Mar 2026 08:16:22 -0400 Subject: [PATCH 06/62] fix: shallow copy input_schema to avoid caller mutation + add mutation guard test Addresses Greptile review: - dict(_input_schema) before mutation prevents cross-provider state leakage - Test asserts original tool parameters dict is unchanged after call --- litellm/llms/anthropic/chat/transformation.py | 7 +++++++ .../chat/test_anthropic_chat_transformation.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 41fcd3e752a..fd1859f7d17 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -399,6 +399,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # schemas from external sources (MCP servers, OpenAI callers) that # may omit the type field or use a non-object type. if _input_schema.get("type") != "object": + litellm.verbose_logger.debug( + "_map_tool_helper: coercing input_schema type from %r to " + "'object' for Anthropic compatibility (tool: %s)", + _input_schema.get("type"), + tool["function"].get("name"), + ) + _input_schema = dict(_input_schema) # avoid mutating caller's dict _input_schema["type"] = "object" if "properties" not in _input_schema: _input_schema["properties"] = {} diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index a5388d19f2b..6f03f630b5f 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3208,11 +3208,16 @@ def test_map_tool_helper_enforces_object_type_when_missing(): }, } + original_params = tool["function"]["parameters"].copy() result, _ = config._map_tool_helper(tool) assert result is not None assert result["input_schema"]["type"] == "object" assert "properties" in result["input_schema"] assert "query" in result["input_schema"]["properties"] + # Original parameters dict must not be modified in place + assert tool["function"]["parameters"] == original_params, ( + "parameters dict was mutated; _map_tool_helper should not modify caller data" + ) def test_map_tool_helper_enforces_object_type_when_wrong_type(): @@ -3234,9 +3239,17 @@ def test_map_tool_helper_enforces_object_type_when_wrong_type(): }, } + original_params = tool["function"]["parameters"].copy() result, _ = config._map_tool_helper(tool) assert result is not None assert result["input_schema"]["type"] == "object" + assert result["input_schema"].get("properties") == {}, ( + "properties should be injected as {} when schema has non-object type and no properties key" + ) + # Original parameters dict must not be modified in place + assert tool["function"]["parameters"] == original_params, ( + "parameters dict was mutated; _map_tool_helper should not modify caller data" + ) def test_map_tool_helper_preserves_valid_object_schema(): From 9c07325396d4058974b0663968eb1e189a1622da Mon Sep 17 00:00:00 2001 From: Yong woo Song Date: Mon, 9 Mar 2026 08:58:10 +0000 Subject: [PATCH 07/62] feat: add qwen3.5 series for openrouter --- model_prices_and_context_window.json | 129 +++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ae256ed0784..ac0bb53a86a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27444,6 +27444,135 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/qwen/qwen3.5-35b-a3b": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 655536, + "max_tokens": 655536, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-35b-a3b", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "openrouter/qwen/qwen3.5-27b": { + "input_cost_per_token": 3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 655536, + "max_tokens": 655536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-27b", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "openrouter/qwen/qwen3.5-122b-a10b": { + "input_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 655536, + "max_tokens": 655536, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-122b-a10b", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + + "openrouter/qwen/qwen3.5-flash-02-23": { + "input_cost_per_token": 1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 655536, + "max_tokens": 655536, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://openrouter.ai/qwen/qwen3.5-flash-02-23", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "openrouter/qwen/qwen3.5-plus-02-15": { + "input_cost_per_token": 4e-07, + "input_cost_per_token_above_256k_tokens": 5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 655536, + "max_tokens": 655536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "output_cost_per_token_above_256k_tokens": 3e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-plus-02-15", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "openrouter/qwen/qwen3.5-397b-a17b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 655536, + "max_tokens": 655536, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-397b-a17b", + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/switchpoint/router": { "input_cost_per_token": 8.5e-07, "litellm_provider": "openrouter", From 2683fa714c14154d3d8d449da44ea766afe42fb6 Mon Sep 17 00:00:00 2001 From: Yong woo Song Date: Mon, 9 Mar 2026 09:11:31 +0000 Subject: [PATCH 08/62] fix: typo on max_output_tokens and max_tokens from qwen3.5 series --- model_prices_and_context_window.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ac0bb53a86a..e5900d7d9c8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27448,8 +27448,8 @@ "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 655536, - "max_tokens": 655536, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://openrouter.ai/qwen/qwen3.5-35b-a3b", @@ -27469,8 +27469,8 @@ "input_cost_per_token": 3e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 655536, - "max_tokens": 655536, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://openrouter.ai/qwen/qwen3.5-27b", @@ -27490,8 +27490,8 @@ "input_cost_per_token": 4e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 655536, - "max_tokens": 655536, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://openrouter.ai/qwen/qwen3.5-122b-a10b", @@ -27512,8 +27512,8 @@ "input_cost_per_token": 1e-07, "litellm_provider": "openrouter", "max_input_tokens": 1000000, - "max_output_tokens": 655536, - "max_tokens": 655536, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://openrouter.ai/qwen/qwen3.5-flash-02-23", @@ -27534,8 +27534,8 @@ "input_cost_per_token_above_256k_tokens": 5e-07, "litellm_provider": "openrouter", "max_input_tokens": 1000000, - "max_output_tokens": 655536, - "max_tokens": 655536, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_above_256k_tokens": 3e-06, @@ -27556,8 +27556,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 655536, - "max_tokens": 655536, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://openrouter.ai/qwen/qwen3.5-397b-a17b", From 0d8880ab9f09a84e71181855b76cbfa79c26a0f8 Mon Sep 17 00:00:00 2001 From: Yong woo Song Date: Mon, 9 Mar 2026 11:03:39 +0000 Subject: [PATCH 09/62] chore: fix --- model_prices_and_context_window.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e5900d7d9c8..fb5c6e036a4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27507,7 +27507,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "openrouter/qwen/qwen3.5-flash-02-23": { "input_cost_per_token": 1e-07, "litellm_provider": "openrouter", @@ -27528,7 +27527,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true - }, + }, "openrouter/qwen/qwen3.5-plus-02-15": { "input_cost_per_token": 4e-07, "input_cost_per_token_above_256k_tokens": 5e-07, From 7ffaa6f74e531f8fcc50769fdb8100a5f82da44f Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 9 Mar 2026 18:48:27 -0400 Subject: [PATCH 10/62] fix(tag-usage): remove broken compute_tag_metadata_totals causing cost panel to show 0 The tag daily activity endpoint used compute_tag_metadata_totals which deduplicates by request_id, but LiteLLM_DailyTagSpend stores aggregated daily records where request_id is either NULL or stale. This caused all metadata totals (total_spend, total_requests, etc.) to be 0 in the UI cost panel. Now uses the same standard totals as every other entity type. Co-Authored-By: Claude Opus 4.6 --- .../tag_management_endpoints.py | 3 +- .../test_common_daily_activity.py | 87 +++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 95b7300992c..f085fa4145f 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -21,7 +21,6 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, - compute_tag_metadata_totals, get_daily_activity, ) from litellm.proxy.management_helpers.utils import handle_budget_for_entity @@ -554,5 +553,5 @@ async def get_tag_daily_activity( api_key=api_key, page=page, page_size=page_size, - metadata_metrics_func=compute_tag_metadata_totals, + metadata_metrics_func=None, ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 1e357d2f02e..00a22f9bf7d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -405,6 +405,93 @@ async def test_get_api_key_metadata_regenerated_key_uses_most_recent_deleted_rec assert result["old-key-hash"]["team_id"] == "latest-team" +@pytest.mark.asyncio +async def test_tag_daily_activity_metadata_totals_not_zero(): + """Test that tag daily activity returns correct metadata totals. + + Regression test: previously compute_tag_metadata_totals skipped records + with NULL request_id, causing metadata totals (total_spend, etc.) to be 0. + """ + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + + # Create mock tag spend records (request_id is NULL for aggregated rows) + mock_record_1 = MagicMock() + mock_record_1.request_id = None # NULL in aggregated daily rows + mock_record_1.tag = "production" + mock_record_1.date = "2024-01-01" + mock_record_1.api_key = "key-1" + mock_record_1.model = "gpt-4" + mock_record_1.model_group = "gpt-4" + mock_record_1.custom_llm_provider = "openai" + mock_record_1.mcp_namespaced_tool_name = None + mock_record_1.endpoint = "/chat/completions" + mock_record_1.spend = 25.0 + mock_record_1.prompt_tokens = 500 + mock_record_1.completion_tokens = 200 + mock_record_1.cache_read_input_tokens = 0 + mock_record_1.cache_creation_input_tokens = 0 + mock_record_1.api_requests = 10 + mock_record_1.successful_requests = 9 + mock_record_1.failed_requests = 1 + + mock_record_2 = MagicMock() + mock_record_2.request_id = None + mock_record_2.tag = "staging" + mock_record_2.date = "2024-01-01" + mock_record_2.api_key = "key-2" + mock_record_2.model = "gpt-3.5-turbo" + mock_record_2.model_group = "gpt-3.5-turbo" + mock_record_2.custom_llm_provider = "openai" + mock_record_2.mcp_namespaced_tool_name = None + mock_record_2.endpoint = "/chat/completions" + mock_record_2.spend = 5.0 + mock_record_2.prompt_tokens = 300 + mock_record_2.completion_tokens = 100 + mock_record_2.cache_read_input_tokens = 0 + mock_record_2.cache_creation_input_tokens = 0 + mock_record_2.api_requests = 5 + mock_record_2.successful_requests = 5 + mock_record_2.failed_requests = 0 + + mock_table = MagicMock() + mock_table.count = AsyncMock(return_value=2) + mock_table.find_many = AsyncMock(return_value=[mock_record_1, mock_record_2]) + mock_prisma.db.litellm_dailytagspend = mock_table + mock_prisma.db.litellm_verificationtoken = MagicMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + + result = await get_daily_activity( + prisma_client=mock_prisma, + table_name="litellm_dailytagspend", + entity_id_field="tag", + entity_id=None, + entity_metadata_field=None, + start_date="2024-01-01", + end_date="2024-01-01", + model=None, + api_key=None, + page=1, + page_size=1000, + metadata_metrics_func=None, # No custom func — matches the fix + ) + + # Metadata totals must reflect actual spend, NOT be zero + assert result.metadata.total_spend == 30.0 # 25.0 + 5.0 + assert result.metadata.total_api_requests == 15 # 10 + 5 + assert result.metadata.total_successful_requests == 14 # 9 + 5 + assert result.metadata.total_failed_requests == 1 + assert result.metadata.total_tokens == 1100 # (500+200) + (300+100) + + # Verify breakdown still works + assert len(result.results) == 1 + daily = result.results[0] + assert "production" in daily.breakdown.entities + assert "staging" in daily.breakdown.entities + assert daily.breakdown.entities["production"].metrics.spend == 25.0 + assert daily.breakdown.entities["staging"].metrics.spend == 5.0 + + @pytest.mark.asyncio async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): """Test that the full aggregation pipeline should preserve metadata for deleted keys.""" From 3127d79da864ecfbf381d166093645dd16e3ca74 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 10 Mar 2026 05:49:46 +0530 Subject: [PATCH 11/62] feat: add strategy to deployment for helmchart --- deploy/charts/litellm-helm/templates/deployment.yaml | 4 ++++ deploy/charts/litellm-helm/values.yaml | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index df483ab927d..51af22b7a46 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -13,6 +13,10 @@ spec: {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }} replicas: {{ .Values.replicaCount }} {{- end }} + {{- with .Values.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} selector: matchLabels: {{- include "litellm.selectorLabels" . | nindent 6 }} diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index d62f5b29c2b..02950d9f5da 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -35,6 +35,14 @@ deploymentLabels: {} podAnnotations: {} podLabels: {} +# -- Deployment strategy configuration +# Example: +# type: RollingUpdate +# rollingUpdate: +# maxUnavailable: 0 +# maxSurge: 1 +strategy: {} + terminationGracePeriodSeconds: 90 topologySpreadConstraints: [] From 976a2afedee615f9631de4c118b7287db7fb9195 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Mon, 9 Mar 2026 17:20:48 -0700 Subject: [PATCH 12/62] Migrate user management buttons from Tremor to Ant Design Replace Tremor Button components with antd Button in the user management flow (Invite User, Bulk Invite Users, Select Users, Bulk Edit, and onboarding modal copy link button). --- .../src/components/CreateUserButton.tsx | 6 +- .../components/bulk_create_users_button.tsx | 57 +++++++++---------- .../src/components/onboarding_link.tsx | 6 +- .../src/components/view_users.tsx | 6 +- 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.tsx index c7c195835d0..fbfcb402766 100644 --- a/ui/litellm-dashboard/src/components/CreateUserButton.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.tsx @@ -1,7 +1,7 @@ import { InfoCircleOutlined, UserAddOutlined } from "@ant-design/icons"; import { useQueryClient } from "@tanstack/react-query"; import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; -import { Accordion, AccordionBody, AccordionHeader, Button as Button2, SelectItem, TextInput } from "@tremor/react"; +import { Accordion, AccordionBody, AccordionHeader, SelectItem, TextInput } from "@tremor/react"; import { Alert, Button, Form, Input, Modal, Select, Select as Select2, Space, Tooltip, Typography } from "antd"; import React, { useEffect, useMemo, useState } from "react"; import BulkCreateUsers from "./bulk_create_users_button"; @@ -229,9 +229,9 @@ export const CreateUserButton: React.FC = ({ // Original return for standalone mode return (
- setIsModalVisible(true)}> + = ({ return ( <> - setIsModalVisible(true)}> + = ({
- - Download CSV Template - +
@@ -662,14 +662,14 @@ const BulkCreateUsersButton: React.FC = ({
- } > - Remove - + Remove + {fileError ? ( @@ -694,7 +694,7 @@ const BulkCreateUsersButton: React.FC = ({

Drag and drop your CSV file here

or

- Browse files +

Only CSV files (.csv) are supported

@@ -781,21 +781,21 @@ const BulkCreateUsersButton: React.FC = ({ {!parsedData.some((user) => user.status === "success" || user.status === "failed") && (
- { setParsedData([]); setParseError(null); }} - variant="secondary" > Back - - +
)} @@ -829,40 +829,39 @@ const BulkCreateUsersButton: React.FC = ({ {!parsedData.some((user) => user.status === "success" || user.status === "failed") && (
- { setParsedData([]); setParseError(null); }} - variant="secondary" className="mr-3" > Back - - +
)} {parsedData.some((user) => user.status === "success" || user.status === "failed") && (
- { setParsedData([]); setParseError(null); }} - variant="secondary" className="mr-3" > Start New Bulk Import - - - Download User Credentials - + +
)} diff --git a/ui/litellm-dashboard/src/components/onboarding_link.tsx b/ui/litellm-dashboard/src/components/onboarding_link.tsx index e6461d26a47..7eb337a970a 100644 --- a/ui/litellm-dashboard/src/components/onboarding_link.tsx +++ b/ui/litellm-dashboard/src/components/onboarding_link.tsx @@ -1,7 +1,7 @@ import React from "react"; -import { Modal, Typography } from "antd"; +import { Button, Modal, Typography } from "antd"; import { CopyToClipboard } from "react-copy-to-clipboard"; -import { Text, Button } from "@tremor/react"; +import { Text } from "@tremor/react"; import NotificationsManager from "./molecules/notifications_manager"; export interface InvitationLink { @@ -86,7 +86,7 @@ export default function OnboardingModal({
NotificationsManager.success("Copied!")}> - diff --git a/ui/litellm-dashboard/src/components/view_users.tsx b/ui/litellm-dashboard/src/components/view_users.tsx index f4c821fb01e..50123a83955 100644 --- a/ui/litellm-dashboard/src/components/view_users.tsx +++ b/ui/litellm-dashboard/src/components/view_users.tsx @@ -1,7 +1,7 @@ import { Tab, TabGroup, TabList, TabPanel, TabPanels } from "@tremor/react"; import React, { useEffect, useState } from "react"; -import { Button } from "@tremor/react"; +import { Button } from "antd"; import BulkEditUserModal from "./BulkEditUsers"; import { CreateUserButton } from "./CreateUserButton"; import EditUserModal from "./edit_user"; @@ -309,7 +309,7 @@ const ViewUserDashboard: React.FC = ({ accessToken, toke {isProxyAdmin && ( )} From 23cf360be768f77eb1b85a0ce113056cec0ee57b Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 9 Mar 2026 23:53:55 -0400 Subject: [PATCH 13/62] fix: explicit type conversion for prompt caching --- .../anthropic_cache_control_hook.py | 4 +- .../test_anthropic_cache_control_hook.py | 72 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 5df79580d3e..67b95c7694b 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -82,8 +82,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): _targetted_index: Optional[Union[int, str]] = point.get("index", None) targetted_index: Optional[int] = None if isinstance(_targetted_index, str): - if _targetted_index.isdigit(): + try: targetted_index = int(_targetted_index) + except ValueError: + pass else: targetted_index = _targetted_index diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 38baaedef14..afeeb4a1ba6 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -903,3 +903,75 @@ async def test_anthropic_cache_control_hook_document_analysis_multiple_pages(): if isinstance(item, dict) and "cachePoint" in item ) assert cache_control_count == 1, f"Expected exactly 1 cache control point (last item only), found {cache_control_count}. Before fix, this would be 6 (one for each content item)." + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_string_negative_index(): + """ + Test that string negative indices like "-1" are handled correctly. + + When cache_control_injection_points are stored in DB/config as JSON, indices + like -1 become the string "-1". Previously, str.isdigit() returned False for + "-1" so the cache control was silently skipped. This tests the fix. + """ + with patch.dict( + os.environ, + { + "AWS_ACCESS_KEY_ID": "fake_access_key_id", + "AWS_SECRET_ACCESS_KEY": "fake_secret_access_key", + "AWS_REGION_NAME": "us-west-2", + }, + ): + anthropic_cache_control_hook = AnthropicCacheControlHook() + litellm.callbacks = [anthropic_cache_control_hook] + + mock_response = MagicMock() + mock_response.json.return_value = { + "output": { + "message": { + "role": "assistant", + "content": "Response", + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 100, + "outputTokens": 50, + "totalTokens": 150, + }, + } + mock_response.status_code = 200 + + client = AsyncHTTPHandler() + with patch.object(client, "post", return_value=mock_response) as mock_post: + await litellm.acompletion( + model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + messages=[ + {"role": "user", "content": "First message"}, + {"role": "assistant", "content": "First response"}, + {"role": "user", "content": "Second message"}, + ], + # index is a string "-1" (as stored in DB/config JSON) + cache_control_injection_points=[ + {"location": "message", "index": "-1"}, + ], + client=client, + ) + + mock_post.assert_called_once() + request_body = json.loads(mock_post.call_args.kwargs["data"]) + + # The last user message should have cache control applied + last_message = request_body["messages"][-1] + last_message_content = last_message["content"] + assert isinstance(last_message_content, list), ( + f"Expected list content, got {type(last_message_content)}" + ) + has_cache_point = any( + isinstance(item, dict) and "cachePoint" in item + for item in last_message_content + ) + assert has_cache_point, ( + f"Expected cachePoint in last message content, got: {last_message_content}. " + "String index '-1' was not parsed correctly (str.isdigit() returns False for negative strings)." + ) From 6bb9f9ceb6d8e4d8b7f149deda1f2aed9a89d6be Mon Sep 17 00:00:00 2001 From: Yong woo Song Date: Tue, 10 Mar 2026 04:05:42 +0000 Subject: [PATCH 14/62] chore: fix --- model_prices_and_context_window.json | 66 +++++----------------------- 1 file changed, 12 insertions(+), 54 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index fb5c6e036a4..1dd94bdcdd4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27453,17 +27453,10 @@ "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://openrouter.ai/qwen/qwen3.5-35b-a3b", - "supported_modalities": [ - "text", - "image", - "video" - ], - "supported_output_modalities": [ - "text" - ], "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "openrouter/qwen/qwen3.5-27b": { "input_cost_per_token": 3e-07, @@ -27474,17 +27467,10 @@ "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://openrouter.ai/qwen/qwen3.5-27b", - "supported_modalities": [ - "text", - "image", - "video" - ], - "supported_output_modalities": [ - "text" - ], "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "openrouter/qwen/qwen3.5-122b-a10b": { "input_cost_per_token": 4e-07, @@ -27495,17 +27481,10 @@ "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://openrouter.ai/qwen/qwen3.5-122b-a10b", - "supported_modalities": [ - "text", - "image", - "video" - ], - "supported_output_modalities": [ - "text" - ], "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "openrouter/qwen/qwen3.5-flash-02-23": { "input_cost_per_token": 1e-07, @@ -27516,17 +27495,10 @@ "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://openrouter.ai/qwen/qwen3.5-flash-02-23", - "supported_modalities": [ - "text", - "image", - "video" - ], - "supported_output_modalities": [ - "text" - ], "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "openrouter/qwen/qwen3.5-plus-02-15": { "input_cost_per_token": 4e-07, @@ -27539,17 +27511,10 @@ "output_cost_per_token": 2.4e-06, "output_cost_per_token_above_256k_tokens": 3e-06, "source": "https://openrouter.ai/qwen/qwen3.5-plus-02-15", - "supported_modalities": [ - "text", - "image", - "video" - ], - "supported_output_modalities": [ - "text" - ], "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "openrouter/qwen/qwen3.5-397b-a17b": { "input_cost_per_token": 6e-07, @@ -27560,17 +27525,10 @@ "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://openrouter.ai/qwen/qwen3.5-397b-a17b", - "supported_modalities": [ - "text", - "image", - "video" - ], - "supported_output_modalities": [ - "text" - ], "supports_function_calling": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "openrouter/switchpoint/router": { "input_cost_per_token": 8.5e-07, From b731e432f88ee026f38ef93a6974782c62f68d9f Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 10 Mar 2026 12:45:48 +0530 Subject: [PATCH 15/62] feat: record silent metrics --- litellm/router.py | 107 +++++++++++------- .../test_router_silent_experiment.py | 40 +++++-- 2 files changed, 98 insertions(+), 49 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 43b53d14d79..fac9409b359 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -727,15 +727,11 @@ class Router: startup_nodes = cache_config.get("startup_nodes") if not startup_nodes: _env_cluster_nodes = get_secret("REDIS_CLUSTER_NODES") - if _env_cluster_nodes is not None and isinstance( - _env_cluster_nodes, str - ): + if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str): startup_nodes = json.loads(_env_cluster_nodes) if startup_nodes: - return RedisClusterCache( - **{**cache_config, "startup_nodes": startup_nodes} - ) + return RedisClusterCache(**{**cache_config, "startup_nodes": startup_nodes}) else: return RedisCache(**cache_config) @@ -1466,12 +1462,15 @@ class Router: silent_kwargs["metadata"]["is_silent_experiment"] = True + # Force stream=False so the response is fully consumed and callbacks fire + silent_kwargs["stream"] = False + # Pop logging objects and call IDs to ensure a fresh logging context # This prevents collisions in the Proxy's database (spend_logs) silent_kwargs.pop("litellm_call_id", None) silent_kwargs.pop("litellm_logging_obj", None) silent_kwargs.pop("standard_logging_object", None) - silent_kwargs.pop("proxy_server_request", None) + # DON'T pop proxy_server_request — it's needed for spend log metadata return silent_kwargs @@ -1494,12 +1493,30 @@ class Router: silent_kwargs = self._get_silent_experiment_kwargs(**kwargs) - # Trigger the silent request - self.completion( - model=silent_model, - messages=cast(List[Dict[str, str]], messages), - **silent_kwargs, - ) + # Override model_group to correctly attribute metrics to the silent model + silent_kwargs["metadata"]["model_group"] = silent_model + + # Create a new event loop for this thread so that async success + # callbacks (e.g. _ProxyDBLogger) can schedule and run DB writes. + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + loop.run_until_complete( + self.acompletion( + model=silent_model, + messages=cast(List[AllMessageValues], messages), + **silent_kwargs, + ) + ) + # Drain any remaining fire-and-forget tasks (e.g. alerting hooks) + # scheduled via asyncio.create_task during the acompletion call. + pending = asyncio.all_tasks(loop) + if pending: + loop.run_until_complete( + asyncio.gather(*pending, return_exceptions=True) + ) + finally: + loop.close() except Exception as e: verbose_router_logger.error( f"Silent experiment failed for model {silent_model}: {str(e)}" @@ -1705,7 +1722,9 @@ class Router: and isinstance(fallback_item, ModelResponseStream) and hasattr(fallback_item, "usage") ): - self._combine_fallback_usage(fallback_item, complete_response_object_usage) + self._combine_fallback_usage( + fallback_item, complete_response_object_usage + ) yield fallback_item else: # If fallback returns a non-streaming response, yield None @@ -1825,13 +1844,11 @@ class Router: router_self._update_kwargs_before_fallbacks( model=model_group, kwargs=initial_kwargs ) - fallback_response = ( - router_self.function_with_fallbacks( - **initial_kwargs, - fallbacks=fallbacks, - context_window_fallbacks=context_window_fallbacks, - content_policy_fallbacks=content_policy_fallbacks, - ) + fallback_response = router_self.function_with_fallbacks( + **initial_kwargs, + fallbacks=fallbacks, + context_window_fallbacks=context_window_fallbacks, + content_policy_fallbacks=content_policy_fallbacks, ) if hasattr(fallback_response, "__iter__"): @@ -1841,7 +1858,9 @@ class Router: and isinstance(fallback_item, ModelResponseStream) and hasattr(fallback_item, "usage") ): - router_self._combine_fallback_usage(fallback_item, complete_response_object_usage) + router_self._combine_fallback_usage( + fallback_item, complete_response_object_usage + ) yield fallback_item else: yield None @@ -1891,6 +1910,8 @@ class Router: ) silent_kwargs = self._get_silent_experiment_kwargs(**kwargs) + # Override model_group to correctly attribute metrics to the silent model + silent_kwargs["metadata"]["model_group"] = silent_model # Trigger the silent request await self.acompletion( @@ -2753,10 +2774,9 @@ class Router: litellm_model = data.get("model", None) # litellm_agent/ prefix only strips the model name, no prompt_id needed - is_litellm_agent_model = ( - isinstance(litellm_model, str) - and litellm_model.startswith("litellm_agent/") - ) + is_litellm_agent_model = isinstance( + litellm_model, str + ) and litellm_model.startswith("litellm_agent/") prompt_id = kwargs.get("prompt_id") or prompt_management_deployment[ "litellm_params" @@ -6538,7 +6558,7 @@ class Router: tiers = complexity_router_config.get("tiers", {}) # Use MEDIUM tier as fallback default default_model = tiers.get("MEDIUM") or tiers.get("SIMPLE") - + if default_model is None: raise ValueError( "complexity_router_default_model is required for complexity-router deployments, " @@ -6771,7 +6791,9 @@ class Router: ######################################################### # Check if this is a complexity-router deployment ######################################################### - if self._is_complexity_router_deployment(litellm_params=deployment.litellm_params): + if self._is_complexity_router_deployment( + litellm_params=deployment.litellm_params + ): self.init_complexity_router_deployment(deployment=deployment) return deployment @@ -6863,9 +6885,7 @@ class Router: # zero-cost models, causing budget checks to block free models. _model_id = deployment.model_info.id if _model_id is not None: - _model_info_dict: dict = deployment.model_info.model_dump( - exclude_none=True - ) + _model_info_dict: dict = deployment.model_info.model_dump(exclude_none=True) for field in CustomPricingLiteLLMParams.model_fields.keys(): field_value = deployment.litellm_params.get(field) if field_value is not None: @@ -7157,7 +7177,10 @@ class Router: @overload def get_router_model_info( - self, deployment: Union[dict, "Deployment"], received_model_name: str, id: None = None + self, + deployment: Union[dict, "Deployment"], + received_model_name: str, + id: None = None, ) -> ModelMapInfo: pass @@ -7197,7 +7220,9 @@ class Router: ## GET BASE MODEL base_model = (deployment.get("model_info") or {}).get("base_model", None) if base_model is None: - base_model = (deployment.get("litellm_params") or {}).get("base_model", None) + base_model = (deployment.get("litellm_params") or {}).get( + "base_model", None + ) model = base_model @@ -7232,12 +7257,12 @@ class Router: if potential_models is not None: for potential_model in potential_models: try: - if (potential_model.get("model_info") or {}).get( - "id" - ) == (deployment.get("model_info") or {}).get("id"): - model = (potential_model.get("litellm_params") or {}).get( - "model" - ) + if (potential_model.get("model_info") or {}).get("id") == ( + deployment.get("model_info") or {} + ).get("id"): + model = ( + potential_model.get("litellm_params") or {} + ).get("model") break except Exception: pass @@ -8160,7 +8185,9 @@ class Router: - team_id: Optional[str] - the team id, to resolve team-specific models """ # Check if this is the no-args hot path (cacheable) - _use_cache = model_name is None and model_access_group is None and team_id is None + _use_cache = ( + model_name is None and model_access_group is None and team_id is None + ) # Return cached result for the no-args hot path if _use_cache and self._access_groups_cache is not None: diff --git a/tests/test_litellm/test_router_silent_experiment.py b/tests/test_litellm/test_router_silent_experiment.py index a23ea80f7ce..79056805d54 100644 --- a/tests/test_litellm/test_router_silent_experiment.py +++ b/tests/test_litellm/test_router_silent_experiment.py @@ -19,11 +19,20 @@ def test_get_silent_experiment_kwargs(): }, ] router = Router(model_list=model_list) - kwargs = {"metadata": {"foo": "bar"}, "litellm_call_id": "call-123"} + kwargs = { + "metadata": {"foo": "bar"}, + "litellm_call_id": "call-123", + "stream": True, + "proxy_server_request": {"body": {"model": "test"}}, + } result = router._get_silent_experiment_kwargs(**kwargs) assert result["metadata"]["is_silent_experiment"] is True assert result["metadata"]["foo"] == "bar" assert "litellm_call_id" not in result + # stream must be forced to False so callbacks fire in background + assert result["stream"] is False + # proxy_server_request must be preserved for spend log metadata + assert "proxy_server_request" in result def test_silent_experiment_completion_direct(): @@ -39,7 +48,7 @@ def test_silent_experiment_completion_direct(): ] router = Router(model_list=model_list) messages = [{"role": "user", "content": "hi"}] - with patch.object(router, "completion", return_value=None): + with patch.object(router, "acompletion", new_callable=AsyncMock, return_value=None): router._silent_experiment_completion( silent_model="gpt-3.5-turbo", messages=messages, @@ -173,12 +182,20 @@ def test_router_silent_experiment_completion(): router = Router(model_list=model_list) - # Mock litellm.completion + # Mock litellm.acompletion mock_response = litellm.ModelResponse(choices=[{"message": {"content": "hello"}}]) - mock_completion = MagicMock(return_value=mock_response) + + # We need an async mock for acompletion + async def mock_acompletion(*args, **kwargs): + return mock_response + + mock_acompletion_mock = AsyncMock(side_effect=mock_acompletion) + mock_completion_mock = MagicMock(return_value=mock_response) # Patch at the litellm module level - with patch.object(litellm, "completion", mock_completion): + with patch.object(litellm, "acompletion", mock_acompletion_mock), patch.object( + litellm, "completion", mock_completion_mock + ): response = router.completion( model="primary-model", messages=[{"role": "user", "content": "hi"}], @@ -189,12 +206,15 @@ def test_router_silent_experiment_completion(): # The sync background call uses a thread pool. We might need to wait a bit. import time - time.sleep(0.5) + time.sleep(2.0) - # Should have 2 calls - assert mock_completion.call_count == 2 + # Should have 1 acompletion call (the silent background call) + # The primary completion call still goes to the real litellm.completion (or we can mock it separately, but here it's testing the background one) + # Wait, the primary call in the test is router.completion. + # Actually, let's just mock both to avoid real network calls if it's hitting one. + assert mock_acompletion_mock.call_count == 1 - call_args_list = mock_completion.call_args_list + call_args_list = mock_acompletion_mock.call_args_list # Verify no silent_model in any call for call in call_args_list: @@ -212,3 +232,5 @@ def test_router_silent_experiment_completion(): ) assert silent_call is not None assert silent_call[1]["model"] == "openai/gpt-4" + # Verify model_group is set to the silent model name for correct metric attribution + assert silent_call[1]["metadata"]["model_group"] == "silent-model" From afb117f5d325b7c4f3a5afb12a08f953d8eaaf7e Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 10 Mar 2026 13:03:29 +0530 Subject: [PATCH 16/62] fix: req changes from greptile --- litellm/router.py | 21 ++++++++++--------- .../test_router_silent_experiment.py | 8 ++----- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index fac9409b359..06def6ceb4d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1501,20 +1501,21 @@ class Router: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: - loop.run_until_complete( - self.acompletion( + + async def _run_silent_completion(): + await self.acompletion( model=silent_model, messages=cast(List[AllMessageValues], messages), **silent_kwargs, ) - ) - # Drain any remaining fire-and-forget tasks (e.g. alerting hooks) - # scheduled via asyncio.create_task during the acompletion call. - pending = asyncio.all_tasks(loop) - if pending: - loop.run_until_complete( - asyncio.gather(*pending, return_exceptions=True) - ) + # Drain any fire-and-forget tasks (e.g. alerting hooks) + # scheduled via asyncio.create_task during acompletion. + pending = asyncio.all_tasks() + pending.discard(asyncio.current_task()) + if pending: + await asyncio.gather(*pending, return_exceptions=True) + + loop.run_until_complete(_run_silent_completion()) finally: loop.close() except Exception as e: diff --git a/tests/test_litellm/test_router_silent_experiment.py b/tests/test_litellm/test_router_silent_experiment.py index 79056805d54..67d262f83d4 100644 --- a/tests/test_litellm/test_router_silent_experiment.py +++ b/tests/test_litellm/test_router_silent_experiment.py @@ -1,4 +1,5 @@ import asyncio +import time from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -203,15 +204,10 @@ def test_router_silent_experiment_completion(): assert response.choices[0].message.content == "hello" - # The sync background call uses a thread pool. We might need to wait a bit. - import time - + # The sync background call uses a thread pool. We might need to wait. time.sleep(2.0) # Should have 1 acompletion call (the silent background call) - # The primary completion call still goes to the real litellm.completion (or we can mock it separately, but here it's testing the background one) - # Wait, the primary call in the test is router.completion. - # Actually, let's just mock both to avoid real network calls if it's hitting one. assert mock_acompletion_mock.call_count == 1 call_args_list = mock_acompletion_mock.call_args_list From 861db111a08ecd58d825ef2e86cf07d52374b008 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 3 Mar 2026 11:19:12 +0530 Subject: [PATCH 17/62] fix: presidio edge case with antropic handle on pii token leak --- .../guardrails/guardrail_hooks/presidio.py | 33 ++++-- .../guardrail_hooks/test_presidio.py | 103 ++++++++++++++++++ 2 files changed, 127 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index ce32ebf54f8..3d2ab334402 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -100,6 +100,16 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): self.mock_redacted_text = mock_redacted_text self.output_parse_pii = output_parse_pii or False self.apply_to_output = apply_to_output + + # When output_parse_pii or apply_to_output is enabled, the guardrail must + # also run on post_call to unmask/mask the response. Expand the event_hook + # so should_run_guardrail returns True for both pre_call and post_call. + if (self.output_parse_pii or self.apply_to_output) and not logging_only: + current_hook = self.event_hook + if isinstance(current_hook, str) and current_hook == "pre_call": + self.event_hook = ["pre_call", "post_call"] + elif isinstance(current_hook, list) and "post_call" not in current_hook: + self.event_hook = current_hook + ["post_call"] self.pii_entities_config: Dict[Union[PiiEntityType, str], PiiAction] = ( pii_entities_config or {} ) @@ -489,9 +499,13 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): "This may indicate a missing caller update." ) request_data = {} - if "pii_tokens" not in request_data: - request_data["pii_tokens"] = {} - pii_tokens = request_data["pii_tokens"] + # Store pii_tokens in metadata to avoid leaking to LLM providers. + # Providers like Anthropic reject unknown top-level fields. + if "metadata" not in request_data: + request_data["metadata"] = {} + if "pii_tokens" not in request_data["metadata"]: + request_data["metadata"]["pii_tokens"] = {} + pii_tokens = request_data["metadata"]["pii_tokens"] # Always append a UUID to ensure the replacement token is unique to this request and session. # This prevents collisions where the LLM might hallucinate a generic token like [PHONE_NUMBER]. @@ -544,8 +558,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): filtered_results: List[PresidioAnalyzeResponseItem] = [] deny_list_strings = [ - getattr(x, "value", str(x)) - for x in self.presidio_entities_deny_list + getattr(x, "value", str(x)) for x in self.presidio_entities_deny_list ] for item in analyze_results: entity_type = item.get("entity_type") @@ -937,10 +950,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): Helper to recursively process a ModelResponse for PII. Handles all choices and tool calls. """ - pii_tokens = request_data.get("pii_tokens", {}) if request_data else {} + metadata = request_data.get("metadata", {}) if request_data else {} + pii_tokens = metadata.get("pii_tokens", {}) if not pii_tokens and mode == "unmask": verbose_proxy_logger.debug( - "No pii_tokens found in request_data — nothing to unmask" + "No pii_tokens found in request_data['metadata'] — nothing to unmask" ) presidio_config = self.get_presidio_settings_from_request_data( request_data or {} @@ -1099,10 +1113,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return # --- PII unmasking path (output_parse_pii=True) --- - pii_tokens = request_data.get("pii_tokens", {}) if request_data else {} + metadata = request_data.get("metadata", {}) if request_data else {} + pii_tokens = metadata.get("pii_tokens", {}) if not pii_tokens and request_data: verbose_proxy_logger.debug( - "No pii_tokens in request_data for streaming unmask path" + "No pii_tokens in request_data['metadata'] for streaming unmask path" ) if not (self.output_parse_pii and pii_tokens): async for chunk in response: diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 76f9c39acd0..e336f2f1bf8 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -1608,3 +1608,106 @@ async def test_anonymize_text_http_error_status(): output_parse_pii=False, masked_entity_count={}, ) + + +@pytest.mark.asyncio +async def test_pii_tokens_stored_in_metadata_not_top_level(presidio_guardrail): + """ + Regression test: pii_tokens must be stored in data['metadata']['pii_tokens'], + NOT in data['pii_tokens']. Storing at the top level leaks the field to LLM + providers like Anthropic, which reject unknown fields with + 'pii_tokens: Extra inputs are not permitted'. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + output_parse_pii=True, + pii_entities_config={ + PiiEntityType.PERSON: PiiAction.MASK, + PiiEntityType.PHONE_NUMBER: PiiAction.MASK, + }, + ) + + mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + mock_cache = DualCache() + + test_data = { + "messages": [ + {"role": "user", "content": "My name is John and my phone is 555-123-4567"} + ], + "model": "claude-haiku-4-5-20251001", + "metadata": {}, + } + + async def mock_check_pii(text, output_parse_pii, presidio_config, request_data): + # Simulate PII masking with token storage (mimics real anonymize_text behavior) + import uuid + + if request_data is not None and output_parse_pii: + if "metadata" not in request_data: + request_data["metadata"] = {} + if "pii_tokens" not in request_data["metadata"]: + request_data["metadata"]["pii_tokens"] = {} + pii_tokens = request_data["metadata"]["pii_tokens"] + token = f"_{str(uuid.uuid4())[:12]}" + pii_tokens[token] = "John" + text = text.replace("John", token) + return text + + guardrail.check_pii = mock_check_pii + + result = await guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key, + cache=mock_cache, + data=test_data, + call_type="completion", + ) + + # pii_tokens must NOT be at the top level of data (would leak to providers) + assert "pii_tokens" not in result, ( + "pii_tokens must not be a top-level key in request data — " + "it would leak to LLM providers and cause 'Extra inputs are not permitted' errors" + ) + + # pii_tokens must be inside metadata (safe from provider leakage) + assert "metadata" in result + assert "pii_tokens" in result["metadata"] + assert len(result["metadata"]["pii_tokens"]) > 0 + + +@pytest.mark.asyncio +async def test_pii_tokens_in_metadata_used_for_unmasking(): + """ + Regression test: _process_response_for_pii must read pii_tokens from + data['metadata']['pii_tokens'] and correctly unmask the response. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + output_parse_pii=True, + ) + + token_key = "_abc123def456" + request_data = { + "model": "claude-haiku-4-5-20251001", + "metadata": {"pii_tokens": {token_key: "John"}}, + } + + response = ModelResponse( + choices=[ + Choices( + message=Message( + role="assistant", + content=f"Hello {token_key}, how can I help you?", + ), + index=0, + finish_reason="stop", + ) + ] + ) + + await guardrail._process_response_for_pii( + response=response, + request_data=request_data, + mode="unmask", + ) + + assert response.choices[0].message.content == "Hello John, how can I help you?" From 1ba42d1d994177f5009fd33cb647f4b0018c311f Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Sat, 7 Mar 2026 13:16:01 +0530 Subject: [PATCH 18/62] fix: address req changes --- tests/guardrails_tests/test_presidio_pii.py | 226 ++++++++++---------- 1 file changed, 109 insertions(+), 117 deletions(-) diff --git a/tests/guardrails_tests/test_presidio_pii.py b/tests/guardrails_tests/test_presidio_pii.py index 0d730288e63..eda0c7bb5b5 100644 --- a/tests/guardrails_tests/test_presidio_pii.py +++ b/tests/guardrails_tests/test_presidio_pii.py @@ -1,20 +1,19 @@ import sys import os -import io, asyncio import pytest -import time from litellm import mock_completion -from unittest.mock import MagicMock, AsyncMock, patch +from unittest.mock import patch + sys.path.insert(0, os.path.abspath("../..")) import litellm -from litellm.proxy.guardrails.guardrail_hooks.presidio import _OPTIONAL_PresidioPIIMasking, PresidioPerRequestConfig +from litellm.proxy.guardrails.guardrail_hooks.presidio import ( + _OPTIONAL_PresidioPIIMasking, + PresidioPerRequestConfig, +) from litellm.types.guardrails import PiiEntityType, PiiAction from litellm.proxy._types import UserAPIKeyAuth from litellm.caching.caching import DualCache from litellm.exceptions import BlockedPiiEntityError -from litellm.types.utils import CallTypes as LitellmCallTypes - - @pytest.mark.asyncio @@ -26,42 +25,37 @@ async def test_presidio_with_entities_config(): PiiEntityType.CREDIT_CARD: PiiAction.MASK, PiiEntityType.EMAIL_ADDRESS: PiiAction.MASK, } - + presidio_guardrail = _OPTIONAL_PresidioPIIMasking( pii_entities_config=pii_entities_config, presidio_analyzer_api_base=os.environ.get("PRESIDIO_ANALYZER_API_BASE"), - presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE") + presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE"), ) - + # Test text with different PII types test_text = "My credit card number is 4111-1111-1111-1111, my email is test@example.com, and my phone is 555-123-4567" - + # Test the analyze request configuration analyze_request = presidio_guardrail._get_presidio_analyze_request_payload( - text=test_text, - presidio_config=None, - request_data={} + text=test_text, presidio_config=None, request_data={} ) - + # Verify entities were passed correctly assert "entities" in analyze_request assert set(analyze_request["entities"]) == set(pii_entities_config.keys()) - + # Test the check_pii method - this will call the actual Presidio API redacted_text = await presidio_guardrail.check_pii( - text=test_text, - output_parse_pii=True, - presidio_config=None, - request_data={} + text=test_text, output_parse_pii=True, presidio_config=None, request_data={} ) - + # Verify PII has been masked/replaced/redacted in the result assert "4111-1111-1111-1111" not in redacted_text assert "test@example.com" not in redacted_text # Since this entity is not in the config, it should not be masked assert "555-123-4567" in redacted_text - + # The specific replacements will vary based on Presidio's implementation print(f"Redacted text: {redacted_text}") @@ -73,10 +67,12 @@ async def test_presidio_apply_guardrail(): presidio_guardrail = _OPTIONAL_PresidioPIIMasking( pii_entities_config={}, presidio_analyzer_api_base=os.environ.get("PRESIDIO_ANALYZER_API_BASE"), - presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE") + presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE"), ) - test_text = "My credit card number is 4111-1111-1111-1111 and my email is test@example.com" + test_text = ( + "My credit card number is 4111-1111-1111-1111 and my email is test@example.com" + ) response = await presidio_guardrail.apply_guardrail( inputs={"texts": [test_text]}, request_data={}, @@ -91,6 +87,7 @@ async def test_presidio_apply_guardrail(): assert "4111-1111-1111-1111" not in modified_text assert "test@example.com" not in modified_text + @pytest.mark.asyncio async def test_presidio_with_blocked_entities(): """Test for Presidio guardrail with blocked entities - requires actual Presidio API""" @@ -100,36 +97,33 @@ async def test_presidio_with_blocked_entities(): PiiEntityType.CREDIT_CARD: PiiAction.BLOCK, # This entity should cause a block PiiEntityType.EMAIL_ADDRESS: PiiAction.MASK, # This entity should be masked } - + presidio_guardrail = _OPTIONAL_PresidioPIIMasking( pii_entities_config=pii_entities_config, presidio_analyzer_api_base=os.environ.get("PRESIDIO_ANALYZER_API_BASE"), - presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE") + presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE"), ) - + # Test text with blocked PII type - test_text = "My credit card number is 4111-1111-1111-1111 and my email is test@example.com" - + test_text = ( + "My credit card number is 4111-1111-1111-1111 and my email is test@example.com" + ) + # Verify the analyze request configuration analyze_request = presidio_guardrail._get_presidio_analyze_request_payload( - text=test_text, - presidio_config=None, - request_data={} + text=test_text, presidio_config=None, request_data={} ) - + # Verify entities were passed correctly assert "entities" in analyze_request assert set(analyze_request["entities"]) == set(pii_entities_config.keys()) - + # Test that BlockedPiiEntityError is raised when check_pii is called with pytest.raises(BlockedPiiEntityError) as excinfo: await presidio_guardrail.check_pii( - text=test_text, - output_parse_pii=True, - presidio_config=None, - request_data={} + text=test_text, output_parse_pii=True, presidio_config=None, request_data={} ) - + # Verify the error contains the correct entity type assert excinfo.value.entity_type == PiiEntityType.CREDIT_CARD assert excinfo.value.guardrail_name == presidio_guardrail.guardrail_name @@ -143,37 +137,40 @@ async def test_presidio_pre_call_hook_with_blocked_entities(): PiiEntityType.CREDIT_CARD: PiiAction.BLOCK, # This entity should cause a block PiiEntityType.EMAIL_ADDRESS: PiiAction.MASK, # This entity should be masked } - + presidio_guardrail = _OPTIONAL_PresidioPIIMasking( pii_entities_config=pii_entities_config, presidio_analyzer_api_base=os.environ.get("PRESIDIO_ANALYZER_API_BASE"), - presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE") + presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE"), ) - + # Create a sample chat completion request with PII data data = { "messages": [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "My credit card is 4111-1111-1111-1111 and my email is test@example.com."} + { + "role": "user", + "content": "My credit card is 4111-1111-1111-1111 and my email is test@example.com.", + }, ], - "model": "gpt-3.5-turbo" + "model": "gpt-3.5-turbo", } - + # Mock objects needed for the pre-call hook user_api_key_dict = UserAPIKeyAuth(api_key="test_key") cache = DualCache() - + # Call the pre-call hook and expect BlockedPiiEntityError with pytest.raises(BlockedPiiEntityError) as excinfo: await presidio_guardrail.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, data=data, - call_type="completion" + call_type="completion", ) - + print(f"got error: {excinfo}") - + # Verify the error contains the correct entity type assert excinfo.value.entity_type == PiiEntityType.CREDIT_CARD assert excinfo.value.guardrail_name == presidio_guardrail.guardrail_name @@ -188,44 +185,46 @@ async def test_presidio_pre_call_hook_with_different_call_types(call_type): PiiEntityType.CREDIT_CARD: PiiAction.MASK, PiiEntityType.EMAIL_ADDRESS: PiiAction.MASK, } - + presidio_guardrail = _OPTIONAL_PresidioPIIMasking( pii_entities_config=pii_entities_config, presidio_analyzer_api_base=os.environ.get("PRESIDIO_ANALYZER_API_BASE"), - presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE") + presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE"), ) - + # Create a sample request with PII data data = { "messages": [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "My credit card is 4111-1111-1111-1111 and my email is test@example.com. My phone number is 555-123-4567"} + { + "role": "user", + "content": "My credit card is 4111-1111-1111-1111 and my email is test@example.com. My phone number is 555-123-4567", + }, ], - "model": "gpt-3.5-turbo" + "model": "gpt-3.5-turbo", } - + # Mock objects needed for the pre-call hook user_api_key_dict = UserAPIKeyAuth(api_key="test_key") cache = DualCache() - + # Call the pre-call hook with the specified call type modified_data = await presidio_guardrail.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=cache, - data=data, - call_type=call_type + user_api_key_dict=user_api_key_dict, cache=cache, data=data, call_type=call_type ) - + # Verify the messages have been modified to mask PII - assert modified_data["messages"][0]["content"] == "You are a helpful assistant." # System prompt should be unchanged - + assert ( + modified_data["messages"][0]["content"] == "You are a helpful assistant." + ) # System prompt should be unchanged + user_message = modified_data["messages"][1]["content"] assert "4111-1111-1111-1111" not in user_message assert "test@example.com" not in user_message # Since this entity is not in the config, it should not be masked assert "555-123-4567" in user_message - + print(f"Modified user message for call_type={call_type}: {user_message}") @@ -243,7 +242,7 @@ def test_validate_environment_missing_http(base_url): # Use patch.dict to temporarily modify environment variables only for this test env_vars = { "PRESIDIO_ANALYZER_API_BASE": f"{base_url}/analyze", - "PRESIDIO_ANONYMIZER_API_BASE": f"{base_url}/anonymize" + "PRESIDIO_ANONYMIZER_API_BASE": f"{base_url}/anonymize", } with patch.dict(os.environ, env_vars): pii_masking.validate_environment() @@ -294,8 +293,12 @@ async def test_output_parsing(): new_response = await pii_masking.async_post_call_success_hook( user_api_key_dict=UserAPIKeyAuth(), data={ - "messages": [{"role": "system", "content": "You are an helpfull assistant"}], - "pii_tokens": {"": "Jane Doe", "": "034453334"}, + "messages": [ + {"role": "system", "content": "You are an helpfull assistant"} + ], + "metadata": { + "pii_tokens": {"": "Jane Doe", "": "034453334"} + }, }, response=response, ) @@ -440,24 +443,26 @@ async def test_presidio_pii_masking_logging_output_only_no_pre_api_hook(): @pytest.mark.asyncio -@patch.dict(os.environ, { - "PRESIDIO_ANALYZER_API_BASE": "http://localhost:5002", - "PRESIDIO_ANONYMIZER_API_BASE": "http://localhost:5001" -}) +@patch.dict( + os.environ, + { + "PRESIDIO_ANALYZER_API_BASE": "http://localhost:5002", + "PRESIDIO_ANONYMIZER_API_BASE": "http://localhost:5001", + }, +) async def test_presidio_pii_masking_logging_output_only_logged_response_guardrails_config(): from typing import Dict, List, Optional import litellm from litellm.proxy.guardrails.init_guardrails import initialize_guardrails from litellm.types.guardrails import ( - GuardrailItem, GuardrailItemSpec, GuardrailEventHooks, ) litellm.set_verbose = True # Environment variables are now patched via the decorator instead of setting them directly - + guardrails_config: List[Dict[str, GuardrailItemSpec]] = [ { "pii_masking": { @@ -499,60 +504,53 @@ async def test_presidio_pii_masking_logging_output_only_logged_response_guardrai async def test_presidio_language_configuration(): """Test that presidio_language parameter is properly set and used in analyze requests""" litellm._turn_on_debug() - + # Test with German language using mock testing to avoid API calls presidio_guardrail_de = _OPTIONAL_PresidioPIIMasking( pii_entities_config={}, presidio_language="de", - mock_testing=True # This bypasses the API validation + mock_testing=True, # This bypasses the API validation ) - + test_text = "Meine Telefonnummer ist +49 30 12345678" - + # Test the analyze request configuration analyze_request = presidio_guardrail_de._get_presidio_analyze_request_payload( - text=test_text, - presidio_config=None, - request_data={} + text=test_text, presidio_config=None, request_data={} ) - + # Verify the language is set to German assert analyze_request["language"] == "de" assert analyze_request["text"] == test_text - + # Test with Spanish language presidio_guardrail_es = _OPTIONAL_PresidioPIIMasking( - pii_entities_config={}, - presidio_language="es", - mock_testing=True + pii_entities_config={}, presidio_language="es", mock_testing=True ) - + test_text_es = "Mi número de teléfono es +34 912 345 678" - + analyze_request_es = presidio_guardrail_es._get_presidio_analyze_request_payload( - text=test_text_es, - presidio_config=None, - request_data={} + text=test_text_es, presidio_config=None, request_data={} ) - + # Verify the language is set to Spanish assert analyze_request_es["language"] == "es" assert analyze_request_es["text"] == test_text_es - + # Test default language (English) when not specified presidio_guardrail_default = _OPTIONAL_PresidioPIIMasking( - pii_entities_config={}, - mock_testing=True + pii_entities_config={}, mock_testing=True ) - + test_text_en = "My phone number is +1 555-123-4567" - - analyze_request_default = presidio_guardrail_default._get_presidio_analyze_request_payload( - text=test_text_en, - presidio_config=None, - request_data={} + + analyze_request_default = ( + presidio_guardrail_default._get_presidio_analyze_request_payload( + text=test_text_en, presidio_config=None, request_data={} + ) ) - + # Verify the language defaults to English assert analyze_request_default["language"] == "en" assert analyze_request_default["text"] == test_text_en @@ -562,36 +560,30 @@ async def test_presidio_language_configuration(): async def test_presidio_language_configuration_with_per_request_override(): """Test that per-request language configuration overrides the default configured language""" litellm._turn_on_debug() - + # Set up guardrail with German as default language presidio_guardrail = _OPTIONAL_PresidioPIIMasking( - pii_entities_config={}, - presidio_language="de", - mock_testing=True + pii_entities_config={}, presidio_language="de", mock_testing=True ) - + test_text = "Test text with PII" - + # Test with per-request config overriding the default language presidio_config = PresidioPerRequestConfig(language="fr") - + analyze_request = presidio_guardrail._get_presidio_analyze_request_payload( - text=test_text, - presidio_config=presidio_config, - request_data={} + text=test_text, presidio_config=presidio_config, request_data={} ) - + # Verify the per-request language (French) overrides the default (German) assert analyze_request["language"] == "fr" assert analyze_request["text"] == test_text - + # Test without per-request config - should use default language analyze_request_default = presidio_guardrail._get_presidio_analyze_request_payload( - text=test_text, - presidio_config=None, - request_data={} + text=test_text, presidio_config=None, request_data={} ) - + # Verify the default language (German) is used assert analyze_request_default["language"] == "de" assert analyze_request_default["text"] == test_text From 1bfd88a33caec6c16aab4d7a0847c33640376781 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Sat, 7 Mar 2026 13:30:09 +0530 Subject: [PATCH 19/62] fix: req changes --- .../guardrails/guardrail_hooks/presidio.py | 4 +-- .../guardrail_hooks/test_presidio.py | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 3d2ab334402..e749b167c75 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -106,8 +106,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): # so should_run_guardrail returns True for both pre_call and post_call. if (self.output_parse_pii or self.apply_to_output) and not logging_only: current_hook = self.event_hook - if isinstance(current_hook, str) and current_hook == "pre_call": - self.event_hook = ["pre_call", "post_call"] + if isinstance(current_hook, str) and current_hook != "post_call": + self.event_hook = [current_hook, "post_call"] elif isinstance(current_hook, list) and "post_call" not in current_hook: self.event_hook = current_hook + ["post_call"] self.pii_entities_config: Dict[Union[PiiEntityType, str], PiiAction] = ( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index e336f2f1bf8..55137176a60 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -1711,3 +1711,34 @@ async def test_pii_tokens_in_metadata_used_for_unmasking(): ) assert response.choices[0].message.content == "Hello John, how can I help you?" + + +@pytest.mark.parametrize( + "initial_hook", + ["pre_call", "during_call", "pre_mcp_call"], +) +def test_event_hook_auto_expansion_for_all_string_hooks(initial_hook): + """ + Regression test: when output_parse_pii is True, the guardrail must add + 'post_call' to event_hook regardless of the initial string hook value, + not just when it's 'pre_call'. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + output_parse_pii=True, + event_hook=initial_hook, + ) + assert isinstance(guardrail.event_hook, list) + assert initial_hook in guardrail.event_hook + assert "post_call" in guardrail.event_hook + + +def test_event_hook_no_expansion_when_already_post_call(): + """post_call alone should stay as-is — no expansion needed.""" + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + output_parse_pii=True, + event_hook="post_call", + ) + # Should remain a string "post_call", not expanded to a list + assert guardrail.event_hook == "post_call" From a28fbba3b17d5a4014f5624361b4ad6740b2c467 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Sat, 7 Mar 2026 13:48:00 +0530 Subject: [PATCH 20/62] fix: req changes to improve score --- litellm/proxy/guardrails/guardrail_hooks/presidio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index e749b167c75..a3138b417dc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -950,7 +950,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): Helper to recursively process a ModelResponse for PII. Handles all choices and tool calls. """ - metadata = request_data.get("metadata", {}) if request_data else {} + metadata = (request_data.get("metadata") or {}) if request_data else {} pii_tokens = metadata.get("pii_tokens", {}) if not pii_tokens and mode == "unmask": verbose_proxy_logger.debug( @@ -1113,7 +1113,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return # --- PII unmasking path (output_parse_pii=True) --- - metadata = request_data.get("metadata", {}) if request_data else {} + metadata = (request_data.get("metadata") or {}) if request_data else {} pii_tokens = metadata.get("pii_tokens", {}) if not pii_tokens and request_data: verbose_proxy_logger.debug( From 14ecc79760b50d5d857923840cd036e46b64b8ff Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Sat, 7 Mar 2026 14:00:26 +0530 Subject: [PATCH 21/62] fix: req changes greptile hallucinates --- .../guardrails/guardrail_hooks/presidio.py | 2 +- .../guardrail_hooks/test_presidio.py | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index a3138b417dc..ec139020704 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -501,7 +501,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): request_data = {} # Store pii_tokens in metadata to avoid leaking to LLM providers. # Providers like Anthropic reject unknown top-level fields. - if "metadata" not in request_data: + if not request_data.get("metadata"): request_data["metadata"] = {} if "pii_tokens" not in request_data["metadata"]: request_data["metadata"]["pii_tokens"] = {} diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 55137176a60..57f55fadb22 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -1742,3 +1742,47 @@ def test_event_hook_no_expansion_when_already_post_call(): ) # Should remain a string "post_call", not expanded to a list assert guardrail.event_hook == "post_call" + + +@pytest.mark.asyncio +async def test_metadata_none_does_not_crash(): + """ + Regression test: if metadata is explicitly None in request_data, + the guardrail must not crash with TypeError on the write or read path. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + output_parse_pii=True, + ) + + token_key = "_abc123def456" + # metadata explicitly None — must not crash + request_data = { + "model": "gpt-3.5-turbo", + "metadata": None, + } + + response = ModelResponse( + choices=[ + Choices( + message=Message( + role="assistant", + content=f"Hello {token_key}, how can I help you?", + ), + index=0, + finish_reason="stop", + ) + ] + ) + + # Should not raise TypeError + await guardrail._process_response_for_pii( + response=response, + request_data=request_data, + mode="unmask", + ) + + # No pii_tokens to unmask, so content stays as-is + assert ( + response.choices[0].message.content == f"Hello {token_key}, how can I help you?" + ) From 12de8a724f578a68d83529839b6b2718e6df4a98 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Sun, 8 Mar 2026 05:43:54 +0530 Subject: [PATCH 22/62] fix: clean approach instead of UUID --- .../guardrails/guardrail_hooks/presidio.py | 27 +++++--- .../guardrail_hooks/test_presidio.py | 67 +++++++++++++++++-- 2 files changed, 78 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index ec139020704..8496a63ec31 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -36,7 +36,6 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm._uuid import uuid from litellm.caching.caching import DualCache from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import ( @@ -490,8 +489,6 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): end = item["end"] replacement = item["text"] # replacement token if item["operator"] == "replace" and output_parse_pii is True: - # check if token in dict - # if exists, add a uuid to the replacement token for swapping back to the original text in llm response output parsing if request_data is None: verbose_proxy_logger.warning( "Presidio anonymize_text called without request_data — " @@ -507,9 +504,16 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): request_data["metadata"]["pii_tokens"] = {} pii_tokens = request_data["metadata"]["pii_tokens"] - # Always append a UUID to ensure the replacement token is unique to this request and session. - # This prevents collisions where the LLM might hallucinate a generic token like [PHONE_NUMBER]. - replacement = f"{replacement}_{str(uuid.uuid4())[:12]}" + # Append a sequential number to make each token unique + # per request, so unmasking maps back to the correct + # original value. Format: , + # This is LLM-friendly and degrades gracefully if the + # LLM doesn't echo the token verbatim. + seq = len(pii_tokens) + 1 + if replacement.endswith(">"): + replacement = f"{replacement[:-1]}_{seq}>" + else: + replacement = f"{replacement}_{seq}" pii_tokens[replacement] = new_text[ start:end @@ -521,12 +525,13 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): masked_entity_count[entity_type] = ( masked_entity_count.get(entity_type, 0) + 1 ) - # When output_parse_pii is True, new_text contains UUID-suffixed - # tokens that match the keys in pii_tokens. Returning - # redacted_text["text"] (Presidio's original output) would send - # un-suffixed tokens to the LLM, making unmasking impossible. + # When output_parse_pii is True, new_text contains sequentially + # numbered tokens (e.g. ) that match the keys + # in pii_tokens. Returning redacted_text["text"] (Presidio's + # original output) would send un-numbered tokens to the LLM, + # making unmasking impossible. # When output_parse_pii is False, new_text == redacted_text["text"] - # because no UUID suffix is appended. + # because no suffix is appended. return new_text else: raise Exception("Invalid anonymizer response: received None") diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 57f55fadb22..6b3fdf2a81f 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -1640,15 +1640,14 @@ async def test_pii_tokens_stored_in_metadata_not_top_level(presidio_guardrail): async def mock_check_pii(text, output_parse_pii, presidio_config, request_data): # Simulate PII masking with token storage (mimics real anonymize_text behavior) - import uuid - if request_data is not None and output_parse_pii: if "metadata" not in request_data: request_data["metadata"] = {} if "pii_tokens" not in request_data["metadata"]: request_data["metadata"]["pii_tokens"] = {} pii_tokens = request_data["metadata"]["pii_tokens"] - token = f"_{str(uuid.uuid4())[:12]}" + seq = len(pii_tokens) + 1 + token = f"" pii_tokens[token] = "John" text = text.replace("John", token) return text @@ -1685,7 +1684,7 @@ async def test_pii_tokens_in_metadata_used_for_unmasking(): output_parse_pii=True, ) - token_key = "_abc123def456" + token_key = "" request_data = { "model": "claude-haiku-4-5-20251001", "metadata": {"pii_tokens": {token_key: "John"}}, @@ -1755,7 +1754,7 @@ async def test_metadata_none_does_not_crash(): output_parse_pii=True, ) - token_key = "_abc123def456" + token_key = "" # metadata explicitly None — must not crash request_data = { "model": "gpt-3.5-turbo", @@ -1786,3 +1785,61 @@ async def test_metadata_none_does_not_crash(): assert ( response.choices[0].message.content == f"Hello {token_key}, how can I help you?" ) + + +# --------------------------------------------------------------------------- +# Tests for sequential-numbered token unmasking in _unmask_pii_text +# --------------------------------------------------------------------------- + + +def test_unmask_exact_match_with_sequential_tokens(): + """ + Normal unmasking: LLM echoes numbered tokens verbatim → original PII restored. + """ + from litellm.proxy.guardrails.guardrail_hooks.presidio import ( + _OPTIONAL_PresidioPIIMasking, + ) + + pii_tokens = { + "": "John Smith", + "": "555-123-4567", + } + text = "Hello , your number is ." + result = _OPTIONAL_PresidioPIIMasking._unmask_pii_text(text, pii_tokens) + assert result == "Hello John Smith, your number is 555-123-4567." + + +def test_unmask_multiple_same_entity_type(): + """ + Two phone numbers get distinct numbered tokens and unmask correctly. + """ + from litellm.proxy.guardrails.guardrail_hooks.presidio import ( + _OPTIONAL_PresidioPIIMasking, + ) + + pii_tokens = { + "": "555-111-0000", + "": "555-222-0000", + } + text = "Call or ." + result = _OPTIONAL_PresidioPIIMasking._unmask_pii_text(text, pii_tokens) + assert result == "Call 555-111-0000 or 555-222-0000." + + +def test_unmask_graceful_degradation(): + """ + If the LLM doesn't echo the token back, the numbered label stays + in the output — clean and readable, not garbage hex. + """ + from litellm.proxy.guardrails.guardrail_hooks.presidio import ( + _OPTIONAL_PresidioPIIMasking, + ) + + pii_tokens = { + "": "John", + } + # LLM paraphrased instead of echoing the token + text = "I see you provided a name." + result = _OPTIONAL_PresidioPIIMasking._unmask_pii_text(text, pii_tokens) + # No change — no garbage, just clean text + assert result == text From 212059cd118029f5f65df2883ad662f02730c6b9 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 10 Mar 2026 16:33:41 +0530 Subject: [PATCH 23/62] fix: presidio improvements --- .../guardrails/guardrail_hooks/presidio.py | 106 +++++- .../guardrail_hooks/test_presidio.py | 342 ++++++++++++++++++ 2 files changed, 435 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 8496a63ec31..507a0b89fc9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -484,7 +484,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): new_text = text if redacted_text is not None: verbose_proxy_logger.debug("redacted_text: %s", redacted_text) - for item in redacted_text["items"]: + # Process items in reverse order by start position so that + # replacing later spans first does not shift earlier coordinates. + for item in sorted( + redacted_text["items"], key=lambda x: x["start"], reverse=True + ): start = item["start"] end = item["end"] replacement = item["text"] # replacement token @@ -515,9 +519,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): else: replacement = f"{replacement}_{seq}" - pii_tokens[replacement] = new_text[ - start:end - ] # get text it'll replace + # Use ORIGINAL text (not new_text) since start/end + # reference the original text's coordinates. + pii_tokens[replacement] = text[start:end] new_text = new_text[:start] + replacement + new_text[end:] entity_type = item.get("entity_type", None) @@ -902,6 +906,10 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) if self.apply_to_output is True: + if self._is_anthropic_message_response(response): + return await self._process_anthropic_response_for_pii( + response=response, request_data=data, mode="mask" + ) return await self._mask_output_response( response=response, request_data=data ) @@ -917,6 +925,10 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): request_data=data, mode="unmask", ) + elif self._is_anthropic_message_response(response): + await self._process_anthropic_response_for_pii( + response=response, request_data=data, mode="unmask" + ) return response @staticmethod @@ -945,6 +957,57 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): break return text + @staticmethod + def _is_anthropic_message_response(response: Any) -> bool: + """Check if the response is an Anthropic native message dict.""" + return ( + isinstance(response, dict) + and response.get("type") == "message" + and isinstance(response.get("content"), list) + ) + + async def _process_anthropic_response_for_pii( + self, + response: dict, + request_data: dict, + mode: Literal["mask", "unmask"], + ) -> dict: + """ + Process an Anthropic native message dict for PII masking/unmasking. + Handles content blocks with type == "text". + """ + metadata = (request_data.get("metadata") or {}) if request_data else {} + pii_tokens = metadata.get("pii_tokens", {}) + if not pii_tokens and mode == "unmask": + verbose_proxy_logger.debug( + "No pii_tokens in metadata for Anthropic response unmask" + ) + presidio_config = self.get_presidio_settings_from_request_data( + request_data or {} + ) + + content = response.get("content") + if not isinstance(content, list): + return response + + for block in content: + if not isinstance(block, dict) or block.get("type") != "text": + continue + text_value = block.get("text") + if text_value is None: + continue + if mode == "unmask": + block["text"] = self._unmask_pii_text(text_value, pii_tokens) + elif mode == "mask": + block["text"] = await self.check_pii( + text=text_value, + output_parse_pii=False, + presidio_config=presidio_config, + request_data=request_data, + ) + + return response + async def _process_response_for_pii( self, response: ModelResponse, @@ -1064,7 +1127,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): user_api_key_dict: UserAPIKeyAuth, response: Any, request_data: dict, - ) -> AsyncGenerator[ModelResponseStream, None]: + ) -> AsyncGenerator[Union[ModelResponseStream, bytes], None]: """ Process streaming response chunks to unmask PII tokens when needed. """ @@ -1081,6 +1144,10 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): async for chunk in response: if isinstance(chunk, ModelResponseStream): all_chunks.append(chunk) + elif isinstance(chunk, bytes): + # Anthropic native SSE: pass through as-is + yield chunk # type: ignore[misc] + continue if not all_chunks: return @@ -1134,6 +1201,10 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): async for chunk in response: if isinstance(chunk, ModelResponseStream): remaining_chunks.append(chunk) + elif isinstance(chunk, bytes): + # Anthropic native SSE: pass through as-is + yield chunk # type: ignore[misc] + continue if not remaining_chunks: return @@ -1211,15 +1282,24 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): """ texts = inputs.get("texts", []) + # When input_type is "response" and pii_tokens are available, + # unmask the text instead of masking it. + metadata = (request_data.get("metadata") or {}) if request_data else {} + pii_tokens = metadata.get("pii_tokens", {}) + new_texts = [] - for text in texts: - modified_text = await self.check_pii( - text=text, - output_parse_pii=self.output_parse_pii, - presidio_config=None, - request_data=request_data or {}, - ) - new_texts.append(modified_text) + if input_type == "response" and pii_tokens: + for text in texts: + new_texts.append(self._unmask_pii_text(text, pii_tokens)) + else: + for text in texts: + modified_text = await self.check_pii( + text=text, + output_parse_pii=self.output_parse_pii, + presidio_config=None, + request_data=request_data or {}, + ) + new_texts.append(modified_text) inputs["texts"] = new_texts return inputs diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 6b3fdf2a81f..296faffa1a5 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -1843,3 +1843,345 @@ def test_unmask_graceful_degradation(): result = _OPTIONAL_PresidioPIIMasking._unmask_pii_text(text, pii_tokens) # No change — no garbage, just clean text assert result == text + + +# --------------------------------------------------------------------------- +# Fix 1: Position bug — reverse sort + original text coordinates +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_anonymize_text_multiple_items_position_correctness(): + """ + Regression test: when multiple PII items exist, coordinates reference the + ORIGINAL text. Processing in reverse order prevents coordinate drift. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + presidio_analyzer_api_base="http://test-analyzer/", + presidio_anonymizer_api_base="http://test-anonymizer/", + mock_testing=False, + ) + + # "Call John at 555-123-4567" + # "John" at [5:9], "555-123-4567" at [13:25] + anonymizer_response = { + "text": "Call at ", + "items": [ + { + "start": 5, + "end": 9, + "entity_type": "PERSON", + "text": "", + "operator": "replace", + }, + { + "start": 13, + "end": 25, + "entity_type": "PHONE_NUMBER", + "text": "", + "operator": "replace", + }, + ], + } + + mock_iterator = _make_mock_session_iterator(anonymizer_response) + + request_data = {"metadata": {}} + with patch.object(guardrail, "_get_session_iterator", mock_iterator): + result = await guardrail.anonymize_text( + text="Call John at 555-123-4567", + analyze_results=[ + {"start": 5, "end": 9, "entity_type": "PERSON", "score": 0.9}, + {"start": 13, "end": 25, "entity_type": "PHONE_NUMBER", "score": 0.95}, + ], + output_parse_pii=True, + masked_entity_count={}, + request_data=request_data, + ) + + pii_tokens = request_data["metadata"]["pii_tokens"] + + # Verify tokens captured the correct ORIGINAL text values + person_token = [k for k in pii_tokens if "PERSON" in k][0] + phone_token = [k for k in pii_tokens if "PHONE" in k][0] + assert pii_tokens[person_token] == "John" + assert pii_tokens[phone_token] == "555-123-4567" + + # Verify both PII values are masked in the result + assert "John" not in result + assert "555-123-4567" not in result + + +# --------------------------------------------------------------------------- +# Fix 2: Anthropic native dict response handling +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_anthropic_native_response_unmasking(): + """ + Anthropic native dict responses (type='message') should be unmasked + when output_parse_pii is enabled. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + output_parse_pii=True, + ) + + request_data = { + "model": "claude-3-haiku", + "metadata": { + "pii_tokens": { + "": "John Smith", + "": "555-123-4567", + } + }, + } + + anthropic_response = { + "type": "message", + "id": "msg_123", + "model": "claude-3-haiku", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Hello , your number is .", + } + ], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 20}, + } + + mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + + result = await guardrail.async_post_call_success_hook( + data=request_data, + user_api_key_dict=mock_user_api_key, + response=anthropic_response, + ) + + assert result["content"][0]["text"] == ( + "Hello John Smith, your number is 555-123-4567." + ) + + +@pytest.mark.asyncio +async def test_anthropic_native_response_masking(): + """ + Anthropic native dict responses should be masked when + apply_to_output is enabled. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + ) + + async def mock_check_pii(text, output_parse_pii, presidio_config, request_data): + return text.replace("John Smith", "[PERSON]").replace("555-123-4567", "[PHONE]") + + guardrail.check_pii = mock_check_pii + + anthropic_response = { + "type": "message", + "id": "msg_123", + "model": "claude-3-haiku", + "role": "assistant", + "content": [{"type": "text", "text": "Hello John Smith, call 555-123-4567."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 20}, + } + + mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + + result = await guardrail.async_post_call_success_hook( + data={}, + user_api_key_dict=mock_user_api_key, + response=anthropic_response, + ) + + assert "[PERSON]" in result["content"][0]["text"] + assert "[PHONE]" in result["content"][0]["text"] + assert "John Smith" not in result["content"][0]["text"] + + +@pytest.mark.asyncio +async def test_anthropic_native_response_non_text_blocks_untouched(): + """ + Non-text blocks (tool_use, thinking) in Anthropic responses + should be left untouched during unmasking. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + output_parse_pii=True, + ) + + request_data = { + "model": "claude-3-haiku", + "metadata": {"pii_tokens": {"": "John"}}, + } + + anthropic_response = { + "type": "message", + "id": "msg_123", + "content": [ + {"type": "text", "text": "Hello "}, + { + "type": "tool_use", + "id": "call_1", + "name": "search", + "input": {"q": "test"}, + }, + ], + "role": "assistant", + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 20}, + } + + mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + + result = await guardrail.async_post_call_success_hook( + data=request_data, + user_api_key_dict=mock_user_api_key, + response=anthropic_response, + ) + + assert result["content"][0]["text"] == "Hello John" + assert result["content"][1]["type"] == "tool_use" + assert result["content"][1]["name"] == "search" + + +# --------------------------------------------------------------------------- +# Fix 3: Anthropic native SSE streaming — bytes passthrough +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_streaming_bytes_chunks_are_yielded_not_discarded(): + """ + Regression test: bytes chunks (Anthropic native SSE) should be yielded + through the streaming hook, not silently discarded. + """ + + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + ) + + byte_chunk = b'data: {"type":"content_block_delta","delta":{"text":"Hello"}}\n\n' + + async def mock_stream(): + yield byte_chunk + + mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + chunks = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=mock_user_api_key, + response=mock_stream(), + request_data={}, + ): + chunks.append(chunk) + + assert any( + isinstance(c, bytes) for c in chunks + ), "bytes chunks must not be discarded" + assert byte_chunk in chunks + + +@pytest.mark.asyncio +async def test_streaming_unmask_path_bytes_passthrough(): + """ + Bytes chunks in the unmasking path should also pass through. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + output_parse_pii=True, + ) + + byte_chunk = b'data: {"type":"content_block_delta"}\n\n' + request_data = { + "metadata": {"pii_tokens": {"": "John"}}, + } + + async def mock_stream(): + yield byte_chunk + + mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + chunks = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=mock_user_api_key, + response=mock_stream(), + request_data=request_data, + ): + chunks.append(chunk) + + assert len(chunks) == 1 + assert chunks[0] == byte_chunk + + +# --------------------------------------------------------------------------- +# Fix 4: apply_guardrail unmask path for input_type="response" +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_apply_guardrail_unmask_on_response(): + """ + When input_type is 'response' and pii_tokens exist, apply_guardrail + should unmask text instead of masking it. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + guardrail_name="test_presidio", + output_parse_pii=True, + mock_testing=True, + ) + + request_data = { + "model": "gpt-4o", + "metadata": { + "pii_tokens": { + "": "John Smith", + "": "555-123-4567", + } + }, + } + + inputs = { + "texts": [ + "Hello , your number is .", + ] + } + + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="response", + ) + + assert result["texts"][0] == "Hello John Smith, your number is 555-123-4567." + + +@pytest.mark.asyncio +async def test_apply_guardrail_masks_on_request(): + """ + When input_type is 'request', apply_guardrail should mask as before. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + guardrail_name="test_presidio", + output_parse_pii=True, + mock_testing=True, + ) + + async def mock_check_pii(text, output_parse_pii, presidio_config, request_data): + return text.replace("John Smith", "") + + guardrail.check_pii = mock_check_pii + + result = await guardrail.apply_guardrail( + inputs={"texts": ["Hello John Smith"]}, + request_data={"model": "gpt-4o", "metadata": {}}, + input_type="request", + ) + + assert "" in result["texts"][0] + assert "John Smith" not in result["texts"][0] From 631aefea18e407922122b7ffc10a6d672801a3a5 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 10 Mar 2026 16:45:22 +0530 Subject: [PATCH 24/62] fix: req changes on feedback from greptile --- .../guardrails/guardrail_hooks/presidio.py | 8 ++++ .../guardrail_hooks/test_presidio.py | 45 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 507a0b89fc9..ddeba2100c3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -1150,6 +1150,14 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): continue if not all_chunks: + # All chunks were Anthropic native SSE bytes — output + # masking cannot be applied to raw bytes. Log a warning + # so operators know PII masking was skipped for this stream. + verbose_proxy_logger.warning( + "Presidio apply_to_output: streaming response contained only " + "bytes chunks (Anthropic native SSE). Output PII masking was " + "skipped for this response." + ) return assembled_model_response = stream_chunk_builder( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 296faffa1a5..32a8c1b1070 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -2185,3 +2185,48 @@ async def test_apply_guardrail_masks_on_request(): assert "" in result["texts"][0] assert "John Smith" not in result["texts"][0] + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_bytes_only_logs_warning(): + """ + Regression test: when apply_to_output=True and the stream contains only + bytes chunks (Anthropic native SSE), output masking is skipped. + A warning must be logged so operators are aware. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + ) + + byte_chunks = [ + b'data: {"type":"content_block_delta","delta":{"text":"Hello"}}\n\n', + b'data: {"type":"content_block_delta","delta":{"text":" world"}}\n\n', + ] + + async def mock_stream(): + for b in byte_chunks: + yield b + + mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + + collected = [] + with patch( + "litellm.proxy.guardrails.guardrail_hooks.presidio.verbose_proxy_logger" + ) as mock_logger: + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=mock_user_api_key, + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + # All bytes should be yielded through + assert len(collected) == len(byte_chunks) + for original, received in zip(byte_chunks, collected): + assert original == received + + # Warning must be logged about skipped masking + mock_logger.warning.assert_called_once() + warning_msg = mock_logger.warning.call_args[0][0] + assert "Output PII masking was skipped" in warning_msg From bec12db6358e0ef7117dbd634b97a8d8e214ce8d Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 10 Mar 2026 09:50:08 -0300 Subject: [PATCH 25/62] docs(responses): add tool_search & namespaces section for gpt-5.4 Add documentation for OpenAI's tool_search feature (Responses API) with SDK and Proxy examples showing namespace-based deferred tool loading. Closes #23206. --- .../docs/providers/openai/responses_api.md | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md index 7799c93ccf2..afe6fcfe106 100644 --- a/docs/my-website/docs/providers/openai/responses_api.md +++ b/docs/my-website/docs/providers/openai/responses_api.md @@ -693,6 +693,159 @@ print(final_response.output) Set `parallel_tool_calls=False` to ensure zero or one tool is called per turn. [More details](https://platform.openai.com/docs/guides/function-calling#parallel-function-calling). +## Tool Search & Namespaces + +Tool search lets models dynamically load tools at runtime instead of sending every tool definition in the prompt. Group functions into **namespaces** and mark them with `defer_loading: true` — the model only loads the schemas it actually needs, saving tokens. + +Requires `gpt-5.4` or later. See [OpenAI Tool Search docs](https://developers.openai.com/api/docs/guides/tools-tool-search) for full details. + + + + +```python showLineNumbers title="Tool Search with Namespaces" +import litellm +import json + +# Define namespaces with deferred tools +tools = [ + {"type": "tool_search"}, # Enable tool search + { + "type": "namespace", + "name": "crm", + "description": "CRM tools for customer management", + "tools": [ + { + "type": "function", + "name": "get_customer", + "description": "Get customer details by ID", + "parameters": { + "type": "object", + "properties": { + "customer_id": {"type": "string"} + }, + "required": ["customer_id"], + }, + "defer_loading": True, + }, + { + "type": "function", + "name": "list_customers", + "description": "List customers with optional filters", + "parameters": { + "type": "object", + "properties": { + "status": {"type": "string", "enum": ["active", "inactive"]}, + }, + }, + "defer_loading": True, + }, + ], + }, + { + "type": "namespace", + "name": "billing", + "description": "Billing and invoicing tools", + "tools": [ + { + "type": "function", + "name": "get_invoice", + "description": "Get an invoice by ID", + "parameters": { + "type": "object", + "properties": { + "invoice_id": {"type": "string"} + }, + "required": ["invoice_id"], + }, + "defer_loading": True, + }, + ], + }, +] + +response = litellm.responses( + model="openai/gpt-5.4", + input="Look up invoice INV-2024-001 from the billing system", + tools=tools, +) + +# The response contains tool_search_call, tool_search_output, and function_call items +for item in response.output: + if isinstance(item, dict): + if item["type"] == "tool_search_call": + print(f"Searched namespaces: {item['arguments']['paths']}") + elif item["type"] == "tool_search_output": + print(f"Loaded {len(item['tools'])} tool(s)") + elif item["type"] == "function_call": + print(f"Called: {item.get('namespace', '')}.{item['name']}({item['arguments']})") + else: + if item.type == "function_call": + print(f"Called: {item.namespace}.{item.name}({item.arguments})") +``` + + + + +1. Set up config.yaml + +```yaml showLineNumbers title="OpenAI Proxy Configuration" +model_list: + - model_name: openai/gpt-5.4 + litellm_params: + model: openai/gpt-5.4 + api_key: os.environ/OPENAI_API_KEY +``` + +2. Start LiteLLM Proxy Server + +```bash title="Start LiteLLM Proxy Server" +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +3. Test it! + +```python showLineNumbers title="Tool Search via OpenAI SDK with LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-api-key" +) + +response = client.responses.create( + model="openai/gpt-5.4", + input="Look up invoice INV-2024-001 from the billing system", + tools=[ + {"type": "tool_search"}, + { + "type": "namespace", + "name": "billing", + "description": "Billing and invoicing tools", + "tools": [ + { + "type": "function", + "name": "get_invoice", + "description": "Get an invoice by ID", + "parameters": { + "type": "object", + "properties": {"invoice_id": {"type": "string"}}, + "required": ["invoice_id"], + }, + "defer_loading": True, + }, + ], + }, + ], +) + +print(response.output) +``` + + + + ## Free-form Function Calling From 8c3d6db4824dbf0b5b4ffd8d7209b73d5f6e2c8e Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 10 Mar 2026 18:21:42 +0530 Subject: [PATCH 26/62] fix: claude code req traces on langfuse --- litellm/utils.py | 57 +++-- .../test_litellm_logging.py | 200 ++++++++++++++---- 2 files changed, 192 insertions(+), 65 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index b7caf0edd7e..b91fc97b74d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -780,9 +780,9 @@ def function_setup( # noqa: PLR0915 coroutine_checker = get_coroutine_checker_fn() ## DYNAMIC CALLBACKS ## - dynamic_callbacks: Optional[List[Union[str, Callable, "CustomLogger"]]] = ( - kwargs.pop("callbacks", None) - ) + dynamic_callbacks: Optional[ + List[Union[str, Callable, "CustomLogger"]] + ] = kwargs.pop("callbacks", None) all_callbacks = get_dynamic_callbacks(dynamic_callbacks=dynamic_callbacks) if len(all_callbacks) > 0: @@ -1143,6 +1143,14 @@ def function_setup( # noqa: PLR0915 litellm_params: Dict[str, Any] = {"api_base": ""} if "metadata" in kwargs: litellm_params["metadata"] = kwargs["metadata"] + if "litellm_metadata" in kwargs: + litellm_params["litellm_metadata"] = kwargs["litellm_metadata"] + # For endpoints like /v1/messages that use "litellm_metadata" instead + # of "metadata" (to avoid conflicting with provider API metadata fields), + # populate litellm_params["metadata"] so callbacks (e.g. Langfuse) that + # read API key info from litellm_params["metadata"] see the fields. + if litellm_params.get("metadata") is None: + litellm_params["metadata"] = kwargs["litellm_metadata"] logging_obj.update_environment_variables( model=model, @@ -1682,9 +1690,9 @@ def client(original_function): # noqa: PLR0915 exception=e, retry_policy=kwargs.get("retry_policy"), ) - kwargs["retry_policy"] = ( - reset_retry_policy() - ) # prevent infinite loops + kwargs[ + "retry_policy" + ] = reset_retry_policy() # prevent infinite loops litellm.num_retries = ( None # set retries to None to prevent infinite loops ) @@ -1731,9 +1739,9 @@ def client(original_function): # noqa: PLR0915 exception=e, retry_policy=kwargs.get("retry_policy"), ) - kwargs["retry_policy"] = ( - reset_retry_policy() - ) # prevent infinite loops + kwargs[ + "retry_policy" + ] = reset_retry_policy() # prevent infinite loops litellm.num_retries = ( None # set retries to None to prevent infinite loops ) @@ -3686,10 +3694,10 @@ def pre_process_non_default_params( if "response_format" in non_default_params: if provider_config is not None: - non_default_params["response_format"] = ( - provider_config.get_json_schema_from_pydantic_object( - response_format=non_default_params["response_format"] - ) + non_default_params[ + "response_format" + ] = provider_config.get_json_schema_from_pydantic_object( + response_format=non_default_params["response_format"] ) else: non_default_params["response_format"] = type_to_response_format_param( @@ -3818,16 +3826,16 @@ def pre_process_optional_params( True # so that main.py adds the function call to the prompt ) if "tools" in non_default_params: - optional_params["functions_unsupported_model"] = ( - non_default_params.pop("tools") - ) + optional_params[ + "functions_unsupported_model" + ] = non_default_params.pop("tools") non_default_params.pop( "tool_choice", None ) # causes ollama requests to hang elif "functions" in non_default_params: - optional_params["functions_unsupported_model"] = ( - non_default_params.pop("functions") - ) + optional_params[ + "functions_unsupported_model" + ] = non_default_params.pop("functions") elif ( litellm.add_function_to_prompt ): # if user opts to add it to prompt instead @@ -7428,9 +7436,9 @@ class ModelResponseIterator: if convert_to_delta is True: _stream_response = ModelResponseStream() _stream_response.choices[0].delta.content = model_response.choices[0].message.content # type: ignore - self.model_response: Union[ModelResponse, ModelResponseStream] = ( - _stream_response - ) + self.model_response: Union[ + ModelResponse, ModelResponseStream + ] = _stream_response else: self.model_response = model_response self.is_done = False @@ -7901,7 +7909,10 @@ class ProviderConfigManager: # Simple provider mappings (no model parameter needed) LlmProviders.DEEPSEEK: (lambda: litellm.DeepSeekChatConfig(), False), LlmProviders.GROQ: (lambda: litellm.GroqChatConfig(), False), - LlmProviders.BEDROCK_MANTLE: (lambda: litellm.BedrockMantleChatConfig(), False), + LlmProviders.BEDROCK_MANTLE: ( + lambda: litellm.BedrockMantleChatConfig(), + False, + ), LlmProviders.A2A: (lambda: litellm.A2AConfig(), False), LlmProviders.BYTEZ: (lambda: litellm.BytezChatConfig(), False), LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False), diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 28624ea8b20..3195ce14d08 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1,4 +1,3 @@ -import json import os import sys from unittest.mock import MagicMock, patch @@ -190,8 +189,13 @@ async def test_datadog_logger_not_shadowed_by_llm_obs(monkeypatch): # Regression check: we expect a distinct DataDogLogger, not the LLM Obs logger assert type(datadog_logger) is DataDogLogger - assert any(isinstance(cb, DataDogLLMObsLogger) for cb in logging_module._in_memory_loggers) - assert any(type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers) + assert any( + isinstance(cb, DataDogLLMObsLogger) + for cb in logging_module._in_memory_loggers + ) + assert any( + type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers + ) finally: logging_module._in_memory_loggers.clear() @@ -202,7 +206,9 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch): # Required env vars for Logfire integration monkeypatch.setenv("LOGFIRE_TOKEN", "test-token") - monkeypatch.setenv("LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev") # no trailing slash on purpose + monkeypatch.setenv( + "LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev" + ) # no trailing slash on purpose # Import after env vars are set (important if module-level caching exists) from litellm.integrations.opentelemetry import OpenTelemetry # logger class @@ -221,7 +227,9 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch): # Sanity: we got the right logger type and it is cached assert type(logger) is OpenTelemetry - assert any(type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers) + assert any( + type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers + ) # Core regression check: base URL env var should influence the exporter endpoint. # @@ -232,7 +240,9 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch): or getattr(logger, "config", None) or getattr(logger, "_otel_config", None) ) - assert cfg is not None, "Expected OpenTelemetry logger to keep an otel config on the instance" + assert ( + cfg is not None + ), "Expected OpenTelemetry logger to keep an otel config on the instance" endpoint = getattr(cfg, "endpoint", None) or getattr(cfg, "otlp_endpoint", None) assert endpoint is not None, "Expected otel config to expose the OTLP endpoint" @@ -297,7 +307,7 @@ async def test_logging_non_streaming_request(): mock_response="Hello, world!", ) await asyncio.sleep(1) - + # Filter calls to only count the one with the expected input message "Hey" # Bridge models may make internal calls that also log, so we filter by the actual input calls_with_expected_input = [] @@ -307,13 +317,13 @@ async def test_logging_non_streaming_request(): first_message_content = messages[0].get("content") if first_message_content == "Hey": calls_with_expected_input.append(call) - + # Assert that we have exactly one call with the expected input assert len(calls_with_expected_input) == 1, ( f"Expected 1 call with input 'Hey', but got {len(calls_with_expected_input)}. " f"Total calls: {mock_async_log_success_event.call_count}" ) - + # Use the filtered call for assertions call_args = calls_with_expected_input[0] standard_logging_object = call_args.kwargs["kwargs"][ @@ -326,14 +336,18 @@ async def test_logging_non_streaming_request(): @pytest.mark.parametrize("async_flag", ["acompletion", "aresponses"]) -def test_success_handler_skips_sync_callbacks_for_async_requests(logging_obj, async_flag): +def test_success_handler_skips_sync_callbacks_for_async_requests( + logging_obj, async_flag +): """Ensure sync success callbacks are skipped when async call type flags are set.""" from litellm.integrations.custom_logger import CustomLogger class DummyLogger(CustomLogger): pass - logging_obj.stream = False # simulate non-streaming request where sync callbacks would normally run + logging_obj.stream = ( + False # simulate non-streaming request where sync callbacks would normally run + ) logging_obj.model_call_details["litellm_params"] = {async_flag: True} logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] @@ -523,7 +537,7 @@ def test_success_handler_runs_guardrail_logging_hook_when_enabled(logging_obj): assert guardrail_call_kwargs["event_type"] == GuardrailEventHooks.logging_only guardrail.logging_hook.assert_called_once() assert logging_obj.model_call_details.get("guardrail_hook_ran") is True - + def test_get_user_agent_tags(): from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup @@ -676,21 +690,29 @@ def test_get_request_tags_does_not_mutate_original_tags(): ) # Verify the original tags list was NOT mutated - assert original_tags == ["custom-tag-1", "custom-tag-2"], ( - f"Original tags list was mutated: {original_tags}" - ) - assert metadata["tags"] == ["custom-tag-1", "custom-tag-2"], ( - f"metadata['tags'] was mutated: {metadata['tags']}" - ) + assert original_tags == [ + "custom-tag-1", + "custom-tag-2", + ], f"Original tags list was mutated: {original_tags}" + assert metadata["tags"] == [ + "custom-tag-1", + "custom-tag-2", + ], f"metadata['tags'] was mutated: {metadata['tags']}" # Verify each returned list has exactly 2 User-Agent tags (not duplicated) user_agent_count_1 = len([t for t in tags1 if t.startswith("User-Agent:")]) user_agent_count_2 = len([t for t in tags2 if t.startswith("User-Agent:")]) user_agent_count_3 = len([t for t in tags3 if t.startswith("User-Agent:")]) - assert user_agent_count_1 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_1}" - assert user_agent_count_2 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_2}" - assert user_agent_count_3 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_3}" + assert ( + user_agent_count_1 == 2 + ), f"Expected 2 User-Agent tags, got {user_agent_count_1}" + assert ( + user_agent_count_2 == 2 + ), f"Expected 2 User-Agent tags, got {user_agent_count_2}" + assert ( + user_agent_count_3 == 2 + ), f"Expected 2 User-Agent tags, got {user_agent_count_3}" # Verify all returned lists are independent (different objects) assert tags1 is not tags2 @@ -795,7 +817,6 @@ def test_get_extra_header_tags(): def test_response_cost_calculator_with_response_cost_in_hidden_params(logging_obj): from litellm import Router - from litellm.litellm_core_utils.litellm_logging import Logging router = Router( model_list=[ @@ -933,7 +954,6 @@ async def test_e2e_generate_cold_storage_object_key_successful(): with patch("litellm.cold_storage_custom_logger", return_value="s3"), patch( "litellm.integrations.s3.get_s3_object_key" ) as mock_get_s3_key: - # Mock the S3 object key generation to return a predictable result mock_get_s3_key.return_value = ( "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" @@ -981,7 +1001,6 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path() ) as mock_get_logger, patch( "litellm.integrations.s3.get_s3_object_key" ) as mock_get_s3_key: - # Setup mocks mock_get_logger.return_value = mock_custom_logger mock_get_s3_key.return_value = ( @@ -1033,7 +1052,6 @@ async def test_e2e_generate_cold_storage_object_key_with_logger_no_s3_path(): ) as mock_get_logger, patch( "litellm.integrations.s3.get_s3_object_key" ) as mock_get_s3_key: - # Setup mocks mock_get_logger.return_value = mock_custom_logger mock_get_s3_key.return_value = ( @@ -1279,9 +1297,9 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu "standard_logging_object should be set for pass-through endpoints " "even when complete_streaming_response is None" ) - assert logging_obj.model_call_details["standard_logging_object"] is not None, ( - "standard_logging_object should not be None for pass-through endpoints" - ) + assert ( + logging_obj.model_call_details["standard_logging_object"] is not None + ), "standard_logging_object should not be None for pass-through endpoints" # Verify that async_complete_streaming_response was set to prevent re-processing # This is consistent with the existing code pattern for regular streaming @@ -1289,15 +1307,15 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu "async_complete_streaming_response should be set to prevent re-processing, " "consistent with the existing code pattern" ) - assert logging_obj.model_call_details["async_complete_streaming_response"] is result, ( - "async_complete_streaming_response should be set to the result" - ) + assert ( + logging_obj.model_call_details["async_complete_streaming_response"] is result + ), "async_complete_streaming_response should be set to the result" # Verify that response_cost is set to None (cost calculation not possible for pass-through) # This is consistent with the error handling in the non-pass-through code path - assert "response_cost" in logging_obj.model_call_details, ( - "response_cost should be set for pass-through endpoints" - ) + assert ( + "response_cost" in logging_obj.model_call_details + ), "response_cost should be set for pass-through endpoints" assert logging_obj.model_call_details["response_cost"] is None, ( "response_cost should be None for pass-through endpoints since " "StandardPassThroughResponseObject doesn't have standard usage info" @@ -1356,10 +1374,14 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp # Verify first call set the values assert "standard_logging_object" in logging_obj.model_call_details assert "async_complete_streaming_response" in logging_obj.model_call_details - first_standard_logging_object = logging_obj.model_call_details["standard_logging_object"] + first_standard_logging_object = logging_obj.model_call_details[ + "standard_logging_object" + ] # Second call - should return early due to async_complete_streaming_response guard - with patch.object(logging_obj, "get_combined_callback_list", return_value=[]) as mock_callbacks: + with patch.object( + logging_obj, "get_combined_callback_list", return_value=[] + ) as mock_callbacks: await logging_obj.async_success_handler( result=result, start_time=start_time, @@ -1370,9 +1392,10 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp mock_callbacks.assert_not_called() # Verify standard_logging_object wasn't modified by second call - assert logging_obj.model_call_details["standard_logging_object"] is first_standard_logging_object, ( - "standard_logging_object should not be modified on re-processing" - ) + assert ( + logging_obj.model_call_details["standard_logging_object"] + is first_standard_logging_object + ), "standard_logging_object should not be modified on re-processing" @pytest.mark.asyncio @@ -1433,9 +1456,11 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_ "standard_logging_object should be set for streaming pass-through endpoints " "even when the response cannot be parsed into a ModelResponse" ) - assert logging_obj.model_call_details["standard_logging_object"] is not None, ( - "standard_logging_object should not be None for streaming pass-through endpoints" - ) + assert ( + logging_obj.model_call_details["standard_logging_object"] is not None + ), "standard_logging_object should not be None for streaming pass-through endpoints" + + def test_get_error_information_error_code_priority(): """ Test get_error_information prioritizes 'code' attribute over 'status_code' attribute @@ -1680,3 +1705,94 @@ async def test_async_success_handler_preserves_response_cost_for_pass_through_en slo = logging_obj.model_call_details.get("standard_logging_object") assert slo is not None assert slo["response_cost"] > 0 + + +def test_function_setup_litellm_metadata_populates_metadata(): + """ + Test that function_setup() properly handles litellm_metadata (used by /v1/messages, + /batches, /responses, /files endpoints) and populates litellm_params["metadata"] + so callbacks like Langfuse can read API key fields. + + This is the root cause of: Claude Code requests missing user_api_key_hash in Langfuse. + """ + import litellm + + test_api_key_hash = "sk-hashed-1234567890abcdef" + test_team_id = "team-test-123" + test_key_alias = "my-test-key" + + # Simulate what happens for /v1/messages: metadata is in "litellm_metadata", not "metadata" + kwargs = { + "model": "claude-3-5-sonnet", + "messages": [{"role": "user", "content": "hello"}], + "litellm_call_id": "test-call-id-123", + "litellm_metadata": { + "user_api_key_hash": test_api_key_hash, + "user_api_key_alias": test_key_alias, + "user_api_key_team_id": test_team_id, + "user_api_key_user_id": "user-123", + "user_api_key": test_api_key_hash, + }, + } + + logging_obj, returned_kwargs = litellm.utils.function_setup( + original_function="anthropic_messages", + rules_obj=litellm.utils.Rules(), + start_time=time.time(), + **kwargs, + ) + + # litellm_params["metadata"] must contain the API key fields + litellm_params = logging_obj.model_call_details.get("litellm_params", {}) + metadata = litellm_params.get("metadata") + assert metadata is not None, "litellm_params['metadata'] should not be None" + assert isinstance(metadata, dict), "litellm_params['metadata'] should be a dict" + assert metadata.get("user_api_key_hash") == test_api_key_hash + assert metadata.get("user_api_key_alias") == test_key_alias + assert metadata.get("user_api_key_team_id") == test_team_id + + # litellm_metadata should also be preserved + litellm_metadata = litellm_params.get("litellm_metadata") + assert litellm_metadata is not None + assert litellm_metadata.get("user_api_key_hash") == test_api_key_hash + + +def test_function_setup_metadata_takes_precedence_over_litellm_metadata(): + """ + Test that when BOTH metadata and litellm_metadata are present (e.g., user sets + Anthropic API metadata AND proxy adds litellm_metadata), metadata is used as + litellm_params["metadata"] and litellm_metadata is stored separately. + """ + import litellm + + kwargs = { + "model": "claude-3-5-sonnet", + "messages": [{"role": "user", "content": "hello"}], + "litellm_call_id": "test-call-id-456", + "metadata": { + "user_id": "anthropic-user-id", + }, + "litellm_metadata": { + "user_api_key_hash": "sk-hashed-xyz", + "user_api_key_team_id": "team-xyz", + }, + } + + logging_obj, _ = litellm.utils.function_setup( + original_function="anthropic_messages", + rules_obj=litellm.utils.Rules(), + start_time=time.time(), + **kwargs, + ) + + litellm_params = logging_obj.model_call_details.get("litellm_params", {}) + + # When both are present, metadata should be the explicit "metadata" dict + metadata = litellm_params.get("metadata") + assert metadata is not None + assert metadata.get("user_id") == "anthropic-user-id" + + # litellm_metadata should be preserved separately for merge_litellm_metadata() + litellm_metadata = litellm_params.get("litellm_metadata") + assert litellm_metadata is not None + assert litellm_metadata.get("user_api_key_hash") == "sk-hashed-xyz" From 558523fd75b730041c0f800e8fa435e647d63b85 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Tue, 10 Mar 2026 18:33:17 +0530 Subject: [PATCH 27/62] fix: req changes by greptile --- litellm/utils.py | 4 +- .../test_litellm_logging.py | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index b91fc97b74d..9e6dd2491c1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1149,8 +1149,8 @@ def function_setup( # noqa: PLR0915 # of "metadata" (to avoid conflicting with provider API metadata fields), # populate litellm_params["metadata"] so callbacks (e.g. Langfuse) that # read API key info from litellm_params["metadata"] see the fields. - if litellm_params.get("metadata") is None: - litellm_params["metadata"] = kwargs["litellm_metadata"] + if not litellm_params.get("metadata"): + litellm_params["metadata"] = kwargs["litellm_metadata"].copy() logging_obj.update_environment_variables( model=model, diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 3195ce14d08..f4aeb27b31a 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1756,6 +1756,11 @@ def test_function_setup_litellm_metadata_populates_metadata(): assert litellm_metadata is not None assert litellm_metadata.get("user_api_key_hash") == test_api_key_hash + # metadata should be a COPY, not an alias — mutating one must not affect the other + assert ( + metadata is not litellm_metadata + ), "litellm_params['metadata'] should be a copy, not the same object" + def test_function_setup_metadata_takes_precedence_over_litellm_metadata(): """ @@ -1796,3 +1801,35 @@ def test_function_setup_metadata_takes_precedence_over_litellm_metadata(): litellm_metadata = litellm_params.get("litellm_metadata") assert litellm_metadata is not None assert litellm_metadata.get("user_api_key_hash") == "sk-hashed-xyz" + + +def test_function_setup_empty_metadata_falls_back_to_litellm_metadata(): + """ + Test that when metadata is explicitly set to {} (empty dict), litellm_metadata + is still used to populate litellm_params["metadata"] so API key fields are visible. + """ + import litellm + + kwargs = { + "model": "claude-3-5-sonnet", + "messages": [{"role": "user", "content": "hello"}], + "litellm_call_id": "test-call-id-789", + "metadata": {}, + "litellm_metadata": { + "user_api_key_hash": "sk-hashed-empty-test", + "user_api_key_team_id": "team-empty-test", + }, + } + + logging_obj, _ = litellm.utils.function_setup( + original_function="anthropic_messages", + rules_obj=litellm.utils.Rules(), + start_time=time.time(), + **kwargs, + ) + + litellm_params = logging_obj.model_call_details.get("litellm_params", {}) + metadata = litellm_params.get("metadata") + assert metadata is not None + assert metadata.get("user_api_key_hash") == "sk-hashed-empty-test" + assert metadata.get("user_api_key_team_id") == "team-empty-test" From 81bd62e8b03cac3bdbc532f061689eb2e9b36052 Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Tue, 10 Mar 2026 18:39:40 +0530 Subject: [PATCH 28/62] Update litellm/utils.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 9e6dd2491c1..5216f79882e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1143,7 +1143,7 @@ def function_setup( # noqa: PLR0915 litellm_params: Dict[str, Any] = {"api_base": ""} if "metadata" in kwargs: litellm_params["metadata"] = kwargs["metadata"] - if "litellm_metadata" in kwargs: + if "litellm_metadata" in kwargs and isinstance(kwargs["litellm_metadata"], dict): litellm_params["litellm_metadata"] = kwargs["litellm_metadata"] # For endpoints like /v1/messages that use "litellm_metadata" instead # of "metadata" (to avoid conflicting with provider API metadata fields), From 17804edc7887343adfce25606203de3ab465376c Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Tue, 10 Mar 2026 18:44:57 +0530 Subject: [PATCH 29/62] Update litellm/utils.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 5216f79882e..4367ec789b3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1144,7 +1144,7 @@ def function_setup( # noqa: PLR0915 if "metadata" in kwargs: litellm_params["metadata"] = kwargs["metadata"] if "litellm_metadata" in kwargs and isinstance(kwargs["litellm_metadata"], dict): - litellm_params["litellm_metadata"] = kwargs["litellm_metadata"] + litellm_params["litellm_metadata"] = kwargs["litellm_metadata"].copy() # For endpoints like /v1/messages that use "litellm_metadata" instead # of "metadata" (to avoid conflicting with provider API metadata fields), # populate litellm_params["metadata"] so callbacks (e.g. Langfuse) that From 8fac04208d046ca6b432ef7229090553e9280746 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 10 Mar 2026 10:27:40 -0300 Subject: [PATCH 30/62] docs(responses): add tool_search bridge examples for chat completions Add examples showing tool_search with namespaces via the chat completions bridge (openai/responses/ prefix) for both SDK and proxy. --- .../docs/providers/openai/responses_api.md | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md index afe6fcfe106..a0a1eda199a 100644 --- a/docs/my-website/docs/providers/openai/responses_api.md +++ b/docs/my-website/docs/providers/openai/responses_api.md @@ -846,6 +846,84 @@ print(response.output) +### Tool Search via Chat Completions Bridge + +You can also use tool search through the `/v1/chat/completions` endpoint by prefixing the model with `openai/responses/`. The request is routed through the Responses API but returns a standard chat completions response. + + + + +```python showLineNumbers title="Tool Search via Chat Completions Bridge" +import litellm + +response = litellm.completion( + model="openai/responses/gpt-5.4", + messages=[{"role": "user", "content": "Look up invoice INV-2024-001"}], + tools=[ + {"type": "tool_search"}, + { + "type": "namespace", + "name": "billing", + "description": "Billing and invoicing tools", + "tools": [ + { + "type": "function", + "name": "get_invoice", + "description": "Get an invoice by ID", + "parameters": { + "type": "object", + "properties": {"invoice_id": {"type": "string"}}, + "required": ["invoice_id"], + }, + "defer_loading": True, + }, + ], + }, + ], +) + +# Standard chat completions response +for tool_call in response.choices[0].message.tool_calls: + print(f"Called: {tool_call.function.name}({tool_call.function.arguments})") +``` + + + + +```bash showLineNumbers title="Tool Search via /v1/chat/completions" +curl http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "openai/responses/gpt-5.4", + "messages": [{"role": "user", "content": "Look up invoice INV-2024-001"}], + "tools": [ + {"type": "tool_search"}, + { + "type": "namespace", + "name": "billing", + "description": "Billing and invoicing tools", + "tools": [ + { + "type": "function", + "name": "get_invoice", + "description": "Get an invoice by ID", + "parameters": { + "type": "object", + "properties": {"invoice_id": {"type": "string"}}, + "required": ["invoice_id"] + }, + "defer_loading": true + } + ] + } + ] + }' +``` + + + + ## Free-form Function Calling From d232d0de6c22ef09912086d2ca19f0dde1f28794 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 10 Mar 2026 12:04:55 -0300 Subject: [PATCH 31/62] docs(openai): document gpt-5.4 reasoning_effort + tools limitation Add tip boxes explaining that gpt-5.4 does not support reasoning_effort with function tools in /v1/chat/completions, and that the responses bridge (openai/responses/gpt-5.4) should be used instead. --- docs/my-website/docs/providers/openai.md | 17 ++++++++++++++++- docs/my-website/docs/reasoning_content.md | 6 ++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 782c7072e50..546cf6540c9 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -628,7 +628,22 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ ## OpenAI Chat Completion to Responses API Bridge -Call any Responses API model from OpenAI's `/chat/completions` endpoint. +Call any Responses API model from OpenAI's `/chat/completions` endpoint. + +:::tip gpt-5.4 + reasoning_effort + function tools + +OpenAI does not support `reasoning_effort` with function tools for `gpt-5.4` in `/v1/chat/completions`. Use the responses bridge instead: + +```python +response = litellm.completion( + model="openai/responses/gpt-5.4", # routes to /v1/responses + messages=[{"role": "user", "content": "What's the weather?"}], + tools=[...], + reasoning_effort="low", +) +``` + +::: diff --git a/docs/my-website/docs/reasoning_content.md b/docs/my-website/docs/reasoning_content.md index b5a5809bd4e..5dd40122c71 100644 --- a/docs/my-website/docs/reasoning_content.md +++ b/docs/my-website/docs/reasoning_content.md @@ -592,6 +592,12 @@ Expected Response +:::tip gpt-5.4: reasoning_effort + function tools + +OpenAI does not support `reasoning_effort` with function tools for `gpt-5.4` in `/v1/chat/completions`. Use `openai/responses/gpt-5.4` to route through the Responses API instead. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details. + +::: + ## OpenAI Responses API - Auto-Summary Control When using OpenAI Responses API models (like `gpt-5`) via `/chat/completions` with `reasoning_effort`, you can control whether `summary="detailed"` is automatically added to the reasoning parameter. From 8c2c379cb5c083cbcc9c535d3456917967fec656 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 10 Mar 2026 12:35:23 -0300 Subject: [PATCH 32/62] fix(openai): add missing gpt-5.3 model entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #23267 — plain `gpt-5.3` was missing from the model pricing JSON, causing tool_choice (and other capability flags) to default to unsupported. Copied fields from gpt-5.3-chat-latest. --- model_prices_and_context_window.json | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index fc8c90ad773..408979bfbae 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20716,6 +20716,43 @@ "supports_tool_choice": true, "supports_vision": true }, + "gpt-5.3": { + "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_priority": 3.5e-07, + "input_cost_per_token": 1.75e-06, + "input_cost_per_token_priority": 3.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1.4e-05, + "output_cost_per_token_priority": 2.8e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": false + }, "gpt-5.3-chat-latest": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, From 336fb0cfebcbd2c7b1436e248776954893d61742 Mon Sep 17 00:00:00 2001 From: milan-berri Date: Tue, 10 Mar 2026 17:58:18 +0200 Subject: [PATCH 33/62] fix(security): strip secret_fields from guardrail logging response (#23162) When guardrails return the full data dict (e.g. guardrails_ai), the guardrail response logged to spend logs and OTEL traces could contain data["secret_fields"].raw_headers with plaintext Authorization headers. This adds a pop("secret_fields") in the guardrail logging path, matching the existing pattern used by Langfuse and Arize integrations. Tested: Verified fix removes secret_fields/raw_headers/authorization from both /spend/logs/ui responses and OTEL trace span attributes. --- litellm/integrations/custom_guardrail.py | 10 ++ .../integrations/test_custom_guardrail.py | 112 ++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index aed77ab2b3e..aa2a8121ee8 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -589,6 +589,16 @@ class CustomGuardrail(CustomLogger): guardrail_json_response ) + # Strip secret_fields to prevent plaintext Authorization headers from + # being persisted to spend logs, OTEL traces, or other logging backends. + # This matches the pattern used by Langfuse and Arize integrations. + if isinstance(clean_guardrail_response, dict): + clean_guardrail_response.pop("secret_fields", None) + elif isinstance(clean_guardrail_response, list): + for item in clean_guardrail_response: + if isinstance(item, dict): + item.pop("secret_fields", None) + slg = StandardLoggingGuardrailInformation( guardrail_name=self.guardrail_name, guardrail_provider=guardrail_provider, diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index ae4082662f9..7c60cbb52ee 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -386,6 +386,118 @@ class TestGuardrailLoggingAggregation: assert info[1]["guardrail_name"] == "test_guardrail" +class TestGuardrailSensitiveFieldStripping: + """Tests that secret_fields is stripped from guardrail responses before logging. + + Matches the pattern used by Langfuse and Arize integrations which also + pop("secret_fields") to prevent raw Authorization headers from being persisted. + """ + + def _make_guardrail(self): + from litellm.types.guardrails import GuardrailEventHooks + + return CustomGuardrail( + guardrail_name="test_guardrail", + event_hook=GuardrailEventHooks.pre_call, + ) + + def test_secret_fields_stripped_from_guardrail_response(self): + """Ensure secret_fields (containing raw Authorization headers) is not persisted.""" + guardrail = self._make_guardrail() + request_data = {"metadata": {}} + + guardrail_response_with_secrets = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "hello"}], + "secret_fields": { + "raw_headers": { + "authorization": "Bearer sk-live-secret-key-12345", + "content-type": "application/json", + } + }, + "proxy_server_request": {"url": "http://localhost:4000/chat/completions"}, + } + + guardrail.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=guardrail_response_with_secrets, + request_data=request_data, + guardrail_status="success", + duration=1.0, + ) + + info = request_data["metadata"]["standard_logging_guardrail_information"] + assert len(info) == 1 + logged_response = info[0]["guardrail_response"] + + # secret_fields must be stripped + assert "secret_fields" not in logged_response + + # Other fields should be preserved + assert "model" in logged_response + assert "messages" in logged_response + assert "proxy_server_request" in logged_response + + def test_string_guardrail_response_not_affected(self): + """String responses (e.g. 'allow', 'deny') should pass through unchanged.""" + guardrail = self._make_guardrail() + request_data = {"metadata": {}} + + guardrail.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response="allow", + request_data=request_data, + guardrail_status="success", + duration=0.5, + ) + + info = request_data["metadata"]["standard_logging_guardrail_information"] + assert info[0]["guardrail_response"] == "allow" + + def test_no_authorization_header_in_logged_response(self): + """Verify no plaintext Authorization header ends up in the logged guardrail response.""" + import json + + guardrail = self._make_guardrail() + request_data = {"metadata": {}} + + guardrail.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response={ + "model": "gpt-4", + "secret_fields": { + "raw_headers": { + "authorization": "Bearer sk-live-SHOULD-NOT-APPEAR", + } + }, + }, + request_data=request_data, + guardrail_status="success", + duration=1.0, + ) + + logged_response = request_data["metadata"]["standard_logging_guardrail_information"][0]["guardrail_response"] + assert "secret_fields" not in logged_response + assert "sk-live-SHOULD-NOT-APPEAR" not in json.dumps(logged_response) + + def test_secret_fields_stripped_from_list_dict_response(self): + """Ensure secret_fields is stripped from List[dict] guardrail responses too.""" + guardrail = self._make_guardrail() + request_data = {"metadata": {}} + + guardrail.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=[ + {"result": "ok", "secret_fields": {"raw_headers": {"authorization": "Bearer sk-secret"}}}, + {"result": "also_ok"}, + ], + request_data=request_data, + guardrail_status="success", + duration=1.0, + ) + + import json + serialized = json.dumps(request_data) + assert "secret_fields" not in serialized + assert "sk-secret" not in serialized + + class TestCustomGuardrailPassthroughSupport: """Tests for passthrough endpoint guardrail support - Issue fixes.""" From 4c9220bdec68d5f6f551077716351db193c627c9 Mon Sep 17 00:00:00 2001 From: Milan Date: Tue, 10 Mar 2026 14:48:48 +0200 Subject: [PATCH 34/62] feat(mcp): add token authentication support for MCP servers - Add 'token' to MCPAuth enum for custom token auth format - Implement token auth in MCP client (_get_auth_headers) - Add token auth support for OpenAPI-based MCP tools - Add comprehensive unit tests to existing test_mcp_client.py - Fixes issue where MCP servers expecting 'Authorization: token ' header could not connect --- litellm/experimental_mcp_client/client.py | 2 + .../mcp_server/mcp_server_manager.py | 2 + litellm/types/mcp.py | 2 + .../test_mcp_client.py | 68 ++++++++++++++++++- .../mcp_tools/create_mcp_server.tsx | 4 +- .../components/mcp_tools/mcp_server_edit.tsx | 3 +- .../src/components/mcp_tools/types.tsx | 1 + 7 files changed, 79 insertions(+), 3 deletions(-) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 849ce023109..e4f241880d8 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -212,6 +212,8 @@ class MCPClient: headers["Authorization"] = self._mcp_auth_value elif self.auth_type == MCPAuth.oauth2: headers["Authorization"] = f"Bearer {self._mcp_auth_value}" + elif self.auth_type == MCPAuth.token: + headers["Authorization"] = f"token {self._mcp_auth_value}" elif isinstance(self._mcp_auth_value, dict): headers.update(self._mcp_auth_value) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 0b58009fcf6..1946e69fd68 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -418,6 +418,8 @@ class MCPServerManager: headers["Authorization"] = f"ApiKey {server.authentication_token}" elif server.auth_type == MCPAuth.basic: headers["Authorization"] = f"Basic {server.authentication_token}" + elif server.auth_type == MCPAuth.token: + headers["Authorization"] = f"token {server.authentication_token}" # Add any static headers from server config. # diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index 884dfefb42c..7b2ea820acc 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -35,6 +35,7 @@ class MCPAuth(str, enum.Enum): basic = "basic" authorization = "authorization" oauth2 = "oauth2" + token = "token" # MCP Literals @@ -50,6 +51,7 @@ MCPAuthType = Optional[ MCPAuth.basic, MCPAuth.authorization, MCPAuth.oauth2, + MCPAuth.token, ] ] diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index febc7c454bd..13a09f54e68 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -11,7 +11,7 @@ sys.path.insert(0, "../../../") import litellm.experimental_mcp_client.client as mcp_client_module from litellm.experimental_mcp_client.client import MCPClient -from litellm.types.mcp import MCPStdioConfig, MCPTransport +from litellm.types.mcp import MCPAuth, MCPStdioConfig, MCPTransport class TestMCPClient: @@ -245,6 +245,72 @@ class TestMCPClient: assert test_client.headers is not None await test_client.aclose() + def test_token_auth_header_generation(self): + """Test that token auth generates correct Authorization header""" + client = MCPClient( + server_url="http://example.com/sse", + transport_type="sse", + auth_type=MCPAuth.token, + auth_value="my-secret-token" + ) + + headers = client._get_auth_headers() + + assert "Authorization" in headers + assert headers["Authorization"] == "token my-secret-token" + + def test_token_auth_compatibility_with_existing_auth_types(self): + """Verify existing auth types are not affected by token auth addition""" + # Test bearer token + client = MCPClient( + server_url="http://example.com/sse", + transport_type="sse", + auth_type=MCPAuth.bearer_token, + auth_value="bearer-token" + ) + headers = client._get_auth_headers() + assert headers["Authorization"] == "Bearer bearer-token" + + # Test API key + client = MCPClient( + server_url="http://example.com/sse", + transport_type="sse", + auth_type=MCPAuth.api_key, + auth_value="api-key" + ) + headers = client._get_auth_headers() + assert headers["X-API-Key"] == "api-key" + + # Test basic auth (gets base64 encoded) + client = MCPClient( + server_url="http://example.com/sse", + transport_type="sse", + auth_type=MCPAuth.basic, + auth_value="user:pass" + ) + headers = client._get_auth_headers() + assert headers["Authorization"].startswith("Basic ") + + def test_token_auth_with_extra_headers(self): + """Test that token auth works alongside extra headers""" + client = MCPClient( + server_url="http://example.com/sse", + transport_type="sse", + auth_type=MCPAuth.token, + auth_value="my-token", + extra_headers={"X-Custom-Header": "custom-value"} + ) + + headers = client._get_auth_headers() + + assert headers["Authorization"] == "token my-token" + assert headers["X-Custom-Header"] == "custom-value" + + def test_token_auth_enum_value(self): + """Test that MCPAuth.token enum exists and has correct value""" + assert hasattr(MCPAuth, "token") + assert MCPAuth.token.value == "token" + if __name__ == "__main__": pytest.main([__file__]) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 6ca58ffae24..6f1e103bc6e 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -29,7 +29,7 @@ interface CreateMCPServerProps { onBackToDiscovery?: () => void; } -const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.BASIC]; +const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.TOKEN, AUTH_TYPE.BASIC]; const AUTH_TYPES_REQUIRING_CREDENTIALS = [...AUTH_TYPES_REQUIRING_AUTH_VALUE, AUTH_TYPE.OAUTH2]; const CREATE_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-create-state"; @@ -654,6 +654,7 @@ const CreateMCPServer: React.FC = ({ User keys will be sent as:{" "} {getFieldValue("auth_type") === "bearer_token" && "Authorization: Bearer {key}"} + {getFieldValue("auth_type") === "token" && "Authorization: token {key}"} {getFieldValue("auth_type") === "api_key" && "x-api-key: {key}"} {getFieldValue("auth_type") === "basic" && "Authorization: Basic {key}"} {getFieldValue("auth_type") === "authorization" && "Authorization: {key}"} @@ -718,6 +719,7 @@ const CreateMCPServer: React.FC = ({ None API Key Bearer Token + Token Basic Auth OAuth diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 00060658ea1..fc55542a0c9 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -20,7 +20,7 @@ interface MCPServerEditProps { availableAccessGroups: string[]; } -const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.BASIC]; +const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.TOKEN, AUTH_TYPE.BASIC]; const AUTH_TYPES_REQUIRING_CREDENTIALS = [...AUTH_TYPES_REQUIRING_AUTH_VALUE, AUTH_TYPE.OAUTH2]; const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state"; @@ -658,6 +658,7 @@ const MCPServerEdit: React.FC = ({ None API Key Bearer Token + Token Basic Auth OAuth diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index 6ba25012197..27b96fd5635 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -9,6 +9,7 @@ export const AUTH_TYPE = { NONE: "none", API_KEY: "api_key", BEARER_TOKEN: "bearer_token", + TOKEN: "token", BASIC: "basic", OAUTH2: "oauth2", }; From 88d0f9d8348db8928849b4807c13c91de2f0015e Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 10 Mar 2026 13:56:15 -0400 Subject: [PATCH 35/62] added comments --- .../tag_management_endpoints.py | 7 +++ .../test_common_daily_activity.py | 59 ++----------------- 2 files changed, 12 insertions(+), 54 deletions(-) diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index f085fa4145f..b7714d3f866 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -553,5 +553,12 @@ async def get_tag_daily_activity( api_key=api_key, page=page, page_size=page_size, + # metadata_metrics_func=None because litellm_dailytagspend rows are + # pre-aggregated per (date, tag, model, …) and have no request_id. + # Deduplication across tags is therefore not possible at this level — + # a request tagged with N tags contributes its spend to N separate rows, + # so passing compute_tag_metadata_totals would double-count spend when + # multiple tags are present. The panel is primarily used to inspect + # individual tags, making this trade-off acceptable. metadata_metrics_func=None, ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 00a22f9bf7d..54fbac1264b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -10,7 +10,6 @@ sys.path.insert( from litellm.proxy.management_endpoints.common_daily_activity import ( _is_user_agent_tag, - compute_tag_metadata_totals, get_api_key_metadata, get_daily_activity, get_daily_activity_aggregated, @@ -77,57 +76,6 @@ def test_is_user_agent_tag(): assert _is_user_agent_tag("user-agent-tag") is False # no colon -def test_compute_tag_metadata_totals(): - """Test compute_tag_metadata_totals function.""" - # Create mock records - class MockRecord: - def __init__(self, request_id, tag, spend, prompt_tokens=10, completion_tokens=5): - self.request_id = request_id - self.tag = tag - self.spend = spend - self.prompt_tokens = prompt_tokens - self.completion_tokens = completion_tokens - self.total_tokens = prompt_tokens + completion_tokens - self.cache_read_input_tokens = 0 - self.cache_creation_input_tokens = 0 - self.api_requests = 1 - self.successful_requests = 1 - self.failed_requests = 0 - - # Test deduplication by request_id (keeps max spend) - records = [ - MockRecord("req-1", "production", spend=10.0), - MockRecord("req-1", "staging", spend=20.0), # Higher spend, should be kept - MockRecord("req-2", "production", spend=15.0), - ] - result = compute_tag_metadata_totals(records) - assert result.spend == 35.0 # 20.0 + 15.0 (deduplicated req-1) - assert result.prompt_tokens == 20 # 10 + 10 (only deduplicated records) - assert result.completion_tokens == 10 # 5 + 5 (only deduplicated records) - - # Test ignoring user-agent tags - records_with_ua = [ - MockRecord("req-1", "production", spend=10.0), - MockRecord("req-1", "user-agent:chrome", spend=50.0), # Should be ignored - MockRecord("req-2", "staging", spend=15.0), - ] - result = compute_tag_metadata_totals(records_with_ua) - assert result.spend == 25.0 # 10.0 + 15.0 (user-agent ignored) - - # Test ignoring records without request_id - records_no_req_id = [ - MockRecord("req-1", "production", spend=10.0), - MockRecord(None, "staging", spend=20.0), # Should be ignored - ] - result = compute_tag_metadata_totals(records_no_req_id) - assert result.spend == 10.0 - - # Test empty records - result = compute_tag_metadata_totals([]) - assert result.spend == 0.0 - assert result.prompt_tokens == 0 - - @pytest.mark.asyncio async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): """Test that endpoint breakdown is included in aggregated daily activity.""" @@ -409,8 +357,11 @@ async def test_get_api_key_metadata_regenerated_key_uses_most_recent_deleted_rec async def test_tag_daily_activity_metadata_totals_not_zero(): """Test that tag daily activity returns correct metadata totals. - Regression test: previously compute_tag_metadata_totals skipped records - with NULL request_id, causing metadata totals (total_spend, etc.) to be 0. + Regression test: the tag endpoint previously passed metadata_metrics_func= + compute_tag_metadata_totals, which skipped every row whose request_id is + NULL. Rows in litellm_dailytagspend are pre-aggregated and always have + NULL request_id, so the totals panel showed $0. The fix is to pass + metadata_metrics_func=None so the fallback aggregation path is used instead. """ mock_prisma = MagicMock() mock_prisma.db = MagicMock() From 9543d785b59c95eb801a5f2b909f53bba0aae1b5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 10 Mar 2026 10:59:49 -0700 Subject: [PATCH 36/62] fix(mcp): don't auto-detect M2M OAuth from field presence (#23187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(mcp): require explicit opt-in for OAuth2 M2M client_credentials flow Auto-detecting M2M from client_id+secret+token_url presence broke existing interactive OAuth setups (e.g. GitHub Enterprise). Add oauth2_flow field and default has_client_credentials to False — M2M must be explicitly opted into with oauth2_flow: client_credentials. * test(mcp): add regression tests for oauth2_flow M2M opt-in behavior --- .../mcp_server/mcp_server_manager.py | 2 + .../types/mcp_server/mcp_server_manager.py | 16 +++- .../mcp_server/test_mcp_server_manager.py | 92 ++++++++++++++++++- 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 1946e69fd68..19582ff63fb 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -318,6 +318,7 @@ class MCPServerManager: # oauth specific fields client_id=server_config.get("client_id", None), client_secret=server_config.get("client_secret", None), + oauth2_flow=server_config.get("oauth2_flow", None), scopes=resolved_scopes, authorization_url=resolved_authorization_url, token_url=resolved_token_url, @@ -634,6 +635,7 @@ class MCPServerManager: client_id=client_id_value or getattr(mcp_server, "client_id", None), client_secret=client_secret_value or getattr(mcp_server, "client_secret", None), + oauth2_flow=getattr(mcp_server, "oauth2_flow", None), scopes=resolved_scopes, authorization_url=mcp_server.authorization_url or getattr(mcp_oauth_metadata, "authorization_url", None), diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index d94795fda2e..2af2dcb88b5 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional from pydantic import BaseModel, ConfigDict @@ -60,12 +60,22 @@ class MCPServer(BaseModel): byok_api_key_help_url: Optional[str] = None created_at: Optional[datetime] = None updated_at: Optional[datetime] = None + # OAuth2 flow type. Defaults to None (interactive / authorization_code). + # Set to "client_credentials" to enable M2M token fetching. + oauth2_flow: Optional[Literal["client_credentials", "authorization_code"]] = None model_config = ConfigDict(arbitrary_types_allowed=True) @property def has_client_credentials(self) -> bool: - """True if this server has OAuth2 client_credentials config (client_id, client_secret, token_url).""" - return bool(self.client_id and self.client_secret and self.token_url) + """True if this server should use the OAuth2 client_credentials (M2M) flow. + + M2M flow must be opted into explicitly via ``oauth2_flow: client_credentials``. + Having client_id / client_secret / token_url present is NOT sufficient — + those fields are also used for interactive (authorization_code) OAuth, + e.g. GitHub Enterprise. Auto-detecting M2M from field presence was a + breaking regression introduced with the M2M feature. + """ + return self.oauth2_flow == "client_credentials" @property def needs_user_oauth_token(self) -> bool: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index acc76221cbb..656a9c616e8 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -1195,7 +1195,7 @@ class TestMCPServerManager: @pytest.mark.asyncio async def test_requires_per_user_auth_property_oauth2_with_client_creds(self): """Test that requires_per_user_auth returns False for OAuth2 with client credentials""" - # OAuth2 with client credentials + # M2M must be opted in explicitly with oauth2_flow="client_credentials" server = MCPServer( server_id="oauth-server", name="oauth-server", @@ -1205,6 +1205,7 @@ class TestMCPServerManager: client_id="client-id", client_secret="client-secret", token_url="http://oauth-server.com/token", + oauth2_flow="client_credentials", ) assert server.requires_per_user_auth is False assert server.has_client_credentials is True @@ -2393,5 +2394,94 @@ class TestMCPServerTimestamps: assert rebuilt_table.updated_at == updated +class TestHasClientCredentialsOAuth2Flow: + """ + Regression tests for the M2M auto-detection bug. + + Before the fix, has_client_credentials returned True whenever + client_id + client_secret + token_url were all set, even for + interactive OAuth setups (e.g. GitHub Enterprise). This silently + dropped user tokens and fetched M2M tokens instead. + + The fix: M2M must be opted in explicitly via oauth2_flow="client_credentials". + """ + + def _make_server(self, **kwargs) -> MCPServer: + return MCPServer( + server_id="test-server", + name="test-server", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + url="https://github.example.com/mcp", + **kwargs, + ) + + def test_all_three_fields_set_without_oauth2_flow_is_not_m2m(self): + """ + GitHub Enterprise regression: client_id + client_secret + token_url + should NOT trigger M2M flow unless oauth2_flow is explicitly set. + """ + server = self._make_server( + client_id="gh-client-id", + client_secret="gh-client-secret", + token_url="https://github.example.com/login/oauth/access_token", + ) + assert server.has_client_credentials is False + + def test_explicit_client_credentials_flow_enables_m2m(self): + """oauth2_flow='client_credentials' opts in to M2M.""" + server = self._make_server( + client_id="svc-client-id", + client_secret="svc-client-secret", + token_url="https://idp.example.com/token", + oauth2_flow="client_credentials", + ) + assert server.has_client_credentials is True + + def test_explicit_authorization_code_flow_disables_m2m(self): + """oauth2_flow='authorization_code' always returns False.""" + server = self._make_server( + client_id="gh-client-id", + client_secret="gh-client-secret", + token_url="https://github.example.com/login/oauth/access_token", + oauth2_flow="authorization_code", + ) + assert server.has_client_credentials is False + + def test_no_fields_no_flow_is_not_m2m(self): + """No credentials configured — not M2M.""" + server = self._make_server() + assert server.has_client_credentials is False + + def test_partial_fields_without_flow_is_not_m2m(self): + """Partial credential fields without explicit flow — not M2M.""" + server = self._make_server( + client_id="only-client-id", + ) + assert server.has_client_credentials is False + + def test_needs_user_oauth_token_true_without_explicit_m2m(self): + """ + Without oauth2_flow='client_credentials', an oauth2 server with + client fields set still needs a user OAuth token (interactive flow). + """ + server = self._make_server( + client_id="gh-client-id", + client_secret="gh-client-secret", + token_url="https://github.example.com/login/oauth/access_token", + ) + assert server.needs_user_oauth_token is True + + def test_needs_user_oauth_token_false_with_explicit_m2m(self): + """With oauth2_flow='client_credentials', no per-user token needed.""" + server = self._make_server( + client_id="svc-client-id", + client_secret="svc-client-secret", + token_url="https://idp.example.com/token", + oauth2_flow="client_credentials", + ) + assert server.needs_user_oauth_token is False + + if __name__ == "__main__": pytest.main([__file__]) From ffc89e4ef6334e78617f622d1e2a21153279b928 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Tue, 10 Mar 2026 19:11:20 +0100 Subject: [PATCH 37/62] fix(mcp): add AWS SigV4 auth for Bedrock AgentCore MCP servers (#22782) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(mcp): add AWS SigV4 auth for Bedrock AgentCore MCP servers Add aws_sigv4 auth type to MCP client via httpx.Auth subclass that signs each request with SigV4 using botocore. Enables mcp_servers config to connect to AgentCore-hosted MCP servers. * docs(mcp): add AWS SigV4 auth documentation for Bedrock AgentCore Add dedicated docs page for configuring MCP servers with AWS SigV4 authentication, update MCP overview with aws_sigv4 auth type and config example, and link from Bedrock AgentCore provider docs. Co-Authored-By: Claude Opus 4.6 * fix(mcp): address Greptile review — requires_request_body, full header signing, health check - Add requires_request_body = True to MCPSigV4Auth so httpx buffers the request body before calling auth_flow (prevents empty body hash for streaming requests) - Pass all request headers to AWSRequest for canonical SigV4 signing instead of only Content-Type - Exclude aws_sigv4 from health check skip logic since it has its own credential fields (not authentication_token) - Fix docs: mark aws_access_key_id/aws_secret_access_key as optional (falls back to boto3 credential chain) - Add test for requires_request_body flag Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Krish Dholakia --- docs/my-website/docs/mcp.md | 11 + docs/my-website/docs/mcp_aws_sigv4.md | 144 ++++++++ .../docs/providers/bedrock_agentcore.md | 2 +- docs/my-website/sidebars.js | 1 + litellm/experimental_mcp_client/client.py | 95 +++++- .../mcp_server/mcp_server_manager.py | 27 +- litellm/types/mcp.py | 2 + .../types/mcp_server/mcp_server_manager.py | 6 + .../mcp_server/test_mcp_sigv4_auth.py | 317 ++++++++++++++++++ 9 files changed, 601 insertions(+), 4 deletions(-) create mode 100644 docs/my-website/docs/mcp_aws_sigv4.md create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 57bc1d57ffd..600f69547d4 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -217,6 +217,7 @@ mcp_servers: | `bearer_token` | `Authorization: Bearer ` | | `basic` | `Authorization: Basic ` | | `authorization` | `Authorization: ` | + | `aws_sigv4` | Per-request AWS SigV4 signature ([details](./mcp_aws_sigv4.md)) | - **Extra Headers**: Optional list of additional header names that should be forwarded from client to the MCP server - **Static Headers**: Optional map of header key/value pairs to include every request to the MCP server. @@ -257,6 +258,16 @@ mcp_servers: auth_type: "authorization" auth_value: "Token example123" # headers={"Authorization": "Token example123"} + # AWS SigV4 for Bedrock AgentCore MCP servers + agentcore_mcp: + url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" + transport: "http" + auth_type: "aws_sigv4" + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-east-1 + aws_service_name: bedrock-agentcore + # Example with extra headers forwarding github_mcp: url: "https://api.githubcopilot.com/mcp" diff --git a/docs/my-website/docs/mcp_aws_sigv4.md b/docs/my-website/docs/mcp_aws_sigv4.md new file mode 100644 index 00000000000..e00cee4fd52 --- /dev/null +++ b/docs/my-website/docs/mcp_aws_sigv4.md @@ -0,0 +1,144 @@ +# MCP - AWS SigV4 Auth + +Use AWS SigV4 authentication to connect LiteLLM to MCP servers hosted on [AWS Bedrock AgentCore](https://docs.aws.amazon.com/bedrock/latest/userguide/agentcore.html). + +## Why SigV4? + +AWS services authenticate requests using [Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) — a per-request signing protocol that includes the request body in the cryptographic signature. This is fundamentally different from static-header auth types (`api_key`, `bearer_token`, etc.) which send the same header on every request. + +LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP request is signed with your AWS credentials before it's sent. + +## Quick Start + +### 1. Set AWS credentials + +```bash +export AWS_ACCESS_KEY_ID="AKIA..." +export AWS_SECRET_ACCESS_KEY="..." +export AWS_REGION_NAME="us-east-1" +``` + +### 2. Add your AgentCore MCP server to config.yaml + +```yaml title="config.yaml" showLineNumbers +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + +mcp_servers: + my_agentcore_mcp: + url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" + transport: "http" + auth_type: "aws_sigv4" + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: "us-east-1" + aws_service_name: "bedrock-agentcore" +``` + +:::info URL encoding + +The AgentCore runtime ARN must be URL-encoded in the `url` field. For example: + +``` +arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/my-mcp-server +``` + +becomes: + +``` +arn%3Aaws%3Abedrock-agentcore%3Aus-east-1%3A123456789012%3Aruntime%2Fmy-mcp-server +``` + +::: + +### 3. Start the proxy + +```bash +litellm --config config.yaml +``` + +### 4. Use the MCP tools + +Once started, your AgentCore MCP tools are available through LiteLLM like any other MCP server: + +```bash title="List available tools" +curl http://localhost:4000/mcp-rest/tools/list \ + -H "Authorization: Bearer sk-1234" +``` + +```bash title="Call a tool" +curl http://localhost:4000/mcp-rest/tools/call \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "name": "my_agentcore_mcp_your_tool_name", + "arguments": {"key": "value"} + }' +``` + +## Config Reference + +| Field | Required | Description | +|-------|----------|-------------| +| `url` | Yes | AgentCore MCP server URL (with URL-encoded ARN) | +| `transport` | Yes | Must be `"http"` | +| `auth_type` | Yes | Must be `"aws_sigv4"` | +| `aws_access_key_id` | No | AWS access key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted | +| `aws_secret_access_key` | No | AWS secret key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted | +| `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) | +| `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` | +| `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` | + +## How It Works + +LiteLLM uses an `httpx.Auth` subclass (`MCPSigV4Auth`) that hooks into the HTTP request lifecycle: + +1. For every outgoing MCP request, the auth handler computes a SHA-256 hash of the request body +2. It creates a SigV4 signature using your AWS credentials, the request URL, headers, and body hash +3. The signed `Authorization` and `x-amz-date` headers are added to the request +4. AWS validates the signature and processes the MCP request + +This happens transparently — no manual token management required. + +## Using Temporary Credentials (STS) + +If you use AWS STS temporary credentials (e.g., from IAM roles or SSO), include the session token: + +```yaml title="config.yaml with STS credentials" showLineNumbers +mcp_servers: + my_agentcore_mcp: + url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes//invocations" + transport: "http" + auth_type: "aws_sigv4" + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_session_token: os.environ/AWS_SESSION_TOKEN + aws_region_name: "us-east-1" + aws_service_name: "bedrock-agentcore" +``` + +## Troubleshooting + +### 403 Forbidden from AWS + +- Verify your AWS credentials are valid and not expired +- Check that `aws_region_name` matches the region in your AgentCore URL +- Ensure `aws_service_name` is set to `bedrock-agentcore` +- If using STS credentials, confirm `aws_session_token` is set and not expired + +### Health check errors on startup + +SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked. + +### "botocore not found" error + +Install the `botocore` package: + +```bash +pip install botocore +``` + +`botocore` is used for SigV4 credential handling and is required when using `aws_sigv4` auth. diff --git a/docs/my-website/docs/providers/bedrock_agentcore.md b/docs/my-website/docs/providers/bedrock_agentcore.md index e3e352f7ab6..7802624fccd 100644 --- a/docs/my-website/docs/providers/bedrock_agentcore.md +++ b/docs/my-website/docs/providers/bedrock_agentcore.md @@ -13,7 +13,7 @@ Call Bedrock AgentCore in the OpenAI Request/Response format. :::info -This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers, add them as you would any other MCP server. See the [MCP documentation](https://docs.litellm.ai/docs/mcp) for details. +This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers with LiteLLM, see the [MCP AWS SigV4 Auth](https://docs.litellm.ai/docs/mcp_aws_sigv4) guide for setup instructions. ::: diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index b4a1337d54e..8b7aad29a34 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -614,6 +614,7 @@ const sidebars = { "mcp_usage", "mcp_openapi", "mcp_oauth", + "mcp_aws_sigv4", "mcp_public_internet", "mcp_semantic_filter", "mcp_control", diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index e4f241880d8..30a1ac20d0c 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -4,7 +4,7 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers. import asyncio import base64 -from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, TypeVar, Union +from typing import Any, Awaitable, Callable, Dict, Generator, List, Optional, Tuple, TypeVar, Union import httpx from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters @@ -50,6 +50,86 @@ def to_basic_auth(auth_value: str) -> str: TSessionResult = TypeVar("TSessionResult") +class MCPSigV4Auth(httpx.Auth): + """ + httpx Auth class that signs each request with AWS SigV4. + + This is used for MCP servers that require AWS SigV4 authentication, + such as AWS Bedrock AgentCore MCP servers. httpx calls auth_flow() + for every outgoing request, enabling per-request signature computation. + """ + + requires_request_body = True + + def __init__( + self, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + aws_region_name: Optional[str] = None, + aws_service_name: Optional[str] = None, + ): + try: + from botocore.credentials import Credentials + except ImportError: + raise ImportError( + "Missing botocore to use AWS SigV4 authentication. " + "Run 'pip install boto3'." + ) + + self.service_name = aws_service_name or "bedrock-agentcore" + self.region_name = aws_region_name or "us-east-1" + + # Note: os.environ/ prefixed values are already resolved by + # ProxyConfig._check_for_os_environ_vars() at config load time. + # Values arrive here as plain strings. + if aws_access_key_id and aws_secret_access_key: + self.credentials = Credentials( + access_key=aws_access_key_id, + secret_key=aws_secret_access_key, + token=aws_session_token, + ) + else: + # Fall back to default boto3 credential chain + import botocore.session + + session = botocore.session.get_session() + self.credentials = session.get_credentials() + if self.credentials is None: + raise ValueError( + "No AWS credentials found. Provide aws_access_key_id and " + "aws_secret_access_key, or configure default credentials " + "(env vars, ~/.aws/credentials, instance profile)." + ) + + def auth_flow( + self, request: httpx.Request + ) -> Generator[httpx.Request, httpx.Response, None]: + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + + # Build AWSRequest from the httpx Request. + # Pass all request headers so the canonical SigV4 signature covers them. + aws_request = AWSRequest( + method=request.method, + url=str(request.url), + data=request.content, + headers=dict(request.headers), + ) + + # Sign the request — SigV4Auth.add_auth() adds Authorization, + # X-Amz-Date, and X-Amz-Security-Token (if session token present). + # Host header is derived automatically from the URL. + sigv4 = SigV4Auth(self.credentials, self.service_name, self.region_name) + sigv4.add_auth(aws_request) + + # Copy SigV4 headers back to the httpx request + for header_name, header_value in aws_request.headers.items(): + request.headers[header_name] = header_value + + yield request + + class MCPClient: """ MCP Client supporting: @@ -68,6 +148,7 @@ class MCPClient: stdio_config: Optional[MCPStdioConfig] = None, extra_headers: Optional[Dict[str, str]] = None, ssl_verify: Optional[VerifyTypes] = None, + aws_auth: Optional[httpx.Auth] = None, ): self.server_url: str = server_url self.transport_type: MCPTransport = transport_type @@ -77,6 +158,7 @@ class MCPClient: self.stdio_config: Optional[MCPStdioConfig] = stdio_config self.extra_headers: Optional[Dict[str, str]] = extra_headers self.ssl_verify: Optional[VerifyTypes] = ssl_verify + self._aws_auth: Optional[httpx.Auth] = aws_auth # handle the basic auth value if provided if auth_value: self.update_auth_value(auth_value) @@ -216,6 +298,9 @@ class MCPClient: headers["Authorization"] = f"token {self._mcp_auth_value}" elif isinstance(self._mcp_auth_value, dict): headers.update(self._mcp_auth_value) + # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request + # signing (including the body hash), so it uses httpx.Auth flow instead + # of static headers. See MCPSigV4Auth and _create_httpx_client_factory(). # update the headers with the extra headers if self.extra_headers: @@ -248,10 +333,16 @@ class MCPClient: f"MCP client using SSL configuration: {type(ssl_config).__name__}" ) + # Use SigV4 auth if configured and no explicit auth provided. + # The MCP SDK's sse_client and streamable_http_client call this + # factory without passing auth=, so self._aws_auth is used. + # For non-SigV4 clients, self._aws_auth is None — no behavior change. + effective_auth = auth if auth is not None else self._aws_auth + return httpx.AsyncClient( headers=headers, timeout=timeout, - auth=auth, + auth=effective_auth, verify=ssl_config, follow_redirects=True, ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 19582ff63fb..2bdc47bf2c2 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -38,7 +38,7 @@ from litellm.constants import ( MCP_TOOL_LISTING_TIMEOUT, ) from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException -from litellm.experimental_mcp_client.client import MCPClient +from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, @@ -340,6 +340,12 @@ class MCPServerManager: available_on_public_internet=bool( server_config.get("available_on_public_internet", True) ), + # AWS SigV4 fields + aws_access_key_id=server_config.get("aws_access_key_id", None), + aws_secret_access_key=server_config.get("aws_secret_access_key", None), + aws_session_token=server_config.get("aws_session_token", None), + aws_region_name=server_config.get("aws_region_name", None), + aws_service_name=server_config.get("aws_service_name", None), ) self.config_mcp_servers[server_id] = new_server @@ -591,6 +597,10 @@ class MCPServerManager: else: client_secret_value = encrypted_client_secret + # TODO: Add AWS SigV4 credential decryption here when DB-stored + # SigV4 MCP servers are supported. Requires corresponding changes + # to encrypt_credentials() in db.py and MCPCredentials TypedDict. + scopes: Optional[List[str]] = None if credentials_dict: scopes_value = credentials_dict.get("scopes") @@ -977,6 +987,18 @@ class MCPServerManager: else: # For HTTP/SSE transports server_url = server.url or "" + + # Create SigV4 auth if configured + aws_auth = None + if server.auth_type == MCPAuth.aws_sigv4: + aws_auth = MCPSigV4Auth( + aws_access_key_id=server.aws_access_key_id, + aws_secret_access_key=server.aws_secret_access_key, + aws_session_token=server.aws_session_token, + aws_region_name=server.aws_region_name, + aws_service_name=server.aws_service_name, + ) + return MCPClient( server_url=server_url, transport_type=transport, @@ -984,6 +1006,7 @@ class MCPServerManager: auth_value=auth_value, timeout=MCP_CLIENT_TIMEOUT, extra_headers=extra_headers, + aws_auth=aws_auth, ) async def _get_tools_from_server( @@ -2510,9 +2533,11 @@ class MCPServerManager: if server.requires_per_user_auth: should_skip_health_check = True # Skip if auth_type is not none and authentication_token is missing + # (except aws_sigv4 which uses its own credential fields) elif ( server.auth_type and server.auth_type != MCPAuth.none + and server.auth_type != MCPAuth.aws_sigv4 and not server.authentication_token ): should_skip_health_check = True diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index 7b2ea820acc..33e55f9bed9 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -35,6 +35,7 @@ class MCPAuth(str, enum.Enum): basic = "basic" authorization = "authorization" oauth2 = "oauth2" + aws_sigv4 = "aws_sigv4" token = "token" @@ -51,6 +52,7 @@ MCPAuthType = Optional[ MCPAuth.basic, MCPAuth.authorization, MCPAuth.oauth2, + MCPAuth.aws_sigv4, MCPAuth.token, ] ] diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 2af2dcb88b5..511cfc958a2 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -48,6 +48,12 @@ class MCPServer(BaseModel): authorization_url: Optional[str] = None token_url: Optional[str] = None registration_url: Optional[str] = None + # AWS SigV4 fields + aws_access_key_id: Optional[str] = None + aws_secret_access_key: Optional[str] = None + aws_session_token: Optional[str] = None + aws_region_name: Optional[str] = None + aws_service_name: Optional[str] = None # defaults to "bedrock-agentcore" # Stdio-specific fields command: Optional[str] = None args: Optional[List[str]] = None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py new file mode 100644 index 00000000000..715bb8e8aee --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py @@ -0,0 +1,317 @@ +""" +Tests for AWS SigV4 authentication in MCP client. + +Tests the MCPSigV4Auth httpx.Auth subclass that enables per-request +SigV4 signing for Bedrock AgentCore MCP servers. +""" + +import pytest +from unittest.mock import patch, MagicMock + +import httpx + +from litellm.experimental_mcp_client.client import MCPSigV4Auth, MCPClient +from litellm.types.mcp import MCPAuth, MCPTransport + + +class TestMCPSigV4Auth: + """Unit tests for the MCPSigV4Auth class.""" + + def test_init_with_explicit_credentials(self): + """MCPSigV4Auth initializes with explicit AWS credentials.""" + auth = MCPSigV4Auth( + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + aws_session_token="FwoGZXIvYXdzEBYaDH...", + aws_region_name="us-east-1", + aws_service_name="bedrock-agentcore", + ) + assert auth.credentials is not None + assert auth.credentials.access_key == "AKIAIOSFODNN7EXAMPLE" + assert auth.credentials.secret_key == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + assert auth.credentials.token == "FwoGZXIvYXdzEBYaDH..." + assert auth.region_name == "us-east-1" + assert auth.service_name == "bedrock-agentcore" + + def test_requires_request_body_flag(self): + """MCPSigV4Auth sets requires_request_body so httpx buffers the body before signing.""" + assert MCPSigV4Auth.requires_request_body is True + + def test_init_defaults(self): + """MCPSigV4Auth uses correct defaults for region and service.""" + auth = MCPSigV4Auth( + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + ) + assert auth.region_name == "us-east-1" + assert auth.service_name == "bedrock-agentcore" + + def test_init_with_resolved_env_values(self): + """MCPSigV4Auth works with pre-resolved values (os.environ/ is resolved at config load time).""" + # Values arrive already resolved by ProxyConfig._check_for_os_environ_vars(), + # so MCPSigV4Auth receives plain strings, not os.environ/ prefixed values. + auth = MCPSigV4Auth( + aws_access_key_id="RESOLVED_KEY_FROM_ENV", + aws_secret_access_key="RESOLVED_SECRET_FROM_ENV", + aws_region_name="us-west-2", + ) + assert auth.credentials.access_key == "RESOLVED_KEY_FROM_ENV" + assert auth.credentials.secret_key == "RESOLVED_SECRET_FROM_ENV" + assert auth.region_name == "us-west-2" + + def test_init_falls_back_to_boto_session(self): + """MCPSigV4Auth falls back to boto3 credential chain when no explicit creds.""" + mock_creds = MagicMock() + mock_creds.access_key = "SESSION_KEY" + mock_creds.secret_key = "SESSION_SECRET" + + mock_session = MagicMock() + mock_session.get_credentials.return_value = mock_creds + + with patch("botocore.session.get_session", return_value=mock_session): + auth = MCPSigV4Auth( + aws_region_name="eu-west-1", + aws_service_name="custom-service", + ) + assert auth.credentials == mock_creds + assert auth.region_name == "eu-west-1" + assert auth.service_name == "custom-service" + + def test_init_raises_when_no_credentials(self): + """MCPSigV4Auth raises ValueError when no credentials are available.""" + mock_session = MagicMock() + mock_session.get_credentials.return_value = None + + with patch("botocore.session.get_session", return_value=mock_session): + with pytest.raises(ValueError, match="No AWS credentials found"): + MCPSigV4Auth() + + def test_auth_flow_signs_request(self): + """MCPSigV4Auth.auth_flow adds SigV4 headers to the request.""" + auth = MCPSigV4Auth( + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + aws_region_name="us-east-1", + aws_service_name="bedrock-agentcore", + ) + + request = httpx.Request( + method="POST", + url="https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/test/invocations", + headers={"Content-Type": "application/json"}, + content=b'{"jsonrpc":"2.0","method":"tools/list","id":1}', + ) + + # Execute auth_flow generator + flow = auth.auth_flow(request) + signed_request = next(flow) + + # Verify SigV4 headers were added + assert "Authorization" in signed_request.headers + assert "AWS4-HMAC-SHA256" in signed_request.headers["Authorization"] + assert "x-amz-date" in signed_request.headers + assert "bedrock-agentcore" in signed_request.headers["Authorization"] + + def test_auth_flow_different_bodies_produce_different_signatures(self): + """Each request gets a unique signature based on its body.""" + auth = MCPSigV4Auth( + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + aws_region_name="us-east-1", + ) + + request1 = httpx.Request( + method="POST", + url="https://example.com/mcp", + headers={"Content-Type": "application/json"}, + content=b'{"jsonrpc":"2.0","method":"tools/list","id":1}', + ) + request2 = httpx.Request( + method="POST", + url="https://example.com/mcp", + headers={"Content-Type": "application/json"}, + content=b'{"jsonrpc":"2.0","method":"tools/call","id":2,"params":{"name":"search"}}', + ) + + signed1 = next(auth.auth_flow(request1)) + signed2 = next(auth.auth_flow(request2)) + + # Signatures must differ because body content differs + assert signed1.headers["Authorization"] != signed2.headers["Authorization"] + + def test_auth_flow_includes_security_token(self): + """SigV4 signing includes X-Amz-Security-Token when session token is present.""" + auth = MCPSigV4Auth( + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + aws_session_token="SESSION_TOKEN_EXAMPLE", + aws_region_name="us-east-1", + ) + + request = httpx.Request( + method="POST", + url="https://example.com/mcp", + headers={"Content-Type": "application/json"}, + content=b'{"jsonrpc":"2.0","method":"initialize","id":0}', + ) + + signed_request = next(auth.auth_flow(request)) + assert "x-amz-security-token" in signed_request.headers + + +class TestMCPClientSigV4Integration: + """Tests for MCPClient with SigV4 auth wired through.""" + + def test_mcp_client_stores_aws_auth(self): + """MCPClient stores the aws_auth parameter.""" + mock_auth = MagicMock(spec=httpx.Auth) + client = MCPClient( + server_url="https://example.com/mcp", + transport_type=MCPTransport.http, + auth_type=MCPAuth.aws_sigv4, + aws_auth=mock_auth, + ) + assert client._aws_auth is mock_auth + + def test_mcp_client_factory_uses_aws_auth(self): + """The httpx client factory uses aws_auth when no explicit auth is passed.""" + mock_auth = MCPSigV4Auth( + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + ) + client = MCPClient( + server_url="https://example.com/mcp", + transport_type=MCPTransport.http, + aws_auth=mock_auth, + ) + + factory = client._create_httpx_client_factory() + httpx_client = factory( + headers={"Content-Type": "application/json"}, + timeout=httpx.Timeout(30.0), + ) + + # Verify the auth object was actually wired into the httpx client + assert httpx_client._auth is mock_auth + + def test_mcp_client_factory_explicit_auth_takes_precedence(self): + """When explicit auth= is passed to the factory, it takes precedence over aws_auth.""" + aws_auth = MCPSigV4Auth( + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + ) + explicit_auth = MagicMock(spec=httpx.Auth) + + client = MCPClient( + server_url="https://example.com/mcp", + transport_type=MCPTransport.http, + aws_auth=aws_auth, + ) + + factory = client._create_httpx_client_factory() + httpx_client = factory( + headers={"Content-Type": "application/json"}, + timeout=httpx.Timeout(30.0), + auth=explicit_auth, + ) + + # Explicit auth should win over aws_auth + assert httpx_client._auth is explicit_auth + + def test_mcp_client_factory_no_aws_auth(self): + """The httpx client factory works normally when no aws_auth is set.""" + client = MCPClient( + server_url="https://example.com/mcp", + transport_type=MCPTransport.http, + ) + + factory = client._create_httpx_client_factory() + httpx_client = factory( + headers={"Content-Type": "application/json"}, + timeout=httpx.Timeout(30.0), + ) + # No auth should be set when aws_auth is not configured + assert httpx_client._auth is None + + +class TestMCPServerManagerSigV4: + """Tests for MCPServerManager config loading with SigV4.""" + + @pytest.mark.asyncio + async def test_load_config_with_aws_sigv4(self): + """Config loading correctly parses aws_sigv4 auth type and AWS fields.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + + config = { + "agentcore_tools": { + "url": "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/test/invocations", + "transport": "http", + "auth_type": "aws_sigv4", + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "aws_region_name": "us-east-1", + "aws_service_name": "bedrock-agentcore", + } + } + + manager = MCPServerManager() + await manager.load_servers_from_config(config) + + server = next(iter(manager.config_mcp_servers.values())) + assert server.auth_type == MCPAuth.aws_sigv4 + assert server.aws_access_key_id == "AKIAIOSFODNN7EXAMPLE" + assert server.aws_secret_access_key == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + assert server.aws_region_name == "us-east-1" + assert server.aws_service_name == "bedrock-agentcore" + + @pytest.mark.asyncio + async def test_create_mcp_client_with_sigv4(self): + """_create_mcp_client creates client with SigV4 auth when auth_type is aws_sigv4.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="test-sigv4", + name="test_sigv4_server", + server_name="test_sigv4", + url="https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/test/invocations", + transport=MCPTransport.http, + auth_type=MCPAuth.aws_sigv4, + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + aws_region_name="us-east-1", + ) + + manager = MCPServerManager() + client = await manager._create_mcp_client(server=server) + + assert client.auth_type == MCPAuth.aws_sigv4 + assert client._aws_auth is not None + assert isinstance(client._aws_auth, MCPSigV4Auth) + + @pytest.mark.asyncio + async def test_create_mcp_client_without_sigv4(self): + """_create_mcp_client does not create SigV4 auth for non-SigV4 servers.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="test-bearer", + name="test_bearer_server", + server_name="test_bearer", + url="https://example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.bearer_token, + authentication_token="test-token", + ) + + manager = MCPServerManager() + client = await manager._create_mcp_client(server=server) + + assert client._aws_auth is None From af297dc082aa5b688f0385e54f421ea979261070 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 10 Mar 2026 16:04:31 -0300 Subject: [PATCH 38/62] fix(openai): clean up gpt-5.3 model entry fields - Remove dead fields: supports_none_reasoning_effort, supports_xhigh_reasoning_effort (not referenced anywhere in the codebase) - Remove supports_web_search (inconsistent with other base models) - Add supports_service_tier (consistent with gpt-5, gpt-5.1, gpt-5.2) --- model_prices_and_context_window.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 408979bfbae..15fe1f0ce67 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20747,11 +20747,9 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, + "supports_service_tier": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_vision": true }, "gpt-5.3-chat-latest": { "cache_read_input_token_cost": 1.75e-07, From fa330ed96bb5434128582892c3c2e8e8bdf18cd1 Mon Sep 17 00:00:00 2001 From: shivam Date: Tue, 10 Mar 2026 12:09:00 -0700 Subject: [PATCH 39/62] policy builder --- .../guardrail_pipeline_flow_builder.md | 270 ++++++++++++++++++ .../proxy/guardrails/guardrail_policies.md | 15 + docs/my-website/sidebars.js | 1 + 3 files changed, 286 insertions(+) create mode 100644 docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md diff --git a/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md b/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md new file mode 100644 index 00000000000..0cd458b24fa --- /dev/null +++ b/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md @@ -0,0 +1,270 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Guardrail Pipeline Flow Builder + +The **Flow Builder** lets you design guardrail policies with **conditional, sequential execution**. Instead of running guardrails independently, you chain them into a pipeline where each step has configurable **ON PASS** and **ON FAIL** actions. This enables multi-tier fallbacks, retries, and escalation paths. + +## When to use the Flow Builder + +| Use Case | Simple Policy | Pipeline (Flow Builder) | +|----------|---------------|-------------------------| +| Run multiple guardrails together | ✅ | ✅ | +| All guardrails run independently | ✅ | ❌ | +| Conditional execution (if A fails → try B) | ❌ | ✅ | +| Fallback to different guardrail on failure | ❌ | ✅ | +| Retry same guardrail before blocking | ❌ | ✅ | +| Pass modified data (e.g., PII-masked) to next step | ❌ | ✅ | + +**Use the Flow Builder when** you need: +- **Fallbacks** — Try a fast/simple guardrail first; if it fails, escalate to a stricter one +- **Retries** — Run the same guardrail multiple times before blocking (e.g., for flaky APIs) +- **Escalation** — Route to different guardrails based on pass/fail outcomes + +## Quick Start + + + + +1. Go to **Policies** → **+ Create New Policy** +2. Choose **Flow Builder** (instead of Simple) +3. Click **Continue to Builder** to open the full-screen Flow Builder +4. Add steps, select guardrails, and configure ON PASS / ON FAIL actions +5. Use **Test** to run a sample message through the pipeline before saving +6. Save the policy + + + + +```yaml showLineNumbers title="config.yaml" +guardrails: + - guardrail_name: strict-filter + litellm_params: + guardrail: lakera + mode: pre_call + api_key: os.environ/LAKERA_API_KEY + - guardrail_name: permissive-filter + litellm_params: + guardrail: presidio + mode: pre_call + +policies: + content-safety: + guardrails: + add: [strict-filter, permissive-filter] + pipeline: + mode: pre_call + steps: + - guardrail: strict-filter + on_fail: next + on_pass: allow + - guardrail: permissive-filter + on_fail: block + on_pass: allow + +policy_attachments: + - policy: content-safety + scope: "*" +``` + + + + +## Step Actions + +Each pipeline step has two actions: + +| Action | When | Description | +|--------|------|-------------| +| **Next Step** | ON PASS or ON FAIL | Continue to the next step in the pipeline | +| **Allow** | ON PASS or ON FAIL | Stop the pipeline and allow the request | +| **Block** | ON PASS or ON FAIL | Stop the pipeline and block the request | +| **Custom Response** | ON PASS or ON FAIL | Stop and return a custom message instead of the default block/allow | + +### Common patterns + +**Fallback chain** — Try strict first, escalate to permissive on failure: + +```yaml +steps: + - guardrail: strict-filter + on_fail: next # strict failed → try next + on_pass: allow + - guardrail: permissive-filter + on_fail: block # permissive failed → block + on_pass: allow +``` + +**Retry same guardrail** — Run the same guardrail twice before blocking: + +```yaml +steps: + - guardrail: lakera-pii + on_fail: next + on_pass: allow + - guardrail: lakera-pii + on_fail: block + on_pass: allow +``` + +**Pass modified data** — Forward PII-masked content to the next step: + +```yaml +steps: + - guardrail: presidio-pii + on_fail: block + on_pass: next + pass_data: true # PII-masked request/response sent to next step + - guardrail: prompt-injection + on_fail: block + on_pass: allow +``` + +## Pipeline Fields + +### `pipeline` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `mode` | `pre_call` \| `post_call` | Yes | When the pipeline runs (before or after the LLM call) | +| `steps` | `list[PipelineStep]` | Yes | Ordered list of steps (at least 1) | + +### `PipelineStep` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `guardrail` | `string` | — | Name of the guardrail to run | +| `on_fail` | `next` \| `block` \| `allow` \| `modify_response` | `block` | Action when guardrail rejects | +| `on_pass` | `next` \| `block` \| `allow` \| `modify_response` | `allow` | Action when guardrail passes | +| `pass_data` | `bool` | `false` | Forward modified request/response to next step | +| `modify_response_message` | `string` | Optional | Custom message for `modify_response` action | + +## Example: Multi-tier content safety + +```yaml showLineNumbers title="config.yaml" +guardrails: + - guardrail_name: "pii-masking" + litellm_params: + guardrail: presidio + mode: pre_call + - guardrail_name: "prompt-injection" + litellm_params: + guardrail: lakera + mode: pre_call + api_key: os.environ/LAKERA_API_KEY + +policies: + content-safety-pipeline: + description: "PII mask → prompt injection check → allow or block" + guardrails: + add: [pii-masking, prompt-injection] + pipeline: + mode: pre_call + steps: + - guardrail: pii-masking + on_fail: block + on_pass: next + pass_data: true + - guardrail: prompt-injection + on_fail: block + on_pass: allow + +policy_attachments: + - policy: content-safety-pipeline + scope: "*" +``` + +**Flow:** 1) Mask PII → 2) Check masked content for prompt injection → 3) Allow or block. + +## Example: Retry with same guardrail + +Useful when a guardrail API is flaky or rate-limited: + +```yaml showLineNumbers title="config.yaml" +policies: + retry-on-failure: + guardrails: + add: [pii_masking] + pipeline: + mode: pre_call + steps: + - guardrail: pii_masking + on_fail: next + on_pass: allow + - guardrail: pii_masking + on_fail: block + on_pass: allow +``` + +**Flow:** Run `pii_masking` twice. Block only if it fails both times. + +## Example: Custom response on failure + +Return a branded message instead of the default block: + +```yaml +steps: + - guardrail: strict-filter + on_fail: modify_response + modify_response_message: "Your request was blocked. Please remove sensitive content and try again." + on_pass: allow +``` + +## Pipeline vs Simple Policy + +**Simple policy** — All guardrails run independently. If any fail, the request is blocked (or handled per guardrail config). + +**Pipeline policy** — Guardrails run in order. Each step has conditional actions. You control the flow (fallback, retry, escalate). + +```mermaid +flowchart TD + subgraph Simple["Simple Policy"] + S1[Guardrail A] --> S2[Guardrail B] + S1 & S2 + end + + subgraph Pipeline["Pipeline Policy"] + P1[Step 1: Guardrail A] -->|on_fail: next| P2[Step 2: Guardrail B] + P1 -->|on_pass: allow| Allow + P2 -->|on_fail: block| Block + P2 -->|on_pass: allow| Allow + end +``` + +## Testing the pipeline + +### In the UI + +The Flow Builder includes a **Test** panel. Enter a sample message and click **Run** to see which steps pass or fail and what action is taken. + +### Via API + +Use the [Test Playground](/docs/proxy/guardrails/test_playground) or send a request with the policy attached: + +```bash +curl -X POST http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer sk-..." \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Test message"}], + "guardrails": ["content-safety-pipeline"] + }' +``` + +## Response headers + +When a pipeline runs, response headers include: + +| Header | Description | +|--------|-------------| +| `x-litellm-applied-policies` | Policies that matched | +| `x-litellm-applied-guardrails` | Guardrails that ran | +| `x-litellm-policy-sources` | Why each policy matched | + +## Related + +- [Guardrail Policies](/docs/proxy/guardrails/guardrail_policies) — Policies overview, attachments, inheritance +- [Policy Templates](/docs/proxy/guardrails/policy_templates) — Pre-configured policy templates +- [Guardrails Quick Start](/docs/proxy/guardrails/quick_start) — Defining guardrails diff --git a/docs/my-website/docs/proxy/guardrails/guardrail_policies.md b/docs/my-website/docs/proxy/guardrails/guardrail_policies.md index e2cb839203e..72cb15afe0a 100644 --- a/docs/my-website/docs/proxy/guardrails/guardrail_policies.md +++ b/docs/my-website/docs/proxy/guardrails/guardrail_policies.md @@ -12,6 +12,17 @@ Use policies to group guardrails and control which ones run for specific teams, - Group guardrails into a single policy - Inherit from existing policies and override what you need +## Policy types + +LiteLLM supports two ways to run guardrails in a policy: + +| Type | Description | Use when | +|------|--------------|----------| +| **Simple** | All guardrails run independently. If any fail, the request is blocked. | You want a flat list of guardrails with no conditional logic. | +| **Pipeline (Flow Builder)** | Guardrails run sequentially with configurable ON PASS / ON FAIL actions per step. Supports fallbacks, retries, and escalation. | You need conditional execution (e.g., try strict filter first, fallback to permissive on failure). | + +For pipelines, see [Guardrail Pipeline Flow Builder](/docs/proxy/guardrails/guardrail_pipeline_flow_builder) for detailed documentation. + ## Quick Start @@ -321,6 +332,9 @@ policies: guardrails: add: [...] remove: [...] + pipeline: # Optional. Enables sequential, conditional execution. + mode: pre_call # or post_call + steps: [...] condition: model: ... ``` @@ -331,6 +345,7 @@ policies: | `inherit` | `string` | Optional. Parent policy to inherit guardrails from. | | `guardrails.add` | `list[string]` | Guardrails to enable. | | `guardrails.remove` | `list[string]` | Guardrails to disable (useful with inheritance). | +| `pipeline` | `object` | Optional. Enables [Flow Builder](/docs/proxy/guardrails/guardrail_pipeline_flow_builder) — sequential execution with ON PASS/ON FAIL actions. | | `condition.model` | `string` or `list[string]` | Optional. Only apply when model matches. Supports regex. | ### `policy_attachments` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index b4a1337d54e..0a0941de05d 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -100,6 +100,7 @@ const sidebars = { label: "Policies", items: [ "proxy/guardrails/guardrail_policies", + "proxy/guardrails/guardrail_pipeline_flow_builder", "proxy/guardrails/policy_templates", "proxy/guardrails/policy_tags", ], From 6bc4cc8f0f93823e7e6acf487636c17c9d92f205 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 10 Mar 2026 16:28:50 -0300 Subject: [PATCH 40/62] feat(openai): add supports_web_search to OpenAI models with Responses API Add `supports_web_search: true` to 31 OpenAI models that support the `web_search_preview` tool via the Responses API. This enables the Router to correctly include these deployments when requests use web search tools. Models excluded (tested, confirmed unsupported): - o1-pro (Tool 'web_search_preview' is not supported) - gpt-audio / gpt-audio-mini (not supported) - gpt-4.1-nano (not supported) - codex-mini-latest (model not found) Also removes the invalid `gpt-5.3` entry added in prior commit (model name does not exist in OpenAI API; use gpt-5.3-chat-latest). --- model_prices_and_context_window.json | 128 +++++++++++++-------------- 1 file changed, 62 insertions(+), 66 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 15fe1f0ce67..c68f3deeec6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19187,7 +19187,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-4.1-2025-04-14": { "cache_read_input_token_cost": 5e-07, @@ -19221,7 +19222,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-4.1-mini": { "cache_read_input_token_cost": 1e-07, @@ -19258,7 +19260,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-4.1-mini-2025-04-14": { "cache_read_input_token_cost": 1e-07, @@ -19292,7 +19295,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-4.1-nano": { "cache_read_input_token_cost": 2.5e-08, @@ -20499,7 +20503,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.1": { "cache_read_input_token_cost": 1.25e-07, @@ -20535,7 +20540,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -20571,7 +20577,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.1-chat-latest": { "cache_read_input_token_cost": 1.25e-07, @@ -20606,7 +20613,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": false, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.2": { "cache_read_input_token_cost": 1.75e-07, @@ -20643,7 +20651,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.2-2025-12-11": { "cache_read_input_token_cost": 1.75e-07, @@ -20680,7 +20689,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.2-chat-latest": { "cache_read_input_token_cost": 1.75e-07, @@ -20714,42 +20724,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true - }, - "gpt-5.3": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_service_tier": true, - "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.3-chat-latest": { "cache_read_input_token_cost": 1.75e-07, @@ -20783,7 +20759,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, @@ -20950,7 +20927,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -21044,7 +21022,8 @@ "supports_response_schema": true, "supports_system_messages": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.1-codex": { "cache_read_input_token_cost": 1.25e-07, @@ -21077,7 +21056,8 @@ "supports_response_schema": true, "supports_system_messages": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.1-codex-max": { "cache_read_input_token_cost": 1.25e-07, @@ -21107,7 +21087,8 @@ "supports_response_schema": true, "supports_system_messages": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.1-codex-mini": { "cache_read_input_token_cost": 2.5e-08, @@ -21140,7 +21121,8 @@ "supports_response_schema": true, "supports_system_messages": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, @@ -21173,7 +21155,8 @@ "supports_response_schema": true, "supports_system_messages": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, @@ -21206,7 +21189,8 @@ "supports_response_schema": true, "supports_system_messages": false, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, @@ -21245,7 +21229,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, @@ -21284,7 +21269,8 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, @@ -21320,7 +21306,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, @@ -21355,7 +21342,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, @@ -25128,7 +25116,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "o3-2025-04-16": { "cache_read_input_token_cost": 5e-07, @@ -25160,7 +25149,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "o3-deep-research": { "cache_read_input_token_cost": 2.5e-06, @@ -25193,7 +25183,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "o3-deep-research-2025-06-26": { "cache_read_input_token_cost": 2.5e-06, @@ -25226,7 +25217,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "o3-mini": { "cache_read_input_token_cost": 5.5e-07, @@ -25290,7 +25282,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "o3-pro-2025-06-10": { "input_cost_per_token": 2e-05, @@ -25320,7 +25313,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "o4-mini": { "cache_read_input_token_cost": 2.75e-07, @@ -25397,7 +25391,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "o4-mini-deep-research-2025-06-26": { "cache_read_input_token_cost": 5e-07, @@ -25430,7 +25425,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "oci/meta.llama-3.1-405b-instruct": { "input_cost_per_token": 1.068e-05, From effaef3cfb7d8e208f7fae49de67df27259660c7 Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 10 Mar 2026 16:49:45 -0300 Subject: [PATCH 41/62] fix: remove duplicate supports_web_search keys 7 models already had supports_web_search from upstream, causing duplicate JSON keys. Re-serialized to remove duplicates. --- model_prices_and_context_window.json | 7 ------- 1 file changed, 7 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 56b6f7e7e53..b9db5f406a3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20910,7 +20910,6 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": false }, @@ -20950,7 +20949,6 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": false }, @@ -20989,7 +20987,6 @@ "supports_tool_choice": false, "supports_vision": true, "supports_web_search": true, - "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": false }, @@ -21030,7 +21027,6 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true }, @@ -21071,7 +21067,6 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_web_search": true, "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true }, @@ -21109,7 +21104,6 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false }, @@ -21147,7 +21141,6 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false }, From bead0b7d908c9dd353e94d5946a6133a36383a2c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 10 Mar 2026 13:42:30 -0700 Subject: [PATCH 42/62] [Test] UI - Logs: Add unit tests for 5 untested view_logs components Add vitest tests for TypeBadges, ErrorViewer, ConfigInfoMessage, TimeCell, and TruncatedValue covering rendering, user interactions, and edge cases. Co-Authored-By: Claude Opus 4.6 --- .../view_logs/ConfigInfoMessage.test.tsx | 41 +++++++++ .../components/view_logs/ErrorViewer.test.tsx | 87 +++++++++++++++++++ .../LogDetailsDrawer/TruncatedValue.test.tsx | 32 +++++++ .../components/view_logs/TypeBadges.test.tsx | 46 ++++++++++ .../components/view_logs/time_cell.test.tsx | 36 ++++++++ 5 files changed, 242 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/view_logs/ConfigInfoMessage.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/ErrorViewer.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/time_cell.test.tsx diff --git a/ui/litellm-dashboard/src/components/view_logs/ConfigInfoMessage.test.tsx b/ui/litellm-dashboard/src/components/view_logs/ConfigInfoMessage.test.tsx new file mode 100644 index 00000000000..9e28b27cece --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/ConfigInfoMessage.test.tsx @@ -0,0 +1,41 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { ConfigInfoMessage } from "./ConfigInfoMessage"; + +describe("ConfigInfoMessage", () => { + it("should render the info message when show is true", () => { + render(); + expect(screen.getByText("Request/Response Data Not Available")).toBeInTheDocument(); + }); + + it("should render nothing when show is false", () => { + const { container } = render(); + expect(container.innerHTML).toBe(""); + }); + + it("should display the YAML config snippet", () => { + render(); + expect(screen.getByText(/store_prompts_in_spend_logs: true/)).toBeInTheDocument(); + }); + + it("should render the settings button when onOpenSettings is provided", () => { + render( {}} />); + expect(screen.getByText("open the settings")).toBeInTheDocument(); + }); + + it("should not render the settings button when onOpenSettings is omitted", () => { + render(); + expect(screen.queryByText("open the settings")).not.toBeInTheDocument(); + }); + + it("should call onOpenSettings when the settings button is clicked", async () => { + const user = userEvent.setup(); + const onOpenSettings = vi.fn(); + + render(); + await user.click(screen.getByText("open the settings")); + + expect(onOpenSettings).toHaveBeenCalledOnce(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/ErrorViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/ErrorViewer.test.tsx new file mode 100644 index 00000000000..64d020b61c7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/ErrorViewer.test.tsx @@ -0,0 +1,87 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { ErrorViewer } from "./ErrorViewer"; + +const basicError = { + error_class: "NotFoundError", + error_message: "Model gpt-5 not found", +}; + +const errorWithTraceback = { + error_class: "AuthenticationError", + error_message: "Invalid API key", + traceback: `Traceback (most recent call last): + File "/app/main.py", line 42, in handle_request + result = await client.chat(model="gpt-4") + File "/app/llms/openai.py", line 100, in chat + response = self._make_request(payload) + File "/app/llms/base.py", line 55, in _make_request + raise AuthenticationError("Invalid API key")`, +}; + +describe("ErrorViewer", () => { + it("should render error type and message", () => { + render(); + expect(screen.getByText("NotFoundError")).toBeInTheDocument(); + expect(screen.getByText("Model gpt-5 not found")).toBeInTheDocument(); + }); + + it("should show 'Unknown Error' when error_class is missing", () => { + render(); + expect(screen.getByText("Unknown Error")).toBeInTheDocument(); + }); + + it("should show 'Unknown error occurred' when error_message is missing", () => { + render(); + expect(screen.getByText("Unknown error occurred")).toBeInTheDocument(); + }); + + it("should render traceback frames when traceback is present", () => { + render(); + expect(screen.getByText("Traceback")).toBeInTheDocument(); + expect(screen.getByText("main.py")).toBeInTheDocument(); + expect(screen.getByText("openai.py")).toBeInTheDocument(); + expect(screen.getByText("base.py")).toBeInTheDocument(); + }); + + it("should expand a frame when clicked", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText("main.py")); + + expect( + screen.getByText("result = await client.chat(model=\"gpt-4\")") + ).toBeInTheDocument(); + }); + + it("should expand all frames when 'Expand All' is clicked", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByText("Expand All")); + + expect(screen.getByText("Collapse All")).toBeInTheDocument(); + }); + + it("should copy traceback to clipboard when copy button is clicked", async () => { + const user = userEvent.setup(); + const mockWriteText = vi.fn().mockResolvedValue(undefined); + Object.defineProperty(navigator, "clipboard", { + value: { writeText: mockWriteText }, + writable: true, + configurable: true, + }); + + render(); + + await user.click(screen.getByTitle("Copy traceback")); + expect(mockWriteText).toHaveBeenCalledWith(errorWithTraceback.traceback); + }); + + it("should not render traceback section when traceback is absent", () => { + render(); + expect(screen.queryByText("Traceback")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.test.tsx new file mode 100644 index 00000000000..0da1f2cc3ab --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.test.tsx @@ -0,0 +1,32 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { TruncatedValue } from "./TruncatedValue"; + +describe("TruncatedValue", () => { + it("should render the value text", () => { + render(); + expect(screen.getByText("chatcmpl-abc123")).toBeInTheDocument(); + }); + + it("should render a dash when value is undefined", () => { + render(); + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("should render a dash when value is empty string", () => { + render(); + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("should apply the default maxWidth when not specified", () => { + render(); + const el = screen.getByText("some-long-id-value"); + expect(el).toHaveStyle({ maxWidth: "180px" }); + }); + + it("should apply custom maxWidth when provided", () => { + render(); + const el = screen.getByText("test-value"); + expect(el).toHaveStyle({ maxWidth: "300px" }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx new file mode 100644 index 00000000000..e3467310265 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx @@ -0,0 +1,46 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { LlmBadge, McpBadge, AgentBadge } from "./TypeBadges"; + +describe("TypeBadges", () => { + describe("LlmBadge", () => { + it("should render with default 'LLM' text when no count is provided", () => { + render(); + expect(screen.getByText("LLM")).toBeInTheDocument(); + }); + + it("should render the count when provided", () => { + render(); + expect(screen.getByText("5")).toBeInTheDocument(); + }); + + it("should render count of 0 instead of default text", () => { + render(); + expect(screen.getByText("0")).toBeInTheDocument(); + }); + }); + + describe("McpBadge", () => { + it("should render with default 'MCP' text when no count is provided", () => { + render(); + expect(screen.getByText("MCP")).toBeInTheDocument(); + }); + + it("should render the count when provided", () => { + render(); + expect(screen.getByText("3")).toBeInTheDocument(); + }); + }); + + describe("AgentBadge", () => { + it("should render with default 'Agent' text when no count is provided", () => { + render(); + expect(screen.getByText("Agent")).toBeInTheDocument(); + }); + + it("should render the count when provided", () => { + render(); + expect(screen.getByText("12")).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/time_cell.test.tsx b/ui/litellm-dashboard/src/components/view_logs/time_cell.test.tsx new file mode 100644 index 00000000000..95a8b43b2c1 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/time_cell.test.tsx @@ -0,0 +1,36 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { TimeCell, getTimeZone } from "./time_cell"; + +describe("TimeCell", () => { + it("should render a formatted time string", () => { + render(); + // The global toLocaleString mock in setupTests returns "YYYY-MM-DD HH:MM:SS" + expect(screen.getByText(/2025/)).toBeInTheDocument(); + }); + + it("should render 'Error converting time' for invalid dates", () => { + // toLocaleString on an Invalid Date returns "Invalid Date", not throwing, + // but the component catches exceptions. Force an error by passing something + // that causes Date constructor to produce NaN. + render(); + // The mock returns "NaN-NaN-NaN NaN:NaN:NaN" for invalid dates + // The component has a try/catch that returns "Error converting time" on exception + const el = screen.getByText(/NaN|Error/); + expect(el).toBeInTheDocument(); + }); + + it("should render with monospace font", () => { + render(); + const span = screen.getByText(/2025/); + expect(span).toHaveStyle({ fontFamily: "monospace" }); + }); +}); + +describe("getTimeZone", () => { + it("should return a non-empty timezone string", () => { + const tz = getTimeZone(); + expect(typeof tz).toBe("string"); + expect(tz.length).toBeGreaterThan(0); + }); +}); From f3844d835607dfacfe85867d8efe0eae5568a3b7 Mon Sep 17 00:00:00 2001 From: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:42:36 -0700 Subject: [PATCH 43/62] Update docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../docs/proxy/guardrails/guardrail_pipeline_flow_builder.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md b/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md index 0cd458b24fa..d98354426d0 100644 --- a/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md +++ b/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md @@ -1,4 +1,5 @@ -import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; From 592232e8350758f97b3271b348f15f18225f71d8 Mon Sep 17 00:00:00 2001 From: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:42:46 -0700 Subject: [PATCH 44/62] Update docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../docs/proxy/guardrails/guardrail_pipeline_flow_builder.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md b/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md index d98354426d0..0ea56eef081 100644 --- a/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md +++ b/docs/my-website/docs/proxy/guardrails/guardrail_pipeline_flow_builder.md @@ -221,8 +221,8 @@ steps: ```mermaid flowchart TD subgraph Simple["Simple Policy"] - S1[Guardrail A] --> S2[Guardrail B] - S1 & S2 + S1[Guardrail A] --> R[Result: block if any fail] + S2[Guardrail B] --> R end subgraph Pipeline["Pipeline Policy"] @@ -231,7 +231,6 @@ flowchart TD P2 -->|on_fail: block| Block P2 -->|on_pass: allow| Allow end -``` ## Testing the pipeline From e88dc2e428c5cb6b77e8799dfb441ad24c3b199c Mon Sep 17 00:00:00 2001 From: Chesars Date: Tue, 10 Mar 2026 17:49:39 -0300 Subject: [PATCH 45/62] feat(openai): add supports_web_search to o4-mini models Tested and confirmed both o4-mini and o4-mini-2025-04-16 support web_search_preview via the Responses API. --- model_prices_and_context_window.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b9db5f406a3..bb4a678b541 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25960,7 +25960,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "o4-mini-2025-04-16": { "cache_read_input_token_cost": 2.75e-07, @@ -25979,7 +25980,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_web_search": true }, "o4-mini-deep-research": { "cache_read_input_token_cost": 5e-07, From 9100e167765179c8561eb28c9c9cc27314b623a5 Mon Sep 17 00:00:00 2001 From: milan-berri Date: Tue, 10 Mar 2026 22:53:54 +0200 Subject: [PATCH 46/62] docs: pip venv upgrade workflow (#23290) * docs: add pip/venv upgrade workflow guide - Add comprehensive guide for upgrading LiteLLM proxy via pip - Covers Prisma client regeneration and DB migration steps - Includes verification commands and troubleshooting tips - Links to existing Prisma migration troubleshooting doc * docs: clarify Python version in prisma generate command - Update example to show multiple Python versions (3.11, 3.12, 3.13) - Make it clear LiteLLM supports multiple Python versions, not just 3.11 * docs: emphasize venv activation before running commands - Add info box at top reminding users to activate venv - Include venv activation step before starting proxy (both options) - Add Windows activation command for cross-platform clarity - Make it clear all commands assume activated venv * docs: add pip_venv_upgrade to sidebar navigation - Add new page to Troubleshooting section in sidebars.js - Positioned after Performance/Latency category and before rollback - Makes the upgrade guide discoverable through docs navigation * docs: show explicit --schema flag in prisma migrate deploy - Add explicit --schema path to Option B migration command - Remove ambiguous instruction about running from litellm_proxy_extras - Include path variable guidance for clarity - Makes the command immediately runnable without directory navigation * Update docs/my-website/docs/troubleshoot/pip_venv_upgrade.md Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update docs/my-website/docs/troubleshoot/pip_venv_upgrade.md Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix: close code block and add missing section in pip_venv_upgrade.md * docs: define schema-path placeholder in verification section --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../docs/troubleshoot/pip_venv_upgrade.md | 121 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 122 insertions(+) create mode 100644 docs/my-website/docs/troubleshoot/pip_venv_upgrade.md diff --git a/docs/my-website/docs/troubleshoot/pip_venv_upgrade.md b/docs/my-website/docs/troubleshoot/pip_venv_upgrade.md new file mode 100644 index 00000000000..6f5699e3fb0 --- /dev/null +++ b/docs/my-website/docs/troubleshoot/pip_venv_upgrade.md @@ -0,0 +1,121 @@ +# Upgrading LiteLLM Proxy (pip/venv) + +Guide for upgrading LiteLLM Proxy when installed via pip in a virtual environment. + +:::info Important +Always activate your virtual environment before running any `litellm` or `prisma` commands. All commands in this guide assume you're working inside an activated venv. +::: + +## How pip/venv Upgrades Work + +There are two pieces that need to stay in sync: + +1. **Prisma client** - Generated Python code that talks to the DB +2. **DB schema** - Tables/columns in PostgreSQL + +When you upgrade via pip, the `litellm-proxy-extras` package ships with a new `schema.prisma` and a `migrations/` directory. But unlike the Docker image, pip install does NOT automatically regenerate the Prisma client or run migrations. You have to do both manually. + +## Upgrade Workflow (pip/venv) + +### 1. Stop the proxy + +Stop your running LiteLLM proxy instance. + +### 2. (Optional) Back up your DB + +```bash +pg_dump -h -U -d -F c -f backup_$(date +%Y%m%d).dump +``` + +### 3. Upgrade the package + +```bash +pip install 'litellm[proxy]==' +``` + +### 4. Regenerate the Prisma client + +```bash +prisma generate --schema /lib/python/site-packages/litellm_proxy_extras/schema.prisma +``` + +Replace `` with your virtual environment path and `` with your Python version (e.g., `python3.11`, `python3.12`, `python3.13`). + +### 5. Apply DB migrations + +You have two options: + +**Option A: Just start the proxy** (simplest) + +The proxy automatically runs `prisma migrate deploy` on startup, which applies any new migrations. + +First, activate your virtual environment: + +```bash +source /bin/activate +``` + +Then start the proxy: + +```bash +litellm --config your_config.yaml --port 4000 +``` + +**Option B: Run manually before starting** + +Activate your virtual environment first: + +```bash +source /bin/activate +``` + +Then run the migration with the explicit schema path: + +```bash +prisma migrate deploy --schema /lib/python/site-packages/litellm_proxy_extras/schema.prisma +``` + +Replace `` with your virtual environment path and `` with your Python version (e.g., `python3.11`, `python3.12`, `python3.13`). + +### 6. Start the proxy + +If you used Option B above, now start the proxy (with venv still activated): + +```bash +litellm --config your_config.yaml --port 4000 +``` + +## How to Verify Migrations + +> **Note:** `` = `/lib/python/site-packages/litellm_proxy_extras/schema.prisma` + +### Before applying migrations: Preview what will change + +Run `pip install 'litellm[proxy]=='` first (Step 3) so the new `schema.prisma` is available. + +```bash +prisma migrate diff \ + --from-url $DATABASE_URL \ + --to-schema-datamodel \ + --script +``` + +### After applying migrations: Check status + +```bash +prisma migrate status --schema +``` + +All migrations should have a `finished_at` timestamp and no `rolled_back_at`. + +## Key Things to Know + +- **`DISABLE_SCHEMA_UPDATE=true`** env var prevents auto-migration on startup - useful if you want full manual control + +- **`prisma db push`** is the nuclear option: force-syncs the DB to match the schema, bypassing migration history. Safe when all changes are additive (new columns/tables), but always have a backup. + +- **The `schema.prisma` inside `litellm_proxy_extras` is the source of truth** - always use that one, not one from a different version or from the git repo + +## Troubleshooting + +If you encounter migration errors, see the [Prisma Migration Troubleshooting Guide](./prisma_migrations). diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 83a9b0d9648..c36c0cd167e 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -1158,6 +1158,7 @@ const sidebars = { "troubleshoot/prisma_migrations", ], }, + "troubleshoot/pip_venv_upgrade", "troubleshoot/rollback", "troubleshoot", ], From 373e5e316b13fe06f21d58dd38fe546229bc22da Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 10 Mar 2026 13:58:59 -0700 Subject: [PATCH 47/62] =?UTF-8?q?feat(mcp):=20BYOM=20=E2=80=94=20non-admin?= =?UTF-8?q?=20MCP=20server=20submission=20+=20admin=20review=20workflow=20?= =?UTF-8?q?(#23205)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(mcp): add BYOM (Bring Your Own MCPs) submission + admin review workflow Non-admins can now submit MCP servers for review via POST /v1/mcp/server/register. Admins get a Submissions tab in the UI to approve or reject pending servers. Approved servers enter the active runtime; rejected ones stay out with notes. - DB: add approval_status, submitted_by, submitted_at, reviewed_at, review_notes to LiteLLM_MCPServerTable with migration - Backend: new endpoints register, submissions, approve, reject - reload_servers_from_database now only loads approval_status=active servers - UI: Submissions tab with stat cards, card list, confirm dialogs; non-admin "Submit MCP Server" button wired to /register endpoint - Fix get_mcp_submissions to filter by submitted_at IS NOT NULL (not submitted_by, which can be null for team-scoped keys without an associated user) * feat(mcp): rename nav item to Team MCPs + add New badge * fix(mcp): revert nav label, rename Submissions tab to Team MCPs + New badge * feat(mcp): add MCP Standards — required fields config + CI-style checks on submissions Adds a "Standards" tab (admin-only) to MCP Servers where admins define which server fields are required for a submission to pass. Each submission card in Team MCPs then shows a green ✓ or red ✗ for each required field, with a summary "N/M checks" badge in the header — like GitHub CI status rows. Also adds a `source_url` field (GitHub / Source URL) to the MCP server schema so non-admins can link to the source repo when submitting a server. - schema.prisma: add `source_url String?` to LiteLLM_MCPServerTable - migration: 20260309000001_add_mcp_source_url - _types.py: source_url on NewMCPServerRequest, UpdateMCPServerRequest, LiteLLM_MCPServerTable - types.tsx: source_url on MCPServer interface - create_mcp_server.tsx: GitHub/Source URL form field - MCPStandardsSettings.tsx: new — toggle which fields are required (stored in general settings as mcp_required_fields) - mcp_servers.tsx: Standards tab (admin-only) - MCPSubmissionsTab.tsx: load required fields + CI-style check pills on each card * refactor(mcp): move submission rules into Team MCPs tab, grouped free-form UI Folds the Standards tab into Team MCPs. Submission Rules panel now lives at the top of the Team MCPs tab — collapsible, shows active rules as chips when closed, expands to a grouped checkbox editor (Documentation / Source / Connection / Security). Removes the separate Standards tab from the nav. MCPStandardsSettings.tsx is now constants-only (FIELD_GROUPS, MCP_REQUIRED_FIELD_DEFS, SETTINGS_KEY) — the UI lives in MCPSubmissionsTab. * feat(mcp): add mcp_required_fields to ConfigGeneralSettings + config/list endpoint Registers mcp_required_fields as a proper general_settings field so the UI can read/write it via /config/list and /config/field/update without the "Invalid field" error. Also fixes a pre-existing pyright None-check issue in _sync_ui_settings_to_general_settings. * ui(mcp): GitHub-style PR checks panel on submission cards * ui: rename Team MCPs -> Submitted Tools, Team Guardrails -> Submitted Guardrails * address greptile review feedback (greploop iteration 1) * fix: inline import, add approval workflow tests, rename Submitted MCPs * fix(mcp): allow re-approval of rejected MCP server submissions * fix(mcp): evict rejected servers from runtime; enforce mcp_required_fields on /register * fix(mcp): sort submissions newest-first; force active status on admin-created servers * fix(mcp): add missing mock in test, show Approve for rejected, clear submission metadata, drop spurious Content-Type * fix(mcp/ui): show Reject for active servers; show submit form to non-admins with team-key note * fix(mcp): conditional reload on reject; view-only admin for submissions; block admin from /register * fix(mcp): match auth_type required-field validation to UI compliance check (reject 'none') * fix(mcp): block view-only admin from /register; log settings failure; warn on active server reject * fix(mcp): allow view-only admin to use /register; add _validate_mcp_required_fields tests * fix(mcp): validate field names in mcp_required_fields; surface backend error in submit UI * fix(mcp): fix falsy field check; add field-name validation; add take limit; document server-managed fields; close dialog on error --- .../migration.sql | 11 + .../migration.sql | 3 + litellm/proxy/_experimental/mcp_server/db.py | 82 ++- .../mcp_server/mcp_server_manager.py | 2 +- litellm/proxy/_types.py | 45 ++ .../mcp_management_endpoints.py | 249 ++++++- litellm/proxy/proxy_server.py | 21 +- litellm/proxy/schema.prisma | 9 + .../test_mcp_management_endpoints.py | 342 +++++++++ .../src/components/guardrails.tsx | 2 +- .../mcp_tools/MCPStandardsSettings.tsx | 72 ++ .../mcp_tools/MCPSubmissionsTab.tsx | 660 ++++++++++++++++++ .../mcp_tools/create_mcp_server.tsx | 43 +- .../src/components/mcp_tools/mcp_servers.tsx | 32 +- .../src/components/mcp_tools/types.tsx | 18 + .../src/components/networking.tsx | 93 +++ 16 files changed, 1659 insertions(+), 25 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000000_add_mcp_approval_status/migration.sql create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000001_add_mcp_source_url/migration.sql create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/MCPStandardsSettings.tsx create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/MCPSubmissionsTab.tsx diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000000_add_mcp_approval_status/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000000_add_mcp_approval_status/migration.sql new file mode 100644 index 00000000000..184caef0809 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000000_add_mcp_approval_status/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable: Add BYOM approval workflow fields to LiteLLM_MCPServerTable +ALTER TABLE "LiteLLM_MCPServerTable" + ADD COLUMN IF NOT EXISTS "approval_status" TEXT DEFAULT 'active', + ADD COLUMN IF NOT EXISTS "submitted_by" TEXT, + ADD COLUMN IF NOT EXISTS "submitted_at" TIMESTAMP(3), + ADD COLUMN IF NOT EXISTS "reviewed_at" TIMESTAMP(3), + ADD COLUMN IF NOT EXISTS "review_notes" TEXT; + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "LiteLLM_MCPServerTable_approval_status_idx" + ON "LiteLLM_MCPServerTable"("approval_status"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000001_add_mcp_source_url/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000001_add_mcp_source_url/migration.sql new file mode 100644 index 00000000000..dc468b82061 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000001_add_mcp_source_url/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable: Add source_url field to LiteLLM_MCPServerTable for GitHub/docs link +ALTER TABLE "LiteLLM_MCPServerTable" + ADD COLUMN IF NOT EXISTS "source_url" TEXT; diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 4c6735bacd3..93580b54305 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from typing import Any, Dict, Iterable, List, Optional, Set, Union, cast from litellm._logging import verbose_proxy_logger @@ -6,6 +7,8 @@ from litellm.proxy._types import ( LiteLLM_MCPServerTable, LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, + MCPApprovalStatus, + MCPSubmissionsSummary, NewMCPServerRequest, SpecialMCPServerName, UpdateMCPServerRequest, @@ -102,12 +105,19 @@ def encrypt_credentials( async def get_all_mcp_servers( prisma_client: PrismaClient, + approval_status: Optional[str] = None, ) -> List[LiteLLM_MCPServerTable]: """ - Returns all of the mcp servers from the db + Returns mcp servers from the db, optionally filtered by approval_status. + Pass approval_status=None to return all servers regardless of approval state. """ try: - mcp_servers = await prisma_client.db.litellm_mcpservertable.find_many() + where: Dict[str, Any] = {} + if approval_status is not None: + where["approval_status"] = approval_status + mcp_servers = await prisma_client.db.litellm_mcpservertable.find_many( + where=where if where else {} + ) return [ LiteLLM_MCPServerTable(**mcp_server.model_dump()) @@ -451,3 +461,71 @@ async def delete_user_credential( await prisma_client.db.litellm_mcpusercredentials.delete( where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} ) + + +async def approve_mcp_server( + prisma_client: PrismaClient, + server_id: str, + touched_by: str, +) -> LiteLLM_MCPServerTable: + """Set approval_status=active and record reviewed_at.""" + now = datetime.now(timezone.utc) + updated = await prisma_client.db.litellm_mcpservertable.update( + where={"server_id": server_id}, + data={ + "approval_status": MCPApprovalStatus.active, + "reviewed_at": now, + "updated_by": touched_by, + }, + ) + return LiteLLM_MCPServerTable(**updated.model_dump()) + + +async def reject_mcp_server( + prisma_client: PrismaClient, + server_id: str, + touched_by: str, + review_notes: Optional[str] = None, +) -> LiteLLM_MCPServerTable: + """Set approval_status=rejected, record reviewed_at and review_notes.""" + now = datetime.now(timezone.utc) + data: Dict[str, Any] = { + "approval_status": MCPApprovalStatus.rejected, + "reviewed_at": now, + "updated_by": touched_by, + } + if review_notes is not None: + data["review_notes"] = review_notes + updated = await prisma_client.db.litellm_mcpservertable.update( + where={"server_id": server_id}, + data=data, + ) + return LiteLLM_MCPServerTable(**updated.model_dump()) + + +async def get_mcp_submissions( + prisma_client: PrismaClient, +) -> MCPSubmissionsSummary: + """ + Returns all MCP servers that were submitted by non-admin users (submitted_at IS NOT NULL), + along with a summary count breakdown by approval_status. + Mirrors get_guardrail_submissions() from guardrail_endpoints.py. + """ + rows = await prisma_client.db.litellm_mcpservertable.find_many( + where={"submitted_at": {"not": None}}, + order={"submitted_at": "desc"}, + take=500, # safety cap; paginate if needed in a future iteration + ) + items = [LiteLLM_MCPServerTable(**r.model_dump()) for r in rows] + + pending = sum(1 for i in items if i.approval_status == MCPApprovalStatus.pending_review) + active = sum(1 for i in items if i.approval_status == MCPApprovalStatus.active) + rejected = sum(1 for i in items if i.approval_status == MCPApprovalStatus.rejected) + + return MCPSubmissionsSummary( + total=len(items), + pending_review=pending, + active=active, + rejected=rejected, + items=items, + ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 2bdc47bf2c2..48ea3d384a9 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -2297,7 +2297,7 @@ class MCPServerManager: prisma_client = get_prisma_client_or_throw( "Database not connected. Connect a database to your proxy" ) - db_mcp_servers = await get_all_mcp_servers(prisma_client) + db_mcp_servers = await get_all_mcp_servers(prisma_client, approval_status="active") verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database") previous_registry = self.registry diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d797d9c7e0a..36790e9feae 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1087,6 +1087,12 @@ class SpecialMCPServerName(str, enum.Enum): all_proxy_servers = "all-proxy-mcpservers" +class MCPApprovalStatus(str, enum.Enum): + pending_review = "pending_review" + active = "active" + rejected = "rejected" + + # MCP Proxy Request Types class NewMCPServerRequest(LiteLLMPydanticObjectBase): server_id: Optional[str] = None @@ -1117,6 +1123,18 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase): is_byok: bool = False byok_description: List[str] = Field(default_factory=list) byok_api_key_help_url: Optional[str] = None + source_url: Optional[str] = None + # BYOM submission fields — set by the endpoint, not by the caller. + # Any caller-provided values are silently overridden before persistence. + approval_status: Optional[str] = Field( + None, description="Server-managed: set by the endpoint; caller values are overridden." + ) + submitted_by: Optional[str] = Field( + None, description="Server-managed: set by the endpoint; caller values are overridden." + ) + submitted_at: Optional[datetime] = Field( + None, description="Server-managed: set by the endpoint; caller values are overridden." + ) @model_validator(mode="before") @classmethod @@ -1176,6 +1194,7 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase): is_byok: bool = False byok_description: List[str] = Field(default_factory=list) byok_api_key_help_url: Optional[str] = None + source_url: Optional[str] = None @model_validator(mode="before") @classmethod @@ -1239,6 +1258,16 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase): byok_description: List[str] = Field(default_factory=list) byok_api_key_help_url: Optional[str] = None has_user_credential: Optional[bool] = None + source_url: Optional[str] = None + # BYOM submission fields + approval_status: Optional[str] = Field( + default="active", + description="Approval status: 'pending_review', 'active', 'rejected'", + ) + submitted_by: Optional[str] = None + submitted_at: Optional[datetime] = None + reviewed_at: Optional[datetime] = None + review_notes: Optional[str] = None class MakeMCPServersPublicRequest(LiteLLMPydanticObjectBase): @@ -1255,6 +1284,18 @@ class MCPUserCredentialResponse(LiteLLMPydanticObjectBase): has_credential: bool +class RejectMCPServerRequest(LiteLLMPydanticObjectBase): + review_notes: Optional[str] = None + + +class MCPSubmissionsSummary(LiteLLMPydanticObjectBase): + total: int + pending_review: int + active: int + rejected: int + items: List["LiteLLM_MCPServerTable"] + + ######## Skills API Types ######## @@ -2203,6 +2244,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, forwards client headers (e.g. Authorization) to the LLM API. Required for Claude Code with Max subscription.", ) + mcp_required_fields: Optional[List[str]] = Field( + None, + description="List of MCP server fields that must be filled in for a submission to pass standards checks (e.g. ['description', 'source_url', 'alias']).", + ) class ConfigYAML(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index f7a4cec301b..14d9e6d0b20 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -18,7 +18,7 @@ import importlib import json import os from dataclasses import dataclass -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, Iterable, List, Literal, Optional from fastapi import ( @@ -76,11 +76,14 @@ if MCP_AVAILABLE: return _ToolNameValidationResult() from litellm.proxy._experimental.mcp_server.db import ( + approve_mcp_server, create_mcp_server, delete_mcp_server, delete_user_credential, get_all_mcp_servers_for_user, get_mcp_server, + get_mcp_submissions, + reject_mcp_server, store_user_credential, update_mcp_server, ) @@ -100,9 +103,12 @@ if MCP_AVAILABLE: LiteLLM_MCPServerTable, LitellmUserRoles, MakeMCPServersPublicRequest, + MCPApprovalStatus, + MCPSubmissionsSummary, MCPUserCredentialRequest, MCPUserCredentialResponse, NewMCPServerRequest, + RejectMCPServerRequest, SpecialMCPServerName, UpdateMCPServerRequest, UserAPIKeyAuth, @@ -155,6 +161,59 @@ if MCP_AVAILABLE: _base_validate_and_normalize_mcp_server_payload(payload) _validate_mcp_server_name_fields(payload) + _VALID_MCP_REQUIRED_FIELDS: frozenset = frozenset( + NewMCPServerRequest.model_fields + ) + + def _validate_mcp_required_fields(payload: Any) -> None: + """Validate submission payload against admin-configured mcp_required_fields.""" + from litellm.proxy.proxy_server import ( + general_settings as proxy_general_settings, + ) + + required_fields: Optional[List[str]] = proxy_general_settings.get( + "mcp_required_fields" + ) + if not required_fields: + return + + # Fail fast on unknown field names — a typo in the config would silently + # block every submission with a confusing "missing fields" error. + unknown = [f for f in required_fields if f not in _VALID_MCP_REQUIRED_FIELDS] + if unknown: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "error": f"mcp_required_fields contains unknown field names: {unknown}. " + "Check general_settings.mcp_required_fields in your proxy config." + }, + ) + + # Mirror the UI's compliance checks (MCPStandardsSettings.tsx FIELD_GROUPS): + # auth_type requires a real value — "none" is treated as absent. + _AUTH_TYPE_SENTINEL = "none" + + def _field_present(field_name: str) -> bool: + value = getattr(payload, field_name, None) + if value is None: + return False + # Treat empty string and empty list as absent (mirrors UI compliance check) + if isinstance(value, (str, list)) and not value: + return False + if field_name == "auth_type" and value == _AUTH_TYPE_SENTINEL: + return False + return True + + missing = [f for f in required_fields if not _field_present(f)] + if missing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": f"Submission is missing required fields: {missing}. " + "Configure required fields via general_settings.mcp_required_fields." + }, + ) + def _is_public_registry_enabled() -> bool: from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, @@ -689,6 +748,187 @@ if MCP_AVAILABLE: for server_id, status in server_status_map.items() ] + @router.post( + "/server/register", + description="Submit a new MCP server for admin review (non-admin users). Mirrors POST /guardrails/register.", + dependencies=[Depends(user_api_key_auth)], + response_model=LiteLLM_MCPServerTable, + status_code=status.HTTP_201_CREATED, + ) + @management_endpoint_wrapper + async def register_mcp_server( + payload: NewMCPServerRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + """ + Allow team members to submit an MCP server for admin review. + Creates the server with approval_status=pending_review. + Requires a team-scoped API key. + """ + if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": "PROXY_ADMIN users should use POST /v1/mcp/server to create servers directly instead of the submission workflow." + }, + ) + + if not user_api_key_dict.team_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "error": "Registration requires an API key associated with a team. Use a team-scoped key." + }, + ) + + prisma_client = get_prisma_client_or_throw( + "Database not connected. Connect a database to your proxy" + ) + + validate_and_normalize_mcp_server_payload(payload) + _validate_mcp_required_fields(payload) + + payload.approval_status = MCPApprovalStatus.pending_review + payload.submitted_by = user_api_key_dict.user_id + payload.submitted_at = datetime.now(timezone.utc) + + try: + new_mcp_server = await create_mcp_server( + prisma_client, + payload, + touched_by=user_api_key_dict.user_id or user_api_key_dict.team_id, + ) + except Exception as e: + verbose_proxy_logger.exception(f"Error registering mcp server: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": f"Error registering mcp server: {str(e)}"}, + ) + # Do NOT add to runtime registry — pending servers are not active + return _redact_mcp_credentials(new_mcp_server) + + @router.get( + "/server/submissions", + description="Returns all MCP servers submitted by non-admin users (admin review queue). Mirrors GET /guardrails/submissions.", + dependencies=[Depends(user_api_key_auth)], + response_model=MCPSubmissionsSummary, + ) + @management_endpoint_wrapper + async def get_mcp_server_submissions( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + """ + Admin-only endpoint to view all user-submitted MCP servers pending review. + """ + if user_api_key_dict.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": "Admin access required to view MCP server submissions."}, + ) + + prisma_client = get_prisma_client_or_throw( + "Database not connected. Connect a database to your proxy" + ) + + return await get_mcp_submissions(prisma_client) + + @router.put( + "/server/{server_id}/approve", + description="Approve a pending MCP server submission (admin only). Mirrors PUT /guardrails/{id}/approve.", + dependencies=[Depends(user_api_key_auth)], + response_model=LiteLLM_MCPServerTable, + ) + @management_endpoint_wrapper + async def approve_mcp_server_submission( + server_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + """ + Admin approves a pending or previously-rejected MCP server — sets approval_status=active and loads it into the runtime registry. + """ + if LitellmUserRoles.PROXY_ADMIN != user_api_key_dict.user_role: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": "Admin access required to approve MCP server submissions."}, + ) + + prisma_client = get_prisma_client_or_throw( + "Database not connected. Connect a database to your proxy" + ) + + existing = await get_mcp_server(prisma_client, server_id) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"error": f"MCP server '{server_id}' not found."}, + ) + if existing.approval_status == MCPApprovalStatus.active: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "MCP server is already active."}, + ) + + approved = await approve_mcp_server( + prisma_client, + server_id, + touched_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME, + ) + await global_mcp_server_manager.reload_servers_from_database() + + return _redact_mcp_credentials(approved) + + @router.put( + "/server/{server_id}/reject", + description="Reject a pending MCP server submission (admin only). Mirrors PUT /guardrails/{id}/reject.", + dependencies=[Depends(user_api_key_auth)], + response_model=LiteLLM_MCPServerTable, + ) + @management_endpoint_wrapper + async def reject_mcp_server_submission( + server_id: str, + payload: RejectMCPServerRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + """ + Admin rejects a pending MCP server — sets approval_status=rejected with optional review_notes. + """ + if LitellmUserRoles.PROXY_ADMIN != user_api_key_dict.user_role: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": "Admin access required to reject MCP server submissions."}, + ) + + prisma_client = get_prisma_client_or_throw( + "Database not connected. Connect a database to your proxy" + ) + + existing = await get_mcp_server(prisma_client, server_id) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"error": f"MCP server '{server_id}' not found."}, + ) + if existing.approval_status == MCPApprovalStatus.rejected: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "MCP server is already rejected."}, + ) + + was_active = existing.approval_status == MCPApprovalStatus.active + rejected = await reject_mcp_server( + prisma_client, + server_id, + touched_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME, + review_notes=payload.review_notes, + ) + # Only evict from the runtime registry if the server was previously active + if was_active: + await global_mcp_server_manager.reload_servers_from_database() + return _redact_mcp_credentials(rejected) + @router.get( "/server/{server_id}", description="Returns the mcp server info", @@ -829,6 +1069,13 @@ if MCP_AVAILABLE: # TODO: audit log for create + # Admin-created servers are always active — clear any submission lifecycle + # fields the caller may have provided to prevent fake entries appearing in + # the submissions queue. + payload.approval_status = MCPApprovalStatus.active + payload.submitted_by = None + payload.submitted_at = None + # Attempt to create the mcp server try: new_mcp_server = await create_mcp_server( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f3bc4b08037..e6bb3ee412e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -351,9 +351,6 @@ from litellm.proxy.management_endpoints.cache_settings_endpoints import ( from litellm.proxy.management_endpoints.callback_management_endpoints import ( router as callback_management_endpoints_router, ) -from litellm.proxy.management_endpoints.config_override_endpoints import ( - router as config_override_router, -) from litellm.proxy.management_endpoints.common_utils import ( _user_has_admin_privileges, admin_can_invite_user, @@ -361,6 +358,9 @@ from litellm.proxy.management_endpoints.common_utils import ( from litellm.proxy.management_endpoints.compliance_endpoints import ( router as compliance_router, ) +from litellm.proxy.management_endpoints.config_override_endpoints import ( + router as config_override_router, +) from litellm.proxy.management_endpoints.cost_tracking_settings import ( router as cost_tracking_settings_router, ) @@ -373,7 +373,9 @@ from litellm.proxy.management_endpoints.fallback_management_endpoints import ( from litellm.proxy.management_endpoints.internal_user_endpoints import ( router as internal_user_router, ) -from litellm.proxy.management_endpoints.internal_user_endpoints import user_update +from litellm.proxy.management_endpoints.internal_user_endpoints import ( + user_update, +) from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import ( router as jwt_key_mapping_router, ) @@ -442,7 +444,9 @@ from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_route from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) -from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config +from litellm.proxy.openai_files_endpoints.files_endpoints import ( + set_files_config, +) from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( passthrough_endpoint_router, ) @@ -541,7 +545,9 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( LiteLLM_UpperboundKeyGenerateParams, ) from litellm.types.realtime import RealtimeQueryParams -from litellm.types.router import DeploymentTypedDict +from litellm.types.router import ( + DeploymentTypedDict, +) from litellm.types.router import ModelInfo as RouterModelInfo from litellm.types.router import ( RouterGeneralSettings, @@ -5788,6 +5794,8 @@ class ProxyStartupEvent: _RUNTIME_GENERAL_SETTINGS_FLAGS, ) + if prisma_client is None: + return db_record = await prisma_client.db.litellm_uisettings.find_unique( where={"id": "ui_settings"} ) @@ -11983,6 +11991,7 @@ async def get_config_list( "mcp_trusted_proxy_ranges": {"type": "List"}, "always_include_stream_usage": {"type": "Boolean"}, "forward_client_headers_to_llm_api": {"type": "Boolean"}, + "mcp_required_fields": {"type": "List"}, } return_val = [] diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 8d4bdffb2dd..721c3e404d2 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -315,6 +315,15 @@ model LiteLLM_MCPServerTable { is_byok Boolean @default(false) byok_description String[] @default([]) byok_api_key_help_url String? + source_url String? + // BYOM submission lifecycle + approval_status String? @default("active") + submitted_by String? + submitted_at DateTime? + reviewed_at DateTime? + review_notes String? + + @@index([approval_status]) } // Per-user BYOK credentials for MCP servers diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index e81c6264f7b..30b3be4a3eb 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -1512,3 +1512,345 @@ class TestManagementPayloadValidation: assert len(result) == 1 assert result[0]["server_id"] == "server-1" assert result[0]["status"] == "healthy" + + +class TestMCPApprovalWorkflow: + """Tests for BYOM submission: register, list submissions, approve, reject.""" + + @pytest.mark.asyncio + async def test_register_mcp_server_requires_team_key(self): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + register_mcp_server, + ) + + payload = NewMCPServerRequest( + alias="My Server", + url="https://example.com/mcp", + transport=MCPTransport.sse, + ) + # No team_id → should raise 400 + user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER, + team_id=None, + ) + with pytest.raises(HTTPException) as exc_info: + await register_mcp_server(payload=payload, user_api_key_dict=user_auth) + assert exc_info.value.status_code == 400 + assert "team" in str(exc_info.value.detail).lower() + + @pytest.mark.asyncio + async def test_register_mcp_server_sets_pending_review(self): + from litellm.proxy._types import MCPApprovalStatus + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + register_mcp_server, + ) + + payload = NewMCPServerRequest( + alias="My Server", + url="https://example.com/mcp", + transport=MCPTransport.sse, + ) + user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER, + team_id="team-123", + user_id="user-abc", + ) + created_record = generate_mock_mcp_server_db_record( + alias="My Server", + url="https://example.com/mcp", + ) + created_record.approval_status = MCPApprovalStatus.pending_review + created_record.submitted_by = "user-abc" + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.validate_and_normalize_mcp_server_payload", + MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.create_mcp_server", + AsyncMock(return_value=created_record), + ) as mock_create, + ): + result = await register_mcp_server( + payload=payload, user_api_key_dict=user_auth + ) + + # Endpoint sets pending_review before calling create_mcp_server + call_payload = mock_create.call_args[0][1] + assert call_payload.approval_status == MCPApprovalStatus.pending_review + assert call_payload.submitted_by == "user-abc" + assert result is not None + + @pytest.mark.asyncio + async def test_get_submissions_non_admin_forbidden(self): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + get_mcp_server_submissions, + ) + + non_admin = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER, + ) + with pytest.raises(HTTPException) as exc_info: + await get_mcp_server_submissions(user_api_key_dict=non_admin) + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_get_submissions_admin_returns_summary(self): + from litellm.proxy._types import MCPSubmissionsSummary + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + get_mcp_server_submissions, + ) + + admin = generate_mock_user_api_key_auth(user_role=LitellmUserRoles.PROXY_ADMIN) + pending = generate_mock_mcp_server_db_record(alias="Pending") + pending.approval_status = "pending_review" + summary = MCPSubmissionsSummary( + total=1, pending_review=1, active=0, rejected=0, items=[pending] + ) + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_submissions", + AsyncMock(return_value=summary), + ), + ): + result = await get_mcp_server_submissions(user_api_key_dict=admin) + + assert result.total == 1 + assert result.pending_review == 1 + + @pytest.mark.asyncio + async def test_approve_non_pending_server_raises_400(self): + from litellm.proxy._types import MCPApprovalStatus + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + approve_mcp_server_submission, + ) + + admin = generate_mock_user_api_key_auth(user_role=LitellmUserRoles.PROXY_ADMIN) + active_server = generate_mock_mcp_server_db_record() + active_server.approval_status = MCPApprovalStatus.active + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(return_value=active_server), + ), + ): + with pytest.raises(HTTPException) as exc_info: + await approve_mcp_server_submission( + server_id="server-1", user_api_key_dict=admin + ) + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_approve_pending_server_loads_into_registry(self): + from litellm.proxy._types import MCPApprovalStatus + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + approve_mcp_server_submission, + ) + + admin = generate_mock_user_api_key_auth(user_role=LitellmUserRoles.PROXY_ADMIN) + pending_server = generate_mock_mcp_server_db_record() + pending_server.approval_status = MCPApprovalStatus.pending_review + approved_server = generate_mock_mcp_server_db_record() + approved_server.approval_status = MCPApprovalStatus.active + + mock_manager = MagicMock() + mock_manager.reload_servers_from_database = AsyncMock() + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(return_value=pending_server), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.approve_mcp_server", + AsyncMock(return_value=approved_server), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + ): + result = await approve_mcp_server_submission( + server_id=pending_server.server_id, user_api_key_dict=admin + ) + + mock_manager.reload_servers_from_database.assert_awaited_once() + assert result is not None + + @pytest.mark.asyncio + async def test_reject_already_rejected_raises_400(self): + from litellm.proxy._types import MCPApprovalStatus, RejectMCPServerRequest + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + reject_mcp_server_submission, + ) + + admin = generate_mock_user_api_key_auth(user_role=LitellmUserRoles.PROXY_ADMIN) + rejected_server = generate_mock_mcp_server_db_record() + rejected_server.approval_status = MCPApprovalStatus.rejected + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(return_value=rejected_server), + ), + ): + with pytest.raises(HTTPException) as exc_info: + await reject_mcp_server_submission( + server_id="server-1", + payload=RejectMCPServerRequest(review_notes="duplicate"), + user_api_key_dict=admin, + ) + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_reject_active_server_allowed(self): + """Admin can deactivate an already-approved server via the reject endpoint.""" + from litellm.proxy._types import MCPApprovalStatus, RejectMCPServerRequest + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + reject_mcp_server_submission, + ) + + admin = generate_mock_user_api_key_auth(user_role=LitellmUserRoles.PROXY_ADMIN) + active_server = generate_mock_mcp_server_db_record() + active_server.approval_status = MCPApprovalStatus.active + now_rejected = generate_mock_mcp_server_db_record() + now_rejected.approval_status = MCPApprovalStatus.rejected + + mock_manager = MagicMock() + mock_manager.reload_servers_from_database = AsyncMock() + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(return_value=active_server), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.reject_mcp_server", + AsyncMock(return_value=now_rejected), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + ): + result = await reject_mcp_server_submission( + server_id=active_server.server_id, + payload=RejectMCPServerRequest(review_notes="policy violation"), + user_api_key_dict=admin, + ) + assert result is not None + mock_manager.reload_servers_from_database.assert_awaited_once() + + +class TestValidateMCPRequiredFields: + """Tests for _validate_mcp_required_fields.""" + + def test_missing_required_field_raises_400(self): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + _validate_mcp_required_fields, + ) + + payload = NewMCPServerRequest( + alias="My Server", + url="https://example.com/mcp", + transport=MCPTransport.sse, + # source_url is absent + ) + with patch_proxy_general_settings({"mcp_required_fields": ["source_url"]}): + with pytest.raises(HTTPException) as exc_info: + _validate_mcp_required_fields(payload) + assert exc_info.value.status_code == 400 + assert "source_url" in str(exc_info.value.detail) + + def test_auth_type_sentinel_treated_as_absent(self): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + _validate_mcp_required_fields, + ) + + payload = NewMCPServerRequest( + alias="My Server", + url="https://example.com/mcp", + transport=MCPTransport.sse, + auth_type=MCPAuth.none, # sentinel value — treated as absent + ) + with patch_proxy_general_settings({"mcp_required_fields": ["auth_type"]}): + with pytest.raises(HTTPException) as exc_info: + _validate_mcp_required_fields(payload) + assert exc_info.value.status_code == 400 + assert "auth_type" in str(exc_info.value.detail) + + def test_all_required_fields_present_passes(self): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + _validate_mcp_required_fields, + ) + + payload = NewMCPServerRequest( + alias="My Server", + url="https://example.com/mcp", + transport=MCPTransport.sse, + source_url="https://github.com/org/repo", + auth_type=MCPAuth.bearer_token, + ) + with patch_proxy_general_settings( + {"mcp_required_fields": ["source_url", "auth_type"]} + ): + # Should not raise + _validate_mcp_required_fields(payload) + + def test_no_required_fields_configured_always_passes(self): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + _validate_mcp_required_fields, + ) + + payload = NewMCPServerRequest( + alias="Minimal", + url="https://example.com/mcp", + transport=MCPTransport.sse, + ) + with patch_proxy_general_settings({}): + # Should not raise when no required fields are configured + _validate_mcp_required_fields(payload) + + def test_unknown_field_name_in_config_raises_500(self): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + _validate_mcp_required_fields, + ) + + payload = NewMCPServerRequest( + alias="My Server", + url="https://example.com/mcp", + transport=MCPTransport.sse, + ) + # "source_Url" is a typo — not a real field on NewMCPServerRequest + with patch_proxy_general_settings({"mcp_required_fields": ["source_Url"]}): + with pytest.raises(HTTPException) as exc_info: + _validate_mcp_required_fields(payload) + assert exc_info.value.status_code == 500 + assert "source_Url" in str(exc_info.value.detail) diff --git a/ui/litellm-dashboard/src/components/guardrails.tsx b/ui/litellm-dashboard/src/components/guardrails.tsx index aa31c3af613..56bba2724d0 100644 --- a/ui/litellm-dashboard/src/components/guardrails.tsx +++ b/ui/litellm-dashboard/src/components/guardrails.tsx @@ -140,7 +140,7 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole Guardrail Garden Guardrails Test Playground - Team Guardrails + Submitted Guardrails diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPStandardsSettings.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPStandardsSettings.tsx new file mode 100644 index 00000000000..fb38e392631 --- /dev/null +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPStandardsSettings.tsx @@ -0,0 +1,72 @@ +"use client"; + +import { MCPServer } from "./types"; + +export interface RequiredFieldDef { + key: string; + label: string; + description: string; + check: (server: MCPServer) => boolean; +} + +export interface FieldGroup { + label: string; + fields: RequiredFieldDef[]; +} + +export const FIELD_GROUPS: FieldGroup[] = [ + { + label: "Documentation", + fields: [ + { + key: "description", + label: "Description", + description: "Must have a non-empty description", + check: (s) => !!s.description?.trim(), + }, + { + key: "alias", + label: "Alias", + description: "Must have a display alias", + check: (s) => !!s.alias?.trim(), + }, + ], + }, + { + label: "Source", + fields: [ + { + key: "source_url", + label: "GitHub / Source URL", + description: "Must link to a source repository", + check: (s) => !!s.source_url?.trim(), + }, + ], + }, + { + label: "Connection", + fields: [ + { + key: "url", + label: "Server URL", + description: "Must have a URL configured", + check: (s) => !!s.url?.trim(), + }, + ], + }, + { + label: "Security", + fields: [ + { + key: "auth_type", + label: "Auth configured", + description: "Must use authentication (not 'none')", + check: (s) => !!s.auth_type && s.auth_type !== "none", + }, + ], + }, +]; + +export const MCP_REQUIRED_FIELD_DEFS: RequiredFieldDef[] = FIELD_GROUPS.flatMap((g) => g.fields); + +export const SETTINGS_KEY = "mcp_required_fields"; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPSubmissionsTab.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPSubmissionsTab.tsx new file mode 100644 index 00000000000..4ce7423f1b5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPSubmissionsTab.tsx @@ -0,0 +1,660 @@ +"use client"; + +import React, { useState, useEffect, useCallback } from "react"; +import { + SearchIcon, + CheckIcon, + XIcon, + AlertCircleIcon, + ServerIcon, + ChevronDownIcon, + ChevronUpIcon, + SettingsIcon, +} from "lucide-react"; +import { + fetchMCPSubmissions, + approveMCPServer, + rejectMCPServer, + getGeneralSettingsCall, + updateConfigFieldSetting, +} from "@/components/networking"; +import { MCPServer, MCPSubmissionsSummary } from "./types"; +import { FIELD_GROUPS, MCP_REQUIRED_FIELD_DEFS, SETTINGS_KEY } from "./MCPStandardsSettings"; +import NotificationsManager from "@/components/molecules/notifications_manager"; + +type MCPStatus = "active" | "pending_review" | "rejected"; + +const STATUS_CONFIG: Record< + MCPStatus, + { label: string; bg: string; text: string; dot: string } +> = { + active: { + label: "Active", + bg: "bg-green-50", + text: "text-green-700", + dot: "bg-green-500", + }, + pending_review: { + label: "Pending Review", + bg: "bg-yellow-50", + text: "text-yellow-700", + dot: "bg-yellow-500", + }, + rejected: { + label: "Rejected", + bg: "bg-red-50", + text: "text-red-700", + dot: "bg-red-500", + }, +}; + +function formatDate(value: string | null | undefined): string { + if (!value) return "—"; + try { + const d = new Date(value); + return isNaN(d.getTime()) ? value : d.toISOString().slice(0, 10); + } catch { + return value; + } +} + +function StatCard({ + label, + value, + color, +}: { + label: string; + value: number; + color: string; +}) { + return ( +
+
{value}
+
{label}
+
+ ); +} + +type ConfirmDialogProps = { + action: "approve" | "reject"; + serverName: string; + isCurrentlyActive?: boolean; + onConfirm: (reviewNotes?: string) => void; + onCancel: () => void; +}; + +function ConfirmDialog({ action, serverName, isCurrentlyActive, onConfirm, onCancel }: ConfirmDialogProps) { + const [reviewNotes, setReviewNotes] = useState(""); + const isApprove = action === "approve"; + const rejectBody = isCurrentlyActive + ? "This server is currently live. Rejecting it will immediately remove it from the proxy runtime." + : "This will mark the submission as rejected."; + return ( +
+
+
+ {isApprove ? ( + + ) : ( + + )} +
+

+ {isApprove ? "Approve MCP Server" : "Reject MCP Server"} +

+

+ Are you sure you want to {action}{" "} + "{serverName}"?{" "} + {isApprove + ? "This will make it active and available for use." + : rejectBody} +

+ {!isApprove && ( +